Merge pull request #82 from abnamro/2033043-finding-column

add column start and column end to finding

Author: Peter-v-d-Spek

components/resc-backend/alembic/versions/9c5fa6db20f1_finding_column.py

@@ -0,0 +1,33 @@
+"""empty message
+
+Revision ID: 9c5fa6db20f1
+Revises: ar399258p714
+Create Date: 2023-03-06 13:56:47.958406
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9c5fa6db20f1'
+down_revision = 'ar399258p714'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('finding', sa.Column('column_start', sa.Integer(), nullable=False, server_default=sa.text("0")))
+    op.add_column('finding', sa.Column('column_end', sa.Integer(), nullable=False, server_default=sa.text("0")))
+    op.drop_constraint('uc_finding_per_branch', 'finding', type_='unique')
+    op.create_unique_constraint('uc_finding_per_branch', 'finding',
+                                ['commit_id', 'branch_id', 'rule_name', 'file_path', 'line_number',
+                                 'column_start', 'column_end'])
+
+
+def downgrade():
+    op.drop_constraint('uc_finding_per_branch', 'finding', type_='unique')
+    op.create_unique_constraint('uc_finding_per_branch', 'finding',
+                                ['commit_id', 'branch_id', 'rule_name', 'file_path', 'line_number'])
+    op.drop_column('finding', 'column_start')
+    op.drop_column('finding', 'column_end')

components/resc-backend/setup.cfg

@@ -1,7 +1,7 @@
 [metadata]
 name = resc_backend
 description = Repository Scanner - Backend
-version = 1.0.3
+version = 1.1.0
 author = ABN AMRO
 author_email = resc@nl.abnamro.com
 url = https://github.com/ABNAMRO/repository-scanner

components/resc-backend/src/resc_backend/db/model/finding.py

@@ -18,6 +18,8 @@ class DBfinding(Base):
     rule_name = Column(String(400), nullable=False)
     file_path = Column(String(500), nullable=False)
     line_number = Column(Integer, nullable=False)
+    column_start = Column(Integer, nullable=False, default=0)
+    column_end = Column(Integer, nullable=False, default=0)
     commit_id = Column(String(120))
     commit_message = Column(Text)
     commit_timestamp = Column(DateTime, default=datetime.utcnow().isoformat())
@@ -29,10 +31,10 @@ class DBfinding(Base):
     event_sent_on = Column(DateTime, nullable=True)
 
     __table_args__ = (UniqueConstraint("commit_id", "branch_id", "rule_name", "file_path", "line_number",
-                                       name="uc_finding_per_branch"),)
+                                       "column_start", "column_end", name="uc_finding_per_branch"),)
 
     def __init__(self, rule_name, file_path, line_number, commit_id, commit_message, commit_timestamp, author,
-                 email, status, comment, event_sent_on, branch_id):
+                 email, status, comment, event_sent_on, branch_id, column_start, column_end):
         self.email = email
         self.author = author
         self.commit_timestamp = commit_timestamp
@@ -45,6 +47,8 @@ def __init__(self, rule_name, file_path, line_number, commit_id, commit_message,
         self.comment = comment
         self.event_sent_on = event_sent_on
         self.branch_id = branch_id
+        self.column_start = column_start
+        self.column_end = column_end
 
     @staticmethod
     def create_from_finding(finding):
@@ -61,6 +65,8 @@ def create_from_finding(finding):
             status=finding.status,
             comment=sanitized_comment,
             event_sent_on=finding.event_sent_on,
-            branch_id=finding.branch_id
+            branch_id=finding.branch_id,
+            column_start=finding.column_start,
+            column_end=finding.column_end
         )
         return db_finding

components/resc-backend/src/resc_backend/resc_web_service/crud/detailed_finding.py

@@ -49,6 +49,8 @@ def get_detailed_findings(db_connection: Session, findings_filter: FindingsFilte
         model.DBfinding.id_,
         model.DBfinding.file_path,
         model.DBfinding.line_number,
+        model.DBfinding.column_start,
+        model.DBfinding.column_end,
         model.DBfinding.commit_id,
         model.DBfinding.commit_message,
         model.DBfinding.commit_timestamp,
@@ -201,6 +203,8 @@ def get_detailed_finding(db_connection: Session, finding_id: int) -> detailed_fi
         model.DBfinding.id_,
         model.DBfinding.file_path,
         model.DBfinding.line_number,
+        model.DBfinding.column_start,
+        model.DBfinding.column_end,
         model.DBfinding.commit_id,
         model.DBfinding.commit_message,
         model.DBfinding.commit_timestamp,

components/resc-backend/src/resc_backend/resc_web_service/crud/finding.py

@@ -86,7 +86,9 @@ def create_findings(db_connection: Session, findings: List[finding_schema.Findin
             if branch_finding.commit_id == finding.commit_id and \
                     branch_finding.rule_name == finding.rule_name and \
                     branch_finding.file_path == finding.file_path and \
-                    branch_finding.line_number == finding.line_number:
+                    branch_finding.line_number == finding.line_number and \
+                    branch_finding.column_start == finding.column_start and \
+                    branch_finding.column_end == finding.column_end:
                 # Store the already known finding
                 db_findings.append(branch_finding)
                 # Remove from the db_branch_findings to increase performance for the next loop

components/resc-backend/src/resc_backend/resc_web_service/endpoints/scans.py

@@ -198,6 +198,8 @@ def create_scan_findings(scan_id: int,
     - **scan_id**:  Id of the scan for which findings need to be inserted
     - **file_path**: file path
     - **line_number**: Line number
+    - **column_start**: Column start
+    - **column_end**: Column end
     - **commit_id**: commit hash
     - **commit_message**: Commit message
     - **commit_timestamp**: Commit timestamp

components/resc-backend/src/resc_backend/resc_web_service/schema/detailed_finding.py

@@ -16,6 +16,8 @@
 class DetailedFindingBase(BaseModel):
     file_path: str
     line_number: conint(gt=-1)
+    column_start: conint(gt=-1)
+    column_end: conint(gt=-1)
     commit_id: constr(max_length=120)
     commit_message: str
     commit_timestamp: datetime.datetime

components/resc-backend/src/resc_backend/resc_web_service/schema/finding.py

@@ -15,6 +15,8 @@
 class FindingBase(BaseModel):
     file_path: constr(max_length=500)
     line_number: conint(gt=-1)
+    column_start: conint(gt=-1)
+    column_end: conint(gt=-1)
     commit_id: constr(max_length=120)
     commit_message: str
     commit_timestamp: datetime.datetime
@@ -60,6 +62,8 @@ def create_from_db_entities(cls, db_finding: DBfinding, scan_ids: List[int]):
             id_=db_finding.id_,
             file_path=db_finding.file_path,
             line_number=db_finding.line_number,
+            column_start=db_finding.column_start,
+            column_end=db_finding.column_end,
             commit_id=db_finding.commit_id,
             commit_message=db_finding.commit_message,
             commit_timestamp=db_finding.commit_timestamp,

components/resc-backend/test_data/database_dummy_data.sql

@@ -31,14 +31,14 @@ INSERT INTO scan (branch_id, [timestamp], scan_type, last_scanned_commit, increm
     (3, '2022-02-24 17:00:00.000', 'BASE', 'qwerty1', 0, '0.0.0'),
     (3, '2022-03-24 17:00:00.000', 'BASE', 'qwerty1', 0, '0.0.0');
 
-INSERT INTO finding (branch_id, file_path, line_number, commit_id, commit_message, commit_timestamp, author, email, rule_name, status, comment) VALUES
-    (1, '/path/to/file', 123, 'qwerty1', 'this is commit 1', '2021-01-01 00:00:00.000', 'developer', 'developer@abn.com', 'rule#1', 'NOT_ANALYZED', NULL),
-    (1, '/path/to/file', 123, 'qwerty1', 'this is commit 1', '2021-01-01 00:00:00.000', 'developer', 'developer@abn.com', 'rule#2', 'NOT_ANALYZED', NULL),
-    (2, '/path/to/file', 123, 'qwerty1', 'this is commit 1', '2021-01-01 00:00:00.000', 'developer', 'developer@abn.com', 'rule#1', 'NOT_ANALYZED', NULL),
-    (2, '/path/to/file', 123, 'qwerty1', 'this is commit 1', '2021-01-01 00:00:00.000', 'developer', 'developer@abn.com', 'rule#2', 'NOT_ANALYZED', NULL),
-    (3, '/path/to/file', 123, 'qwerty1', 'this is commit 1', '2021-01-01 00:00:00.000', 'developer', 'developer@abn.com', 'rule#1', 'NOT_ANALYZED', NULL),
-    (3, '/path/to/file', 123, 'qwerty1', 'this is commit 1', '2021-01-01 00:00:00.000', 'developer', 'developer@abn.com', 'rule#2', 'NOT_ANALYZED', NULL),
-    (3, '/path/to/file', 123, 'qwerty2', 'this is commit 2', '2021-01-01 00:00:00.000', 'developer', 'developer@abn.com', 'rule#1', 'NOT_ANALYZED', NULL);
+INSERT INTO finding (branch_id, file_path, line_number, commit_id, commit_message, commit_timestamp, author, email, rule_name, status, comment, column_start, column_end) VALUES
+    (1, '/path/to/file', 123, 'qwerty1', 'this is commit 1', '2021-01-01 00:00:00.000', 'developer', 'developer@abn.com', 'rule#1', 'NOT_ANALYZED', NULL, 1, 100),
+    (1, '/path/to/file', 123, 'qwerty1', 'this is commit 1', '2021-01-01 00:00:00.000', 'developer', 'developer@abn.com', 'rule#2', 'NOT_ANALYZED', NULL, 0, 0),
+    (2, '/path/to/file', 123, 'qwerty1', 'this is commit 1', '2021-01-01 00:00:00.000', 'developer', 'developer@abn.com', 'rule#1', 'NOT_ANALYZED', NULL, 1, 50),
+    (2, '/path/to/file', 123, 'qwerty1', 'this is commit 1', '2021-01-01 00:00:00.000', 'developer', 'developer@abn.com', 'rule#2', 'NOT_ANALYZED', NULL, 42, 43),
+    (3, '/path/to/file', 123, 'qwerty1', 'this is commit 1', '2021-01-01 00:00:00.000', 'developer', 'developer@abn.com', 'rule#1', 'NOT_ANALYZED', NULL, 12, 34),
+    (3, '/path/to/file', 123, 'qwerty1', 'this is commit 1', '2021-01-01 00:00:00.000', 'developer', 'developer@abn.com', 'rule#2', 'NOT_ANALYZED', NULL, 21, 34),
+    (3, '/path/to/file', 123, 'qwerty2', 'this is commit 2', '2021-01-01 00:00:00.000', 'developer', 'developer@abn.com', 'rule#1', 'NOT_ANALYZED', NULL, 12, 34);
 
 INSERT INTO scan_finding(scan_id, finding_id) VALUES
     (1, 1),