Merge pull request #124 from abnamro/2375656-remove-branch

2375656 remove branch

Author: amrityamrout

README.md

@@ -96,8 +96,8 @@ components of RESC.
 
 ### VCS Scanner Worker flow diagram
 The flow diagram below shows the different stages that a VCS Scanner Worker goes through and the choices it is confronted with to
-come to the desired result. It first picks up a branch from the queue where it is decided, in conjunction with user input, what type of scan to run. 
-If it is a base scan, a full scan of all commits will be carried out to look for secrets. Possible findings are stored inside the database along with the last scanned commit hash of the branch. An incremental scan, where the branch was scanned earlier, only looks at the commits that were made after the last scanned commit hash. The process of finding secrets and storing them in the database is similar as previously described. 
+come to the desired result. It first picks up a repository from the queue where it is decided, in conjunction with user input, what type of scan to run. 
+If it is a base scan, a full scan of all commits will be carried out to look for secrets. Possible findings are stored inside the database along with the last scanned commit hash of the repository. An incremental scan, where the repository was scanned earlier, only looks at the commits that were made after the last scanned commit hash. The process of finding secrets and storing them in the database is similar as previously described. 
 
 ![product-screenshot!](images/RESC_Scan_Flow_Diagram.png)
 

components/resc-backend/.coveragerc

@@ -1,5 +1,5 @@
 [report]
-fail_under=61
+fail_under=60
 exclude_lines =
     pragma: no cover
     if __name__ == .__main__.:

components/resc-backend/alembic/versions/44ac9602612b_remove_branch.py

@@ -0,0 +1,73 @@
+"""remove branch
+
+Revision ID: 44ac9602612b
+Revises: 8dd0f349b5ad
+Create Date: 2023-06-27 10:03:22.197295
+
+"""
+import logging
+import sys
+
+from alembic import op
+import sqlalchemy as sa
+
+from sqlalchemy.engine import Inspector
+
+# revision identifiers, used by Alembic.
+revision = '44ac9602612b'
+down_revision = '8dd0f349b5ad'
+branch_labels = None
+depends_on = None
+
+# Logger
+logger = logging.getLogger()
+
+
+def upgrade():
+    inspector = Inspector.from_engine(op.get_bind())
+
+    # add column repository_id to scan and finding
+    op.add_column('finding', sa.Column('repository_id', sa.Integer(), nullable=True))
+    op.add_column('scan', sa.Column('repository_id', sa.Integer(), nullable=True))
+    # Fill it with corresponding contents
+    op.execute("update finding "
+               "set finding.repository_id = branch.repository_id "
+               "from branch "
+               "where branch.id = finding.branch_id")
+    op.execute("update scan "
+               "set scan.repository_id = branch.repository_id "
+               "from branch "
+               "where branch.id = scan.branch_id")
+    # make repository_id not nullable
+    op.alter_column('finding', 'repository_id', existing_type=sa.Integer(), nullable=False)
+    op.alter_column('scan', 'repository_id', existing_type=sa.Integer(), nullable=False)
+    # Add foreign key constraint from scan and finding to repository
+    op.create_foreign_key('fk_finding_repository_id', 'finding', 'repository', ['repository_id'], ['id'])
+    op.create_foreign_key('fk_scan_repository_id', 'scan', 'repository', ['repository_id'], ['id'])
+    # Update unique constraint in finding with repository_id instead of branch_id
+    op.drop_constraint('uc_finding_per_branch', 'finding', type_='unique')
+    op.create_unique_constraint('uc_finding_per_repository', 'finding',
+                                ['commit_id', 'repository_id', 'rule_name', 'file_path', 'line_number',
+                                 'column_start', 'column_end'])
+    # Drop column branch_id from finding and scan
+    op.drop_constraint(get_foreign_key_name(inspector, 'finding', 'branch'), 'finding', type_='foreignkey')
+    op.drop_column('finding', 'branch_id')
+    op.drop_constraint(get_foreign_key_name(inspector, 'scan', 'branch'), 'scan', type_='foreignkey')
+    op.drop_column('scan', 'branch_id')
+    # Drop table branch
+    op.drop_table('branch')
+
+
+def downgrade():
+    # Unable to make a reliable downgrade here as there would not be enough information in the database to restore the
+    # branch table and re-link the finding and scan tables to it. Meaning that all findings would be invalidated
+    pass
+
+
+def get_foreign_key_name(inspector: Inspector, table_name: str, reference_table: str):
+    foreign_keys = inspector.get_foreign_keys(table_name=table_name)
+    for foreign_key in foreign_keys:
+        if foreign_key["referred_table"] == reference_table:
+            return foreign_key["name"]
+    logger.error(f"Unable to find foreign key name for {table_name} referencing {reference_table}")
+    sys.exit(-1)

components/resc-backend/src/resc_backend/constants.py

@@ -8,7 +8,6 @@
 # RWS: RESC Web Service
 RWS_VERSION_PREFIX = "/resc/v1"
 RWS_ROUTE_REPOSITORIES = "/repositories"
-RWS_ROUTE_BRANCHES = "/branches"
 RWS_ROUTE_SCANS = "/scans"
 RWS_ROUTE_LAST_SCAN = "/last-scan"
 RWS_ROUTE_FINDINGS = "/findings"
@@ -42,7 +41,6 @@
 RWS_ROUTE_HEALTH = "/health"
 
 REPOSITORIES_TAG = "resc-repositories"
-BRANCHES_TAG = "resc-branches"
 SCANS_TAG = "resc-scans"
 FINDINGS_TAG = "resc-findings"
 RULES_TAG = "resc-rules"

components/resc-backend/src/resc_backend/db/model/__init__.py

@@ -13,7 +13,6 @@
 
 # First Party
 from resc_backend.db.model.audit import DBaudit
-from resc_backend.db.model.branch import DBbranch
 from resc_backend.db.model.finding import DBfinding
 from resc_backend.db.model.repository import DBrepository
 from resc_backend.db.model.rule import DBrule

components/resc-backend/src/resc_backend/db/model/branch.py

@@ -12,7 +12,7 @@
 class DBfinding(Base):
     __tablename__ = "finding"
     id_ = Column("id", Integer, primary_key=True)
-    branch_id = Column(Integer, ForeignKey("branch.id"), nullable=False)
+    repository_id = Column(Integer, ForeignKey("repository.id"), nullable=False)
     rule_name = Column(String(400), nullable=False)
     file_path = Column(String(500), nullable=False)
     line_number = Column(Integer, nullable=False)
@@ -25,11 +25,11 @@ class DBfinding(Base):
     email = Column(String(100))
     event_sent_on = Column(DateTime, nullable=True)
 
-    __table_args__ = (UniqueConstraint("commit_id", "branch_id", "rule_name", "file_path", "line_number",
-                                       "column_start", "column_end", name="uc_finding_per_branch"),)
+    __table_args__ = (UniqueConstraint("commit_id", "repository_id", "rule_name", "file_path", "line_number",
+                                       "column_start", "column_end", name="uc_finding_per_repository"),)
 
     def __init__(self, rule_name, file_path, line_number, commit_id, commit_message, commit_timestamp, author,
-                 email, event_sent_on, branch_id, column_start, column_end):
+                 email, event_sent_on, repository_id, column_start, column_end):
         self.email = email
         self.author = author
         self.commit_timestamp = commit_timestamp
@@ -39,7 +39,7 @@ def __init__(self, rule_name, file_path, line_number, commit_id, commit_message,
         self.file_path = file_path
         self.rule_name = rule_name
         self.event_sent_on = event_sent_on
-        self.branch_id = branch_id
+        self.repository_id = repository_id
         self.column_start = column_start
         self.column_end = column_end
 
@@ -55,7 +55,7 @@ def create_from_finding(finding):
             commit_timestamp=finding.commit_timestamp,
             author=finding.author,
             event_sent_on=finding.event_sent_on,
-            branch_id=finding.branch_id,
+            repository_id=finding.repository_id,
             column_start=finding.column_start,
             column_end=finding.column_end
         )

components/resc-backend/src/resc_backend/db/model/finding.py

@@ -10,22 +10,22 @@
 from resc_backend.db.model.rule_pack import DBrulePack
 from resc_backend.resc_web_service.schema.scan_type import ScanType
 
-BRANCH_ID = "branch.id"
+REPOSITORY_ID = "repository.id"
 
 
 class DBscan(Base):
     __tablename__ = "scan"
     id_ = Column("id", Integer, primary_key=True)
-    branch_id = Column(Integer, ForeignKey(BRANCH_ID))
+    repository_id = Column(Integer, ForeignKey(REPOSITORY_ID))
     rule_pack = Column(String(100), ForeignKey(DBrulePack.version), nullable=False)
     scan_type = Column(Enum(ScanType), default=ScanType.BASE, server_default=BASE_SCAN, nullable=False)
     last_scanned_commit = Column(String(100), nullable=False)
     timestamp = Column(DateTime, nullable=False, default=datetime.utcnow)
     increment_number = Column(Integer, server_default=text("0"), default=0, nullable=False)
 
-    def __init__(self, branch_id: int, scan_type: ScanType, last_scanned_commit: str, timestamp: datetime,
+    def __init__(self, repository_id: int, scan_type: ScanType, last_scanned_commit: str, timestamp: datetime,
                  increment_number: int, rule_pack: str):
-        self.branch_id = branch_id
+        self.repository_id = repository_id
         self.scan_type = scan_type
         self.last_scanned_commit = last_scanned_commit
         self.timestamp = timestamp
@@ -34,13 +34,13 @@ def __init__(self, branch_id: int, scan_type: ScanType, last_scanned_commit: str
 
     @staticmethod
     def create_from_metadata(timestamp: datetime, scan_type: ScanType, last_scanned_commit: str, increment_number: int,
-                             rule_pack: str, branch_id: int):
+                             rule_pack: str, repository_id: int):
         db_scan = DBscan(
             timestamp=timestamp,
             scan_type=scan_type,
             last_scanned_commit=last_scanned_commit,
             increment_number=increment_number,
             rule_pack=rule_pack,
-            branch_id=branch_id
+            repository_id=repository_id
         )
         return db_scan

components/resc-backend/src/resc_backend/db/model/scan.py

@@ -12,16 +12,13 @@
 
 
 def clone_repository(repository_url: str,
-                     branch_name: str,
                      repo_clone_path: str,
                      username: str = "",
                      personal_access_token: str = ""):
     """
         Clones the given repository
     :param repository_url:
         Repository url to clone
-    :param branch_name:
-        Branch name of the repository url to clone
     :param repo_clone_path:
         Path where to clone the repository
     :param username:
@@ -36,5 +33,5 @@ def clone_repository(repository_url: str,
         repo_clone_url = f"https://{personal_access_token}@{url}"
         logger.debug(f"username is not specified, so cloning the repository with only personal access token: {url}")
 
-    Repo.clone_from(repo_clone_url, repo_clone_path, branch=branch_name)
-    logger.debug(f"Repository {repository_url}:{branch_name} cloned successfully")
+    Repo.clone_from(repo_clone_url, repo_clone_path)
+    logger.debug(f"Repository {repository_url} cloned successfully")

components/resc-backend/src/resc_backend/helpers/git_operation.py

@@ -20,7 +20,6 @@
     requires_no_auth
 )
 from resc_backend.resc_web_service.endpoints import (
-    branches,
     common,
     detailed_findings,
     findings,
@@ -92,7 +91,6 @@ def generate_logger_config(log_file_path, debug=True):
     {"name": "resc-rules", "description": "Manage rule information"},
     {"name": "resc-rule-packs", "description": "Manage rule pack information"},
     {"name": "resc-repositories", "description": "Manage repository information"},
-    {"name": "resc-branches", "description": "Manage branch information"},
     {"name": "resc-scans", "description": "Manage scan information"},
     {"name": "resc-findings", "description": "Manage findings information"},
     {"name": "resc-vcs-instances", "description": "Manage vcs instance information"},
@@ -120,7 +118,6 @@ def generate_logger_config(log_file_path, debug=True):
 
 app.include_router(health.router, prefix=RWS_VERSION_PREFIX)
 app.include_router(common.router, prefix=RWS_VERSION_PREFIX)
-app.include_router(branches.router, prefix=RWS_VERSION_PREFIX)
 app.include_router(rules.router, prefix=RWS_VERSION_PREFIX)
 app.include_router(rule_packs.router, prefix=RWS_VERSION_PREFIX)
 app.include_router(findings.router, prefix=RWS_VERSION_PREFIX)