[#2241513] Rule categorization

Author: Peter-v-d-Spek

components/resc-backend/alembic/versions/8dd0f349b5ad_rule_tags.py

@@ -0,0 +1,85 @@
+"""rule tags
+
+Revision ID: 8dd0f349b5ad
+Revises: ar7fh3ac2071
+Create Date: 2023-05-02 10:10:20.423021
+
+"""
+import logging
+
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+from sqlalchemy.engine import Inspector
+
+revision = '8dd0f349b5ad'
+down_revision = 'ar7fh3ac2071'
+branch_labels = None
+depends_on = None
+
+# Logger
+logger = logging.getLogger()
+
+
+def upgrade():
+    inspector = Inspector.from_engine(op.get_bind())
+
+    drop_if_exist(inspector, "tag")
+    if not inspector.has_table("tag"):
+        logger.info("Creating table tag")
+        op.create_table("tag",
+                        sa.Column('id', sa.Integer(), nullable=False),
+                        sa.Column('name', sa.String(length=200), nullable=False),
+                        sa.PrimaryKeyConstraint('id')
+                        )
+
+    drop_if_exist(inspector, "rule_tag")
+    if not inspector.has_table("rule_tag"):
+        logger.info("Creating table rule_tag")
+        op.create_table('rule_tag',
+                        sa.Column('rule_id', sa.Integer(), nullable=False),
+                        sa.Column('tag_id', sa.Integer(), nullable=False),
+                        sa.ForeignKeyConstraint(['rule_id'], ['rules.id'], ),
+                        sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], ),
+                        sa.PrimaryKeyConstraint('rule_id', 'tag_id'),
+                        )
+    # insert data in to tag table
+    logger.info("Inserting data in to tag table")
+    op.execute("INSERT INTO tag (name) "
+               "select distinct cs.Value as name from rules "
+               "cross apply STRING_SPLIT(tags, ',') cs")
+
+    # insert data in to rule_tag table
+    logger.info("Inserting data in to rule_tag table")
+    op.execute("INSERT INTO rule_tag (rule_id, tag_id) "
+               "select rules.id, tag.id from rules "
+               "cross apply STRING_SPLIT(tags, ',') cs join tag on tag.name = cs.Value")
+
+    # Drop tags column from rules table
+    logger.info("Drop tags column from rules table")
+    op.drop_column('rules', 'tags')
+
+
+def downgrade():
+    # add tags column to rules
+    op.add_column('rules', sa.Column('tags', sa.String(length=2000), nullable=True))
+
+    # insert data in to rules table
+    logger.info("Update tags in rules table")
+    op.execute("update rules set tags = ("
+               "SELECT STRING_AGG(tag.name, ',') as tags "
+               "FROM rule_tag "
+               "JOIN tag ON tag.id = rule_tag.tag_id "
+               "where rule_id = rules.id "
+               "GROUP BY rule_id)")
+
+    # drop rule_tag and tag tables
+    op.drop_table('tag')
+    op.drop_table('rule_tag')
+
+
+def drop_if_exist(inspector: Inspector, table_name: str):
+    if inspector.has_table(table_name):
+        logger.info(f"Dropping table {table_name}")
+        op.drop_table(table_name)

components/resc-backend/setup.cfg

@@ -1,7 +1,7 @@
 [metadata]
 name = resc_backend
 description = Repository Scanner - Backend
-version = 1.2.0
+version = 1.3.0
 author = ABN AMRO
 author_email = resc@nl.abnamro.com
 url = https://github.com/ABNAMRO/repository-scanner

components/resc-backend/src/resc_backend/db/model/__init__.py

@@ -19,6 +19,8 @@
 from resc_backend.db.model.rule import DBrule
 from resc_backend.db.model.rule_allow_list import DBruleAllowList
 from resc_backend.db.model.rule_pack import DBrulePack
+from resc_backend.db.model.rule_tag import DBruleTag
 from resc_backend.db.model.scan import DBscan
 from resc_backend.db.model.scan_finding import DBscanFinding
+from resc_backend.db.model.tag import DBtag
 from resc_backend.db.model.vcs_instance import DBVcsInstance

components/resc-backend/src/resc_backend/db/model/rule.py

@@ -15,7 +15,6 @@ class DBrule(Base):
     allow_list = Column(Integer, ForeignKey(DBruleAllowList.id_), nullable=True)
     rule_name = Column(String(400), nullable=False)
     description = Column(String(2000), nullable=True)
-    tags = Column(String(2000), nullable=True)
     entropy = Column(Float, nullable=True)
     secret_group = Column(Integer, nullable=True)
     regex = Column(Text, nullable=True)
@@ -24,7 +23,7 @@ class DBrule(Base):
     __table_args__ = (UniqueConstraint("rule_name", "rule_pack", name="unique_rule_name_per_rule_pack_version"),)
 
     def __init__(self, rule_pack: str, rule_name: str, description: str, allow_list: int = None,
-                 entropy: float = None, secret_group: str = None, regex: str = None, path: str = None, tags: str = None,
+                 entropy: float = None, secret_group: str = None, regex: str = None, path: str = None,
                  keywords: str = None):
         self.rule_pack = rule_pack
         self.allow_list = allow_list
@@ -34,18 +33,16 @@ def __init__(self, rule_pack: str, rule_name: str, description: str, allow_list:
         self.secret_group = secret_group
         self.regex = regex
         self.path = path
-        self.tags = tags
         self.keywords = keywords
 
     @staticmethod
-    def create_from_metadata(rule_pack: str, rule_name: str, description: str, tags: str, entropy: float,
+    def create_from_metadata(rule_pack: str, rule_name: str, description: str, entropy: float,
                              secret_group: str, regex: str, path: str, keywords: str,
                              allow_list: int):
         db_rule = DBrule(
             rule_pack=rule_pack,
             rule_name=rule_name,
             description=description,
-            tags=tags,
             entropy=entropy,
             secret_group=secret_group,
             regex=regex,

components/resc-backend/src/resc_backend/db/model/rule_tag.py

@@ -0,0 +1,16 @@
+# Third Party
+from sqlalchemy import Column, ForeignKey, Integer
+
+# First Party
+from resc_backend.db.model import Base
+
+
+class DBruleTag(Base):
+    __tablename__ = "rule_tag"
+    rule_id = Column(Integer, ForeignKey("rules.id"), primary_key=True)
+    tag_id = Column(Integer, ForeignKey("tag.id"), primary_key=True)
+
+    def __init__(self, rule_id: int, tag_id: int):
+
+        self.rule_id = rule_id
+        self.tag_id = tag_id

components/resc-backend/src/resc_backend/db/model/tag.py

@@ -0,0 +1,16 @@
+# pylint: disable=R0902
+# Third Party
+from sqlalchemy import Column, Integer, String, UniqueConstraint
+
+# First Party
+from resc_backend.db.model import Base
+
+
+class DBtag(Base):
+    __tablename__ = "tag"
+    id_ = Column("id", Integer, primary_key=True)
+    name = Column(String(200), nullable=False)
+    __table_args__ = (UniqueConstraint("name", name="unique_tag"),)
+
+    def __init__(self, name: str):
+        self.name = name

components/resc-backend/src/resc_backend/resc_web_service/crud/detailed_finding.py

@@ -1,4 +1,4 @@
-# pylint: disable=R0912,C0121
+# pylint: disable=R0912,C0121,R0915
 # Standard Library
 from typing import List
 
@@ -46,6 +46,18 @@ def get_detailed_findings(db_connection: Session, findings_filter: FindingsFilte
                                              func.max(model.DBaudit.id_).label("audit_id")) \
         .group_by(model.DBaudit.finding_id).subquery()
 
+    rule_tag_subquery = db_connection.query(model.DBruleTag.rule_id) \
+        .join(model.DBtag, model.DBruleTag.tag_id == model.DBtag.id_)
+    if findings_filter.rule_tags:
+        rule_tag_subquery = rule_tag_subquery.filter(model.DBtag.name.in_(findings_filter.rule_tags))
+    if findings_filter.rule_pack_versions or findings_filter.rule_names:
+        rule_tag_subquery = rule_tag_subquery.join(model.DBrule, model.DBrule.id_ == model.DBruleTag.rule_id)
+        if findings_filter.rule_pack_versions:
+            rule_tag_subquery = rule_tag_subquery.filter(model.DBrule.rule_pack.in_(findings_filter.rule_pack_versions))
+        if findings_filter.rule_names:
+            rule_tag_subquery = rule_tag_subquery.filter(model.DBrule.rule_name.in_(findings_filter.rule_names))
+    rule_tag_subquery = rule_tag_subquery.group_by(model.DBruleTag.rule_id).subquery()
+
     limit_val = MAX_RECORDS_PER_PAGE_LIMIT if limit > MAX_RECORDS_PER_PAGE_LIMIT else limit
 
     query = db_connection.query(
@@ -96,8 +108,8 @@ def get_detailed_findings(db_connection: Session, findings_filter: FindingsFilte
     if findings_filter.rule_tags:
         query = query.join(model.DBrule, and_(model.DBrule.rule_name == model.DBfinding.rule_name,
                                               model.DBrule.rule_pack == model.DBscan.rule_pack))
-        for tag in findings_filter.rule_tags:
-            query = query.filter(model.DBrule.tags.like(f"%{tag}%"))
+        query = query.join(rule_tag_subquery, model.DBrule.id_ == rule_tag_subquery.c.rule_id)
+
     if findings_filter.rule_pack_versions:
         query = query.filter(model.DBscan.rule_pack.in_(findings_filter.rule_pack_versions))
     if findings_filter.start_date_time:
@@ -158,6 +170,18 @@ def get_detailed_findings_count(db_connection: Session, findings_filter: Finding
             model.DBscan.rule_pack.in_(findings_filter.rule_pack_versions))
     max_base_scan_subquery = max_base_scan_subquery.group_by(model.DBscan.branch_id).subquery()
 
+    rule_tag_subquery = db_connection.query(model.DBruleTag.rule_id) \
+        .join(model.DBtag, model.DBruleTag.tag_id == model.DBtag.id_)
+    if findings_filter.rule_tags:
+        rule_tag_subquery = rule_tag_subquery.filter(model.DBtag.name.in_(findings_filter.rule_tags))
+    if findings_filter.rule_pack_versions or findings_filter.rule_names:
+        rule_tag_subquery = rule_tag_subquery.join(model.DBrule, model.DBrule.id_ == model.DBruleTag.rule_id)
+        if findings_filter.rule_pack_versions:
+            rule_tag_subquery = rule_tag_subquery.filter(model.DBrule.rule_pack.in_(findings_filter.rule_pack_versions))
+        if findings_filter.rule_names:
+            rule_tag_subquery = rule_tag_subquery.filter(model.DBrule.rule_name.in_(findings_filter.rule_names))
+    rule_tag_subquery = rule_tag_subquery.group_by(model.DBruleTag.rule_id).subquery()
+
     query = db_connection.query(func.count(model.DBfinding.id_))
 
     query = query.join(model.DBscanFinding, model.DBfinding.id_ == model.DBscanFinding.finding_id)
@@ -184,8 +208,7 @@ def get_detailed_findings_count(db_connection: Session, findings_filter: Finding
     if findings_filter.rule_tags:
         query = query.join(model.DBrule, and_(model.DBrule.rule_name == model.DBfinding.rule_name,
                                               model.DBrule.rule_pack == model.DBscan.rule_pack))
-        for tag in findings_filter.rule_tags:
-            query = query.filter(model.DBrule.tags.like(f"%{tag}%"))
+        query = query.join(rule_tag_subquery, model.DBrule.id_ == rule_tag_subquery.c.rule_id)
 
     if findings_filter.rule_pack_versions:
         query = query.filter(model.DBscan.rule_pack.in_(findings_filter.rule_pack_versions))

components/resc-backend/src/resc_backend/resc_web_service/crud/finding.py

@@ -189,20 +189,6 @@ def get_total_findings_count(db_connection: Session, findings_filter: FindingsFi
                       model.repository.DBrepository.id_ == model.branch.DBbranch.repository_id) \
                 .join(model.DBVcsInstance,
                       model.vcs_instance.DBVcsInstance.id_ == model.repository.DBrepository.vcs_instance)
-        elif findings_filter.rule_tags:
-            max_scan_subquery = db_connection.query(model.DBscanFinding.finding_id,
-                                                    func.max(model.DBscanFinding.scan_id).label("scan_id"))
-            max_scan_subquery = max_scan_subquery.group_by(model.DBscanFinding.finding_id).subquery()
-            total_count_query = total_count_query.join(max_scan_subquery,
-                                                       model.finding.DBfinding.id_ == max_scan_subquery.c.finding_id) \
-                .join(model.DBscan, model.scan.DBscan.id_ == max_scan_subquery.c.scan_id)
-
-        if findings_filter.rule_tags:
-            total_count_query = total_count_query.join(model.DBrule,
-                                                       and_(model.DBrule.rule_name == model.DBfinding.rule_name,
-                                                            model.DBrule.rule_pack == model.DBscan.rule_pack))
-            for tag in findings_filter.rule_tags:
-                total_count_query = total_count_query.filter(model.DBrule.tags.like(f"%{tag}%"))
 
         if findings_filter.start_date_time:
             total_count_query = total_count_query.filter(
@@ -401,13 +387,16 @@ def get_findings_count_by_status(db_connection: Session, scan_ids: List[int] = N
     return findings_count_by_status
 
 
-def get_rule_findings_count_by_status(db_connection: Session, rule_pack_versions: [str] = None):
+def get_rule_findings_count_by_status(db_connection: Session, rule_pack_versions: [str] = None,
+                                      rule_tags: [str] = None):
     """
         Retrieve count of findings based on rulename and status
     :param db_connection:
         Session of the database connection
     :param rule_pack_versions:
         optional, filter on rule pack version
+    :param rule_tags:
+        optional, filter on rule tag
     :return: findings_count
         per rulename and status the count of findings
     """
@@ -430,6 +419,20 @@ def get_rule_findings_count_by_status(db_connection: Session, rule_pack_versions
     query = query.join(max_base_scan_subquery, model.DBfinding.branch_id == max_base_scan_subquery.c.branch_id)
     query = query.join(model.DBscan, and_(model.DBscanFinding.scan_id == model.DBscan.id_,
                                           model.DBscan.id_ >= max_base_scan_subquery.c.latest_base_scan_id))
+    if rule_tags:
+        rule_tag_subquery = db_connection.query(model.DBruleTag.rule_id) \
+            .join(model.DBtag, model.DBruleTag.tag_id == model.DBtag.id_)
+        if rule_pack_versions:
+            rule_tag_subquery = rule_tag_subquery.join(model.DBrule, model.DBrule.id_ == model.DBruleTag.rule_id)
+            rule_tag_subquery = rule_tag_subquery.filter(model.DBrule.rule_pack.in_(rule_pack_versions))
+
+        rule_tag_subquery = rule_tag_subquery.filter(model.DBtag.name.in_(rule_tags))
+        rule_tag_subquery = rule_tag_subquery.group_by(model.DBruleTag.rule_id).subquery()
+
+        query = query.join(model.DBrule, and_(model.DBrule.rule_name == model.DBfinding.rule_name,
+                                              model.DBrule.rule_pack == model.DBscan.rule_pack))
+        query = query.join(rule_tag_subquery, model.DBrule.id_ == rule_tag_subquery.c.rule_id)
+
     if rule_pack_versions:
         query = query.filter(model.DBscan.rule_pack.in_(rule_pack_versions))
 

components/resc-backend/src/resc_backend/resc_web_service/crud/rule.py

@@ -62,7 +62,6 @@ def create_rule(db_connection: Session, rule: RuleCreate):
     db_rule = model.rule.DBrule(
         rule_name=rule.rule_name,
         description=rule.description,
-        tags=rule.tags,
         entropy=rule.entropy,
         secret_group=rule.secret_group,
         regex=rule.regex,
@@ -92,7 +91,6 @@ def get_rules_by_rule_pack_version(db_connection: Session, rule_pack_version: st
         model.DBrule.id_,
         model.DBrule.rule_pack,
         model.DBrule.rule_name,
-        model.DBrule.tags,
         model.DBrule.entropy,
         model.DBrule.secret_group,
         model.DBrule.regex,