Qa view spike

.github/scripts/qa-data.py

@@ -0,0 +1,65 @@
+import os
+import pyodbc
+import json
+import struct
+from azure import identity
+
+def query_to_json(connection_string, sql_query):
+
+    if os.getenv("ENV") != "dev":
+        credential = identity.DefaultAzureCredential(exclude_interactive_browser_credential=True)
+        token_bytes = credential.get_token("https://database.windows.net/.default").token.encode("UTF-16-LE")
+        token_struct = struct.pack(f'<I{len(token_bytes)}s', len(token_bytes), token_bytes)
+        SQL_COPT_SS_ACCESS_TOKEN = 1256
+        connection = pyodbc.connect(connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct})
+    else:
+        connection = pyodbc.connect(connection_string)
+    cursor = connection.cursor()
+    cursor.execute(sql_query)
+    result = cursor.fetchall()
+    connection.close()
+    return cursor, result
+
+def main():
+
+    connection_string = os.getenv("SQL_CONNECTION_STRING")
+
+    data_folder = 'data'
+    if not os.path.exists(data_folder):
+        os.makedirs(data_folder)
+
+    section_query = "SELECT id, name FROM [Contentful].[Sections]"
+    cursor, sections = query_to_json(connection_string, section_query)
+
+    for section in sections:
+        section_id, section_name = section
+
+        sql_query = f'''
+            SELECT
+                Q.Id AS QuestionId,
+                Q.Text AS QuestionText,
+                O.[Order] AS QuestionOrder,
+                A.Text,
+                A.NextQuestionId
+            FROM
+                [Contentful].[Questions] Q
+            JOIN
+                [Contentful].[ContentComponents] O ON Q.Id = O.Id
+            LEFT JOIN
+                [Contentful].[Answers] A ON Q.Id = A.ParentQuestionId
+            WHERE
+                Q.SectionId = '{section_id}'
+            ORDER BY
+                O.[Order] ASC;
+        '''
+
+        cursor, result = query_to_json(connection_string, sql_query)
+
+        json_result = json.dumps([dict(zip([column[0] for column in cursor.description], row)) for row in result], indent=2)
+
+        output_file = os.path.join(data_folder, f'{section_name}.json')
+        with open(output_file, 'w') as file:
+            file.write(json_result)
+
+if __name__ == "__main__":
+    main()

.github/scripts/qa-visualisations.py

@@ -0,0 +1,46 @@
+import os
+import json
+from graphviz import Digraph
+
+def create_questionnaire_flowchart(question_data):
+    tree = Digraph(comment='Questionnaire Flowchart', format='png')
+
+    end_nodes = set()
+
+    for item in question_data:
+        current_question = item["QuestionId"]
+        next_question_id = item["NextQuestionId"]
+        answer_text = item["Text"]
+
+        tree.node(current_question, item["QuestionText"], shape='box')
+
+        if next_question_id:
+            tree.node(next_question_id, "", shape='box')
+            tree.edge(current_question, next_question_id, label=answer_text)
+        else:
+            end_nodes.add(current_question)
+
+    for end_node in end_nodes:
+        tree.node("end", "End", shape='box')
+        tree.edge(end_node, "end", label="No More Questions")
+
+    return tree
+
+def process_data_files():
+    data_folder = 'data'
+    png_folder = 'visualisations'
+    if not os.path.exists(png_folder):
+        os.makedirs(png_folder)
+
+    for filename in os.listdir(data_folder):
+        if filename.endswith('.json'):
+            file_path = os.path.join(data_folder, filename)
+            with open(file_path, 'r') as file:
+                json_data = json.load(file)
+
+            flowchart = create_questionnaire_flowchart(json_data)
+            output_file = os.path.join(png_folder, f'{os.path.splitext(filename)[0]}')
+            flowchart.render(output_file, cleanup=True)
+
+if __name__ == '__main__':
+    process_data_files()

.github/workflows/qa-viz.yml

@@ -0,0 +1,101 @@
+name: QA Visualisation Workflow
+
+
+on:
+  schedule:
+    - cron: "*/30 * * * *" # Every 30 minutes for now - we'll fine tune frequency.
+  workflow_dispatch:
+    inputs: 
+      environment:
+        description: Environment to run the visualisation for
+        required: true
+        options: ['Dev', 'Tst', 'Staging']
+
+jobs:
+  generate_images:
+    runs-on: ubuntu-latest
+    env:
+      az_keyvault_name: ${{ secrets.AZ_ENVIRONMENT }}${{ secrets.DFE_PROJECT_NAME }}-kv
+      az_resource_group_name: ${{ secrets.AZ_ENVIRONMENT }}${{ secrets.DFE_PROJECT_NAME }}
+      az_sql_database_server_name: ${{ secrets.AZ_ENVIRONMENT }}${{ secrets.DFE_PROJECT_NAME }}
+      SQL_IP_NAME: qa-viz
+    environment: ${{ inputs.environment || 'Staging' }}  #Assuming default trigger is cron job so will set env to Staging.
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Login with AZ
+        uses: ./.github/actions/azure-login
+        with:
+          az_tenant_id: ${{ secrets.AZ_TENANT_ID }}
+          az_subscription_id: ${{ secrets.AZ_SUBSCRIPTION_ID }}
+          az_client_id: ${{ secrets.AZ_CLIENT_ID }}
+          az_client_secret: ${{ secrets.AZ_CLIENT_SECRET }}
+
+      - name: Get workflow IP address
+        id: whats-my-ip
+        uses: ./.github/actions/whats-my-ip-address
+
+      - name: Add Azure firewall rules
+        uses: ./.github/actions/azure-ip-whitelist
+        with:
+          ip_address: ${{ steps.whats-my-ip.outputs.ip }}
+          verb: "add"
+          az_keyvault_name: ${{ env.az_keyvault_name }}
+          az_ip_name: ${{ env.SQL_IP_NAME }}
+          az_resource_group: ${{ env.az_resource_group_name}}
+          az_sql_database_server_name: ${{ env.az_sql_database_server_name }}
+
+      - name: Install Graphviz
+        run: sudo apt-get install graphviz 
+
+      - name: Install Microsoft ODBC
+        run: sudo ACCEPT_EULA=Y apt-get install msodbcsql17 -y
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+
+      - name: Install pyodbc
+        run: pip install pyodbc
+
+      - name: Install graphviz
+        run: pip install graphviz
+
+      - name: Install azure identity
+        run: pip install azure-identity
+
+      - name: Run data script
+        run: python ./.github/scripts/qa-data.py
+        env:
+          SQL_CONNECTION_STRING: "Driver={ODBC Driver 17 for SQL Server};Server=tcp:${{ env.az_sql_database_server_name }}.database.windows.net,1433;Database=${{ env.az_sql_database_server_name }}-sqldb;Encrypt=yes;TrustServerCertificate=no;Connection Timeout=30"
+
+      - name: List data files
+        run: ls -l ./data
+
+      - name: Remove Azure firewall rules
+        if: always()
+        uses: ./.github/actions/azure-ip-whitelist
+        with:
+          ip_address: ${{ steps.whats-my-ip.outputs.ip }}
+          verb: "remove"
+          az_keyvault_name: ${{ env.az_keyvault_name }}
+          az_ip_name: ${{ env.SQL_IP_NAME }}
+          az_resource_group: ${{ env.az_resource_group_name}}
+          az_sql_database_server_name: ${{ env.az_sql_database_server_name }}
+
+      - name: Run Visaulisation Script
+        run: python ./.github/scripts/qa-visualisations.py 
+
+      - name: Upload image
+        id: artifact-upload-step
+        uses: actions/upload-artifact@v4
+        with:
+          name: qa-visualisations
+          path: ./visualisations
+
+      - name: Call Teams Webhook
+        run: |
+            curl -X POST -H 'Content-Type: application/json' -d '{"text": "New Visualisations Available for ${{ inputs.environment }} : https://github.com/DFE-Digital/plan-technology-for-your-school/actions/runs/${{ github.run_id }}/artifacts/${{ steps.artifact-upload-step.outputs.artifact-id }}"}' ${{ secrets.PLANTECH_TEAMS_WEBHOOK }}
+