Merge branch 'development' into wip-csv-creator

.github/workflows/qa-viz.yml

@@ -10,6 +10,11 @@ on:
         type: choice
         options: ['Dev', 'Tst', 'Staging', 'Production']
         default: 'Staging'
+      recommendations:
+        type: boolean
+        description: Include recommendations
+        required: false
+        default: false
   workflow_call:
     inputs:
       environment:
@@ -106,6 +111,7 @@ jobs:
           PLANTECH_API_URL: https://localhost:8081/api/cms
           PLANTECH_API_KEY: ${{ steps.get-plantech-api-key.outputs.secret_value }}
           REQUESTS_CA_BUNDLE: localhost.crt
+          DISPLAY_RECOMMENDATIONS: ${{ inputs.recommendations }}
 
       - name: Remove Azure firewall rules
         uses: ./.github/actions/azure-ip-whitelist

contentful/qa-visualiser/README.md

@@ -26,6 +26,7 @@ To run the qa-visualiser locally:
    |------------------|--------------------------------------------------------|----------------------------------------------------|
    | PLANTECH_API_KEY | The API key the cms controller uses for authentication | Keyvault value for `api--authentication--keyvalue` |
    | PLANTECH_API_URL | Base url of the cms controller                         | https://localhost:8080/api/cms                     |
+    | DISPLAY_RECOMMENDATIONS | Boolean option to display recommendation header text linked to the answer on the image                    | true/false             |
 
 4. create and activate a virtual environment with:
     ```bash

contentful/qa-visualiser/main.py

@@ -1,6 +1,7 @@
 import logging
+import os
 
-from src.fetch_sections import fetch_sections
+from src.fetch_sections import fetch_recommendation_chunks, fetch_sections
 from src.generate_visualisations import process_sections
 
 logging.basicConfig(
@@ -10,8 +11,17 @@
 
 
 def main():
+    display_recommendations = os.getenv("DISPLAY_RECOMMENDATIONS", "false").lower() in [
+        "true",
+        "1",
+    ]
+    recommendation_map = []
+
+    if display_recommendations:
+        recommendation_map = fetch_recommendation_chunks()
+
     sections = fetch_sections()
-    process_sections(sections)
+    process_sections(sections, recommendation_map)
 
 
 if __name__ == "__main__":

contentful/qa-visualiser/src/fetch_sections.py

@@ -36,3 +36,70 @@ def fetch_sections() -> list[Section]:
     except (ValidationError, TypeError) as ex:
         logger.error(f"Error converting response to Sections: {ex}")
         raise ex
+
+
+def fetch_recommendation_chunks() -> dict[str, list[str]]:
+    """
+    Fetches all RecommendationChunks from the chunks API in /api/cms
+    Returns a dictionary mapping answerId to RecommendationHeader.
+    """
+    token = os.getenv("PLANTECH_API_KEY")
+    base_url = os.getenv("PLANTECH_API_URL")
+
+    total_items = []  # Store all results
+    page_number = 1  # Start from the first page
+
+    try:
+        logger.info(f"Fetching recommendation chunks from {base_url}/chunks/1")
+
+        while True:
+            response = requests.get(
+                f"{base_url}/chunks/{page_number}",
+                headers={
+                    "Accept": "application/json",
+                    "Authorization": f"Bearer {token}",
+                },
+            )
+            response.raise_for_status()
+
+            data = response.json()
+
+            items = data.get("items", [])
+
+            if not items:
+                logger.info(
+                    f"No more items on page {page_number}. Stopping pagination."
+                )
+                break
+
+            total_items.extend(items)
+            logger.info(
+                f"Retrieved {len(items)} items from page {page_number}, total so far: {len(total_items)}"
+            )
+
+            page_number += 1
+
+        recommendation_map = {}
+
+        for item in total_items:
+            answer_id = item.get("answerId")
+            recommendation_header = item.get(
+                "recommendationHeader"
+            )  # Ensure correct field name
+
+            if answer_id and recommendation_header:
+                if answer_id not in recommendation_map:
+                    recommendation_map[answer_id] = []  # Initialize as list
+
+                recommendation_map[answer_id].append(
+                    recommendation_header
+                )  # Store multiple recommendations
+
+        logger.info(
+            f"Successfully retrieved {len(recommendation_map)} unique answer mappings."
+        )
+        return recommendation_map
+
+    except (RequestException, TypeError) as ex:
+        logger.error(f"Error fetching recommendation chunks: {ex}")
+        return {}

contentful/qa-visualiser/src/generate_visualisations.py

@@ -25,16 +25,21 @@ def _create_blank_digraph() -> Digraph:
         format="png",
         graph_attr={
             "rankdir": "LR",
-            "beautify": "true",
+            "nodesep": "0.2",
+            "ranksep": "0.5",
+            "splines": "true",
         },
         edge_attr={
             "arrowhead": "vee",
-            "arrowsize": "0.5",
+            "arrowsize": "0.7",
+            "minlen": "2",
         },
     )
 
 
-def create_questionnaire_flowchart(section: Section) -> Digraph:
+def create_questionnaire_flowchart(
+    section: Section, recommendation_map: dict[str, str]
+) -> Digraph:
     """Create a graph of all possible paths you can take through a section"""
     tree = _create_blank_digraph()
 
@@ -62,9 +67,23 @@ def create_questionnaire_flowchart(section: Section) -> Digraph:
             gradient="200",
         )
 
+        created_recommendation_nodes = {}
+
         for answer in question.answers:
             answer_text = _wrap_text(answer.text, 20)
-
+            answer_id = answer.sys.id
+            answer_node_id = f"ans_{answer_id}"
+
+            tree.node(
+                answer_node_id,
+                answer_text,
+                shape="ellipse",
+                style="filled",
+                fillcolor="lightgrey:white",
+                width="1.5",
+            )
+
+            tree.edge(current_question_id, answer_node_id)  # Connect question to answer
             if next_question := answer.next_question:
                 next_question_id = next_question.sys.id
                 tree.node(
@@ -75,20 +94,68 @@ def create_questionnaire_flowchart(section: Section) -> Digraph:
                     fillcolor="red:white",
                     width="2",
                 )
-                tree.edge(current_question_id, next_question_id, label=answer_text)
+
+            if answer.next_question:
+                next_question_id = answer.next_question.sys.id
+
+                tree.node(
+                    next_question_id,
+                    "Next Question",
+                    shape="diamond",
+                    style="filled",
+                    fillcolor="blue:white",
+                    width="2",
+                )
+                tree.edge(
+                    answer_node_id, next_question_id
+                )  # Connect answer to next question
             else:
-                tree.edge(current_question_id, "end", label=answer_text)
+                tree.edge(answer_node_id, "end")  # Connect answer to end
+
+            # Ensure answer.sys.id exists and is in recommendation_map
+            if answer_id in recommendation_map:
+                recommendations = recommendation_map[
+                    answer_id
+                ]  # Get list of recommendations
+
+                for recommendation_text in recommendations:
+                    wrapped_text = _wrap_text(recommendation_text, 20)
+
+                    # Check if this recommendation text already has a node
+                    if wrapped_text not in created_recommendation_nodes:
+                        recommendation_node_id = f"rec_{hash(wrapped_text)}"  # Use hash to create a unique ID
+                        tree.node(
+                            recommendation_node_id,
+                            wrapped_text,
+                            shape="note",
+                            style="filled",
+                            fillcolor="yellow:white",
+                            width="2",
+                        )
+                        created_recommendation_nodes[wrapped_text] = (
+                            recommendation_node_id  # Store the node ID
+                        )
+
+                    # Connect answer to existing recommendation node
+                    tree.edge(
+                        answer_node_id,
+                        created_recommendation_nodes[wrapped_text],
+                        label="",
+                        color="red",
+                    )
 
     return tree
 
 
-def process_sections(sections: list[Section]) -> None:
+def process_sections(
+    sections: list[Section], recommendation_map: dict[str, str]
+) -> None:
     """Generates a graph for each section and saves it to the visualisations folder by section name"""
     png_folder = Path("visualisations")
     png_folder.mkdir(exist_ok=True)
 
     for section in sections:
         logger.info(f"Generating visualisation for section {section.name}")
         output_file = Path(png_folder, section.name)
-        flowchart = create_questionnaire_flowchart(section)
+        flowchart = create_questionnaire_flowchart(section, recommendation_map)
         flowchart.render(output_file, cleanup=True)

contentful/qa-visualiser/tests/test_generate_visualisations.py

@@ -1,23 +1,44 @@
 from unittest.mock import patch
 
-from src.generate_visualisations import create_questionnaire_flowchart
+from src.generate_visualisations import _wrap_text, create_questionnaire_flowchart
 from src.models import Section
 
 
 @patch("src.generate_visualisations.Digraph.node")
 @patch("src.generate_visualisations.Digraph.edge")
 def test_create_questionnaire_flowchart(mock_edge, mock_node, mock_sections):
+    mock_recommendation_map = {
+        "answer-1": ["Keep backups up-to-date"],
+        "answer-2": ["Ensure security protocols are followed"],
+    }
+
+    wrapped_recommendation_1 = _wrap_text("Keep backups up-to-date", 20)
+    wrapped_recommendation_2 = _wrap_text("Ensure security protocols are followed", 20)
+
+    rec_id_1 = f"rec_{hash(wrapped_recommendation_1)}"
+    rec_id_2 = f"rec_{hash(wrapped_recommendation_2)}"
+
     expected_nodes = {
         ("question-1", "First Question?"),
-        ("question-2", "Missing Content"),
         ("end", "Check Answers"),
+        ("question-2", "Next Question"),
+        ("ans_answer-1", "first answer"),
+        ("ans_answer-2", "second answer"),
+        (rec_id_1, wrapped_recommendation_1),
+        (rec_id_2, wrapped_recommendation_2),
+        ("question-2", "Missing Content"),
     }
     expected_edges = {
-        ("question-1", "question-2"),
-        ("question-1", "end"),
+        ("question-1", "ans_answer-1"),
+        ("question-1", "ans_answer-2"),
+        ("ans_answer-1", "question-2"),
+        ("ans_answer-2", "end"),
+        ("ans_answer-1", rec_id_1),
+        ("ans_answer-2", rec_id_2),
     }
+
     section = Section.model_validate(mock_sections[0])
-    create_questionnaire_flowchart(section)
+    create_questionnaire_flowchart(section, mock_recommendation_map)
 
     assert mock_node.call_count == len(expected_nodes)
     assert mock_edge.call_count == len(expected_edges)

src/Dfe.PlanTech.Application/Persistence/Interfaces/IContentRepository.cs

@@ -36,6 +36,10 @@ public interface IContentRepository
     /// <returns></returns>
     Task<IEnumerable<TEntity>> GetEntities<TEntity>(IGetEntitiesOptions options, CancellationToken cancellationToken = default);
 
+    Task<IEnumerable<TEntity>> GetPaginatedEntities<TEntity>(IGetEntitiesOptions options, CancellationToken cancellationToken = default);
+
+    Task<int> GetEntitiesCount<TEntity>(CancellationToken cancellationToken = default);
+
     /// <summary>
     /// Get entities without filtering
     /// </summary>

src/Dfe.PlanTech.Application/Persistence/Models/GetEntitiesOptions.cs

@@ -26,6 +26,10 @@ public GetEntitiesOptions()
     {
     }
 
+    public int Page { get; init; } = 1;
+
+    public int? Limit { get; init; }
+
     public IEnumerable<string>? Select { get; set; }
 
     public IEnumerable<IContentQuery>? Queries { get; init; }

src/Dfe.PlanTech.Application/Questionnaire/Queries/GetRecommendationQuery.cs

@@ -0,0 +1,39 @@
+using Dfe.PlanTech.Application.Core;
+using Dfe.PlanTech.Application.Exceptions;
+using Dfe.PlanTech.Application.Persistence.Interfaces;
+using Dfe.PlanTech.Application.Persistence.Models;
+using Dfe.PlanTech.Domain.Common;
+using Dfe.PlanTech.Domain.Questionnaire.Interfaces;
+using Dfe.PlanTech.Domain.Questionnaire.Models;
+
+namespace Dfe.PlanTech.Application.Questionnaire.Queries
+{
+    public class GetRecommendationQuery : ContentRetriever, IGetRecommendationQuery
+    {
+
+        public GetRecommendationQuery(IContentRepository repository) : base(repository)
+        {
+        }
+
+        /// <summary>
+        /// Returns recommendation chunks from contentful but only containing the system details ID and the header.
+        /// </summary>
+        public async Task<(IEnumerable<RecommendationChunk> Chunks, Pagination Pagination)> GetChunksByPage(int page, CancellationToken cancellationToken = default)
+        {
+            try
+            {
+                var totalEntries = await repository.GetEntitiesCount<RecommendationChunk>(cancellationToken);
+
+                var options = new GetEntitiesOptions(include: 3) { Page = page };
+                var result = await repository.GetPaginatedEntities<RecommendationChunk>(options, cancellationToken);
+
+                return (result, new Pagination() { Page = page, Total = totalEntries });
+
+            }
+            catch (Exception ex)
+            {
+                throw new ContentfulDataUnavailableException("Error getting recommendation chunks from Contentful", ex);
+            }
+        }
+    }
+}

src/Dfe.PlanTech.Domain/Common/Pagination.cs

@@ -0,0 +1,8 @@
+namespace Dfe.PlanTech.Domain.Common
+{
+    public class Pagination
+    {
+        public int Total { get; set; }
+        public int Page { get; set; }
+    }
+}

src/Dfe.PlanTech.Domain/Persistence/Interfaces/IGetEntitiesOptions.cs

@@ -20,5 +20,8 @@ public interface IGetEntitiesOptions
     /// </remarks>
     public IEnumerable<string>? Select { get; set; }
 
+    public int? Limit { get; init; }
+    public int Page { get; init; }
+
     public string SerializeToRedisFormat();
 }

src/Dfe.PlanTech.Domain/Questionnaire/Interfaces/IGetRecommendationQuery.cs

@@ -0,0 +1,11 @@
+using Dfe.PlanTech.Domain.Common;
+using Dfe.PlanTech.Domain.Questionnaire.Models;
+
+namespace Dfe.PlanTech.Domain.Questionnaire.Interfaces
+{
+    public interface IGetRecommendationQuery
+    {
+        public Task<(IEnumerable<RecommendationChunk> Chunks, Pagination Pagination)> GetChunksByPage(int page, CancellationToken cancellationToken = default);
+
+    }
+}