Load the list of domains from Datahub

Previously we hardcoded the list of domains shown in the search filter,
and had different lists per environment.

This was useful in alpha when we had some junk domains we wanted to
filter out, but now we're at a point where every domain in Datahub
should be one we want to use.

This commit means we now fetch every domain that has something linked to
it, and display that in alphabetical order.

Author: MatMoore

home/forms/domain_model.py

@@ -1,6 +1,10 @@
-import os
+import logging
 from typing import NamedTuple
 
+from home.service.search_facet_fetcher import SearchFacetFetcher
+
+logger = logging.getLogger(__name__)
+
 
 class Domain(NamedTuple):
     urn: str
@@ -12,109 +16,22 @@ class DomainModel:
     Store information about domains and subdomains
     """
 
-    def __init__(self):
+    def __init__(self, search_facet_fetcher: SearchFacetFetcher):
         self.labels = {}
-        # This is temporary whilst we still have the dev enviroment connected to a
-        # datahub with different domains.
-        if os.environ.get("ENV") == "dev":
-            self.top_level_domains = [
-                Domain("urn:li:domain:HMCTS", "HMCTS"),
-                Domain("urn:li:domain:HMPPS", "HMPPS"),
-                Domain("urn:li:domain:HQ", "HQ"),
-                Domain("urn:li:domain:LAA", "LAA"),
-                Domain("urn:li:domain:OPG", "OPG"),
-            ]
 
-            for urn, label in self.top_level_domains:
-                self.labels[urn] = label
+        search_facets = search_facet_fetcher.fetch()
+        self.top_level_domains = [
+            Domain(option.value, option.label)
+            for option in search_facets.options("domains")
+        ]
+        self.top_level_domains.sort(key=lambda d: d.label)
+
+        logger.info(f"{self.top_level_domains=}")
 
-            self.subdomains = {
-                "urn:li:domain:HMPPS": [
-                    Domain(
-                        "urn:li:domain:2feb789b-44d3-4412-b998-1f26819fabf9", "Prisons"
-                    ),
-                    Domain(
-                        "urn:li:domain:abe153c1-416b-4abb-be7f-6accf2abb10a",
-                        "Probation",
-                    ),
-                ],
-                "urn:li:domain:HMCTS": [
-                    Domain(
-                        "urn:li:domain:4d77af6d-9eca-4c44-b189-5f1addffae55",
-                        "Civil courts",
-                    ),
-                    Domain(
-                        "urn:li:domain:31754f66-33df-4a73-b039-532518bc765e",
-                        "Crown courts",
-                    ),
-                    Domain(
-                        "urn:li:domain:81adfe94-1284-46a2-9179-945ad2a76c14",
-                        "Family courts",
-                    ),
-                    Domain(
-                        "urn:li:domain:b261176c-d8eb-4111-8454-c0a1fa95005f",
-                        "Magistrates courts",
-                    ),
-                ],
-                "urn:li:domain:OPG": [
-                    Domain(
-                        "urn:li:domain:bc091f6c-7674-4c82-a315-f5489398f099",
-                        "Lasting power of attourney",
-                    ),
-                    Domain(
-                        "urn:li:domain:efb9ade3-3c5d-4c5c-b451-df9f2d8136f5",
-                        "Supervision orders",
-                    ),
-                ],
-                "urn:li:domain:HQ": [
-                    Domain(
-                        "urn:li:domain:9fb7ff13-6c7e-47ef-bef1-b13b23fd8c7a", "Estates"
-                    ),
-                    Domain(
-                        "urn:li:domain:e4476e66-37a1-40fd-83b9-c908f805d8f4", "Finance"
-                    ),
-                    Domain(
-                        "urn:li:domain:0985731b-8e1c-4b4a-bfc0-38e58d8ba8a1", "People"
-                    ),
-                    Domain(
-                        "urn:li:domain:a320c915-0b43-4277-9769-66615aab4adc",
-                        "Performance",
-                    ),
-                ],
-                "urn:li:domain:LAA": [
-                    Domain(
-                        "urn:li:domain:24344488-d770-437a-ba6f-e6129203b927",
-                        "Civil legal advice",
-                    ),
-                    Domain("urn:li:domain:Legal%20Aid", "Legal aid"),
-                    Domain(
-                        "urn:li:domain:5c423c06-d328-431f-8634-7a7e86928819",
-                        "Public defender",
-                    ),
-                ],
-            }
-            for domain, subdomains in self.subdomains.items():
-                domain_label = self.labels[domain]
-                for urn, subdoman_label in subdomains:
-                    self.labels[urn] = f"{domain_label} - {subdoman_label}"
-        else:
-            self.top_level_domains = [
-                Domain("urn:li:domain:courts", "Courts"),
-                Domain("urn:li:domain:electronic_monitoring", "Electronic monitoring"),
-                Domain("urn:li:domain:general", "General"),
-                Domain("urn:li:domain:interventions", "Interventions"),
-                Domain("urn:li:domain:opg", "OPG"),
-                Domain("urn:li:domain:prison", "Prison"),
-                Domain("urn:li:domain:probation", "Probation"),
-                Domain("urn:li:domain:risk", "Risk"),
-                Domain(
-                    "urn:li:domain:victims_case_management", "Victims case management"
-                ),
-            ]
-            self.subdomains = {}
+        self.subdomains = {}
 
-            for urn, label in self.top_level_domains:
-                self.labels[urn] = label
+        for urn, label in self.top_level_domains:
+            self.labels[urn] = label
 
     def all_subdomains(self) -> list[Domain]:  # -> list[Any]
         """

home/forms/search.py

@@ -4,6 +4,7 @@
 from data_platform_catalogue.search_types import ResultType
 from django import forms
 
+from ..service.search_facet_fetcher import SearchFacetFetcher
 from .domain_model import Domain, DomainModel
 
 
@@ -12,13 +13,13 @@ def get_domain_choices() -> list[Domain]:
     choices = [
         Domain("", "All domains"),
     ]
-    choices.extend(DomainModel().top_level_domains)
+    choices.extend(DomainModel(SearchFacetFetcher()).top_level_domains)
     return choices
 
 
 def get_subdomain_choices() -> list[Domain]:
     choices = [Domain("", "All subdomains")]
-    choices.extend(DomainModel().all_subdomains())
+    choices.extend(DomainModel(SearchFacetFetcher()).all_subdomains())
     return choices
 
 
@@ -47,8 +48,7 @@ def get_entity_types():
 class SelectWithOptionAttribute(forms.Select):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-
-        self.domain_model = DomainModel()
+        self.domain_model = None
 
     def create_option(
         self, name, urn, label, selected, index, subindex=None, attrs=None
@@ -57,6 +57,8 @@ def create_option(
             name, urn, label, selected, index, subindex, attrs
         )
 
+        self.domain_model = self.domain_model or DomainModel(SearchFacetFetcher())
+
         if urn:
             option["attrs"]["data-parent"] = self.domain_model.get_parent_urn(urn)
 

home/service/search.py

@@ -16,9 +16,12 @@
 from home.forms.search import SearchForm
 
 from .base import GenericService
+from .search_facet_fetcher import SearchFacetFetcher
 
 
-def domains_with_their_subdomains(domain: str, subdomain: str) -> list[str]:
+def domains_with_their_subdomains(
+    domain: str, subdomain: str, domain_model: DomainModel
+) -> list[str]:
     """
     Users can search by domain, and optionally by subdomain.
     When subdomain is passed, then we can filter on that directly.
@@ -30,14 +33,15 @@ def domains_with_their_subdomains(domain: str, subdomain: str) -> list[str]:
     if subdomain:
         return [subdomain]
 
-    subdomains = DomainModel().subdomains.get(domain, [])
+    subdomains = domain_model.subdomains.get(domain, [])
     subdomains = [subdomain[0] for subdomain in subdomains]
     return [domain, *subdomains] if not domain == "" else []
 
 
 class SearchService(GenericService):
     def __init__(self, form: SearchForm, page: str, items_per_page: int = 20):
-        self.domain_model = DomainModel()
+        search_facet_fetcher = SearchFacetFetcher()
+        self.domain_model = DomainModel(search_facet_fetcher)
         self.stemmer = PorterStemmer()
         self.form = form
         if self.form.is_bound:
@@ -76,7 +80,9 @@ def _get_search_results(self, page: str, items_per_page: int) -> SearchResponse:
         sort = form_data.get("sort", "relevance")
         domain = form_data.get("domain", "")
         subdomain = form_data.get("subdomain", "")
-        domains_and_subdomains = domains_with_their_subdomains(domain, subdomain)
+        domains_and_subdomains = domains_with_their_subdomains(
+            domain, subdomain, self.domain_model
+        )
         where_to_access = self._build_custom_property_filter(
             "whereToAccessDataset=", form_data.get("where_to_access", [])
         )

home/service/search_facet_fetcher.py

@@ -0,0 +1,16 @@
+from data_platform_catalogue.search_types import SearchFacets
+
+from .base import GenericService
+
+
+class SearchFacetFetcher(GenericService):
+    def __init__(self):
+        self.client = self._get_catalogue_client()
+
+    def fetch(self) -> SearchFacets:
+        """
+        Fetch a static list of options that is independent of the search query
+        and any applied filters.
+        TODO: this can be cached in memory to speed up future requests
+        """
+        return self.client.search_facets()

tests/conftest.py

@@ -3,8 +3,6 @@
 from typing import Any
 from unittest.mock import MagicMock, patch
 
-from django.conf import settings
-
 import pytest
 from data_platform_catalogue.client.datahub_client import DataHubCatalogueClient
 from data_platform_catalogue.entities import (
@@ -29,13 +27,15 @@
     SearchResponse,
     SearchResult,
 )
-
+from django.conf import settings
 from django.test import Client
 from faker import Faker
 
+from home.forms.domain_model import DomainModel
 from home.forms.search import SearchForm
 from home.service.details import DatabaseDetailsService
 from home.service.search import SearchService
+from home.service.search_facet_fetcher import SearchFacetFetcher
 
 fake = Faker()
 
@@ -146,22 +146,6 @@ def generate_page(page_size=20, result_type: ResultType | None = None):
     return results
 
 
-def generate_options(num_options=5):
-    """
-    Generate a list of options for the search facets
-    """
-    results = []
-    for _ in range(num_options):
-        results.append(
-            FacetOption(
-                value=fake.name(),
-                label=fake.name(),
-                count=fake.random_int(min=0, max=100),
-            )
-        )
-    return results
-
-
 @pytest.fixture(autouse=True)
 def client():
     client = Client()
@@ -181,7 +165,26 @@ def mock_catalogue(request):
     mock_search_response(
         mock_catalogue, page_results=generate_page(), total_results=100
     )
-    mock_search_facets_response(mock_catalogue, domains=generate_options())
+    mock_search_facets_response(
+        mock_catalogue,
+        domains=[
+            FacetOption(
+                value="urn:li:domain:prisons",
+                label="Prisons",
+                count=fake.random_int(min=0, max=100),
+            ),
+            FacetOption(
+                value="urn:li:domain:courts",
+                label="Courts",
+                count=fake.random_int(min=0, max=100),
+            ),
+            FacetOption(
+                value="urn:li:domain:finance",
+                label="Finance",
+                count=fake.random_int(min=0, max=100),
+            ),
+        ],
+    )
     mock_get_glossary_terms_response(mock_catalogue)
     mock_list_database_tables_response(
         mock_catalogue,
@@ -296,11 +299,16 @@ def mock_get_glossary_terms_response(mock_catalogue):
 
 
 @pytest.fixture
-def valid_form():
+def valid_domain():
+    return DomainModel(SearchFacetFetcher()).top_level_domains[0]
+
+
+@pytest.fixture
+def valid_form(valid_domain):
     valid_form = SearchForm(
         data={
             "query": "test",
-            "domain": "urn:li:domain:prison",
+            "domain": valid_domain.urn,
             "entity_types": ["TABLE"],
             "where_to_access": ["analytical_platform"],
             "sort": "ascending",

tests/home/service/test_search.py

@@ -1,6 +1,7 @@
 import os
 import re
 from types import GeneratorType
+from urllib.parse import quote
 
 import pytest
 
@@ -29,9 +30,9 @@ def test_get_context_paginator(self, search_context):
     def test_get_context_h1_value(self, search_context):
         assert search_context["h1_value"] == "Search"
 
-    def test_get_context_label_clear_href(self, search_context):
+    def test_get_context_label_clear_href(self, search_context, valid_domain):
         assert search_context["label_clear_href"]["domain"] == {
-            "Prison": (
+            valid_domain.label: (
                 "?query=test&"
                 "where_to_access=analytical_platform&"
                 "entity_types=TABLE&"
@@ -44,7 +45,7 @@ def test_get_context_label_clear_href(self, search_context):
         assert search_context["label_clear_href"]["Where To Access"] == {
             "analytical_platform": (
                 "?query=test&"
-                "domain=urn%3Ali%3Adomain%3Aprison&"
+                f"domain={quote(valid_domain.urn)}&"
                 "subdomain=&"
                 "entity_types=TABLE&"
                 "sort=ascending&"
@@ -56,7 +57,7 @@ def test_get_context_label_clear_href(self, search_context):
         assert search_context["label_clear_href"]["Entity Types"] == {
             "Table": (
                 "?query=test&"
-                "domain=urn%3Ali%3Adomain%3Aprison&"
+                f"domain={quote(valid_domain.urn)}&"
                 "subdomain=&"
                 "where_to_access=analytical_platform&"
                 "sort=ascending&"