Fmd 366 add dataset lineage link (#416)

* add upstream and downstream lineage to getDatasetDetails graphql query

* refactor parse_relations() helper to handle more relations

* add upstream and downstream lineage to RelationshipType enum

* update parse_relations() input args

* update parse_relations() input args in search

* add has_lineage and lineage_url to dataset details context

* add lineage link to details_table template

* remove redundant block in query for data product relationships

* return entity name for lineage

* have only 1 RelationshipType for lineage

* simplfy `parse_relations()` helper function

* update DatasetDetails to use single lineage type

* align url to rest of table

* update tests

* add default value for url

* design suggestions for lineage label, from Alex and Jess

* spell it right

* suggestions from Mat

Author: LavMatt

home/service/details.py

@@ -1,6 +1,8 @@
+import os
 from data_platform_catalogue.entities import RelationshipType
 from data_platform_catalogue.search_types import MultiSelectFilter, ResultType
 from django.core.exceptions import ObjectDoesNotExist
+from urllib.parse import urlsplit
 
 from .base import GenericService
 
@@ -86,13 +88,32 @@ def __init__(self, urn: str):
         self.context = self._get_context()
 
     def _get_context(self):
+        split_datahub_url = urlsplit(
+            os.getenv("CATALOGUE_URL", "https://test-catalogue.gov.uk")
+        )
+
         return {
             "table": self.table_metadata,
             "parent_entity": self.parent_entity,
             "dataset_parent_type": self.dataset_parent_type,
             "h1_value": "Details",
+            "has_lineage": self.has_lineage(),
+            "lineage_url": f"{split_datahub_url.scheme}://{split_datahub_url.netloc}/dataset/{self.table_metadata.urn}/Lineage?is_lineage_mode=true&",
         }
 
+    def has_lineage(self) -> bool:
+        """
+        Inspects the relationships property of the Table model to establish if a
+        Dataset has any lineage recorded in datahub.
+        """
+        has_lineage = (
+            len(
+                self.table_metadata.relationships.get(RelationshipType.DATA_LINEAGE, [])
+            )
+            > 0
+        )
+        return has_lineage
+
 
 class ChartDetailsService(GenericService):
     def __init__(self, urn: str):

lib/datahub-client/data_platform_catalogue/client/datahub_client.py

@@ -232,21 +232,23 @@ def get_table_details(self, urn) -> Table:
             created, modified = parse_created_and_modified(properties)
             name, display_name, qualified_name = parse_names(response, properties)
 
-            # A dataset can't have multiple parents, but if we did
-            # start to use in that we'd need to change this
-            if response["container_relations"]["total"] > 0:
-                relations = parse_relations(
-                    RelationshipType.PARENT, response["container_relations"]
-                )
-            else:
-                relations = {}
+            lineage_relations = parse_relations(
+                RelationshipType.DATA_LINEAGE,
+                [
+                    response.get("downstream_lineage_relations", {}),
+                    response.get("upstream_lineage_relations", {}),
+                ],
+            )
+            parent_relations = parse_relations(
+                RelationshipType.PARENT, [response["parent_container_relations"]]
+            )
             return Table(
                 urn=urn,
                 display_name=display_name,
                 name=name,
                 fully_qualified_name=qualified_name,
                 description=properties.get("description", ""),
-                relationships=relations,
+                relationships={**lineage_relations, **parent_relations},
                 domain=domain,
                 governance=Governance(
                     data_owner=owner,

lib/datahub-client/data_platform_catalogue/client/graphql/getDatasetDetails.graphql

@@ -14,10 +14,50 @@ query getDatasetDetails($urn: String!) {
         }
       }
     }
+    downstream_lineage_relations: lineage (
+      input: {direction: DOWNSTREAM
+      start:0,
+      count:10}
+    ) {
+      total
+      relationships{
+        type
+        entity{
+          urn
+          ... on Dataset {
+            name
+            properties{
+              name
+            }
+          }
+          type
+        }
+      }
+    }
+    upstream_lineage_relations: lineage (
+      input: {direction: UPSTREAM
+      start:0,
+      count:10}
+    ) {
+      total
+      relationships{
+        type
+        entity{
+          urn
+          ... on Dataset {
+            name
+            properties{
+              name
+            }
+          }
+          type
+        }
+      }
+    }
     subTypes {
       typeNames
     }
-    container_relations: relationships(
+    parent_container_relations: relationships(
       input: { types: ["IsPartOf"], direction: OUTGOING, count: 10 }
     ) {
       total

lib/datahub-client/data_platform_catalogue/client/graphql_helpers.py

@@ -256,21 +256,26 @@ def parse_columns(entity: dict[str, Any]) -> list[Column]:
 
 
 def parse_relations(
-    relationship_type: RelationshipType, relations_dict: dict
+    relationship_type: RelationshipType, relations_list: list[dict]
 ) -> dict[RelationshipType, list[EntityRef]]:
     """
     parse the relationships results returned from a graphql querys
     """
-    # we may want to do soemthing with total realtion if we are returning child
+
+    # we may want to do something with total relations if we are returning child
     # relations and need to paginate through relations - 10 relations returned as is
-    # total_relations = relations_dict.get("total", 0)
-    parent_entities = relations_dict.get("relationships", [])
-    related_entities = [
-        EntityRef(
-            urn=i["entity"]["urn"], display_name=i["entity"]["properties"]["name"]
-        )
-        for i in parent_entities
-    ]
+    # There may be more than 10 lineage entities but since we currently only care
+    # if lineage exists for a dataset we don't need to capture everything    
+    related_entities = []
+    for j in relations_list:
+        for i in j["relationships"]:
+            urn = i.get("entity").get("urn")
+            display_name = (
+                i.get("entity").get("properties").get("name")
+                if i.get("entity", {}).get("properties") is not None
+                else i.get("entity").get("name")
+            )
+            related_entities.append(EntityRef(urn=urn, display_name=display_name))
 
     relations_return = {relationship_type: related_entities}
     return relations_return

lib/datahub-client/data_platform_catalogue/client/search.py

@@ -12,7 +12,6 @@
     parse_names,
     parse_owner,
     parse_properties,
-    parse_relations,
     parse_tags,
 )
 from data_platform_catalogue.entities import RelationshipType
@@ -259,16 +258,12 @@ def _parse_result(
         last_modified = parse_last_modified(entity)
         name, display_name, qualified_name = parse_names(entity, properties)
 
-        relations = parse_relations(
-            RelationshipType.PARENT, entity.get("relationships", {})
-        )
         domain = parse_domain(entity)
 
         metadata = {
             "owner": owner.display_name,
             "owner_email": owner.email,
             "total_parents": entity.get("relationships", {}).get("total", 0),
-            "parents": relations[RelationshipType.PARENT],
             "domain_name": domain.display_name,
             "domain_id": domain.urn,
             "entity_types": self._parse_types_and_sub_types(entity, "Dataset"),

lib/datahub-client/data_platform_catalogue/entities.py

@@ -10,6 +10,7 @@
 class RelationshipType(Enum):
     PARENT = "PARENT"
     PLATFORM = "PLATFORM"
+    DATA_LINEAGE = "DATA_LINEAGE"
 
 
 class EntityRef(BaseModel):

lib/datahub-client/tests/client/datahub/test_datahub_client.py

@@ -243,17 +243,22 @@ def test_get_dataset(
         urn = "abc"
         datahub_response = {
             "dataset": {
+                "type": "DATASET",
                 "platform": {"name": "datahub"},
                 "ownership": None,
                 "subTypes": None,
-                "container_relations": {
+                "downstream_lineage_relations": {"total": 0, "relationships": []},
+                "upstream_lineage_relations": {"total": 0, "relationships": []},
+                "parent_container_relations": {
                     "total": 1,
                     "relationships": [
                         {
+                            "type": "IsPartOf",
+                            "direction": "OUTGOING",
                             "entity": {
                                 "urn": "urn:li:container:database",
                                 "properties": {"name": "database"},
-                            }
+                            },
                         }
                     ],
                 },
@@ -325,7 +330,8 @@ def test_get_dataset(
             relationships={
                 RelationshipType.PARENT: [
                     EntityRef(urn="urn:li:container:database", display_name="database")
-                ]
+                ],
+                RelationshipType.DATA_LINEAGE: [],
             },
             domain=DomainRef(display_name="", urn=""),
             governance=Governance(
@@ -369,9 +375,10 @@ def test_get_dataset_minimal_properties(
                 "platform": {"name": "datahub"},
                 "name": "notinproperties",
                 "properties": {},
-                "container_relations": {
-                    "total": 0,
-                },
+                "downstream_lineage_relations": {"total": 0, "relationships": []},
+                "upstream_lineage_relations": {"total": 0, "relationships": []},
+                "parent_container_relations": {"total": 0, "relationships": []},
+                "data_product_relations": {"total": 0, "relationships": []},
                 "schemaMetadata": {"fields": []},
             }
         }
@@ -389,7 +396,10 @@ def test_get_dataset_minimal_properties(
             name="notinproperties",
             fully_qualified_name="notinproperties",
             description="",
-            relationships={},
+            relationships={
+                RelationshipType.PARENT: [],
+                RelationshipType.DATA_LINEAGE: [],
+            },
             domain=DomainRef(display_name="", urn=""),
             governance=Governance(
                 data_owner=OwnerRef(display_name="", email="", urn=""),

lib/datahub-client/tests/client/datahub/test_graphql_helpers.py

@@ -191,7 +191,7 @@ def test_parse_relations():
             ],
         }
     }
-    result = parse_relations(RelationshipType.PARENT, relations["relationships"])
+    result = parse_relations(RelationshipType.PARENT, [relations["relationships"]])
     assert result == {
         RelationshipType.PARENT: [
             EntityRef(urn="urn:li:dataProduct:test", display_name="test")
@@ -201,7 +201,7 @@ def test_parse_relations():
 
 def test_parse_relations_blank():
     relations = {"relationships": {"total": 0, "relationships": []}}
-    result = parse_relations(RelationshipType.PARENT, relations["relationships"])
+    result = parse_relations(RelationshipType.PARENT, [relations["relationships"]])
     assert result == {RelationshipType.PARENT: []}
 
 

lib/datahub-client/tests/client/datahub/test_search.py

@@ -123,7 +123,6 @@ def test_one_search_result(mock_graph, searcher):
                     "owner": "",
                     "owner_email": "",
                     "total_parents": 0,
-                    "parents": [],
                     "domain_name": "HMPPS",
                     "domain_id": "urn:li:domain:3dc18e48-c062-4407-84a9-73e23f768023",
                     "entity_types": {
@@ -210,7 +209,6 @@ def test_dataset_result(mock_graph, searcher):
                     "owner": "",
                     "owner_email": "",
                     "total_parents": 0,
-                    "parents": [],
                     "domain_name": "HMPPS",
                     "domain_id": "urn:li:domain:3dc18e48-c062-4407-84a9-73e23f768023",
                     "entity_types": {
@@ -302,7 +300,6 @@ def test_full_page(mock_graph, searcher):
                     "owner": "",
                     "owner_email": "",
                     "total_parents": 0,
-                    "parents": [],
                     "domain_name": "",
                     "domain_id": "",
                     "entity_types": {
@@ -334,7 +331,6 @@ def test_full_page(mock_graph, searcher):
                     "owner": "",
                     "owner_email": "",
                     "total_parents": 0,
-                    "parents": [],
                     "domain_name": "",
                     "domain_id": "",
                     "entity_types": {
@@ -364,7 +360,6 @@ def test_full_page(mock_graph, searcher):
                     "owner": "",
                     "owner_email": "",
                     "total_parents": 0,
-                    "parents": [],
                     "domain_name": "",
                     "domain_id": "",
                     "entity_types": {
@@ -444,7 +439,6 @@ def test_query_match(mock_graph, searcher):
                     "owner": "",
                     "owner_email": "",
                     "total_parents": 0,
-                    "parents": [],
                     "domain_name": "",
                     "domain_id": "",
                     "entity_types": {
@@ -520,7 +514,6 @@ def test_result_with_owner(mock_graph, searcher):
                     "owner": "Shannon Lovett",
                     "owner_email": "shannon@longtail.com",
                     "total_parents": 0,
-                    "parents": [],
                     "domain_name": "",
                     "domain_id": "",
                     "entity_types": {
@@ -891,7 +884,6 @@ def test_search_for_charts(mock_graph, searcher):
                     "owner": "",
                     "owner_email": "",
                     "total_parents": 0,
-                    "parents": [],
                     "domain_name": "",
                     "domain_id": "",
                     "entity_types": {
@@ -1091,7 +1083,6 @@ def test_list_database_tables(mock_graph, searcher):
                     "owner": "",
                     "owner_email": "",
                     "total_parents": 0,
-                    "parents": [],
                     "domain_name": "",
                     "domain_id": "",
                     "entity_types": {

templates/details_table.html

@@ -78,9 +78,17 @@ <h3 class="govuk-heading-s govuk-!-margin-top-3">
         <h2 class="govuk-heading-m">Table schema</h2>
         <p class="govuk-body">The schema for this table is not available.</p>
       {% endif %}
-
-
-    </div>
+      {% if has_lineage %}
+      <h2 class="govuk-heading-m">Lineage</h2>
+      <div class="govuk-body-m" >
+        If you are interested to find out what data were used to create this table or if this table is used to create any further tables, you can see that information via the lineage.
+      </div class="govuk-body-m">  
+      <div class="govuk-body">
+            <a href="{{lineage_url}}" class="govuk-link">
+              View lineage in DataHub
+            </a>
+        </div>
+  {% endif %}
   </div>
 
 {% endblock content %}

tests/benchmark/test_exact_matches.py

@@ -15,8 +15,8 @@
     "query,expected_urn",
     [
         (
-            "prison_population_history.chunk_assignment",
-            "urn:li:dataset:(urn:li:dataPlatform:dbt,cadet.awsdatacatalog.prison_population_history.chunk_assignment,PROD)",
+            "bold_common_platform_linked_tables.all_offence",
+            "urn:li:dataset:(urn:li:dataPlatform:dbt,cadet.awsdatacatalog.bold_common_platform_linked_tables.all_offence,PROD)",
         ),
         (
             "Accommodation on the first night following release",

tests/conftest.py

@@ -98,7 +98,8 @@ def generate_table_metadata(
         name=name or fake.unique.name(),
         fully_qualified_name="Foo.Dataset",
         description=description or fake.paragraph(),
-        relationships=relations or {RelationshipType.PARENT: []},
+        relationships=relations
+        or {RelationshipType.PARENT: [], RelationshipType.DATA_LINEAGE: []},
         domain=DomainRef(display_name="LAA", urn="LAA"),
         governance=Governance(
             data_owner=OwnerRef(