Support disconnected environment for Pipeline SDK tests

ods_ci/libs/DataSciencePipelinesKfp.py

@@ -2,9 +2,11 @@
 import json
 import os
 import sys
+import tempfile
 import time
 
 from DataSciencePipelinesAPI import DataSciencePipelinesAPI
+from kfp import compiler
 from kfp.client import Client
 from robotlibcore import keyword
 
@@ -89,7 +91,17 @@ def delete_run(self, run_id):
 
     @keyword
     def create_run_from_pipeline_func(
-        self, user, pwd, project, source_code, fn, pipeline_params={}, current_path=None, route_name="ds-pipeline-dspa"
+        self,
+        user,
+        pwd,
+        project,
+        source_code,
+        fn,
+        pipeline_params={},
+        current_path=None,
+        route_name="ds-pipeline-dspa",
+        pip_index_url=None,
+        pip_trusted_host=None,
     ):
         print(f"pipeline_params: {pipeline_params}")
         client, api = self.get_client(user, pwd, project, route_name)
@@ -101,7 +113,7 @@ def create_run_from_pipeline_func(
         if current_path is None:
             current_path = os.getcwd()
         my_source = self.import_souce_code(f"{current_path}/tests/Resources/Files/pipeline-samples/v2/{source_code}")
-        pipeline = getattr(my_source, fn)
+        pipeline_func = getattr(my_source, fn)
 
         # pipeline_params
         # there are some special keys to retrieve argument values dynamically
@@ -118,7 +130,27 @@ def create_run_from_pipeline_func(
 
         # create_run_from_pipeline_func will compile the code
         # if you need to see the yaml, for debugging purpose, call: TektonCompiler().compile(pipeline, f'{fn}.yaml')
-        result = client.create_run_from_pipeline_func(pipeline_func=pipeline, arguments=pipeline_params)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            pipeline_package_path = os.path.join(tmpdir, "pipeline.yaml")
+            compiler.Compiler().compile(
+                pipeline_func=pipeline_func,
+                package_path=pipeline_package_path,
+            )
+
+            if pip_index_url is not None:
+                assert pip_trusted_host is not None
+                with open(pipeline_package_path, "r") as file:
+                    file_content = file.read()
+                file_content = file_content.replace(
+                    "python3 -m pip install",
+                    f"python3 -m pip install --index-url {pip_index_url} --trusted-host {pip_trusted_host}",
+                )
+                with open(pipeline_package_path, "w") as file:
+                    file.write(file_content)
+
+            result = client.create_run_from_pipeline_package(
+                pipeline_file=pipeline_package_path, arguments=pipeline_params
+            )
         # easy to debug and double check failures
         print(result)
         return result.run_id

ods_ci/tests/Tests/1100__data_science_pipelines/1100__data-science-pipelines-kfp.robot

@@ -95,8 +95,14 @@ End To End Pipeline Workflow Using Kfp
     IF    ${ray} == ${TRUE}
         Setup Kueue Resources    ${project}    cluster-queue-user    resource-flavor-user    local-queue-user
     END
+    # The run_robot_test.sh is sending the --variablefile ${TEST_VARIABLES_FILE} which may contain the `PIP_INDEX_URL`
+    # and `PIP_TRUSTED_HOST` variables, e.g. for disconnected testing.
+    ${pip_index_url} =    Get Variable Value    ${PIP_INDEX_URL}    ${NONE}
+    ${pip_trusted_host} =    Get Variable Value    ${PIP_TRUSTED_HOST}    ${NONE}
+    Log    pip_index_url = ${pip_index_url} / pip_trusted_host = ${pip_trusted_host}
     ${run_id}    Create Run From Pipeline Func    ${username}    ${password}    ${project}
-    ...    ${python_file}    ${method_name}    pipeline_params=${pipeline_params}
+    ...    ${python_file}    ${method_name}    pipeline_params=${pipeline_params}    pip_index_url=${pip_index_url}
+    ...    pip_trusted_host=${pip_trusted_host}
     ${run_status}    Check Run Status    ${run_id}    timeout=500
     Should Be Equal As Strings    ${run_status}    SUCCEEDED    Pipeline run doesn't have a status that means success. Check the logs
     Remove Pipeline Project    ${project}