Add SEP TC for Monitoring

tests/Resources/Page/ODH/Prometheus/Prometheus.robot

@@ -6,16 +6,29 @@ Library             Process
 Library             RequestsLibrary
 
 *** Keywords ***
+Get Observatorium Token
+    [Documentation]  Returns the token to access observatorium
+    ${data}=    Create Dictionary     grant_type=client_credentials  client_id=${STARBURST.OBS_CLIENT_ID}  client_secret=${STARBURST.OBS_CLIENT_SECRET}
+    ${resp}=     RequestsLibrary.POST  ${STARBURST.OBS_TOKEN_URL}   data=${data}
+    [Return]    ${resp.json()['access_token']}
+
 Run Query
     [Documentation]    Runs a prometheus query, obtaining the current value. More info at:
     ...                - https://promlabs.com/blog/2020/06/18/the-anatomy-of-a-promql-query
     ...                - https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
-    [Arguments]    ${pm_url}    ${pm_token}    ${pm_query}
+    [Arguments]    ${pm_url}    ${pm_token}    ${pm_query}  ${project}=RHODS
     ${pm_headers}=    Create Dictionary    Authorization=Bearer ${pm_token}
-    ${resp}=    RequestsLibrary.GET    url=${pm_url}/api/v1/query?query=${pm_query}
-    ...    headers=${pm_headers}    verify=${False}
-    Status Should Be    200    ${resp}
-    # Log To Console    ${resp.json()}
+
+    IF  "${project}" == "SERH"
+        ${resp}=    RequestsLibrary.GET    url=${pm_url}?query=${pm_query}
+        ...    headers=${pm_headers}    verify=${False}
+        Status Should Be    200    ${resp}
+
+    ELSE
+        ${resp}=    RequestsLibrary.GET    url=${pm_url}/api/v1/query?query=${pm_query}
+        ...    headers=${pm_headers}    verify=${False}
+        Status Should Be    200    ${resp}
+    END
     [Return]    ${resp}
 
 Run Range Query
@@ -216,15 +229,15 @@ Get Target Endpoints Which Have State Up
     ${links}=    Replace String    ${links}    "    ${EMPTY}
     @{links}=    Split String  ${links}  \n
     [Return]    ${links}
-    
-Get Date When Availability Value Matches Expected Value 
+
+Get Date When Availability Value Matches Expected Value
     [Documentation]    Returns date when availability value matches expected value
     ...    Args:
     ...        expected_value: expected availability value
     [Arguments]    ${expected_value}
     ${resp}=    Prometheus.Run Query    ${RHODS_PROMETHEUS_URL}    ${RHODS_PROMETHEUS_TOKEN}   rhods_aggregate_availability
     ${date}=    Convert Date    ${resp.json()["data"]["result"][0]["value"][0]}    epoch
-    ${value}=    Set Variable    ${resp.json()["data"]["result"][0]["value"][1]} 
+    ${value}=    Set Variable    ${resp.json()["data"]["result"][0]["value"][1]}
     Should Match    ${value}    ${expected_value}
     [Return]    ${date}
 
@@ -254,9 +267,9 @@ Verify ODS Availability Range
         Set Local Variable    ${component}   instance="combined"
     ELSE
         Fail   msg="Unknown component: ${component} (expected: jupyterhub, rhods-dashboard, combined)"
-    END  
+    END
 
-    &{expression}=    Create Dictionary    query=rhods_aggregate_availability{${component}}    start=${start}    end=${end}    step=${step}   
+    &{expression}=    Create Dictionary    query=rhods_aggregate_availability{${component}}    start=${start}    end=${end}    step=${step}
     ${resp}=    Run Query Range        &{expression}
     @{values}=    Set Variable    ${resp.json()["data"]["result"][0]["values"]}
     @{downtime}=    Create List
@@ -273,7 +286,7 @@ Verify ODS Availability Range
         ELSE IF   ${downtime_length} == 1
             Fail  msg=There is a Downtime at ${downtime-duration}[0] in ODS
         ELSE
-            ${downtime_lower_value} =    Convert Date    ${downtime}[0]       
+            ${downtime_lower_value} =    Convert Date    ${downtime}[0]
             ${downtime_upper_value} =    Convert Date    ${downtime}[-1]
             ${downtime-duration} =  Subtract Date From Date    ${downtime_upper_value}    ${downtime_lower_value}    compact
             Fail    msg=There is a Downtime of ${downtime-duration} in ODS
@@ -289,4 +302,4 @@ Run Query Range
     ${resp}=    RequestsLibrary.GET    url=${RHODS_PROMETHEUS_URL}/api/v1/query_range   params=&{pm_query}
     ...    headers=${pm_headers}    verify=${False}
     Request Should Be Successful
-    [Return]    ${resp}
+    [Return]    ${resp}

tests/Resources/RHOSi.resource

@@ -30,6 +30,7 @@ Resource            Page/OCPDashboard/InstalledOperators/InstalledOperators.robo
 ...                         Check Managed Starburst Addon Is Not Installed
 ...                         Delete Managed Starburst CRs
 ...                         Create RedHat Marketplace Secret
+...                         Get Observatorium Token
 
 
 *** Keywords ***

tests/Tests/600__ai_apps/680__starburst/681__managed_starburst_monitoring.robot

@@ -0,0 +1,53 @@
+*** Settings ***
+Documentation       Test suite testing SERH Metrics
+Resource            ../../../Resources/RHOSi.resource
+Resource            ../../../Resources/ODS.robot
+Resource            ../../../Resources/Common.robot
+Suite Setup         RHOSi Setup
+Suite Teardown      RHOSi Teardown
+
+
+*** Variables ***
+@{serh_querys}   node_namespace_pod_container:container_memory_working_set_bytes  node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate
+                ...  namespace_workload_pod:kube_pod_owner:relabel  kube_pod_container_info  kube_pod_status_ready  kube_namespace_status_phase
+                ...  node_namespace_pod:kube_pod_info  kube_pod_container_status_last_terminated_reason  kube_pod_container_status_waiting  kube_service_info
+                ...  cluster:namespace:pod_memory:active:kube_pod_container_resource_limits  container_cpu_cfs_throttled_seconds_total  container_fs_usage_bytes
+                ...  container_network_transmit_bytes_total  kube_pod_container_resource_requests  container_memory_usage_bytes  container_network_receive_bytes_total
+                ...  kube_deployment_status_replicas_available  kube_node_status_capacity  kube_deployment_status_replicas_unavailable
+                ...  kube_persistentvolumeclaim_status_phase  kube_pod_container_resource_limits  cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits
+                ...  container_network_receive_packets_total  container_network_transmit_packets_total  kube_running_pod_ready  container_cpu_usage_seconds_total
+                ...  kube_pod_container_status_restarts_total  kube_pod_status_phase  cluster:namespace:pod_memory:active:kube_pod_container_resource_requests
+                ...  jmx_config_reload_success_total  jmx_scrape_duration_seconds  jmx_scrape_cached_beans  jmx_scrape_error  jmx_exporter_build_info
+                ...  jmx_config_reload_failure_total  jmx_config_reload_failure_created  jmx_config_reload_success_created  jvm_threads_current  jvm_threads_daemon
+                ...  jvm_threads_peak  jvm_threads_started_total  jvm_threads_deadlocked  jvm_threads_deadlocked_monitor  jvm_threads_state  jvm_buffer_pool_used_bytes
+                ...  jvm_buffer_pool_capacity_bytes  jvm_buffer_pool_used_buffers  jvm_info  jvm_heap_memory_used  jvm_heap_memory_commited
+                ...  jvm_memory_pool_allocated_bytes_total  jvm_memory_bytes_used  jvm_memory_bytes_committed  jvm_memory_bytes_max  jvm_memory_bytes_init
+                ...  jvm_memory_pool_bytes_used  jvm_memory_pool_bytes_committed  jvm_memory_pool_bytes_max  jvm_memory_pool_bytes_init  jvm_classes_loaded
+                ...  jvm_classes_loaded_total  jvm_classes_unloaded_total  jvm_gc_collection_seconds_count  jvm_gc_collection_seconds_sum
+                ...  jvm_memory_pool_allocated_bytes_created  failed_queries  thread_count  trino_active_nodes  trino_free_memory  queries_killed_oom
+                ...  trino_active_queries  trino_queries_started  trino_queries_failed_external  trino_queries_failed_internal  trino_queries_failed_all
+                ...  trino_failed_queries_user  trino_execution_latency  trino_input_data_rate  input_data_bytes  input_rows  cluster_memory_bytes  tasks_killed_oom
+
+
+*** Test Cases ***
+Verify Query And Check Values Are Not Empty
+    [Documentation]    Verifies the Observatorium metrics values are not none
+    [Tags]    MISV-94
+    ${SSO_TOKEN}    Get Observatorium Token
+    Run Query And Check Values Are Not Empty   ${SSO_TOKEN}
+
+
+*** Keywords ***
+Run Query And Check Values Are Not Empty
+    [Documentation]  Run query and and check if Values Are Not Empty
+    [Arguments]     ${SSO_TOKEN}
+    FOR  ${query}   IN   @{serh_querys}
+        ${obs_query_op}=    Prometheus.Run Query    ${STARBURST.OBS_URL}    ${SSO_TOKEN}
+        ...   ${query}{namespace="redhat-starburst-operator"}   project=SERH
+        Should Be Equal    ${obs_query_op.json()['status']}    success
+        FOR  ${data}    IN   @{obs_query_op.json()['data']['result']}
+            Should Not Be Empty    ${data['value']}
+            Log    ${data['metric']['__name__']} |${data['metric']['pod']}| ${data['value']}
+            Append To List  ${value}    ${data['value']}
+        END
+    END