Merge pull request #3434 from jack-morrison/jackm/2857

[feat] Support reference values of zero

Author: vkarak

reframe/core/pipeline.py

@@ -672,7 +672,13 @@ def pipeline_hooks(cls):
     #: performance variables defined in :attr:`perf_patterns` and scoped under
     #: the system/partition combinations.
     #: The reference itself is a four-tuple that contains the reference value,
-    #: the lower and upper thresholds and the measurement unit.
+    #: the lower and upper thresholds, and the measurement unit.
+    #: 
+    #: For non-zero reference values, lower and upper thresholds are
+    #: percentages -/+ from the reference value in decimal form.
+    #:
+    #: When a reference value of ``0`` is expected, lower and upper 
+    #: thresholds are interpreted as absolute values.
     #:
     #: An example follows:
     #:
@@ -690,7 +696,7 @@ def pipeline_hooks(cls):
     #:    }
     #:
     #: To better understand how to set the performance reference tuple, here
-    #: are some examples with both positive and negative reference values:
+    #: are some examples with positive, negative, and zero reference values:
     #:
     #:   ============================== ============  ==========  ===========
     #:   **Performance Tuple**          **Expected**  **Lowest**  **Highest**
@@ -700,13 +706,14 @@ def pipeline_hooks(cls):
     #:   ``(-100, -0.01, 0.02, 'C')``     -100 C        -101 C      -98 C
     #:   ``(-100, -0.01, None, 'C')``     -100 C        -101 C      inf C
     #:   ``(-100, None, 0.02, 'C')``      -100 C        -inf C      -98 C
+    #:   ``(0, -2, 5, 'C')``                 0 C          -2 C        5 C
     #:   ============================== ============  ==========  ===========
     #:
     #: During the performance stage of the pipeline, the reference tuple
     #: elements, except the unit, are passed to the
     #: :func:`~reframe.utility.sanity.assert_reference` function along with the
-    #: obtained performance value in order to actually assess whether the test
-    #: passes the performance check or not.
+    #: obtained performance value to assess whether the test
+    #: passes or fails the performance check.
     #:
     #: :type: A scoped dictionary with system names as scopes, performance
     #:   variables as keys and reference tuples as values.

reframe/utility/sanity.py

@@ -550,7 +550,13 @@ def assert_reference(val, ref, lower_thres=None, upper_thres=None, msg=None):
         lower and upper thresholds do not have appropriate values.
     '''
     if lower_thres is not None:
-        lower_thres_limit = -1 if ref >= 0 else None
+        if ref > 0:
+            lower_thres_limit = -1
+        elif ref == 0:
+            lower_thres_limit = -math.inf
+        else:
+            lower_thres_limit = None
+
         try:
             evaluate(assert_bounded(lower_thres, lower_thres_limit, 0))
         except SanityError:
@@ -559,7 +565,13 @@ def assert_reference(val, ref, lower_thres=None, upper_thres=None, msg=None):
             ) from None
 
     if upper_thres is not None:
-        upper_thres_limit = None if ref >= 0 else 1
+        if ref > 0:
+            upper_thres_limit = None
+        elif ref == 0:
+            upper_thres_limit = math.inf
+        else:
+            upper_thres_limit = 1
+
         try:
             evaluate(assert_bounded(upper_thres, 0, upper_thres_limit))
         except SanityError:
@@ -577,11 +589,16 @@ def calc_bound(thres):
 
         return ref*(1 + thres)
 
-    lower = calc_bound(lower_thres)
+    if ref != 0:
+        lower = calc_bound(lower_thres)
+        upper = calc_bound(upper_thres)
+    else:
+        lower = lower_thres
+        upper = upper_thres
+
     if lower is None:
         lower = -math.inf
 
-    upper = calc_bound(upper_thres)
     if upper is None:
         upper = math.inf
 

unittests/test_sanity_functions.py

@@ -448,6 +448,14 @@ def test_assert_reference():
     assert sn.assert_reference(-0.9, -1, upper_thres=0.1)
     assert sn.assert_reference(-0.9, -1)
 
+    # Check reference values of 0
+    assert sn.assert_reference(0, 0, 0, 0)
+    assert sn.assert_reference(-1, 0, -2, 2)
+    assert sn.assert_reference(2, 0, -2, 2)
+    with pytest.raises(SanityError, match=r'3 is beyond reference value 0 '
+                                          r'\(l=-10, u=1\)'):
+        sn.evaluate(sn.assert_reference(3, 0, -10, 1))
+
     # Check upper threshold values greater than 1
     assert sn.assert_reference(20.0, 10.0, None, 3.0)
     assert sn.assert_reference(-50.0, -20.0, -2.0, 0.5)