Re-allow using cov directly

Author: paulf81

flasc/analysis/expected_power_analysis.py

@@ -378,12 +378,10 @@ def _total_uplift_expected_power_with_standard_error(
     # # Synchronize any null values in the covariance back to df_bin
     # df_bin = _synchronize_cov_nulls_back_to_mean(df_bin=df_bin, test_cols=test_cols)
 
+    # Set to null any mean values associated with null variance by row/turbine
     df_bin = _synchronize_var_nulls_back_to_mean(df_bin=df_bin, test_cols=test_cols)
 
-    with pl.Config(tbl_cols=-1):
-        print(df_bin)
-
-    # Synchronize the null values in the mean columns
+    # Synchronize the null values in the mean columns across uplift pairs
     df_bin = _synchronize_nulls(
         df_bin=df_bin,
         sync_cols=[f"{col}_mean" for col in test_cols],

flasc/analysis/expected_power_analysis_utilities.py

@@ -322,8 +322,8 @@ def _fill_cov_with_var(
             n_col = f"count_{t1}_{t2}"
             var_1_col = f"cov_{t1}_{t1}"
             var_2_col = f"cov_{t2}_{t2}"
-            # n_1_col = f"count_{t1}_{t1}"
-            # n_2_col = f"count_{t2}_{t2}"
+            n_1_col = f"count_{t1}_{t1}"
+            n_2_col = f"count_{t2}_{t2}"
 
             # For the rows where cov_col is null, fill the cov_col with the product of the square
             # root of the variances of the two test columns and the n_col with the minimum of the
@@ -334,30 +334,27 @@ def _fill_cov_with_var(
             if fill_all:
                 df_cov = df_cov.with_columns(pl.lit(True).alias("null_map"))
 
-            with pl.Config(tbl_cols=-1):
-                print(cov_col)
-                print(df_cov)
-
             df_cov = df_cov.with_columns(
                 pl.when(pl.col("null_map"))
                 .then((pl.col(var_1_col).sqrt() * pl.col(var_2_col).sqrt()))
                 .otherwise(pl.col(cov_col))
                 .alias(cov_col)
             )
 
-            # Leave n_col as is (number of joint points)
-            # df_cov = df_cov.with_columns(
-            #     pl.when(pl.col("null_map"))
-            #     .then(pl.min_horizontal(n_1_col, n_2_col))
-            #     .otherwise(pl.col(n_col))
-            #     .alias(n_col)
-            # )
+            # Num points as the minimum between the values of the two variances
+            df_cov = df_cov.with_columns(
+                pl.when(pl.col("null_map"))
+                .then(pl.min_horizontal(n_1_col, n_2_col))
+                .otherwise(pl.col(n_col))
+                .alias(n_col)
+            )
 
-            # For any rows where n_col is 0 or null, set n_col and cov_col to null
+            # For any rows where n_col is < 2 set n_col to null
             df_cov = df_cov.with_columns(
-                pl.when(pl.col(n_col) == 0).then(None).otherwise(pl.col(n_col)).alias(n_col)
+                pl.when(pl.col(n_col) < 2).then(None).otherwise(pl.col(n_col)).alias(n_col)
             )
 
+            # Wherever n_col is null, set cov_col to null
             df_cov = df_cov.with_columns(
                 pl.when(pl.col(n_col).is_null())
                 .then(None)

tests/expected_power_analysis_test.py

@@ -467,7 +467,7 @@ def test_fill_cov_with_var_dont_fill_all():
             "cov_pow_001_pow_000": [1, 1],
             "cov_pow_001_pow_001": [4, 4],
             "count_pow_000_pow_000": [1, 2],
-            "count_pow_000_pow_001": [3, 4],  # Note values not updated here
+            "count_pow_000_pow_001": [3, 2],  # Note values are updated here too as min
             "count_pow_001_pow_000": [5, 6],
             "count_pow_001_pow_001": [7, 8],
         }
@@ -502,12 +502,18 @@ def test_fill_cov_with_var_fill_all():
             "ws_bin": [0, 0],
             "df_name": ["baseline"] * 2,
             "cov_pow_000_pow_000": [4, 4],
-            "cov_pow_000_pow_001": [4, 6],  # Note filled values
-            "cov_pow_001_pow_000": [4, 6],  # Note filled values
+            "cov_pow_000_pow_001": [
+                None,
+                6,
+            ],  # Note filled values (None because count goes below 2)
+            "cov_pow_001_pow_000": [
+                None,
+                6,
+            ],  # Note filled values (None because count goes below 2)
             "cov_pow_001_pow_001": [4, 9],
             "count_pow_000_pow_000": [1, 2],
-            "count_pow_000_pow_001": [3, 4],  # Note values not updated here
-            "count_pow_001_pow_000": [5, 6],  # Note values not updated here
+            "count_pow_000_pow_001": [None, 2],  # Note values updated here to minimum
+            "count_pow_001_pow_000": [None, 2],  # Note values updated here to minimum
             "count_pow_001_pow_001": [7, 8],
         }
     )