Project Headache

expan/core/__init__.py

@@ -3,9 +3,9 @@
 
 from __future__ import absolute_import
 
-# __all__ = ["binning", "experiment", "experimentdata", "results", "statistics", "util", "version"]
-__all__ = ["binning", "experiment", "statistics", "util", "version"]
-
 from expan.core.version import __version__, version
 
+__all__ = ["binning", "early_stopping", "experiment", "statistics", "util",
+           "version", "results", "correction", "statistical_test"]
+
 print(('ExpAn core init: {}'.format(version())))

expan/core/binning.py

@@ -1,10 +1,12 @@
+# TODO: This module is deprecated
+
 import logging
 import warnings
 from heapq import heapify, heappush, heappop
 
 import numpy as np
 
-from expan.core.util import is_number_and_nan
+from expan.core.util import is_nan
 
 logger = logging.getLogger(__name__)
 
@@ -174,7 +176,7 @@ def create_bins(data, n_bins):
         raise ValueError('Less than one bin makes no sense.')
 
     insufficient_distinct = False
-    n_unique_values = len(np.unique([value for value in data if not is_number_and_nan(value)]))
+    n_unique_values = len(np.unique([value for value in data if not is_nan(value)]))
     if n_unique_values < n_bins:
         insufficient_distinct = True
         warnings.warn("Insufficient unique values for requested number of bins. " +

expan/core/results.py

@@ -0,0 +1,117 @@
+from expan.core.util import JsonSerializable
+
+
+# --------- Below are the data structure of statistics --------- #
+class BaseTestStatistics(JsonSerializable):
+    """ Holds only statistics for the control and treatment group. 
+
+    :param control_statistics: statistics within the control group
+    :type  control_statistics: SampleStatistics
+    :param treatment_statistics: statistics within the treatment group
+    :type  treatment_statistics: SampleStatistics
+    """
+    def __init__(self, control_statistics, treatment_statistics):
+        self.control_statistics   = control_statistics
+        self.treatment_statistics = treatment_statistics
+
+
+class SampleStatistics(JsonSerializable):
+    """ This class holds sample size, mean and variance.
+
+    :type sample_size: int
+    :type mean: float
+    :type variance: float
+    """
+    def __init__(self, sample_size, mean, variance):
+        self.sample_size = sample_size
+        self.mean        = mean
+        self.variance    = variance
+
+
+class SimpleTestStatistics(BaseTestStatistics):
+    """ Additionally to BaseTestStatistics, holds delta, confidence interval, statistical power, and p value.
+
+    :type control_statistics: SampleStatistics
+    :type treatment_statistics: SampleStatistics
+    :type delta: float
+    :type p: float
+    :type statistical_power: float
+    :param ci: a dict where keys are percentiles and values are the corresponding value for the statistic.
+    :type  ci: dict
+    """
+    def __init__(self, control_statistics, treatment_statistics, delta, ci, p, statistical_power):
+        super(SimpleTestStatistics, self).__init__(control_statistics, treatment_statistics)
+        self.delta               = delta
+        self.p                   = p
+        self.statistical_power   = statistical_power
+        # TODO: think of structure {p: v} (the same as ci)
+        self.confidence_interval = [{'percentile': p, 'value': v} for (p, v) in ci.items()]
+
+
+class EarlyStoppingTestStatistics(SimpleTestStatistics):
+    """ Additionally to SimpleTestStatistics, holds boolean flag for early stopping.
+
+    :type control_statistics: SampleStatistics
+    :type treatment_statistics: SampleStatistics
+    :type delta: float
+    :type p: float
+    :type statistical_power: float
+    :param ci: a dict where keys are percentiles and values are the corresponding value for the statistic.
+    :type  ci: dict
+    :type stop: bool
+    """
+    def __init__(self, control_statistics, treatment_statistics, delta, ci, p, statistical_power, stop):
+        super(EarlyStoppingTestStatistics, self).__init__(control_statistics, treatment_statistics, delta, ci, p, statistical_power)
+        self.stop = stop
+
+
+class CorrectedTestStatistics(JsonSerializable):
+    """ Holds original and corrected statistics. This class should be used to hold statistics for multiple testing.
+    original_test_statistics and corrected_test_statistics should have the same type.
+
+    :param original_test_statistics: test result before correction
+    :type  original_test_statistics: SimpleTestStatistics or EarlyStoppingTestStatistics
+    :param corrected_test_statistics: test result after correction
+    :type  corrected_test_statistics: SimpleTestStatistics or EarlyStoppingTestStatistics
+    """
+    def __init__(self, original_test_statistics, corrected_test_statistics):
+        type1 = type(original_test_statistics)
+        type2 = type(corrected_test_statistics)
+        if type1 != type2:
+            raise RuntimeError("Type mismatch for type " + str(type1) + " and " + str(type2))
+        if not isinstance(original_test_statistics, BaseTestStatistics):
+            raise RuntimeError("Input should be instances of BaseTestStatistics or its subclass")
+        self.original_test_statistics  = original_test_statistics
+        self.corrected_test_statistics = corrected_test_statistics
+
+
+# --------- Below are the data structure of test results --------- #
+class StatisticalTestResult(JsonSerializable):
+    """ This class holds the results of a single statistical test.
+
+    :param test: information about the statistical test
+    :type  test: StatisticalTest
+    :param result: result of this statistical test
+    :type  result: BaseTestStatistics or its subclasses or CorrectedTestStatistics  #TODO: better approach?
+    """
+    def __init__(self, test, result):
+        self.test   = test
+        self.result = result
+
+
+class MultipleTestSuiteResult(JsonSerializable):
+    """ This class holds the results of a MultipleTestSuite.
+
+    :param statistical_test_results: test results for all statistical testing unit
+    :type  statistical_test_results: list[StatisticalTestResult]
+    :param correction_method: method used for multiple testing correction. Possible values are:
+                              "none": no correction
+                              "bh": benjamini hochberg correction
+                              "bf": bonferroni correction
+    :type  correction_method: str
+    """
+    def __init__(self, statistical_test_results, correction_method="none"):
+        self.statistical_test_results = statistical_test_results
+        if correction_method not in ["none", "bh", "bf"]:
+            raise ValueError('Correction method is not implemented. We support "none", "bh", and "bf".')
+        self.correction_method = correction_method

expan/core/statistical_test.py

@@ -0,0 +1,114 @@
+import pandas as pd
+from expan.core.util import JsonSerializable
+
+
+class StatisticalTest(JsonSerializable):
+    """ This class describes what has to be tested against what and represent a unit of statistical testing.
+
+    :param kpi: the kpi to perform on
+    :type  kpi: KPI or its subclass
+    :param features: list of features used for subgroups
+    :type  features: list[FeatureFilter]
+    :param variants: variant column name and their values
+    :type  variants: Variants
+    """
+    def __init__(self, kpi, features, variants):
+        if not isinstance(features, list):
+            raise TypeError("Features should be a list.")
+        if not all(isinstance(n, FeatureFilter) for n in features):
+            raise TypeError("Some features are not of the type FeatureFilter.")
+        self.kpi       = kpi
+        self.features  = features
+        self.variants  = variants
+
+
+class KPI(JsonSerializable):
+    """ This class represents a basic kpi.
+    :param name: name of the kpi
+    :type  name: str
+    """
+    def __init__(self, name):
+        self.name = name
+
+
+class DerivedKPI(KPI):
+    """ This class represents a derived KPI which is a ratio of two columns. 
+    Names of the the two columns are passed as numerator and denominator.
+
+    :param name: name of the kpi
+    :type  name: str
+    :param numerator: the numerator for the derived KPI
+    :type  numerator: str
+    :param denominator: the denominator for the derived KPI
+    :type  denominator: str
+    """
+    def __init__(self, name, numerator, denominator):
+        super(DerivedKPI, self).__init__(name)
+        self.numerator = numerator
+        self.denominator = denominator
+
+    def make_derived_kpi(self, data):
+        """ Create the derived kpi column if it is not yet created. """
+        if self.name not in data.columns:
+            data.loc[:, self.name] = data[self.numerator]/data[self.denominator].astype(float)
+
+
+class StatisticalTestSuite(JsonSerializable):
+    """ This class consists of a number of tests plus choice of the correction method.
+
+    :param tests: list of statistical tests in the suite
+    :type  tests: list[StatisticalTest]
+    :param correction_method: method used for multiple testing correction. Possible values are:
+                              "none": no correction
+                              "bh": benjamini hochberg correction
+                              "bf": bonferroni correction
+    :type  correction_method: str
+    """
+    def __init__(self, tests, correction_method="none"):
+        self.tests = tests
+        if correction_method not in ["none", "bh", "bf"]:
+            raise ValueError('Correction method is not implemented. We support "none", "bh", and "bf".')
+        self.correction_method = correction_method
+
+    @property
+    def size(self):
+        return len(self.tests)
+
+
+class FeatureFilter(JsonSerializable):
+    """ This class represents a filter, restricting a DataFrame to rows with column_value in column_name.
+
+    It can be used to specify subgroup conditions.
+    :param column_name: name of the column to perform filter on
+    :type  column_name: str
+    :param column_value: value of the column to perform filter on
+    :type  column_value: str
+    """
+    def __init__(self, column_name, column_value):
+        self.column_name  = column_name
+        self.column_value = column_value
+
+    def apply_to_data(self, data):
+        return data[data[self.column_name] == self.column_value]
+
+
+class Variants(JsonSerializable):
+    """ This class represents information of variants.
+
+    :param variant_column_name: name of the column that represents variant
+    :type  variant_column_name: str
+    :param control_name: value of the variant that represents control group
+    :type  control_name: str
+    :param treatment_name: value of the variant that represents control group
+    :type  treatment_name: str
+    """
+    def __init__(self, variant_column_name, control_name, treatment_name):
+        self.variant_column_name = variant_column_name
+        self.control_name        = control_name
+        self.treatment_name      = treatment_name
+
+    def get_variant(self, data, variant_name):
+        result = data[data[self.variant_column_name] == variant_name]
+        if not isinstance(result, pd.DataFrame):
+            result = pd.DataFrame([result])
+        return result