Handle parallel testing

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 12.2.0
+
+* Use `nose` in the `openfisca-run-test` script
+  - This avoids boilerplate code on country packages, and makes parallelism easier to set on Circle CI.
+
 ## 12.1.4
 
 * Fix package naming conflict between the preview API and the official one.

openfisca_core/scripts/run_test.py

@@ -31,19 +31,12 @@ def main():
         'name_filter': args.name_filter,
         }
 
-    tests_found = False
+    paths = map(os.path.abspath, args.path)
+    tests_ok = run_tests(tax_benefit_system, paths, options)
 
-    for path in args.path:
-        path = os.path.abspath(path)
-        nb_tests = run_tests(tax_benefit_system, path, options)
-        tests_found = tests_found or nb_tests > 0
-
-    if not tests_found:
-        print("No tests found!")
+    if not tests_ok:
         sys.exit(1)
 
-    sys.exit(0)
-
 
 if __name__ == "__main__":
     main()

openfisca_core/tools/test_runner.py

@@ -8,8 +8,12 @@
 import copy
 import glob
 import os
-import yaml
+import sys
+import unittest
+
+import nose
 import numpy as np
+import yaml
 
 from openfisca_core import conv, periods, scenarios
 from openfisca_core.tools import assert_near
@@ -56,7 +60,7 @@ def dict_constructor(loader, node):
 
 # Exposed methods
 
-def generate_tests(tax_benefit_system, path, options = {}):
+def generate_tests(tax_benefit_system, paths, options = {}):
     """
     Generates a lazy iterator of all the YAML tests contained in a file or a directory.
 
@@ -66,20 +70,26 @@ def generate_tests(tax_benefit_system, path, options = {}):
 
     """
 
-    if os.path.isdir(path):
-        return _generate_tests_from_directory(tax_benefit_system, path, options)
-    else:
-        return _generate_tests_from_file(tax_benefit_system, path, options)
+    if isinstance(paths, str):
+        paths = [paths]
 
+    for path in paths:
+        if os.path.isdir(path):
+            for test in _generate_tests_from_directory(tax_benefit_system, path, options):
+                yield test
+        else:
+            for test in _generate_tests_from_file(tax_benefit_system, path, options):
+                yield test
 
-def run_tests(tax_benefit_system, path, options = {}):
+
+def run_tests(tax_benefit_system, paths, options = {}):
     """
     Runs all the YAML tests contained in a file or a directory.
 
     If `path` is a directory, subdirectories will be recursively explored.
 
     :param TaxBenefitSystem tax_benefit_system: the tax-benefit system to use to run the tests
-    :param str path: the path towards the file or directory containing thes tests. If it is a directory, subdirectories will be recursively explored.
+    :param (str/list) paths: A path, or a list of paths, towards the files or directories containing the tests to run. If a path is a directory, subdirectories will be recursively explored.
     :param dict options: See more details below.
 
     :raises AssertionError: if a test does not pass
@@ -92,18 +102,18 @@ def run_tests(tax_benefit_system, path, options = {}):
     | Key                           | Type      | Role                                      |
     +===============================+===========+===========================================+
     | verbose                       | ``bool``  |                                           |
-    +-------------------------------+-----------+                                           +
-    | name_filter                   | ``str``   | See :any:`openfisca-run-test` options doc |
+    +-------------------------------+-----------+ See :any:`openfisca-run-test` options doc +
+    | name_filter                   | ``str``   |                                           |
     +-------------------------------+-----------+-------------------------------------------+
 
     """
-
-    nb_tests = 0
-    for test in generate_tests(tax_benefit_system, path, options):
-        test()
-        nb_tests += 1
-
-    return nb_tests  # Nb of sucessful tests
+    return nose.run(
+        # The suite argument must be a lambda for nose to run the tests lazily
+        suite = lambda: generate_tests(tax_benefit_system, paths, options),
+        # Nose crashes if it gets any unexpected argument.
+        argv = sys.argv[:1],
+        # testRunner = nose.core.TextTestRunner(stream = open(os.devnull, 'w')),
+        )
 
 
 # Internal methods
@@ -137,7 +147,7 @@ def check():
             print("=" * len(title))
             _run_test(period_str, test, verbose, options)
 
-        yield check
+        yield unittest.FunctionTestCase(check)
 
 
 def _generate_tests_from_directory(tax_benefit_system, path_to_dir, options):

setup.py

@@ -7,7 +7,7 @@
 
 setup(
     name = 'OpenFisca-Core',
-    version = '12.1.4',
+    version = '12.2.0',
     author = 'OpenFisca Team',
     author_email = 'contact@openfisca.fr',
     classifiers = [

tests/core/test_countries.py

@@ -8,17 +8,17 @@
 from openfisca_core import periods
 from openfisca_core.formulas import DIVIDE
 from openfisca_country_template import CountryTaxBenefitSystem
+from openfisca_country_template.entities import Person
 from openfisca_core.tools import assert_near
 from openfisca_core.columns import FloatCol
-from openfisca_dummy_country.entities import Individu
 
 
 tax_benefit_system = CountryTaxBenefitSystem()
 
 
 class income_tax_no_period(Variable):
     column = FloatCol
-    entity = Individu
+    entity = Person
     label = u"Salaire net (buggy)"
     definition_period = MONTH
 

tests/core/test_yaml.py

@@ -2,6 +2,7 @@
 
 import pkg_resources
 import os
+import sys
 import subprocess
 
 from nose.tools import nottest, raises
@@ -28,84 +29,85 @@
 
 
 @nottest
-def run_yaml_test(file_name, options = {}):
-    yaml_path = os.path.join(yaml_tests_dir, '{}.yaml'.format(file_name))
-    return run_tests(tax_benefit_system, yaml_path, options)
+def run_yaml_test(path, options = {}):
+    yaml_path = os.path.join(yaml_tests_dir, path)
+
+    # We are testing tests, and don't want the latter to print anything, so we temporarily deactivate stderr.
+    old_stderr = sys.stderr
+    sys.stderr = open(os.devnull, 'w')
+    result = run_tests(tax_benefit_system, yaml_path, options)
+    sys.stderr = old_stderr
+    return result
 
 
 def test_success():
-    run_yaml_test('test_success')
+    assert run_yaml_test('test_success.yaml')
 
 
-@raises(AssertionError)
 def test_fail():
-    run_yaml_test('test_failure')
+    assert run_yaml_test('test_failure.yaml') is False
 
 
 def test_relative_error_margin_success():
-    run_yaml_test('test_relative_error_margin')
+    assert run_yaml_test('test_relative_error_margin.yaml')
 
 
-@raises(AssertionError)
 def test_relative_error_margin_fail():
-    run_yaml_test('failing_test_relative_error_margin')
+    assert run_yaml_test('failing_test_relative_error_margin.yaml') is False
 
 
 def test_absolute_error_margin_success():
-    run_yaml_test('test_absolute_error_margin')
+    assert run_yaml_test('test_absolute_error_margin.yaml')
 
 
-@raises(AssertionError)
 def test_absolute_error_margin_fail():
-    run_yaml_test('failing_test_absolute_error_margin')
+    assert run_yaml_test('failing_test_absolute_error_margin.yaml') is False
 
 
 def test_run_tests_from_directory():
     dir_path = os.path.join(yaml_tests_dir, 'directory')
-    assert run_tests(tax_benefit_system, dir_path) == 4
+    assert run_yaml_test(dir_path)
 
 
 def test_with_reform():
-    run_yaml_test('test_with_reform')
+    assert run_yaml_test('test_with_reform.yaml')
 
 
-@raises(AssertionError)
 def test_run_tests_from_directory_fail():
-    run_tests(tax_benefit_system, yaml_tests_dir)
+    assert run_yaml_test(yaml_tests_dir) is False
 
 
 def test_name_filter():
-    nb_tests = run_tests(
-        tax_benefit_system,
+    assert run_yaml_test(
         yaml_tests_dir,
         options = {'name_filter': 'success'}
         )
 
-    assert nb_tests == 3
-
-
-def test_nose_style():
-    dir_path = os.path.join(yaml_tests_dir, 'directory')
-    for test in generate_tests(tax_benefit_system, dir_path):
-        yield test
-
 
 def test_shell_script():
     yaml_path = os.path.join(yaml_tests_dir, 'test_success.yaml')
     command = ['openfisca-run-test', yaml_path, '-c', 'openfisca_country_template']
     with open(os.devnull, 'wb') as devnull:
-        subprocess.check_call(command, stdout = devnull)
+        subprocess.check_call(command, stdout = devnull, stderr = devnull)
+
+
+@raises(subprocess.CalledProcessError)
+def test_failing_shell_script():
+    yaml_path = os.path.join(yaml_tests_dir, 'test_failure.yaml')
+    command = ['openfisca-run-test', yaml_path, '-c', 'openfisca_dummy_country']
+    with open(os.devnull, 'wb') as devnull:
+        subprocess.check_call(command, stdout = devnull, stderr = devnull)
 
 
 def test_shell_script_with_reform():
     yaml_path = os.path.join(yaml_tests_dir, 'test_with_reform_2.yaml')
     command = ['openfisca-run-test', yaml_path, '-c', 'openfisca_country_template', '-r', 'openfisca_country_template.reforms.removal_basic_income.removal_basic_income']
     with open(os.devnull, 'wb') as devnull:
-        subprocess.check_call(command, stdout = devnull)
+        subprocess.check_call(command, stdout = devnull, stderr = devnull)
 
 
 def test_shell_script_with_extension():
     extension_dir = openfisca_extension_template.__path__[0]
     command = ['openfisca-run-test', extension_dir, '-c', 'openfisca_country_template', '-e', extension_dir]
     with open(os.devnull, 'wb') as devnull:
-        subprocess.check_call(command, stdout = devnull)
+        subprocess.check_call(command, stdout = devnull, stderr = devnull)