Merge branch 'provenance_redesign' into issue_2228

.ci/README

@@ -1,5 +1,2 @@
-Configuration files to run Jenkins
-
-Note that it (re)uses many files from the ../.ci folder!
-The doubler.sh is copied here because the Dockerfile requires the
-file to be in the same folder (or a subfolder) and it cannot be a symlink
+Files for continuous integration tests,
+both on Travis and on Jenkins

.coveragerc

@@ -3,4 +3,4 @@ source = aiida
 omit = aiida/test*.py,aiida/*/test*.py,aiida/*/*/test*.py,aiida/*/*/*/test*.py,aiida/*/*/*/*/test*.py,aiida/*/*/*/*/*/test*.py,aiida/*/migrations/*.py,aiida/*/migrations/versions/*.py
 
 [html]
-directory = coverage/html
+directory = .ci/coverage/html

.pre-commit-config.yaml

@@ -45,7 +45,6 @@
         aiida/backends/djsite/queries.py|
         aiida/backends/djsite/settings/__init__.py|
         aiida/backends/djsite/settings/settings_profile.py|
-        aiida/backends/djsite/settings/wsgi.py|
         aiida/backends/general/abstractqueries.py|
         aiida/backends/__init__.py|
         aiida/backends/profile.py|
@@ -189,6 +188,7 @@
         aiida/common/links.py|
         aiida/common/orbital/__init__.py|
         aiida/common/orbital/realhydrogen.py|
+        aiida/common/setup.py|
         aiida/common/test_extendeddicts.py|
         aiida/common/test_logging.py|
         aiida/control/tests/test_postgres.py|
@@ -341,9 +341,6 @@
         aiida/work/__init__.py|
         aiida/work/job_processes.py|
         aiida/work/run.py|
-        fastentrypoints.py|
-        setup.py|
-        setup_requirements.py|
         utils/create_requirements.py|
         utils/validate_pyproject.py|
         .ci/test_daemon.py|
@@ -369,7 +366,8 @@
     language: system
     files: >-
       (?x)^(
-        setup_requirements.py|
+        setup.json|
+        setup.py|
         docs/requirements_for_rtd.txt|
         docs/update_req_for_rtd.py|
       )$
@@ -381,7 +379,8 @@
     language: system
     files: >-
       (?x)^(
-        setup_requirements.py|
+        setup.json|
+        setup.py|
         utils/validate_pyproject|
       )$
     pass_filenames: false
@@ -393,6 +392,17 @@
     language: ruby
     additional_dependencies: ['travis']
 
+  - id: version-number
+    name: Check version numbers
+    entry: python ./utils/validate_version.py
+    language: system
+    files: >-
+      (?x)^(
+        setup.json|
+        aiida/__init__.py
+      )$
+    pass_filenames: false
+
 # modernizer: make sure our code-base is Python 3 ready
 - repo: https://github.com/python-modernize/python-modernize.git
   sha: a234ce4e185cf77a55632888f1811d83b4ad9ef2

.pylintrc

@@ -7,7 +7,7 @@ extension-pkg-whitelist=
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
-ignore=CVS,test,examples,setup.py
+ignore=
 
 # Add files or directories matching the regex patterns to the blacklist. The
 # regex matches against base names, not paths.

MANIFEST.in

@@ -6,8 +6,8 @@ include aiida/cmdline/templates/*.tpl
 include aiida/common/additions/backup_script/backup_info.json.tmpl
 include examples/testdata/qepseudos/*
 include docs/source/images/*.png
-include fastentrypoints.py
-include setup_requirements.py
+include utils/*
+include setup.json
 include AUTHORS.txt
 include CHANGELOG.md
 include pyproject.toml

aiida/backends/djsite/db/migrations/0019_migrate_builtin_calculations.py

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
+# pylint: disable=invalid-name,too-few-public-methods
+"""Migration to reflect the name change of the built in calculation entry points in the database."""
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import absolute_import
+
+# Remove when https://github.com/PyCQA/pylint/issues/1931 is fixed
+# pylint: disable=no-name-in-module,import-error
+from django.db import migrations
+
+from aiida.backends.djsite.db.migrations import upgrade_schema_version
+
+REVISION = '1.0.19'
+DOWN_REVISION = '1.0.18'
+
+
+class Migration(migrations.Migration):
+    """Migration to remove entry point groups from process type strings and prefix unknown types with a marker."""
+
+    dependencies = [
+        ('db', '0018_django_1_11'),
+    ]
+
+    operations = [
+        # The built in calculation plugins `arithmetic.add` and `templatereplacer` have been moved and their entry point
+        # renamed. In the change the `simpleplugins` namespace was dropped so we migrate the existing nodes.
+        migrations.RunSQL(
+            sql="""
+            UPDATE db_dbnode SET type = 'calculation.job.arithmetic.add.ArithmeticAddCalculation.'
+            WHERE type = 'calculation.job.simpleplugins.arithmetic.add.ArithmeticAddCalculation.';
+
+            UPDATE db_dbnode SET type = 'calculation.job.templatereplacer.TemplatereplacerCalculation.'
+            WHERE type = 'calculation.job.simpleplugins.templatereplacer.TemplatereplacerCalculation.';
+
+            UPDATE db_dbnode SET process_type = 'aiida.calculations:arithmetic.add'
+            WHERE process_type = 'aiida.calculations:simpleplugins.arithmetic.add';
+
+            UPDATE db_dbnode SET process_type = 'aiida.calculations:templatereplacer'
+            WHERE process_type = 'aiida.calculations:simpleplugins.templatereplacer';
+
+            UPDATE db_dbattribute AS a SET tval = 'arithmetic.add'
+            FROM db_dbnode AS n WHERE a.dbnode_id = n.id
+                AND a.key = 'input_plugin'
+                AND a.tval = 'simpleplugins.arithmetic.add'
+                AND n.type = 'data.code.Code.';
+
+            UPDATE db_dbattribute AS a SET tval = 'templatereplacer'
+            FROM db_dbnode AS n WHERE a.dbnode_id = n.id
+                AND a.key = 'input_plugin'
+                AND a.tval = 'simpleplugins.templatereplacer'
+                AND n.type = 'data.code.Code.';
+            """,
+            reverse_sql="""
+            UPDATE db_dbnode SET type = 'calculation.job.simpleplugins.arithmetic.add.ArithmeticAddCalculation.'
+            WHERE type = 'calculation.job.arithmetic.add.ArithmeticAddCalculation.';
+
+            UPDATE db_dbnode SET type = 'calculation.job.simpleplugins.templatereplacer.TemplatereplacerCalculation.'
+            WHERE type = 'calculation.job.templatereplacer.TemplatereplacerCalculation.';
+
+            UPDATE db_dbnode SET process_type = 'aiida.calculations:simpleplugins.arithmetic.add'
+            WHERE process_type = 'aiida.calculations:arithmetic.add';
+
+            UPDATE db_dbnode SET process_type = 'aiida.calculations:simpleplugins.templatereplacer'
+            WHERE process_type = 'aiida.calculations:templatereplacer';
+
+            UPDATE db_dbattribute AS a SET tval = 'simpleplugins.arithmetic.add'
+            FROM db_dbnode AS n WHERE a.dbnode_id = n.id
+                AND a.key = 'input_plugin'
+                AND a.tval = 'arithmetic.add'
+                AND n.type = 'data.code.Code.';
+
+            UPDATE db_dbattribute AS a SET tval = 'simpleplugins.templatereplacer'
+            FROM db_dbnode AS n WHERE a.dbnode_id = n.id
+                AND a.key = 'input_plugin'
+                AND a.tval = 'templatereplacer'
+                AND n.type = 'data.code.Code.';
+            """),
+        upgrade_schema_version(REVISION, DOWN_REVISION)
+    ]

aiida/backends/djsite/db/migrations/__init__.py

@@ -11,7 +11,7 @@
 from __future__ import print_function
 from __future__ import absolute_import
 
-LATEST_MIGRATION = '0018_django_1_11'
+LATEST_MIGRATION = '0019_migrate_builtin_calculations'
 
 def _update_schema_version(version, apps, schema_editor):
     from aiida.backends.djsite.utils import set_db_schema_version

aiida/backends/djsite/db/subtests/generic.py

@@ -16,7 +16,7 @@
 from __future__ import absolute_import
 from aiida.backends.testbase import AiidaTestCase
 from aiida.common import exceptions
-from aiida.orm.node import Node
+from aiida.orm import Node, Data
 from aiida import orm
 
 
@@ -71,10 +71,10 @@ def test_query(self):
         g2 = backend.groups.create(name='testquery2', user=default_user).store()
         self.addCleanup(lambda: backend.groups.delete(g2.id))
 
-        n1 = Node().store()
-        n2 = Node().store()
-        n3 = Node().store()
-        n4 = Node().store()
+        n1 = Data().store()
+        n2 = Data().store()
+        n3 = Data().store()
+        n4 = Data().store()
 
         g1.add_nodes([n1, n2])
         g2.add_nodes([n1, n3])
@@ -157,7 +157,7 @@ def test_rename_existing(self):
     def test_creation_from_dbgroup(self):
         backend = self.backend
 
-        n = Node().store()
+        n = Data().store()
 
         default_user = backend.users.create("{}@aiida.net".format(self.id())).store()
 
@@ -182,8 +182,8 @@ class TestDbExtrasDjango(AiidaTestCase):
     def test_replacement_1(self):
         from aiida.backends.djsite.db.models import DbExtra
 
-        n1 = Node().store()
-        n2 = Node().store()
+        n1 = Data().store()
+        n2 = Data().store()
 
         DbExtra.set_value_for_node(n1._dbnode, "pippo", [1, 2, 'a'])
         DbExtra.set_value_for_node(n1._dbnode, "pippobis", [5, 6, 'c'])

aiida/backends/djsite/db/subtests/migrations.py

@@ -33,6 +33,7 @@ def app(self):
     migrate_to = None
 
     def setUp(self):
+        """Go to a specific schema version before running tests."""
         assert self.migrate_from and self.migrate_to, \
             "TestCase '{}' must define migrate_from and migrate_to properties".format(type(self).__name__)
         self.migrate_from = [(self.app, self.migrate_from)]
@@ -43,24 +44,38 @@ def setUp(self):
         # Reverse to the original migration
         executor.migrate(self.migrate_from)
 
-        self.setUpBeforeMigration(old_apps)
+        try:
+            self.setUpBeforeMigration(old_apps)
+            # Run the migration to test
+            executor = MigrationExecutor(connection)
+            executor.loader.build_graph()
+            executor.migrate(self.migrate_to)
 
-        # Run the migration to test
-        executor = MigrationExecutor(connection)
-        executor.loader.build_graph()
-        executor.migrate(self.migrate_to)
+            self.apps = executor.loader.project_state(self.migrate_to).apps
+        except Exception:
+            # Bring back the DB to the correct state if this setup part fails
+            self._revert_database_schema()
+            raise
 
-        self.apps = executor.loader.project_state(self.migrate_to).apps
 
     def tearDown(self):
+        """At the end make sure we go back to the latest schema version."""
+        self._revert_database_schema()
+
+    def setUpBeforeMigration(self, apps):
+        """
+        Anything to do before running the migrations. 
+        This is typically implemented in test subclasses.
+        """
+        pass
+
+    def _revert_database_schema(self):
+        """Bring back the DB to the correct state."""
         from ..migrations import LATEST_MIGRATION
         self.migrate_to = [(self.app, LATEST_MIGRATION)]
         executor = MigrationExecutor(connection)
         executor.migrate(self.migrate_to)
 
-    def setUpBeforeMigration(self, apps):
-        pass
-
 
 class TestDuplicateNodeUuidMigration(TestMigrations):
 
@@ -144,9 +159,9 @@ class TestUuidMigration(TestMigrations):
     migrate_to = '0018_django_1_11'
 
     def setUpBeforeMigration(self, apps):
-        from aiida.orm import Node
+        from aiida.orm import Data
 
-        n = Node().store()
+        n = Data().store()
         self.node_uuid = n.uuid
         self.node_id = n.id