Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rework export archive extraction #3242

Closed
wants to merge 4 commits into from
Closed

Rework export archive extraction #3242

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
66c8f13
Rewrite `extract_tree` for archive import
CasperWA Jul 23, 2019
a5824d6
Rework `extract_tree` and unpacking archives
CasperWA Aug 29, 2019
765d6a7
Archive class is used for all migrations
CasperWA Sep 9, 2019
bda6519
Addressing some suggestions by @sphuber
CasperWA Sep 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 0 additions & 15 deletions aiida/backends/tests/cmdline/commands/test_export.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,21 +212,6 @@ def test_migrate_silent(self):
finally:
delete_temporary_file(filename_output)

def test_migrate_tar_gz(self):
"""Test that -F/--archive-format option can be used to write a tar.gz instead."""
filename_input = get_archive_file(self.penultimate_archive, filepath=self.fixture_archive)
filename_output = next(tempfile._get_candidate_names()) # pylint: disable=protected-access

for option in ['-F', '--archive-format']:
try:
options = [option, 'tar.gz', filename_input, filename_output]
result = self.cli_runner.invoke(cmd_export.migrate, options)
self.assertIsNone(result.exception, result.output)
self.assertTrue(os.path.isfile(filename_output))
self.assertTrue(tarfile.is_tarfile(filename_output))
finally:
delete_temporary_file(filename_output)

def test_inspect(self):
"""Test the functionality of `verdi export inspect`."""
archives = []
Expand Down
21 changes: 21 additions & 0 deletions aiida/backends/tests/cmdline/commands/test_import.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,27 @@ def test_import_archive(self):
self.assertIsNone(result.exception, result.output)
self.assertEqual(result.exit_code, 0, result.output)

def test_import_folder(self):
"""Test import for archive folder from disk"""
import os

from aiida.tools.importexport import Archive

archive_filepath = get_archive_file(self.newest_archive, filepath=self.archive_path)

with Archive(archive_filepath, silent=True) as archive:
archive.unpack()

# Make sure the JSON files and the nodes subfolder exists and are correctly extracted,
# then try to import it by passing the extracted folder to `verdi import`.
for name in {'metadata.json', 'data.json', 'nodes'}:
self.assertTrue(os.path.exists(os.path.join(archive.folder.abspath, name)))

result = self.cli_runner.invoke(cmd_import.cmd_import, [archive.folder.abspath])

self.assertIsNone(result.exception, msg=result.output)
self.assertEqual(result.exit_code, 0, msg=result.output)

def test_import_to_group(self):
"""
Test import to existing Group and that Nodes are added correctly for multiple imports of the same,
Expand Down
5 changes: 2 additions & 3 deletions aiida/backends/tests/tools/importexport/common/test_archive.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,16 @@
"""Tests for the Archive class."""

from aiida.backends.testbase import AiidaTestCase
from aiida.common.exceptions import InvalidOperation
from aiida.backends.tests.utils.archives import get_archive_file
from aiida.tools.importexport import Archive, CorruptArchive
from aiida.tools.importexport import CorruptArchive, InvalidArchiveOperation, Archive


class TestCommonArchive(AiidaTestCase):
"""Tests for the :py:class:`~aiida.tools.importexport.common.archive.Archive` class."""

def test_context_required(self):
"""Verify that accessing a property of an Archive outside of a context manager raises."""
with self.assertRaises(InvalidOperation):
with self.assertRaises(InvalidArchiveOperation):
filepath = get_archive_file('export_v0.1_simple.aiida', filepath='export/migrate')
archive = Archive(filepath)
archive.version_format # pylint: disable=pointless-statement
Expand Down
139 changes: 55 additions & 84 deletions aiida/backends/tests/tools/importexport/migration/test_migration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@

from aiida import orm
from aiida.backends.testbase import AiidaTestCase
from aiida.backends.tests.utils.archives import get_archive_file, get_json_files, migrate_archive
from aiida.backends.tests.utils.archives import get_archive_file, NoContextArchive
from aiida.backends.tests.utils.configuration import with_temp_dir
from aiida.tools.importexport import import_data, EXPORT_VERSION as newest_version
from aiida.tools.importexport.migration import migrate_recursively, verify_metadata_version
from aiida.common.utils import Capturing
from aiida.tools.importexport import import_data, Archive, EXPORT_VERSION as newest_version
from aiida.tools.importexport.common.exceptions import MigrationValidationError
from aiida.tools.importexport.migration import migrate_recursively, verify_archive_version, migrate_archive


class TestExportFileMigration(AiidaTestCase):
Expand Down Expand Up @@ -63,44 +63,16 @@ def setUp(self):
super().setUp()
self.reset_database()

def test_migrate_recursively(self):
"""Test function 'migrate_recursively'"""
import io
import tarfile
import zipfile

from aiida.common.exceptions import NotExistent
from aiida.common.folders import SandboxFolder
from aiida.common.json import load as jsonload
from aiida.tools.importexport.common.archive import extract_tar, extract_zip

# Get metadata.json and data.json as dicts from v0.1 file archive
# Cannot use 'get_json_files' for 'export_v0.1_simple.aiida',
# because we need to pass the SandboxFolder to 'migrate_recursively'
dirpath_archive = get_archive_file('export_v0.1_simple.aiida', **self.core_archive)

with SandboxFolder(sandbox_in_repo=False) as folder:
if zipfile.is_zipfile(dirpath_archive):
extract_zip(dirpath_archive, folder, silent=True)
elif tarfile.is_tarfile(dirpath_archive):
extract_tar(dirpath_archive, folder, silent=True)
else:
raise ValueError('invalid file format, expected either a zip archive or gzipped tarball')

try:
with io.open(folder.get_abs_path('data.json'), 'r', encoding='utf8') as fhandle:
data = jsonload(fhandle)
with io.open(folder.get_abs_path('metadata.json'), 'r', encoding='utf8') as fhandle:
metadata = jsonload(fhandle)
except IOError:
raise NotExistent('export archive does not contain the required file {}'.format(fhandle.filename))

verify_metadata_version(metadata, version='0.1')

# Migrate to newest version
new_version = migrate_recursively(metadata, data, folder)
verify_metadata_version(metadata, version=newest_version)
self.assertEqual(new_version, newest_version)
@with_temp_dir
def test_migrate_archive(self, temp_dir):
"""Test function 'migrate_archive'"""
input_file = get_archive_file('export_v0.1_simple.aiida', **self.core_archive)
migrated_file = os.path.join(temp_dir, 'migrated_file.aiida')

# Migrate to newest version
old_version, new_version = migrate_archive(input_file, migrated_file, silent=True)
self.assertEqual(old_version, '0.1')
self.assertEqual(new_version, newest_version)

@with_temp_dir
def test_no_node_export(self, temp_dir):
Expand All @@ -119,7 +91,7 @@ def test_no_node_export(self, temp_dir):
user_emails.append('aiida@localhost')

# Perform the migration
migrate_archive(input_file, output_file)
migrate_archive(input_file, output_file, silent=True)

# Load the migrated file
import_data(output_file, silent=True)
Expand Down Expand Up @@ -156,51 +128,50 @@ def test_wrong_versions(self):
msg="'{}' was not expected to be a legal version, legal version: {}".format(version, legal_versions)
)

# Make sure migrate_recursively throws a critical message and raises SystemExit
# Make sure migrate_recursively throws a critical message and raises MigrationValidationError
for metadata in wrong_version_metadatas:
with self.assertRaises(SystemExit) as exception:
with Capturing(capture_stderr=True):
new_version = migrate_recursively(metadata, {}, None)

self.assertIn(
'Critical: Cannot migrate from version {}'.format(metadata['export_version']),
exception.exception,
msg="Expected a critical statement for the wrong export version '{}', "
'instead got {}'.format(metadata['export_version'], exception.exception)
)
archive = NoContextArchive(metadata=metadata)
with self.assertRaises(MigrationValidationError) as exception:
new_version = migrate_recursively(archive)

self.assertIsNone(
new_version,
msg='migrate_recursively should not return anything, '
"hence the 'return' should be None, but instead it is {}".format(new_version)
)

self.assertIn(
'Cannot migrate from version {}'.format(metadata['export_version']),
str(exception.exception),
msg="Expected a critical statement for the wrong export version '{}', "
'instead got {}'.format(metadata['export_version'], str(exception.exception))
)

def test_migrate_newest_version(self):
"""
Test critical message and SystemExit is raised, when an export file with the newest export version is migrated
"""
# Initialization
metadata = {'export_version': newest_version}
archive = NoContextArchive(metadata=metadata)

# Check
with self.assertRaises(SystemExit) as exception:

with Capturing(capture_stderr=True):
new_version = migrate_recursively(metadata, {}, None)
with self.assertRaises(MigrationValidationError) as exception:
new_version = migrate_recursively(archive)

self.assertIn(
'Critical: Your export file is already at the newest export version {}'.format(
metadata['export_version']
),
exception.exception,
msg="Expected a critical statement that the export version '{}' is the newest export version '{}', "
'instead got {}'.format(metadata['export_version'], newest_version, exception.exception)
)
self.assertIsNone(
new_version,
msg='migrate_recursively should not return anything, '
"hence the 'return' should be None, but instead it is {}".format(new_version)
)

self.assertIn(
'Your export file is already at the newest export version {}'.format(metadata['export_version']),
str(exception.exception),
msg="Expected a critical statement that the export version '{}' is the newest export version '{}', "
'instead got {}'.format(metadata['export_version'], newest_version, str(exception.exception))
)

@with_temp_dir
def test_v02_to_newest(self, temp_dir):
"""Test migration of exported files from v0.2 to newest export version"""
Expand All @@ -209,9 +180,9 @@ def test_v02_to_newest(self, temp_dir):
output_file = os.path.join(temp_dir, 'output_file.aiida')

# Perform the migration
migrate_archive(input_file, output_file)
metadata, _ = get_json_files(output_file)
verify_metadata_version(metadata, version=newest_version)
migrate_archive(input_file, output=output_file, silent=True)
with Archive(output_file) as archive:
verify_archive_version(archive.version_format, version=newest_version)

# Load the migrated file
import_data(output_file, silent=True)
Expand Down Expand Up @@ -264,9 +235,9 @@ def test_v03_to_newest(self, temp_dir):
output_file = os.path.join(temp_dir, 'output_file.aiida')

# Perform the migration
migrate_archive(input_file, output_file)
metadata, _ = get_json_files(output_file)
verify_metadata_version(metadata, version=newest_version)
migrate_archive(input_file, output_file, silent=True)
with Archive(output_file) as archive:
verify_archive_version(archive.version_format, version=newest_version)

# Load the migrated file
import_data(output_file, silent=True)
Expand Down Expand Up @@ -319,9 +290,9 @@ def test_v04_to_newest(self, temp_dir):
output_file = os.path.join(temp_dir, 'output_file.aiida')

# Perform the migration
migrate_archive(input_file, output_file)
metadata, _ = get_json_files(output_file)
verify_metadata_version(metadata, version=newest_version)
migrate_archive(input_file, output_file, silent=True)
with Archive(output_file) as archive:
verify_archive_version(archive.version_format, version=newest_version)

# Load the migrated file
import_data(output_file, silent=True)
Expand Down Expand Up @@ -374,9 +345,9 @@ def test_v05_to_newest(self, temp_dir):
output_file = os.path.join(temp_dir, 'output_file.aiida')

# Perform the migration
migrate_archive(input_file, output_file)
metadata, _ = get_json_files(output_file)
verify_metadata_version(metadata, version=newest_version)
migrate_archive(input_file, output_file, silent=True)
with Archive(output_file) as archive:
verify_archive_version(archive.version_format, version=newest_version)

# Load the migrated file
import_data(output_file, silent=True)
Expand Down Expand Up @@ -429,9 +400,9 @@ def test_v06_to_newest(self, temp_dir):
output_file = os.path.join(temp_dir, 'output_file.aiida')

# Perform the migration
migrate_archive(input_file, output_file)
metadata, _ = get_json_files(output_file)
verify_metadata_version(metadata, version=newest_version)
migrate_archive(input_file, output_file, silent=True)
with Archive(output_file) as archive:
verify_archive_version(archive.version_format, version=newest_version)

# Load the migrated file
import_data(output_file, silent=True)
Expand Down Expand Up @@ -484,9 +455,9 @@ def test_v07_to_newest(self, temp_dir):
output_file = os.path.join(temp_dir, 'output_file.aiida')

# Perform the migration
migrate_archive(input_file, output_file)
metadata, _ = get_json_files(output_file)
verify_metadata_version(metadata, version=newest_version)
migrate_archive(input_file, output_file, silent=True)
with Archive(output_file) as archive:
verify_archive_version(archive.version_format, version=newest_version)

# Load the migrated file
import_data(output_file, silent=True)
Expand Down
28 changes: 17 additions & 11 deletions aiida/backends/tests/tools/importexport/migration/test_v01_to_v02.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@

from aiida import get_version
from aiida.backends.testbase import AiidaTestCase
from aiida.backends.tests.utils.archives import get_json_files
from aiida.tools.importexport.migration.utils import verify_metadata_version
from aiida.backends.tests.utils.archives import get_archive_file
from aiida.tools.importexport import Archive
from aiida.tools.importexport.migration.utils import verify_archive_version
from aiida.tools.importexport.migration.v01_to_v02 import migrate_v1_to_v2


Expand All @@ -21,17 +22,22 @@ class TestMigrateV01toV02(AiidaTestCase):

def test_migrate_v1_to_v2(self):
"""Test function migrate_v1_to_v2"""
# Get metadata.json and data.json as dicts from v0.1 file archive
metadata_v1, data_v1 = get_json_files('export_v0.1_simple.aiida', filepath='export/migrate')
verify_metadata_version(metadata_v1, version='0.1')
archive_v1 = get_archive_file('export_v0.1_simple.aiida', filepath='export/migrate')
archive_v2 = get_archive_file('export_v0.2_simple.aiida', filepath='export/migrate')

# Get metadata.json and data.json as dicts from v0.2 file archive
metadata_v2, data_v2 = get_json_files('export_v0.2_simple.aiida', filepath='export/migrate')
verify_metadata_version(metadata_v2, version='0.2')
with Archive(archive_v1) as archive:
verify_archive_version(archive.version_format, '0.1')
migrate_v1_to_v2(archive)
verify_archive_version(archive.version_format, '0.2')

# Migrate to v0.2
migrate_v1_to_v2(metadata_v1, data_v1)
verify_metadata_version(metadata_v1, version='0.2')
data_v1 = archive.data
metadata_v1 = archive.meta_data

with Archive(archive_v2) as archive:
verify_archive_version(archive.version_format, '0.2')

data_v2 = archive.data
metadata_v2 = archive.meta_data

# Remove AiiDA version, since this may change irregardless of the migration function
metadata_v1.pop('aiida_version')
Expand Down
Loading