From 35cbc3e21e940cc128fcb08aad201c519e0e2947 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Fri, 3 Dec 2021 08:59:07 +0100 Subject: [PATCH] Implemented fetch_tracker to RepositorySimulator This commit implements a feature in Repository Simulator to track the fetch calls to the metadata and targets. This feature was mentioned in PR #1666 that generated issue #1682. This commit adds RepositorySimulator.fetch_tracker. It also changes the tests/test_updater_consistent_snapshot.py to use the fetch_tracker instead of using mock. It implements a dataclass that stores the calls to fetch metadata (_fetch_metadata) in fetch_tracker.metadata and targets (_fetch_targets) in fetch_tracker.targets. The fetch calls for metadata, and targets are stored as lists. Signed-off-by: Kairo de Araujo --- tests/repository_simulator.py | 16 ++++- tests/test_updater_consistent_snapshot.py | 87 +++++++++++------------ 2 files changed, 57 insertions(+), 46 deletions(-) diff --git a/tests/repository_simulator.py b/tests/repository_simulator.py index 666accb54d..4e530cd32e 100644 --- a/tests/repository_simulator.py +++ b/tests/repository_simulator.py @@ -48,7 +48,7 @@ import os import tempfile from collections import OrderedDict -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timedelta from typing import Dict, Iterator, List, Optional, Tuple from urllib import parse @@ -81,6 +81,14 @@ SPEC_VER = ".".join(SPECIFICATION_VERSION) +@dataclass +class FetchTracker: + """Fetcher counter for metadata and targets.""" + + metadata: List[Tuple[str, Optional[int]]] = field(default_factory=list) + targets: List[Tuple[str, Optional[str]]] = field(default_factory=list) + + @dataclass class RepositoryTarget: """Contains actual target data and the related target metadata.""" @@ -116,6 +124,8 @@ def __init__(self) -> None: self.dump_dir: Optional[str] = None self.dump_version = 0 + self.fetch_tracker = FetchTracker() + now = datetime.utcnow() self.safe_expiry = now.replace(microsecond=0) + timedelta(days=30) @@ -229,6 +239,8 @@ def _fetch_target( If hash is None, then consistent_snapshot is not used. """ + self.fetch_tracker.targets.append((target_path, target_hash)) + repo_target = self.target_files.get(target_path) if repo_target is None: raise FetcherHTTPError(f"No target {target_path}", 404) @@ -248,6 +260,8 @@ def _fetch_metadata( If version is None, non-versioned metadata is being requested. """ + self.fetch_tracker.metadata.append((role, version)) + if role == Root.type: # return a version previously serialized in publish_root() if version is None or version > len(self.signed_roots): diff --git a/tests/test_updater_consistent_snapshot.py b/tests/test_updater_consistent_snapshot.py index 4289d7b860..518693c8f9 100644 --- a/tests/test_updater_consistent_snapshot.py +++ b/tests/test_updater_consistent_snapshot.py @@ -10,7 +10,6 @@ import tempfile import unittest from typing import Any, Dict, Iterable, List, Optional -from unittest.mock import call, patch from tests import utils from tests.repository_simulator import RepositorySimulator @@ -90,19 +89,19 @@ def _assert_targets_files_exist(self, filenames: Iterable[str]) -> None: "consistent_snaphot disabled": { "consistent_snapshot": False, "calls": [ - call("root", 3), - call("timestamp", None), - call("snapshot", None), - call("targets", None), + ("root", 3), + ("timestamp", None), + ("snapshot", None), + ("targets", None), ], }, "consistent_snaphot enabled": { "consistent_snapshot": True, "calls": [ - call("root", 3), - call("timestamp", None), - call("snapshot", 1), - call("targets", 1), + ("root", 3), + ("timestamp", None), + ("snapshot", 1), + ("targets", 1), ], }, } @@ -117,15 +116,14 @@ def test_top_level_roles_update(self, test_case_data: Dict[str, Any]): sim = self._init_repo(consistent_snapshot) updater = self._init_updater(sim) - with patch.object( - sim, "_fetch_metadata", wraps=sim._fetch_metadata - ) as wrapped_fetch: - updater.refresh() + # cleanup fetch tracker metadata + sim.fetch_tracker.metadata.clear() + updater.refresh() - # metadata files are fetched with the expected version (or None) - self.assertListEqual(wrapped_fetch.call_args_list, expected_calls) - # metadata files are always persisted without a version prefix - self._assert_metadata_files_exist(TOP_LEVEL_ROLE_NAMES) + # metadata files are fetched with the expected version (or None) + self.assertListEqual(sim.fetch_tracker.metadata, expected_calls) + # metadata files are always persisted without a version prefix + self._assert_metadata_files_exist(TOP_LEVEL_ROLE_NAMES) self._cleanup_dir(self.metadata_dir) @@ -147,7 +145,7 @@ def test_delegated_roles_update(self, test_case_data: Dict[str, Any]): consistent_snapshot: bool = test_case_data["consistent_snapshot"] expected_version: Optional[int] = test_case_data["expected_version"] rolenames = ["role1", "..", "."] - expected_calls = [call(role, expected_version) for role in rolenames] + expected_calls = [(role, expected_version) for role in rolenames] sim = self._init_repo(consistent_snapshot) # Add new delegated targets @@ -159,15 +157,14 @@ def test_delegated_roles_update(self, test_case_data: Dict[str, Any]): updater = self._init_updater(sim) updater.refresh() - with patch.object( - sim, "_fetch_metadata", wraps=sim._fetch_metadata - ) as wrapped_fetch: - # trigger updater to fetch the delegated metadata - updater.get_targetinfo("anything") - # metadata files are fetched with the expected version (or None) - self.assertListEqual(wrapped_fetch.call_args_list, expected_calls) - # metadata files are always persisted without a version prefix - self._assert_metadata_files_exist(rolenames) + # cleanup fetch tracker metadata + sim.fetch_tracker.metadata.clear() + # trigger updater to fetch the delegated metadata + updater.get_targetinfo("anything") + # metadata files are fetched with the expected version (or None) + self.assertListEqual(sim.fetch_tracker.metadata, expected_calls) + # metadata files are always persisted without a version prefix + self._assert_metadata_files_exist(rolenames) self._cleanup_dir(self.metadata_dir) @@ -176,16 +173,19 @@ def test_delegated_roles_update(self, test_case_data: Dict[str, Any]): "consistent_snapshot": False, "prefix_targets": True, "hash_algo": None, + "targetpaths": ["file", "file.txt", "..file.ext", "f.le"], }, "consistent_snaphot enabled without prefixed targets": { "consistent_snapshot": True, "prefix_targets": False, "hash_algo": None, + "targetpaths": ["file", "file.txt", "..file.ext", "f.le"], }, "consistent_snaphot enabled with prefixed targets": { "consistent_snapshot": True, "prefix_targets": True, "hash_algo": "sha256", + "targetpaths": ["file", "file.txt", "..file.ext", "f.le"], }, } @@ -197,7 +197,7 @@ def test_download_targets(self, test_case_data: Dict[str, Any]): consistent_snapshot: bool = test_case_data["consistent_snapshot"] prefix_targets_with_hash: bool = test_case_data["prefix_targets"] hash_algo: Optional[str] = test_case_data["hash_algo"] - targetpaths = ["file", "file.txt", "..file.ext", "f.le"] + targetpaths: List[str] = test_case_data["targetpaths"] sim = self._init_repo(consistent_snapshot, prefix_targets_with_hash) # Add targets to repository @@ -210,23 +210,20 @@ def test_download_targets(self, test_case_data: Dict[str, Any]): updater.config.prefix_targets_with_hash = prefix_targets_with_hash updater.refresh() - with patch.object( - sim, "_fetch_target", wraps=sim._fetch_target - ) as wrapped_fetch_target: - - for targetpath in targetpaths: - info = updater.get_targetinfo(targetpath) - updater.download_target(info) - expected_prefix = ( - None if not hash_algo else info.hashes[hash_algo] - ) - # files are fetched with the expected hash prefix (or None) - wrapped_fetch_target.assert_called_once_with( - info.path, expected_prefix - ) - # target files are always persisted without hash prefix - self._assert_targets_files_exist([info.path]) - wrapped_fetch_target.reset_mock() + for targetpath in targetpaths: + info = updater.get_targetinfo(targetpath) + updater.download_target(info) + + # target files are always persisted without hash prefix + self._assert_targets_files_exist([info.path]) + + # files are fetched with the expected hash prefix (or None) + expected_fetches = [ + (targetpath, None if not hash_algo else info.hashes[hash_algo]) + ] + + self.assertListEqual(sim.fetch_tracker.targets, expected_fetches) + sim.fetch_tracker.targets.clear() self._cleanup_dir(self.targets_dir)