From 142a11cd5bc0565240b0ecb84de01c8eb7e1d5ba Mon Sep 17 00:00:00 2001 From: lomnido <135008914+lomnido@users.noreply.github.com> Date: Mon, 20 Jan 2025 11:09:04 +0100 Subject: [PATCH] sync '--clean', fix bare_repos, additional tests --- tsrc/cleaner.py | 40 ++ tsrc/cli/__init__.py | 2 +- tsrc/cli/dump_manifest.py | 18 +- tsrc/cli/manifest.py | 12 +- tsrc/cli/status.py | 14 +- tsrc/cli/sync.py | 30 +- tsrc/cloner.py | 42 +- tsrc/config_status.py | 2 + tsrc/dump_manifest.py | 22 +- tsrc/dump_manifest_args.py | 10 +- tsrc/dump_manifest_args_data.py | 3 +- tsrc/dump_manifest_helper.py | 12 +- tsrc/git.py | 45 +- tsrc/groups_to_find.py | 5 +- tsrc/manifest_common.py | 8 +- tsrc/repo.py | 5 +- tsrc/repo_grabber.py | 3 + tsrc/status_endpoint.py | 4 +- tsrc/status_header.py | 31 +- tsrc/test/cli/test_dirty_after_sync.py | 328 ++++++++++++++ tsrc/test/cli/test_display_bare_repos.py | 22 +- tsrc/test/cli/test_dump_manifest.py | 29 +- .../test_dump_manifest__filter_bo_manifest.py | 44 +- tsrc/test/cli/test_dump_manifest__groups.py | 11 +- tsrc/test/cli/test_dump_manifest_sha1.py | 336 +++++++++++++- tsrc/test/cli/test_groups_extra.py | 6 +- tsrc/test/cli/test_manifest.py | 2 +- tsrc/test/cli/test_sync__groups.py | 412 ++++++++++++++++++ tsrc/test/cli/test_sync_extended.py | 219 +++++++++- tsrc/workspace.py | 31 ++ tsrc/workspace_repos_summary.py | 46 +- 31 files changed, 1625 insertions(+), 169 deletions(-) create mode 100644 tsrc/cleaner.py create mode 100644 tsrc/test/cli/test_dirty_after_sync.py create mode 100644 tsrc/test/cli/test_sync__groups.py diff --git a/tsrc/cleaner.py b/tsrc/cleaner.py new file mode 100644 index 00000000..71fe8b93 --- /dev/null +++ b/tsrc/cleaner.py @@ -0,0 +1,40 @@ +from pathlib import Path +from typing import List + +import cli_ui as ui + +from tsrc.executor import Outcome, Task +from tsrc.repo import Repo + + +class Cleaner(Task[Repo]): + def __init__( + self, + workspace_path: Path, + *, + do_hard_clean: bool = False, + ) -> None: + self.workspace_path = workspace_path + self.do_hard_clean = do_hard_clean + + def describe_item(self, item: Repo) -> str: + return item.dest + + def describe_process_start(self, item: Repo) -> List[ui.Token]: + return ["Cleaning", item.dest] + + def describe_process_end(self, item: Repo) -> List[ui.Token]: + return [ui.green, "ok", ui.reset, item.dest] + + def process(self, index: int, count: int, repo: Repo) -> Outcome: + """ + Clean each repo so it will be ready for next 'sync' + """ + self.info_count(index, count, "Cleaning", repo.dest) + + repo_path = self.workspace_path / repo.dest + self.run_git(repo_path, "clean", "-f", "-d") + if self.do_hard_clean is True: + self.run_git(repo_path, "clean", "-f", "-X", "-d") + + return Outcome.empty() diff --git a/tsrc/cli/__init__.py b/tsrc/cli/__init__.py index 42b0a092..a52b8338 100644 --- a/tsrc/cli/__init__.py +++ b/tsrc/cli/__init__.py @@ -334,7 +334,7 @@ def repos_from_config( ", ".join(repo_groups), ) # fmt: on - return manifest.get_repos(groups=repo_groups) + return manifest.get_repos(groups=repo_groups, ignore_if_group_not_found=silent) else: # workspace config does not specify clone_all_repos nor # a list of groups, ask the manifest for the list of default diff --git a/tsrc/cli/dump_manifest.py b/tsrc/cli/dump_manifest.py index cbe753cf..4a53ca11 100644 --- a/tsrc/cli/dump_manifest.py +++ b/tsrc/cli/dump_manifest.py @@ -76,7 +76,7 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None: parser.add_argument( "-U", "--update-on", - help="Set UPDATE operation mode, by setting the UPDATE source and DESTINATION default to provided UPDATE_AT path to YAML file. Such path must exists", # noqa: E501 + help="Set UPDATE operation mode, by setting the UPDATE source and DESTINATION default to provided UPDATE_ON path to YAML file. Such path must exists", # noqa: E501 type=Path, dest="update_on", ) @@ -87,14 +87,20 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None: dest="no_repo_delete", ) parser.add_argument( - "--sha1-only", + "--sha1-on", action="store_true", - help="Use SHA1 as only value (with branch if available) for every considered Repo. This is particulary useful when we want to point to exact point of Repos states", # noqa: E501 - dest="sha1_only", + help="Explicitly set SHA1 for every considered Repo. This is particulary useful when we want to point to exact commit in Repos", # noqa: E501 + dest="sha1_on", + ) + parser.add_argument( + "--sha1-off", + action="store_true", + help="Tell dumping mechanism that we do not care about excat Repo commit, as long as it keep 'branch' and 'tag's. This option is ignored, when there is no 'branch' or 'tag'", # noqa: E501 + dest="sha1_off", ) parser.add_argument( "-X", - "--skip-manifest", + "--skip-manifest-repo", help="Skip manifest repository if found. If not, it is ignored. For this filter to work, the Workspace needs to be present. And it is only applied after the processing of the Repositories", # noqa: E501 dest="skip_manifest", default=False, @@ -102,7 +108,7 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None: ) parser.add_argument( "-M", - "--only-manifest", + "--only-manifest-repo", help="Only work with manifest repository if found. If not, the Error is thrown that list of Repositories ends up empty. For this filter to work, the Workspace needs to be present. And it is only applied after the processing of the Repositories", # noqa: E501 dest="only_manifest", default=False, diff --git a/tsrc/cli/manifest.py b/tsrc/cli/manifest.py index e239487b..6675b43d 100644 --- a/tsrc/cli/manifest.py +++ b/tsrc/cli/manifest.py @@ -84,7 +84,7 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None: def run(args: argparse.Namespace) -> None: - gtf = GroupsToFind(args.groups) + gtf = GroupsToFind(args.groups, args.ignore_if_group_not_found) groups_seen = simulate_get_workspace_with_repos(args) gtf.found_these(groups_seen) @@ -129,9 +129,13 @@ def run(args: argparse.Namespace) -> None: ) if args.manifest_branch: cfg_update_data = ConfigUpdateData(manifest_branch=args.manifest_branch) - status_header.register_change( - cfg_update_data, [ConfigUpdateType.MANIFEST_BRANCH] - ) + if ( + status_header.register_change( + cfg_update_data, [ConfigUpdateType.MANIFEST_BRANCH] + ) + is False + ): + return status_header.display() status_collector = StatusCollector( workspace, ignore_group_item=args.ignore_group_item diff --git a/tsrc/cli/status.py b/tsrc/cli/status.py index 4f9ac313..a6f2b367 100644 --- a/tsrc/cli/status.py +++ b/tsrc/cli/status.py @@ -21,7 +21,7 @@ ready_tmp_bare_repos, ) from tsrc.manifest_common_data import ManifestsTypeOfData -from tsrc.pcs_repo import get_deep_manifest_from_local_manifest_pcsrepo +from tsrc.pcs_repo import PCSRepo, get_deep_manifest_from_local_manifest_pcsrepo from tsrc.repo import Repo from tsrc.status_endpoint import ( BareStatus, @@ -102,7 +102,7 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None: def run(args: argparse.Namespace) -> None: - gtf = GroupsToFind(args.groups) + gtf = GroupsToFind(args.groups, args.ignore_if_group_not_found) groups_seen = simulate_get_workspace_with_repos(args) gtf.found_these(groups_seen) @@ -124,23 +124,23 @@ def run(args: argparse.Namespace) -> None: ) # DM (if present) + bare DM (if DM and present) - dm = None + dm_pcsr: Union[PCSRepo, None] = None bare_dm_repos: List[Repo] = [] if args.use_deep_manifest is True: - dm, gtf = get_deep_manifest_from_local_manifest_pcsrepo( + dm_pcsr, gtf = get_deep_manifest_from_local_manifest_pcsrepo( workspace, gtf, ) - if dm and args.local_git_only is False: + if dm_pcsr and args.local_git_only is False: # this require to check remote bare_dm_repos = prepare_tmp_bare_dm_repos( - workspace, dm, gtf, num_jobs=get_num_jobs(args) + workspace, dm_pcsr, gtf, num_jobs=get_num_jobs(args) ) wrs = WorkspaceReposSummary( workspace, gtf, - dm, + dm_pcsr, manifest_marker=args.use_manifest_marker, future_manifest=args.use_future_manifest, use_same_future_manifest=args.use_same_future_manifest, diff --git a/tsrc/cli/sync.py b/tsrc/cli/sync.py index 05194eed..7e89d6e6 100644 --- a/tsrc/cli/sync.py +++ b/tsrc/cli/sync.py @@ -50,6 +50,18 @@ def configure_parser(subparser: argparse._SubParsersAction) -> None: dest="ignore_group_item", help="ignore group element if it is not found among Manifest's Repos. WARNING: If you end up in need of this option, you have to understand that you end up with useles Manifest. Warnings will be printed for each Group element that is missing, so it may be easier to fix that. Using this option is NOT RECOMMENDED for normal use", # noqa: E501 ) + parser.add_argument( + "--clean", + action="store_true", + dest="do_clean", + help="WARNING: you may loose files that are not under the version control. like such files that are ignored by '.gitignore'. sync to clean state, so the next sync can run smoothly. use with care.", # noqa: E501 + ) + parser.add_argument( + "--hard-clean", + action="store_true", + dest="do_hard_clean", + help="WARNING: you may loose files that are not under the version control and also files ignored by '.gitignore'. sync to clean state, that does not even contain ignored files. use with care.", # noqa: E501 + ) parser.add_argument( "--no-correct-branch", action="store_false", @@ -77,6 +89,8 @@ def run(args: argparse.Namespace) -> None: correct_branch = args.correct_branch workspace = get_workspace(args) num_jobs = get_num_jobs(args) + do_clean = args.do_clean + do_hard_clean = args.do_hard_clean ignore_if_group_not_found: bool = False report_update_repo_groups: bool = False @@ -93,10 +107,11 @@ def run(args: argparse.Namespace) -> None: found_groups = list( set(groups).intersection(local_manifest.group_list.groups) ) - workspace.update_config_repo_groups( - groups=found_groups, ignore_group_item=args.ignore_group_item - ) - report_update_repo_groups = True + if update_config_repo_groups is True: + workspace.update_config_repo_groups( + groups=found_groups, ignore_group_item=args.ignore_group_item + ) + report_update_repo_groups = True if update_config_repo_groups is True: if args.ignore_if_group_not_found is True: @@ -112,6 +127,8 @@ def run(args: argparse.Namespace) -> None: ui.info_2("Leaving repo_groups intact") else: ui.info_2("Not updating manifest") + if args.ignore_if_group_not_found is True: + ignore_if_group_not_found = True workspace.repos = resolve_repos( workspace, @@ -123,7 +140,9 @@ def run(args: argparse.Namespace) -> None: ignore_if_group_not_found=ignore_if_group_not_found, ignore_group_item=args.ignore_group_item, ) - + if len(workspace.repos) == 0: + ui.info_1("Nothing to synchronize, skipping") + return workspace.clone_missing(num_jobs=num_jobs) workspace.set_remotes(num_jobs=num_jobs) workspace.sync( @@ -132,5 +151,6 @@ def run(args: argparse.Namespace) -> None: correct_branch=correct_branch, num_jobs=num_jobs, ) + workspace.clean(do_clean=do_clean, do_hard_clean=do_hard_clean, num_jobs=num_jobs) workspace.perform_filesystem_operations(ignore_group_item=args.ignore_group_item) ui.info_1("Workspace synchronized") diff --git a/tsrc/cloner.py b/tsrc/cloner.py index f55eea09..46423044 100644 --- a/tsrc/cloner.py +++ b/tsrc/cloner.py @@ -180,7 +180,7 @@ def _choose_remote(self, repo: Repo) -> Remote: return repo.remotes[0] - def bare_clone_repo(self, repo: Repo) -> None: + def bare_clone_repo(self, repo: Repo) -> Path: # check if our Repo is bare repo_path = self.workspace_path / repo.dest parent = repo_path.parent @@ -188,23 +188,28 @@ def bare_clone_repo(self, repo: Repo) -> None: remote = self._choose_remote(repo) remote_url = remote.url if Path(str(repo_path) + os.sep + ".git").is_dir(): - return - if repo._bare_clone_path: - clone_args = [ - "clone", - "--mirror", - str(repo._bare_clone_path), - str(repo_path) + os.sep + ".git", - ] - else: - clone_args = [ - "clone", - "--mirror", - remote_url, - str(repo_path) + os.sep + ".git", - ] + return repo_path + clone_args = [ + "clone", + "--mirror", + remote_url, + str(repo_path) + os.sep + ".git", + ] - self.run_git(parent, *clone_args) + run_git_captured(parent, *clone_args) + + # make sure from this moment on to use 'repo_path' + + run_git_captured( + repo_path, "config", "--bool", "core.bare", "false", check=False + ) + + run_git_captured(repo_path, "remote", "remove", remote.name, check=False) + run_git_captured( + repo_path, "remote", "add", remote.name, remote_url, check=False + ) + + return repo_path def bare_set_branch(self, repo: Repo) -> bool: @@ -244,7 +249,8 @@ def bare_reset_repo(self, repo: Repo) -> None: def process(self, index: int, count: int, repo: Repo) -> Outcome: self.info_count(index, count, repo.dest, end="\r") - self.bare_clone_repo(repo) + repo_path = self.bare_clone_repo(repo) + self.run_git(repo_path, "fetch", "--all", "--prune") if self.bare_set_branch(repo) is True: self.bare_reset_repo(repo) # NOTE: not considering submodules (not useful for bare Repo) diff --git a/tsrc/config_status.py b/tsrc/config_status.py index 35fd4fcb..dd12d277 100644 --- a/tsrc/config_status.py +++ b/tsrc/config_status.py @@ -15,6 +15,7 @@ from tsrc.config_data import ConfigUpdateData, ConfigUpdateType from tsrc.config_status_rc import ConfigStatusReturnCode from tsrc.config_tools import ConfigTools +from tsrc.errors import Error from tsrc.status_header_dm import StatusHeaderDisplayMode from tsrc.workspace import Workspace @@ -110,6 +111,7 @@ def _manifest_branch_report_issue( "ignoring", ui.reset, ) + raise Error("aborting Manifest branch change") if rc == ConfigStatusReturnCode.CANCEL: branch_0 = self.workspace.config.manifest_branch_0 if branch == branch_0: diff --git a/tsrc/dump_manifest.py b/tsrc/dump_manifest.py index c65a9665..be76b374 100644 --- a/tsrc/dump_manifest.py +++ b/tsrc/dump_manifest.py @@ -584,9 +584,7 @@ def _add_repos_based_on_mris( rr["branch"] = mri.branch if mri.tag: rr["tag"] = mri.tag - if ( - not mri.branch and not mri.tag and mri.sha1 - ) or mdo.sha1_only is True: + if (not mri.branch and not mri.tag and mri.sha1) or mdo.sha1_on is True: rr["sha1"] = mri.sha1 # TODO: add comment in form of '\n' just to better separate Repos @@ -718,7 +716,12 @@ def _update_on_update_on_items_on_repo( and mri.sha1 # noqa: W503 and y[u_i] != mri.sha1 # noqa: W503 ) - or (mdo.sha1_only is True and y[u_i] != mri.sha1) # noqa: W503 + or (mdo.sha1_on is True and y[u_i] != mri.sha1) # noqa: W503 + or ( + mdo.sha1_off is False + and (mri.ahead > 0 or mri.behind > 0) + and y[u_i] != mri.sha1 + ) ): y[u_i] = mri.sha1 ret_updated = True @@ -833,7 +836,10 @@ def _update_on_items_on_repo( if not s_item: if mri.sha1: s_item.append("sha1") - if "sha1" not in s_item and mdo.sha1_only is True: + if "sha1" not in s_item and ( + mdo.sha1_on is True + or (mdo.sha1_off is False and (mri.behind > 0 or mri.ahead > 0)) + ): s_item.append("sha1") # add these on item @@ -1145,8 +1151,10 @@ def do_create( if items.tag: rr["tag"] = items.tag if ( - not items.branch and not items.tag and items.sha1 - ) or mdo.sha1_only is True: + (not items.branch and not items.tag and items.sha1) + or mdo.sha1_on is True + or (mdo.sha1_off is False and (items.ahead > 0 or items.behind > 0)) + ): rr["sha1"] = items.sha1 y["repos"].append(rr) diff --git a/tsrc/dump_manifest_args.py b/tsrc/dump_manifest_args.py index a2de0b7c..6be32e0d 100644 --- a/tsrc/dump_manifest_args.py +++ b/tsrc/dump_manifest_args.py @@ -67,15 +67,19 @@ def __init__(self, args: argparse.Namespace) -> None: def _get_manifest_data_options(self) -> ManifestDataOptions: mdo = ManifestDataOptions() - if self.args.sha1_only is True: - mdo.sha1_only = True + if self.args.sha1_on is True and self.args.sha1_off is True: + raise Exception("'--sha1-on' and '--sha1-off' are mutually exclusive") + elif self.args.sha1_on is True: + mdo.sha1_on = True + elif self.args.sha1_off is True: + mdo.sha1_off = True if self.args.skip_manifest is True: mdo.skip_manifest = True if self.args.only_manifest is True: mdo.only_manifest = True if self.args.skip_manifest is True and self.args.only_manifest is True: raise Exception( - "'--skip-manifest' and '--only-manifest' are mutually exclusive" + "'--skip-manifest-repo' and '--only-manifest-repo' are mutually exclusive" ) return mdo diff --git a/tsrc/dump_manifest_args_data.py b/tsrc/dump_manifest_args_data.py index d3233c16..ffa90ec9 100644 --- a/tsrc/dump_manifest_args_data.py +++ b/tsrc/dump_manifest_args_data.py @@ -23,7 +23,8 @@ class UpdateSourceEnum(Enum): @dataclass class ManifestDataOptions: - sha1_only: bool = False + sha1_on: bool = False + sha1_off: bool = False skip_manifest: bool = False only_manifest: bool = False ignore_groups: bool = False # not implemented diff --git a/tsrc/dump_manifest_helper.py b/tsrc/dump_manifest_helper.py index e6dc20d8..8355c7ff 100644 --- a/tsrc/dump_manifest_helper.py +++ b/tsrc/dump_manifest_helper.py @@ -24,6 +24,10 @@ class ManifestRepoItem: ignore_submodules: Optional[bool] = False remotes: Optional[List[Remote]] = None groups_considered: Optional[bool] = False + + # positon-related + ahead: int = 0 + behind: int = 0 # TODO: implement test if required variables are set @property @@ -54,10 +58,12 @@ def _repo_to_mri( return ManifestRepoItem( branch=repo.branch, tag=repo.tag, - # sha1=repo.sha1_full, - sha1=repo.sha1, + sha1=repo.sha1_full, + # sha1=repo.sha1, ignore_submodules=repo.ignore_submodules, remotes=repo.remotes, + ahead=repo._grabbed_ahead, + behind=repo._grabbed_behind, ) def _repos_to_mris( @@ -86,6 +92,8 @@ def _status_to_mri( ignore_submodules=w_repo.ignore_submodules, remotes=w_repo.remotes, groups_considered=True, + ahead=status.git.ahead, + behind=status.git.behind, ) return ManifestRepoItem() diff --git a/tsrc/git.py b/tsrc/git.py index 865d7142..3a877c36 100644 --- a/tsrc/git.py +++ b/tsrc/git.py @@ -61,8 +61,10 @@ class GitBareStatus: as only very few information is needed for related Use-Case """ - def __init__(self, working_path: Path) -> None: + def __init__(self, working_path: Path, remote_name: str, remote_url: str) -> None: self.working_path = working_path + self.remote_name = remote_name + self.remote_url = remote_url self.branch: Optional[str] = None self.ahead = 0 self.behind = 0 @@ -84,12 +86,45 @@ def update_branch(self) -> None: def update_upstream(self) -> None: if self.branch: + + # check if upstream is already set + rc, is_upstr = run_git_captured( + self.working_path, + "rev-parse", + "--symbolic-full-name", + f"{self.branch}@{{u}}", + check=False, + ) + if rc == 0: + self.is_upstreamed = True + return + + # if we are here, than we need to take some measures + # so to setting upstream will be possible + + # refresh remotes so setting upstream will be possible later + run_git_captured( + self.working_path, "remote", "remove", self.remote_name, check=False + ) + run_git_captured( + self.working_path, + "remote", + "add", + self.remote_name, + self.remote_url, + check=False, + ) + run_git_captured( + self.working_path, "fetch", "--all", "--prune", check=False + ) + + # only set upstream if it is not set rc, _ = run_git_captured( self.working_path, "branch", self.branch, "--set-upstream-to", - f"origin/{self.branch}", + f"{self.remote_name}/{self.branch}", check=False, ) if rc == 0: @@ -450,8 +485,10 @@ def get_git_status(working_path: Path) -> GitStatus: return status -def get_git_bare_status(working_path: Path) -> GitBareStatus: - bare_status = GitBareStatus(working_path) +def get_git_bare_status( + working_path: Path, remote_name: str, remote_url: str +) -> GitBareStatus: + bare_status = GitBareStatus(working_path, remote_name, remote_url) bare_status.update() return bare_status diff --git a/tsrc/groups_to_find.py b/tsrc/groups_to_find.py index 175a7717..663d8fb6 100644 --- a/tsrc/groups_to_find.py +++ b/tsrc/groups_to_find.py @@ -16,8 +16,11 @@ class GroupsToFind: - def __init__(self, groups: Union[List[str], None]) -> None: + def __init__( + self, groups: Union[List[str], None], ignore_missing_groups: bool = False + ) -> None: self.groups = groups + self.ignore_missing_groups = ignore_missing_groups self.found_groups: List[str] = [] def found_some(self) -> bool: diff --git a/tsrc/manifest_common.py b/tsrc/manifest_common.py index bd476a33..876c53ca 100644 --- a/tsrc/manifest_common.py +++ b/tsrc/manifest_common.py @@ -161,6 +161,8 @@ def _with_groups_consider_local(self, m_group_items: List[str]) -> List[Repo]: repos: List[Repo] = [] found_items = list(set(w_group_items).intersection(m_group_items)) + found_items_wd = found_items + m_group_items + found_items = list(set(found_items_wd)) for item in found_items: repos.append(self.manifest.get_repo(item)) return repos @@ -188,18 +190,20 @@ def _without_groups(self) -> List[Repo]: # m_group_items = self.manifest.get_repos(all_=True) for repo in self.manifest.get_repos(all_=True): m_group_items.append(repo.dest) - pass else: # we need to consider all groups in such case return self.manifest.get_repos(all_=True) if self._local_m.group_list and self._local_m.group_list.groups: w_group_items = self._local_m.group_list.get_elements( - self.workspace.config.repo_groups + self.workspace.config.repo_groups, + ignore_if_group_not_found=self.gtf.ignore_missing_groups, ) else: return self._local_m.get_repos(all_=True) found_items = list(set(w_group_items).intersection(m_group_items)) + found_items_wd = found_items + m_group_items + found_items = list(set(found_items_wd)) repos: List[Repo] = [] for item in found_items: diff --git a/tsrc/repo.py b/tsrc/repo.py index f4fa8fd2..4decfc24 100644 --- a/tsrc/repo.py +++ b/tsrc/repo.py @@ -48,13 +48,15 @@ class Repo: # see 'test/cli/test_sync_to_ref.py' for even more details orig_branch: Optional[str] = None sha1: Optional[str] = None - # sha1_full: Optional[str] = None + sha1_full: Optional[str] = None # used for RepoGrabber tag: Optional[str] = None shallow: bool = False ignore_submodules: bool = False is_bare: bool = False # only used by RepoGrabber _grabbed_from_path: Optional[Path] = None + _grabbed_ahead: int = 0 # position-related data from Status + _grabbed_behind: int = 0 # position-related data from Status # only for BareCloner class _bare_clone_path: Optional[Path] = None _bare_clone_mtod: Optional[ManifestsTypeOfData] = None @@ -168,7 +170,6 @@ def _describe_to_token_output( # noqa: C901 ) res += tmp_res able += tmp_able - # TODO: any correction for 'ljust'? else: if self.sha1: res += [ui.red, f"?? {sha1}".ljust(this_ljust), ui.reset] diff --git a/tsrc/repo_grabber.py b/tsrc/repo_grabber.py index e2d4fcc3..051a29a6 100644 --- a/tsrc/repo_grabber.py +++ b/tsrc/repo_grabber.py @@ -63,8 +63,11 @@ def process(self, index: int, count: int, repo: Repo) -> Outcome: is_default_branch=False, orig_branch=gits.branch, sha1=gits.sha1, + sha1_full=gits.sha1_full, tag=gits.tag, remotes=gitr.remotes, + _grabbed_ahead=gits.ahead, + _grabbed_behind=gits.behind, ) ) diff --git a/tsrc/status_endpoint.py b/tsrc/status_endpoint.py index 050331dc..3f16a724 100644 --- a/tsrc/status_endpoint.py +++ b/tsrc/status_endpoint.py @@ -111,7 +111,9 @@ def process(self, index: int, count: int, repo: Repo) -> Outcome: return Outcome.empty() def _process_bare(self, full_path: Path, repo: Repo) -> None: - git_bare_status = get_git_bare_status(full_path) + git_bare_status = get_git_bare_status( + full_path, repo.remotes[0].name, repo.remotes[0].url + ) if repo._bare_clone_is_ok is False: git_bare_status.is_ok = False bare_status = BareStatus(git=git_bare_status) diff --git a/tsrc/status_header.py b/tsrc/status_header.py index ccb242b9..b79af09f 100644 --- a/tsrc/status_header.py +++ b/tsrc/status_header.py @@ -22,6 +22,7 @@ # import tsrc.config_status_rc from tsrc.config_status_rc import ConfigStatusReturnCode +from tsrc.errors import Error from tsrc.manifest_common_data import ManifestsTypeOfData, mtod_get_main_color from tsrc.status_header_dm import StatusHeaderDisplayMode from tsrc.workspace import Workspace @@ -53,7 +54,7 @@ def register_change( self, cfgud: ConfigUpdateData, cfguts: List[ConfigUpdateType], - ) -> None: + ) -> bool: """this function should be called only once (as only once will work)""" if not self._config_update_data: self._config_update_data = cfgud @@ -61,17 +62,23 @@ def register_change( # pre-check all provided updates to config cs = ConfigStatus(self.workspace, self.shdms) - found_some = False - ( - self._config_status_rc, - self._config_update_type, - found_some, - ) = cs.pre_check_change(cfgud, cfguts) - if found_some is True: - self.shdms += [StatusHeaderDisplayMode.CONFIG_CHANGE] - - # do not care if you want to display it, if data are OK, config will be updated - cs.proceed_to_change(cfgud, self._config_update_type) + try: + found_some = False + ( + self._config_status_rc, + self._config_update_type, + found_some, + ) = cs.pre_check_change(cfgud, cfguts) + except Error as e: + ui.error(e) + return False + else: + if found_some is True: + self.shdms += [StatusHeaderDisplayMode.CONFIG_CHANGE] + + # do not care if you want to display it, if data are OK, config will be updated + cs.proceed_to_change(cfgud, self._config_update_type) + return True def display(self) -> None: if StatusHeaderDisplayMode.CONFIG_CHANGE in self.shdms: diff --git a/tsrc/test/cli/test_dirty_after_sync.py b/tsrc/test/cli/test_dirty_after_sync.py new file mode 100644 index 00000000..67bf3512 --- /dev/null +++ b/tsrc/test/cli/test_dirty_after_sync.py @@ -0,0 +1,328 @@ +""" +special case when some Repo end up 'dirty' +right after successful sync +""" + +import os +from pathlib import Path +from typing import List + +# import pytest +import ruamel.yaml +from cli_ui.tests import MessageRecorder + +from tsrc.errors import Error +from tsrc.git import run_git +from tsrc.test.helpers.cli import CLI +from tsrc.test.helpers.git_server import GitServer +from tsrc.workspace_config import WorkspaceConfig + + +def test_dirty_after_sync__case_1( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + test unusual case when some Repo may end up with + '(dirty)' flag right after the successful 'sync' + + in this CASE_1: + we just let it dirty + """ + # 1st: Create repository + backend_path: Path = workspace_path / "main-proj-backend" + git_server.add_repo("main-proj-backend") + git_server.push_file("main-proj-backend", "CMakeLists.txt") + manifest_url = git_server.manifest_url + + # 2nd: add Manifest repository + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3rd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: introduce '.gitignore' to root + with open(backend_path / ".gitignore", "a") as gir: + gir.write(".*\n") + gir.write("data\n") + run_git(backend_path, "add", "-f", ".gitignore") + run_git(backend_path, "commit", "-m", "introducing root gitignore") + run_git(backend_path, "push") + + # 5th: ready next version + run_git(backend_path, "checkout", "-b", "dev") + # update '.gitignore' + with open(backend_path / ".gitignore", "a") as gir: + gir.write("test.txt\n") + run_git(backend_path, "add", "-f", ".gitignore") + # throw there file not under the version control + with open(backend_path / "test.txt", "a") as tf: + tf.write("dummy text") + + # "data" directory + os.mkdir(backend_path / "data") + with open(backend_path / "data" / ".gitignore", "a") as gif: + gif.write("*\n") + gif.write(".*") + new_add: Path = Path("data") / ".gitignore" + run_git(backend_path, "add", "-f", str(new_add)) + run_git(backend_path, "commit", "-m", "introducing gitignore on data") + run_git(backend_path, "push", "-u", "origin", "dev") + + # "extra-data", not in version control + os.mkdir(backend_path / "data" / "extra-data") + with open(backend_path / "data" / "extra-data" / "dummy-file.txt", "a") as duf: + duf.write("just a dummy") + + # update Manifest file, add, commit, push + manifest_path = workspace_path / "manifest" + ad_hoc_update_dm_repo_branch_only(workspace_path) + run_git(manifest_path, "checkout", "-b", "cmp-1") + run_git(manifest_path, "add", "manifest.yml") + run_git(manifest_path, "commit", "-m", "new cmp-1 branch") + run_git(manifest_path, "push", "-u", "origin", "cmp-1") + + """ + we are now ready sync forward and than back + by this, we will invoke the state when right after the + 'sync' the Repo 'main-proj-backend' will end up with 'dirty' + GIT status. + """ + + # 6th: change branch forward and back + # ready change forward (to 'cmp-1') + tsrc_cli.run("manifest", "--branch", "cmp-1") + tsrc_cli.run("sync") + + # and back to 'master' + tsrc_cli.run("manifest", "--branch", "master") + tsrc_cli.run("sync") + + # 7th: check the dirty flag + message_recorder.reset() + tsrc_cli.run("status") + assert message_recorder.find(r"\* main-proj-backend \[ master \] master \(dirty\)") + + if Path(backend_path / "data" / "extra-data" / "dummy-file.txt").is_file() is False: + raise Error("ignored file was cleaned. that is not good") + + +def test_dirty_after_sync__case_2( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + test unusual case when some Repo may end up with + '(dirty)' flag right after the successful 'sync' + + in this CASE_2: + clean 'sync' - but ignored files should stay there + """ + # 1st: Create repository + backend_path: Path = workspace_path / "main-proj-backend" + git_server.add_repo("main-proj-backend") + git_server.push_file("main-proj-backend", "CMakeLists.txt") + manifest_url = git_server.manifest_url + + # 2nd: add Manifest repository + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3rd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: introduce '.gitignore' to root + with open(backend_path / ".gitignore", "a") as gir: + gir.write(".*\n") + gir.write("data\n") + run_git(backend_path, "add", "-f", ".gitignore") + run_git(backend_path, "commit", "-m", "introducing root gitignore") + run_git(backend_path, "push") + + # 5th: ready next version + run_git(backend_path, "checkout", "-b", "dev") + # update '.gitignore' + with open(backend_path / ".gitignore", "a") as gir: + gir.write("test.txt\n") + run_git(backend_path, "add", "-f", ".gitignore") + # throw there file not under the version control + with open(backend_path / "test.txt", "a") as tf: + tf.write("dummy text") + + # "data" directory + os.mkdir(backend_path / "data") + with open(backend_path / "data" / ".gitignore", "a") as gif: + gif.write("*\n") + gif.write(".*") + new_add: Path = Path("data") / ".gitignore" + run_git(backend_path, "add", "-f", str(new_add)) + run_git(backend_path, "commit", "-m", "introducing gitignore on data") + run_git(backend_path, "push", "-u", "origin", "dev") + + # "extra-data", not in version control + os.mkdir(backend_path / "data" / "extra-data") + with open(backend_path / "data" / "extra-data" / "dummy-file.txt", "a") as duf: + duf.write("just a dummy") + + # update Manifest file, add, commit, push + manifest_path = workspace_path / "manifest" + ad_hoc_update_dm_repo_branch_only(workspace_path) + run_git(manifest_path, "checkout", "-b", "cmp-1") + run_git(manifest_path, "add", "manifest.yml") + run_git(manifest_path, "commit", "-m", "new cmp-1 branch") + run_git(manifest_path, "push", "-u", "origin", "cmp-1") + + """ + we are now ready sync forward and than back + by this, we will invoke the state when right after the + 'sync' the Repo 'main-proj-backend' will end up with 'dirty' + GIT status. + """ + + # 6th: change branch forward and back + # ready change forward (to 'cmp-1') + tsrc_cli.run("manifest", "--branch", "cmp-1") + tsrc_cli.run("sync") + + # and back to 'master' + tsrc_cli.run("manifest", "--branch", "master") + tsrc_cli.run("sync", "--clean") + + # 7th: dirty flag should not be there + message_recorder.reset() + tsrc_cli.run("status") + assert not message_recorder.find( + r"\* main-proj-backend \[ master \] master \(dirty\)" + ) + assert message_recorder.find(r"\* main-proj-backend \[ master \] master") + + if Path(backend_path / "data" / "extra-data" / "dummy-file.txt").is_file() is False: + raise Error("ignored file was cleaned. that is not good") + + +def test_dirty_after_sync__case_3( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + test unusual case when some Repo may end up with + '(dirty)' flag right after the successful 'sync' + + in this CASE_3: + clean 'sync' ALSO clean ignored files + """ + # 1st: Create repository + backend_path: Path = workspace_path / "main-proj-backend" + git_server.add_repo("main-proj-backend") + git_server.push_file("main-proj-backend", "CMakeLists.txt") + manifest_url = git_server.manifest_url + + # 2nd: add Manifest repository + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3rd: init workspace on master + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: introduce '.gitignore' to root + with open(backend_path / ".gitignore", "a") as gir: + gir.write(".*\n") + gir.write("data\n") + run_git(backend_path, "add", "-f", ".gitignore") + run_git(backend_path, "commit", "-m", "introducing root gitignore") + run_git(backend_path, "push") + + # 5th: ready next version + run_git(backend_path, "checkout", "-b", "dev") + # update '.gitignore' + with open(backend_path / ".gitignore", "a") as gir: + gir.write("test.txt\n") + run_git(backend_path, "add", "-f", ".gitignore") + # throw there file not under the version control + with open(backend_path / "test.txt", "a") as tf: + tf.write("dummy text") + + # "data" directory + os.mkdir(backend_path / "data") + with open(backend_path / "data" / ".gitignore", "a") as gif: + gif.write("*\n") + gif.write(".*") + new_add: Path = Path("data") / ".gitignore" + run_git(backend_path, "add", "-f", str(new_add)) + run_git(backend_path, "commit", "-m", "introducing gitignore on data") + run_git(backend_path, "push", "-u", "origin", "dev") + + # "extra-data", not in version control + os.mkdir(backend_path / "data" / "extra-data") + with open(backend_path / "data" / "extra-data" / "dummy-file.txt", "a") as duf: + duf.write("just a dummy") + + # update Manifest file, add, commit, push + manifest_path = workspace_path / "manifest" + ad_hoc_update_dm_repo_branch_only(workspace_path) + run_git(manifest_path, "checkout", "-b", "cmp-1") + run_git(manifest_path, "add", "manifest.yml") + run_git(manifest_path, "commit", "-m", "new cmp-1 branch") + run_git(manifest_path, "push", "-u", "origin", "cmp-1") + + """ + we are now ready sync forward and than back + by this, we will invoke the state when right after the + 'sync' the Repo 'main-proj-backend' will end up with 'dirty' + GIT status. + """ + + # 6th: change branch forward and back + # ready change forward (to 'cmp-1') + tsrc_cli.run("manifest", "--branch", "cmp-1") + tsrc_cli.run("sync") + + # and back to 'master' + tsrc_cli.run("manifest", "--branch", "master") + tsrc_cli.run("sync", "--hard-clean") + + # 7th: dirty flag should not be there + message_recorder.reset() + tsrc_cli.run("status") + assert not message_recorder.find( + r"\* main-proj-backend \[ master \] master \(dirty\)" + ) + assert message_recorder.find(r"\* main-proj-backend \[ master \] master") + + if Path(backend_path / "data" / "extra-data" / "dummy-file.txt").is_file() is True: + raise Error("not cleaned properly, ignored file is still there") + + +def ad_hoc_update_dm_repo_branch_only( + workspace_path: Path, +) -> None: + """change Repo's branch only + so we will be able to introduce change for next 'sync' + """ + manifest_path = workspace_path / "manifest" / "manifest.yml" + manifest_path.parent.mkdir(parents=True, exist_ok=True) + yaml = ruamel.yaml.YAML(typ="rt") + parsed = yaml.load(manifest_path.read_text()) + + for _, value in parsed.items(): + if isinstance(value, List): + for x in value: + if isinstance(x, ruamel.yaml.comments.CommentedMap): + if "dest" in x: + if x["dest"] == "main-proj-backend": + x["branch"] = "dev" + if x["dest"] == "manifest": + x["branch"] = "cmp-1" + # write the file down + with open(manifest_path, "w") as file: + yaml.dump(parsed, file) diff --git a/tsrc/test/cli/test_display_bare_repos.py b/tsrc/test/cli/test_display_bare_repos.py index cd972e93..e72ea1d5 100644 --- a/tsrc/test/cli/test_display_bare_repos.py +++ b/tsrc/test/cli/test_display_bare_repos.py @@ -70,7 +70,7 @@ def test_create_new_assembly_chain_by_tag_and_sha1( run_git(el, "tag", "-a", "ver_x", "-m", "version x") _, sha1_of_tag = run_git_captured(el, "rev-parse", "HEAD", check=False) run_git(el, "commit", "-m", "adding latest changes") - run_git(el, "push", "origin", "master") + run_git(el, "push", "--tags", "origin", "master") # 5th: consider reaching some consistant state, thus # start to create new assembly chain @@ -79,7 +79,7 @@ def test_create_new_assembly_chain_by_tag_and_sha1( run_git(mp, "checkout", "-b", "ac_1.0") # 6th: create Manifest with SHA1 marks while skipping Manifest Repo - tsrc_cli.run("dump-manifest", "--sha1-only", "--update", "--skip-manifest") + tsrc_cli.run("dump-manifest", "--sha1-on", "--update", "--skip-manifest") # 7th: let us update Manifest, but only with its branch (no SHA1) tsrc_cli.run("dump-manifest", "--update", "--only-manifest", "--force") @@ -116,7 +116,7 @@ def test_create_new_assembly_chain_by_tag_and_sha1( # 10th: dumping and updating Manifest tsrc_cli.run( - "dump-manifest", "--raw", ".", "--sha1-only", "--update", "--skip-manifest" + "dump-manifest", "--raw", ".", "--sha1-on", "--update", "--skip-manifest" ) # checkout new branch for Manifest in order to dump it to Manifest later @@ -159,7 +159,7 @@ def test_create_new_assembly_chain_by_tag_and_sha1( message_recorder.reset() tsrc_cli.run("status") assert message_recorder.find( - r"\* extra-lib \[ master on ver_x .1 commit \] master .1 commit" + r"\* extra-lib \[ master on ver_x .2 commits \] master .1 commit" ) @@ -257,7 +257,7 @@ def test_create_new_assembly_chain_by_sha1( run_git(mp, "checkout", "-b", "ac_1.0") # 6th: create Manifest with SHA1 marks while skipping Manifest Repo - tsrc_cli.run("dump-manifest", "--sha1-only", "--update", "--skip-manifest") + tsrc_cli.run("dump-manifest", "--sha1-on", "--update", "--skip-manifest") # 7th: let us update Manifest, but only with its branch (no SHA1) tsrc_cli.run("dump-manifest", "--update", "--only-manifest", "--force") @@ -315,7 +315,7 @@ def test_create_new_assembly_chain_by_sha1( # 10th: dumping and updating Manifest tsrc_cli.run( - "dump-manifest", "--raw", ".", "--sha1-only", "--update", "--skip-manifest" + "dump-manifest", "--raw", ".", "--sha1-on", "--update", "--skip-manifest" ) # checkout new branch for Manifest in order to dump it to Manifest later @@ -429,11 +429,11 @@ def test_create_new_assembly_chain_by_sha1( tsrc_cli.run("status", "--show-leftovers-status") if os.name == "nt": assert message_recorder.find( - r"\+ inside\\repo_inside \[ master \] \( master ~~ commit << master \) \(dirty\)" + r"\+ inside\\repo_inside \[ master \] \( master ~~ commit << master \) \(dirty\)" ) else: assert message_recorder.find( - rf"\+ inside{os.sep}repo_inside \[ master \] \( master ~~ commit << master \) \(dirty\)" + rf"\+ inside{os.sep}repo_inside \[ master \] \( master ~~ commit << master \) \(dirty\)" ) # 24th: get rid of dirty repo @@ -442,14 +442,14 @@ def test_create_new_assembly_chain_by_sha1( tsrc_cli.run("status") if os.name == "nt": assert message_recorder.find( - r"\+ inside\\repo_inside \[ master \] \( master ~~ commit == master \)" + r"\+ inside\\repo_inside \[ master \] \( master ~~ commit == master \)" ) else: assert message_recorder.find( - rf"\+ inside{os.sep}repo_inside \[ master \] \( master ~~ commit == master \)" + rf"\+ inside{os.sep}repo_inside \[ master \] \( master ~~ commit == master \)" ) assert message_recorder.find( - r"\* backend-proj \[ master \] \( master ~~ commit << master \) .1 commit" + r"\* backend-proj \[ master .1 commit \] \( master ~~ commit << master \) .1 commit" ) diff --git a/tsrc/test/cli/test_dump_manifest.py b/tsrc/test/cli/test_dump_manifest.py index dec696dc..4ffdc5dd 100644 --- a/tsrc/test/cli/test_dump_manifest.py +++ b/tsrc/test/cli/test_dump_manifest.py @@ -902,12 +902,16 @@ def test_raw_dump_update__use_workspace__without_workspace( message_recorder: MessageRecorder, ) -> None: """ - Check if Workspace gets ignored even if it is called to be used + Check if Workspace gets ignored even if it is called to be used. + Also check if '--skip-manifest-repo'|'--only-manifest-repo' throws a Warning + as without Workspace it is not possible to determine Deep Manifest Scenario: - # 1nd: create 'repo 1', GIT init, add, commit - # 2nd: try to dump manifest by RAW mode, while want to update DM + * 1nd: create 'repo 1', GIT init, add, commit + * 2nd: try to dump manifest by RAW mode, while want to update DM + * 3rd: test Warning when '--skip-manifest-repo' + * 4th: test Warning and Error when '--only-manifest-repo' """ # 1nd: create 'repo 1', GIT init, add, commit sub1_path = workspace_path @@ -932,6 +936,25 @@ def test_raw_dump_update__use_workspace__without_workspace( assert message_recorder.find(r"Error: Could not find current workspace") + # 3rd: test Warning when '--skip-manifest-repo' + # without Workspace, we cannot know which is manifest, + # thus we cannot skip it. This is just a Warning + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--raw", ".", "--skip-manifest-repo") + assert message_recorder.find( + r"Warning: Cannot skip Deep Manifest if there is no Workspace" + ) + + # 4th: test Warning and Error when '--only-manifest-repo' + # not only manifest without Workspace can be determined, + # there is no data, thus Error is also throwed + message_recorder.reset() + tsrc_cli.run("dump-manifest", "--raw", ".", "--only-manifest-repo", "--force") + assert message_recorder.find( + r"Warning: Cannot look for Deep Manifest if there is no Workspace" + ) + assert message_recorder.find(r"Error: cannot obtain data: no Repos were found") + # flake8: noqa: C901 def ad_hoc_delete_remote_from_manifest( diff --git a/tsrc/test/cli/test_dump_manifest__filter_bo_manifest.py b/tsrc/test/cli/test_dump_manifest__filter_bo_manifest.py index f116308b..3c17d717 100644 --- a/tsrc/test/cli/test_dump_manifest__filter_bo_manifest.py +++ b/tsrc/test/cli/test_dump_manifest__filter_bo_manifest.py @@ -1,12 +1,12 @@ """ Dump Manifest: filter by: -* considering only Manifest Repo ('--only-manifest') -* disregarding other then Manifest Repo ('--skip-manifest') +* considering only Manifest Repo ('--only-manifest-repo') +* disregarding other then Manifest Repo ('--skip-manifest-repo') -* test if '--only-manifest' fails when no Workspace is there -* same for '--skip-manifest' +* test if '--only-manifest-repo' fails when no Workspace is there +* same for '--skip-manifest-repo' -* test if it disregard using '--skip-manifest' and '--only-manifest' at the same time +* test if it disregard using '--skip-manifest-repo' and '--only-manifest-repo' at the same time """ from pathlib import Path @@ -23,7 +23,7 @@ """ ================================= -'--skip-manifest' section follows +'--skip-manifest-repo' section follows """ @@ -35,7 +35,7 @@ def test_skip_manifest__on_raw( ) -> None: """ Description: - Test if '--skip-manifest' is respected when + Test if '--skip-manifest-repo' is respected when on RAW dump Scenario: @@ -61,7 +61,7 @@ def test_skip_manifest__on_raw( WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") # 4th: RAW dump, while skipping manifest - tsrc_cli.run("dump-manifest", "--raw", ".", "--skip-manifest") + tsrc_cli.run("dump-manifest", "--raw", ".", "--skip-manifest-repo") # 5th: test if 'manifest' repository is not present m_path = workspace_path / "manifest.yml" @@ -79,7 +79,7 @@ def test_skip_manifest__on_workspace( ) -> None: """ Description: - Test if '--skip-manifest' is respected when + Test if '--skip-manifest-repo' is respected when dump from Workspace (no UPDATE) Scenario: @@ -106,7 +106,7 @@ def test_skip_manifest__on_workspace( # 4th: dump manifest, while skipping DM message_recorder.reset() - tsrc_cli.run("dump-manifest", "--skip-manifest") + tsrc_cli.run("dump-manifest", "--skip-manifest-repo") assert message_recorder.find(r"=> Creating NEW file 'manifest\.yml'") assert message_recorder.find(r":: Dump complete") @@ -126,7 +126,7 @@ def test_skip_manifest__on_workspace__on_update( ) -> None: """ Description: - Test if '--skip-manifest' is respected when + Test if '--skip-manifest-repo' is respected when dump from Workspace no UPDATE Scenario: @@ -157,7 +157,7 @@ def test_skip_manifest__on_workspace__on_update( ad_hoc_update_manifest_repo_dest(dm_path_file) # 5th: dump manifest, while skipping DM - tsrc_cli.run("dump-manifest", "--update", "--skip-manifest", "--force") + tsrc_cli.run("dump-manifest", "--update", "--skip-manifest-repo", "--force") # 6th: verify if Manifest's dest was not updated w_m_path = workspace_path / "manifest" / "manifest.yml" @@ -188,7 +188,7 @@ def ad_hoc_update_manifest_repo_dest( """ ================================= -'--only-manifest' section follows +'--only-manifest-repo' section follows """ @@ -201,7 +201,7 @@ def test_if_stop_on_mutually_exclusive( """ Description: Test if it stops when using mutually exclusive options: - '--skip-manifest' and '--only-manifest' at the same time + '--skip-manifest-repo' and '--only-manifest-repo' at the same time Scenario: * 1st: Create repositories @@ -226,9 +226,9 @@ def test_if_stop_on_mutually_exclusive( # 4th: test conflicting options message_recorder.reset() - tsrc_cli.run("dump-manifest", "--only-manifest", "--skip-manifest") + tsrc_cli.run("dump-manifest", "--only-manifest-repo", "--skip-manifest-repo") assert message_recorder.find( - r"Error: '--skip-manifest' and '--only-manifest' are mutually exclusive" + r"Error: '--skip-manifest-repo' and '--only-manifest-repo' are mutually exclusive" ) @@ -240,7 +240,7 @@ def test_only_manifest__on_workspace( ) -> None: """ Description: - Test if '--only-manifest' is respected when + Test if '--only-manifest-repo' is respected when dump from Workspace (no UPDATE) Scenario: @@ -267,7 +267,7 @@ def test_only_manifest__on_workspace( # 4th: dump manifest, while only considering DM message_recorder.reset() - tsrc_cli.run("dump-manifest", "--only-manifest") + tsrc_cli.run("dump-manifest", "--only-manifest-repo") assert message_recorder.find(r"=> Creating NEW file 'manifest\.yml'") assert message_recorder.find(r":: Dump complete") @@ -293,7 +293,7 @@ def test_only_manifest__on_raw( ) -> None: """ Description: - Test if '--skip-manifest' is respected when + Test if '--skip-manifest-repo' is respected when on RAW dump Scenario: @@ -319,7 +319,7 @@ def test_only_manifest__on_raw( WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") # 4th: RAW dump, while we are interrestend only in manifest's Repo - tsrc_cli.run("dump-manifest", "--raw", ".", "--only-manifest") + tsrc_cli.run("dump-manifest", "--raw", ".", "--only-manifest-repo") # 5th: test if 'manifest' is only Repo in Manifest m_path = workspace_path / "manifest.yml" @@ -344,7 +344,7 @@ def test_only_manifest__on_update( Description: 'dump-manifest': - test option '--only-manifest' when Workspace dump on UPDATE + test option '--only-manifest-repo' when Workspace dump on UPDATE Scenario: @@ -381,7 +381,7 @@ def test_only_manifest__on_update( # 6th: dump-manifest, only manifest, UPDATE existin DM message_recorder.reset() - tsrc_cli.run("dump-manifest", "--update", "--only-manifest", "--force") + tsrc_cli.run("dump-manifest", "--update", "--only-manifest-repo", "--force") assert message_recorder.find(r"=> UPDATING Deep Manifest on") assert message_recorder.find(r":: Dump complete") diff --git a/tsrc/test/cli/test_dump_manifest__groups.py b/tsrc/test/cli/test_dump_manifest__groups.py index 99122b2a..02446910 100644 --- a/tsrc/test/cli/test_dump_manifest__groups.py +++ b/tsrc/test/cli/test_dump_manifest__groups.py @@ -44,7 +44,6 @@ # from tsrc.test.helpers.message_recorder_ext import MessageRecorderExt -# @pytest.mark.last def test_dump_manifest_raw__constraints__incl_excl( tsrc_cli: CLI, git_server: GitServer, @@ -255,7 +254,7 @@ def test_dump_manifest_workspace__update_with_constraints__add_repo( Story: when using RAW dump and UPDATE and we want - to limit such update on selected Group(s), + to limit such update to only selected Group(s), then other Repos that does not match such Group(s) should be left alone (not updated). However when there is Repo in Manifest we @@ -375,7 +374,13 @@ def test_dump_manifest_workspace__update_with_constraints__add_repo( sub1_1_1_file.touch() run_git(full1_path, "add", "in_repo.txt") run_git(full1_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") - run_git(full1_path, "remote", "add", "origin", repo_4_url) + # take care about remote + sub_1_1_url = git_server.get_url(str(sub1_1_path)) + sub_1_1_url_path = Path(git_server.url_to_local_path(sub_1_1_url)) + sub_1_1_url_path.mkdir() + run_git(sub_1_1_url_path, "init", "--bare") + run_git(full1_path, "remote", "add", "origin", sub_1_1_url) + run_git(full1_path, "push", "-u", "origin", "refs/heads/master") # 12th: RAW dump when with Group 'group_1' # First, let us try how RAW dump + UPDATE + Group constraints diff --git a/tsrc/test/cli/test_dump_manifest_sha1.py b/tsrc/test/cli/test_dump_manifest_sha1.py index 8ca07a07..cd2d5698 100644 --- a/tsrc/test/cli/test_dump_manifest_sha1.py +++ b/tsrc/test/cli/test_dump_manifest_sha1.py @@ -1,14 +1,15 @@ """ test SHA1 related manifest dump -like: '--sha1-only' +like: '--sha1-on' """ import re from pathlib import Path -import pytest +# import pytest from cli_ui.tests import MessageRecorder +from tsrc.git import run_git, run_git_captured from tsrc.manifest import load_manifest_safe_mode from tsrc.manifest_common_data import ManifestsTypeOfData from tsrc.test.helpers.cli import CLI @@ -16,15 +17,186 @@ from tsrc.workspace_config import WorkspaceConfig -def test_create_from_workspace__sha1_only( +# flake8: noqa: C901 +def test_dump_when_repo_behind__sha1_on( tsrc_cli: CLI, git_server: GitServer, workspace_path: Path, message_recorder: MessageRecorder, ) -> None: """ - Create Manifest (by 'dump-manifest') - from Workspace while using '--sha1-only' option + test dump-manifest: (only) create dump when Repo is ahead/behind + also test '--sha1-off' to disable SHA1 information in the same case + + Scenario: + + # 1st: create repositories representing project + # 2nd: add there a Manifest Repo + # 3rd: init Workspace + # 4th: introduce some changes, make some Repos behind remote + # 5th: consider reaching some consistant state, thus + # 6th: create Manifest with SHA1 marks while skipping Manifest Repo + # 7th: let us update Manifest, but only with its branch (no SHA1) + # 8th: commit and push such Manifest to remote + # 9th: adding repos, making them ahead/behind + ___ here is where the real tests begins ___ + # 10th A: in 3 Repos the SHA1 has to be present + # 10th B: same but now turn off sha1 + # 11th A: same as 10th, but for RAW dump + # 11th B: same but now turn off sha1 + """ + # 1st: create repositories representing project + git_server.add_repo("frontend-proj") + git_server.push_file("frontend-proj", "frontend-proj.txt") + git_server.add_repo("backend-proj") + git_server.push_file("backend-proj", "backend-proj.txt") + git_server.add_repo("extra-lib") + git_server.push_file("extra-lib", "extra-lib.txt") + manifest_url = git_server.manifest_url + + # 2nd: add there a Manifest Repo + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + # 3rd: init Workspace + tsrc_cli.run("init", "--branch", "master", manifest_url) + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # 4th: introduce some changes, make some Repos behind remote + # simulating development process + fp = Path(workspace_path / "frontend-proj") + Path(fp / "latest-changes.txt").touch() + run_git(fp, "add", "latest-changes.txt") + run_git(fp, "commit", "-m", "adding latest changes") + run_git(fp, "push", "origin", "master") + + bp = Path(workspace_path / "backend-proj") + Path(bp / "latest-changes.txt").touch() + run_git(bp, "add", "latest-changes.txt") + run_git(bp, "commit", "-m", "adding latest changes") + run_git(bp, "push", "origin", "master") + + el = Path(workspace_path / "extra-lib") + Path(el / "latest-changes.txt").touch() + run_git(el, "add", "latest-changes.txt") + run_git(el, "commit", "-m", "adding latest changes") + run_git(el, "push", "origin", "master") + + # 5th: consider reaching some consistant state, thus + # start to create new assembly chain + # by checking out new branch on Manifest + mp = Path(workspace_path / "manifest") + run_git(mp, "checkout", "-b", "ac_1.0") + + # 6th: create Manifest with SHA1 marks while skipping Manifest Repo + tsrc_cli.run("dump-manifest", "--sha1-on", "--update", "--skip-manifest-repo") + + # 7th: let us update Manifest, but only with its branch (no SHA1) + tsrc_cli.run("dump-manifest", "--update", "--only-manifest-repo", "--force") + + # 8th: commit and push such Manifest to remote + run_git(mp, "add", "manifest.yml") + run_git(mp, "commit", "-m", "new assembly chain of version 1.0") + run_git(mp, "push", "-u", "origin", "ac_1.0") + + # 9th: adding repos, making them ahead/behind + # AHEAD + Path(fp / "new_files.txt").touch() + run_git(fp, "add", "new_files.txt") + run_git(fp, "commit", "-m", "adding new changes") + # run_git(fp, "push", "origin", "master") + + # BEHIND + Path(bp / "new_files.txt").touch() + run_git(bp, "add", "new_files.txt") + run_git(bp, "commit", "-m", "adding new changes") + run_git(bp, "push", "origin", "master") + run_git(bp, "reset", "--hard", "HEAD~1") + + # BEHIND + Path(el / "new_files.txt").touch() + run_git(el, "add", "new_files.txt") + run_git(el, "commit", "-m", "adding new changes") + run_git(el, "push", "origin", "master") + run_git(el, "reset", "--hard", "HEAD~1") + + # 10th A: in 3 Repos the SHA1 has to be present + tsrc_cli.run("dump-manifest", "--save-to", "m_1.yml") + m_file = workspace_path / "m_1.yml" + if m_file.is_file() is False: + raise Exception("Manifest file does not exists") + m = load_manifest_safe_mode(m_file, ManifestsTypeOfData.SAVED) + count: int = 0 + pattern = re.compile("^[0-9a-f]{40}$") + for repo in m.get_repos(): + if repo.sha1 and pattern.match(repo.sha1): + count += 1 + + if count != 3: + raise Exception("mismatch on Manifest SHA1 records") + + # 10th B: same but now turn off sha1 + # now ignore position by not setting SHA1 + tsrc_cli.run("dump-manifest", "--save-to", "m_1_off.yml", "--sha1-off") + m_file = workspace_path / "m_1_off.yml" + if m_file.is_file() is False: + raise Exception("Manifest file does not exists") + m = load_manifest_safe_mode(m_file, ManifestsTypeOfData.SAVED) + count = 0 + pattern = re.compile("^[0-9a-f]{40}$") + for repo in m.get_repos(): + if repo.sha1 and pattern.match(repo.sha1): + count += 1 + + if count != 0: + raise Exception("mismatch on Manifest SHA1 records") + + # 11th A: same as before but for RAW dump + tsrc_cli.run("dump-manifest", "--raw", ".", "--save-to", "m_2.yml") + m_file = workspace_path / "m_2.yml" + if m_file.is_file() is False: + raise Exception("Manifest file does not exists") + m = load_manifest_safe_mode(m_file, ManifestsTypeOfData.SAVED) + count = 0 + pattern = re.compile("^[0-9a-f]{40}$") + for repo in m.get_repos(): + if repo.sha1 and pattern.match(repo.sha1): + count += 1 + + if count != 3: + raise Exception("mismatch on Manifest SHA1 records") + + # 11th B: same but now turn off sha1 + # now ignore position by not setting SHA1 + tsrc_cli.run("dump-manifest", "--raw", ".", "--save-to", "m_2.yml") + m_file = workspace_path / "m_2.yml" + if m_file.is_file() is False: + raise Exception("Manifest file does not exists") + m = load_manifest_safe_mode(m_file, ManifestsTypeOfData.SAVED) + count = 0 + pattern = re.compile("^[0-9a-f]{40}$") + for repo in m.get_repos(): + if repo.sha1 and pattern.match(repo.sha1): + count += 1 + + if count != 3: + raise Exception("mismatch on Manifest SHA1 records") + + +def test_create_from_workspace_or_raw__sha1_on( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + (only) Create Manifest (by 'dump-manifest') + from: + * Workspace + * RAW + while using '--sha1-on' option (force SHA1 information) + + test by loading proper Manifest and counting SHA1 records """ # 1st: create bunch of repos @@ -46,9 +218,9 @@ def test_create_from_workspace__sha1_only( tsrc_cli.run("init", "--branch", "master", manifest_url) WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") - # 4th: create Manifest by '--sha1-only' + # 4th: create Manifest by '--sha1-on' # tsrc_cli.run("dump-manifest") - tsrc_cli.run("dump-manifest", "--sha1-only") + tsrc_cli.run("dump-manifest", "--sha1-on") # 5th: test by load_manifest and pattern match m_file = workspace_path / "manifest.yml" @@ -64,9 +236,26 @@ def test_create_from_workspace__sha1_only( if count != 5: raise Exception("mismatch on Manifest SHA1 records") + # 6th: also test RAW mode + tsrc_cli.run( + "dump-manifest", "--raw", ".", "--sha1-on", "--save-to", "raw_manifest.yml" + ) -@pytest.mark.last -def test_update__sha1_only( + raw_m_file = workspace_path / "raw_manifest.yml" + if raw_m_file.is_file() is False: + raise Exception("Manifest file does not exists") + raw_m = load_manifest_safe_mode(raw_m_file, ManifestsTypeOfData.SAVED) + count = 0 + pattern = re.compile("^[0-9a-f]{40}$") + for repo in raw_m.get_repos(): + if repo.sha1 and pattern.match(repo.sha1): + count += 1 + + if count != 5: + raise Exception("mismatch on RAW Manifest SHA1 records") + + +def test_update__ahead_behind__sha1_on( tsrc_cli: CLI, git_server: GitServer, workspace_path: Path, @@ -74,21 +263,36 @@ def test_update__sha1_only( ) -> None: """ Update Deep Manifest (by 'dump-manifest') - from Workspace while using '--sha1-only' option + from Workspace: + * while using '--sha1-on' option + * ahead/behind status of Repo + * same with '--sha1-off' + Note: Even if we did not change any Repo, writing SHA1 to Manifest update it still. + + Scenario: + + # 1st: create bunch of repos + # 2nd: create Manifest repo + # 3rd: init Workspace + # 4th: create Manifest by '--sha1-on' + # 5th: test by load_manifest and pattern match + # 6th: checkout changes to Manifest back + # 7th: introduce some commits to Repos + # 8th: adding repos, making them ahead/behind to remote + # 9th: exact check for proper SHA1 on UPDATE + # 10th: turn off sha1 again with UPDATE """ # 1st: create bunch of repos - git_server.add_repo("repo1-mr") - git_server.push_file("repo1-mr", "test.txt") - git_server.add_repo("repo2") - git_server.push_file("repo2", "test.txt") - git_server.add_repo("repo3") - git_server.push_file("repo3", "test.txt") - git_server.add_repo("repo4") - git_server.push_file("repo4", "test.txt") + git_server.add_repo("frontend-proj") + git_server.push_file("frontend-proj", "frontend-proj.txt") + git_server.add_repo("backend-proj") + git_server.push_file("backend-proj", "backend-proj.txt") + git_server.add_repo("extra-lib") + git_server.push_file("extra-lib", "extra-lib.txt") # 2nd: create Manifest repo git_server.add_manifest_repo("manifest") @@ -99,9 +303,8 @@ def test_update__sha1_only( tsrc_cli.run("init", "--branch", "master", manifest_url) WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") - # 4th: create Manifest by '--sha1-only' - # tsrc_cli.run("dump-manifest") - tsrc_cli.run("dump-manifest", "--update", "--sha1-only") + # 4th: create Manifest by '--sha1-on' + tsrc_cli.run("dump-manifest", "--update", "--sha1-on") # 5th: test by load_manifest and pattern match m_file = workspace_path / "manifest" / "manifest.yml" @@ -114,5 +317,94 @@ def test_update__sha1_only( if repo.sha1 and pattern.match(repo.sha1): count += 1 - if count != 5: + if count != 4: raise Exception("mismatch on Manifest SHA1 records") + + # 6th: checkout changes to Manifest back + mp = Path(workspace_path / "manifest") + run_git(mp, "checkout", ".") + + # 7th: introduce some commits to Repos + fp = Path(workspace_path / "frontend-proj") + Path(fp / "latest-changes.txt").touch() + run_git(fp, "add", "latest-changes.txt") + run_git(fp, "commit", "-m", "adding latest changes") + run_git(fp, "push", "origin", "master") + + bp = Path(workspace_path / "backend-proj") + Path(bp / "latest-changes.txt").touch() + run_git(bp, "add", "latest-changes.txt") + run_git(bp, "commit", "-m", "adding latest changes") + run_git(bp, "push", "origin", "master") + + el = Path(workspace_path / "extra-lib") + Path(el / "latest-changes.txt").touch() + run_git(el, "add", "latest-changes.txt") + run_git(el, "commit", "-m", "adding latest changes") + run_git(el, "push", "origin", "master") + + # 8th: adding repos, making them ahead/behind to remote + # AHEAD + Path(fp / "new_files.txt").touch() + run_git(fp, "add", "new_files.txt") + run_git(fp, "commit", "-m", "adding new changes") + # run_git(fp, "push", "origin", "master") + _, fp_sha1 = run_git_captured(fp, "rev-parse", "HEAD", check=False) + + # BEHIND + Path(bp / "new_files.txt").touch() + run_git(bp, "add", "new_files.txt") + run_git(bp, "commit", "-m", "adding new changes") + run_git(bp, "push", "origin", "master") + run_git(bp, "reset", "--hard", "HEAD~1") + _, bp_sha1 = run_git_captured(bp, "rev-parse", "HEAD", check=False) + + # BEHIND + Path(el / "new_files.txt").touch() + run_git(el, "add", "new_files.txt") + run_git(el, "commit", "-m", "adding new changes") + run_git(el, "push", "origin", "master") + run_git(el, "reset", "--hard", "HEAD~1") + _, el_sha1 = run_git_captured(el, "rev-parse", "HEAD", check=False) + + # 9th: exact check for proper SHA1 on UPDATE + tsrc_cli.run("dump-manifest", "--update") + m_file = workspace_path / "manifest" / "manifest.yml" + if m_file.is_file() is False: + raise Exception("Manifest file does not exists") + m = load_manifest_safe_mode(m_file, ManifestsTypeOfData.SAVED) + for repo in m.get_repos(): + if repo.dest == "frontend-proj": + if repo.sha1 != fp_sha1: + raise Exception("mismatch on Manifest SHA1 records") + elif repo.dest == "backend-proj": + if repo.sha1 != bp_sha1: + raise Exception("mismatch on Manifest SHA1 records") + elif repo.dest == "extra-lib": + if repo.sha1 != el_sha1: + raise Exception("mismatch on Manifest SHA1 records") + elif repo.dest == "manifest": + pass + else: + raise Exception("mismatch on Manifest's data") + + # 10th: turn off sha1 again with UPDATE + tsrc_cli.run("dump-manifest", "--update", "--sha1-off", "--force") + m_file = workspace_path / "manifest" / "manifest.yml" + if m_file.is_file() is False: + raise Exception("Manifest file does not exists") + m = load_manifest_safe_mode(m_file, ManifestsTypeOfData.SAVED) + for repo in m.get_repos(): + if repo.dest == "frontend-proj": + if repo.sha1: + raise Exception("mismatch on Manifest SHA1 records") + elif repo.dest == "backend-proj": + if repo.sha1: + raise Exception("mismatch on Manifest SHA1 records") + elif repo.dest == "extra-lib": + if repo.sha1: + raise Exception("mismatch on Manifest SHA1 records") + elif repo.dest == "manifest": + pass + else: + raise Exception("mismatch on Manifest's data") diff --git a/tsrc/test/cli/test_groups_extra.py b/tsrc/test/cli/test_groups_extra.py index 245e06a1..f40531c7 100644 --- a/tsrc/test/cli/test_groups_extra.py +++ b/tsrc/test/cli/test_groups_extra.py @@ -295,10 +295,8 @@ def test_intersectioned_groups( # selected groups: 'group_1','group_3','group_4' # which translated to: # 'repo_1','manifest','repo_3','repo_5' - # However: - # configuration also have 'clone_all_repos'=True - # Therefore: - # we should consider all defined repository for Deep Manifest as well + # Deep Manifest respects '--clone-all-repos', which + # translated to take all repos from Deep Manifest message_recorder.reset() tsrc_cli.run("status") assert message_recorder.find(r"\* repo_1 \[ master \] master") diff --git a/tsrc/test/cli/test_manifest.py b/tsrc/test/cli/test_manifest.py index 876b56eb..8e707e87 100644 --- a/tsrc/test/cli/test_manifest.py +++ b/tsrc/test/cli/test_manifest.py @@ -390,7 +390,7 @@ def test_manifest_cmd_branch( r"=> Such Manifest's branch: xxx was not found on remote, ignoring" ), "manifest branch change must be resistant against non-existant branch" assert message_recorder.find( - r":: Manifest's branch: devel" + r"Error: aborting Manifest branch change" ), "report that wrong value does not impact current state" # 9th: change to already present branch (should not change) diff --git a/tsrc/test/cli/test_sync__groups.py b/tsrc/test/cli/test_sync__groups.py new file mode 100644 index 00000000..abf36517 --- /dev/null +++ b/tsrc/test/cli/test_sync__groups.py @@ -0,0 +1,412 @@ +""" +test how 'sync' should work when using Groups. +Try out different scenarios +""" + +import os +import shutil +from pathlib import Path + +# import pytest +from cli_ui.tests import MessageRecorder + +from tsrc.git import run_git +from tsrc.test.helpers.cli import CLI +from tsrc.test.helpers.git_server import GitServer +from tsrc.workspace_config import WorkspaceConfig + + +def test_sync__group__fm_groups_not_in_config( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + BUG: Future Manifest is ignored completely for some reason + + configured Groups [group_3, group_4, group_5] + does not match any from Future Manifest and thus Future Manifest + block is not even displayed + + How it should be: + + If we do not provide any specific Groups, the configured Groups + (from Workspace) should be used + DM Groups + FM Groups + """ + + sub1_path = workspace_path + sub1_1_path = Path("repo_1") + os.mkdir(sub1_1_path) + os.chdir(sub1_1_path) + full_sub1_path: Path = Path(os.path.join(workspace_path, sub1_path, sub1_1_path)) + run_git(full_sub1_path, "init") + sub1_1_1_file = Path("in_repo.txt") + sub1_1_1_file.touch() + run_git(full_sub1_path, "add", "in_repo.txt") + run_git(full_sub1_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + # take care of remote + sub1_1_url = git_server.get_url(str(sub1_1_path)) + sub1_1_url_path = Path(git_server.url_to_local_path(sub1_1_url)) + sub1_1_url_path.mkdir() + run_git(sub1_1_url_path, "init", "--bare") + run_git(full_sub1_path, "remote", "add", "origin", sub1_1_url) + run_git(full_sub1_path, "push", "-u", "origin", "refs/heads/master") + + os.chdir(workspace_path) + + sub2_path = workspace_path + sub2_1_path = Path("repo_2") + os.mkdir(sub2_1_path) + os.chdir(sub2_1_path) + full_sub2_path: Path = Path(os.path.join(workspace_path, sub2_path, sub2_1_path)) + run_git(full_sub2_path, "init") + sub2_1_1_file = Path("in_repo.txt") + sub2_1_1_file.touch() + run_git(full_sub2_path, "add", "in_repo.txt") + run_git(full_sub2_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + # take care of remote + sub2_1_url = git_server.get_url(str(sub2_1_path)) + sub2_1_url_path = Path(git_server.url_to_local_path(sub2_1_url)) + sub2_1_url_path.mkdir() + run_git(sub2_1_url_path, "init", "--bare") + run_git(full_sub2_path, "remote", "add", "origin", sub2_1_url) + run_git(full_sub2_path, "push", "-u", "origin", "refs/heads/master") + + os.chdir(workspace_path) + + tsrc_cli.run("dump-manifest", "--raw", ".", "--save-to", "later_manifest.yml") + + """ + ==================================================== + now: let us create Workspace with Repos and Manifest + """ + + git_server.add_repo("repo_3") + git_server.push_file("repo_3", "repo_3_file.txt") + git_server.add_repo("repo_4") + git_server.push_file("repo_4", "repo_4_file.txt") + git_server.add_repo("repo_5") + git_server.push_file("repo_5", "repo_5_file.txt") + manifest_url = git_server.manifest_url + + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + git_server.add_group("group_3", ["manifest", "repo_3"]) + git_server.add_group("group_4", ["manifest", "repo_4"]) + git_server.add_group("group_5", ["manifest", "repo_5"]) + + # git_server.manifest.configure_group("group_3", ["manifest", "repo_3"]) + + tsrc_cli.run("init", "--branch", "master", manifest_url) + tsrc_cli.run("sync") + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # move previous repos so we can determine if they was broad again + shutil.move(workspace_path / "repo_1", workspace_path / "__repo_1") + + """ + ========== + now: introduce previous manifest + """ + + shutil.copyfile("later_manifest.yml", Path("manifest") / "manifest.yml") + manifest_path = workspace_path / "manifest" + + # write groups to manifest + with open(manifest_path / "manifest.yml", "a") as dm_file: + dm_file.write("groups:\n group_all:\n repos: [repo_1, repo_2]\n") + + run_git(manifest_path, "checkout", "-b", "dev") + run_git(manifest_path, "add", "manifest.yml") + run_git(manifest_path, "commit", "-m", "new version - dev") + run_git(manifest_path, "push", "-u", "origin", "dev") + + # set to new manifest branch + tsrc_cli.run("manifest", "--branch", "dev") + + # should display all Groups from config + DM Groups + FM Groups + message_recorder.reset() + tsrc_cli.run("status") + + assert message_recorder.find(r"=> Destination \(Future Manifest description\)") + assert message_recorder.find( + r"\* manifest \( << dev \) \(expected: master\) ~~ MANIFEST" + ) + assert message_recorder.find(r"\* repo_3 \( << master \)") + assert message_recorder.find(r"\* repo_4 \( << master \)") + assert message_recorder.find(r"\* repo_5 \( << master \)") + assert message_recorder.find(r"- repo_1 \( master << ::: \)") + assert message_recorder.find(r"\+ repo_2 \( master == master \)") + + +def test_sync__group__dm_group_found_but_no_item_intersection_with_workspace( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Test corner cases when ready to 'sync' + (manifest_branch != manifest_branch_0) + + Cases + + 1st: DM has intersectioned Group, however it items does not intersect + - in such case, the DM item will be displayed as leftover + + """ + + sub1_path = workspace_path + sub1_1_path = Path("repo_1") + os.mkdir(sub1_1_path) + os.chdir(sub1_1_path) + full_sub1_path: Path = Path(os.path.join(workspace_path, sub1_path, sub1_1_path)) + run_git(full_sub1_path, "init") + sub1_1_1_file = Path("in_repo.txt") + sub1_1_1_file.touch() + run_git(full_sub1_path, "add", "in_repo.txt") + run_git(full_sub1_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + # take care of remote + sub1_1_url = git_server.get_url(str(sub1_1_path)) + sub1_1_url_path = Path(git_server.url_to_local_path(sub1_1_url)) + sub1_1_url_path.mkdir() + run_git(sub1_1_url_path, "init", "--bare") + run_git(full_sub1_path, "remote", "add", "origin", sub1_1_url) + run_git(full_sub1_path, "push", "-u", "origin", "refs/heads/master") + + os.chdir(workspace_path) + + sub2_path = workspace_path + sub2_1_path = Path("repo_2") + os.mkdir(sub2_1_path) + os.chdir(sub2_1_path) + full_sub2_path: Path = Path(os.path.join(workspace_path, sub2_path, sub2_1_path)) + run_git(full_sub2_path, "init") + sub2_1_1_file = Path("in_repo.txt") + sub2_1_1_file.touch() + run_git(full_sub2_path, "add", "in_repo.txt") + run_git(full_sub2_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + # take care of remote + sub2_1_url = git_server.get_url(str(sub2_1_path)) + sub2_1_url_path = Path(git_server.url_to_local_path(sub2_1_url)) + sub2_1_url_path.mkdir() + run_git(sub2_1_url_path, "init", "--bare") + run_git(full_sub2_path, "remote", "add", "origin", sub2_1_url) + run_git(full_sub2_path, "push", "-u", "origin", "refs/heads/master") + + os.chdir(workspace_path) + + tsrc_cli.run("dump-manifest", "--raw", ".", "--save-to", "later_manifest.yml") + + """ + ==================================================== + now: let us create Workspace with Repos and Manifest + """ + + git_server.add_repo("repo_3") + git_server.push_file("repo_3", "repo_3_file.txt") + git_server.add_repo("repo_4") + git_server.push_file("repo_4", "repo_4_file.txt") + git_server.add_repo("repo_5") + git_server.push_file("repo_5", "repo_5_file.txt") + manifest_url = git_server.manifest_url + + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + git_server.add_group("group_3", ["manifest", "repo_3"]) + git_server.add_group("group_4", ["manifest", "repo_4"]) + git_server.add_group("group_5", ["manifest", "repo_5"]) + + # git_server.manifest.configure_group("group_3", ["manifest", "repo_3"]) + + tsrc_cli.run("init", "--branch", "master", manifest_url) + tsrc_cli.run("sync") + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # move previous repos so we can determine if they was broad again + shutil.move(workspace_path / "repo_1", workspace_path / "__repo_1") + + """ + ========== + now: introduce previous manifest + """ + + shutil.copyfile("later_manifest.yml", Path("manifest") / "manifest.yml") + manifest_path = workspace_path / "manifest" + + # write groups to manifest + with open(manifest_path / "manifest.yml", "a") as fm_file: + fm_file.write("groups:\n group_all:\n repos: [repo_1, repo_2]\n") + + run_git(manifest_path, "checkout", "-b", "dev") + run_git(manifest_path, "add", "manifest.yml") + run_git(manifest_path, "commit", "-m", "new version - dev") + run_git(manifest_path, "push", "-u", "origin", "dev") + + # set to new manifest branch + tsrc_cli.run("manifest", "--branch", "dev") + + # change Deep Manifest + with open(manifest_path / "manifest.yml", "a") as dm_file: + dm_file.write(" group_3:\n repos: [repo_1]\n") + + # HERE: DM leftover ('repo_1') will be displayed + # only DM repo is displayed that match configured Group ('repo_1') + # all FM repos is displayed due to by default config is updated + # with it also Groups is updated (unless '--no-update-config') + # according to Future Manifest's defined Groups. + message_recorder.reset() + tsrc_cli.run("status") + assert message_recorder.find( + r"=> Destination \[Deep Manifest description\] \(Future Manifest description\)" + ) + assert message_recorder.find( + r"\* manifest \( << dev \) \(dirty\) \(expected: master\) ~~ MANIFEST" + ) + assert message_recorder.find(r"\* repo_4 \( << master \)") + assert message_recorder.find(r"\* repo_5 \( << master \)") + assert message_recorder.find(r"\* repo_3 \( << master \)") + assert message_recorder.find(r"- repo_1 \[ master \] \( master << ::: \)") + assert message_recorder.find(r"\+ repo_2 \( master == master \)") + + # put changes so it can be seen in apprise block (FM comparsion) + run_git(manifest_path, "add", "manifest.yml") + run_git(manifest_path, "commit", "-m", "extra group_3") + run_git(manifest_path, "push", "-u", "origin", "dev") + + # now with requrested Groups + message_recorder.reset() + tsrc_cli.run("status", "--groups", "group_3", "group_4", "group_5") + assert message_recorder.find( + r"=> Destination \[Deep Manifest description\] \(Future Manifest description\)" + ) + assert message_recorder.find(r"\* repo_3 \( << master \)") + assert message_recorder.find(r"\* repo_5 \( << master \)") + assert message_recorder.find( + r"\* manifest \( << dev \) \(expected: master\) ~~ MANIFEST" + ) + assert message_recorder.find(r"\* repo_4 \( << master \)") + assert message_recorder.find(r"- repo_1 \[ master \] \( master << ::: \)") + + +def test_sync__group__fm_no_groups( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + when going to sync: + configured Groups, but not specified + but Deep Manifest does not have any Groups defined + """ + + sub1_path = workspace_path + sub1_1_path = Path("repo_1") + os.mkdir(sub1_1_path) + os.chdir(sub1_1_path) + full_sub1_path: Path = Path(os.path.join(workspace_path, sub1_path, sub1_1_path)) + run_git(full_sub1_path, "init") + sub1_1_1_file = Path("in_repo.txt") + sub1_1_1_file.touch() + run_git(full_sub1_path, "add", "in_repo.txt") + run_git(full_sub1_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + # take care of remote + sub1_1_url = git_server.get_url(str(sub1_1_path)) + sub1_1_url_path = Path(git_server.url_to_local_path(sub1_1_url)) + sub1_1_url_path.mkdir() + run_git(sub1_1_url_path, "init", "--bare") + run_git(full_sub1_path, "remote", "add", "origin", sub1_1_url) + run_git(full_sub1_path, "push", "-u", "origin", "refs/heads/master") + + os.chdir(workspace_path) + + sub2_path = workspace_path + sub2_1_path = Path("repo_2") + os.mkdir(sub2_1_path) + os.chdir(sub2_1_path) + full_sub2_path: Path = Path(os.path.join(workspace_path, sub2_path, sub2_1_path)) + run_git(full_sub2_path, "init") + sub2_1_1_file = Path("in_repo.txt") + sub2_1_1_file.touch() + run_git(full_sub2_path, "add", "in_repo.txt") + run_git(full_sub2_path, "commit", "in_repo.txt", "-m", "adding in_repo.txt file") + # take care of remote + sub2_1_url = git_server.get_url(str(sub2_1_path)) + sub2_1_url_path = Path(git_server.url_to_local_path(sub2_1_url)) + sub2_1_url_path.mkdir() + run_git(sub2_1_url_path, "init", "--bare") + run_git(full_sub2_path, "remote", "add", "origin", sub2_1_url) + run_git(full_sub2_path, "push", "-u", "origin", "refs/heads/master") + + os.chdir(workspace_path) + + tsrc_cli.run("dump-manifest", "--raw", ".", "--save-to", "later_manifest.yml") + + """ + ==================================================== + now: let us create Workspace with Repos and Manifest + """ + + git_server.add_repo("repo_3") + git_server.push_file("repo_3", "repo_3_file.txt") + git_server.add_repo("repo_4") + git_server.push_file("repo_4", "repo_4_file.txt") + git_server.add_repo("repo_5") + git_server.push_file("repo_5", "repo_5_file.txt") + manifest_url = git_server.manifest_url + + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + + git_server.add_group("group_3", ["manifest", "repo_3"]) + git_server.add_group("group_4", ["manifest", "repo_4"]) + git_server.add_group("group_5", ["manifest", "repo_5"]) + + # git_server.manifest.configure_group("group_3", ["manifest", "repo_3"]) + + tsrc_cli.run("init", "--branch", "master", manifest_url) + tsrc_cli.run("sync") + WorkspaceConfig.from_file(workspace_path / ".tsrc" / "config.yml") + + # move previous repos so we can determine if they was broad again + shutil.move(workspace_path / "repo_1", workspace_path / "__repo_1") + + """ + ========== + now: introduce previous manifest + """ + + shutil.copyfile("later_manifest.yml", Path("manifest") / "manifest.yml") + manifest_path = workspace_path / "manifest" + + # write groups to manifest + # with open(manifest_path / "manifest.yml", "a") as dm_file: + # dm_file.write("groups:\n group_all:\n repos: [repo_1, repo_2]\n") + + run_git(manifest_path, "checkout", "-b", "dev") + run_git(manifest_path, "add", "manifest.yml") + run_git(manifest_path, "commit", "-m", "new version - dev") + run_git(manifest_path, "push", "-u", "origin", "dev") + + # set to new manifest branch + tsrc_cli.run("manifest", "--branch", "dev") + + # status: we did not specify Groups, there are configured Groups + # but that does not hold Deep Manifest back + message_recorder.reset() + tsrc_cli.run("status") + assert message_recorder.find( + r"=> Destination \[Deep Manifest description\] \(Future Manifest description\)" + ) + assert message_recorder.find(r"\* repo_3 \( << master \)") + assert message_recorder.find( + r"\* manifest \( << dev \) \(expected: master\) ~~ MANIFEST" + ) + assert message_recorder.find(r"\* repo_4 \( << master \)") + assert message_recorder.find(r"\* repo_5 \( << master \)") + assert message_recorder.find(r"- repo_1 \[ master \] \( master << ::: \)") + assert message_recorder.find(r"\+ repo_2 \[ master \] \( master == master \)") diff --git a/tsrc/test/cli/test_sync_extended.py b/tsrc/test/cli/test_sync_extended.py index 1305193a..ac4a9a51 100644 --- a/tsrc/test/cli/test_sync_extended.py +++ b/tsrc/test/cli/test_sync_extended.py @@ -172,7 +172,7 @@ def test_sync_on_groups_intersection__case_2( tsrc_cli.run("manifest", "--branch", "dev") # ================== from here it is different ================== - # case: 2 + # 9th: case: 2 tsrc_cli.run("sync", "--no-update-config") # 10th: very by 'status' output @@ -276,13 +276,13 @@ def test_sync_on_groups_intersection__case_3_a( tsrc_cli.run("manifest", "--branch", "dev") # ================== from here it is different ================== - # 9th: sync (use specific case for given test) + # 10th: sync (use specific case for given test) # case: 3 A # here: if we do not provide '--ignore-missing-groups' # we will end up with error that 'group_2' is not found tsrc_cli.run("sync", "--ignore-missing-groups") - # 10th: very by 'status' output + # 11th: veryfy by 'status' output message_recorder.reset() tsrc_cli.run("status") assert message_recorder.find(r"\* manifest \[ master \]= master ~~ MANIFEST") @@ -304,7 +304,7 @@ def test_sync_on_groups_intersection__case_3_b( have different Groups defined, we may want to update them to config. If we do not do it, we can end up with error - even for default listing. + even for default listing (tsrc status). (only when 'clone_all_repos'=false) This is not desired and by default when @@ -382,10 +382,26 @@ def test_sync_on_groups_intersection__case_3_b( tsrc_cli.run("manifest", "--branch", "dev") # ================== from here it is different ================== - # 9th: sync (use specific case for given test) + # 10th: check if 'status' respects Groups + message_recorder.reset() + tsrc_cli.run("status") + assert message_recorder.find( + r"=> Destination \[Deep Manifest description\] \(Future Manifest description\)" + ) + assert message_recorder.find(r"\* repo_2 \( << master \)") + assert message_recorder.find(r"\* repo_6 \( << master \)") + assert message_recorder.find(r"\* repo_4 \( << master \)") + assert message_recorder.find( + r"\* manifest \[ master \]= \( master << dev \) \(expected: master\) ~~ MANIFEST" + ) + assert message_recorder.find(r"\* repo_3 \[ master \] \( master == master \)") + assert not message_recorder.find(r"\* repo_5") + assert not message_recorder.find(r"\* repo_1") + + # 11th: sync (use specific case for given test) # variant: 3 B # here: the 'group_2' is not configured, - # but with '--ignore-missing-groups' thus it will be ignored + # but with '--ignore-missing-groups' it will be ignored # from provided list, and config will be updated only with # proper Groups # (skipping 'group_5' as it was not entered in '--groups') @@ -393,7 +409,7 @@ def test_sync_on_groups_intersection__case_3_b( "sync", "--ignore-missing-groups", "--groups", "group_1", "group_2", "group_3" ) - # 10th: very by 'status' output + # 12th: veryfy by 'status' output message_recorder.reset() tsrc_cli.run("status") assert message_recorder.find(r"\* manifest \[ master \]= master ~~ MANIFEST") @@ -401,3 +417,192 @@ def test_sync_on_groups_intersection__case_3_b( assert message_recorder.find(r"\* repo_3 \[ master \] master") # exclude assert not message_recorder.find(r"\* repo_5") + + +def test_sync_on_groups_intersection__case_3_b__no_update_config( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Reason: + + Test keeping config file ('.tsrc/config.yml') + unchanged on 'sync' + + Such king of action have conquences. + Like we may no longer use 'tsrc status' just like that. + + This si extension of earlier tests + """ + + # 1st: Create bunch of repos + git_server.add_repo("repo_1") + git_server.push_file("repo_1", "my_file_in_repo_1.txt") + git_server.add_repo("repo_2") + git_server.push_file("repo_2", "my_file_in_repo_2.txt") + git_server.add_repo("repo_3") + git_server.push_file("repo_3", "my_file_in_repo_3.txt") + git_server.add_repo("repo_4") + git_server.push_file("repo_4", "my_file_in_repo_4.txt") + git_server.add_repo("repo_5") + git_server.push_file("repo_5", "my_file_in_repo_5.txt") + + # 2nd: add Manifest repo + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + manifest_url = git_server.manifest_url + + # 3rd: add bunch of Groups + git_server.add_group("group_1", ["manifest", "repo_1"]) + git_server.add_group("group_3", ["manifest", "repo_3"]) + git_server.add_group("group_5", ["manifest", "repo_5"]) + + # 4th: init Workspace with only selected Groups + tsrc_cli.run("init", manifest_url, "--groups", "group_1", "group_3", "group_5") + + # 5th: save and push current Manifest to different Git Branch (dev) + manifest_path = workspace_path / "manifest" + run_git(manifest_path, "checkout", "-B", "dev") + run_git(manifest_path, "push", "-u", "origin", "dev") + + # 6th: clean tsrc configuration + # rmtree(workspace_path / ".tsrc") + move(workspace_path / ".tsrc", workspace_path / ".old_tsrc") + + # ================== from here it is different ================== + # 7th: add some other groups + git_server.add_group("group_2", ["manifest", "repo_2"]) + git_server.add_group("group_4", ["manifest", "repo_4"]) + git_server.add_group("group_6", ["manifest", "repo_6"]) + + # 8th: init workspace with bunch of groups + tsrc_cli.run( + "init", + "--branch", + "master", + manifest_url, + "--groups", + "group_2", + "group_4", + "group_6", + "group_3", + ) + + # 9th: change branch for next sync + tsrc_cli.run("manifest", "--branch", "dev") + + # 10th: sync without update config + tsrc_cli.run( + "sync", + "--no-update-config", + "--ignore-missing-groups", + "--groups", + "group_1", + "group_2", + "group_3", + ) + # let us see what this does: + # * 'group_1' gets ignored as it is not configured in workspace configuration ('.tsrc/config.yml') # noqa E501 + # * 'group_2' gets ignored due to it is missing on Future Manifest + # * skipping 'group_5' from Future Manifest as it was not present in '--groups' + # * skipping 'group_6' from config ('.tsrc/config.yml') as it wos not present in '--groups' + # * only 'group_3' is usefull than and thus synced + + # 11th: veryfy by 'status' output + # unline before, we have to use '--ignore-missing-groups', due to config file was not updated + # and old groups are present there + message_recorder.reset() + tsrc_cli.run("status", "--ignore-missing-groups") + assert message_recorder.find(r"\* manifest \[ master \]= master ~~ MANIFEST") + assert message_recorder.find(r"\* repo_3 \[ master \] master") + # exclude + assert not message_recorder.find(r"\* repo_5") + assert not message_recorder.find(r"\* repo_1") + + +def test_sync_on_groups_intersection__case_3_b__no_update_config__0_repos( + tsrc_cli: CLI, + git_server: GitServer, + workspace_path: Path, + message_recorder: MessageRecorder, +) -> None: + """ + Reason: + + Our filters of Groups AND ignoring missing groups + may lead 'sync' to 0 Repos. + In such case further action should be stopped + + This si extension of earlier tests + """ + + # 1st: Create bunch of repos + git_server.add_repo("repo_1") + git_server.push_file("repo_1", "my_file_in_repo_1.txt") + git_server.add_repo("repo_2") + git_server.push_file("repo_2", "my_file_in_repo_2.txt") + git_server.add_repo("repo_3") + git_server.push_file("repo_3", "my_file_in_repo_3.txt") + git_server.add_repo("repo_4") + git_server.push_file("repo_4", "my_file_in_repo_4.txt") + git_server.add_repo("repo_5") + git_server.push_file("repo_5", "my_file_in_repo_5.txt") + + # 2nd: add Manifest repo + git_server.add_manifest_repo("manifest") + git_server.manifest.change_branch("master") + manifest_url = git_server.manifest_url + + # 3rd: add bunch of Groups + git_server.add_group("group_1", ["manifest", "repo_1"]) + git_server.add_group("group_3", ["manifest", "repo_3"]) + git_server.add_group("group_5", ["manifest", "repo_5"]) + + # 4th: init Workspace with only selected Groups + tsrc_cli.run("init", manifest_url, "--groups", "group_1", "group_3", "group_5") + + # 5th: save and push current Manifest to different Git Branch (dev) + manifest_path = workspace_path / "manifest" + run_git(manifest_path, "checkout", "-B", "dev") + run_git(manifest_path, "push", "-u", "origin", "dev") + + # 6th: clean tsrc configuration + # rmtree(workspace_path / ".tsrc") + move(workspace_path / ".tsrc", workspace_path / ".old_tsrc") + + # ================== from here it is different ================== + # 7th: add some other groups + git_server.add_group("group_2", ["manifest", "repo_2"]) + git_server.add_group("group_4", ["manifest", "repo_4"]) + git_server.add_group("group_6", ["manifest", "repo_6"]) + + # 8th: init workspace with bunch of groups + tsrc_cli.run( + "init", + "--branch", + "master", + manifest_url, + "--groups", + "group_2", + "group_4", + "group_6", + "group_3", + ) + + # 9th: change branch for next sync + tsrc_cli.run("manifest", "--branch", "dev") + + # 10th: sync without update config + # none of Groups are present in Future Manifest + message_recorder.reset() + tsrc_cli.run( + "sync", + "--no-update-config", + "--ignore-missing-groups", + "--groups", + "group_2", + "group_6", + ) + assert message_recorder.find(r":: Nothing to synchronize, skipping") diff --git a/tsrc/workspace.py b/tsrc/workspace.py index a2a1ecb4..ab33c486 100644 --- a/tsrc/workspace.py +++ b/tsrc/workspace.py @@ -7,6 +7,7 @@ import cli_ui as ui import ruamel.yaml +from tsrc.cleaner import Cleaner from tsrc.cloner import Cloner from tsrc.errors import Error from tsrc.executor import process_items @@ -189,6 +190,36 @@ def sync( collection.print_errors() raise SyncError + def clean( + self, *, do_clean: bool = False, do_hard_clean: bool = False, num_jobs: int = 1 + ) -> None: + """ + optional. only runs when some of the cleans are True + WARNING: this may lead to data loss of files that are not under the + version control + """ + if do_clean is True and do_hard_clean is True: + ui.warning( + "'--hard-clean' also performs '--clean', no need for extra option" + ) + + clean_mode: int = 0 + if do_hard_clean is True: + clean_mode = 2 + else: + if do_clean is True: + clean_mode = 1 + + if clean_mode > 0: + this_hard: bool = False + if clean_mode == 2: + this_hard = True + + cleaner = Cleaner(self.root_path, do_hard_clean=this_hard) + repos = self.repos + ui.info_2("Cleaning repos:") + process_items(repos, cleaner, num_jobs=num_jobs) + class SyncError(Error): pass diff --git a/tsrc/workspace_repos_summary.py b/tsrc/workspace_repos_summary.py index 625ed229..8e8767ca 100644 --- a/tsrc/workspace_repos_summary.py +++ b/tsrc/workspace_repos_summary.py @@ -40,7 +40,7 @@ def __init__( self, workspace: Workspace, gtf: GroupsToFind, - dm: Union[PCSRepo, None], + dm_pcsr: Union[PCSRepo, None], only_manifest: bool = False, manifest_marker: bool = True, future_manifest: bool = True, @@ -49,7 +49,7 @@ def __init__( ) -> None: self.workspace = workspace self.gtf = gtf - self.dm = dm # presence is also a marker + self.dm_pcsr = dm_pcsr # presence is also a marker self.is_manifest_marker = manifest_marker self.is_future_manifest = future_manifest self.use_same_future_manifest = use_same_future_manifest @@ -229,17 +229,16 @@ def summary(self) -> None: Called to print all the reasonable data of Workspace, when there are some """ - # no need to perform dry run check and calculation self.is_dry_run = False # side-quest: check Deep Manifest for root point - if self.deep_manifest and self.dm: + if self.deep_manifest and self.dm_pcsr: self.d_m_root_point = self._check_d_m_root_point( self.workspace, cast(Dict[str, StatusOrError], self.statuses), self.deep_manifest, - self.dm.dest, + self.dm_pcsr.dest, ) # alignment for 'dest' @@ -279,6 +278,9 @@ def summary(self) -> None: self.gtf, self.must_find_all_groups ) + # updating 'd_m_repos' may cause different 'max_dm_desc' + self.max_dm_desc = self._calculate_max_dm_desc() + # print main part with current workspace repositories self._core_message_print( self.deep_manifest, @@ -509,7 +511,7 @@ def _sort_based_on_d_m( ) -> "OrderedDict[str, bool]": # sort based on: bool: is there a Deep Manifest corelated repository? s_has_d_m_d: OrderedDict[str, bool] = OrderedDict() - if not self.dm: # do not sort if there is no reason + if not self.dm_pcsr: # do not sort if there is no reason s_has_d_m_d = OrderedDict(has_d_m_d) return s_has_d_m_d for key in sorted(has_d_m_d, key=has_d_m_d.__getitem__): @@ -547,10 +549,12 @@ def _prepare_for_sort_on_d_m( # for dest in o_stats.keys(): for dest in self.statuses.keys(): # following condition is only here to minimize execution - if self.only_manifest is False or (self.dm and dest == self.dm.dest): + if self.only_manifest is False or ( + self.dm_pcsr and dest == self.dm_pcsr.dest + ): # produce just [True|False] to be used as key in sorting items d_m_repo_found: bool = False - if self.dm: + if self.dm_pcsr: d_m_repo_found, _ = self._repo_matched_manifest_dest( self.workspace, deep_manifest, @@ -568,14 +572,14 @@ def _prepare_for_sort_on_d_m( def _ready_d_m_repos( self, ) -> Tuple[Union[List[Repo], None], Union[Manifest, None]]: - if self.dm: - path = self.workspace.root_path / self.dm.dest + if self.dm_pcsr: + path = self.workspace.root_path / self.dm_pcsr.dest ldm = LocalManifest(path) try: ldmm = ldm.get_manifest_safe_mode(ManifestsTypeOfData.DEEP) except LoadManifestSchemaError as lmse: ui.warning(lmse) - self.dm = None # unset Deep Manifest + self.dm_pcsr = None # unset Deep Manifest return None, None mgr = ManifestGetRepos( @@ -847,9 +851,9 @@ def _calculate_max_fm_desc( def _calculate_max_dm_desc(self) -> int: max_len: int = 0 - if self.dm and self.d_m_repos: + if self.dm_pcsr and self.d_m_repos: max_len = self._check_max_dm_desc( - self.dm, + self.dm_pcsr, self.d_m_repos, ) return max_len @@ -930,14 +934,20 @@ def _core_message_print( d_m_repo_found = False d_m_repo = None # following condition is only here to minimize execution - if self.only_manifest is False or (self.dm and dest == self.dm.dest): + if self.only_manifest is False or ( + self.dm_pcsr and dest == self.dm_pcsr.dest + ): d_m_repo_found, d_m_repo = self._repo_matched_manifest_dest( self.workspace, deep_manifest, dest, ) - if self.dm and dest != self.dm.dest and self.only_manifest is True: + if ( + self.dm_pcsr + and dest != self.dm_pcsr.dest # noqa: 503 + and self.only_manifest is True # noqa: 503 + ): continue message = [ui.green, "*", ui.reset, dest.ljust(self.max_dest)] @@ -1039,7 +1049,7 @@ def _describe_deep_manifest_column( d_m_repo_found, r_d_m_repo, dest, - self.dm, + self.dm_pcsr, self.max_dm_desc, ) return message @@ -1148,7 +1158,6 @@ def _describe_status_apprise_part_get_gd_message( git: Union[GitStatus, GitBareStatus], apprise_repo: Union[Repo, None], ) -> List[ui.Token]: - # TODO: WIP """ produce comparable list of tokens to take place in '_compare_ui_token()'. this produce reference part @@ -1166,9 +1175,6 @@ def _describe_status_apprise_part_get_gd_message( if apprise_repo and apprise_repo.sha1: gd_message += [ui.blue, f"~~ {git.sha1}", ui.reset] gd_message += git.describe_dirty() - # TODO: add also (missing remote) - # when there is missing remote, it should not be compared - # instead it should be always marked as '<<' (trasition to) return gd_message