From 176d927027830001be7d2e660924fa784c98ec59 Mon Sep 17 00:00:00 2001 From: huangyi Date: Thu, 20 Jun 2024 01:08:25 +0800 Subject: [PATCH] Problem: testground image not pushed to registry Solution: - push to github container registry --- .github/workflows/container.yml | 38 +++++ flake.lock | 18 +-- flake.nix | 6 +- nix/benchmark.nix | 32 ++++ nix/testground-image.nix | 16 ++ testground/benchmark/flake.lock | 259 ------------------------------- testground/benchmark/flake.nix | 93 ----------- testground/benchmark/overlay.nix | 46 ++++++ 8 files changed, 145 insertions(+), 363 deletions(-) create mode 100644 .github/workflows/container.yml create mode 100644 nix/benchmark.nix create mode 100644 nix/testground-image.nix delete mode 100644 testground/benchmark/flake.lock delete mode 100644 testground/benchmark/flake.nix create mode 100644 testground/benchmark/overlay.nix diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml new file mode 100644 index 0000000000..fe40053d3f --- /dev/null +++ b/.github/workflows/container.yml @@ -0,0 +1,38 @@ +name: Push Testground Image + +on: + push: + branches: + - main + tags: + - "v*.*.*" + +env: + IMAGE_NAME: testground-testcase + +jobs: + + push: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v23 + with: + nix_path: nixpkgs=channel:nixos-22.11 + extra_nix_config: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + - uses: cachix/cachix-action@v12 + with: + name: cronos + signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}" + - name: build and push image + run: | + REPO_NAME=${{ github.event.repository.name }} + TAG=${{ github.ref_name }} + OCI_ARCHIVE=$(nix build --no-link --print-out-paths .#testground-image) + echo $REPO_NAME $TAG $OCI_ARCHIVE + skopeo --insecure-policy copy --dest-creds="${REPO_NAME}:${{ secrets.GITHUB_TOKEN }}" "docker-archive:${OCI_ARCHIVE}" "docker://${REPO_NAME}/testground-image:${TAG}" diff --git a/flake.lock b/flake.lock index dc81b1613a..26a3262253 100644 --- a/flake.lock +++ b/flake.lock @@ -77,11 +77,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1708341091, - "narHash": "sha256-3R7doGV1AoB5VKFifEd5elj8t4cld6VpJRpn9NaYr1Y=", + "lastModified": 1716715802, + "narHash": "sha256-usk0vE7VlxPX8jOavrtpOqphdfqEQpf9lgedlY/r66c=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "86ef6bd96b6279e1a4a53236d341f5df1ede3803", + "rev": "e2dd4e18cc1c7314e24154331bae07df76eb582f", "type": "github" }, "original": { @@ -104,11 +104,11 @@ "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1708589824, - "narHash": "sha256-2GOiFTkvs5MtVF65sC78KNVxQSmsxtk0WmV1wJ9V2ck=", + "lastModified": 1718745582, + "narHash": "sha256-TFlVP4YXg6n+MbP/Iv/RIwqvRKuV9KA1JAPihoFmPfo=", "owner": "nix-community", "repo": "poetry2nix", - "rev": "3c92540611f42d3fb2d0d084a6c694cd6544b609", + "rev": "48e7ed4ef7832efa5a5558e573986c4128fc478f", "type": "github" }, "original": { @@ -148,11 +148,11 @@ ] }, "locked": { - "lastModified": 1708335038, - "narHash": "sha256-ETLZNFBVCabo7lJrpjD6cAbnE11eDOjaQnznmg/6hAE=", + "lastModified": 1718522839, + "narHash": "sha256-ULzoKzEaBOiLRtjeY3YoGFJMwWSKRYOic6VNw2UyTls=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "e504621290a1fd896631ddbc5e9c16f4366c9f65", + "rev": "68eb1dc333ce82d0ab0c0357363ea17c31ea1f81", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 5fea04c927..e87b64f50a 100644 --- a/flake.nix +++ b/flake.nix @@ -31,13 +31,15 @@ let pkgs = import nixpkgs { inherit system; - overlays = self.overlays.default; + overlays = self.overlays.default ++ [ + (import ./testground/benchmark/overlay.nix) + ]; config = { }; }; in rec { packages = pkgs.cronos-matrix // { - inherit (pkgs) rocksdb; + inherit (pkgs) rocksdb testground-image; }; apps = { cronosd = mkApp packages.cronosd; diff --git a/nix/benchmark.nix b/nix/benchmark.nix new file mode 100644 index 0000000000..0014fb15df --- /dev/null +++ b/nix/benchmark.nix @@ -0,0 +1,32 @@ +{ lib, poetry2nix, python311, nix-gitignore }: +let + overrides = poetry2nix.overrides.withDefaults + (self: super: + let + buildSystems = { + pystarport = [ "poetry-core" ]; + durations = [ "setuptools" ]; + multitail2 = [ "setuptools" ]; + docker = [ "hatchling" "hatch-vcs" ]; + pyunormalize = [ "setuptools" ]; + }; + in + lib.mapAttrs + (attr: systems: super.${attr}.overridePythonAttrs + (old: { + nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ map (a: self.${a}) systems; + })) + buildSystems + ); +in +poetry2nix.mkPoetryApplication { + projectDir = nix-gitignore.gitignoreSourcePure [ + "/*" # ignore all, then add whitelists + "!/benchmark/" + "!poetry.lock" + "!pyproject.toml" + ] ../testground/benchmark; + python = python311; + inherit overrides; +} + diff --git a/nix/testground-image.nix b/nix/testground-image.nix new file mode 100644 index 0000000000..9c227bb016 --- /dev/null +++ b/nix/testground-image.nix @@ -0,0 +1,16 @@ +{ dockerTools, cronos-matrix, test-env, testground-testcase }: +dockerTools.buildLayeredImage { + name = "cronos-testground"; + contents = [ + testground-testcase + test-env + cronos-matrix.cronosd + ]; + config = { + Expose = [ 9090 26657 26656 1317 26658 26660 26659 30000 ]; + Cmd = [ "/bin/testground-testcase" ]; + Env = [ + "PYTHONUNBUFFERED=1" + ]; + }; +} diff --git a/testground/benchmark/flake.lock b/testground/benchmark/flake.lock deleted file mode 100644 index 22feaa71c5..0000000000 --- a/testground/benchmark/flake.lock +++ /dev/null @@ -1,259 +0,0 @@ -{ - "nodes": { - "cronos": { - "inputs": { - "flake-utils": [ - "flake-utils" - ], - "gomod2nix": "gomod2nix", - "nix-bundle-exe": "nix-bundle-exe", - "nixpkgs": [ - "nixpkgs" - ], - "poetry2nix": "poetry2nix" - }, - "locked": { - "lastModified": 1716448317, - "narHash": "sha256-MY2S2YyKWvyj2z086VaFcrWpjjMzb7R10Rprq3zeE+o=", - "owner": "crypto-org-chain", - "repo": "cronos", - "rev": "b2a47a2bdf95dfe8b5d81428f7024ef36b81f32e", - "type": "github" - }, - "original": { - "owner": "crypto-org-chain", - "ref": "main", - "repo": "cronos", - "type": "github" - } - }, - "flake-utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "flake-utils_2": { - "inputs": { - "systems": "systems_3" - }, - "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "gomod2nix": { - "inputs": { - "flake-utils": [ - "cronos", - "flake-utils" - ], - "nixpkgs": [ - "cronos", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1705314449, - "narHash": "sha256-yfQQ67dLejP0FLK76LKHbkzcQqNIrux6MFe32MMFGNQ=", - "owner": "nix-community", - "repo": "gomod2nix", - "rev": "30e3c3a9ec4ac8453282ca7f67fca9e1da12c3e6", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "gomod2nix", - "type": "github" - } - }, - "nix-bundle-exe": { - "flake": false, - "locked": { - "lastModified": 1660176694, - "narHash": "sha256-cJGZ/3CjVkoyk1W9mFVs6P/5LbJ8C+42chGYiB/wB/A=", - "owner": "3noch", - "repo": "nix-bundle-exe", - "rev": "91416cec283a33ae3448aacdc5cabdece9c08793", - "type": "github" - }, - "original": { - "owner": "3noch", - "repo": "nix-bundle-exe", - "type": "github" - } - }, - "nix-github-actions": { - "inputs": { - "nixpkgs": [ - "cronos", - "poetry2nix", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1703863825, - "narHash": "sha256-rXwqjtwiGKJheXB43ybM8NwWB8rO2dSRrEqes0S7F5Y=", - "owner": "nix-community", - "repo": "nix-github-actions", - "rev": "5163432afc817cf8bd1f031418d1869e4c9d5547", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nix-github-actions", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1716924492, - "narHash": "sha256-9/Ro5/MfI+PNMF8jzh7+gXDPUHeOzL1e/iw3p4z6Ttc=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "4ae13643e7f2cd4bc6555fce074865d9d14e7c24", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-unstable-small", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_2": { - "locked": { - "lastModified": 1716715802, - "narHash": "sha256-usk0vE7VlxPX8jOavrtpOqphdfqEQpf9lgedlY/r66c=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "e2dd4e18cc1c7314e24154331bae07df76eb582f", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "poetry2nix": { - "inputs": { - "flake-utils": "flake-utils", - "nix-github-actions": "nix-github-actions", - "nixpkgs": "nixpkgs", - "systems": "systems_2", - "treefmt-nix": "treefmt-nix" - }, - "locked": { - "lastModified": 1716986791, - "narHash": "sha256-T9Lra3MFVrto23zj2dDZ6bgxz/9gp/wh2x7QhFkpWEs=", - "owner": "nix-community", - "repo": "poetry2nix", - "rev": "1f32bb76c8e2502d896d38dc08433ac8d1e27468", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "poetry2nix", - "type": "github" - } - }, - "root": { - "inputs": { - "cronos": "cronos", - "flake-utils": "flake-utils_2", - "nixpkgs": "nixpkgs_2" - } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - }, - "systems_2": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "id": "systems", - "type": "indirect" - } - }, - "systems_3": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - }, - "treefmt-nix": { - "inputs": { - "nixpkgs": [ - "cronos", - "poetry2nix", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1708335038, - "narHash": "sha256-ETLZNFBVCabo7lJrpjD6cAbnE11eDOjaQnznmg/6hAE=", - "owner": "numtide", - "repo": "treefmt-nix", - "rev": "e504621290a1fd896631ddbc5e9c16f4366c9f65", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "treefmt-nix", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/testground/benchmark/flake.nix b/testground/benchmark/flake.nix deleted file mode 100644 index c75f5a1496..0000000000 --- a/testground/benchmark/flake.nix +++ /dev/null @@ -1,93 +0,0 @@ -{ - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; - flake-utils.url = "github:numtide/flake-utils"; - cronos = { - url = "github:crypto-org-chain/cronos/main"; - inputs.nixpkgs.follows = "nixpkgs"; - inputs.flake-utils.follows = "flake-utils"; - inputs.poetry2nix.url = "github:nix-community/poetry2nix"; - }; - }; - - outputs = { self, nixpkgs, flake-utils, cronos }: - let - overrides = { lib, poetry2nix }: poetry2nix.overrides.withDefaults - (self: super: - let - buildSystems = { - pystarport = [ "poetry-core" ]; - durations = [ "setuptools" ]; - multitail2 = [ "setuptools" ]; - docker = [ "hatchling" "hatch-vcs" ]; - pyunormalize = [ "setuptools" ]; - }; - in - lib.mapAttrs - (attr: systems: super.${attr}.overridePythonAttrs - (old: { - nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ map (a: self.${a}) systems; - })) - buildSystems - ); - - src = nix-gitignore: nix-gitignore.gitignoreSourcePure [ - "/*" # ignore all, then add whitelists - "!/benchmark/" - "!poetry.lock" - "!pyproject.toml" - ] ./.; - - benchmark = { lib, poetry2nix, python311, nix-gitignore }: poetry2nix.mkPoetryApplication { - projectDir = src nix-gitignore; - python = python311; - overrides = overrides { inherit lib poetry2nix; }; - }; - - benchmark-env = { lib, poetry2nix, python311, nix-gitignore }: poetry2nix.mkPoetryEnv { - projectDir = src nix-gitignore; - python = python311; - overrides = overrides { inherit lib poetry2nix; }; - }; - - image = { dockerTools, cronos-matrix, test-env, benchmark }: - dockerTools.buildLayeredImage { - name = "cronos-testground"; - contents = [ - benchmark - test-env - cronos-matrix.cronosd - ]; - config = { - Expose = [ 9090 26657 26656 1317 26658 26660 26659 30000 ]; - Cmd = [ "/bin/benchmark" ]; - Env = [ - "PYTHONUNBUFFERED=1" - ]; - }; - }; - in - (flake-utils.lib.eachDefaultSystem - (system: - let - pkgs = import nixpkgs { - inherit system; - overlays = cronos.overlays.default ++ [ - (final: _: { - benchmark = final.callPackage benchmark { }; - benchmark-env = final.callPackage benchmark-env { }; - benchmark-image = final.callPackage image { }; - }) - ]; - config = { }; - }; - in - rec { - packages.default = pkgs.benchmark-image; - devShells.default = pkgs.mkShell { - buildInputs = [ pkgs.benchmark-env ]; - }; - legacyPackages = pkgs; - }) - ); -} diff --git a/testground/benchmark/overlay.nix b/testground/benchmark/overlay.nix new file mode 100644 index 0000000000..ccacdd939c --- /dev/null +++ b/testground/benchmark/overlay.nix @@ -0,0 +1,46 @@ +final: _: +let + overrides = { lib, poetry2nix }: poetry2nix.overrides.withDefaults + (self: super: + let + buildSystems = { + pystarport = [ "poetry-core" ]; + durations = [ "setuptools" ]; + multitail2 = [ "setuptools" ]; + docker = [ "hatchling" "hatch-vcs" ]; + pyunormalize = [ "setuptools" ]; + }; + in + lib.mapAttrs + (attr: systems: super.${attr}.overridePythonAttrs + (old: { + nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ map (a: self.${a}) systems; + })) + buildSystems + ); + + src = nix-gitignore: nix-gitignore.gitignoreSourcePure [ + "/*" # ignore all, then add whitelists + "!/benchmark/" + "!poetry.lock" + "!pyproject.toml" + ] ./.; + + benchmark = { lib, poetry2nix, python311, nix-gitignore }: poetry2nix.mkPoetryApplication { + projectDir = src nix-gitignore; + python = python311; + overrides = overrides { inherit lib poetry2nix; }; + }; + + benchmark-env = { lib, poetry2nix, python311, nix-gitignore }: poetry2nix.mkPoetryEnv { + projectDir = src nix-gitignore; + python = python311; + overrides = overrides { inherit lib poetry2nix; }; + }; + +in +{ + testground-testcase = final.callPackage benchmark { }; + testground-image = final.callPackage ../../nix/testground-image.nix { }; + testground-testcase-env = final.callPackage benchmark-env { }; +}