fix: ci integration tests chopsticks

.github/workflows/check-code.yml

@@ -132,7 +132,13 @@ jobs:
     env:
       working-dir: ./integration-tests/chopsticks
       CI: true
-      PEREGRINE_WASM_OVERRIDE: ../../target/debug/wbuild/peregrine-runtime/peregrine_runtime.wasm
+      SPIRITNET_BLOCK_NUMBER: 7850499
+      HYDRATION_BLOCK_NUMBER: 5235787
+      POLKADOT_BLOCK_NUMBER: 21010819
+      ASSETHUB_BLOCK_NUMBER: 7934113
+      SPIRITNET_WASM_OVERRIDE: ../../target/wasm32-unknown-unknown/debug/wbuild/spiritnet-runtime/spiritnet_runtime.wasm
+      PEREGRINE_WASM_OVERRIDE: ../../target/wasm32-unknown-unknown/debug/wbuild/peregrine-runtime/peregrine_runtime.wasm
+
     defaults:
       run:
         working-directory: ${{ env.working-dir }}
@@ -148,6 +154,18 @@ jobs:
         with:
           tool-cache: true
 
+      - name: Set up Cargo cache
+        uses: actions/cache@v4
+        with:
+          # These paths are mounted inside the Docker container.
+          # We cannot mount the `.cargo/bin` folder since the container already contains binaries, and overriding with an empty one breaks compilation.
+          path: |
+            ~/.cargo/registry/index
+            ~/.cargo/registry/cache
+            ~/.cargo/git/db
+          key: ${{ github.job }}-${{ github.ref }}-${{ matrix.features }}-${{ hashFiles('**/Cargo.lock') }}
+          save-always: true
+
       - name: Setup environment
         uses: actions/setup-node@v4
         with:
@@ -162,23 +180,11 @@ jobs:
       - name: Check lints
         run: yarn lint
 
-      - name: Set up Cargo cache
-        uses: actions/cache@v4
-        with:
-          # Not executed in a container, so we cache the local paths as on the GH worker.
-          path: |
-            ${HOME}/.cargo/bin/
-            ${HOME}/.cargo/registry/index/
-            ${HOME}/.cargo/registry/cache/
-            ${HOME}/.cargo/git/db/
-          key: ${{ github.job }}-${{ github.ref }}-${{ hashFiles('**/Cargo.lock') }}
-          save-always: true
-
-      - name: Build Peregrine runtime
-        run: cargo build -p peregrine-runtime
+      - name: Build runtime wasms
+        run: cargo build -p peregrine-runtime -p spiritnet-runtime --no-default-features --target wasm32-unknown-unknown
 
       - name: Run Chopsticks tests
-        run: yarn test:CI
+        run: yarn test
 
   cargo-test:
     name: Run Cargo tests

.gitignore

@@ -26,3 +26,6 @@ runtimes/spiritnet/src/xcm_tests/e2e/out
 chopsticks/**/db.sqlite*
 chopsticks/**/extra.yaml
 chopsticks/**/.tmp.yaml
+**/preview
+
+integration-tests/**/db

integration-tests/chopsticks/.env-example

@@ -1,6 +1,38 @@
-HYDRADX_WS=
-HYDRADX_PORT=
-POLKADOT_WS=
-POLKADOT_PORT=
+# These env variables are mandatory for the tests to work.
+
+# The Spiritnet block number to be used in the tests
+SPIRITNET_BLOCK_NUMBER=
+# The Hydration block number to be used in the tests
+HYDRATION_BLOCK_NUMBER=
+# The Polkadot block number to be used in the tests
+POLKADOT_BLOCK_NUMBER=
+# Directory path to the new WASM blob
+SPIRITNET_WASM_OVERRIDE=
+# The AssetHub block number to be used in the tests 
+ASSETHUB_BLOCK_NUMBER=
+
+
+# Optional env variables
+HYDRATION_TEST_WS=
+HYDRATION_TEST_PORT=
+
+HYDRATION_WS=
+HYDRATION_PORT=
+
+ASSETHUB_WSS=
+ASSETHUB_MAIN_PORT=
+
+ASSETHUB_TEST_WSS=
+ASSETHUB_TEST_PORT=
+
 SPIRITNET_WS=
 SPIRITNET_PORT=
+
+PEREGRINE_WS=
+PEREGRINE_PORT=
+
+POLKADOT_WS=
+POLKADOT_PORT=
+
+PASEO_WS=
+PASEO_PORT=

integration-tests/chopsticks/README.md

@@ -0,0 +1,148 @@
+# e2e Chopsticks tests
+
+This project is a set of end-to-end tests for the KILT protocol against other parachains.
+Other functionalities such as a creation of DID can be easily added.
+
+## Getting Started
+
+These instructions will allow you to run the tests on your local machine for development and testing purposes.
+
+### Prerequisites
+
+- Node.js v20.11.0 (as specified in the [`.nvmrc`](https://github.com/nvm-sh/nvm)). With `nvm use` the right node version will be installed.
+- npm (comes with Node.js)
+
+### Installing
+
+To install the node modules call:
+
+```sh
+yarn 
+```
+
+### Running the tests
+
+In the package.json a script is provided.
+
+By calling the command below, the test will be executed:
+
+```sh
+yarn test 
+```
+
+Please make sure an appropriate env is set.
+
+### Env
+
+The project uses environment variables for configuration.
+You can find an example in the `.env-example` file. Copy this file to
+.env and fill in the appropriate values.
+Explanation for the values are in the `.env-example` file provided.
+
+## Code Style
+
+This project uses Prettier and ESLint for code formatting and linting.
+The configuration for these tools can be found in the `.prettierrc` and `.eslintrc.json` files respectively.
+
+To check your code for style issues, run:
+
+```sh
+yarn lint
+```
+
+To automatically fix style issues, run:
+
+```sh
+yarn lint:fix
+```
+
+## Adding a new test case
+
+To add a new test case, you need to insert a new object into the list of test cases.
+For example, if you want to add a new instance of `LimitedReserveTestConfiguration`, you would insert it into the `testPairsLimitedReserveTransfers` list.
+
+The tests are configuration-driven, meaning they can be easily customized for different scenarios.
+The test framework doesn't make assumptions about which parachain is sending which coin to which destination over which relay chain.
+Fundamental events such as the moving of coins or the creation of a new account should be emitted during the test.
+
+The test cases live in the tests folder and use mocks from the network directory.
+The network directory contains helper functions to set the blockchain state and provides an abstraction over the chopsticks functionalities, such as creating a network.
+
+For Adding a new test case scenario, a template folder is provided.
+
+## Debugging Existence Tests
+
+Each test case should have a unique ID. To execute a specific test case, run:
+
+```sh
+yarn test -t "REGULAR_EXPRESSION"
+```
+
+## UI
+
+Vitest supports a UI to manage the test cases. To spin up the UI, call:
+
+```sh
+yarn ui
+```
+
+## CLI
+
+The project provides a CLI to interact with the test framework.
+To execute the cli run:
+
+```sh
+yarn cli [COMMAND]
+```
+
+Below are the available commands and their descriptions:
+
+`spinUp`
+
+Spins up the network using the definition in `./src/command/network.ts`.
+The network configuration can be adjusted as needed.
+A detailed step-by-step explanation of how to modify the network to a specific state is provided in the `network.ts` file.
+
+```sh
+yarn cli spinUp
+```
+
+`scheduleTx`
+
+Executes a transaction on the network, creating a new Chopsticks instance.
+
+```sh
+yarn cli scheduleTx endpoint rawTx [options]
+```
+
+- endpoint: The endpoint of the network.
+- rawTx: The raw transaction to execute.
+- --origin: The origin of the transaction (default: Root).
+- --port: The RPC port (default: 8888).
+
+`stateTransition`
+
+Shows the state transition of the network based on the latest block.
+The command creates a preview folder containing an HTML file, which can be opened in a browser to inspect the state transition.
+
+```sh
+yarn cli stateTransition endpoint [option]
+```
+
+- endpoint: The endpoint of the network.
+- --block: The block to do the state transition
+
+
+## Recommended VS Code Extensions
+
+- [ESLint](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint): Integrates ESLint into VS Code, a tool for identifying and reporting on patterns in JavaScript.
+- [Prettier - Code formatter](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode): An opinionated code formatter that supports many languages.
+- [TypeScript Hero](https://marketplace.visualstudio.com/items?itemName=rbbit.typescript-hero): Provides additional features for TypeScript development, such as import management and code navigation.
+- [Live Server](https://marketplace.visualstudio.com/items?itemName=ritwickdey.LiveServer): Renders HTML files, useful for state transition HTML.
+- [GitLens](https://marketplace.visualstudio.com/items?itemName=eamodio.gitlens): Supercharges the built-in Git capabilities of VS Code.
+
+## Built With
+
+[TypeScript](https://www.typescriptlang.org/)
+[Chopsticks](https://github.com/AcalaNetwork/chopsticks)
+[Polkadot API](https://github.com/polkadot-js/api)

integration-tests/chopsticks/package.json

@@ -8,12 +8,13 @@
   "author": "[\"KILT <info@kilt.io>\"]",
   "license": "MIT",
   "devDependencies": {
-    "@acala-network/chopsticks": "0.12.2",
-    "@acala-network/chopsticks-testing": "0.12.2",
-    "@polkadot/api": "^10.11.2",
+    "@acala-network/chopsticks": "1.0.1",
+    "@acala-network/chopsticks-testing": "1.0.1",
+    "@polkadot/api": "^14.3.1",
     "@types/node": "^20.11.30",
     "@typescript-eslint/eslint-plugin": "^7.7.0",
     "@typescript-eslint/parser": "^7.7.0",
+    "@vitest/ui": "2.1.8",
     "eslint": "^8.0.1",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-prettier": "^9.1.0",
@@ -24,17 +25,21 @@
     "eslint-plugin-prettier": "^5.1.3",
     "eslint-plugin-promise": "^6.0.0",
     "prettier": "^3.2.5",
-    "ts-node": "^10.9.2",
-    "tsx": "^4.7.1",
+    "tsx": "^4.19.2",
     "typescript": "*",
-    "vitest": "^1.4.0"
+    "vitest": "^2.1.8"
   },
   "scripts": {
     "ts-check": "tsc --noEmit",
     "lint": "eslint src && prettier --check src",
     "lint:fix": "eslint --fix src && prettier --write src",
     "clean": "rm -rf ./db",
-    "test": "LOG_LEVEL=error vitest",
-    "test:CI": "vitest --retry 3 --no-file-parallelism"
+    "test": "LOG_LEVEL=error vitest --config ./src/vitest.config.ts",
+    "ui": "vitest --ui --config ./src/vitest.config.ts",
+    "cli": "tsx ./src/cli.ts"
+  },
+  "dependencies": {
+    "commander": "^13.1.0",
+    "execa": "^9.5.2"
   }
 }

integration-tests/chopsticks/src/cli.ts

@@ -0,0 +1,50 @@
+import { Command } from 'commander'
+
+import { createTestNetwork, scheduleTxCommand, stateTransition } from './command/index.js'
+
+const program = new Command()
+
+program.name('chopsticks utils').description('CLI to mange some chopsticks instances').version('0.0.1')
+
+program
+	.command('spinUp')
+	.description('Spin up the network')
+	.action(async () => {
+		await createTestNetwork()
+	})
+
+program
+	.command('scheduleTx')
+	.description('Executes a transaction on the network')
+	.argument('<endpoint>', 'The endpoint of the network')
+	.argument('<rawTx>', 'The raw transaction to execute')
+	.option(
+		'--origin, <origin>',
+		'The origin of the transaction Either "Root" or "Signed""',
+		(value) => {
+			if (value !== 'Root' && value !== 'Signed') {
+				throw new Error('Invalid origin. Must be either "Root" or "Signed"')
+			}
+			return value
+		},
+		'Root'
+	)
+	.option('--port <port>', 'The RPC port', '8888')
+	.action(async (endpoint, rawTx, options) => {
+		const { origin, port } = options
+
+		await scheduleTxCommand(endpoint, rawTx, origin, +port)
+	})
+
+program
+	.command('stateTransition')
+	.description('Shows the state transition of the network by the latest block')
+	.argument('<endpoint>', 'The endpoint of the network')
+	.option('--block', 'The block number to do the state transition', 'undefined')
+	.action(async (endpoint, options) => {
+		const { block } = options
+		const blockNumber = block === 'undefined' ? undefined : +block
+		await stateTransition(endpoint, blockNumber)
+	})
+
+program.parse()

integration-tests/chopsticks/src/command/index.ts

@@ -0,0 +1,5 @@
+import { createTestNetwork } from './network.js'
+import { scheduleTxCommand } from './scheduleTx.js'
+import { stateTransition } from './stateTransition.js'
+
+export { createTestNetwork, scheduleTxCommand, stateTransition }

integration-tests/chopsticks/src/command/network.ts

@@ -0,0 +1,24 @@
+import { mainChains } from '../network/index.js'
+import { setStorage, setupNetwork } from '../network/utils.js'
+
+export async function createTestNetwork() {
+	const relayConfig = mainChains.polkadot.getConfig({})
+
+	// 1. Create the network specific state
+	const kiltStorage = mainChains.kilt.storage.assignNativeTokensToAccounts([
+		'4pF5Y2Eo6doQHPLQj5AkndZwtomVB8ab2sRftRS2D9JDdELr',
+	])
+
+	// 2. Create the network specific config
+	const kiltConfig = mainChains.kilt.getConfig({})
+
+	// 2. Add the different parachainContexts here. The order of parachainContexts and parachainStorage should match.
+	const parachainOptions = [kiltConfig]
+	const parachainStorage = [kiltStorage]
+
+	const { relayChainContext, parachainContexts } = await setupNetwork(relayConfig, parachainOptions)
+
+	await Promise.all(parachainStorage.map((storage, index) => setStorage(parachainContexts[index], storage)))
+
+	await Promise.all([...parachainContexts, relayChainContext].map((context) => context.pause()))
+}