Reorganising the repository - external renames and moves (#4074)

* Adding first rough ouline of the repository structure

* Remove old CI stuff

* add title

* formatting fixes

* move node-exits job's script to scripts dir

* Move docs into subdir

* move to bin

* move maintainence scripts, configs and helpers into its own dir

* add .local to ignore

* move core->client

* start up 'test' area

* move test client

* move test runtime

* make test move compile

* Add dependencies rule enforcement.

* Fix indexing.

* Update docs to reflect latest changes

* Moving /srml->/paint

* update docs

* move client/sr-* -> primitives/

* clean old readme

* remove old broken code in rhd

* update lock

* Step 1.

* starting to untangle client

* Fix after merge.

* start splitting out client interfaces

* move children and blockchain interfaces

* Move trie and state-machine to primitives.

* Fix WASM builds.

* fixing broken imports

* more interface moves

* move backend and light to interfaces

* move CallExecutor

* move cli off client

* moving around more interfaces

* re-add consensus crates into the mix

* fix subkey path

* relieve client from executor

* starting to pull out client from grandpa

* move is_decendent_of out of client

* grandpa still depends on client directly

* lemme tests pass

* rename srml->paint

* Make it compile.

* rename interfaces->client-api

* Move keyring to primitives.

* fixup libp2p dep

* fix broken use

* allow dependency enforcement to fail

* move fork-tree

* Moving wasm-builder

* make env

* move build-script-utils

* fixup broken crate depdencies and names

* fix imports for authority discovery

* fix typo

* update cargo.lock

* fixing imports

* Fix paths and add missing crates

* re-add missing crates

Author: gnunicorn

.dockerignore

@@ -3,3 +3,4 @@ doc
 .idea/
 Dockerfile
 .dockerignore
+.local

.gitignore

@@ -21,3 +21,4 @@ rls*.log
 *.orig
 *.rej
 **/wip/*.stderr
+.local

.gitlab-ci.yml

@@ -78,7 +78,7 @@ check-runtime:
     GITLAB_API:                    "https://gitlab.parity.io/api/v4"
     GITHUB_API_PROJECT:            "parity%2Finfrastructure%2Fgithub-api"
   script:
-    - ./scripts/gitlab/check_runtime.sh
+    - ./.maintain/gitlab/check_runtime.sh
   allow_failure:                   true
 
 
@@ -89,7 +89,7 @@ check-line-width:
   only:
     - /^[0-9]+$/
   script:
-    - ./scripts/gitlab/check_line_width.sh
+    - ./.maintain/gitlab/check_line_width.sh
   allow_failure:                   true
 
 
@@ -117,7 +117,7 @@ cargo-check-subkey:
   except:
     - /^v[0-9]+\.[0-9]+.*$/        # i.e. v1.0, v2.1rc1
   script:
-    - cd ./subkey
+    - cd ./bin/subkey
     - BUILD_DUMMY_WASM_BINARY=1 time cargo check --release
     - sccache -s
 
@@ -145,8 +145,17 @@ test-linux-stable:                 &test-linux
     paths:
       - ${CI_COMMIT_SHORT_SHA}_warnings.log
 
+test-dependency-rules:             &test-linux
+  stage:                           test
+  <<:                              *docker-env
+  allow_failure:                   true
+  except:
+    variables:
+      - $DEPLOY_TAG
+  script:
+    - .maintain/ensure-deps.sh
 
-test-srml-staking:                 &test-srml-staking
+test-paint-staking:                 &test-paint-staking
   stage:                           test
   <<:                              *docker-env
   variables:
@@ -158,7 +167,7 @@ test-srml-staking:                 &test-srml-staking
     variables:
       - $DEPLOY_TAG
   script:
-    - cd srml/staking/
+    - cd paint/staking/
     - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --no-default-features --features std
     - sccache -s
 
@@ -174,7 +183,7 @@ test-wasmtime:                 &test-wasmtime
     variables:
       - $DEPLOY_TAG
   script:
-    - cd core/executor
+    - cd client/executor
     - WASM_BUILD_NO_COLOR=1 time cargo test --release --verbose --features wasmtime
     - sccache -s
 
@@ -219,7 +228,7 @@ check-web-wasm:
     - time cargo web build -p substrate-consensus-common
     - time cargo web build -p substrate-telemetry
     # Note: the command below is a bit weird because several Cargo issues prevent us from compiling the node in a more straight-forward way.
-    - time cargo build --manifest-path=node/cli/Cargo.toml --no-default-features --features "browser" --target=wasm32-unknown-unknown
+    - time cargo build --manifest-path=bin/node/cli/Cargo.toml --no-default-features --features "browser" --target=wasm32-unknown-unknown
     - sccache -s
 
 node-exits:
@@ -228,7 +237,7 @@ node-exits:
   except:
     - /^v[0-9]+\.[0-9]+.*$/        # i.e. v1.0, v2.1rc1
   script:
-    - ./ci/check_for_exit.sh
+    - ./.maintain/check_for_exit.sh
 
 
 test-full-crypto-feature:          &test-full-crypto-feature
@@ -243,7 +252,7 @@ test-full-crypto-feature:          &test-full-crypto-feature
     variables:
       - $DEPLOY_TAG
   script:
-    - cd core/primitives/
+    - cd primitives/core/
     - time cargo +nightly build --verbose --no-default-features --features full_crypto
     - cd ../application-crypto
     - time cargo +nightly build --verbose --no-default-features --features full_crypto
@@ -274,8 +283,8 @@ build-linux-substrate:
       fi
     - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256
     - printf '\n# building node-template\n\n'
-    - ./scripts/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
-    - cp -r scripts/docker/substrate.Dockerfile ./artifacts/substrate/
+    - ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
+    - cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/
     - sccache -s
 
 build-linux-subkey:
@@ -298,7 +307,7 @@ build-linux-subkey:
         sed -n -r 's/^subkey ([0-9.]+.*)/\1/p' |
           tee ./artifacts/subkey/VERSION;
     - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
-    - cp -r scripts/docker/subkey.Dockerfile ./artifacts/subkey/
+    - cp -r .maintain/docker/subkey.Dockerfile ./artifacts/subkey/
     - sccache -s
 
 build-rust-doc-release:
@@ -314,7 +323,7 @@ build-rust-doc-release:
   <<:                              *build-only
   script:
     - rm -f ./crate-docs/index.html # use it as an indicator if the job succeeds
-    - BUILD_DUMMY_WASM_BINARY=1 RUSTDOCFLAGS="--html-in-header $(pwd)/rustdoc-header.html" time cargo +nightly doc --release --all --verbose
+    - BUILD_DUMMY_WASM_BINARY=1 RUSTDOCFLAGS="--html-in-header $(pwd).maintain/rustdoc-header.html" time cargo +nightly doc --release --all --verbose
     - cp -R ./target/doc ./crate-docs
     - echo "<meta http-equiv=refresh content=0;url=substrate_service/index.html>" > ./crate-docs/index.html
     - sccache -s
@@ -533,10 +542,10 @@ publish-gh-doc:
     - echo "Substrate version = ${DEPLOY_TAG}"
     # or use helm to render the template
     - helm template
-      --values ./scripts/kubernetes/values.yaml
+      --values ./.maintain/kubernetes/values.yaml
       --set image.tag=${DEPLOY_TAG}
       --set validator.keys=${VALIDATOR_KEYS}
-      ./scripts/kubernetes | kubectl apply -f - --dry-run=false
+      ./.maintain/kubernetes | kubectl apply -f - --dry-run=false
     - echo "# substrate namespace ${KUBE_NAMESPACE}"
     - kubectl -n ${KUBE_NAMESPACE} get all
     - echo "# substrate's nodes' external ip addresses:"
@@ -598,16 +607,16 @@ deploy-ue1-tag:
 validator 1 4:
   <<:                              *validator-deploy
   script:
-    - ./scripts/flamingfir-deploy.sh flamingfir-validator1
+    - ./.maintain/flamingfir-deploy.sh flamingfir-validator1
 validator 2 4:
   <<:                              *validator-deploy
   script:
-    - ./scripts/flamingfir-deploy.sh flamingfir-validator2
+    - ./.maintain/flamingfir-deploy.sh flamingfir-validator2
 validator 3 4:
   <<:                              *validator-deploy
   script:
-    - ./scripts/flamingfir-deploy.sh flamingfir-validator3
+    - ./.maintain/flamingfir-deploy.sh flamingfir-validator3
 validator 4 4:
   <<:                              *validator-deploy
   script:
-    - ./scripts/flamingfir-deploy.sh flamingfir-validator4
+    - ./.maintain/flamingfir-deploy.sh flamingfir-validator4

Dockerfile .maintain/Dockerfile

@@ -8,7 +8,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
 	io.parity.image.title="parity/subkey" \
 	io.parity.image.description="Subkey: key generating utility for Substrate." \
-	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/docker/subkey.Dockerfile" \
+	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/.maintain/docker/subkey.Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \
 	io.parity.image.created="${BUILD_DATE}" \
 	io.parity.image.documentation="https://github.com/paritytech/substrate/tree/${VCS_REF}/subkey"

scripts/build-only-wasm.sh .maintain/build-only-wasm.sh

@@ -8,7 +8,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
 	io.parity.image.title="parity/substrate" \
 	io.parity.image.description="Substrate: The platform for blockchain innovators." \
-	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/docker/Dockerfile" \
+	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/.maintain/docker/Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \
 	io.parity.image.created="${BUILD_DATE}" \
 	io.parity.image.documentation="https://wiki.parity.io/Parity-Substrate"

ci/check_for_exit.sh .maintain/check_for_exit.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# The script is meant to check if the rules regarding packages
+# dependencies are satisfied.
+# The general format is:
+# [top-lvl-dir]<[crate-name-prefix]
+
+# For instance no crate within `./client` directory
+# is allowed to import any crate with a directory path containing `paint`.
+# Such rule is just: `client<paint`.
+
+# The script should be run from the main repo directory!
+
+set -u
+
+PLEASE_DONT=(
+	"client<paint"
+	"client<node"
+	"paint<node"
+	"paint<client"
+	"primitives<paint"
+	"primitives<client"
+)
+
+VIOLATIONS=()
+PACKAGES=()
+
+for rule in "${PLEASE_DONT[@]}"
+do
+	from=$(echo $rule | cut -f1 -d\<)
+	to=$(echo $rule | cut -f2 -d\<)
+
+	cd $from
+	echo "Checking rule $rule"
+	packages=$(find -name Cargo.toml | xargs grep -wn "path.*$to")
+	has_references=$(echo -n $packages | wc -c)
+	if [ "$has_references" != "0" ]; then
+		VIOLATIONS+=("$rule")
+		# Find packages that violate:
+		PACKAGES+=("$packages")
+	fi
+	cd - > /dev/null
+done
+
+# Display violations and fail
+I=0
+for v in "${VIOLATIONS[@]}"
+do
+	cat << EOF
+
+===========================================
+======= Violation of rule: $v
+===========================================
+${PACKAGES[$I]}
+
+
+EOF
+	I=$I+1
+done
+
+exit ${#VIOLATIONS[@]}

scripts/common.sh .maintain/common.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 #
 #
-# check for any changes in the node/src/runtime, srml/ and core/sr_* trees. if
+# check for any changes in the node/src/runtime, paint/ and core/sr_* trees. if
 # there are any changes found, it should mark the PR breaksconsensus and
 # "auto-fail" the PR if there isn't a change in the runtime/src/lib.rs file 
 # that alters the version.
@@ -32,7 +32,7 @@ github_label () {
 
 # check if the wasm sources changed
 if ! git diff --name-only origin/master...${CI_COMMIT_SHA} \
-	| grep -q -e '^node/src/runtime' -e '^srml/' -e '^core/sr-' | grep -v -e '^core/sr-arithmetic/fuzzer'
+	| grep -q -e '^node/src/runtime' -e '^paint/' -e '^core/sr-' | grep -v -e '^core/sr-arithmetic/fuzzer'
 then
 	cat <<-EOT
 	
@@ -102,7 +102,7 @@ else
 
 	source file directories:
 	- node/src/runtime
-	- srml
+	- paint
 	- core/sr-*
 
 	versions file: ${VERSIONS_FILE}

scripts/docker/subkey.Dockerfile .maintain/docker/subkey.Dockerfile

@@ -11,6 +11,6 @@ if [ "$#" -ne 1 ]; then
 fi
 
 PATH_TO_ARCHIVE=$(pwd)/$1
-cd $PROJECT_ROOT/scripts/node-template-release
+cd $PROJECT_ROOT/.maintain/node-template-release
 
 cargo run $PROJECT_ROOT/node-template $PATH_TO_ARCHIVE

scripts/docker/substrate.Dockerfile .maintain/docker/substrate.Dockerfile

@@ -4,7 +4,7 @@
 # pip install toml
 # sudo apt-get install graphviz
 # the first parameter is the runtime folder
-# python ./scripts/runtime-dep.py ./substrate/runtime | dot -Tpng -o output.png
+# python ./.maintain/runtime-dep.py ./substrate/runtime | dot -Tpng -o output.png
 import sys
 import os
 import toml

.maintain/ensure-deps.sh

@@ -12,7 +12,7 @@
 #
 # 1. Build `target/release/substrate` binary: `cargo build --release`
 #
-# 2. Start networks and containers: `sudo docker-compose -f scripts/sentry-node/docker-compose.yml up`
+# 2. Start networks and containers: `sudo docker-compose -f .maintain/sentry-node/docker-compose.yml up`
 #
 # 3. Reach:
 #   - polkadot/apps on localhost:3000