From 2ce6a39b52d09ce86350fc4dda0d6ff3ddece912 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Wed, 19 Feb 2025 15:50:32 -0800 Subject: [PATCH 01/16] testers.testBuildFailure': init --- doc/build-helpers/testers.chapter.md | 61 +++++++++++ doc/redirects.json | 12 +++ pkgs/build-support/testers/default.nix | 66 ++++++++++++ pkgs/build-support/testers/test/default.nix | 114 ++++++++++++++++++++ 4 files changed, 253 insertions(+) diff --git a/doc/build-helpers/testers.chapter.md b/doc/build-helpers/testers.chapter.md index e0eb0cd1a5de6..81e13043e896e 100644 --- a/doc/build-helpers/testers.chapter.md +++ b/doc/build-helpers/testers.chapter.md @@ -255,6 +255,67 @@ runCommand "example" { ::: +## `testBuildFailure'` {#tester-testBuildFailurePrime} + +This tester wraps the functionality provided by [`testers.testBuildFailure`](#tester-testBuildFailure) to make writing checks easier by simplifying checking the exit code of the builder and asserting the existence of entries in the builder's log. +Additionally, users may specify a script containing additional checks, accessing the result of applying `testers.testBuildFailure` through the variable `failed`. + +:::{.example #ex-testBuildFailurePrime-doc-example} + +# Check that a build fails, and verify the changes made during build + +```nix +testers.testBuildFailure' { + drv = runCommand "doc-example" { } '' + echo ok-ish >"$out" + echo failing though + exit 3 + ''; + expectedBuilderExitCode = 3; + expectedBuilderLogEntries = [ "failing though" ]; + script = '' + grep --silent -F 'ok-ish' "$failed/result" + touch "$out" + ''; +} +``` + +::: + +### Inputs {#tester-testBuildFailurePrime-inputs} + +`drv` (derivation) + +: The failing derivation to wrap with `testBuildFailure`. + +`name` (string, optional) + +: The name of the test. + When not provided, this value defaults to `testBuildFailure-${(testers.testBuildFailure drv).name}`. + +`expectedBuilderExitCode` (integer, optional) + +: The expected exit code of the builder of `drv`. + When not provided, this value defaults to `1`. + +`expectedBuilderLogEntries` (array of string-like values, optional) + +: A list of string-like values which must be found in the builder's log by exact match. + When not provided, this value defaults to `[ ]`. + + NOTE: Patterns and regular expressions are not supported. + +`script` (string, optional) + +: A string containing additional checks to run. + When not provided, this value defaults to `""`. + The result of `testers.testBuildFailure drv` is available through the variable `failed`. + As an example, the builder's log is at `"$failed/testBuildFailure.log"`. + +### Return value {#tester-testBuildFailurePrime-return} + +The tester produces an empty output and only succeeds when the checks using `expectedBuilderExitCode`, `expectedBuilderLogEntries`, and `script` succeed. + ## `testEqualContents` {#tester-testEqualContents} Check that two paths have the same contents. diff --git a/doc/redirects.json b/doc/redirects.json index ddf3626bbd340..fd3388f3b3b42 100644 --- a/doc/redirects.json +++ b/doc/redirects.json @@ -8,6 +8,9 @@ "ex-build-helpers-extendMkDerivation": [ "index.html#ex-build-helpers-extendMkDerivation" ], + "ex-testBuildFailurePrime-doc-example": [ + "index.html#ex-testBuildFailurePrime-doc-example" + ], "neovim": [ "index.html#neovim" ], @@ -332,6 +335,15 @@ "footnote-stdenv-find-inputs-location.__back.0": [ "index.html#footnote-stdenv-find-inputs-location.__back.0" ], + "tester-testBuildFailurePrime": [ + "index.html#tester-testBuildFailurePrime" + ], + "tester-testBuildFailurePrime-inputs": [ + "index.html#tester-testBuildFailurePrime-inputs" + ], + "tester-testBuildFailurePrime-return": [ + "index.html#tester-testBuildFailurePrime-return" + ], "variables-specifying-dependencies": [ "index.html#variables-specifying-dependencies" ], diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index 0393895d53aff..ec3ddae2e29a8 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -34,6 +34,72 @@ ] ++ orig.args or ["-e" ../../stdenv/generic/source-stdenv.sh (orig.builder or ../../stdenv/generic/default-builder.sh)]; }); + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime + # or doc/build-helpers/testers.chapter.md + testBuildFailure' = + let + mkBuildCommand = + script: + '' + if [[ -n ''${expectedBuilderExitCode:-} ]]; then + nixLog "checking original builder exit code" + builderExitCode=$(<"$failed/testBuildFailure.exit") + if ((expectedBuilderExitCode == builderExitCode)); then + nixLog "original builder exit code matches expected value of $expectedBuilderExitCode" + else + nixErrorLog "original builder produced exit code $builderExitCode but was expected to produce $expectedBuilderExitCode" + exit 1 + fi + unset builderExitCode + fi + + if ((''${#expectedBuilderLogEntries[@]})); then + nixLog "checking original builder log" + builderLogEntries="$(<"$failed/testBuildFailure.log")" + shouldExit=0 + for expectedBuilderLogEntry in "''${expectedBuilderLogEntries[@]}"; do + if [[ ''${builderLogEntries} == *"$expectedBuilderLogEntry"* ]]; then + nixLog "original builder log contains ''${expectedBuilderLogEntry@Q}" + else + nixErrorLog "original builder log does not contain ''${expectedBuilderLogEntry@Q}" + shouldExit=1 + fi + done + unset builderLogEntries + ((shouldExit)) && exit 1 + unset shouldExit + fi + '' + + lib.optionalString (script != "") '' + nixLog "running additional checks from user-provided script" + ${script} + '' + + '' + touch "$out" + ''; + final = + { + drv, + name ? null, + expectedBuilderExitCode ? 1, # NOTE: Should be an integer. + expectedBuilderLogEntries ? [ ], # NOTE: Should be an array of string-coercible values. TODO: Only checks for inclusion, not order! + script ? "", # Succeed by default if checks pass. + }: + (runCommand name { + __structuredAttrs = true; + strictDeps = true; + failed = testers.testBuildFailure drv; + inherit expectedBuilderExitCode expectedBuilderLogEntries; + } (mkBuildCommand script)).overrideAttrs + ( + finalAttrs: _: { + # Fix name so the default value uses whatever failed ends up as. + name = if name != null then name else "testBuildFailure-${finalAttrs.failed.name}"; + } + ); + in + lib.makeOverridable final; + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualDerivation # or doc/build-helpers/testers.chapter.md testEqualDerivation = callPackage ./test-equal-derivation.nix { }; diff --git a/pkgs/build-support/testers/test/default.nix b/pkgs/build-support/testers/test/default.nix index b0f2b4c1d391a..ac05f95f54a98 100644 --- a/pkgs/build-support/testers/test/default.nix +++ b/pkgs/build-support/testers/test/default.nix @@ -220,6 +220,120 @@ lib.recurseIntoAttrs { sideEffectStructuredAttrs = overrideStructuredAttrs true sideEffects; }; + testBuildFailure' = lib.recurseIntoAttrs rec { + # NOTE: This example is used in the docs. + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime + # or doc/build-helpers/testers.chapter.md + doc-example = testers.testBuildFailure' { + drv = runCommand "doc-example" { } '' + echo ok-ish >"$out" + echo failing though + exit 3 + ''; + expectedBuilderExitCode = 3; + expectedBuilderLogEntries = [ "failing though" ]; + script = '' + grep --silent -F 'ok-ish' "$failed/result" + touch "$out" + ''; + }; + + happy = testers.testBuildFailure' { + drv = runCommand "happy" { } '' + echo ok-ish >$out + + echo failing though + echo also stderr 1>&2 + echo 'line\nwith-\bbackslashes' + printf "incomplete line - no newline" + + exit 3 + ''; + expectedBuilderExitCode = 3; + expectedBuilderLogEntries = [ + "failing though" + "also stderr" + ''line\nwith-\bbackslashes'' + "incomplete line - no newline" + ]; + script = '' + grep --silent -F 'ok-ish' "$failed/result" + touch "$out" + ''; + }; + + happyStructuredAttrs = overrideStructuredAttrs true happy; + + helloDoesNotFail = testers.testBuildFailure' { + drv = testers.testBuildFailure hello; + expectedBuilderLogEntries = [ + "testBuildFailure: The builder did not fail, but a failure was expected" + ]; + }; + + multiOutput = testers.testBuildFailure' { + drv = + runCommand "multiOutput" + { + # dev will be the default output + outputs = [ + "dev" + "doc" + "out" + ]; + } + '' + echo i am failing + exit 1 + ''; + expectedBuilderLogEntries = [ + "i am failing" + ]; + script = '' + # Checking our note that dev is the default output + echo $failed/_ | grep -- '-dev/_' >/dev/null + echo 'All good.' + touch $out + ''; + }; + + multiOutputStructuredAttrs = overrideStructuredAttrs true multiOutput; + + sideEffects = testers.testBuildFailure' { + drv = stdenvNoCC.mkDerivation { + name = "fail-with-side-effects"; + src = emptyDirectory; + + postHook = '' + echo touching side-effect... + # Assert that the side-effect doesn't exist yet... + # We're checking that this hook isn't run by expect-failure.sh + if [[ -e side-effect ]]; then + echo "side-effect already exists" + exit 1 + fi + touch side-effect + ''; + + buildPhase = '' + echo i am failing + exit 1 + ''; + }; + expectedBuilderLogEntries = [ + "touching side-effect..." + "i am failing" + ]; + script = '' + [[ ! -e side-effect ]] + + touch $out + ''; + }; + + sideEffectStructuredAttrs = overrideStructuredAttrs true sideEffects; + }; + testEqualContents = lib.recurseIntoAttrs { equalDir = testers.testEqualContents { assertion = "The same directory contents at different paths are recognized as equal"; From a01f55037d3c42f5e48787f06687e7a91a347251 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Thu, 20 Feb 2025 10:19:33 -0800 Subject: [PATCH 02/16] testers.testBuildFailure': output is created so long as checks succeed --- doc/build-helpers/testers.chapter.md | 5 ++++- pkgs/build-support/testers/test/default.nix | 5 ----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/doc/build-helpers/testers.chapter.md b/doc/build-helpers/testers.chapter.md index 81e13043e896e..74e70162ed3d5 100644 --- a/doc/build-helpers/testers.chapter.md +++ b/doc/build-helpers/testers.chapter.md @@ -260,10 +260,14 @@ runCommand "example" { This tester wraps the functionality provided by [`testers.testBuildFailure`](#tester-testBuildFailure) to make writing checks easier by simplifying checking the exit code of the builder and asserting the existence of entries in the builder's log. Additionally, users may specify a script containing additional checks, accessing the result of applying `testers.testBuildFailure` through the variable `failed`. +NOTE: This tester will produce an empty output and exit with success if none of the checks fail; there is no need to `touch "$out"` in `script`. + :::{.example #ex-testBuildFailurePrime-doc-example} # Check that a build fails, and verify the changes made during build +Re-using the example from [`testers.testBuildFailure`](#ex-testBuildFailure-showingenvironmentchanges), we can see how common checks are made easier and remove the need for `runCommand`: + ```nix testers.testBuildFailure' { drv = runCommand "doc-example" { } '' @@ -275,7 +279,6 @@ testers.testBuildFailure' { expectedBuilderLogEntries = [ "failing though" ]; script = '' grep --silent -F 'ok-ish' "$failed/result" - touch "$out" ''; } ``` diff --git a/pkgs/build-support/testers/test/default.nix b/pkgs/build-support/testers/test/default.nix index ac05f95f54a98..a7187a7dbfbf8 100644 --- a/pkgs/build-support/testers/test/default.nix +++ b/pkgs/build-support/testers/test/default.nix @@ -234,7 +234,6 @@ lib.recurseIntoAttrs { expectedBuilderLogEntries = [ "failing though" ]; script = '' grep --silent -F 'ok-ish' "$failed/result" - touch "$out" ''; }; @@ -258,7 +257,6 @@ lib.recurseIntoAttrs { ]; script = '' grep --silent -F 'ok-ish' "$failed/result" - touch "$out" ''; }; @@ -293,7 +291,6 @@ lib.recurseIntoAttrs { # Checking our note that dev is the default output echo $failed/_ | grep -- '-dev/_' >/dev/null echo 'All good.' - touch $out ''; }; @@ -326,8 +323,6 @@ lib.recurseIntoAttrs { ]; script = '' [[ ! -e side-effect ]] - - touch $out ''; }; From 5d541923d28301a568d97812dbfda9a0f2cbdbf5 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Thu, 20 Feb 2025 10:19:52 -0800 Subject: [PATCH 03/16] testers.testBuildFailure': use more strict bash for checks --- pkgs/build-support/testers/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index ec3ddae2e29a8..83c85c3a9f7d7 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -41,6 +41,8 @@ mkBuildCommand = script: '' + set -euo pipefail + if [[ -n ''${expectedBuilderExitCode:-} ]]; then nixLog "checking original builder exit code" builderExitCode=$(<"$failed/testBuildFailure.exit") From efa7ac1648cad38ca250513a8bcedd73914827e5 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Thu, 20 Feb 2025 23:16:16 +0000 Subject: [PATCH 04/16] testers.testBuildFailure': move to separate directory and use buildCommandPath --- pkgs/build-support/testers/default.nix | 67 +--------- pkgs/build-support/testers/test/default.nix | 111 +--------------- .../testBuildFailurePrime/build-command.sh | 40 ++++++ .../testers/testBuildFailurePrime/tester.nix | 47 +++++++ .../testers/testBuildFailurePrime/tests.nix | 119 ++++++++++++++++++ 5 files changed, 211 insertions(+), 173 deletions(-) create mode 100644 pkgs/build-support/testers/testBuildFailurePrime/build-command.sh create mode 100644 pkgs/build-support/testers/testBuildFailurePrime/tester.nix create mode 100644 pkgs/build-support/testers/testBuildFailurePrime/tests.nix diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index 83c85c3a9f7d7..0fcbeff9f0d0f 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -36,71 +36,8 @@ # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime # or doc/build-helpers/testers.chapter.md - testBuildFailure' = - let - mkBuildCommand = - script: - '' - set -euo pipefail - - if [[ -n ''${expectedBuilderExitCode:-} ]]; then - nixLog "checking original builder exit code" - builderExitCode=$(<"$failed/testBuildFailure.exit") - if ((expectedBuilderExitCode == builderExitCode)); then - nixLog "original builder exit code matches expected value of $expectedBuilderExitCode" - else - nixErrorLog "original builder produced exit code $builderExitCode but was expected to produce $expectedBuilderExitCode" - exit 1 - fi - unset builderExitCode - fi - - if ((''${#expectedBuilderLogEntries[@]})); then - nixLog "checking original builder log" - builderLogEntries="$(<"$failed/testBuildFailure.log")" - shouldExit=0 - for expectedBuilderLogEntry in "''${expectedBuilderLogEntries[@]}"; do - if [[ ''${builderLogEntries} == *"$expectedBuilderLogEntry"* ]]; then - nixLog "original builder log contains ''${expectedBuilderLogEntry@Q}" - else - nixErrorLog "original builder log does not contain ''${expectedBuilderLogEntry@Q}" - shouldExit=1 - fi - done - unset builderLogEntries - ((shouldExit)) && exit 1 - unset shouldExit - fi - '' - + lib.optionalString (script != "") '' - nixLog "running additional checks from user-provided script" - ${script} - '' - + '' - touch "$out" - ''; - final = - { - drv, - name ? null, - expectedBuilderExitCode ? 1, # NOTE: Should be an integer. - expectedBuilderLogEntries ? [ ], # NOTE: Should be an array of string-coercible values. TODO: Only checks for inclusion, not order! - script ? "", # Succeed by default if checks pass. - }: - (runCommand name { - __structuredAttrs = true; - strictDeps = true; - failed = testers.testBuildFailure drv; - inherit expectedBuilderExitCode expectedBuilderLogEntries; - } (mkBuildCommand script)).overrideAttrs - ( - finalAttrs: _: { - # Fix name so the default value uses whatever failed ends up as. - name = if name != null then name else "testBuildFailure-${finalAttrs.failed.name}"; - } - ); - in - lib.makeOverridable final; + # NOTE: Must be `import`-ed rather than `callPackage`-d to preserve the `override` attribute. + testBuildFailure' = import ./testBuildFailurePrime/tester.nix { inherit lib stdenvNoCC testers; }; # See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualDerivation # or doc/build-helpers/testers.chapter.md diff --git a/pkgs/build-support/testers/test/default.nix b/pkgs/build-support/testers/test/default.nix index a7187a7dbfbf8..7e4df128391da 100644 --- a/pkgs/build-support/testers/test/default.nix +++ b/pkgs/build-support/testers/test/default.nix @@ -220,114 +220,9 @@ lib.recurseIntoAttrs { sideEffectStructuredAttrs = overrideStructuredAttrs true sideEffects; }; - testBuildFailure' = lib.recurseIntoAttrs rec { - # NOTE: This example is used in the docs. - # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime - # or doc/build-helpers/testers.chapter.md - doc-example = testers.testBuildFailure' { - drv = runCommand "doc-example" { } '' - echo ok-ish >"$out" - echo failing though - exit 3 - ''; - expectedBuilderExitCode = 3; - expectedBuilderLogEntries = [ "failing though" ]; - script = '' - grep --silent -F 'ok-ish' "$failed/result" - ''; - }; - - happy = testers.testBuildFailure' { - drv = runCommand "happy" { } '' - echo ok-ish >$out - - echo failing though - echo also stderr 1>&2 - echo 'line\nwith-\bbackslashes' - printf "incomplete line - no newline" - - exit 3 - ''; - expectedBuilderExitCode = 3; - expectedBuilderLogEntries = [ - "failing though" - "also stderr" - ''line\nwith-\bbackslashes'' - "incomplete line - no newline" - ]; - script = '' - grep --silent -F 'ok-ish' "$failed/result" - ''; - }; - - happyStructuredAttrs = overrideStructuredAttrs true happy; - - helloDoesNotFail = testers.testBuildFailure' { - drv = testers.testBuildFailure hello; - expectedBuilderLogEntries = [ - "testBuildFailure: The builder did not fail, but a failure was expected" - ]; - }; - - multiOutput = testers.testBuildFailure' { - drv = - runCommand "multiOutput" - { - # dev will be the default output - outputs = [ - "dev" - "doc" - "out" - ]; - } - '' - echo i am failing - exit 1 - ''; - expectedBuilderLogEntries = [ - "i am failing" - ]; - script = '' - # Checking our note that dev is the default output - echo $failed/_ | grep -- '-dev/_' >/dev/null - echo 'All good.' - ''; - }; - - multiOutputStructuredAttrs = overrideStructuredAttrs true multiOutput; - - sideEffects = testers.testBuildFailure' { - drv = stdenvNoCC.mkDerivation { - name = "fail-with-side-effects"; - src = emptyDirectory; - - postHook = '' - echo touching side-effect... - # Assert that the side-effect doesn't exist yet... - # We're checking that this hook isn't run by expect-failure.sh - if [[ -e side-effect ]]; then - echo "side-effect already exists" - exit 1 - fi - touch side-effect - ''; - - buildPhase = '' - echo i am failing - exit 1 - ''; - }; - expectedBuilderLogEntries = [ - "touching side-effect..." - "i am failing" - ]; - script = '' - [[ ! -e side-effect ]] - ''; - }; - - sideEffectStructuredAttrs = overrideStructuredAttrs true sideEffects; - }; + testBuildFailure' = lib.recurseIntoAttrs ( + pkgs.callPackages ../testBuildFailurePrime/tests.nix { inherit overrideStructuredAttrs; } + ); testEqualContents = lib.recurseIntoAttrs { equalDir = testers.testEqualContents { diff --git a/pkgs/build-support/testers/testBuildFailurePrime/build-command.sh b/pkgs/build-support/testers/testBuildFailurePrime/build-command.sh new file mode 100644 index 0000000000000..f3f292a5f724f --- /dev/null +++ b/pkgs/build-support/testers/testBuildFailurePrime/build-command.sh @@ -0,0 +1,40 @@ +# shellcheck shell=bash + +set -eu + +scriptPhase() { + runHook preScript + + nixLog "checking original builder exit code" + local -ir builderExitCode=$(<"${failed:?}/testBuildFailure.exit") + # shellcheck disable=SC2154 + if ((expectedBuilderExitCode == builderExitCode)); then + nixLog "original builder exit code matches expected value of $expectedBuilderExitCode" + else + nixErrorLog "original builder produced exit code $builderExitCode but was expected to produce $expectedBuilderExitCode" + exit 1 + fi + + # shellcheck disable=SC2154 + if ((${#expectedBuilderLogEntries[@]})); then + nixLog "checking original builder log" + local -r builderLogEntries="$(<"${failed:?}/testBuildFailure.log")" + local -i shouldExit=0 + for expectedBuilderLogEntry in "${expectedBuilderLogEntries[@]}"; do + if [[ ${builderLogEntries} == *"$expectedBuilderLogEntry"* ]]; then + nixLog "original builder log contains ${expectedBuilderLogEntry@Q}" + else + nixErrorLog "original builder log does not contain ${expectedBuilderLogEntry@Q}" + shouldExit=1 + fi + done + ((shouldExit)) && exit 1 + fi + + runHook script + + runHook postScript +} + +runHook scriptPhase +touch "${out:?}" diff --git a/pkgs/build-support/testers/testBuildFailurePrime/tester.nix b/pkgs/build-support/testers/testBuildFailurePrime/tester.nix new file mode 100644 index 0000000000000..8b28e19dd1d78 --- /dev/null +++ b/pkgs/build-support/testers/testBuildFailurePrime/tester.nix @@ -0,0 +1,47 @@ +# NOTE: Must be `import`-ed rather than `callPackage`-d to preserve the `override` attribute. +{ + lib, + stdenvNoCC, + testers, +}: +let + inherit (lib) maintainers; + inherit (lib.customisation) makeOverridable; + inherit (testers) testBuildFailure; + + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime + # or doc/build-helpers/testers.chapter.md + testBuildFailure' = + { + drv, + name ? "testBuildFailure-${drv.name}", + expectedBuilderExitCode ? 1, + expectedBuilderLogEntries ? [ ], + script ? "", + }: + let + failed = testBuildFailure drv; + in + stdenvNoCC.mkDerivation { + __structuredAttrs = true; + strictDeps = true; + + inherit name; + + nativeBuildInputs = [ failed ]; + + inherit failed; + + inherit expectedBuilderExitCode expectedBuilderLogEntries; + + inherit script; + + buildCommandPath = ./build-command.sh; + + meta = { + description = "A wrapper around testers.testBuildFailure to simplify common use cases"; + maintainers = [ maintainers.connorbaker ]; + }; + }; +in +makeOverridable testBuildFailure' diff --git a/pkgs/build-support/testers/testBuildFailurePrime/tests.nix b/pkgs/build-support/testers/testBuildFailurePrime/tests.nix new file mode 100644 index 0000000000000..9ba65387ae8bc --- /dev/null +++ b/pkgs/build-support/testers/testBuildFailurePrime/tests.nix @@ -0,0 +1,119 @@ +{ + emptyDirectory, + hello, + overrideStructuredAttrs, + runCommand, + stdenvNoCC, + testers, +}: +let + final = { + # NOTE: This example is used in the docs. + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime + # or doc/build-helpers/testers.chapter.md + doc-example = testers.testBuildFailure' { + drv = runCommand "doc-example" { } '' + echo ok-ish >"$out" + echo failing though + exit 3 + ''; + expectedBuilderExitCode = 3; + expectedBuilderLogEntries = [ "failing though" ]; + script = '' + grep --silent -F 'ok-ish' "$failed/result" + ''; + }; + + happy = testers.testBuildFailure' { + drv = runCommand "happy" { } '' + echo ok-ish >$out + + echo failing though + echo also stderr 1>&2 + echo 'line\nwith-\bbackslashes' + printf "incomplete line - no newline" + + exit 3 + ''; + expectedBuilderExitCode = 3; + expectedBuilderLogEntries = [ + "failing though" + "also stderr" + ''line\nwith-\bbackslashes'' + "incomplete line - no newline" + ]; + script = '' + grep --silent -F 'ok-ish' "$failed/result" + ''; + }; + + happyStructuredAttrs = overrideStructuredAttrs true final.happy; + + helloDoesNotFail = testers.testBuildFailure' { + drv = testers.testBuildFailure hello; + expectedBuilderLogEntries = [ + "testBuildFailure: The builder did not fail, but a failure was expected" + ]; + }; + + multiOutput = testers.testBuildFailure' { + drv = + runCommand "multiOutput" + { + # dev will be the default output + outputs = [ + "dev" + "doc" + "out" + ]; + } + '' + echo i am failing + exit 1 + ''; + expectedBuilderLogEntries = [ + "i am failing" + ]; + script = '' + # Checking our note that dev is the default output + echo $failed/_ | grep -- '-dev/_' >/dev/null + echo 'All good.' + ''; + }; + + multiOutputStructuredAttrs = overrideStructuredAttrs true final.multiOutput; + + sideEffects = testers.testBuildFailure' { + drv = stdenvNoCC.mkDerivation { + name = "fail-with-side-effects"; + src = emptyDirectory; + + postHook = '' + echo touching side-effect... + # Assert that the side-effect doesn't exist yet... + # We're checking that this hook isn't run by expect-failure.sh + if [[ -e side-effect ]]; then + echo "side-effect already exists" + exit 1 + fi + touch side-effect + ''; + + buildPhase = '' + echo i am failing + exit 1 + ''; + }; + expectedBuilderLogEntries = [ + "touching side-effect..." + "i am failing" + ]; + script = '' + [[ ! -e side-effect ]] + ''; + }; + + sideEffectsStructuredAttrs = overrideStructuredAttrs true final.sideEffects; + }; +in +final From 98c6d78418bfcf3d3f84356c43ea8e8649f6c8d2 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Fri, 21 Feb 2025 10:11:34 -0800 Subject: [PATCH 05/16] testers.testBuildFailure': move tests to preScriptHooks --- .../testBuildFailurePrime/build-command.sh | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/pkgs/build-support/testers/testBuildFailurePrime/build-command.sh b/pkgs/build-support/testers/testBuildFailurePrime/build-command.sh index f3f292a5f724f..f4747ff3f76c2 100644 --- a/pkgs/build-support/testers/testBuildFailurePrime/build-command.sh +++ b/pkgs/build-support/testers/testBuildFailurePrime/build-command.sh @@ -2,34 +2,40 @@ set -eu -scriptPhase() { - runHook preScript +declare -ag preScriptHooks=(testBuilderExitCode) +# shellcheck disable=SC2154 +((${#expectedBuilderLogEntries[@]})) && preScriptHooks+=(testBuilderLogEntries) +testBuilderExitCode() { nixLog "checking original builder exit code" local -ir builderExitCode=$(<"${failed:?}/testBuildFailure.exit") # shellcheck disable=SC2154 if ((expectedBuilderExitCode == builderExitCode)); then nixLog "original builder exit code matches expected value of $expectedBuilderExitCode" + return 0 else nixErrorLog "original builder produced exit code $builderExitCode but was expected to produce $expectedBuilderExitCode" - exit 1 + return 1 fi +} - # shellcheck disable=SC2154 - if ((${#expectedBuilderLogEntries[@]})); then - nixLog "checking original builder log" - local -r builderLogEntries="$(<"${failed:?}/testBuildFailure.log")" - local -i shouldExit=0 - for expectedBuilderLogEntry in "${expectedBuilderLogEntries[@]}"; do - if [[ ${builderLogEntries} == *"$expectedBuilderLogEntry"* ]]; then - nixLog "original builder log contains ${expectedBuilderLogEntry@Q}" - else - nixErrorLog "original builder log does not contain ${expectedBuilderLogEntry@Q}" - shouldExit=1 - fi - done - ((shouldExit)) && exit 1 - fi +testBuilderLogEntries() { + nixLog "checking original builder log" + local -r builderLogEntries="$(<"${failed:?}/testBuildFailure.log")" + local -i shouldExit=0 + for expectedBuilderLogEntry in "${expectedBuilderLogEntries[@]}"; do + if [[ ${builderLogEntries} == *"$expectedBuilderLogEntry"* ]]; then + nixLog "original builder log contains ${expectedBuilderLogEntry@Q}" + else + nixErrorLog "original builder log does not contain ${expectedBuilderLogEntry@Q}" + shouldExit=1 + fi + done + return $shouldExit +} + +scriptPhase() { + runHook preScript runHook script From f93587f28b5e617a4759fbefe714f6f2ddd0230c Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Fri, 21 Feb 2025 10:12:36 -0800 Subject: [PATCH 06/16] tests.testers.testBuildFailure': add negative tests for exit code check and log check --- .../testers/testBuildFailurePrime/tests.nix | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pkgs/build-support/testers/testBuildFailurePrime/tests.nix b/pkgs/build-support/testers/testBuildFailurePrime/tests.nix index 9ba65387ae8bc..1f1adadccb32f 100644 --- a/pkgs/build-support/testers/testBuildFailurePrime/tests.nix +++ b/pkgs/build-support/testers/testBuildFailurePrime/tests.nix @@ -114,6 +114,34 @@ let }; sideEffectsStructuredAttrs = overrideStructuredAttrs true final.sideEffects; + + exitCodeNegativeTest = testers.testBuildFailure' { + drv = testers.testBuildFailure' { + drv = runCommand "exit-code" { } "exit 3"; + # Default expected exit code is 1 + }; + expectedBuilderLogEntries = [ + "ERROR: testBuilderExitCode: original builder produced exit code 3 but was expected to produce 1" + ]; + }; + + exitCodeNegativeTestStructuredAttrs = overrideStructuredAttrs true final.exitCodeNegativeTest; + + logNegativeTest = testers.testBuildFailure' { + drv = testers.testBuildFailure' { + drv = runCommand "exit-code" { } '' + nixLog "apples" + exit 3 + ''; + expectedBuilderExitCode = 3; + expectedBuilderLogEntries = [ "bees" ]; + }; + expectedBuilderLogEntries = [ + "ERROR: testBuilderLogEntries: original builder log does not contain 'bees'" + ]; + }; + + logNegativeTestStructuredAttrs = overrideStructuredAttrs true final.logNegativeTest; }; in final From 2d556f96aa58b5fd04b69b7cedbf5f8c4cd0c21a Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Tue, 18 Feb 2025 16:26:16 -0800 Subject: [PATCH 07/16] testers.testEqualArrayOrMap: init --- doc/build-helpers/testers.chapter.md | 91 ++++++ doc/redirects.json | 12 + pkgs/build-support/testers/default.nix | 5 + pkgs/build-support/testers/test/default.nix | 2 + .../testEqualArrayOrMap/assert-equal-array.sh | 65 ++++ .../testEqualArrayOrMap/assert-equal-map.sh | 96 ++++++ .../testers/testEqualArrayOrMap/tester.nix | 97 ++++++ .../testers/testEqualArrayOrMap/tests.nix | 279 ++++++++++++++++++ 8 files changed, 647 insertions(+) create mode 100644 pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-array.sh create mode 100644 pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-map.sh create mode 100644 pkgs/build-support/testers/testEqualArrayOrMap/tester.nix create mode 100644 pkgs/build-support/testers/testEqualArrayOrMap/tests.nix diff --git a/doc/build-helpers/testers.chapter.md b/doc/build-helpers/testers.chapter.md index e0eb0cd1a5de6..45c9bf1fc89ce 100644 --- a/doc/build-helpers/testers.chapter.md +++ b/doc/build-helpers/testers.chapter.md @@ -283,6 +283,97 @@ testers.testEqualContents { ::: +## `testEqualArrayOrMap` {#tester-testEqualArrayOrMap} + +Check that bash arrays (including associative arrays, referred to as "maps") are populated correctly. + +This can be used to ensure setup hooks are registered in a certain order, or write unit tests for shell functions which transform arrays. + +:::{.example #ex-testEqualArrayOrMap-test-function-add-cowbell} + +# Test a function which appends a value to an array + +```nix +testers.testEqualArrayOrMap { + name = "test-function-add-cowbell"; + valuesArray = [ + "cowbell" + "cowbell" + ]; + expectedArray = [ + "cowbell" + "cowbell" + "cowbell" + ]; + checkSetupScript = '' + addCowbell() { + local -rn arrayNameRef="$1" + arrayNameRef+=( "cowbell" ) + } + + nixLog "appending all values in valuesArray to actualArray" + for value in "''${valuesArray[@]}"; do + actualArray+=( "$value" ) + done + + nixLog "applying addCowbell" + addCowbell actualArray + ''; +} +``` + +::: + +### Inputs {#tester-testEqualArrayOrMap-inputs} + +NOTE: Internally, this tester uses `__structuredAttrs` to handle marhsalling between Nix expressions and shell variables. +This imposes the restriction that arrays and "maps" have values which are string-coercible. + +NOTE: At least one of `expectedArray` and `expectedMap` must be provided. + +`name` (string) + +: The name of the test. + +`checkSetupScript` (string) + +: The singular task of `checkSetupScript` is to populate `actualArray` or `actualMap` (it may populate both). + To do this, checkSetupScript may access the following shell variables: + + - `valuesArray` + - `valuesMap` + - `actualArray` + - `actualMap` + + While both `expectedArray` and `expectedMap` are in scope during the execution of `checkSetupScript`, they *must not* be accessed or modified from within `checkSetupScript`. + +`valuesArray` (array of string-like values, optional) + +: An array of string-coercible values. + This array may be used within `checkSetupScript`. + +`valuesMap` (attribute set of string-like values, optional) + +: An attribute set of string-coercible values. + This attribute set may be used within `checkSetupScript`. + +`expectedArray` (array of string-like values, optional) + +: An array of string-coercible values. + This array *must not* be accessed or modified from within `checkSetupScript`. + When provided, `checkSetupScript` is expected to populate `actualArray`. + +`expectedMap` (attribute set of string-like values, optional) + +: An attribute set of string-coercible values. + This attribute set *must not* be accessed or modified from within `checkSetupScript`. + When provided, `checkSetupScript` is expected to populate `actualMap`. + +### Return value {#tester-testEqualArrayOrMap-return} + +The tester produces an empty output and only succeeds when `expectedArray` and `expectedMap` match `actualArray` and `actualMap`, respectively, when non-null. +The build log will contain differences encountered. + ## `testEqualDerivation` {#tester-testEqualDerivation} Checks that two packages produce the exact same build instructions. diff --git a/doc/redirects.json b/doc/redirects.json index ddf3626bbd340..5108d6786e9fd 100644 --- a/doc/redirects.json +++ b/doc/redirects.json @@ -8,6 +8,9 @@ "ex-build-helpers-extendMkDerivation": [ "index.html#ex-build-helpers-extendMkDerivation" ], + "ex-testEqualArrayOrMap-test-function-add-cowbell": [ + "index.html#ex-testEqualArrayOrMap-test-function-add-cowbell" + ], "neovim": [ "index.html#neovim" ], @@ -332,6 +335,15 @@ "footnote-stdenv-find-inputs-location.__back.0": [ "index.html#footnote-stdenv-find-inputs-location.__back.0" ], + "tester-testEqualArrayOrMap": [ + "index.html#tester-testEqualArrayOrMap" + ], + "tester-testEqualArrayOrMap-inputs": [ + "index.html#tester-testEqualArrayOrMap-inputs" + ], + "tester-testEqualArrayOrMap-return": [ + "index.html#tester-testEqualArrayOrMap-return" + ], "variables-specifying-dependencies": [ "index.html#variables-specifying-dependencies" ], diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index 0393895d53aff..d8db79215c055 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -65,6 +65,11 @@ fi ''; + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualArrayOrMap + # or doc/build-helpers/testers.chapter.md + # NOTE: Must be `import`-ed rather than `callPackage`-d to preserve the `override` attribute. + testEqualArrayOrMap = import ./testEqualArrayOrMap/tester.nix { inherit lib runCommand; }; + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testVersion # or doc/build-helpers/testers.chapter.md testVersion = diff --git a/pkgs/build-support/testers/test/default.nix b/pkgs/build-support/testers/test/default.nix index b0f2b4c1d391a..2c7f36bea0c29 100644 --- a/pkgs/build-support/testers/test/default.nix +++ b/pkgs/build-support/testers/test/default.nix @@ -352,4 +352,6 @@ lib.recurseIntoAttrs { touch -- "$out" ''; }; + + testEqualArrayOrMap = lib.recurseIntoAttrs (pkgs.callPackages ../testEqualArrayOrMap/tests.nix { }); } diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-array.sh b/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-array.sh new file mode 100644 index 0000000000000..4e68b3b1488c3 --- /dev/null +++ b/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-array.sh @@ -0,0 +1,65 @@ +# shellcheck shell=bash + +# Asserts that two arrays are equal, printing out differences if they are not. +# Does not short circuit on the first difference. +assertEqualArray() { + if (($# != 2)); then + nixErrorLog "expected two arguments!" + nixErrorLog "usage: assertEqualArray expectedArrayRef actualArrayRef" + exit 1 + fi + + local -nr expectedArrayRef="$1" + local -nr actualArrayRef="$2" + + if [[ ! ${expectedArrayRef@a} =~ a ]]; then + nixErrorLog "first arugment expectedArrayRef must be an array reference" + exit 1 + fi + + if [[ ! ${actualArrayRef@a} =~ a ]]; then + nixErrorLog "second arugment actualArrayRef must be an array reference" + exit 1 + fi + + local -ir expectedLength=${#expectedArrayRef[@]} + local -ir actualLength=${#actualArrayRef[@]} + + local -i hasDiff=0 + + if ((expectedLength != actualLength)); then + nixErrorLog "arrays differ in length: expectedArray has length $expectedLength but actualArray has length $actualLength" + hasDiff=1 + fi + + local -i idx=0 + local expectedValue + local actualValue + + # We iterate so long as at least one array has indices we've not considered. + # This means that `idx` is a valid index to *at least one* of the arrays. + for ((idx = 0; idx < expectedLength || idx < actualLength; idx++)); do + # Update values for variables which are still in range/valid. + if ((idx < expectedLength)); then + expectedValue="${expectedArrayRef[idx]}" + fi + + if ((idx < actualLength)); then + actualValue="${actualArrayRef[idx]}" + fi + + # Handle comparisons. + if ((idx >= expectedLength)); then + nixErrorLog "arrays differ at index $idx: expectedArray has no such index but actualArray has value ${actualValue@Q}" + hasDiff=1 + elif ((idx >= actualLength)); then + nixErrorLog "arrays differ at index $idx: expectedArray has value ${expectedValue@Q} but actualArray has no such index" + hasDiff=1 + elif [[ $expectedValue != "$actualValue" ]]; then + nixErrorLog "arrays differ at index $idx: expectedArray has value ${expectedValue@Q} but actualArray has value ${actualValue@Q}" + hasDiff=1 + fi + done + + ((hasDiff)) && exit 1 || return 0 +} diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-map.sh b/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-map.sh new file mode 100644 index 0000000000000..1e07b3aaff023 --- /dev/null +++ b/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-map.sh @@ -0,0 +1,96 @@ +# shellcheck shell=bash + +# Asserts that two maps are equal, printing out differences if they are not. +# Does not short circuit on the first difference. +assertEqualMap() { + if (($# != 2)); then + nixErrorLog "expected two arguments!" + nixErrorLog "usage: assertEqualMap expectedMapRef actualMapRef" + exit 1 + fi + + local -nr expectedMapRef="$1" + local -nr actualMapRef="$2" + + if [[ ! ${expectedMapRef@a} =~ A ]]; then + nixErrorLog "first arugment expectedMapRef must be an associative array reference" + exit 1 + fi + + if [[ ! ${actualMapRef@a} =~ A ]]; then + nixErrorLog "second arugment actualMapRef must be an associative array reference" + exit 1 + fi + + # NOTE: + # From the `sort` manpage: "The locale specified by the environment affects sort order. Set LC_ALL=C to get the + # traditional sort order that uses native byte values." + # We specify the environment variable in a subshell to avoid polluting the caller's environment. + + local -a sortedExpectedKeys + mapfile -d '' -t sortedExpectedKeys < <(printf '%s\0' "${!expectedMapRef[@]}" | LC_ALL=C sort --stable --zero-terminated) + + local -a sortedActualKeys + mapfile -d '' -t sortedActualKeys < <(printf '%s\0' "${!actualMapRef[@]}" | LC_ALL=C sort --stable --zero-terminated) + + local -ir expectedLength=${#expectedMapRef[@]} + local -ir actualLength=${#actualMapRef[@]} + + local -i hasDiff=0 + + if ((expectedLength != actualLength)); then + nixErrorLog "maps differ in length: expectedMap has length $expectedLength but actualMap has length $actualLength" + hasDiff=1 + fi + + local -i expectedKeyIdx=0 + local expectedKey + local expectedValue + local -i actualKeyIdx=0 + local actualKey + local actualValue + + # We iterate so long as at least one map has keys we've not considered. + while ((expectedKeyIdx < expectedLength || actualKeyIdx < actualLength)); do + # Update values for variables which are still in range/valid. + if ((expectedKeyIdx < expectedLength)); then + expectedKey="${sortedExpectedKeys["$expectedKeyIdx"]}" + expectedValue="${expectedMapRef["$expectedKey"]}" + fi + + if ((actualKeyIdx < actualLength)); then + actualKey="${sortedActualKeys["$actualKeyIdx"]}" + actualValue="${actualMapRef["$actualKey"]}" + fi + + # In the case actualKeyIdx is valid and expectedKey comes after actualKey or expectedKeyIdx is invalid, actualMap + # has an extra key relative to expectedMap. + # NOTE: In Bash, && and || have the same precedence, so use the fact they're left-associative to enforce groups. + if ((actualKeyIdx < actualLength)) && [[ $expectedKey > $actualKey ]] || ((expectedKeyIdx >= expectedLength)); then + nixErrorLog "maps differ at key ${actualKey@Q}: expectedMap has no such key but actualMap has value ${actualValue@Q}" + hasDiff=1 + actualKeyIdx+=1 + + # In the case actualKeyIdx is invalid or expectedKey comes before actualKey, expectedMap has an extra key relative + # to actualMap. + # NOTE: By virtue of the previous condition being false, we know the negation is true. Namely, expectedKeyIdx is + # valid AND (actualKeyIdx is invalid OR expectedKey <= actualKey). + elif ((actualKeyIdx >= actualLength)) || [[ $expectedKey < $actualKey ]]; then + nixErrorLog "maps differ at key ${expectedKey@Q}: expectedMap has value ${expectedValue@Q} but actualMap has no such key" + hasDiff=1 + expectedKeyIdx+=1 + + # In the case where both key indices are valid and the keys are equal. + else + if [[ $expectedValue != "$actualValue" ]]; then + nixErrorLog "maps differ at key ${expectedKey@Q}: expectedMap has value ${expectedValue@Q} but actualMap has value ${actualValue@Q}" + hasDiff=1 + fi + + expectedKeyIdx+=1 + actualKeyIdx+=1 + fi + done + + ((hasDiff)) && exit 1 || return 0 +} diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix b/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix new file mode 100644 index 0000000000000..1044903aaca5b --- /dev/null +++ b/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix @@ -0,0 +1,97 @@ +# NOTE: Must be `import`-ed rather than `callPackage`-d to preserve the `override` attribute. +# NOTE: We must use `pkgs.runCommand` instead of `testers.runCommand` to build `testers.testEqualArrayOrMap`, or else +# our negative tests will not work. See ./tests.nix for more information. +{ + lib, + runCommand, +}: +let + inherit (lib) maintainers; + inherit (lib.customisation) makeOverridable; + inherit (lib.strings) optionalString; + + testEqualArrayOrMap = + { + name, + valuesArray ? null, + valuesMap ? null, + expectedArray ? null, + expectedMap ? null, + checkSetupScript ? '' + nixErrorLog "no checkSetupScript provided!" + exit 1 + '', + }: + runCommand name + { + __structuredAttrs = true; + strictDeps = true; + + nativeBuildInputs = [ + ./assert-equal-array.sh + ./assert-equal-map.sh + ]; + + inherit valuesArray valuesMap; + inherit expectedArray expectedMap; + + preCheckSetupScript = + optionalString (expectedArray == null && expectedMap == null) '' + nixErrorLog "neither expectedArray nor expectedMap were set, so test is meaningless!" + exit 1 + '' + + optionalString (valuesArray != null) '' + nixLog "using valuesArray: $(declare -p valuesArray)" + '' + + optionalString (valuesMap != null) '' + nixLog "using valuesMap: $(declare -p valuesMap)" + '' + + optionalString (expectedArray != null) '' + nixLog "using expectedArray: $(declare -p expectedArray)" + declare -ag actualArray + '' + + optionalString (expectedMap != null) '' + nixLog "using expectedMap: $(declare -p expectedMap)" + declare -Ag actualMap + ''; + + # NOTE: + # The singular task of checkSetupScript is to populate actualArray or actualMap. To do this, checkSetupScript + # may access valuesArray, valuesMap, actualArray, and actualMap, but should *never* access or modify expectedArray, + # or expectedMap. + inherit checkSetupScript; + + postCheckSetupScript = + optionalString (expectedArray != null) '' + nixLog "using actualArray: $(declare -p actualArray)" + nixLog "comparing actualArray against expectedArray" + assertEqualArray expectedArray actualArray + nixLog "actualArray matches expectedArray" + '' + + optionalString (expectedMap != null) '' + nixLog "using actualMap: $(declare -p actualMap)" + nixLog "comparing actualMap against expectedMap" + assertEqualMap expectedMap actualMap + nixLog "actualMap matches expectedMap" + ''; + + meta = { + description = "Tests for equality of bash arrays (including associative arrays)"; + maintainers = [ maintainers.connorbaker ]; + }; + } + '' + nixLog "running preCheckSetupScript" + runHook preCheckSetupScript + + nixLog "running checkSetupScript" + runHook checkSetupScript + + nixLog "running postCheckSetupScript" + runHook postCheckSetupScript + + nixLog "test passed" + touch "$out" + ''; +in +makeOverridable testEqualArrayOrMap diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/tests.nix b/pkgs/build-support/testers/testEqualArrayOrMap/tests.nix new file mode 100644 index 0000000000000..92888151d43e5 --- /dev/null +++ b/pkgs/build-support/testers/testEqualArrayOrMap/tests.nix @@ -0,0 +1,279 @@ +# NOTE: We must use `pkgs.runCommand` instead of `testers.runCommand` for negative tests -- those wrapped with +# `testers.testBuildFailure`. This is due to the fact that `testers.testBuildFailure` modifies the derivation such that +# it produces an output containing the exit code, logs, and other things. Since `testers.runCommand` expects the empty +# derivation, it produces a hash mismatch. +{ runCommand, testers, ... }: +let + inherit (testers) testEqualArrayOrMap testBuildFailure; + concatValuesArrayToActualArray = '' + nixLog "appending all values in valuesArray to actualArray" + for value in "''${valuesArray[@]}"; do + actualArray+=( "$value" ) + done + ''; + concatValuesMapToActualMap = '' + nixLog "adding all values in valuesMap to actualMap" + for key in "''${!valuesMap[@]}"; do + actualMap["$key"]="''${valuesMap["$key"]}" + done + ''; +in +{ + # NOTE: This particular test is used in the docs: + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualArrayOrMap + # or doc/build-helpers/testers.chapter.md + docs-test-function-add-cowbell = testEqualArrayOrMap { + name = "test-function-add-cowbell"; + valuesArray = [ + "cowbell" + "cowbell" + ]; + expectedArray = [ + "cowbell" + "cowbell" + "cowbell" + ]; + checkSetupScript = '' + addCowbell() { + local -rn arrayNameRef="$1" + arrayNameRef+=( "cowbell" ) + } + + nixLog "appending all values in valuesArray to actualArray" + for value in "''${valuesArray[@]}"; do + actualArray+=( "$value" ) + done + + nixLog "applying addCowbell" + addCowbell actualArray + ''; + }; + array-append = testEqualArrayOrMap { + name = "testEqualArrayOrMap-array-append"; + valuesArray = [ + "apple" + "bee" + "cat" + ]; + expectedArray = [ + "apple" + "bee" + "cat" + "dog" + ]; + checkSetupScript = '' + ${concatValuesArrayToActualArray} + actualArray+=( "dog" ) + ''; + }; + array-prepend = testEqualArrayOrMap { + name = "testEqualArrayOrMap-array-prepend"; + valuesArray = [ + "apple" + "bee" + "cat" + ]; + expectedArray = [ + "dog" + "apple" + "bee" + "cat" + ]; + checkSetupScript = '' + actualArray+=( "dog" ) + ${concatValuesArrayToActualArray} + ''; + }; + array-empty = testEqualArrayOrMap { + name = "testEqualArrayOrMap-array-empty"; + valuesArray = [ + "apple" + "bee" + "cat" + ]; + expectedArray = [ ]; + checkSetupScript = '' + # doing nothing + ''; + }; + array-missing-value = + let + name = "testEqualArrayOrMap-array-missing-value"; + failure = testEqualArrayOrMap { + name = "${name}-failure"; + valuesArray = [ "apple" ]; + expectedArray = [ ]; + checkSetupScript = concatValuesArrayToActualArray; + }; + in + runCommand name + { + failed = testBuildFailure failure; + passthru = { + inherit failure; + }; + } + '' + nixLog "Checking for exit code 1" + (( 1 == "$(cat "$failed/testBuildFailure.exit")" )) + nixLog "Checking for first error message" + grep -F \ + "ERROR: assertEqualArray: arrays differ in length: expectedArray has length 0 but actualArray has length 1" \ + "$failed/testBuildFailure.log" + nixLog "Checking for second error message" + grep -F \ + "ERROR: assertEqualArray: arrays differ at index 0: expectedArray has no such index but actualArray has value 'apple'" \ + "$failed/testBuildFailure.log" + nixLog "Test passed" + touch $out + ''; + map-insert = testEqualArrayOrMap { + name = "testEqualArrayOrMap-map-insert"; + valuesMap = { + apple = "0"; + bee = "1"; + cat = "2"; + }; + expectedMap = { + apple = "0"; + bee = "1"; + cat = "2"; + dog = "3"; + }; + checkSetupScript = '' + ${concatValuesMapToActualMap} + actualMap["dog"]="3" + ''; + }; + map-remove = testEqualArrayOrMap { + name = "testEqualArrayOrMap-map-remove"; + valuesMap = { + apple = "0"; + bee = "1"; + cat = "2"; + dog = "3"; + }; + expectedMap = { + apple = "0"; + cat = "2"; + dog = "3"; + }; + checkSetupScript = '' + ${concatValuesMapToActualMap} + unset 'actualMap[bee]' + ''; + }; + map-missing-key = + let + name = "testEqualArrayOrMap-map-missing-key"; + failure = testEqualArrayOrMap { + name = "${name}-failure"; + valuesMap = { + bee = "1"; + cat = "2"; + dog = "3"; + }; + expectedMap = { + apple = "0"; + bee = "1"; + cat = "2"; + dog = "3"; + }; + checkSetupScript = concatValuesMapToActualMap; + }; + in + runCommand name + { + failed = testBuildFailure failure; + passthru = { + inherit failure; + }; + } + '' + nixLog "Checking for exit code 1" + (( 1 == "$(cat "$failed/testBuildFailure.exit")" )) + nixLog "Checking for first error message" + grep -F \ + "ERROR: assertEqualMap: maps differ in length: expectedMap has length 4 but actualMap has length 3" \ + "$failed/testBuildFailure.log" + nixLog "Checking for second error message" + grep -F \ + "ERROR: assertEqualMap: maps differ at key 'apple': expectedMap has value '0' but actualMap has no such key" \ + "$failed/testBuildFailure.log" + nixLog "Test passed" + touch $out + ''; + map-missing-key-with-empty = + let + name = "map-missing-key-with-empty"; + failure = testEqualArrayOrMap { + name = "${name}-failure"; + valuesArray = [ ]; + expectedMap.apple = 1; + checkSetupScript = '' + nixLog "doing nothing in checkSetupScript" + ''; + }; + in + runCommand name + { + failed = testBuildFailure failure; + passthru = { + inherit failure; + }; + } + '' + nixLog "Checking for exit code 1" + (( 1 == "$(cat "$failed/testBuildFailure.exit")" )) + nixLog "Checking for first error message" + grep -F \ + "ERROR: assertEqualMap: maps differ in length: expectedMap has length 1 but actualMap has length 0" \ + "$failed/testBuildFailure.log" + nixLog "Checking for second error message" + grep -F \ + "ERROR: assertEqualMap: maps differ at key 'apple': expectedMap has value '1' but actualMap has no such key" \ + "$failed/testBuildFailure.log" + nixLog "Test passed" + touch $out + ''; + map-extra-key = + let + name = "testEqualArrayOrMap-map-extra-key"; + failure = testEqualArrayOrMap { + name = "${name}-failure"; + valuesMap = { + apple = "0"; + bee = "1"; + cat = "2"; + dog = "3"; + }; + expectedMap = { + apple = "0"; + bee = "1"; + dog = "3"; + }; + checkSetupScript = concatValuesMapToActualMap; + }; + in + runCommand + { + failed = testBuildFailure failure; + passthru = { + inherit failure; + }; + } + '' + nixLog "Checking for exit code 1" + (( 1 == "$(cat "$failed/testBuildFailure.exit")" )) + nixLog "Checking for first error message" + grep -F \ + "ERROR: assertEqualMap: maps differ in length: expectedMap has length 3 but actualMap has length 4" \ + "$failed/testBuildFailure.log" + nixLog "Checking for second error message" + grep -F \ + "ERROR: assertEqualMap: maps differ at key 'cat': expectedMap has no such key but actualMap has value '2'" \ + "$failed/testBuildFailure.log" + nixLog "Test passed" + touch $out + ''; +} From 38e4ea07fc9c067675c6753ed6d753d0d09f2030 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Thu, 20 Feb 2025 22:37:59 +0000 Subject: [PATCH 08/16] testers.testEqualArrayOrMap: use buildCommandPath and change checkSetupScript argument to script --- doc/build-helpers/testers.chapter.md | 44 ++++----- pkgs/build-support/testers/default.nix | 2 +- .../testEqualArrayOrMap/assert-equal-array.sh | 14 ++- .../testEqualArrayOrMap/assert-equal-map.sh | 14 ++- .../testEqualArrayOrMap/build-command.sh | 64 +++++++++++++ .../testers/testEqualArrayOrMap/tester.nix | 90 ++++--------------- .../testers/testEqualArrayOrMap/tests.nix | 22 +++-- 7 files changed, 136 insertions(+), 114 deletions(-) create mode 100644 pkgs/build-support/testers/testEqualArrayOrMap/build-command.sh diff --git a/doc/build-helpers/testers.chapter.md b/doc/build-helpers/testers.chapter.md index 45c9bf1fc89ce..4c36027aef820 100644 --- a/doc/build-helpers/testers.chapter.md +++ b/doc/build-helpers/testers.chapter.md @@ -287,7 +287,7 @@ testers.testEqualContents { Check that bash arrays (including associative arrays, referred to as "maps") are populated correctly. -This can be used to ensure setup hooks are registered in a certain order, or write unit tests for shell functions which transform arrays. +This can be used to ensure setup hooks are registered in a certain order, or to write unit tests for shell functions which transform arrays. :::{.example #ex-testEqualArrayOrMap-test-function-add-cowbell} @@ -305,7 +305,7 @@ testers.testEqualArrayOrMap { "cowbell" "cowbell" ]; - checkSetupScript = '' + script = '' addCowbell() { local -rn arrayNameRef="$1" arrayNameRef+=( "cowbell" ) @@ -326,8 +326,8 @@ testers.testEqualArrayOrMap { ### Inputs {#tester-testEqualArrayOrMap-inputs} -NOTE: Internally, this tester uses `__structuredAttrs` to handle marhsalling between Nix expressions and shell variables. -This imposes the restriction that arrays and "maps" have values which are string-coercible. +NOTE: Internally, this tester uses `__structuredAttrs` to handle marshalling between Nix expressions and shell variables. +This imposes the restriction that arrays and "maps" have values which are string-like. NOTE: At least one of `expectedArray` and `expectedMap` must be provided. @@ -335,39 +335,39 @@ NOTE: At least one of `expectedArray` and `expectedMap` must be provided. : The name of the test. -`checkSetupScript` (string) +`script` (string) -: The singular task of `checkSetupScript` is to populate `actualArray` or `actualMap` (it may populate both). - To do this, checkSetupScript may access the following shell variables: +: The singular task of `script` is to populate `actualArray` or `actualMap` (it may populate both). + To do this, `script` may access the following shell variables: - - `valuesArray` - - `valuesMap` - - `actualArray` - - `actualMap` + - `valuesArray` (available when `valuesArray` is provided to the tester) + - `valuesMap` (available when `valuesMap` is provided to the tester) + - `actualArray` (available when `expectedArray` is provided to the tester) + - `actualMap` (available when `expectedMap` is provided to the tester) - While both `expectedArray` and `expectedMap` are in scope during the execution of `checkSetupScript`, they *must not* be accessed or modified from within `checkSetupScript`. + While both `expectedArray` and `expectedMap` are in scope during the execution of `script`, they *must not* be accessed or modified from within `script`. `valuesArray` (array of string-like values, optional) -: An array of string-coercible values. - This array may be used within `checkSetupScript`. +: An array of string-like values. + This array may be used within `script`. `valuesMap` (attribute set of string-like values, optional) -: An attribute set of string-coercible values. - This attribute set may be used within `checkSetupScript`. +: An attribute set of string-like values. + This attribute set may be used within `script`. `expectedArray` (array of string-like values, optional) -: An array of string-coercible values. - This array *must not* be accessed or modified from within `checkSetupScript`. - When provided, `checkSetupScript` is expected to populate `actualArray`. +: An array of string-like values. + This array *must not* be accessed or modified from within `script`. + When provided, `script` is expected to populate `actualArray`. `expectedMap` (attribute set of string-like values, optional) -: An attribute set of string-coercible values. - This attribute set *must not* be accessed or modified from within `checkSetupScript`. - When provided, `checkSetupScript` is expected to populate `actualMap`. +: An attribute set of string-like values. + This attribute set *must not* be accessed or modified from within `script`. + When provided, `script` is expected to populate `actualMap`. ### Return value {#tester-testEqualArrayOrMap-return} diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index d8db79215c055..453c66626fd0f 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -68,7 +68,7 @@ # See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualArrayOrMap # or doc/build-helpers/testers.chapter.md # NOTE: Must be `import`-ed rather than `callPackage`-d to preserve the `override` attribute. - testEqualArrayOrMap = import ./testEqualArrayOrMap/tester.nix { inherit lib runCommand; }; + testEqualArrayOrMap = import ./testEqualArrayOrMap/tester.nix { inherit lib stdenvNoCC; }; # See https://nixos.org/manual/nixpkgs/unstable/#tester-testVersion # or doc/build-helpers/testers.chapter.md diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-array.sh b/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-array.sh index 4e68b3b1488c3..92fa981e44eeb 100644 --- a/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-array.sh +++ b/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-array.sh @@ -1,5 +1,11 @@ # shellcheck shell=bash +# Tests if an array is declared. +isDeclaredArray() { + # shellcheck disable=SC2034 + local -nr arrayRef="$1" && [[ ${!arrayRef@a} =~ a ]] +} + # Asserts that two arrays are equal, printing out differences if they are not. # Does not short circuit on the first difference. assertEqualArray() { @@ -12,13 +18,13 @@ assertEqualArray() { local -nr expectedArrayRef="$1" local -nr actualArrayRef="$2" - if [[ ! ${expectedArrayRef@a} =~ a ]]; then - nixErrorLog "first arugment expectedArrayRef must be an array reference" + if ! isDeclaredArray expectedArrayRef; then + nixErrorLog "first arugment expectedArrayRef must be an array reference to a declared array" exit 1 fi - if [[ ! ${actualArrayRef@a} =~ a ]]; then - nixErrorLog "second arugment actualArrayRef must be an array reference" + if ! isDeclaredArray actualArrayRef; then + nixErrorLog "second arugment actualArrayRef must be an array reference to a declared array" exit 1 fi diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-map.sh b/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-map.sh index 1e07b3aaff023..f66fab4f8468d 100644 --- a/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-map.sh +++ b/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-map.sh @@ -1,5 +1,11 @@ # shellcheck shell=bash +# Tests if a map is declared. +isDeclaredMap() { + # shellcheck disable=SC2034 + local -nr mapRef="$1" && [[ ${!mapRef@a} =~ A ]] +} + # Asserts that two maps are equal, printing out differences if they are not. # Does not short circuit on the first difference. assertEqualMap() { @@ -12,13 +18,13 @@ assertEqualMap() { local -nr expectedMapRef="$1" local -nr actualMapRef="$2" - if [[ ! ${expectedMapRef@a} =~ A ]]; then - nixErrorLog "first arugment expectedMapRef must be an associative array reference" + if ! isDeclaredMap expectedMapRef; then + nixErrorLog "first arugment expectedMapRef must be an associative array reference to a declared associative array" exit 1 fi - if [[ ! ${actualMapRef@a} =~ A ]]; then - nixErrorLog "second arugment actualMapRef must be an associative array reference" + if ! isDeclaredMap actualMapRef; then + nixErrorLog "second arugment actualMapRef must be an associative array reference to a declared associative array" exit 1 fi diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/build-command.sh b/pkgs/build-support/testers/testEqualArrayOrMap/build-command.sh new file mode 100644 index 0000000000000..b1733692d1e9c --- /dev/null +++ b/pkgs/build-support/testers/testEqualArrayOrMap/build-command.sh @@ -0,0 +1,64 @@ +# shellcheck shell=bash + +set -eu + +preScript() { + # If neither expectedArray nor expectedMap are declared, the test is meaningless. + if ! isDeclaredArray expectedArray && ! isDeclaredMap expectedMap; then + nixErrorLog "neither expectedArray nor expectedMap were set, so test is meaningless!" + exit 1 + fi + + if isDeclaredArray valuesArray; then + # shellcheck disable=SC2154 + nixLog "using valuesArray: $(declare -p valuesArray)" + fi + + if isDeclaredMap valuesMap; then + # shellcheck disable=SC2154 + nixLog "using valuesMap: $(declare -p valuesMap)" + fi + + if isDeclaredArray expectedArray; then + # shellcheck disable=SC2154 + nixLog "using expectedArray: $(declare -p expectedArray)" + declare -ag actualArray=() + fi + + if isDeclaredMap expectedMap; then + # shellcheck disable=SC2154 + nixLog "using expectedMap: $(declare -p expectedMap)" + declare -Ag actualMap=() + fi + + return 0 +} + +scriptPhase() { + runHook preScript + + runHook script + + runHook postScript +} + +postScript() { + if isDeclaredArray expectedArray; then + nixLog "using actualArray: $(declare -p actualArray)" + nixLog "comparing actualArray against expectedArray" + assertEqualArray expectedArray actualArray + nixLog "actualArray matches expectedArray" + fi + + if isDeclaredMap expectedMap; then + nixLog "using actualMap: $(declare -p actualMap)" + nixLog "comparing actualMap against expectedMap" + assertEqualMap expectedMap actualMap + nixLog "actualMap matches expectedMap" + fi + + return 0 +} + +runHook scriptPhase +touch "${out:?}" diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix b/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix index 1044903aaca5b..fed20a48f625b 100644 --- a/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix +++ b/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix @@ -3,12 +3,11 @@ # our negative tests will not work. See ./tests.nix for more information. { lib, - runCommand, + stdenvNoCC, }: let inherit (lib) maintainers; inherit (lib.customisation) makeOverridable; - inherit (lib.strings) optionalString; testEqualArrayOrMap = { @@ -17,81 +16,30 @@ let valuesMap ? null, expectedArray ? null, expectedMap ? null, - checkSetupScript ? '' - nixErrorLog "no checkSetupScript provided!" - exit 1 - '', + script, }: - runCommand name - { - __structuredAttrs = true; - strictDeps = true; + stdenvNoCC.mkDerivation (finalAttrs: { + __structuredAttrs = true; + strictDeps = true; - nativeBuildInputs = [ - ./assert-equal-array.sh - ./assert-equal-map.sh - ]; + inherit name; - inherit valuesArray valuesMap; - inherit expectedArray expectedMap; + nativeBuildInputs = [ + ./assert-equal-array.sh + ./assert-equal-map.sh + ]; - preCheckSetupScript = - optionalString (expectedArray == null && expectedMap == null) '' - nixErrorLog "neither expectedArray nor expectedMap were set, so test is meaningless!" - exit 1 - '' - + optionalString (valuesArray != null) '' - nixLog "using valuesArray: $(declare -p valuesArray)" - '' - + optionalString (valuesMap != null) '' - nixLog "using valuesMap: $(declare -p valuesMap)" - '' - + optionalString (expectedArray != null) '' - nixLog "using expectedArray: $(declare -p expectedArray)" - declare -ag actualArray - '' - + optionalString (expectedMap != null) '' - nixLog "using expectedMap: $(declare -p expectedMap)" - declare -Ag actualMap - ''; + inherit valuesArray valuesMap; + inherit expectedArray expectedMap; - # NOTE: - # The singular task of checkSetupScript is to populate actualArray or actualMap. To do this, checkSetupScript - # may access valuesArray, valuesMap, actualArray, and actualMap, but should *never* access or modify expectedArray, - # or expectedMap. - inherit checkSetupScript; + inherit script; - postCheckSetupScript = - optionalString (expectedArray != null) '' - nixLog "using actualArray: $(declare -p actualArray)" - nixLog "comparing actualArray against expectedArray" - assertEqualArray expectedArray actualArray - nixLog "actualArray matches expectedArray" - '' - + optionalString (expectedMap != null) '' - nixLog "using actualMap: $(declare -p actualMap)" - nixLog "comparing actualMap against expectedMap" - assertEqualMap expectedMap actualMap - nixLog "actualMap matches expectedMap" - ''; + buildCommandPath = ./build-command.sh; - meta = { - description = "Tests for equality of bash arrays (including associative arrays)"; - maintainers = [ maintainers.connorbaker ]; - }; - } - '' - nixLog "running preCheckSetupScript" - runHook preCheckSetupScript - - nixLog "running checkSetupScript" - runHook checkSetupScript - - nixLog "running postCheckSetupScript" - runHook postCheckSetupScript - - nixLog "test passed" - touch "$out" - ''; + meta = { + description = "Tests for equality of bash arrays (including associative arrays)"; + maintainers = [ maintainers.connorbaker ]; + }; + }); in makeOverridable testEqualArrayOrMap diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/tests.nix b/pkgs/build-support/testers/testEqualArrayOrMap/tests.nix index 92888151d43e5..bcb2a176a4247 100644 --- a/pkgs/build-support/testers/testEqualArrayOrMap/tests.nix +++ b/pkgs/build-support/testers/testEqualArrayOrMap/tests.nix @@ -33,7 +33,7 @@ in "cowbell" "cowbell" ]; - checkSetupScript = '' + script = '' addCowbell() { local -rn arrayNameRef="$1" arrayNameRef+=( "cowbell" ) @@ -61,7 +61,7 @@ in "cat" "dog" ]; - checkSetupScript = '' + script = '' ${concatValuesArrayToActualArray} actualArray+=( "dog" ) ''; @@ -79,7 +79,7 @@ in "bee" "cat" ]; - checkSetupScript = '' + script = '' actualArray+=( "dog" ) ${concatValuesArrayToActualArray} ''; @@ -92,7 +92,7 @@ in "cat" ]; expectedArray = [ ]; - checkSetupScript = '' + script = '' # doing nothing ''; }; @@ -103,7 +103,7 @@ in name = "${name}-failure"; valuesArray = [ "apple" ]; expectedArray = [ ]; - checkSetupScript = concatValuesArrayToActualArray; + script = concatValuesArrayToActualArray; }; in runCommand name @@ -140,7 +140,7 @@ in cat = "2"; dog = "3"; }; - checkSetupScript = '' + script = '' ${concatValuesMapToActualMap} actualMap["dog"]="3" ''; @@ -158,7 +158,7 @@ in cat = "2"; dog = "3"; }; - checkSetupScript = '' + script = '' ${concatValuesMapToActualMap} unset 'actualMap[bee]' ''; @@ -179,7 +179,7 @@ in cat = "2"; dog = "3"; }; - checkSetupScript = concatValuesMapToActualMap; + script = concatValuesMapToActualMap; }; in runCommand name @@ -210,9 +210,7 @@ in name = "${name}-failure"; valuesArray = [ ]; expectedMap.apple = 1; - checkSetupScript = '' - nixLog "doing nothing in checkSetupScript" - ''; + script = ""; }; in runCommand name @@ -252,7 +250,7 @@ in bee = "1"; dog = "3"; }; - checkSetupScript = concatValuesMapToActualMap; + script = concatValuesMapToActualMap; }; in runCommand From ecc1aaedb3f413c7a189e2e242705ae3cba9ddcf Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Thu, 20 Feb 2025 23:18:13 +0000 Subject: [PATCH 09/16] testers.testEqualArrayOrMap: remove old comment --- pkgs/build-support/testers/testEqualArrayOrMap/tester.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix b/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix index fed20a48f625b..ce67f930cc6a4 100644 --- a/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix +++ b/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix @@ -1,6 +1,4 @@ # NOTE: Must be `import`-ed rather than `callPackage`-d to preserve the `override` attribute. -# NOTE: We must use `pkgs.runCommand` instead of `testers.runCommand` to build `testers.testEqualArrayOrMap`, or else -# our negative tests will not work. See ./tests.nix for more information. { lib, stdenvNoCC, From bf4fd0580ff1bf86cf0ff178ade624fca834e3a0 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Fri, 21 Feb 2025 10:14:31 -0800 Subject: [PATCH 10/16] testers.testEqualArrayOrMap: remove unused finalAttrs --- pkgs/build-support/testers/testEqualArrayOrMap/tester.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix b/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix index ce67f930cc6a4..66dc8dd3f153a 100644 --- a/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix +++ b/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix @@ -16,7 +16,7 @@ let expectedMap ? null, script, }: - stdenvNoCC.mkDerivation (finalAttrs: { + stdenvNoCC.mkDerivation { __structuredAttrs = true; strictDeps = true; @@ -38,6 +38,6 @@ let description = "Tests for equality of bash arrays (including associative arrays)"; maintainers = [ maintainers.connorbaker ]; }; - }); + }; in makeOverridable testEqualArrayOrMap From 85bc4c60e02e729733fc3d3e8667a462a0f4609b Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Fri, 21 Feb 2025 10:22:09 -0800 Subject: [PATCH 11/16] testers.testEqualArrayOrMap: move argument check from bash to Nix --- .../testers/testEqualArrayOrMap/build-command.sh | 9 +++------ .../build-support/testers/testEqualArrayOrMap/tester.nix | 4 ++++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/build-command.sh b/pkgs/build-support/testers/testEqualArrayOrMap/build-command.sh index b1733692d1e9c..3ea8300c8c72e 100644 --- a/pkgs/build-support/testers/testEqualArrayOrMap/build-command.sh +++ b/pkgs/build-support/testers/testEqualArrayOrMap/build-command.sh @@ -2,13 +2,10 @@ set -eu -preScript() { - # If neither expectedArray nor expectedMap are declared, the test is meaningless. - if ! isDeclaredArray expectedArray && ! isDeclaredMap expectedMap; then - nixErrorLog "neither expectedArray nor expectedMap were set, so test is meaningless!" - exit 1 - fi +# NOTE: If neither expectedArray nor expectedMap are declared, the test is meaningless. +# This precondition is checked in the Nix expression through an assert. +preScript() { if isDeclaredArray valuesArray; then # shellcheck disable=SC2154 nixLog "using valuesArray: $(declare -p valuesArray)" diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix b/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix index 66dc8dd3f153a..76df1189c7267 100644 --- a/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix +++ b/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix @@ -5,6 +5,7 @@ }: let inherit (lib) maintainers; + inherit (lib.asserts) assertMsg; inherit (lib.customisation) makeOverridable; testEqualArrayOrMap = @@ -16,6 +17,9 @@ let expectedMap ? null, script, }: + assert assertMsg ( + expectedArray != null || expectedMap != null + ) "testEqualArrayOrMap: at least one of 'expectedArray' or 'expectedMap' must be provided"; stdenvNoCC.mkDerivation { __structuredAttrs = true; strictDeps = true; From f04ae03224b3cbc40719fa08ec908879c20cef9a Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Fri, 21 Feb 2025 12:10:17 -0800 Subject: [PATCH 12/16] testers.testEqualArrayOrMap: use default.nix and move recurseIntoAttrs into tests.nix --- pkgs/build-support/testers/default.nix | 2 +- pkgs/build-support/testers/test/default.nix | 2 +- .../testEqualArrayOrMap/{tester.nix => default.nix} | 0 pkgs/build-support/testers/testEqualArrayOrMap/tests.nix | 9 +++++++-- 4 files changed, 9 insertions(+), 4 deletions(-) rename pkgs/build-support/testers/testEqualArrayOrMap/{tester.nix => default.nix} (100%) diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index 453c66626fd0f..91bfdc0f137aa 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -68,7 +68,7 @@ # See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualArrayOrMap # or doc/build-helpers/testers.chapter.md # NOTE: Must be `import`-ed rather than `callPackage`-d to preserve the `override` attribute. - testEqualArrayOrMap = import ./testEqualArrayOrMap/tester.nix { inherit lib stdenvNoCC; }; + testEqualArrayOrMap = import ./testEqualArrayOrMap { inherit lib stdenvNoCC; }; # See https://nixos.org/manual/nixpkgs/unstable/#tester-testVersion # or doc/build-helpers/testers.chapter.md diff --git a/pkgs/build-support/testers/test/default.nix b/pkgs/build-support/testers/test/default.nix index 2c7f36bea0c29..94ca50ea76cbf 100644 --- a/pkgs/build-support/testers/test/default.nix +++ b/pkgs/build-support/testers/test/default.nix @@ -353,5 +353,5 @@ lib.recurseIntoAttrs { ''; }; - testEqualArrayOrMap = lib.recurseIntoAttrs (pkgs.callPackages ../testEqualArrayOrMap/tests.nix { }); + testEqualArrayOrMap = pkgs.callPackages ../testEqualArrayOrMap/tests.nix { }; } diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/tester.nix b/pkgs/build-support/testers/testEqualArrayOrMap/default.nix similarity index 100% rename from pkgs/build-support/testers/testEqualArrayOrMap/tester.nix rename to pkgs/build-support/testers/testEqualArrayOrMap/default.nix diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/tests.nix b/pkgs/build-support/testers/testEqualArrayOrMap/tests.nix index bcb2a176a4247..2202c8ab82d82 100644 --- a/pkgs/build-support/testers/testEqualArrayOrMap/tests.nix +++ b/pkgs/build-support/testers/testEqualArrayOrMap/tests.nix @@ -2,8 +2,13 @@ # `testers.testBuildFailure`. This is due to the fact that `testers.testBuildFailure` modifies the derivation such that # it produces an output containing the exit code, logs, and other things. Since `testers.runCommand` expects the empty # derivation, it produces a hash mismatch. -{ runCommand, testers, ... }: +{ + lib, + runCommand, + testers, +}: let + inherit (lib.attrsets) recurseIntoAttrs; inherit (testers) testEqualArrayOrMap testBuildFailure; concatValuesArrayToActualArray = '' nixLog "appending all values in valuesArray to actualArray" @@ -18,7 +23,7 @@ let done ''; in -{ +recurseIntoAttrs { # NOTE: This particular test is used in the docs: # See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualArrayOrMap # or doc/build-helpers/testers.chapter.md From c40211222cf671729968ee077ba5abdf52bb3b12 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Fri, 21 Feb 2025 12:11:56 -0800 Subject: [PATCH 13/16] testers.testBuildFailure': use default.nix and move recurseIntoAttrs into tests.nix --- pkgs/build-support/testers/default.nix | 2 +- .../testers/testBuildFailurePrime/{tester.nix => default.nix} | 0 pkgs/build-support/testers/testBuildFailurePrime/tests.nix | 4 +++- 3 files changed, 4 insertions(+), 2 deletions(-) rename pkgs/build-support/testers/testBuildFailurePrime/{tester.nix => default.nix} (100%) diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index 0fcbeff9f0d0f..4e0be62cc886c 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -37,7 +37,7 @@ # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime # or doc/build-helpers/testers.chapter.md # NOTE: Must be `import`-ed rather than `callPackage`-d to preserve the `override` attribute. - testBuildFailure' = import ./testBuildFailurePrime/tester.nix { inherit lib stdenvNoCC testers; }; + testBuildFailure' = import ./testBuildFailurePrime { inherit lib stdenvNoCC testers; }; # See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualDerivation # or doc/build-helpers/testers.chapter.md diff --git a/pkgs/build-support/testers/testBuildFailurePrime/tester.nix b/pkgs/build-support/testers/testBuildFailurePrime/default.nix similarity index 100% rename from pkgs/build-support/testers/testBuildFailurePrime/tester.nix rename to pkgs/build-support/testers/testBuildFailurePrime/default.nix diff --git a/pkgs/build-support/testers/testBuildFailurePrime/tests.nix b/pkgs/build-support/testers/testBuildFailurePrime/tests.nix index 1f1adadccb32f..2664f752cd89d 100644 --- a/pkgs/build-support/testers/testBuildFailurePrime/tests.nix +++ b/pkgs/build-support/testers/testBuildFailurePrime/tests.nix @@ -1,12 +1,14 @@ { emptyDirectory, hello, + lib, overrideStructuredAttrs, runCommand, stdenvNoCC, testers, }: let + inherit (lib.attrsets) recurseIntoAttrs; final = { # NOTE: This example is used in the docs. # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime @@ -144,4 +146,4 @@ let logNegativeTestStructuredAttrs = overrideStructuredAttrs true final.logNegativeTest; }; in -final +recurseIntoAttrs final From 2dac11821fcdd30403af2c7c627ff793ae5db631 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Tue, 25 Feb 2025 18:15:22 +0000 Subject: [PATCH 14/16] testers.testBuildFailure': create local bindings for loop variables to prevent lexical scoping mistakes --- .../build-support/testers/testBuildFailurePrime/build-command.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/build-support/testers/testBuildFailurePrime/build-command.sh b/pkgs/build-support/testers/testBuildFailurePrime/build-command.sh index f4747ff3f76c2..e11bd8a95f553 100644 --- a/pkgs/build-support/testers/testBuildFailurePrime/build-command.sh +++ b/pkgs/build-support/testers/testBuildFailurePrime/build-command.sh @@ -23,6 +23,7 @@ testBuilderLogEntries() { nixLog "checking original builder log" local -r builderLogEntries="$(<"${failed:?}/testBuildFailure.log")" local -i shouldExit=0 + local expectedBuilderLogEntry for expectedBuilderLogEntry in "${expectedBuilderLogEntries[@]}"; do if [[ ${builderLogEntries} == *"$expectedBuilderLogEntry"* ]]; then nixLog "original builder log contains ${expectedBuilderLogEntry@Q}" From a5b14368fa4a9da5adc0f6fb70ad0883cf996874 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Tue, 25 Feb 2025 22:41:28 +0000 Subject: [PATCH 15/16] testers.testEqualArrayOrMap: unwrap namerefs before passing to avoid nesting --- .../testers/testEqualArrayOrMap/assert-equal-array.sh | 4 ++-- .../testers/testEqualArrayOrMap/assert-equal-map.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-array.sh b/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-array.sh index 92fa981e44eeb..ef43dedba625f 100644 --- a/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-array.sh +++ b/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-array.sh @@ -18,12 +18,12 @@ assertEqualArray() { local -nr expectedArrayRef="$1" local -nr actualArrayRef="$2" - if ! isDeclaredArray expectedArrayRef; then + if ! isDeclaredArray "${!expectedArrayRef}"; then nixErrorLog "first arugment expectedArrayRef must be an array reference to a declared array" exit 1 fi - if ! isDeclaredArray actualArrayRef; then + if ! isDeclaredArray "${!actualArrayRef}"; then nixErrorLog "second arugment actualArrayRef must be an array reference to a declared array" exit 1 fi diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-map.sh b/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-map.sh index f66fab4f8468d..b601f9e424e9b 100644 --- a/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-map.sh +++ b/pkgs/build-support/testers/testEqualArrayOrMap/assert-equal-map.sh @@ -18,12 +18,12 @@ assertEqualMap() { local -nr expectedMapRef="$1" local -nr actualMapRef="$2" - if ! isDeclaredMap expectedMapRef; then + if ! isDeclaredMap "${!expectedMapRef}"; then nixErrorLog "first arugment expectedMapRef must be an associative array reference to a declared associative array" exit 1 fi - if ! isDeclaredMap actualMapRef; then + if ! isDeclaredMap "${!actualMapRef}"; then nixErrorLog "second arugment actualMapRef must be an associative array reference to a declared associative array" exit 1 fi From 49fecd727c51e9fd4e9f20b5d42831c9b4995da0 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Wed, 19 Feb 2025 15:51:19 -0800 Subject: [PATCH 16/16] test.testers.testEqualArrayOrMap: switch to testBuildFailure' --- .../testers/testEqualArrayOrMap/tests.nix | 204 ++++++------------ 1 file changed, 62 insertions(+), 142 deletions(-) diff --git a/pkgs/build-support/testers/testEqualArrayOrMap/tests.nix b/pkgs/build-support/testers/testEqualArrayOrMap/tests.nix index 2202c8ab82d82..5a382ded71385 100644 --- a/pkgs/build-support/testers/testEqualArrayOrMap/tests.nix +++ b/pkgs/build-support/testers/testEqualArrayOrMap/tests.nix @@ -2,14 +2,10 @@ # `testers.testBuildFailure`. This is due to the fact that `testers.testBuildFailure` modifies the derivation such that # it produces an output containing the exit code, logs, and other things. Since `testers.runCommand` expects the empty # derivation, it produces a hash mismatch. -{ - lib, - runCommand, - testers, -}: +{ lib, testers }: let inherit (lib.attrsets) recurseIntoAttrs; - inherit (testers) testEqualArrayOrMap testBuildFailure; + inherit (testers) testBuildFailure' testEqualArrayOrMap; concatValuesArrayToActualArray = '' nixLog "appending all values in valuesArray to actualArray" for value in "''${valuesArray[@]}"; do @@ -101,37 +97,18 @@ recurseIntoAttrs { # doing nothing ''; }; - array-missing-value = - let + array-missing-value = testBuildFailure' { + drv = testEqualArrayOrMap { name = "testEqualArrayOrMap-array-missing-value"; - failure = testEqualArrayOrMap { - name = "${name}-failure"; - valuesArray = [ "apple" ]; - expectedArray = [ ]; - script = concatValuesArrayToActualArray; - }; - in - runCommand name - { - failed = testBuildFailure failure; - passthru = { - inherit failure; - }; - } - '' - nixLog "Checking for exit code 1" - (( 1 == "$(cat "$failed/testBuildFailure.exit")" )) - nixLog "Checking for first error message" - grep -F \ - "ERROR: assertEqualArray: arrays differ in length: expectedArray has length 0 but actualArray has length 1" \ - "$failed/testBuildFailure.log" - nixLog "Checking for second error message" - grep -F \ - "ERROR: assertEqualArray: arrays differ at index 0: expectedArray has no such index but actualArray has value 'apple'" \ - "$failed/testBuildFailure.log" - nixLog "Test passed" - touch $out - ''; + valuesArray = [ "apple" ]; + expectedArray = [ ]; + script = concatValuesArrayToActualArray; + }; + expectedBuilderLogEntries = [ + "ERROR: assertEqualArray: arrays differ in length: expectedArray has length 0 but actualArray has length 1" + "ERROR: assertEqualArray: arrays differ at index 0: expectedArray has no such index but actualArray has value 'apple'" + ]; + }; map-insert = testEqualArrayOrMap { name = "testEqualArrayOrMap-map-insert"; valuesMap = { @@ -168,115 +145,58 @@ recurseIntoAttrs { unset 'actualMap[bee]' ''; }; - map-missing-key = - let + map-missing-key = testBuildFailure' { + drv = testEqualArrayOrMap { name = "testEqualArrayOrMap-map-missing-key"; - failure = testEqualArrayOrMap { - name = "${name}-failure"; - valuesMap = { - bee = "1"; - cat = "2"; - dog = "3"; - }; - expectedMap = { - apple = "0"; - bee = "1"; - cat = "2"; - dog = "3"; - }; - script = concatValuesMapToActualMap; + valuesMap = { + bee = "1"; + cat = "2"; + dog = "3"; }; - in - runCommand name - { - failed = testBuildFailure failure; - passthru = { - inherit failure; - }; - } - '' - nixLog "Checking for exit code 1" - (( 1 == "$(cat "$failed/testBuildFailure.exit")" )) - nixLog "Checking for first error message" - grep -F \ - "ERROR: assertEqualMap: maps differ in length: expectedMap has length 4 but actualMap has length 3" \ - "$failed/testBuildFailure.log" - nixLog "Checking for second error message" - grep -F \ - "ERROR: assertEqualMap: maps differ at key 'apple': expectedMap has value '0' but actualMap has no such key" \ - "$failed/testBuildFailure.log" - nixLog "Test passed" - touch $out - ''; - map-missing-key-with-empty = - let - name = "map-missing-key-with-empty"; - failure = testEqualArrayOrMap { - name = "${name}-failure"; - valuesArray = [ ]; - expectedMap.apple = 1; - script = ""; + expectedMap = { + apple = "0"; + bee = "1"; + cat = "2"; + dog = "3"; }; - in - runCommand name - { - failed = testBuildFailure failure; - passthru = { - inherit failure; - }; - } - '' - nixLog "Checking for exit code 1" - (( 1 == "$(cat "$failed/testBuildFailure.exit")" )) - nixLog "Checking for first error message" - grep -F \ - "ERROR: assertEqualMap: maps differ in length: expectedMap has length 1 but actualMap has length 0" \ - "$failed/testBuildFailure.log" - nixLog "Checking for second error message" - grep -F \ - "ERROR: assertEqualMap: maps differ at key 'apple': expectedMap has value '1' but actualMap has no such key" \ - "$failed/testBuildFailure.log" - nixLog "Test passed" - touch $out - ''; - map-extra-key = - let + script = concatValuesMapToActualMap; + }; + expectedBuilderLogEntries = [ + "ERROR: assertEqualMap: maps differ in length: expectedMap has length 4 but actualMap has length 3" + "ERROR: assertEqualMap: maps differ at key 'apple': expectedMap has value '0' but actualMap has no such key" + ]; + }; + map-missing-key-with-empty = testBuildFailure' { + drv = testEqualArrayOrMap { + name = "testEqualArrayOrMap-map-missing-key-with-empty"; + valuesArray = [ ]; + expectedMap.apple = 1; + script = ""; + }; + expectedBuilderLogEntries = [ + "ERROR: assertEqualMap: maps differ in length: expectedMap has length 1 but actualMap has length 0" + "ERROR: assertEqualMap: maps differ at key 'apple': expectedMap has value '1' but actualMap has no such key" + ]; + }; + map-extra-key = testBuildFailure' { + drv = testEqualArrayOrMap { name = "testEqualArrayOrMap-map-extra-key"; - failure = testEqualArrayOrMap { - name = "${name}-failure"; - valuesMap = { - apple = "0"; - bee = "1"; - cat = "2"; - dog = "3"; - }; - expectedMap = { - apple = "0"; - bee = "1"; - dog = "3"; - }; - script = concatValuesMapToActualMap; + valuesMap = { + apple = "0"; + bee = "1"; + cat = "2"; + dog = "3"; }; - in - runCommand - { - failed = testBuildFailure failure; - passthru = { - inherit failure; - }; - } - '' - nixLog "Checking for exit code 1" - (( 1 == "$(cat "$failed/testBuildFailure.exit")" )) - nixLog "Checking for first error message" - grep -F \ - "ERROR: assertEqualMap: maps differ in length: expectedMap has length 3 but actualMap has length 4" \ - "$failed/testBuildFailure.log" - nixLog "Checking for second error message" - grep -F \ - "ERROR: assertEqualMap: maps differ at key 'cat': expectedMap has no such key but actualMap has value '2'" \ - "$failed/testBuildFailure.log" - nixLog "Test passed" - touch $out - ''; + expectedMap = { + apple = "0"; + bee = "1"; + dog = "3"; + }; + script = concatValuesMapToActualMap; + }; + expectedBuilderLogEntries = [ + "ERROR: assertEqualMap: maps differ in length: expectedMap has length 3 but actualMap has length 4" + "ERROR: assertEqualMap: maps differ at key 'cat': expectedMap has no such key but actualMap has value '2'" + ]; + }; }