diff --git a/doc/build-helpers/testers.chapter.md b/doc/build-helpers/testers.chapter.md index e0eb0cd1a5de6..74e70162ed3d5 100644 --- a/doc/build-helpers/testers.chapter.md +++ b/doc/build-helpers/testers.chapter.md @@ -255,6 +255,70 @@ runCommand "example" { ::: +## `testBuildFailure'` {#tester-testBuildFailurePrime} + +This tester wraps the functionality provided by [`testers.testBuildFailure`](#tester-testBuildFailure) to make writing checks easier by simplifying checking the exit code of the builder and asserting the existence of entries in the builder's log. +Additionally, users may specify a script containing additional checks, accessing the result of applying `testers.testBuildFailure` through the variable `failed`. + +NOTE: This tester will produce an empty output and exit with success if none of the checks fail; there is no need to `touch "$out"` in `script`. + +:::{.example #ex-testBuildFailurePrime-doc-example} + +# Check that a build fails, and verify the changes made during build + +Re-using the example from [`testers.testBuildFailure`](#ex-testBuildFailure-showingenvironmentchanges), we can see how common checks are made easier and remove the need for `runCommand`: + +```nix +testers.testBuildFailure' { + drv = runCommand "doc-example" { } '' + echo ok-ish >"$out" + echo failing though + exit 3 + ''; + expectedBuilderExitCode = 3; + expectedBuilderLogEntries = [ "failing though" ]; + script = '' + grep --silent -F 'ok-ish' "$failed/result" + ''; +} +``` + +::: + +### Inputs {#tester-testBuildFailurePrime-inputs} + +`drv` (derivation) + +: The failing derivation to wrap with `testBuildFailure`. + +`name` (string, optional) + +: The name of the test. + When not provided, this value defaults to `testBuildFailure-${(testers.testBuildFailure drv).name}`. + +`expectedBuilderExitCode` (integer, optional) + +: The expected exit code of the builder of `drv`. + When not provided, this value defaults to `1`. + +`expectedBuilderLogEntries` (array of string-like values, optional) + +: A list of string-like values which must be found in the builder's log by exact match. + When not provided, this value defaults to `[ ]`. + + NOTE: Patterns and regular expressions are not supported. + +`script` (string, optional) + +: A string containing additional checks to run. + When not provided, this value defaults to `""`. + The result of `testers.testBuildFailure drv` is available through the variable `failed`. + As an example, the builder's log is at `"$failed/testBuildFailure.log"`. + +### Return value {#tester-testBuildFailurePrime-return} + +The tester produces an empty output and only succeeds when the checks using `expectedBuilderExitCode`, `expectedBuilderLogEntries`, and `script` succeed. + ## `testEqualContents` {#tester-testEqualContents} Check that two paths have the same contents. diff --git a/doc/redirects.json b/doc/redirects.json index 4a830cb926b50..fe713d03384a0 100644 --- a/doc/redirects.json +++ b/doc/redirects.json @@ -8,6 +8,9 @@ "ex-build-helpers-extendMkDerivation": [ "index.html#ex-build-helpers-extendMkDerivation" ], + "ex-testBuildFailurePrime-doc-example": [ + "index.html#ex-testBuildFailurePrime-doc-example" + ], "neovim": [ "index.html#neovim" ], @@ -332,6 +335,15 @@ "footnote-stdenv-find-inputs-location.__back.0": [ "index.html#footnote-stdenv-find-inputs-location.__back.0" ], + "tester-testBuildFailurePrime": [ + "index.html#tester-testBuildFailurePrime" + ], + "tester-testBuildFailurePrime-inputs": [ + "index.html#tester-testBuildFailurePrime-inputs" + ], + "tester-testBuildFailurePrime-return": [ + "index.html#tester-testBuildFailurePrime-return" + ], "variables-specifying-dependencies": [ "index.html#variables-specifying-dependencies" ], diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index 0393895d53aff..9934e84d7b5e9 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -34,6 +34,10 @@ ] ++ orig.args or ["-e" ../../stdenv/generic/source-stdenv.sh (orig.builder or ../../stdenv/generic/default-builder.sh)]; }); + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime + # or doc/build-helpers/testers.chapter.md + testBuildFailure' = callPackage ./testBuildFailurePrime { }; + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualDerivation # or doc/build-helpers/testers.chapter.md testEqualDerivation = callPackage ./test-equal-derivation.nix { }; diff --git a/pkgs/build-support/testers/test/default.nix b/pkgs/build-support/testers/test/default.nix index b0f2b4c1d391a..7e4df128391da 100644 --- a/pkgs/build-support/testers/test/default.nix +++ b/pkgs/build-support/testers/test/default.nix @@ -220,6 +220,10 @@ lib.recurseIntoAttrs { sideEffectStructuredAttrs = overrideStructuredAttrs true sideEffects; }; + testBuildFailure' = lib.recurseIntoAttrs ( + pkgs.callPackages ../testBuildFailurePrime/tests.nix { inherit overrideStructuredAttrs; } + ); + testEqualContents = lib.recurseIntoAttrs { equalDir = testers.testEqualContents { assertion = "The same directory contents at different paths are recognized as equal"; diff --git a/pkgs/build-support/testers/testBuildFailurePrime/build-command.sh b/pkgs/build-support/testers/testBuildFailurePrime/build-command.sh new file mode 100644 index 0000000000000..e11bd8a95f553 --- /dev/null +++ b/pkgs/build-support/testers/testBuildFailurePrime/build-command.sh @@ -0,0 +1,47 @@ +# shellcheck shell=bash + +set -eu + +declare -ag preScriptHooks=(testBuilderExitCode) +# shellcheck disable=SC2154 +((${#expectedBuilderLogEntries[@]})) && preScriptHooks+=(testBuilderLogEntries) + +testBuilderExitCode() { + nixLog "checking original builder exit code" + local -ir builderExitCode=$(<"${failed:?}/testBuildFailure.exit") + # shellcheck disable=SC2154 + if ((expectedBuilderExitCode == builderExitCode)); then + nixLog "original builder exit code matches expected value of $expectedBuilderExitCode" + return 0 + else + nixErrorLog "original builder produced exit code $builderExitCode but was expected to produce $expectedBuilderExitCode" + return 1 + fi +} + +testBuilderLogEntries() { + nixLog "checking original builder log" + local -r builderLogEntries="$(<"${failed:?}/testBuildFailure.log")" + local -i shouldExit=0 + local expectedBuilderLogEntry + for expectedBuilderLogEntry in "${expectedBuilderLogEntries[@]}"; do + if [[ ${builderLogEntries} == *"$expectedBuilderLogEntry"* ]]; then + nixLog "original builder log contains ${expectedBuilderLogEntry@Q}" + else + nixErrorLog "original builder log does not contain ${expectedBuilderLogEntry@Q}" + shouldExit=1 + fi + done + return $shouldExit +} + +scriptPhase() { + runHook preScript + + runHook script + + runHook postScript +} + +runHook scriptPhase +touch "${out:?}" diff --git a/pkgs/build-support/testers/testBuildFailurePrime/default.nix b/pkgs/build-support/testers/testBuildFailurePrime/default.nix new file mode 100644 index 0000000000000..260ab5c0c5eba --- /dev/null +++ b/pkgs/build-support/testers/testBuildFailurePrime/default.nix @@ -0,0 +1,37 @@ +{ + lib, + stdenvNoCC, + testers, +}: +# See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime +# or doc/build-helpers/testers.chapter.md +lib.makeOverridable ( + { + drv, + name ? "testBuildFailure-${drv.name}", + expectedBuilderExitCode ? 1, + expectedBuilderLogEntries ? [ ], + script ? "", + }: + stdenvNoCC.mkDerivation (finalAttrs: { + __structuredAttrs = true; + strictDeps = true; + + inherit name; + + nativeBuildInputs = [ finalAttrs.failed ]; + + failed = testers.testBuildFailure drv; + + inherit expectedBuilderExitCode expectedBuilderLogEntries; + + inherit script; + + buildCommandPath = ./build-command.sh; + + meta = { + description = "A wrapper around testers.testBuildFailure to simplify common use cases"; + maintainers = [ lib.maintainers.connorbaker ]; + }; + }) +) diff --git a/pkgs/build-support/testers/testBuildFailurePrime/tests.nix b/pkgs/build-support/testers/testBuildFailurePrime/tests.nix new file mode 100644 index 0000000000000..2664f752cd89d --- /dev/null +++ b/pkgs/build-support/testers/testBuildFailurePrime/tests.nix @@ -0,0 +1,149 @@ +{ + emptyDirectory, + hello, + lib, + overrideStructuredAttrs, + runCommand, + stdenvNoCC, + testers, +}: +let + inherit (lib.attrsets) recurseIntoAttrs; + final = { + # NOTE: This example is used in the docs. + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime + # or doc/build-helpers/testers.chapter.md + doc-example = testers.testBuildFailure' { + drv = runCommand "doc-example" { } '' + echo ok-ish >"$out" + echo failing though + exit 3 + ''; + expectedBuilderExitCode = 3; + expectedBuilderLogEntries = [ "failing though" ]; + script = '' + grep --silent -F 'ok-ish' "$failed/result" + ''; + }; + + happy = testers.testBuildFailure' { + drv = runCommand "happy" { } '' + echo ok-ish >$out + + echo failing though + echo also stderr 1>&2 + echo 'line\nwith-\bbackslashes' + printf "incomplete line - no newline" + + exit 3 + ''; + expectedBuilderExitCode = 3; + expectedBuilderLogEntries = [ + "failing though" + "also stderr" + ''line\nwith-\bbackslashes'' + "incomplete line - no newline" + ]; + script = '' + grep --silent -F 'ok-ish' "$failed/result" + ''; + }; + + happyStructuredAttrs = overrideStructuredAttrs true final.happy; + + helloDoesNotFail = testers.testBuildFailure' { + drv = testers.testBuildFailure hello; + expectedBuilderLogEntries = [ + "testBuildFailure: The builder did not fail, but a failure was expected" + ]; + }; + + multiOutput = testers.testBuildFailure' { + drv = + runCommand "multiOutput" + { + # dev will be the default output + outputs = [ + "dev" + "doc" + "out" + ]; + } + '' + echo i am failing + exit 1 + ''; + expectedBuilderLogEntries = [ + "i am failing" + ]; + script = '' + # Checking our note that dev is the default output + echo $failed/_ | grep -- '-dev/_' >/dev/null + echo 'All good.' + ''; + }; + + multiOutputStructuredAttrs = overrideStructuredAttrs true final.multiOutput; + + sideEffects = testers.testBuildFailure' { + drv = stdenvNoCC.mkDerivation { + name = "fail-with-side-effects"; + src = emptyDirectory; + + postHook = '' + echo touching side-effect... + # Assert that the side-effect doesn't exist yet... + # We're checking that this hook isn't run by expect-failure.sh + if [[ -e side-effect ]]; then + echo "side-effect already exists" + exit 1 + fi + touch side-effect + ''; + + buildPhase = '' + echo i am failing + exit 1 + ''; + }; + expectedBuilderLogEntries = [ + "touching side-effect..." + "i am failing" + ]; + script = '' + [[ ! -e side-effect ]] + ''; + }; + + sideEffectsStructuredAttrs = overrideStructuredAttrs true final.sideEffects; + + exitCodeNegativeTest = testers.testBuildFailure' { + drv = testers.testBuildFailure' { + drv = runCommand "exit-code" { } "exit 3"; + # Default expected exit code is 1 + }; + expectedBuilderLogEntries = [ + "ERROR: testBuilderExitCode: original builder produced exit code 3 but was expected to produce 1" + ]; + }; + + exitCodeNegativeTestStructuredAttrs = overrideStructuredAttrs true final.exitCodeNegativeTest; + + logNegativeTest = testers.testBuildFailure' { + drv = testers.testBuildFailure' { + drv = runCommand "exit-code" { } '' + nixLog "apples" + exit 3 + ''; + expectedBuilderExitCode = 3; + expectedBuilderLogEntries = [ "bees" ]; + }; + expectedBuilderLogEntries = [ + "ERROR: testBuilderLogEntries: original builder log does not contain 'bees'" + ]; + }; + + logNegativeTestStructuredAttrs = overrideStructuredAttrs true final.logNegativeTest; + }; +in +recurseIntoAttrs final