Open source rewinding integration tests.

Some things to note:

* Tests that compile CPP are skipped when running in bazel mode because we do not yet have `BuildIntegrationTestCase` infrastructure for that. These are still run within Google as part of `RewindingTest`.
* Some test cases exercise rewinding of an action running concurrently with another action that consumes its outputs. This doesn't work with execution strategies (including `standalone`) which rely on local disk for action inputs - an artifact may be unexpectedly deleted at the wrong time. To work around this, tests which exercise this behavior set `--jobs=1`, eliminating execution phase parallelism. These tests still run with a typical value for `--jobs` within Google since the default execution strategy replicates artifacts remotely.
* There are still no execution strategies in bazel which throw `LostInputsExecException`. Tests inject this behavior using `SpawnController`. So while the tests verify that skyframe executes rewinding correctly, rewinding cannot currently be triggered in bazel.

PiperOrigin-RevId: 456785344
Change-Id: Ie202a9a141d3aa9b0a7c6b525200d59f325fc05d

src/main/java/com/google/devtools/build/lib/skyframe/proto/BUILD

@@ -11,7 +11,10 @@ proto_library(
 
 java_proto_library(
     name = "action_rewind_event_java_proto",
-    visibility = ["//src/main/java/com/google/devtools/build/lib:__subpackages__"],
+    visibility = [
+        "//src/main/java/com/google/devtools/build/lib:__subpackages__",
+        "//src/test/java/com/google/devtools/build/lib/testutil:__pkg__",
+    ],
     deps = [":action_rewind_event"],
 )
 

src/test/java/com/google/devtools/build/lib/buildtool/util/BuildIntegrationTestCase.java

@@ -244,6 +244,10 @@ public final void restoreUncaughtExceptionHandler() {
     Thread.setDefaultUncaughtExceptionHandler(oldExceptionHandler);
   }
 
+  public final BlazeRuntimeWrapper getRuntimeWrapper() {
+    return runtimeWrapper;
+  }
+
   /**
    * Creates an uncaught exception handler to be used in {@link
    * Thread#setDefaultUncaughtExceptionHandler}.
@@ -298,8 +302,7 @@ protected final RecordingBugReporter recordBugReportsAndReinitialize() throws Ex
    *
    * <p>The server is reinitialized so that this change is picked up.
    */
-  protected final void setCustomBugReporterAndReinitialize(BugReporter bugReporter)
-      throws Exception {
+  public final void setCustomBugReporterAndReinitialize(BugReporter bugReporter) throws Exception {
     this.bugReporter = checkNotNull(bugReporter);
     reinitializeAndPreserveOptions();
   }
@@ -365,7 +368,7 @@ private static void bestEffortDeleteTreesBelow(Path path, String canSkip) throws
    * <p>{@link BugReport} stores information about crashes in a static variable when running tests.
    * Tests which deliberately cause crashes, need to clear that flag not to taint the environment.
    */
-  protected static void assertAndClearBugReporterStoredCrash(Class<? extends Throwable> expected) {
+  public static void assertAndClearBugReporterStoredCrash(Class<? extends Throwable> expected) {
     assertThrows(expected, BugReport::maybePropagateUnprocessedThrowableIfInTest);
   }
 
@@ -578,7 +581,7 @@ protected void resetOptions() {
     runtimeWrapper.resetOptions();
   }
 
-  protected void addOptions(String... args) {
+  public void addOptions(String... args) {
     runtimeWrapper.addOptions(args);
   }
 
@@ -995,10 +998,8 @@ protected ImmutableList<String> getLabelsOfAnalyzedTests() {
         .collect(toImmutableList());
   }
 
-  /**
-   * Assertion-checks that the expected error was reported,
-   */
-  protected void assertContainsError(String expectedError) {
+  /** Assertion-checks that the expected error was reported, */
+  public final void assertContainsError(String expectedError) {
     for (Event error : events.errors()) {
       if (error.getMessage().contains(expectedError)) {
         return;

src/test/java/com/google/devtools/build/lib/skyframe/BUILD

@@ -8,7 +8,7 @@ package(
 filegroup(
     name = "srcs",
     testonly = 0,
-    srcs = glob(["**"]),
+    srcs = glob(["**"]) + ["//src/test/java/com/google/devtools/build/lib/skyframe/rewinding:srcs"],
     visibility = ["//src/test/java/com/google/devtools/build/lib:__pkg__"],
 )
 

src/test/java/com/google/devtools/build/lib/skyframe/rewinding/BUILD

@@ -0,0 +1,73 @@
+load("@rules_java//java:defs.bzl", "java_library", "java_test")
+
+package(
+    default_testonly = True,
+    default_visibility = ["//src:__subpackages__"],
+)
+
+licenses(["notice"])
+
+filegroup(
+    name = "srcs",
+    testonly = False,
+    srcs = glob(["**"]),
+    visibility = ["//src:__subpackages__"],
+)
+
+java_library(
+    name = "rewinding_tests_helper",
+    srcs = ["RewindingTestsHelper.java"],
+    data = ["//src/test/java/com/google/devtools/build/lib:embedded_scripts"],
+    deps = [
+        "//src/main/java/com/google/devtools/build/lib:keep-going-option",
+        "//src/main/java/com/google/devtools/build/lib:runtime",
+        "//src/main/java/com/google/devtools/build/lib/actions",
+        "//src/main/java/com/google/devtools/build/lib/actions:action_lookup_data",
+        "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
+        "//src/main/java/com/google/devtools/build/lib/actions:middleman_type",
+        "//src/main/java/com/google/devtools/build/lib/analysis:config/core_options",
+        "//src/main/java/com/google/devtools/build/lib/bugreport",
+        "//src/main/java/com/google/devtools/build/lib/cmdline",
+        "//src/main/java/com/google/devtools/build/lib/exec:spawn_exec_exception",
+        "//src/main/java/com/google/devtools/build/lib/skyframe:artifact_nested_set_key",
+        "//src/main/java/com/google/devtools/build/lib/skyframe/rewinding",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/skyframe",
+        "//src/main/java/com/google/devtools/build/skyframe:graph_inconsistency_java_proto",
+        "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
+        "//src/main/protobuf:failure_details_java_proto",
+        "//src/test/java/com/google/devtools/build/lib/buildtool/util",
+        "//src/test/java/com/google/devtools/build/lib/testutil:TestConstants",
+        "//src/test/java/com/google/devtools/build/lib/testutil:TestUtils",
+        "//src/test/java/com/google/devtools/build/lib/testutil:action_event_recorder",
+        "//src/test/java/com/google/devtools/build/lib/testutil:controllable_action_strategy_module",
+        "//src/test/java/com/google/devtools/build/lib/testutil:spawn_controller",
+        "//src/test/java/com/google/devtools/build/lib/testutil:spawn_input_utils",
+        "//src/test/java/com/google/devtools/build/skyframe:testutil",
+        "//third_party:error_prone_annotations",
+        "//third_party:guava",
+        "//third_party:junit4",
+        "//third_party:mockito",
+        "//third_party:truth",
+    ],
+)
+
+java_test(
+    name = "RewindingTest",
+    srcs = ["RewindingTest.java"],
+    shard_count = 6,
+    tags = ["no_windows"],  # BuildIntegrationTestCase isn't fully compatible with Windows.
+    deps = [
+        ":rewinding_tests_helper",
+        "//src/main/java/com/google/devtools/build/lib:runtime",
+        "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
+        "//src/main/java/com/google/devtools/build/lib/includescanning",
+        "//src/test/java/com/google/devtools/build/lib/analysis/util",
+        "//src/test/java/com/google/devtools/build/lib/buildtool/util",
+        "//src/test/java/com/google/devtools/build/lib/testutil:action_event_recorder",
+        "//third_party:guava",
+        "//third_party:junit4",
+        "//third_party:truth",
+        "@com_google_testparameterinjector//:testparameterinjector",
+    ],
+)

src/test/java/com/google/devtools/build/lib/skyframe/rewinding/RewindingTest.java

@@ -0,0 +1,214 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe.rewinding;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.TruthJUnit.assume;
+
+import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.util.AnalysisMock;
+import com.google.devtools.build.lib.buildtool.util.BuildIntegrationTestCase;
+import com.google.devtools.build.lib.includescanning.IncludeScanningModule;
+import com.google.devtools.build.lib.runtime.BlazeRuntime;
+import com.google.devtools.build.lib.testutil.ActionEventRecorder;
+import com.google.testing.junit.testparameterinjector.TestParameter;
+import com.google.testing.junit.testparameterinjector.TestParameterInjector;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Tests for action rewinding on non-incremental builds. */
+// TODO(b/228090759): Add back actionFromPreviousBuildReevaluated when incrementality is supported.
+@RunWith(TestParameterInjector.class)
+public final class RewindingTest extends BuildIntegrationTestCase {
+
+  @TestParameter private boolean keepGoing;
+
+  private final ActionEventRecorder actionEventRecorder = new ActionEventRecorder();
+  private final RewindingTestsHelper helper = new RewindingTestsHelper(this, actionEventRecorder);
+
+  @Override
+  protected BlazeRuntime.Builder getRuntimeBuilder() throws Exception {
+    return super.getRuntimeBuilder()
+        .addBlazeModule(new IncludeScanningModule())
+        .addBlazeModule(helper.makeControllableActionStrategyModule("standalone"));
+  }
+
+  @Override
+  protected void setupOptions() throws Exception {
+    super.setupOptions();
+    addOptions(
+        "--spawn_strategy=standalone",
+        "--notrack_incremental_state",
+        "--nouse_action_cache",
+        "--rewind_lost_inputs",
+        "--features=cc_include_scanning",
+        "--experimental_remote_include_extraction_size_threshold=0",
+        "--keep_going=" + keepGoing);
+    runtimeWrapper.registerSubscriber(actionEventRecorder);
+  }
+
+  /**
+   * Skips test cases that cannot run with bazel.
+   *
+   * <p>{@link BuildIntegrationTestCase} currently does not support CPP compilation on bazel.
+   */
+  // TODO(b/195425240): Remove once CPP compilation on bazel is supported. Assumptions that
+  // generated headers are always under k8-opt will need to be relaxed to support other platforms.
+  private static void skipIfBazel() {
+    assume().that(AnalysisMock.get().isThisBazel()).isFalse();
+  }
+
+  @Test
+  public void noLossSmokeTest() throws Exception {
+    helper.runNoLossSmokeTest();
+  }
+
+  @Test
+  public void buildingParentFoundUndoneChildNotToleratedWithoutRewinding() throws Exception {
+    helper.runBuildingParentFoundUndoneChildNotToleratedWithoutRewinding();
+  }
+
+  @Test
+  public void dependentActionsReevaluated() throws Exception {
+    helper.runDependentActionsReevaluated_spawnFailed();
+  }
+
+  @Test
+  public void inputDiscoveringActionNoticesMissingDep() throws Exception {
+    skipIfBazel();
+    helper.runInputDiscoveringActionNoticesMissingDep();
+  }
+
+  @Test
+  public void multipleLostInputsForRewindPlan() throws Exception {
+    helper.runMultipleLostInputsForRewindPlan();
+  }
+
+  @Test
+  public void multiplyLosingInputsFails() throws Exception {
+    helper.runMultiplyLosingInputsFails();
+    assertOutputForRule2NotCreated();
+  }
+
+  @Test
+  public void interruptedDuringRewindStopsNormally() throws Exception {
+    helper.runInterruptedDuringRewindStopsNormally();
+    assertOutputForRule2NotCreated();
+  }
+
+  @Test
+  public void failureDuringRewindStopsNormally() throws Exception {
+    helper.runFailureDuringRewindStopsNormally();
+    assertOutputForRule2NotCreated();
+  }
+
+  /**
+   * Because this test infrastructure allows builds to write outputs to the filesystem, these
+   * "fail"/"stops normally" tests can assert that the build's output file was not written.
+   */
+  private void assertOutputForRule2NotCreated() throws Exception {
+    Artifact output =
+        Iterables.getOnlyElement(
+            getFilesToBuild(getExistingConfiguredTarget("//test:rule2")).toList());
+    assertThat(output.getPath().exists()).isFalse();
+  }
+
+  @Test
+  public void intermediateActionRewound() throws Exception {
+    helper.runIntermediateActionRewound();
+  }
+
+  @Test
+  public void chainOfActionsRewound() throws Exception {
+    helper.runChainOfActionsRewound();
+  }
+
+  @Test
+  public void nondeterministicActionRewound() throws Exception {
+    helper.runNondeterministicActionRewound();
+  }
+
+  @Test
+  public void parallelTrackSharedActionsRewound() throws Exception {
+    helper.runParallelTrackSharedActionsRewound();
+  }
+
+  @Test
+  public void treeFileArtifactRewound() throws Exception {
+    skipIfBazel();
+    helper.runTreeFileArtifactRewound_spawnFailed();
+  }
+
+  @Test
+  public void treeArtifactRewound_allFilesLost() throws Exception {
+    skipIfBazel();
+    helper.runTreeArtifactRewound_allFilesLost_spawnFailed();
+  }
+
+  @Test
+  public void treeArtifactRewound_oneFileLost() throws Exception {
+    skipIfBazel();
+    helper.runTreeArtifactRewound_oneFileLost_spawnFailed();
+  }
+
+  @Test
+  public void generatedRunfilesRewound_allFilesLost() throws Exception {
+    helper.runGeneratedRunfilesRewound_allFilesLost_spawnFailed();
+  }
+
+  @Test
+  public void generatedRunfilesRewound_oneFileLost() throws Exception {
+    helper.runGeneratedRunfilesRewound_oneFileLost_spawnFailed();
+  }
+
+  @Test
+  public void dupeDirectAndRunfilesDependencyRewound() throws Exception {
+    helper.runDupeDirectAndRunfilesDependencyRewound_spawnFailed();
+  }
+
+  @Test
+  public void treeInRunfilesRewound() throws Exception {
+    helper.runTreeInRunfilesRewound_spawnFailed();
+  }
+
+  @Test
+  public void inputsFromSameGeneratingActionSplitAmongNestedSetChildren() throws Exception {
+    helper.runInputsFromSameGeneratingActionSplitAmongNestedSetChildren();
+  }
+
+  @Test
+  public void generatedHeaderRewound_lostInInputDiscovery() throws Exception {
+    skipIfBazel();
+    helper.runGeneratedHeaderRewound_lostInInputDiscovery_spawnFailed();
+  }
+
+  @Test
+  public void generatedHeaderRewound_lostInActionExecution() throws Exception {
+    skipIfBazel();
+    helper.runGeneratedHeaderRewound_lostInActionExecution_spawnFailed();
+  }
+
+  @Test
+  public void generatedTransitiveHeaderRewound_lostInInputDiscovery() throws Exception {
+    skipIfBazel();
+    helper.runGeneratedTransitiveHeaderRewound_lostInInputDiscovery_spawnFailed();
+  }
+
+  @Test
+  public void generatedTransitiveHeaderRewound_lostInActionExecution() throws Exception {
+    skipIfBazel();
+    helper.runGeneratedTransitiveHeaderRewound_lostInActionExecution_spawnFailed();
+  }
+}