mk: Bootstrap from stable instead of snapshots

Makefile.in

@@ -214,13 +214,6 @@ include $(CFG_SRC_DIR)mk/debuggers.mk
 # Secondary makefiles, conditionalized for speed
 ######################################################################
 
-# Binary snapshots
-ifneq ($(strip $(findstring snap,$(MAKECMDGOALS))   \
-               $(findstring clean,$(MAKECMDGOALS))),)
-  CFG_INFO := $(info cfg: including snap rules)
-  include $(CFG_SRC_DIR)mk/snap.mk
-endif
-
 # The test suite
 ifneq ($(strip $(findstring check,$(MAKECMDGOALS)) \
                $(findstring test,$(MAKECMDGOALS))  \

mk/dist.mk

@@ -61,7 +61,7 @@ PKG_FILES := \
       rtstartup                                \
       rustllvm                                 \
       rustc                                    \
-      snapshots.txt                            \
+      stage0.txt                               \
       rust-installer                           \
       tools                                    \
       test)                                    \

mk/main.mk

@@ -34,6 +34,7 @@ CFG_FILENAME_EXTRA=$(shell printf '%s' $(CFG_RELEASE)$(CFG_EXTRA_FILENAME) | $(C
 # intentionally not "secure" by any definition, this is largely just a deterrent
 # from users enabling unstable features on the stable compiler.
 CFG_BOOTSTRAP_KEY=$(CFG_FILENAME_EXTRA)
+CFG_BOOTSTRAP_KEY_STAGE0=$(shell grep 'rustc_key' $(S)src/stage0.txt | sed 's/rustc_key: '//)
 
 ifeq ($(CFG_RELEASE_CHANNEL),stable)
 # This is the normal semver version string, e.g. "0.12.0", "0.12.0-nightly"

mk/reconfig.mk

@@ -38,6 +38,6 @@ else
 SREL_ROOT := $(SREL)
 endif
 
-config.stamp: $(S)configure $(S)Makefile.in $(S)src/snapshots.txt
+config.stamp: $(S)configure $(S)Makefile.in $(S)src/stage0.txt
 	@$(call E, cfg: reconfiguring)
 	$(SREL_ROOT)configure $(CFG_CONFIGURE_ARGS)

mk/stage0.mk

@@ -10,17 +10,14 @@ $(HLIB0_H_$(CFG_BUILD))/:
 endif
 
 $(SNAPSHOT_RUSTC_POST_CLEANUP): \
-		$(S)src/snapshots.txt \
-		$(S)src/etc/get-snapshot.py $(MKFILE_DEPS) \
+		$(S)src/stage0.txt \
+		$(S)src/etc/get-stage0.py $(MKFILE_DEPS) \
 		| $(HBIN0_H_$(CFG_BUILD))/
-
 	@$(call E, fetch: $@)
-#   Note: the variable "SNAPSHOT_FILE" is generally not set, and so
-#   we generally only pass one argument to this script.
 ifdef CFG_ENABLE_LOCAL_RUST
 	$(Q)$(S)src/etc/local_stage0.sh $(CFG_BUILD) $(CFG_LOCAL_RUST_ROOT) rustlib
 else
-	$(Q)$(CFG_PYTHON) $(S)src/etc/get-snapshot.py $(CFG_BUILD) $(SNAPSHOT_FILE)
+	$(Q)$(CFG_PYTHON) $(S)src/etc/get-stage0.py $(CFG_BUILD)
 endif
 	$(Q)if [ -e "$@" ]; then touch "$@"; else echo "ERROR: snapshot $@ not found"; exit 1; fi
 

mk/target.mk

@@ -65,6 +65,11 @@ $(foreach host,$(CFG_HOST), \
 # $(4) is the crate name
 define RUST_TARGET_STAGE_N
 
+ifeq ($(1),0)
+$$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): \
+	export RUSTC_BOOTSTRAP_KEY := $$(CFG_BOOTSTRAP_KEY_STAGE0)
+endif
+
 $$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): CFG_COMPILER_HOST_TRIPLE = $(2)
 $$(TLIB$(1)_T_$(2)_H_$(3))/stamp.$(4): \
 		$$(CRATEFILE_$(4)) \
@@ -113,6 +118,11 @@ endef
 # $(4) - name of the tool being built
 define TARGET_TOOL
 
+ifeq ($(1),0)
+$$(TBIN$(1)_T_$(2)_H_$(3))/$(4)$$(X_$(2)): \
+	export RUSTC_BOOTSTRAP_KEY := $$(CFG_BOOTSTRAP_KEY_STAGE0)
+endif
+
 $$(TBIN$(1)_T_$(2)_H_$(3))/$(4)$$(X_$(2)): \
 		$$(TOOL_SOURCE_$(4)) \
 		$$(TOOL_INPUTS_$(4)) \
@@ -167,11 +177,15 @@ SNAPSHOT_RUSTC_POST_CLEANUP=$(HBIN0_H_$(CFG_BUILD))/rustc$(X_$(CFG_BUILD))
 
 define TARGET_HOST_RULES
 
-$$(TLIB$(1)_T_$(2)_H_$(3))/:
+$$(TLIB$(1)_T_$(2)_H_$(3))/: $$(SNAPSHOT_RUSTC_POST_CLEANUP)
+	mkdir -p $$@
+
+$$(TBIN$(1)_T_$(2)_H_$(3))/: $$(SNAPSHOT_RUSTC_POST_CLEANUP)
 	mkdir -p $$@
 
 $$(TLIB$(1)_T_$(2)_H_$(3))/%: $$(RT_OUTPUT_DIR_$(2))/% \
-	    | $$(TLIB$(1)_T_$(2)_H_$(3))/ $$(SNAPSHOT_RUSTC_POST_CLEANUP)
+	    $$(SNAPSHOT_RUSTC_POST_CLEANUP) \
+	    | $$(TLIB$(1)_T_$(2)_H_$(3))/
 	@$$(call E, cp: $$@)
 	$$(Q)cp $$< $$@
 endef

mk/tests.mk

@@ -241,13 +241,16 @@ cleantestlibs:
 ######################################################################
 
 .PHONY: tidy
-tidy: $(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD))
+tidy: $(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD)) \
+		$(SNAPSHOT_RUSTC_POST_CLEANUP)
 	$(TARGET_RPATH_VAR0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) $< $(S)src
 
 $(HBIN0_H_$(CFG_BUILD))/tidy$(X_$(CFG_BUILD)): \
 		$(TSREQ0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) \
 		$(TLIB0_T_$(CFG_BUILD)_H_$(CFG_BUILD))/stamp.std \
-		$(call rwildcard,$(S)src/tools/tidy/src,*.rs)
+		$(call rwildcard,$(S)src/tools/tidy/src,*.rs) \
+		$(SNAPSHOT_RUSTC_POST_CLEANUP) | \
+		$(TLIB0_T_$(CFG_BUILD)_H_$(CFG_BUILD))
 	$(STAGE0_T_$(CFG_BUILD)_H_$(CFG_BUILD)) $(S)src/tools/tidy/src/main.rs \
 		--out-dir $(@D) --crate-name tidy
 

src/bootstrap/bootstrap.py

@@ -79,11 +79,22 @@ def run(args, verbose=False):
             raise RuntimeError(err)
         sys.exit(err)
 
+def stage0_data(rust_root):
+    nightlies = os.path.join(rust_root, "src/stage0.txt")
+    with open(nightlies, 'r') as nightlies:
+        data = {}
+        for line in nightlies.read().split("\n"):
+            if line.startswith("#") or line == '':
+                continue
+            a, b = line.split(": ", 1)
+            data[a] = b
+        return data
+
 class RustBuild:
-    def download_rust_nightly(self):
+    def download_stage0(self):
         cache_dst = os.path.join(self.build_dir, "cache")
-        rustc_cache = os.path.join(cache_dst, self.snap_rustc_date())
-        cargo_cache = os.path.join(cache_dst, self.snap_cargo_date())
+        rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
+        cargo_cache = os.path.join(cache_dst, self.stage0_cargo_date())
         if not os.path.exists(rustc_cache):
             os.makedirs(rustc_cache)
         if not os.path.exists(cargo_cache):
@@ -93,41 +104,49 @@ def download_rust_nightly(self):
            (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
             if os.path.exists(self.bin_root()):
                 shutil.rmtree(self.bin_root())
-            filename = "rust-std-nightly-" + self.build + ".tar.gz"
-            url = "https://static.rust-lang.org/dist/" + self.snap_rustc_date()
+            channel = self.stage0_rustc_channel()
+            filename = "rust-std-" + channel + "-" + self.build + ".tar.gz"
+            url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
             tarball = os.path.join(rustc_cache, filename)
             if not os.path.exists(tarball):
                 get(url + "/" + filename, tarball, verbose=self.verbose)
             unpack(tarball, self.bin_root(),
                    match="rust-std-" + self.build,
                    verbose=self.verbose)
 
-            filename = "rustc-nightly-" + self.build + ".tar.gz"
-            url = "https://static.rust-lang.org/dist/" + self.snap_rustc_date()
+            filename = "rustc-" + channel + "-" + self.build + ".tar.gz"
+            url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
             tarball = os.path.join(rustc_cache, filename)
             if not os.path.exists(tarball):
                 get(url + "/" + filename, tarball, verbose=self.verbose)
             unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
             with open(self.rustc_stamp(), 'w') as f:
-                f.write(self.snap_rustc_date())
+                f.write(self.stage0_rustc_date())
 
         if self.cargo().startswith(self.bin_root()) and \
            (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
-            filename = "cargo-nightly-" + self.build + ".tar.gz"
-            url = "https://static.rust-lang.org/cargo-dist/" + self.snap_cargo_date()
+            channel = self.stage0_cargo_channel()
+            filename = "cargo-" + channel + "-" + self.build + ".tar.gz"
+            url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date()
             tarball = os.path.join(cargo_cache, filename)
             if not os.path.exists(tarball):
                 get(url + "/" + filename, tarball, verbose=self.verbose)
             unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
             with open(self.cargo_stamp(), 'w') as f:
-                f.write(self.snap_cargo_date())
+                f.write(self.stage0_cargo_date())
 
-    def snap_cargo_date(self):
+    def stage0_cargo_date(self):
         return self._cargo_date
 
-    def snap_rustc_date(self):
+    def stage0_cargo_channel(self):
+        return self._cargo_channel
+
+    def stage0_rustc_date(self):
         return self._rustc_date
 
+    def stage0_rustc_channel(self):
+        return self._rustc_channel
+
     def rustc_stamp(self):
         return os.path.join(self.bin_root(), '.rustc-stamp')
 
@@ -138,13 +157,13 @@ def rustc_out_of_date(self):
         if not os.path.exists(self.rustc_stamp()):
             return True
         with open(self.rustc_stamp(), 'r') as f:
-            return self.snap_rustc_date() != f.read()
+            return self.stage0_rustc_date() != f.read()
 
     def cargo_out_of_date(self):
         if not os.path.exists(self.cargo_stamp()):
             return True
         with open(self.cargo_stamp(), 'r') as f:
-            return self.snap_cargo_date() != f.read()
+            return self.stage0_cargo_date() != f.read()
 
     def bin_root(self):
         return os.path.join(self.build_dir, self.build, "stage0")
@@ -187,15 +206,6 @@ def exe_suffix(self):
         else:
             return ''
 
-    def parse_nightly_dates(self):
-        nightlies = os.path.join(self.rust_root, "src/nightlies.txt")
-        with open(nightlies, 'r') as nightlies:
-            rustc, cargo = nightlies.read().split("\n")[:2]
-            assert rustc.startswith("rustc: ")
-            assert cargo.startswith("cargo: ")
-            self._rustc_date = rustc[len("rustc: "):]
-            self._cargo_date = cargo[len("cargo: "):]
-
     def build_bootstrap(self):
         env = os.environ.copy()
         env["CARGO_TARGET_DIR"] = os.path.join(self.build_dir, "bootstrap")
@@ -300,46 +310,53 @@ def build_triple(self):
 
         return cputype + '-' + ostype
 
-parser = argparse.ArgumentParser(description='Build rust')
-parser.add_argument('--config')
-parser.add_argument('-v', '--verbose', action='store_true')
-
-args = [a for a in sys.argv if a != '-h']
-args, _ = parser.parse_known_args(args)
-
-# Configure initial bootstrap
-rb = RustBuild()
-rb.config_toml = ''
-rb.config_mk = ''
-rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
-rb.build_dir = os.path.join(os.getcwd(), "build")
-rb.verbose = args.verbose
-
-try:
-    with open(args.config or 'config.toml') as config:
-        rb.config_toml = config.read()
-except:
-    pass
-try:
-    rb.config_mk = open('config.mk').read()
-except:
-    pass
-
-# Fetch/build the bootstrap
-rb.build = rb.build_triple()
-rb.parse_nightly_dates()
-rb.download_rust_nightly()
-sys.stdout.flush()
-rb.build_bootstrap()
-sys.stdout.flush()
-
-# Run the bootstrap
-args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
-args.append('--src')
-args.append(rb.rust_root)
-args.append('--build')
-args.append(rb.build)
-args.extend(sys.argv[1:])
-env = os.environ.copy()
-env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
-rb.run(args, env)
+def main():
+    parser = argparse.ArgumentParser(description='Build rust')
+    parser.add_argument('--config')
+    parser.add_argument('-v', '--verbose', action='store_true')
+
+    args = [a for a in sys.argv if a != '-h']
+    args, _ = parser.parse_known_args(args)
+
+    # Configure initial bootstrap
+    rb = RustBuild()
+    rb.config_toml = ''
+    rb.config_mk = ''
+    rb.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
+    rb.build_dir = os.path.join(os.getcwd(), "build")
+    rb.verbose = args.verbose
+
+    try:
+        with open(args.config or 'config.toml') as config:
+            rb.config_toml = config.read()
+    except:
+        pass
+    try:
+        rb.config_mk = open('config.mk').read()
+    except:
+        pass
+
+    data = stage0_data(rb.rust_root)
+    rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
+    rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)
+
+    # Fetch/build the bootstrap
+    rb.build = rb.build_triple()
+    rb.download_stage0()
+    sys.stdout.flush()
+    rb.build_bootstrap()
+    sys.stdout.flush()
+
+    # Run the bootstrap
+    args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")]
+    args.append('--src')
+    args.append(rb.rust_root)
+    args.append('--build')
+    args.append(rb.build)
+    args.extend(sys.argv[1:])
+    env = os.environ.copy()
+    env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
+    rb.run(args, env)
+
+if __name__ == '__main__':
+    main()

src/bootstrap/build/channel.rs

@@ -84,4 +84,12 @@ pub fn collect(build: &mut Build) {
     build.bootstrap_key = format!("{:02x}{:02x}{:02x}{:02x}",
                                   key[0], key[1], key[2], key[3]);
     env::set_var("RUSTC_BOOTSTRAP_KEY", &build.bootstrap_key);
+
+    let mut s = String::new();
+    t!(t!(File::open(build.src.join("src/stage0.txt"))).read_to_string(&mut s));
+    if let Some(line) = s.lines().find(|l| l.starts_with("rustc_key")) {
+        if let Some(key) = line.split(": ").nth(1) {
+            build.bootstrap_key_stage0 = key.to_string();
+        }
+    }
 }

src/bootstrap/build/compile.rs

@@ -179,7 +179,6 @@ pub fn rustc<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) {
          .env("CFG_VERSION", &build.version)
          .env("CFG_BOOTSTRAP_KEY", &build.bootstrap_key)
          .env("CFG_PREFIX", build.config.prefix.clone().unwrap_or(String::new()))
-         .env("RUSTC_BOOTSTRAP_KEY", &build.bootstrap_key)
          .env("CFG_LIBDIR_RELATIVE", "lib");
 
     if let Some(ref ver_date) = build.ver_date {