add Tarball Pins with optional support for flakes "Lockabel HTTP Tarball Protocol"

Author: pr2502

README.md

@@ -207,6 +207,7 @@ Commands:
   gitlab   Track a GitLab repository
   git      Track a git repository
   pypi     Track a package on PyPi
+  tarball  Track a tarball
   help     Print this message or the help of the given subcommand(s)
 
 Options:

src/cli.rs

@@ -62,9 +62,9 @@ pub struct ChannelAddOpts {
 }
 
 impl ChannelAddOpts {
-    pub fn add(&self) -> Result<(String, Pin)> {
+    pub fn add(&self) -> Result<(Option<String>, Pin)> {
         Ok((
-            self.name.clone(),
+            Some(self.name.clone()),
             channel::Pin {
                 name: self.name.clone(),
             }
@@ -119,9 +119,9 @@ pub struct GitHubAddOpts {
 }
 
 impl GitHubAddOpts {
-    pub fn add(&self) -> Result<(String, Pin)> {
+    pub fn add(&self) -> Result<(Option<String>, Pin)> {
         Ok((
-            self.repository.clone(),
+            Some(self.repository.clone()),
             match &self.more.branch {
                 Some(branch) => {
                     let pin = git::GitPin::github(
@@ -164,7 +164,7 @@ pub struct ForgejoAddOpts {
     pub more: GenericGitAddOpts,
 }
 impl ForgejoAddOpts {
-    pub fn add(&self) -> Result<(String, Pin)> {
+    pub fn add(&self) -> Result<(Option<String>, Pin)> {
         let server_url = Url::parse(&self.server).or_else(|err| match err {
             ParseError::RelativeUrlWithoutBase => {
                 Url::parse(&("https://".to_string() + self.server.as_str()))
@@ -173,7 +173,7 @@ impl ForgejoAddOpts {
         })?;
 
         Ok((
-            self.repository.clone(),
+            Some(self.repository.clone()),
             match &self.more.branch {
                 Some(branch) => {
                     let pin = git::GitPin::forgejo(
@@ -234,26 +234,26 @@ pub struct GitLabAddOpts {
 }
 
 impl GitLabAddOpts {
-    pub fn add(&self) -> Result<(String, Pin)> {
+    pub fn add(&self) -> Result<(Option<String>, Pin)> {
         Ok((
-            self.repo_path
+            Some(self.repo_path
                 .last()
                 .ok_or_else(|| anyhow::format_err!("GitLab repository path must at least have one element (usually two: owner, repo)"))?
-                .clone(),
+                .clone()),
             match &self.more.branch {
-                Some(branch) =>{
+                Some(branch) => {
                     let pin = git::GitPin::gitlab(
                         self.repo_path.join("/"),
                         branch.clone(),
                         Some(self.server.clone()),
                         self.private_token.clone(),
                         self.more.submodules,
                     );
-                    let version = self.more.at.as_ref()
-                    .map(|at| git::GitRevision {
+                    let version = self.more.at.as_ref().map(|at| git::GitRevision {
                         revision: at.clone(),
                     });
-                    (pin, version).into()},
+                    (pin, version).into()
+                },
                 None => {
                     let pin = git::GitReleasePin::gitlab(
                         self.repo_path.join("/"),
@@ -264,10 +264,9 @@ impl GitLabAddOpts {
                         self.more.release_prefix.clone(),
                         self.more.submodules,
                     );
-                    let version = self.more.at.as_ref()
-                        .map(|at| GenericVersion {
-                            version: at.clone(),
-                        });
+                    let version = self.more.at.as_ref().map(|at| GenericVersion {
+                        version: at.clone(),
+                    });
                     (pin, version).into()
                 },
             },
@@ -285,7 +284,7 @@ pub struct GitAddOpts {
 }
 
 impl GitAddOpts {
-    pub fn add(&self) -> Result<(String, Pin)> {
+    pub fn add(&self) -> Result<(Option<String>, Pin)> {
         let url = Url::parse(&self.url)
             .map_err(|e| {
                 match e {
@@ -311,7 +310,7 @@ impl GitAddOpts {
         let name = name.strip_suffix(".git").unwrap_or(&name);
 
         Ok((
-            name.to_owned(),
+            Some(name.to_owned()),
             match &self.more.branch {
                 Some(branch) => {
                     let pin = git::GitPin::git(url, branch.clone(), self.more.submodules);
@@ -354,8 +353,8 @@ pub struct PyPiAddOpts {
 }
 
 impl PyPiAddOpts {
-    pub fn add(&self) -> Result<(String, Pin)> {
-        Ok((self.name.clone(), {
+    pub fn add(&self) -> Result<(Option<String>, Pin)> {
+        Ok((Some(self.name.clone()), {
             let pin = pypi::Pin {
                 name: self.name.clone(),
                 version_upper_bound: self.version_upper_bound.clone(),
@@ -368,6 +367,19 @@ impl PyPiAddOpts {
     }
 }
 
+#[derive(Debug, Parser)]
+pub struct TarballAddOpts {
+    /// Tarball URL
+    pub url: Url,
+}
+
+impl TarballAddOpts {
+    pub fn add(&self) -> Result<(Option<String>, Pin)> {
+        let url = self.url.clone();
+        Ok((None, tarball::TarballPin { url }.into()))
+    }
+}
+
 #[derive(Debug, Subcommand)]
 pub enum AddCommands {
     /// Track a Nix channel
@@ -388,6 +400,12 @@ pub enum AddCommands {
     /// Track a package on PyPi
     #[command(name = "pypi")]
     PyPi(PyPiAddOpts),
+    /// Track a tarball
+    ///
+    /// This can be either a static URL that never changes its contents or a
+    /// URL which supports flakes "Lockable HTTP Tarball" API.
+    #[command(name = "tarball")]
+    Tarball(TarballAddOpts),
 }
 
 #[derive(Debug, Parser)]
@@ -415,20 +433,21 @@ impl AddOpts {
             AddCommands::Forgejo(fg) => fg.add()?,
             AddCommands::GitLab(gl) => gl.add()?,
             AddCommands::PyPi(p) => p.add()?,
+            AddCommands::Tarball(p) => p.add()?,
         };
 
-        let name = if let Some(ref n) = self.name {
-            n.clone()
-        } else {
-            name
+        let name = match (&self.name, name) {
+            (Some(user_specified), _) => user_specified.clone(),
+            (None, Some(guess_from_pin)) => guess_from_pin,
+            (None, None) => {
+                anyhow::bail!(
+                    "Couldn't pick a Pin name automatically. Use --name to specify one manually"
+                )
+            },
         };
         if self.frozen {
             pin.freeze();
         }
-        anyhow::ensure!(
-            !name.is_empty(),
-            "Pin name cannot be empty. Use --name to specify one manually",
-        );
 
         Ok((name, pin))
     }

src/default.nix

@@ -59,6 +59,8 @@ let
           mkPyPiSource spec
         else if spec.type == "Channel" then
           mkChannelSource spec
+        else if spec.type == "Tarball" then
+          mkTarballSource spec
         else
           builtins.throw "Unknown source type ${spec.type}";
     in
@@ -125,8 +127,20 @@ let
       inherit url;
       sha256 = hash;
     };
+
+  mkTarballSource =
+    {
+      url,
+      locked_url ? url,
+      hash,
+      ...
+    }:
+    builtins.fetchTarball {
+      url = locked_url;
+      sha256 = hash;
+    };
 in
-if version == 4 then
+if version == 5 then
   builtins.mapAttrs mkSource data.pins
 else
   throw "Unsupported format version ${toString version} in sources.json. Try running `npins upgrade`"

src/flake.rs

@@ -24,6 +24,7 @@ enum FlakeType {
     Github,
     Git,
     Path,
+    Tarball,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -52,6 +53,8 @@ pub struct FlakeOriginal {
     ref_: Option<String>,
     #[serde(rename = "type")]
     type_: String,
+    /// the url of a lockable tarball
+    url: Option<Url>,
 }
 
 impl FlakePin {
@@ -116,6 +119,13 @@ impl FlakePin {
                 )
                 .into()
             },
+            Tarball => {
+                let url = self
+                    .original
+                    .url
+                    .context("missing url on a tarball flake input")?;
+                tarball::TarballPin { url }.into()
+            },
             Path => anyhow::bail!("Path inputs are currently not supported by npins."),
         })
     }

src/main.rs

@@ -15,6 +15,7 @@ pub mod git;
 pub mod niv;
 pub mod nix;
 pub mod pypi;
+pub mod tarball;
 pub mod versions;
 
 /// Helper method to build you a client.
@@ -250,6 +251,7 @@ mkPin! {
     (GitRelease, git_release, "git release tag", git::GitReleasePin),
     (PyPi, pypi, "pypi package", pypi::Pin),
     (Channel, channel, "Nix channel", channel::Pin),
+    (Tarball, tarball, "tarball", tarball::TarballPin),
 }
 
 /// The main struct the CLI operates on
@@ -285,6 +287,17 @@ impl diff::Diff for GenericVersion {
     }
 }
 
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
+pub struct GenericHash {
+    pub hash: String,
+}
+
+impl diff::Diff for GenericHash {
+    fn properties(&self) -> Vec<(String, String)> {
+        vec![("hash".into(), self.hash.clone())]
+    }
+}
+
 /// The Frozen field in a Pin
 #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
 pub struct Frozen(pub bool);

src/tarball.rs

@@ -0,0 +1,97 @@
+//! Pin a tarball URL source
+//!
+//! Optionally (if the host supports it) can use the "Lockable HTTP Tarball Protocol" from flakes.
+//! Reference: <https://github.com/nixos/nix/blob/56763ff918eb308db23080e560ed2ea3e00c80a7/doc/manual/src/protocols/tarball-fetcher.md>
+
+use anyhow::{Context, Result};
+use reqwest::header::HeaderName;
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+use crate::*;
+
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
+pub struct TarballPin {
+    /// URL provided as user input
+    pub url: Url,
+}
+
+impl diff::Diff for TarballPin {
+    fn properties(&self) -> Vec<(String, String)> {
+        vec![("url".into(), self.url.to_string())]
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq, Hash)]
+pub struct LockedTarball {
+    /// If the given URL supports the Lockable Tarball Protocol we store the
+    /// flakeref here
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub locked_url: Option<Url>,
+}
+
+impl diff::Diff for LockedTarball {
+    fn properties(&self) -> Vec<(String, String)> {
+        self.locked_url
+            .iter()
+            .map(|locked_url| ("locked_url".into(), locked_url.to_string()))
+            .collect()
+    }
+}
+
+#[async_trait::async_trait]
+impl Updatable for TarballPin {
+    type Version = LockedTarball;
+    type Hashes = GenericHash;
+
+    async fn update(&self, old: Option<&LockedTarball>) -> Result<LockedTarball> {
+        const LINK: HeaderName = HeaderName::from_static("link");
+
+        // Attempt to use the Lockable HTTP Tarball Protocol, if that fails (the
+        // expected Link header is missing) we fail back to using whatever was
+        // the input.
+        let headers = build_client()?
+            .head(self.url.clone())
+            .send()
+            .await?
+            .headers()
+            .clone();
+        let flakerefs = headers
+            .get_all(LINK)
+            .into_iter()
+            .filter_map(|header| header.to_str().ok())
+            .filter_map(|link| {
+                // Naive parsing of the `Link: <flakeref>; rel="immutable"` header
+                link.strip_suffix(r#">; rel="immutable""#)?
+                    .strip_prefix("<")
+            })
+            .collect::<Vec<_>>();
+        let locked_url = if let [flakeref] = flakerefs[..] {
+            Some(
+                flakeref
+                    .parse::<Url>()
+                    .context("immutable link contained an invalid URL")?,
+            )
+        } else {
+            if matches!(old, Some(old) if old.locked_url.is_some()) {
+                log::warn!(
+                    "url `{url}` of a locked tarball pin did not respond with the expected `Link` header. \
+                     if you changed the `url` manually to one that doesn't support this protocol make sure to also remove the `locked_url` field. \
+                     https://docs.lix.systems/manual/lix/nightly/protocols/tarball-fetcher.html",
+                    url = &self.url,
+                );
+                return Ok(old.unwrap().clone());
+            } else {
+                // This is a no-op since we started with `old.locked_url.is_none()`
+                None
+            }
+        };
+        Ok(LockedTarball { locked_url })
+    }
+
+    async fn fetch(&self, version: &LockedTarball) -> Result<GenericHash> {
+        let url = version.locked_url.as_ref().unwrap_or(&self.url);
+        let hash = nix::nix_prefetch_tarball(&url).await?;
+        Ok(GenericHash { hash })
+    }
+}