Allow local index references in requirements.txt files (#4525)

## Summary We currently accept `--index-url /path/to/index` on the command line, but confusingly, not in `requirements.txt`. This PR just brings the two in sync. ## Test Plan New snapshot tests.
astral-sh · Jun 25, 2024 · 904957b · 904957b
1 parent e39f5f7
commit 904957b
Show file tree

Hide file tree

Showing 4 changed files with 212 additions and 35 deletions.
diff --git a/crates/distribution-types/src/index_url.rs b/crates/distribution-types/src/index_url.rs
@@ -100,32 +100,18 @@ pub enum IndexUrlError {
     Url(#[from] ParseError),
     #[error(transparent)]
     VerbatimUrl(#[from] VerbatimUrlError),
-    #[error("Index URL must be a valid base URL")]
-    CannotBeABase,
 }
 
 impl FromStr for IndexUrl {
     type Err = IndexUrlError;
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        if let Ok(path) = Path::new(s).canonicalize() {
-            let url = VerbatimUrl::from_path(path)?.with_given(s.to_owned());
-            Ok(Self::Path(url))
+        let url = if let Ok(path) = Path::new(s).canonicalize() {
+            VerbatimUrl::from_path(path)?
         } else {
-            let url = Url::parse(s)?;
-            if url.cannot_be_a_base() {
-                Err(IndexUrlError::CannotBeABase)
-            } else {
-                let url = VerbatimUrl::from_url(url).with_given(s.to_owned());
-                if *url.raw() == *PYPI_URL {
-                    Ok(Self::Pypi(url))
-                } else if url.scheme() == "file" {
-                    Ok(Self::Path(url))
-                } else {
-                    Ok(Self::Url(url))
-                }
-            }
-        }
+            VerbatimUrl::parse_url(s)?
+        };
+        Ok(Self::from(url.with_given(s)))
     }
 }
 
@@ -150,7 +136,9 @@ impl<'de> serde::de::Deserialize<'de> for IndexUrl {
 
 impl From<VerbatimUrl> for IndexUrl {
     fn from(url: VerbatimUrl) -> Self {
-        if *url.raw() == *PYPI_URL {
+        if url.scheme() == "file" {
+            Self::Path(url)
+        } else if *url.raw() == *PYPI_URL {
             Self::Pypi(url)
         } else {
             Self::Url(url)

diff --git a/crates/requirements-txt/src/lib.rs b/crates/requirements-txt/src/lib.rs
@@ -550,27 +550,45 @@ fn parse_entry(
     } else if s.eat_if("-i") || s.eat_if("--index-url") {
         let given = parse_value(content, s, |c: char| !['\n', '\r', '#'].contains(&c))?;
         let expanded = expand_env_vars(given);
-        let url = VerbatimUrl::parse_url(expanded.as_ref())
-            .map(|url| url.with_given(given.to_owned()))
-            .map_err(|err| RequirementsTxtParserError::Url {
+        let url = if let Ok(path) = Path::new(expanded.as_ref()).canonicalize() {
+            VerbatimUrl::from_path(path).map_err(|err| RequirementsTxtParserError::VerbatimUrl {
                 source: err,
                 url: given.to_string(),
                 start,
                 end: s.cursor(),
-            })?;
-        RequirementsTxtStatement::IndexUrl(url)
+            })?
+        } else {
+            VerbatimUrl::parse_url(expanded.as_ref()).map_err(|err| {
+                RequirementsTxtParserError::Url {
+                    source: err,
+                    url: given.to_string(),
+                    start,
+                    end: s.cursor(),
+                }
+            })?
+        };
+        RequirementsTxtStatement::IndexUrl(url.with_given(given))
     } else if s.eat_if("--extra-index-url") {
         let given = parse_value(content, s, |c: char| !['\n', '\r', '#'].contains(&c))?;
         let expanded = expand_env_vars(given);
-        let url = VerbatimUrl::parse_url(expanded.as_ref())
-            .map(|url| url.with_given(given.to_owned()))
-            .map_err(|err| RequirementsTxtParserError::Url {
+        let url = if let Ok(path) = Path::new(expanded.as_ref()).canonicalize() {
+            VerbatimUrl::from_path(path).map_err(|err| RequirementsTxtParserError::VerbatimUrl {
                 source: err,
                 url: given.to_string(),
                 start,
                 end: s.cursor(),
-            })?;
-        RequirementsTxtStatement::ExtraIndexUrl(url)
+            })?
+        } else {
+            VerbatimUrl::parse_url(expanded.as_ref()).map_err(|err| {
+                RequirementsTxtParserError::Url {
+                    source: err,
+                    url: given.to_string(),
+                    start,
+                    end: s.cursor(),
+                }
+            })?
+        };
+        RequirementsTxtStatement::ExtraIndexUrl(url.with_given(given))
     } else if s.eat_if("--no-index") {
         RequirementsTxtStatement::NoIndex
     } else if s.eat_if("--find-links") || s.eat_if("-f") {
@@ -856,6 +874,8 @@ pub enum RequirementsTxtParserError {
     VerbatimUrl {
         source: pep508_rs::VerbatimUrlError,
         url: String,
+        start: usize,
+        end: usize,
     },
     UrlConversion(String),
     UnsupportedUrl(String),
@@ -923,8 +943,8 @@ impl Display for RequirementsTxtParserError {
             Self::FileUrl { url, start, .. } => {
                 write!(f, "Invalid file URL at position {start}: `{url}`")
             }
-            Self::VerbatimUrl { source, url } => {
-                write!(f, "Invalid URL: `{url}`: {source}")
+            Self::VerbatimUrl { url, start, .. } => {
+                write!(f, "Invalid URL at position {start}: `{url}`")
             }
             Self::UrlConversion(given) => {
                 write!(f, "Unable to convert URL to path: {given}")
@@ -1025,8 +1045,12 @@ impl Display for RequirementsTxtFileError {
                     self.file.user_display(),
                 )
             }
-            RequirementsTxtParserError::VerbatimUrl { url, .. } => {
-                write!(f, "Invalid URL in `{}`: `{url}`", self.file.user_display())
+            RequirementsTxtParserError::VerbatimUrl { url, start, .. } => {
+                write!(
+                    f,
+                    "Invalid URL in `{}` at position {start}: `{url}`",
+                    self.file.user_display(),
+                )
             }
             RequirementsTxtParserError::UrlConversion(given) => {
                 write!(

diff --git a/crates/uv/tests/pip_install.rs b/crates/uv/tests/pip_install.rs
@@ -5623,7 +5623,7 @@ fn prefer_editable() -> Result<()> {
 
 /// Resolve against a local directory laid out as a PEP 503-compatible index.
 #[test]
-fn local_index() -> Result<()> {
+fn local_index_absolute() -> Result<()> {
     let context = TestContext::new("3.12");
 
     let root = context.temp_dir.child("simple-html");
@@ -5669,3 +5669,165 @@ fn local_index() -> Result<()> {
 
     Ok(())
 }
+
+/// Resolve against a local directory laid out as a PEP 503-compatible index, provided via a
+/// relative path on the CLI.
+#[test]
+fn local_index_relative() -> Result<()> {
+    let context = TestContext::new("3.12");
+
+    let root = context.temp_dir.child("simple-html");
+    fs_err::create_dir_all(&root)?;
+
+    let tqdm = root.child("tqdm");
+    fs_err::create_dir_all(&tqdm)?;
+
+    let index = tqdm.child("index.html");
+    index.write_str(&indoc::formatdoc! {r#"
+        <!DOCTYPE html>
+        <html>
+          <head>
+            <meta name="pypi:repository-version" content="1.1" />
+          </head>
+          <body>
+            <h1>Links for example-a-961b4c22</h1>
+            <a
+              href="{}/tqdm-1000.0.0-py3-none-any.whl"
+              data-requires-python=">=3.8"
+            >
+              tqdm-1000.0.0-py3-none-any.whl
+            </a>
+          </body>
+        </html>
+    "#, Url::from_directory_path(context.workspace_root.join("scripts/links/")).unwrap().as_str()})?;
+
+    uv_snapshot!(context.filters(), context.install_without_exclude_newer()
+        .arg("tqdm")
+        .arg("--index-url")
+        .arg("./simple-html"), @r###"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+
+    ----- stderr -----
+    Resolved 1 package in [TIME]
+    Prepared 1 package in [TIME]
+    Installed 1 package in [TIME]
+     + tqdm==1000.0.0
+    "###
+    );
+
+    Ok(())
+}
+
+/// Resolve against a local directory laid out as a PEP 503-compatible index, provided via a
+/// `requirements.txt` file.
+#[test]
+fn local_index_requirements_txt_absolute() -> Result<()> {
+    let context = TestContext::new("3.12");
+
+    let root = context.temp_dir.child("simple-html");
+    fs_err::create_dir_all(&root)?;
+
+    let tqdm = root.child("tqdm");
+    fs_err::create_dir_all(&tqdm)?;
+
+    let index = tqdm.child("index.html");
+    index.write_str(&indoc::formatdoc! {r#"
+        <!DOCTYPE html>
+        <html>
+          <head>
+            <meta name="pypi:repository-version" content="1.1" />
+          </head>
+          <body>
+            <h1>Links for example-a-961b4c22</h1>
+            <a
+              href="{}/tqdm-1000.0.0-py3-none-any.whl"
+              data-requires-python=">=3.8"
+            >
+              tqdm-1000.0.0-py3-none-any.whl
+            </a>
+          </body>
+        </html>
+    "#, Url::from_directory_path(context.workspace_root.join("scripts/links/")).unwrap().as_str()})?;
+
+    let requirements_txt = context.temp_dir.child("requirements.txt");
+    requirements_txt.write_str(&indoc::formatdoc! {r#"
+        --index-url {}
+        tqdm
+    "#, Url::from_directory_path(root).unwrap().as_str()})?;
+
+    uv_snapshot!(context.filters(), context.install_without_exclude_newer()
+        .arg("-r")
+        .arg("requirements.txt"), @r###"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+
+    ----- stderr -----
+    Resolved 1 package in [TIME]
+    Prepared 1 package in [TIME]
+    Installed 1 package in [TIME]
+     + tqdm==1000.0.0
+    "###
+    );
+
+    Ok(())
+}
+
+/// Resolve against a local directory laid out as a PEP 503-compatible index, provided via a
+/// relative path in a `requirements.txt` file.
+#[test]
+fn local_index_requirements_txt_relative() -> Result<()> {
+    let context = TestContext::new("3.12");
+
+    let root = context.temp_dir.child("simple-html");
+    fs_err::create_dir_all(&root)?;
+
+    let tqdm = root.child("tqdm");
+    fs_err::create_dir_all(&tqdm)?;
+
+    let index = tqdm.child("index.html");
+    index.write_str(&indoc::formatdoc! {r#"
+        <!DOCTYPE html>
+        <html>
+          <head>
+            <meta name="pypi:repository-version" content="1.1" />
+          </head>
+          <body>
+            <h1>Links for example-a-961b4c22</h1>
+            <a
+              href="{}/tqdm-1000.0.0-py3-none-any.whl"
+              data-requires-python=">=3.8"
+            >
+              tqdm-1000.0.0-py3-none-any.whl
+            </a>
+          </body>
+        </html>
+    "#, Url::from_directory_path(context.workspace_root.join("scripts/links/")).unwrap().as_str()})?;
+
+    let requirements_txt = context.temp_dir.child("requirements.txt");
+    requirements_txt.write_str(
+        r"
+        --index-url ./simple-html
+        tqdm
+    ",
+    )?;
+
+    uv_snapshot!(context.filters(), context.install_without_exclude_newer()
+        .arg("-r")
+        .arg("requirements.txt"), @r###"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+
+    ----- stderr -----
+    Resolved 1 package in [TIME]
+    Prepared 1 package in [TIME]
+    Installed 1 package in [TIME]
+     + tqdm==1000.0.0
+    "###
+    );
+
+    Ok(())
+}
diff --git a/req.txt b/req.txt
@@ -0,0 +1,3 @@
+--index-url file:///Users/crmarsh/workspace/packse/index/simple-html/
+
+example-a-961b4c22
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		--index-url file:///Users/crmarsh/workspace/packse/index/simple-html/

		example-a-961b4c22