Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow users to provide pre-defined metadata for resolution #7442

Merged
merged 6 commits into from
Sep 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions crates/bench/benches/uv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ mod resolver {

use anyhow::Result;

use distribution_types::{IndexCapabilities, IndexLocations};
use distribution_types::{DependencyMetadata, IndexCapabilities, IndexLocations};
use install_wheel_rs::linker::LinkMode;
use pep440_rs::Version;
use pep508_rs::{MarkerEnvironment, MarkerEnvironmentBuilder};
Expand Down Expand Up @@ -139,7 +139,7 @@ mod resolver {
interpreter: &Interpreter,
universal: bool,
) -> Result<ResolutionGraph> {
let build_isolation = BuildIsolation::Isolated;
let build_isolation = BuildIsolation::default();
let build_options = BuildOptions::default();
let concurrency = Concurrency::default();
let config_settings = ConfigSettings::default();
Expand All @@ -150,17 +150,18 @@ mod resolver {
.timestamp()
.into(),
);
let build_constraints = Constraints::default();
let capabilities = IndexCapabilities::default();
let flat_index = FlatIndex::default();
let git = GitResolver::default();
let capabilities = IndexCapabilities::default();
let hashes = HashStrategy::None;
let hashes = HashStrategy::default();
let in_flight = InFlight::default();
let index = InMemoryIndex::default();
let index_locations = IndexLocations::default();
let installed_packages = EmptyInstalledPackages;
let sources = SourceStrategy::default();
let options = OptionsBuilder::new().exclude_newer(exclude_newer).build();
let build_constraints = Constraints::default();
let sources = SourceStrategy::default();
let dependency_metadata = DependencyMetadata::default();

let python_requirement = if universal {
PythonRequirement::from_requires_python(
Expand All @@ -178,6 +179,7 @@ mod resolver {
interpreter,
&index_locations,
&flat_index,
&dependency_metadata,
&index,
&git,
&capabilities,
Expand Down
76 changes: 76 additions & 0 deletions crates/distribution-types/src/dependency_metadata.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
use pep440_rs::{Version, VersionSpecifiers};
use pep508_rs::Requirement;
use pypi_types::{Metadata23, VerbatimParsedUrl};
use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize};
use uv_normalize::{ExtraName, PackageName};

/// Pre-defined [`StaticMetadata`] entries, indexed by [`PackageName`] and [`Version`].
#[derive(Debug, Clone, Default)]
pub struct DependencyMetadata(FxHashMap<PackageName, Vec<StaticMetadata>>);

impl DependencyMetadata {
/// Index a set of [`StaticMetadata`] entries by [`PackageName`] and [`Version`].
pub fn from_entries(entries: impl IntoIterator<Item = StaticMetadata>) -> Self {
let mut map = Self::default();
for entry in entries {
map.0.entry(entry.name.clone()).or_default().push(entry);
}
map
}

/// Retrieve a [`StaticMetadata`] entry by [`PackageName`] and [`Version`].
pub fn get(&self, package: &PackageName, version: &Version) -> Option<Metadata23> {
let versions = self.0.get(package)?;

// Search for an exact, then a global match.
let metadata = versions
.iter()
.find(|v| v.version.as_ref() == Some(version))
.or_else(|| versions.iter().find(|v| v.version.is_none()))?;

Some(Metadata23 {
name: metadata.name.clone(),
version: version.clone(),
requires_dist: metadata.requires_dist.clone(),
requires_python: metadata.requires_python.clone(),
provides_extras: metadata.provides_extras.clone(),
})
}

/// Retrieve all [`StaticMetadata`] entries.
pub fn values(&self) -> impl Iterator<Item = &StaticMetadata> {
self.0.values().flatten()
}
}

/// A subset of the Python Package Metadata 2.3 standard as specified in
/// <https://packaging.python.org/specifications/core-metadata/>.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
#[serde(rename_all = "kebab-case")]
pub struct StaticMetadata {
// Mandatory fields
pub name: PackageName,
#[cfg_attr(
feature = "schemars",
schemars(
with = "String",
description = "PEP 440-style package version, e.g., `1.2.3`"
)
)]
pub version: Option<Version>,
// Optional fields
#[serde(default)]
pub requires_dist: Vec<Requirement<VerbatimParsedUrl>>,
#[cfg_attr(
feature = "schemars",
schemars(
with = "Option<String>",
description = "PEP 508-style Python requirement, e.g., `>=3.10`"
)
)]
pub requires_python: Option<VersionSpecifiers>,
#[serde(default)]
pub provides_extras: Vec<ExtraName>,
}
2 changes: 2 additions & 0 deletions crates/distribution-types/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ pub use crate::annotation::*;
pub use crate::any::*;
pub use crate::buildable::*;
pub use crate::cached::*;
pub use crate::dependency_metadata::*;
pub use crate::diagnostic::*;
pub use crate::error::*;
pub use crate::file::*;
Expand All @@ -68,6 +69,7 @@ mod annotation;
mod any;
mod buildable;
mod cached;
mod dependency_metadata;
mod diagnostic;
mod error;
mod file;
Expand Down
8 changes: 5 additions & 3 deletions crates/pep508-rs/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ create_exception!(
);

/// A PEP 508 dependency specifier.
#[derive(Hash, Debug, Clone, Eq, PartialEq)]
#[derive(Hash, Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
pub struct Requirement<T: Pep508Url = VerbatimUrl> {
/// The distribution name such as `requests` in
/// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`.
Expand Down Expand Up @@ -480,7 +480,9 @@ impl<T: Pep508Url> schemars::JsonSchema for Requirement<T> {
schemars::schema::SchemaObject {
instance_type: Some(schemars::schema::InstanceType::String.into()),
metadata: Some(Box::new(schemars::schema::Metadata {
description: Some("A PEP 508 dependency specifier".to_string()),
description: Some(
"A PEP 508 dependency specifier, e.g., `ruff >= 0.6.0`".to_string(),
),
..schemars::schema::Metadata::default()
})),
..schemars::schema::SchemaObject::default()
Expand Down Expand Up @@ -535,7 +537,7 @@ impl Extras {
}

/// The actual version specifier or URL to install.
#[derive(Debug, Clone, Eq, Hash, PartialEq)]
#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub enum VersionOrUrl<T: Pep508Url = VerbatimUrl> {
/// A PEP 440 version specifier set
VersionSpecifier(VersionSpecifiers),
Expand Down
2 changes: 2 additions & 0 deletions crates/uv-cli/src/options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ pub fn resolver_options(
} else {
prerelease
},
dependency_metadata: None,
config_settings: config_setting
.map(|config_settings| config_settings.into_iter().collect::<ConfigSettings>()),
no_build_isolation: flag(no_build_isolation, build_isolation),
Expand Down Expand Up @@ -364,6 +365,7 @@ pub fn resolver_installer_options(
} else {
prerelease
},
dependency_metadata: None,
config_settings: config_setting
.map(|config_settings| config_settings.into_iter().collect::<ConfigSettings>()),
no_build_isolation: flag(no_build_isolation, build_isolation),
Expand Down
10 changes: 9 additions & 1 deletion crates/uv-dispatch/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ use rustc_hash::FxHashMap;
use tracing::{debug, instrument};

use distribution_types::{
CachedDist, IndexCapabilities, IndexLocations, Name, Resolution, SourceDist, VersionOrUrlRef,
CachedDist, DependencyMetadata, IndexCapabilities, IndexLocations, Name, Resolution,
SourceDist, VersionOrUrlRef,
};
use pypi_types::Requirement;
use uv_build::{SourceBuild, SourceBuildContext};
Expand Down Expand Up @@ -45,6 +46,7 @@ pub struct BuildDispatch<'a> {
index: &'a InMemoryIndex,
git: &'a GitResolver,
capabilities: &'a IndexCapabilities,
dependency_metadata: &'a DependencyMetadata,
in_flight: &'a InFlight,
build_isolation: BuildIsolation<'a>,
link_mode: install_wheel_rs::linker::LinkMode,
Expand All @@ -66,6 +68,7 @@ impl<'a> BuildDispatch<'a> {
interpreter: &'a Interpreter,
index_locations: &'a IndexLocations,
flat_index: &'a FlatIndex,
dependency_metadata: &'a DependencyMetadata,
index: &'a InMemoryIndex,
git: &'a GitResolver,
capabilities: &'a IndexCapabilities,
Expand All @@ -90,6 +93,7 @@ impl<'a> BuildDispatch<'a> {
index,
git,
capabilities,
dependency_metadata,
in_flight,
index_strategy,
config_settings,
Expand Down Expand Up @@ -136,6 +140,10 @@ impl<'a> BuildContext for BuildDispatch<'a> {
self.capabilities
}

fn dependency_metadata(&self) -> &DependencyMetadata {
self.dependency_metadata
}

fn build_options(&self) -> &BuildOptions {
self.build_options
}
Expand Down
22 changes: 22 additions & 0 deletions crates/uv-distribution/src/distribution_database.rs
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,15 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
dist: &BuiltDist,
hashes: HashPolicy<'_>,
) -> Result<ArchiveMetadata, Error> {
// If the metadata was provided by the user directly, prefer it.
if let Some(metadata) = self
.build_context
.dependency_metadata()
.get(dist.name(), dist.version())
{
return Ok(ArchiveMetadata::from_metadata23(metadata.clone()));
}

// If hash generation is enabled, and the distribution isn't hosted on an index, get the
// entire wheel to ensure that the hashes are included in the response. If the distribution
// is hosted on an index, the hashes will be included in the simple metadata response.
Expand Down Expand Up @@ -415,6 +424,19 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
source: &BuildableSource<'_>,
hashes: HashPolicy<'_>,
) -> Result<ArchiveMetadata, Error> {
// If the metadata was provided by the user directly, prefer it.
if let Some(dist) = source.as_dist() {
if let Some(version) = dist.version() {
if let Some(metadata) = self
.build_context
.dependency_metadata()
.get(dist.name(), version)
{
return Ok(ArchiveMetadata::from_metadata23(metadata.clone()));
}
}
}

// Optimization: Skip source dist download when we must not build them anyway.
if self
.build_context
Expand Down
63 changes: 59 additions & 4 deletions crates/uv-resolver/src/lock/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@ use url::Url;
use cache_key::RepositoryUrl;
use distribution_filename::{DistExtension, ExtensionError, SourceDistExtension, WheelFilename};
use distribution_types::{
BuiltDist, DirectUrlBuiltDist, DirectUrlSourceDist, DirectorySourceDist, Dist,
DistributionMetadata, FileLocation, FlatIndexLocation, GitSourceDist, HashPolicy,
BuiltDist, DependencyMetadata, DirectUrlBuiltDist, DirectUrlSourceDist, DirectorySourceDist,
Dist, DistributionMetadata, FileLocation, FlatIndexLocation, GitSourceDist, HashPolicy,
IndexLocations, IndexUrl, Name, PathBuiltDist, PathSourceDist, RegistryBuiltDist,
RegistryBuiltWheel, RegistrySourceDist, RemoteSource, Resolution, ResolvedDist, ToUrlError,
UrlString,
RegistryBuiltWheel, RegistrySourceDist, RemoteSource, Resolution, ResolvedDist, StaticMetadata,
ToUrlError, UrlString,
};
use pep440_rs::Version;
use pep508_rs::{split_scheme, MarkerEnvironment, MarkerTree, VerbatimUrl, VerbatimUrlError};
Expand Down Expand Up @@ -795,6 +795,40 @@ impl Lock {
manifest_table.insert("overrides", value(overrides));
}

if !self.manifest.dependency_metadata.is_empty() {
let mut tables = ArrayOfTables::new();
for metadata in &self.manifest.dependency_metadata {
let mut table = Table::new();
table.insert("name", value(metadata.name.to_string()));
if let Some(version) = metadata.version.as_ref() {
table.insert("version", value(version.to_string()));
}
if !metadata.requires_dist.is_empty() {
table.insert(
"requires-dist",
value(serde::Serialize::serialize(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this just to avoid a use serde::Serialize;?

&metadata.requires_dist,
toml_edit::ser::ValueSerializer::new(),
)?),
);
}
if let Some(requires_python) = metadata.requires_python.as_ref() {
table.insert("requires-python", value(requires_python.to_string()));
}
if !metadata.provides_extras.is_empty() {
table.insert(
"provides-extras",
value(serde::Serialize::serialize(
&metadata.provides_extras,
toml_edit::ser::ValueSerializer::new(),
)?),
);
}
tables.push(table);
}
manifest_table.insert("dependency-metadata", Item::ArrayOfTables(tables));
}

if !manifest_table.is_empty() {
doc.insert("manifest", Item::Table(manifest_table));
}
Expand Down Expand Up @@ -881,6 +915,7 @@ impl Lock {
requirements: &[Requirement],
constraints: &[Requirement],
overrides: &[Requirement],
dependency_metadata: &DependencyMetadata,
indexes: Option<&IndexLocations>,
build_options: &BuildOptions,
tags: &Tags,
Expand Down Expand Up @@ -995,6 +1030,18 @@ impl Lock {
}
}

// Validate that the lockfile was generated with the same static metadata.
{
let expected = dependency_metadata
.values()
.cloned()
.collect::<BTreeSet<_>>();
let actual = &self.manifest.dependency_metadata;
if expected != *actual {
return Ok(SatisfiesResult::MismatchedStaticMetadata(expected, actual));
}
}

// Collect the set of available indexes (both `--index-url` and `--find-links` entries).
let remotes = indexes.map(|locations| {
locations
Expand Down Expand Up @@ -1249,6 +1296,8 @@ pub enum SatisfiesResult<'lock> {
MismatchedConstraints(BTreeSet<Requirement>, BTreeSet<Requirement>),
/// The lockfile uses a different set of overrides.
MismatchedOverrides(BTreeSet<Requirement>, BTreeSet<Requirement>),
/// The lockfile uses different static metadata.
MismatchedStaticMetadata(BTreeSet<StaticMetadata>, &'lock BTreeSet<StaticMetadata>),
/// The lockfile is missing a workspace member.
MissingRoot(PackageName),
/// The lockfile referenced a remote index that was not provided
Expand Down Expand Up @@ -1302,6 +1351,9 @@ pub struct ResolverManifest {
/// The overrides provided to the resolver.
#[serde(default)]
overrides: BTreeSet<Requirement>,
/// The static metadata provided to the resolver.
#[serde(default)]
dependency_metadata: BTreeSet<StaticMetadata>,
}

impl ResolverManifest {
Expand All @@ -1312,12 +1364,14 @@ impl ResolverManifest {
requirements: impl IntoIterator<Item = Requirement>,
constraints: impl IntoIterator<Item = Requirement>,
overrides: impl IntoIterator<Item = Requirement>,
dependency_metadata: impl IntoIterator<Item = StaticMetadata>,
) -> Self {
Self {
members: members.into_iter().collect(),
requirements: requirements.into_iter().collect(),
constraints: constraints.into_iter().collect(),
overrides: overrides.into_iter().collect(),
dependency_metadata: dependency_metadata.into_iter().collect(),
}
}

Expand All @@ -1340,6 +1394,7 @@ impl ResolverManifest {
.into_iter()
.map(|requirement| requirement.relative_to(workspace.install_path()))
.collect::<Result<BTreeSet<_>, _>>()?,
dependency_metadata: self.dependency_metadata,
})
}
}
Expand Down
Loading
Loading