Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Include all extras when generating lockfile #3912

Merged
merged 1 commit into from
May 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion crates/uv-requirements/src/specification.rs
Original file line number Diff line number Diff line change
Expand Up @@ -451,11 +451,13 @@ impl RequirementsSpecification {
}

/// Read the combined requirements and constraints from a set of sources.
///
/// If a [`Workspace`] is provided, it will be used as-is without re-discovering a workspace
/// from the filesystem.
pub async fn from_sources(
requirements: &[RequirementsSource],
constraints: &[RequirementsSource],
overrides: &[RequirementsSource],
// Avoid re-discovering the workspace if we already loaded it.
workspace: Option<&Workspace>,
extras: &ExtrasSpecification,
client_builder: &BaseClientBuilder<'_>,
Expand Down
62 changes: 51 additions & 11 deletions crates/uv-requirements/src/workspace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,17 @@
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};

use distribution_types::{Requirement, RequirementSource};
use glob::{glob, GlobError, PatternError};
use pep508_rs::{VerbatimUrl, VerbatimUrlError};
use rustc_hash::FxHashSet;
use tracing::{debug, trace};

use uv_fs::{absolutize_path, Simplified};
use uv_normalize::PackageName;
use uv_normalize::{ExtraName, PackageName};
use uv_warnings::warn_user;

use crate::pyproject::{PyProjectToml, Source, ToolUvWorkspace};
use crate::RequirementsSource;

#[derive(thiserror::Error, Debug)]
pub enum WorkspaceError {
Expand All @@ -32,6 +33,8 @@ pub enum WorkspaceError {
DynamicNotAllowed(&'static str),
#[error("Failed to normalize workspace member path")]
Normalize(#[source] std::io::Error),
#[error("Failed to normalize workspace member path")]
VerbatimUrl(#[from] VerbatimUrlError),
}

/// A workspace, consisting of a root directory and members. See [`ProjectWorkspace`].
Expand Down Expand Up @@ -172,6 +175,8 @@ pub struct ProjectWorkspace {
project_root: PathBuf,
/// The name of the package.
project_name: PackageName,
/// The extras available in the project.
extras: Vec<ExtraName>,
/// The workspace the project is part of.
workspace: Workspace,
}
Expand Down Expand Up @@ -235,31 +240,46 @@ impl ProjectWorkspace {
))
}

/// The directory containing the closest `pyproject.toml`, defining the current project.
/// Returns the directory containing the closest `pyproject.toml` that defines the current
/// project.
pub fn project_root(&self) -> &Path {
&self.project_root
}

/// The name of the current project.
/// Returns the [`PackageName`] of the current project.
pub fn project_name(&self) -> &PackageName {
&self.project_name
}

/// The workspace definition.
/// Returns the extras available in the project.
pub fn project_extras(&self) -> &[ExtraName] {
&self.extras
}

/// Returns the [`Workspace`] containing the current project.
pub fn workspace(&self) -> &Workspace {
&self.workspace
}

/// The current project.
/// Returns the current project as a [`WorkspaceMember`].
pub fn current_project(&self) -> &WorkspaceMember {
&self.workspace().packages[&self.project_name]
}

/// Return the requirements for the project, which is the current project as editable.
pub fn requirements(&self) -> Vec<RequirementsSource> {
vec![RequirementsSource::Editable(
self.project_root.to_string_lossy().to_string(),
)]
/// Return the [`Requirement`] entries for the project, which is the current project as
/// editable.
pub fn requirements(&self) -> Vec<Requirement> {
vec![Requirement {
name: self.project_name.clone(),
extras: self.extras.clone(),
marker: None,
source: RequirementSource::Path {
path: self.project_root.clone(),
editable: true,
url: VerbatimUrl::from_path(&self.project_root).expect("path is valid URL"),
},
origin: None,
}]
}

/// Find the workspace for a project.
Expand All @@ -272,6 +292,18 @@ impl ProjectWorkspace {
.map_err(WorkspaceError::Normalize)?
.to_path_buf();

// Extract the extras available in the project.
let extras = project
.project
.as_ref()
.and_then(|project| project.optional_dependencies.as_ref())
.map(|optional_dependencies| {
let mut extras = optional_dependencies.keys().cloned().collect::<Vec<_>>();
extras.sort_unstable();
extras
})
.unwrap_or_default();

let mut workspace_members = BTreeMap::new();
// The current project is always a workspace member, especially in a single project
// workspace.
Expand Down Expand Up @@ -305,6 +337,7 @@ impl ProjectWorkspace {
return Ok(Self {
project_root: project_path.clone(),
project_name,
extras,
workspace: Workspace {
root: project_path,
packages: workspace_members,
Expand Down Expand Up @@ -385,6 +418,7 @@ impl ProjectWorkspace {
Ok(Self {
project_root: project_path.clone(),
project_name,
extras,
workspace: Workspace {
root: workspace_root,
packages: workspace_members,
Expand Down Expand Up @@ -412,6 +446,7 @@ impl ProjectWorkspace {
Self {
project_root: root.to_path_buf(),
project_name: project_name.clone(),
extras: Vec::new(),
workspace: Workspace {
root: root.to_path_buf(),
packages: [(project_name.clone(), root_member)].into_iter().collect(),
Expand Down Expand Up @@ -627,6 +662,7 @@ mod tests {
{
"project_root": "[ROOT]/albatross-in-example/examples/bird-feeder",
"project_name": "bird-feeder",
"extras": [],
"workspace": {
"root": "[ROOT]/albatross-in-example/examples/bird-feeder",
"packages": {
Expand Down Expand Up @@ -657,6 +693,7 @@ mod tests {
{
"project_root": "[ROOT]/albatross-project-in-excluded/excluded/bird-feeder",
"project_name": "bird-feeder",
"extras": [],
"workspace": {
"root": "[ROOT]/albatross-project-in-excluded/excluded/bird-feeder",
"packages": {
Expand Down Expand Up @@ -686,6 +723,7 @@ mod tests {
{
"project_root": "[ROOT]/albatross-root-workspace",
"project_name": "albatross",
"extras": [],
"workspace": {
"root": "[ROOT]/albatross-root-workspace",
"packages": {
Expand Down Expand Up @@ -729,6 +767,7 @@ mod tests {
{
"project_root": "[ROOT]/albatross-virtual-workspace/packages/albatross",
"project_name": "albatross",
"extras": [],
"workspace": {
"root": "[ROOT]/albatross-virtual-workspace",
"packages": {
Expand Down Expand Up @@ -766,6 +805,7 @@ mod tests {
{
"project_root": "[ROOT]/albatross-just-project",
"project_name": "albatross",
"extras": [],
"workspace": {
"root": "[ROOT]/albatross-just-project",
"packages": {
Expand Down
4 changes: 4 additions & 0 deletions crates/uv-resolver/src/resolution/graph.rs
Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,10 @@ impl ResolutionGraph {
let mut locked_dists = vec![];
for node_index in self.petgraph.node_indices() {
let dist = &self.petgraph[node_index];
if dist.extra.is_some() {
continue;
}

let mut locked_dist = lock::Distribution::from_annotated_dist(dist)?;
for edge in self.petgraph.neighbors(node_index) {
let dependency_dist = &self.petgraph[edge];
Expand Down
48 changes: 19 additions & 29 deletions crates/uv/src/commands/project/lock.rs
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
use anstream::eprint;
use anyhow::Result;

use distribution_types::IndexLocations;
use distribution_types::{IndexLocations, UnresolvedRequirementSpecification};
use install_wheel_rs::linker::LinkMode;
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, RegistryClientBuilder};
use uv_client::RegistryClientBuilder;
use uv_configuration::{
Concurrency, ConfigSettings, ExtrasSpecification, NoBinary, NoBuild, PreviewMode, Reinstall,
SetupPyStrategy, Upgrade,
};
use uv_dispatch::BuildDispatch;
use uv_interpreter::PythonEnvironment;
use uv_requirements::{ProjectWorkspace, RequirementsSpecification};
use uv_requirements::ProjectWorkspace;
use uv_resolver::{ExcludeNewer, FlatIndex, InMemoryIndex, Lock, OptionsBuilder};
use uv_types::{BuildIsolation, EmptyInstalledPackages, HashStrategy, InFlight};
use uv_warnings::warn_user;
Expand Down Expand Up @@ -39,7 +39,7 @@ pub(crate) async fn lock(
let venv = project::init_environment(&project, preview, cache, printer)?;

// Perform the lock operation.
match do_lock(&project, &venv, exclude_newer, preview, cache, printer).await {
match do_lock(&project, &venv, exclude_newer, cache, printer).await {
Ok(_) => Ok(ExitStatus::Success),
Err(ProjectError::Operation(pip::operations::Error::Resolve(
uv_resolver::ResolveError::NoSolution(err),
Expand All @@ -58,29 +58,19 @@ pub(super) async fn do_lock(
project: &ProjectWorkspace,
venv: &PythonEnvironment,
exclude_newer: Option<ExcludeNewer>,
preview: PreviewMode,
cache: &Cache,
printer: Printer,
) -> Result<Lock, ProjectError> {
// TODO(zanieb): Support client configuration
let client_builder = BaseClientBuilder::default();

// Read all requirements from the provided sources.
// TODO(zanieb): Consider allowing constraints and extras
// TODO(zanieb): Allow specifying extras somehow
let spec = RequirementsSpecification::from_sources(
// TODO(konsti): With workspace (just like with extras), these are the requirements for
// syncing. For locking, we want to use the entire workspace with all extras.
// See https://github.com/astral-sh/uv/issues/3700
&project.requirements(),
&[],
&[],
None,
&ExtrasSpecification::None,
&client_builder,
preview,
)
.await?;
// When locking, include the project itself (as editable).
let requirements = project
.requirements()
.into_iter()
.map(UnresolvedRequirementSpecification::from)
.collect::<Vec<_>>();
let constraints = vec![];
let overrides = vec![];
let source_trees = vec![];
let project_name = project.project_name().clone();

// Determine the tags, markers, and interpreter to use for resolution.
let interpreter = venv.interpreter().clone();
Expand Down Expand Up @@ -133,11 +123,11 @@ pub(super) async fn do_lock(

// Resolve the requirements.
let resolution = pip::operations::resolve(
spec.requirements,
spec.constraints,
spec.overrides,
spec.source_trees,
spec.project,
requirements,
constraints,
overrides,
source_trees,
Some(project_name),
&extras,
EmptyInstalledPackages,
&hasher,
Expand Down
3 changes: 1 addition & 2 deletions crates/uv/src/commands/project/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,7 @@ pub(crate) async fn run(
let venv = project::init_environment(&project, preview, cache, printer)?;

// Lock and sync the environment.
let lock =
project::lock::do_lock(&project, &venv, exclude_newer, preview, cache, printer).await?;
let lock = project::lock::do_lock(&project, &venv, exclude_newer, cache, printer).await?;
project::sync::do_sync(&project, &venv, &lock, cache, printer).await?;

Some(venv)
Expand Down
Loading
Loading