From fe0621a17ae0295d12472c67ffd7316a28e0f37a Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Tue, 3 Sep 2024 21:37:51 +0200 Subject: [PATCH 1/3] refactor(manager/pep621): extract custom managers --- lib/modules/manager/pep621/schema.ts | 68 ++++++++++++++-------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/lib/modules/manager/pep621/schema.ts b/lib/modules/manager/pep621/schema.ts index f63791335fdd32..4fc2b80a7dc697 100644 --- a/lib/modules/manager/pep621/schema.ts +++ b/lib/modules/manager/pep621/schema.ts @@ -8,6 +8,37 @@ const DependencyRecordSchema = z .record(z.string(), z.array(z.string())) .optional(); +const PdmSchema = z.object({ + 'dev-dependencies': DependencyRecordSchema, + source: z + .array( + z.object({ + url: z.string(), + name: z.string(), + verify_ssl: z.boolean().optional(), + }), + ) + .optional(), +}); + +const HatchSchema = z.object({ + envs: z + .record( + z.string(), + z + .object({ + dependencies: DependencyListSchema, + 'extra-dependencies': DependencyListSchema, + }) + .optional(), + ) + .optional(), +}); + +const UvSchema = z.object({ + 'dev-dependencies': DependencyListSchema, +}); + export const PyProjectSchema = z.object({ project: z .object({ @@ -25,40 +56,9 @@ export const PyProjectSchema = z.object({ .optional(), tool: z .object({ - pdm: z - .object({ - 'dev-dependencies': DependencyRecordSchema, - source: z - .array( - z.object({ - url: z.string(), - name: z.string(), - verify_ssl: z.boolean().optional(), - }), - ) - .optional(), - }) - .optional(), - hatch: z - .object({ - envs: z - .record( - z.string(), - z - .object({ - dependencies: DependencyListSchema, - 'extra-dependencies': DependencyListSchema, - }) - .optional(), - ) - .optional(), - }) - .optional(), - uv: z - .object({ - 'dev-dependencies': DependencyListSchema, - }) - .optional(), + pdm: PdmSchema.optional(), + hatch: HatchSchema.optional(), + uv: UvSchema.optional(), }) .optional(), }); From adde4e424b761504db6a393ebaea0f086b4b78d8 Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Tue, 3 Sep 2024 22:13:05 +0200 Subject: [PATCH 2/3] feat(manager/uv): set registry URLs --- .../manager/pep621/processors/uv.spec.ts | 62 ++++++++++++++++++- lib/modules/manager/pep621/processors/uv.ts | 18 +++++- lib/modules/manager/pep621/schema.ts | 4 ++ 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/lib/modules/manager/pep621/processors/uv.spec.ts b/lib/modules/manager/pep621/processors/uv.spec.ts index 7488abe4fc8e74..8d44fd58664b58 100644 --- a/lib/modules/manager/pep621/processors/uv.spec.ts +++ b/lib/modules/manager/pep621/processors/uv.spec.ts @@ -42,7 +42,10 @@ describe('modules/manager/pep621/processors/uv', () => { const result = processor.process(pyproject, dependencies); expect(result).toEqual([ - { packageName: 'dep1' }, + { + packageName: 'dep1', + registryUrls: ['https://pypi.org/pypi/'], + }, { currentValue: '==1.2.3', currentVersion: '1.2.3', @@ -50,6 +53,7 @@ describe('modules/manager/pep621/processors/uv', () => { depName: 'dep2', depType: 'tool.uv.dev-dependencies', packageName: 'dep2', + registryUrls: ['https://pypi.org/pypi/'], }, { currentValue: '==2.3.4', @@ -58,6 +62,62 @@ describe('modules/manager/pep621/processors/uv', () => { depName: 'dep3', depType: 'tool.uv.dev-dependencies', packageName: 'dep3', + registryUrls: ['https://pypi.org/pypi/'], + }, + ]); + }); + + it('uses default PyPI and extra URLs when setting extra-index-url', () => { + const pyproject = { + tool: { + uv: { + 'extra-index-url': [ + 'https://foo.example.com', + 'https://bar.example.com', + ], + }, + }, + }; + const dependencies = [{ packageName: 'dep1' }]; + + const result = processor.process(pyproject, dependencies); + + expect(result).toEqual([ + { + packageName: 'dep1', + registryUrls: [ + 'https://foo.example.com', + 'https://bar.example.com', + 'https://pypi.org/pypi/', + ], + }, + ]); + }); + + it('uses index and extra URLs when setting index-url and extra-index-url', () => { + const pyproject = { + tool: { + uv: { + 'index-url': 'https://foobar.example.com', + 'extra-index-url': [ + 'https://foo.example.com', + 'https://bar.example.com', + ], + }, + }, + }; + const dependencies = [{ packageName: 'dep1' }]; + + const result = processor.process(pyproject, dependencies); + + expect(result).toEqual([ + { + packageName: 'dep1', + registryUrls: [ + 'https://foo.example.com', + 'https://bar.example.com', + 'https://foobar.example.com', + ], }, ]); }); diff --git a/lib/modules/manager/pep621/processors/uv.ts b/lib/modules/manager/pep621/processors/uv.ts index f5b6cd5c7f713f..f0c7a638e38358 100644 --- a/lib/modules/manager/pep621/processors/uv.ts +++ b/lib/modules/manager/pep621/processors/uv.ts @@ -6,13 +6,14 @@ import { exec } from '../../../../util/exec'; import type { ExecOptions, ToolConstraint } from '../../../../util/exec/types'; import { getSiblingFileName, readLocalFile } from '../../../../util/fs'; import { Result } from '../../../../util/result'; +import { PypiDatasource } from '../../../datasource/pypi'; import type { PackageDependency, UpdateArtifact, UpdateArtifactsResult, Upgrade, } from '../../types'; -import { type PyProject, UvLockfileSchema } from '../schema'; +import { type PyProject, type Uv, UvLockfileSchema } from '../schema'; import { depTypes, parseDependencyList } from '../utils'; import type { PyProjectProcessor } from './types'; @@ -32,6 +33,11 @@ export class UvProcessor implements PyProjectProcessor { ), ); + const registryUrls = extractRegistryUrls(uv); + for (const dep of deps) { + dep.registryUrls = [...registryUrls]; + } + return deps; } @@ -142,6 +148,16 @@ export class UvProcessor implements PyProjectProcessor { } } +function extractRegistryUrls(uvManifest: Uv): string[] { + // Extra indexes have priority over default index: https://docs.astral.sh/uv/reference/settings/#extra-index-url + const registryUrls = uvManifest['extra-index-url'] ?? []; + + // If default index URL is not overridden, we need to use default PyPI URL additionally to potential extra indexes. + registryUrls.push(uvManifest['index-url'] ?? PypiDatasource.defaultURL); + + return registryUrls; +} + function generateCMD(updatedDeps: Upgrade[]): string { const deps: string[] = []; diff --git a/lib/modules/manager/pep621/schema.ts b/lib/modules/manager/pep621/schema.ts index 4fc2b80a7dc697..eaf7f1e3719166 100644 --- a/lib/modules/manager/pep621/schema.ts +++ b/lib/modules/manager/pep621/schema.ts @@ -37,8 +37,12 @@ const HatchSchema = z.object({ const UvSchema = z.object({ 'dev-dependencies': DependencyListSchema, + 'index-url': z.string().optional(), + 'extra-index-url': z.array(z.string()).optional(), }); +export type Uv = z.infer; + export const PyProjectSchema = z.object({ project: z .object({ From 104f37d26ccacaee05555d04df4d48ea2afe64cc Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Wed, 4 Sep 2024 00:41:08 +0200 Subject: [PATCH 3/3] feat(manager/uv): support `uv.toml` configuration --- lib/modules/manager/pep621/extract.ts | 2 +- .../manager/pep621/processors/hatch.ts | 6 +- lib/modules/manager/pep621/processors/pdm.ts | 11 +-- .../manager/pep621/processors/types.ts | 5 +- .../manager/pep621/processors/uv.spec.ts | 70 ++++++++++++++++--- lib/modules/manager/pep621/processors/uv.ts | 26 ++++++- 6 files changed, 101 insertions(+), 19 deletions(-) diff --git a/lib/modules/manager/pep621/extract.ts b/lib/modules/manager/pep621/extract.ts index 12a7c1803d4c37..af26538f8f3c82 100644 --- a/lib/modules/manager/pep621/extract.ts +++ b/lib/modules/manager/pep621/extract.ts @@ -53,7 +53,7 @@ export async function extractPackageFile( // process specific tool sets let processedDeps = deps; for (const processor of processors) { - processedDeps = processor.process(def, processedDeps); + processedDeps = await processor.process(def, processedDeps); processedDeps = await processor.extractLockedVersions( def, processedDeps, diff --git a/lib/modules/manager/pep621/processors/hatch.ts b/lib/modules/manager/pep621/processors/hatch.ts index 0cb8cbe1494525..db73dd24e2737c 100644 --- a/lib/modules/manager/pep621/processors/hatch.ts +++ b/lib/modules/manager/pep621/processors/hatch.ts @@ -12,10 +12,10 @@ export class HatchProcessor implements PyProjectProcessor { process( pyproject: PyProject, deps: PackageDependency[], - ): PackageDependency[] { + ): Promise { const hatch_envs = pyproject.tool?.hatch?.envs; if (is.nullOrUndefined(hatch_envs)) { - return deps; + return Promise.resolve(deps); } for (const [envName, env] of Object.entries(hatch_envs)) { @@ -29,7 +29,7 @@ export class HatchProcessor implements PyProjectProcessor { deps.push(...extraDeps); } - return deps; + return Promise.resolve(deps); } extractLockedVersions( diff --git a/lib/modules/manager/pep621/processors/pdm.ts b/lib/modules/manager/pep621/processors/pdm.ts index f1222f9bd315a5..43cbd2134688f2 100644 --- a/lib/modules/manager/pep621/processors/pdm.ts +++ b/lib/modules/manager/pep621/processors/pdm.ts @@ -20,10 +20,13 @@ import type { PyProjectProcessor } from './types'; const pdmUpdateCMD = 'pdm update --no-sync --update-eager'; export class PdmProcessor implements PyProjectProcessor { - process(project: PyProject, deps: PackageDependency[]): PackageDependency[] { + process( + project: PyProject, + deps: PackageDependency[], + ): Promise { const pdm = project.tool?.pdm; if (is.nullOrUndefined(pdm)) { - return deps; + return Promise.resolve(deps); } deps.push( @@ -35,7 +38,7 @@ export class PdmProcessor implements PyProjectProcessor { const pdmSource = pdm.source; if (is.nullOrUndefined(pdmSource)) { - return deps; + return Promise.resolve(deps); } // add pypi default url, if there is no source declared with the name `pypi`. https://daobook.github.io/pdm/pyproject/tool-pdm/#specify-other-sources-for-finding-packages @@ -51,7 +54,7 @@ export class PdmProcessor implements PyProjectProcessor { dep.registryUrls = [...registryUrls]; } - return deps; + return Promise.resolve(deps); } async extractLockedVersions( diff --git a/lib/modules/manager/pep621/processors/types.ts b/lib/modules/manager/pep621/processors/types.ts index cc4c9861e58521..4ccc6708a780f0 100644 --- a/lib/modules/manager/pep621/processors/types.ts +++ b/lib/modules/manager/pep621/processors/types.ts @@ -17,7 +17,10 @@ export interface PyProjectProcessor { * @param project PyProject object * @param deps List of already extracted/processed dependencies */ - process(project: PyProject, deps: PackageDependency[]): PackageDependency[]; + process( + project: PyProject, + deps: PackageDependency[], + ): Promise; extractLockedVersions( project: PyProject, diff --git a/lib/modules/manager/pep621/processors/uv.spec.ts b/lib/modules/manager/pep621/processors/uv.spec.ts index 8d44fd58664b58..108aa313bee2f7 100644 --- a/lib/modules/manager/pep621/processors/uv.spec.ts +++ b/lib/modules/manager/pep621/processors/uv.spec.ts @@ -1,3 +1,4 @@ +import { codeBlock } from 'common-tags'; import { join } from 'upath'; import { mockExecAll } from '../../../../../test/exec-util'; import { fs, mockedFunction } from '../../../../../test/util'; @@ -24,22 +25,75 @@ const processor = new UvProcessor(); describe('modules/manager/pep621/processors/uv', () => { describe('process()', () => { - it('returns initial dependencies if there is no tool.uv section', () => { + it('reads configuration solely from uv.toml if valid', async () => { + const pyproject = { + tool: { uv: { 'index-url': 'https://example.com' } }, + }; + const dependencies = [{ packageName: 'dep1' }]; + + fs.readLocalFile.mockResolvedValueOnce( + codeBlock` + dev-dependencies = ["dep2==1.2.3"] + `, + ); + + const result = await processor.process(pyproject, dependencies); + + expect(result).toEqual([ + { + packageName: 'dep1', + registryUrls: ['https://pypi.org/pypi/'], + }, + { + currentValue: '==1.2.3', + currentVersion: '1.2.3', + datasource: 'pypi', + depName: 'dep2', + depType: 'tool.uv.dev-dependencies', + packageName: 'dep2', + registryUrls: ['https://pypi.org/pypi/'], + }, + ]); + }); + + it('reads configuration from pyproject.toml on invalid uv.toml', async () => { + const pyproject = { + tool: { uv: { 'index-url': 'https://example.com' } }, + }; + const dependencies = [{ packageName: 'dep1' }]; + + fs.readLocalFile.mockResolvedValueOnce( + codeBlock` + dev-dependencies = invalid_toml + `, + ); + + const result = await processor.process(pyproject, dependencies); + + expect(result).toEqual([ + { + packageName: 'dep1', + registryUrls: ['https://example.com'], + }, + ]); + }); + + it('returns initial dependencies if there is no tool.uv section', async () => { const pyproject = { tool: {} }; const dependencies = [{ packageName: 'dep1' }]; - const result = processor.process(pyproject, dependencies); + const result = await processor.process(pyproject, dependencies); expect(result).toEqual(dependencies); }); - it('includes uv dev dependencies if there is a tool.uv section', () => { + it('includes uv dev dependencies if there is a tool.uv section', async () => { const pyproject = { tool: { uv: { 'dev-dependencies': ['dep2==1.2.3', 'dep3==2.3.4'] } }, }; const dependencies = [{ packageName: 'dep1' }]; - const result = processor.process(pyproject, dependencies); + const result = await processor.process(pyproject, dependencies); expect(result).toEqual([ { @@ -67,7 +121,7 @@ describe('modules/manager/pep621/processors/uv', () => { ]); }); - it('uses default PyPI and extra URLs when setting extra-index-url', () => { + it('uses default PyPI and extra URLs when setting extra-index-url', async () => { const pyproject = { tool: { uv: { @@ -80,7 +134,7 @@ describe('modules/manager/pep621/processors/uv', () => { }; const dependencies = [{ packageName: 'dep1' }]; - const result = processor.process(pyproject, dependencies); + const result = await processor.process(pyproject, dependencies); expect(result).toEqual([ { @@ -94,7 +148,7 @@ describe('modules/manager/pep621/processors/uv', () => { ]); }); - it('uses index and extra URLs when setting index-url and extra-index-url', () => { + it('uses index and extra URLs when setting index-url and extra-index-url', async () => { const pyproject = { tool: { uv: { @@ -108,7 +162,7 @@ describe('modules/manager/pep621/processors/uv', () => { }; const dependencies = [{ packageName: 'dep1' }]; - const result = processor.process(pyproject, dependencies); + const result = await processor.process(pyproject, dependencies); expect(result).toEqual([ { diff --git a/lib/modules/manager/pep621/processors/uv.ts b/lib/modules/manager/pep621/processors/uv.ts index f0c7a638e38358..37bdd54eb35525 100644 --- a/lib/modules/manager/pep621/processors/uv.ts +++ b/lib/modules/manager/pep621/processors/uv.ts @@ -6,6 +6,7 @@ import { exec } from '../../../../util/exec'; import type { ExecOptions, ToolConstraint } from '../../../../util/exec/types'; import { getSiblingFileName, readLocalFile } from '../../../../util/fs'; import { Result } from '../../../../util/result'; +import { Toml } from '../../../../util/schema-utils'; import { PypiDatasource } from '../../../datasource/pypi'; import type { PackageDependency, @@ -17,11 +18,16 @@ import { type PyProject, type Uv, UvLockfileSchema } from '../schema'; import { depTypes, parseDependencyList } from '../utils'; import type { PyProjectProcessor } from './types'; +const UV_CONFIG_FILE = 'uv.toml'; + const uvUpdateCMD = 'uv lock'; export class UvProcessor implements PyProjectProcessor { - process(project: PyProject, deps: PackageDependency[]): PackageDependency[] { - const uv = project.tool?.uv; + async process( + project: PyProject, + deps: PackageDependency[], + ): Promise { + const uv = await getUvConfig(project.tool?.uv); if (is.nullOrUndefined(uv)) { return deps; } @@ -158,6 +164,22 @@ function extractRegistryUrls(uvManifest: Uv): string[] { return registryUrls; } +async function getUvConfig( + uvPyprojectSection: Uv | undefined, +): Promise { + const uvTomlContent = await readLocalFile(UV_CONFIG_FILE, 'utf8'); + + if (uvTomlContent) { + try { + return Toml.parse(uvTomlContent) as Uv; + } catch (err) { + logger.debug({ UV_CONFIG_FILE, err }, 'Error parsing uv config file'); + } + } + + return uvPyprojectSection; +} + function generateCMD(updatedDeps: Upgrade[]): string { const deps: string[] = [];