diff --git a/providers/fetch/condaFetch.js b/providers/fetch/condaFetch.js index 17fa5710..c73e722e 100644 --- a/providers/fetch/condaFetch.js +++ b/providers/fetch/condaFetch.js @@ -25,13 +25,13 @@ class CondaFetch extends AbstractFetch { return spec && condaChannels[spec.provider] } - // (conda|condasource)/(conda-forge|anaconda-main|anaconda-r)/-/${package_name}/(((${version}|-)_(${architecture}|-))|-)/(${toolVersion}|-) - // i.e. conda/conda-forge/-/numpy/1.13.0_linux-aarch64/py36 - // conda/conda-forge/-/numpy/-/py36 - // conda/conda-forge/-/numpy/1.13.0_/py36 - // conda/conda-forge/-/numpy/_linux-aarch64/py36 - // conda/anaconda-main/-/numpy/_/py27 - // conda/anaconda-main/-/numpy/_/- + // {type: conda|condasource}/{provider: anaconda-main|anaconda-r|conda-forge}/-/{package name}/[{archictecture | _}:][{version | _}]-[{build version | _}]/[{tool version}] + // i.e. conda/conda-forge/-/numpy/linux-aarch64:1.13.0-py36/ + // conda/conda-forge/-/numpy/-py36/ + // conda/conda-forge/-/numpy/1.13.0-py36/ + // conda/conda-forge/-/numpy/linux-aarch64:_-py36/ + // conda/conda-forge/-/numpy/ + // conda/conda-forge/-/numpy/_:_-_ async handle(request) { const spec = this.toSpec(request) const specStr = { @@ -48,26 +48,36 @@ class CondaFetch extends AbstractFetch { return request.markSkip(`Unrecognized conda provider: ${spec.provider}, must be either of: ${Object.keys(condaChannels)}`) } const channelData = await this._getChannelData(condaChannels[spec.provider], spec.provider) - let [version, architecture] = (spec.revision || '-_-').split('_') - if (channelData.packages[spec.name] == undefined) { + let [architecture, revision] = (spec.revision || '').split(':') + + // both arch and revision or revision only + if (architecture && !revision) { + revision = architecture + architecture = null + } + + let [version, buildVersion] = (revision || '').split('-') + + if (channelData.packages[spec.name] === undefined) { return request.markSkip(`Missing package ${spec.name} in channelData`) } const packageChannelData = channelData.packages[spec.name] - if (spec.type != 'conda' && spec.type != 'condasource') { + if (spec.type !== 'conda' && spec.type !== 'condasource') { return request.markSkip('spec type must either be conda or condasource') } // unless otherwise specified, we fetch the architecture package - if (spec.type != 'conda' && packageChannelData.subdirs.length == 0) { + if (spec.type !== 'conda' && packageChannelData.subdirs.length === 0) { return request.markSkip('No architecture build in package channel data') } - if ((!architecture || architecture == '-') && spec.type == 'conda') { - architecture = packageChannelData.subdirs[0] - this.logger.info(`No binary architecture specified for ${spec.name}, using random architecture: ${architecture}`) + if ((!architecture || architecture === '_') && spec.type === 'conda') { + // prefer no-arch if available + architecture = packageChannelData.subdirs.includes('noarch') ? 'noarch' : packageChannelData.subdirs[0] + this.logger.info(`No binary architecture specified for ${spec.name}, using architecture: ${architecture}`) } - if (spec.type == 'condasource') { - if (version && version != '-' && packageChannelData.version != version) { + if (spec.type === 'condasource') { + if (version && version !== '_' && packageChannelData.version !== version) { return request.markSkip(`Missing source file version ${version} for package ${spec.name}`) } if (!packageChannelData.source_url) { @@ -100,7 +110,7 @@ class CondaFetch extends AbstractFetch { return request } else { let repoData = undefined - if (!(packageChannelData.subdirs.find(x => x == architecture))) { + if (!(packageChannelData.subdirs.find(x => x === architecture))) { return request.markSkip(`Missing architecture ${architecture} in channel`) } repoData = await this._getRepoData(condaChannels[spec.provider], spec.provider, architecture) @@ -109,16 +119,16 @@ class CondaFetch extends AbstractFetch { if (repoData['packages']) { packageRepoEntries = packageRepoEntries.concat(Object.entries(repoData['packages']) - .filter(([, packageData]) => packageData.name == spec.name && ((!version) || version == '-' || version == packageData.version) - && ((!spec.toolVersion) || packageData.build.startsWith(spec.toolVersion)) + .filter(([, packageData]) => packageData.name === spec.name && ((!version) || version === '_' || version === packageData.version) + && ((!buildVersion) || buildVersion === '_' || packageData.build.startsWith(buildVersion)) ) .map(([packageFile, packageData]) => { return { packageFile, packageData } })) } if (repoData['packages.conda']) { packageRepoEntries = packageRepoEntries.concat(Object.entries(repoData['packages.conda']) - .filter(([, packageData]) => packageData.name == spec.name && ((!version) || version == '-' || version == packageData.version) - && ((!spec.toolVersion) || packageData.build.startsWith(spec.toolVersion)) + .filter(([, packageData]) => packageData.name === spec.name && ((!version) || version === '_' || version === packageData.version) + && ((!buildVersion) || buildVersion === '_' || packageData.build.startsWith(buildVersion)) ) .map(([packageFile, packageData]) => { return { packageFile, packageData } })) } @@ -126,7 +136,7 @@ class CondaFetch extends AbstractFetch { packageRepoEntries.sort((a, b) => { if (a.packageData.build < b.packageData.build) { return 1 - } else if (a.packageData.build == b.packageData.build) { + } else if (a.packageData.build === b.packageData.build) { return 0 } else { @@ -136,13 +146,12 @@ class CondaFetch extends AbstractFetch { let packageRepoEntry = packageRepoEntries[0] if (!packageRepoEntry) { - return request.markSkip(`Missing package with matching spec (version: ${version}, toolVersion: ${spec.toolVersion}) in ${architecture} repository`) + return request.markSkip(`Missing package with matching spec (version: ${version}, buildVersion: ${buildVersion}) in ${architecture} repository`) } let downloadUrl = new URL(`${condaChannels[spec.provider]}/${architecture}/${packageRepoEntry.packageFile}`).href - spec.toolVersion = packageRepoEntry.packageData.build - spec.revision = packageRepoEntry.packageData.version + '_' + architecture + spec.revision = architecture + ':' + packageRepoEntry.packageData.version + '-' + packageRepoEntry.packageData.build request.url = spec.toUrl() super.handle(request) diff --git a/providers/process/top.js b/providers/process/top.js index 89545cc7..429416d2 100644 --- a/providers/process/top.js +++ b/providers/process/top.js @@ -193,7 +193,7 @@ class TopProcessor extends AbstractProcessor { for (let subdir of channelData.subdirs) { let repoData = await condaFetch._getRepoData(channelUrl, spec.provider, subdir) let repoCoordinates = Object.entries(repoData.packages). - map(([, packageData]) => `cd:/conda/${spec.provider}/-/${packageData.name}/${packageData.version}_${subdir}/${packageData.build}` + map(([, packageData]) => `cd:/conda/${spec.provider}/-/${packageData.name}/${subdir}:${packageData.version}-${packageData.build}/` ) packagesCoordinates = packagesCoordinates.concat(repoCoordinates) if (start < packagesCoordinates.length && end <= packagesCoordinates.length) { diff --git a/test/unit/providers/fetch/condaFetchTests.js b/test/unit/providers/fetch/condaFetchTests.js index 36bff48d..166efa75 100644 --- a/test/unit/providers/fetch/condaFetchTests.js +++ b/test/unit/providers/fetch/condaFetchTests.js @@ -60,7 +60,7 @@ describe('condaFetch', () => { }) function verifyFetch(result) { - expect(result.url).to.be.contains('cd:/conda/conda-forge/-/21cmfast/3.0.2') + expect(result.url).to.be.contains('cd:/conda/conda-forge/-/21cmfast/linux-64:3.0.2') expect(result.document.hashes).to.be.deep.equal({ sha1: '9b2f4958826956be03cf3793dbdb663a53a8a1f1', sha256: '1154fceeb5c4ee9bb97d245713ac21eb1910237c724d2b7103747215663273c2' @@ -70,38 +70,38 @@ describe('condaFetch', () => { expect(result.document.declaredLicenses).to.equal('MIT') } - it('fetch sourcepackage without version and architecture', async () => { + it('fetch package without version and architecture', async () => { const result = await fetch.handle(new Request('test', 'cd:/conda/conda-forge/-/21cmfast/')) verifyFetch(result.fetchResult) }) - it('fetch sourcepackage with version and architecture sentinel', async () => { - const result = await fetch.handle(new Request('test', 'cd:/conda/conda-forge/-/21cmfast/-_-')) + it('fetch package with version and architecture sentinel', async () => { + const result = await fetch.handle(new Request('test', 'cd:/conda/conda-forge/-/21cmfast/_-_')) verifyFetch(result.fetchResult) }) - it('fetch sourcepackage with version and without architecture', async () => { + it('fetch package with version and without architecture', async () => { const result = await fetch.handle(new Request('test', 'cd:/conda/conda-forge/-/21cmfast/3.0.2')) verifyFetch(result.fetchResult) }) - it('fetch sourcepackage with version and null architecture', async () => { - const result = await fetch.handle(new Request('test', 'cd:/conda/conda-forge/-/21cmfast/3.0.2_')) + it('fetch package with version and null architecture', async () => { + const result = await fetch.handle(new Request('test', 'cd:/conda/conda-forge/-/21cmfast/:3.0.2')) verifyFetch(result.fetchResult) }) - it('fetch sourcepackage with version and architecture sentinel', async () => { - const result = await fetch.handle(new Request('test', 'cd:/conda/conda-forge/-/21cmfast/3.0.2_-')) + it('fetch package with version and architecture sentinel', async () => { + const result = await fetch.handle(new Request('test', 'cd:/conda/conda-forge/-/21cmfast/_:3.0.2')) verifyFetch(result.fetchResult) }) - it('fetch sourcepackage with version and architecture', async () => { - const result = await fetch.handle(new Request('test', 'cd:/conda/conda-forge/-/21cmfast/3.0.2_linux-64')) + it('fetch package with version and architecture', async () => { + const result = await fetch.handle(new Request('test', 'cd:/conda/conda-forge/-/21cmfast/linux-64:3.0.2')) verifyFetch(result.fetchResult) }) - it('fetch sourcepackage with version, architecture, and build version', async () => { - const result = await fetch.handle(new Request('test', 'cd:/conda/conda-forge/-/21cmfast/3.0.2_linux-64/py37hd45b216_1')) + it('fetch package with version, architecture, and build version', async () => { + const result = await fetch.handle(new Request('test', 'cd:/conda/conda-forge/-/21cmfast/linux-64:3.0.2-py37hd45b216_1')) verifyFetch(result.fetchResult) }) diff --git a/test/unit/providers/process/condaExtractTests.js b/test/unit/providers/process/condaExtractTests.js index b5f587d0..eac3f88c 100644 --- a/test/unit/providers/process/condaExtractTests.js +++ b/test/unit/providers/process/condaExtractTests.js @@ -37,7 +37,7 @@ async function setup() { } function createRequest() { - const request = new Request('conda', 'cd:/conda/conda-forge/-/21cmfast/3.0.2_linux-64') + const request = new Request('conda', 'cd:/conda/conda-forge/-/21cmfast/linux-64:3.0.2') request.document = { _metadata: { links: {} }, sourceInfo: { @@ -45,7 +45,7 @@ function createRequest() { provider: 'conda-forge', namespace: '-', name: '21cmfast', - revision: '3.0.2_linux-64' + revision: 'linux-64:3.0.2' }, registryData: { downloadUrl: '21cmfast',