implemented changes from review

changed condafetch date to iso format fixed test update tests update
clearlydefined · Jan 24, 2024 · 676ac80 · 676ac80
1 parent 51119a1
commit 676ac80
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 35 deletions.
diff --git a/config/cdConfig.js b/config/cdConfig.js
@@ -54,7 +54,7 @@ module.exports = {
     cdsource: {},
     component: {},
     conda: { githubToken },
-    condasrc: { githubToken },
+    condasrc: {},
     crate: { githubToken },
     deb: {},
     debsrc: {},

diff --git a/providers/fetch/condaFetch.js b/providers/fetch/condaFetch.js
@@ -28,13 +28,13 @@ class CondaFetch extends AbstractFetch {
     return spec && !!(this.channels[spec.provider])
   }
 
-  //      {type: conda|condasrc}/{provider: anaconda-main|anaconda-r|conda-forge}/{architecture|-}/{package name}/[{version | _}]-[{build version | _}]/
+  //      {type: conda|condasrc}/{provider: anaconda-main|anaconda-r|conda-forge}/{architecture|-}/{package name}/[{version | }]-[{build version | }]/
   // i.e. conda/conda-forge/linux-aarch64/numpy/1.13.0-py36/
   //      conda/conda-forge/-/numpy/-py36/
   //      conda/conda-forge/-/numpy/1.13.0-py36/
-  //      conda/conda-forge/linux-aarch64/numpy/_-py36/
+  //      conda/conda-forge/linux-aarch64/numpy/-py36/
   //      conda/conda-forge/-/numpy/
-  //      conda/conda-forge/-/numpy/_-_
+  //      conda/conda-forge/-/numpy/-
   async handle(request) {
     const spec = this.toSpec(request)
     if (spec.type !== 'conda' && spec.type !== 'condasrc') {
@@ -65,7 +65,7 @@ class CondaFetch extends AbstractFetch {
   }
 
   async _downloadCondaSourcePackage(spec, request, version, packageChannelData) {
-    if (version && version !== '_' && packageChannelData.version !== version) {
+    if (version && packageChannelData.version !== version) {
       return request.markSkip(`Missing source file version ${version} for package ${spec.name}`)
     }
     if (!packageChannelData.source_url) {
@@ -84,7 +84,7 @@ class CondaFetch extends AbstractFetch {
     fetchResult.document = {
       location: dir.name,
       registryData: { 'channelData': packageChannelData, downloadUrl },
-      releaseDate: new Date(packageChannelData.timestamp).toUTCString(),
+      releaseDate: new Date(packageChannelData.timestamp).toISOString(),
       declaredLicenses: packageChannelData.license,
       hashes
     }
@@ -96,8 +96,8 @@ class CondaFetch extends AbstractFetch {
   _matchPackage(name, version, buildVersion, repoData) {
     let packageRepoEntries = []
     let packageMatches = ([, packageData]) => {
-      return packageData.name === name && ((!version) || version === '_' || version === packageData.version)
-        && ((!buildVersion) || buildVersion === '_' || packageData.build.startsWith(buildVersion))
+      return packageData.name === name && ((!version) || version === packageData.version)
+        && ((!buildVersion) || packageData.build.startsWith(buildVersion))
     }
     if (repoData['packages']) {
       packageRepoEntries = packageRepoEntries.concat(Object.entries(repoData['packages'])
@@ -128,7 +128,7 @@ class CondaFetch extends AbstractFetch {
       return request.markSkip(`failed to fetch and parse repodata json file for channel ${spec.provider} in architecture ${architecture}`)
     }
     let packageRepoEntries = this._matchPackage(spec.name, version, buildVersion, repoData)
-    if (packageRepoEntries.length == 0) {
+    if (packageRepoEntries.length === 0) {
       return request.markSkip(`Missing package with matching spec (version: ${version}, buildVersion: ${buildVersion}) in ${architecture} repository`)
     }
     let packageRepoEntry = packageRepoEntries[0]
@@ -146,7 +146,7 @@ class CondaFetch extends AbstractFetch {
     fetchResult.document = {
       location: dir.name,
       registryData: { 'channelData': packageChannelData, 'repoData': packageRepoEntry, downloadUrl },
-      releaseDate: new Date(packageRepoEntry.packageData.timestamp).toUTCString(),
+      releaseDate: new Date(packageRepoEntry.packageData.timestamp).toISOString(),
       declaredLicenses: packageRepoEntry.packageData.license,
       hashes
     }

diff --git a/providers/process/condaExtract.js b/providers/process/condaExtract.js
@@ -24,28 +24,7 @@ class CondaExtract extends AbstractClearlyDefinedProcessor {
       const spec = this.toSpec(request)
       const { releaseDate, registryData, declaredLicenses } = request.document
       request.document = merge(this.clone(request.document), { releaseDate, registryData, declaredLicenses })
-      let sourceCandidates = [
-        registryData.channelData.source_url,
-        registryData.channelData.source_git_url,
-        registryData.channelData.home,
-        registryData.channelData.dev_url,
-        registryData.channelData.doc_url,
-        registryData.channelData.doc_source_url].filter(e => e)
-      let sourceInfo = undefined
-      const githubSource = await this.sourceFinder(
-        registryData.repoData.packageData.version, sourceCandidates, {
-        githubToken: this.options.githubToken,
-        logger: this.logger
-      })
-      if (githubSource) {
-        sourceInfo = githubSource
-      } else {
-        sourceInfo = SourceSpec.fromObject(spec)
-        sourceInfo.type = 'condasrc'
-        sourceInfo.namespace = null
-        sourceInfo.version = null
-      }
-      request.document.sourceInfo = sourceInfo
+      request.document.sourceInfo = await this._discoverSource(spec, registryData)
     }
     this.addLocalToolTasks(request)
     if (request.document.sourceInfo) {
@@ -54,6 +33,31 @@ class CondaExtract extends AbstractClearlyDefinedProcessor {
     }
     return request
   }
+
+  async _discoverSource(spec, registryData) {
+    let sourceCandidates = [
+      registryData.channelData.source_url,
+      registryData.channelData.source_git_url,
+      registryData.channelData.home,
+      registryData.channelData.dev_url,
+      registryData.channelData.doc_url,
+      registryData.channelData.doc_source_url].filter(e => e)
+    let sourceInfo = undefined
+    const githubSource = await this.sourceFinder(
+      registryData.repoData.packageData.version, sourceCandidates, {
+      githubToken: this.options.githubToken,
+      logger: this.logger
+    })
+    if (githubSource) {
+      sourceInfo = githubSource
+    } else {
+      sourceInfo = SourceSpec.fromObject(spec)
+      sourceInfo.type = 'condasrc'
+      sourceInfo.namespace = null
+      sourceInfo.revision = spec.revision.split('-')[0]
+    }
+    return sourceInfo
+  }
 }
 
 module.exports = (options, sourceFinder) => new CondaExtract(options, sourceFinder || sourceDiscovery)
diff --git a/providers/process/top.js b/providers/process/top.js
@@ -202,7 +202,7 @@ class TopProcessor extends AbstractProcessor {
       `Conda top - coordinates: ${packagesCoordinates.length}, start: ${start}, end: ${end}, sliced: ${slicedCoordinates.length}`
     )
 
-    await request.queueRequests(slicedCoordinates.map(coord => new Request(spec.type == 'conda' ? 'package' : 'source', coord)))
+    await request.queueRequests(slicedCoordinates.map(coord => new Request(spec.type === 'conda' ? 'package' : 'source', coord)))
     return request.markNoSave()
   }
 

diff --git a/test/unit/providers/fetch/condaFetchTests.js b/test/unit/providers/fetch/condaFetchTests.js
@@ -51,7 +51,7 @@ describe('condaFetch', () => {
       sha256: '1154fceeb5c4ee9bb97d245713ac21eb1910237c724d2b7103747215663273c2'
     })
     expect(result.document.location).to.be.a.string
-    expect(result.document.releaseDate).to.contain('Wed, 11 Nov 2020 16:04:29 GMT')
+    expect(result.document.releaseDate).to.contain('2020-11-11T16:04:29.311Z')
     expect(result.document.declaredLicenses).to.equal('MIT')
   }
 
@@ -101,6 +101,11 @@ describe('condaFetch', () => {
     verifyFetch(result.fetchResult)
   })
 
+  it('fetch package with architecture and build version', async () => {
+    const result = await fetch.handle(new Request('test', 'cd:/conda/conda-forge/linux-64/21cmfast/-py37hd45b216_1'))
+    verifyFetch(result.fetchResult)
+  })
+
   it('reports failed package matching', async () => {
     const result = await fetch.handle(new Request('test', 'cd:/conda/conda-forge/linux-64/21cmfast/3.0.2-py9999_invalid'))
     expect(result.outcome).to.equal('Missing package with matching spec (version: 3.0.2, buildVersion: py9999_invalid) in linux-64 repository')
@@ -147,7 +152,7 @@ describe('condaSrcFetch', () => {
       sha256: '96f5809d111a8a137c25758fa3f41586ea44cecba7ae191518767895afc7b3c6'
     })
     expect(result.document.location).to.be.a.string
-    expect(result.document.releaseDate).to.contain('Tue, 20 Jan 1970 02:41:00 GMT')
+    expect(result.document.releaseDate).to.contain('1970-01-20T02:41:00.314Z')
     expect(result.document.declaredLicenses).to.equal('MIT')
   }