-
Notifications
You must be signed in to change notification settings - Fork 2.3k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: debian datasource #13463
feat: debian datasource #13463
Changes from all commits
bc79714
1c5a116
270e363
3c2f433
5fe458c
0cf2328
8faee2d
bd88abb
3206401
1de78f1
314f709
096c1f3
c79f809
a8e7a32
bc30c95
b4bc85c
727b292
ddeeced
a2ed06e
01b606e
d9a24f6
ea50609
0bcb6ff
e8f1468
615f8a1
56d4964
8b8f6da
389aa60
c439a68
283055e
6caa868
d95a3bb
27196a0
ba1a006
483bc2a
331a39f
90b3d72
30913ea
c7dd40a
dcae944
193ae88
f01b8ba
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
import { createReadStream } from 'fs'; | ||
import { performance } from 'perf_hooks'; | ||
import readline from 'readline'; | ||
|
||
async function probeExtractedPackage(extractedFile, packageName) { | ||
const requiredPackageKeys = ['Package', 'Version', 'Homepage']; | ||
const rl = readline.createInterface({ | ||
input: createReadStream(extractedFile), | ||
terminal: false, | ||
}); | ||
let pd = {}; | ||
for await (const line of rl) { | ||
if (line === '') { | ||
// now we should have all information available | ||
if (pd.Package === packageName) { | ||
// return { releases: [{ version: pd.Version }], homepage: pd.Homepage }; | ||
} | ||
pd = {}; | ||
continue; | ||
} | ||
|
||
for (let i = 0; i < requiredPackageKeys.length; i++) { | ||
if (line.startsWith(requiredPackageKeys[i])) { | ||
pd[requiredPackageKeys[i]] = line | ||
.substring(requiredPackageKeys[i].length + 1) | ||
.trim(); | ||
break; | ||
} | ||
} | ||
} | ||
|
||
return null; | ||
} | ||
|
||
const startTime = performance.now(); | ||
const p = await probeExtractedPackage( | ||
'/tmp/renovate-deb/packages/6a3a33ecf7f7cfdd630f78c3e840c5bf3bbb9c090f139a178c44dd6dd2de7154.txt', | ||
'curl' | ||
); | ||
var endTime = performance.now(); | ||
console.log(`Took ${endTime - startTime} ms`); |
Original file line number | Diff line number | Diff line change | ||
---|---|---|---|---|
@@ -0,0 +1,182 @@ | ||||
import { copyFile, mkdirp, stat } from 'fs-extra'; | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. file structure changed, please move datasource to right folder. please don't force-push |
||||
import { DirectoryResult, dir } from 'tmp-promise'; | ||||
import upath from 'upath'; | ||||
import { getPkgReleases } from '..'; | ||||
import type { GetPkgReleasesConfig } from '..'; | ||||
import * as httpMock from '../../../test/http-mock'; | ||||
import { GlobalConfig } from '../../config/global'; | ||||
import { DebDatasource } from '.'; | ||||
|
||||
describe('datasource/deb/index', () => { | ||||
describe('getReleases', () => { | ||||
const testPackagesFile = upath.join(__dirname, 'test-data', 'Packages.gz'); | ||||
const extractedTestFile = upath.join(__dirname, 'test-data', 'Packages'); | ||||
let cacheDir: DirectoryResult; | ||||
let extractionFolder: string; | ||||
let extractedPackageFile: string; | ||||
let cfg: GetPkgReleasesConfig; // this can be modified within the test cases | ||||
|
||||
beforeEach(async () => { | ||||
jest.resetAllMocks(); | ||||
cacheDir = await dir({ unsafeCleanup: true }); | ||||
GlobalConfig.set({ cacheDir: cacheDir.path }); | ||||
extractionFolder = upath.join( | ||||
cacheDir.path, | ||||
'others', | ||||
DebDatasource.cacheSubDir | ||||
); | ||||
extractedPackageFile = upath.join( | ||||
extractionFolder, | ||||
'0b01d9df270158d22c09c85f21b0f403d31b0da3cae4930fdb305df8f7749c27.txt' | ||||
); | ||||
|
||||
cfg = { | ||||
datasource: 'deb', | ||||
depName: 'steam-devices', | ||||
registryUrls: [ | ||||
'http://ftp.debian.org/debian?suite=stable&components=non-free&binaryArch=amd64', | ||||
], | ||||
}; | ||||
}); | ||||
|
||||
it('returns a valid version for the package `steam-devices` and does not require redownload', async () => { | ||||
// copy the Packages file to the appropriate location | ||||
await mkdirp(extractionFolder); | ||||
await copyFile(extractedTestFile, extractedPackageFile); | ||||
const stats = await stat(extractedPackageFile); | ||||
const ts = stats.ctime; | ||||
|
||||
httpMock | ||||
.scope('http://ftp.debian.org') | ||||
.head('/debian/dists/stable/non-free/binary-amd64/Packages.gz') | ||||
.reply(304); | ||||
|
||||
const res = await getPkgReleases(cfg); | ||||
expect(res).toBeObject(); | ||||
expect(res.releases).toHaveLength(1); | ||||
|
||||
// validate that the server was called correctly | ||||
expect(httpMock.getTrace()).toHaveLength(1); | ||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
This will be always equal to your mocked http requests, so no benefit. 😉 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Unfulfilled mocks are only evaluated after the test block. In the next line the array access on There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes and no, if you look at the http mock code, you can see a |
||||
const modifiedTs = httpMock.getTrace()[0].headers['if-modified-since']; | ||||
expect(modifiedTs).toBeDefined(); | ||||
expect(modifiedTs).toEqual(ts.toUTCString()); | ||||
}); | ||||
|
||||
describe('parsing of registry url', () => { | ||||
it('returns null when registry url misses components', async () => { | ||||
cfg.registryUrls = [ | ||||
'http://ftp.debian.org/debian?suite=stable&binaryArch=amd64', | ||||
]; | ||||
const res = await getPkgReleases(cfg); | ||||
expect(res).toBeNull(); | ||||
}); | ||||
|
||||
it('returns null when registry url misses binaryArch', async () => { | ||||
cfg.registryUrls = [ | ||||
'http://ftp.debian.org/debian?suite=stable&components=non-free', | ||||
]; | ||||
const res = await getPkgReleases(cfg); | ||||
expect(res).toBeNull(); | ||||
}); | ||||
|
||||
it('returns null when registry url misses suite', async () => { | ||||
cfg.registryUrls = [ | ||||
'http://ftp.debian.org/debian?components=non-free&binaryArch=amd64', | ||||
]; | ||||
const res = await getPkgReleases(cfg); | ||||
expect(res).toBeNull(); | ||||
}); | ||||
}); | ||||
|
||||
describe('without local version', () => { | ||||
beforeEach(() => { | ||||
httpMock | ||||
.scope('http://ftp.debian.org') | ||||
.get('/debian/dists/stable/non-free/binary-amd64/Packages.gz') | ||||
.replyWithFile(200, testPackagesFile); | ||||
}); | ||||
|
||||
it('returns a valid version for the package `steam-devices`', async () => { | ||||
const res = await getPkgReleases(cfg); | ||||
expect(res).toBeObject(); | ||||
expect(res.releases).toHaveLength(1); | ||||
}); | ||||
|
||||
it('returns a valid version for the package `steam-devices` if release is used in the registryUrl', async () => { | ||||
// release is a synonym for suite | ||||
cfg.registryUrls = [ | ||||
'http://ftp.debian.org/debian?release=stable&components=non-free&binaryArch=amd64', | ||||
]; | ||||
const res = await getPkgReleases(cfg); | ||||
expect(res).toBeObject(); | ||||
expect(res.releases).toHaveLength(1); | ||||
}); | ||||
|
||||
it('returns null for an unknown package', async () => { | ||||
cfg.depName = 'you-will-never-find-me'; | ||||
const res = await getPkgReleases(cfg); | ||||
expect(res).toBeNull(); | ||||
}); | ||||
|
||||
describe('with two components', () => { | ||||
const testPackagesFile2 = __dirname + '/test-data/Packages2.gz'; | ||||
|
||||
beforeEach(() => { | ||||
httpMock | ||||
.scope('http://ftp.debian.org') | ||||
.get( | ||||
'/debian/dists/stable/non-free-second/binary-amd64/Packages.gz' | ||||
) | ||||
.replyWithFile(200, testPackagesFile2); | ||||
|
||||
// overwrite the previously set registryUrls to have two components | ||||
cfg.registryUrls = [ | ||||
'http://ftp.debian.org/debian?suite=stable&components=non-free,non-free-second&binaryArch=amd64', | ||||
]; | ||||
}); | ||||
|
||||
it('returns two releases for `steam-devices` which is the same across the components', async () => { | ||||
const res = await getPkgReleases(cfg); | ||||
expect(res).toBeObject(); | ||||
expect(res.releases).toHaveLength(2); | ||||
}); | ||||
|
||||
it('returns two releases for `album` which has different metadata across the components', async () => { | ||||
cfg.depName = 'album'; | ||||
const res = await getPkgReleases(cfg); | ||||
expect(res?.releases).toHaveLength(2); | ||||
}); | ||||
}); | ||||
}); | ||||
|
||||
describe('without server response', () => { | ||||
beforeEach(() => { | ||||
httpMock | ||||
.scope('http://ftp.debian.org') | ||||
.get('/debian/dists/stable/non-free/binary-amd64/Packages.gz') | ||||
.reply(404); | ||||
}); | ||||
|
||||
it('returns null for the package', async () => { | ||||
cfg.depName = 'you-will-never-find-me'; | ||||
const res = await getPkgReleases(cfg); | ||||
expect(res).toBeNull(); | ||||
}); | ||||
}); | ||||
|
||||
it('supports specifying a custom binary arch', async () => { | ||||
httpMock | ||||
.scope('http://ftp.debian.org') | ||||
.get('/debian/dists/stable/non-free/binary-riscv/Packages.gz') | ||||
.replyWithFile(200, testPackagesFile); | ||||
|
||||
cfg.registryUrls = [ | ||||
'http://ftp.debian.org/debian?suite=stable&components=non-free&binaryArch=riscv', | ||||
]; | ||||
|
||||
const res = await getPkgReleases(cfg); | ||||
expect(res).toBeObject(); | ||||
expect(res.releases).toHaveLength(1); | ||||
}); | ||||
}); | ||||
}); |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
why this file?