forked from electron/apps
-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fetch GitHub release data again (electron#332)
* update release data every four hours at most * re-enable release data fetching * fetch releases and readmes in separate scripts
- Loading branch information
Showing
9 changed files
with
32,856 additions
and
3,679 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| const apps = require('./raw-app-list')() | ||
| const parseGitUrl = require('github-url-to-object') | ||
|
|
||
| module.exports = apps | ||
| .filter(app => { | ||
| // inherit repository from website if possible | ||
| if (!app.repository && parseGitUrl(app.website)) app.repository = app.website | ||
| if (!app.repository) return false | ||
| if (!parseGitUrl(app.repository)) return false | ||
| return true | ||
| }) |
Large diffs are not rendered by default.
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| const MAX_CONCURRENCY = Number(process.env.MAX_CONCURRENCY) || 4 // simultaneous open web requests | ||
| const README_CACHE_TTL = require('human-interval')(process.env.README_CACHE_TTL || '4 hours') | ||
|
|
||
| const fs = require('fs') | ||
| const path = require('path') | ||
| const Bottleneck = require('bottleneck') | ||
| const github = require('../lib/github') | ||
| const cheerio = require('cheerio') | ||
| const parseGitUrl = require('github-url-to-object') | ||
|
|
||
| const outputFile = path.join(__dirname, '../meta/readmes.json') | ||
| const oldReadmeData = require(outputFile) | ||
| const output = {} | ||
| const limiter = new Bottleneck(MAX_CONCURRENCY) | ||
|
|
||
| const apps = require('../lib/raw-app-list')() | ||
| const appsWithRepos = require('../lib/apps-with-github-repos') | ||
| const appsToUpdate = appsWithRepos.filter(app => { | ||
| const oldData = oldReadmeData[app.slug] | ||
| if (!oldData) return true | ||
| const oldDate = new Date(oldData.readmeFetchedAt || null).getTime() | ||
| return oldDate + README_CACHE_TTL < Date.now() | ||
| }) | ||
|
|
||
| console.log(`${appsWithRepos.length} of ${apps.length} apps have a GitHub repo.`) | ||
| console.log(`${appsToUpdate.length} of those ${appsWithRepos.length} have missing or outdated README data.`) | ||
|
|
||
| appsToUpdate.forEach(app => { | ||
| limiter.schedule(getReadme, app) | ||
| }) | ||
|
|
||
| limiter.on('idle', () => { | ||
| fs.writeFileSync(outputFile, JSON.stringify(output, null, 2)) | ||
| console.log(`Done fetching README files.\nWrote ${outputFile}`) | ||
| process.exit() | ||
| }) | ||
|
|
||
| function getReadme (app) { | ||
| const {user: owner, repo} = parseGitUrl(app.repository) | ||
| const opts = { | ||
| owner: owner, | ||
| repo: repo, | ||
| headers: { | ||
| Accept: 'application/vnd.github.v3.html' | ||
| } | ||
| } | ||
|
|
||
| return github.repos.getReadme(opts) | ||
| .then(release => { | ||
| console.log(`${app.slug}: got latest README`) | ||
| output[app.slug] = { | ||
| readmeCleaned: cleanReadme(release.data, app), | ||
| readmeOriginal: release.data, | ||
| readmeFetchedAt: new Date() | ||
| } | ||
| }) | ||
| .catch(err => { | ||
| console.error(`${app.slug}: no README found`) | ||
| output[app.slug] = { | ||
| readmeOriginal: null, | ||
| readmeFetchedAt: new Date() | ||
| } | ||
| if (err.code !== 404) console.error(err) | ||
| }) | ||
| } | ||
|
|
||
| function cleanReadme (readme, app) { | ||
| const $ = cheerio.load(readme) | ||
|
|
||
| const $relativeImages = $('img').not('[src^="http"]') | ||
| if ($relativeImages.length) { | ||
| console.log(`${app.slug}: updating ${$relativeImages.length} relative image URLs`) | ||
| $relativeImages.each((i, img) => { | ||
| $(img).attr('src', `${app.repository}/raw/master/${$(img).attr('src')}`) | ||
| }) | ||
| } | ||
| return $('body').html() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,108 +1,62 @@ | ||
| const MAX_CONCURRENCY = Number(process.env.MAX_CONCURRENCY) || 4 // simultaneous open web requests | ||
| const RELEASE_CACHE_TTL = require('human-interval')(process.env.RELEASE_CACHE_TTL || '4 hours') | ||
|
|
||
| const fs = require('fs') | ||
| const path = require('path') | ||
| const Bottleneck = require('bottleneck') | ||
| const github = require('../lib/github') | ||
| const cheerio = require('cheerio') | ||
| const parseGitUrl = require('github-url-to-object') | ||
| const Duration = require('duration') | ||
| const downloadExtensions = [ | ||
| '.deb', | ||
| '.dmg', | ||
| '.exe', | ||
| '.gz', | ||
| '.rpm', | ||
| '.zip' | ||
| ] | ||
| const apps = require('../lib/raw-app-list')() | ||
| .filter(app => { | ||
| if (!app.repository) { | ||
| if (parseGitUrl(app.website)) { | ||
| console.log(`${app.name} website is a giturl: ${app.website}`) | ||
| app.repository = app.website | ||
| } | ||
| } | ||
| if (!app.repository) return false | ||
| if (!parseGitUrl(app.repository)) return false | ||
| let age = new Duration(new Date(app.releases_fetched_at || null), new Date()) | ||
| if (age.hours < 24) return false | ||
| return true | ||
| }) | ||
|
|
||
| const outputFile = path.join(__dirname, '../meta/releases.json') | ||
| const oldReleaseData = require(outputFile) | ||
| const output = {} | ||
| let i = -1 | ||
| const limiter = new Bottleneck(MAX_CONCURRENCY) | ||
|
|
||
| // Don't fetch release data too often | ||
| const outputFileAgeInHours = (new Date() - new Date(fs.statSync(outputFile).mtime)) / 1000 / 60 | ||
| if (outputFileAgeInHours < 1) { | ||
| console.log('Release data was updated less than an hour ago; skipping') | ||
| const apps = require('../lib/raw-app-list')() | ||
| const appsWithRepos = require('../lib/apps-with-github-repos') | ||
| const appsToUpdate = appsWithRepos.filter(app => { | ||
| const oldData = oldReleaseData[app.slug] | ||
| if (!oldData) return true | ||
| const oldDate = new Date(oldData.latestReleaseFetchedAt || null).getTime() | ||
| return oldDate + RELEASE_CACHE_TTL < Date.now() | ||
| }) | ||
|
|
||
| console.log(`${appsWithRepos.length} of ${apps.length} apps have a GitHub repo.`) | ||
| console.log(`${appsToUpdate.length} of those ${appsWithRepos.length} have missing or outdated release data.`) | ||
|
|
||
| appsToUpdate.forEach(app => { | ||
| limiter.schedule(getLatestRelease, app) | ||
| }) | ||
|
|
||
| limiter.on('idle', () => { | ||
| fs.writeFileSync(outputFile, JSON.stringify(output, null, 2)) | ||
| console.log(`Done fetching release data.\nWrote ${outputFile}`) | ||
| process.exit() | ||
| } else { | ||
| console.log('Fetching release data for apps that have a GitHub repo...') | ||
| } | ||
|
|
||
| go() | ||
|
|
||
| function go () { | ||
| ++i | ||
| }) | ||
|
|
||
| if (i === apps.length) { | ||
| fs.writeFileSync(outputFile, JSON.stringify(output, null, 2)) | ||
| process.exit() | ||
| } | ||
|
|
||
| const app = apps[i] | ||
| function getLatestRelease (app) { | ||
| const {user: owner, repo} = parseGitUrl(app.repository) | ||
| const gitHubOptions = { | ||
| const opts = { | ||
| owner: owner, | ||
| repo: repo, | ||
| headers: { | ||
| Accept: 'application/vnd.github.v3.html' | ||
| } | ||
| } | ||
|
|
||
| github.repos.getLatestRelease(gitHubOptions) | ||
| .then(release => { | ||
| console.log(app.slug) | ||
| output[app.slug] = { | ||
| latestRelease: release.data || false, | ||
| release_fetched_at: new Date() | ||
| } | ||
| if (release.data) { | ||
| output[app.slug].latestRelease = { | ||
| releaseUrl: release.data.html_url, | ||
| tagName: release.data.tag_name, | ||
| releaseName: release.data.name, | ||
| releaseNotes: release.data.body_html | ||
| return github.repos.getLatestRelease(opts) | ||
| .then(release => { | ||
| console.log(`${app.slug}: got latest release`) | ||
| output[app.slug] = { | ||
| latestRelease: release.data, | ||
| latestReleaseFetchedAt: new Date() | ||
| } | ||
| output[app.slug].latestRelease.downloads = release.data.assets.filter((asset) => { | ||
| let fileExtension = path.extname(asset.browser_download_url) | ||
| return (downloadExtensions.indexOf(fileExtension) !== -1) | ||
| }).map((asset) => { | ||
| return Object.assign({ | ||
| fileName: asset.name, | ||
| fileUrl: asset.browser_download_url | ||
| }) | ||
| }) | ||
| } | ||
| return github.repos.getReadme(gitHubOptions) | ||
| }).catch(() => { | ||
| output[app.slug] = { | ||
| latestRelease: false | ||
| } | ||
| return github.repos.getReadme(gitHubOptions) | ||
| }).then((response) => { | ||
| let readme = response.data | ||
| let $ = cheerio.load(readme) | ||
|
|
||
| const $relativeImages = $('img').not('[src^="http"]') | ||
| if ($relativeImages.length) { | ||
| console.log(`Updating relative image URLs in ${app.name}`) | ||
| $relativeImages.each((i, img) => { | ||
| $(img).attr('src', `${app.repository}/raw/master/${$(img).attr('src')}`) | ||
| }) | ||
| } | ||
|
|
||
| output[app.slug].originalReadme = readme | ||
| output[app.slug].readme = $('body').html() | ||
| go() | ||
| }) | ||
| }).catch(err => { | ||
| console.error(`${app.slug}: no releases found`) | ||
| output[app.slug] = { | ||
| latestRelease: null, | ||
| latestReleaseFetchedAt: new Date() | ||
| } | ||
| if (err.code !== 404) console.error(err) | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters