Revert "delete unused folder"

This reverts commit 2b66e8b.

lib/app-categories.js

@@ -0,0 +1,28 @@
+module.exports = [
+  'Books',
+  'Business',
+  'Catalogs',
+  'Developer Tools',
+  'Education',
+  'Entertainment',
+  'Finance',
+  'Food & Drink',
+  'Games',
+  'Health & Fitness',
+  'Graphics & Design',
+  'Lifestyle',
+  'Kids',
+  'Magazines & Newspapers',
+  'Music',
+  'Navigation',
+  'News',
+  'Photo & Video',
+  'Productivity',
+  'Reference',
+  'Science & Medicine',
+  'Shopping',
+  'Social Networking',
+  'Sports',
+  'Travel',
+  'Utilities',
+]

lib/apps-with-github-repos.js

@@ -0,0 +1,10 @@
+const apps = require('./raw-app-list')()
+const parseGitUrl = require('github-url-to-object')
+
+module.exports = apps.filter((app) => {
+  // inherit repository from website if possible
+  if (!app.repository && parseGitUrl(app.website)) app.repository = app.website
+  if (!app.repository) return false
+  if (!parseGitUrl(app.repository)) return false
+  return true
+})

lib/broken-links.js

@@ -0,0 +1,93 @@
+const fetch = require('node-fetch')
+const fsPromises = require('fs').promises
+const isUrl = require('is-url')
+const path = require('path')
+const readdirp = require('readdirp')
+const yaml = require('yaml')
+
+const topDir = path.dirname(__dirname)
+
+// walk an object subtree looking for URL strings
+const getObjectUrls = (root) => {
+  const urls = []
+  const queue = [root]
+  while (queue.length !== 0) {
+    const vals = Object.values(queue.shift())
+    urls.push(...vals.filter(isUrl))
+    queue.push(...vals.filter((v) => typeof v === 'object'))
+  }
+  return urls
+}
+
+// scrape a url to see if the link is broken.
+// return a Promise that resolves as { url, err }
+const scrape = (url) =>
+  fetch(url, { method: 'HEAD' }) // just scrape headers; body not needed
+    .then(
+      (res) => ({
+        url,
+        err: res.ok ? null : `${res.status} ${res.statusText}`,
+        status: res.status,
+      }),
+      (err) => ({ url, err, status: -2 })
+    )
+
+// scrape all the urls found in a yml file.
+// report broken links to console.log().
+// return a Promise that resolves as an array of broken links: [ { url, err }, ... ]
+const processYmlEntry = (entry) =>
+  fsPromises
+    .readFile(entry.fullPath, { encoding: 'utf8' })
+    .then((file) => {
+      try {
+        return yaml.parse(file)
+      } catch (error) {
+        console.error(`Failed to parse ${entry.path}. Skipping.`)
+        return { disabled: true }
+      }
+    })
+    .then((o) => (o.disabled ? [] : getObjectUrls(o)))
+    .then(async (urls) => {
+      const results = []
+
+      for (const url of urls) {
+        // Scrape one by one to handle rate limiting
+        const r = await scrape(url)
+        results.push(r)
+      }
+
+      return results
+    })
+    .then((results) => results.filter((res) => !!res.err))
+    .then((fails) => {
+      fails.forEach((f) => console.log(`${entry.path} - ${f.url} (${f.err})`))
+      return fails
+    })
+
+const findBrokenLinks = (start = 0, end = Infinity) =>
+  readdirp
+    .promise(topDir, {
+      fileFilter: '*.yml',
+      directoryFilter: (entry) => entry.path.startsWith('apps'),
+    })
+    .then(async (entries) => {
+      const result = []
+      let limitedEntries = entries
+
+      if (start !== 0 || end !== Infinity) {
+        limitedEntries = entries.slice(start, end)
+      }
+
+      for (const entry of limitedEntries) {
+        console.log(`Processing ${entry.path}`)
+        result.push({
+          entry,
+          result: await processYmlEntry(entry),
+        })
+      }
+
+      return result
+    })
+    .then((arr) => arr.filter((inner) => !!inner.result.length))
+
+module.exports = findBrokenLinks

lib/colors.js

@@ -0,0 +1,90 @@
+'use strict'
+
+const fs = require('fs')
+const colorConvert = require('color-convert')
+const getImageColors = require('get-image-colors')
+const mime = require('mime-types')
+const path = require('path')
+const pickAGoodColor = require('pick-a-good-color')
+const revHash = require('rev-hash')
+const stringify = require('json-stable-stringify')
+
+/**
+ * Generates good colors for an image.
+ *
+ * @param slugsAndIconPaths [ { slug: foo, iconPath: bar } ... ]
+ * @param oldColors: reference colors from previous call to getColors()
+ * @param root: repo toplevel directory so that saved iconPaths are relative to it
+ * @return { slug: { palette, goodColorOnWhite, goodColorOnBlack, faintColorOnWhite, source: { revHash, iconPath } }
+ */
+async function getColors(slugsAndIconPaths, oldColors, root) {
+  return Promise.all(
+    slugsAndIconPaths.map(async (app) => {
+      const slug = app.slug
+      try {
+        const data = fs.readFileSync(app.iconPath)
+        const hash = revHash(data)
+
+        // if nothing's changed, don't recalculate
+        let o = oldColors[slug]
+        if (o && o.source && o.source.revHash === hash) return { [slug]: o }
+
+        console.info(`calculating good colors for ${slug}`)
+        return await getImageColors(data, mime.lookup(app.iconPath)).then(
+          (iconColors) => {
+            const palette = iconColors.map((color) => color.hex())
+            const goodColorOnWhite = pickAGoodColor(palette)
+            const goodColorOnBlack = pickAGoodColor(palette, {
+              background: 'black',
+            })
+            const faintColorOnWhite = `rgba(${colorConvert.hex
+              .rgb(goodColorOnWhite)
+              .join(', ')}, 0.1)`
+            return {
+              [slug]: {
+                source: {
+                  revHash: hash,
+                  path: path.relative(root, app.iconPath),
+                },
+                palette,
+                goodColorOnWhite,
+                goodColorOnBlack,
+                faintColorOnWhite,
+              },
+            }
+          }
+        )
+      } catch (e) {
+        console.error(`Error processing ${app.iconPath}`, e)
+      }
+    })
+  ).then((values) => Object.assign({}, ...values))
+}
+
+/**
+ * Wrapper around getColors() that uses the same file for input & output,
+ * refreshing the file when the data changes
+ *
+ * @param slugsAndIconPaths [ { slug: foo, iconPath: bar } ... ]
+ * @param colorsFile: the file that keeps the list of complimentary colors
+ * @param root: repo toplevel directory so that saved iconPaths are relative to it
+ */
+const rebuildColorFile = (slugsAndIconPaths, colorsFile, root) => {
+  let oldColors
+  try {
+    oldColors = require(colorsFile)
+  } catch (e) {
+    oldColors = {}
+  }
+
+  getColors(slugsAndIconPaths, oldColors, root).then((colors) => {
+    try {
+      fs.writeFileSync(colorsFile, stringify(colors, { space: 2 }))
+    } catch (e) {
+      console.error(`Error writing ${colorsFile}`, e)
+    }
+  })
+}
+
+module.exports = rebuildColorFile
+module.exports.getColors = getColors

lib/github.js

@@ -0,0 +1,10 @@
+if (!process.env.GH_TOKEN) {
+  require('dotenv-safe').config()
+}
+
+const { Octokit } = require('@octokit/rest')
+const github = new Octokit({
+  auth: process.env.GH_TOKEN,
+})
+
+module.exports = github