feat: API (#18)

.gitignore

@@ -1,5 +1,8 @@
 package-lock.json
-
+*.zip
+*.tar.gz
+*.tar
+.DS_Store
 # Logs
 logs
 *.log

api/config/creds.env.sample

@@ -0,0 +1 @@
+INFURA_API_KEY=""

api/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "yearn-tokenlist-api",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "build-list:all": "yarn build-list:erc20-mainnet && yarn build-list:erc721-mainnet && yarn build-list:erc1155-mainnet && yarn scrape-open-graph",
+    "build-list:erc20-mainnet": "ts-node ./scripts/erc20-mainnet.ts && yarn run lint:fix-json",
+    "build-list:erc721-mainnet": "ts-node ./scripts/erc721-mainnet.ts && yarn run lint:fix-json",
+    "build-list:erc1155-mainnet": "ts-node ./scripts/erc1155-mainnet.ts && yarn run lint:fix-json",
+    "scrape-open-graph": "ts-node ./scripts/open-graph-scrapper.ts && yarn run lint:fix-json",
+    "lint:fix-json": "prettier --write \"../index/**/*.json\""
+  },
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@nomiclabs/hardhat-ethers": "^2.0.1",
+    "@openzeppelin/contracts": "^3.3.0",
+    "@types/fs-extra": "^9.0.6",
+    "@types/lodash.isequal": "^4.5.5",
+    "@types/lodash.kebabcase": "^4.1.6",
+    "@types/node": "^14.14.20",
+    "@types/node-fetch": "^2.5.7",
+    "@uniswap/token-lists": "^1.0.0-beta.19",
+    "ajv": "6.12.2",
+    "ajv-formats": "^1.5.1",
+    "cli-progress": "^3.8.2",
+    "dotenv": "^8.2.0",
+    "ethers": "^5.0.26",
+    "fs-extra": "^9.0.1",
+    "hardhat": "^2.0.8",
+    "lodash.isequal": "^4.5.0",
+    "lodash.kebabcase": "^4.1.1",
+    "node-fetch": "^2.6.1",
+    "open-graph-scraper": "^4.7.1",
+    "prettier": "^2.2.1",
+    "ts-node": "^9.1.1",
+    "typescript": "^4.1.3"
+  }
+}

api/scripts/erc1155-mainnet.ts

@@ -0,0 +1,157 @@
+import * as fs from "fs"
+import * as ethers from 'ethers'
+import fetch from "node-fetch"
+import { nextVersion, schema, VersionUpgrade, YearnList, YearnInfo } from '@0xsequence/collectible-lists'
+import { getEnvConfig } from "../src/utils"
+const Ajv = require("ajv")
+const isEqual = require("lodash.isequal")
+const cliProgress = require('cli-progress');
+
+// Loading jsons
+const erc1155json = require("@openzeppelin/contracts/build/contracts/ERC1155.json")
+const erc1155Dump: TokenDump[] = require("../src/data/erc1155_dune_dump_2021_01_25.json")
+const erc1155: YearnList = require("../../index/mainnet/erc1155.json")
+
+// Build ERC-1155 list
+// 1. Load crv dump from Dune analytics
+// 2. Query contract info via opensea API
+// 3. Build list according to @0xsequence/collectible-lists
+
+// List to fetch
+const ERC1155_LIST_PATH = "../index/mainnet/erc1155.json"
+const config = getEnvConfig()
+const provider = new ethers.providers.InfuraProvider('mainnet', config['INFURA_API_KEY'])
+
+interface TokenDump {
+  name: string;
+  address: string;
+  n_transfers: number;
+}
+
+// Building list
+const main = async () => {
+
+  // Create token information array
+  let newYearnList: YearnInfo[] = []
+  const erc1155Contracts: string[] = [...new Set([...erc1155Dump.map(t => t.address)])]
+  const errors: any = []
+
+  // Progress bar init
+  console.log('Building ERC-1155 mainnet list')
+  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+  progressBar.start(erc1155Contracts.length, 0)
+  for (let i= 0; i < erc1155Contracts.length; i++) {
+    let resp
+    try {
+      resp = await fetch('https://api.opensea.io/api/v1/asset_contract/' + erc1155Contracts[i])
+    } catch (err) {
+      console.log(err)
+    }
+    while (resp && resp.status == 429) {
+      await new Promise(r => setTimeout(r, 5000));
+      try {
+        resp = await fetch('https://api.opensea.io/api/v1/asset_contract/' + erc1155Contracts[i])
+      } catch (err) {
+        console.log(err)
+      }
+    }
+    if (!resp || !resp.ok) {
+      errors.push({
+        id: i,
+        address: erc1155Contracts[i],
+        resp: !resp ? null : resp.status + ' - ' + resp.statusText
+      })
+      progressBar.update(i+1)
+      continue
+    }
+    const info = await resp.json()
+
+    // Query symbol on contract if couldn't find it
+    let validSymbol
+    if (!info.symbol || info.symbol === "") {
+      const erc1155contract = new ethers.Contract(erc1155Contracts[i], erc1155json.abi, provider)
+      try {
+        validSymbol = await erc1155contract.symbol()
+      } catch {
+        validSymbol = ""
+      }
+    } else {
+      validSymbol = info.symbol
+    }
+
+    // Force some basic validation so they are compatible with schema
+    validSymbol = validSymbol.length <= 20 ? validSymbol : validSymbol.slice(0,20)
+    const validName = !info.name || info.name.length <= 64 ? info.name : info.name.slice(0,64)
+    const validDescription = !info.description || info.description.length <= 1000 ? info.description : info.description.slice(0,997) + '...'
+
+    // Append token to list
+    newYearnList.push({
+      chainId: 1,
+      address: erc1155Contracts[i],
+      name: validName,
+      standard: "erc1155",
+      symbol: validSymbol === "" ? null : validSymbol,
+      logoURI: !info.image_url || info.image_url === "" ? null : info.image_url,
+      extensions: {
+        "link": !info.external_link || info.external_link === "" ? null : info.external_link,
+        "description": !validDescription || validDescription === "" ? null : validDescription
+      }
+    })
+
+    progressBar.update(i+1)
+  }
+  progressBar.stop()
+
+  // Print contracts that were ignored and why
+  if (errors.length > 0) {
+    console.log('Contracts ignored')
+    console.log(errors)
+    console.log('\n')
+  }
+
+  // Validate the list fetched against current YearnList schema1
+  const ajv = new Ajv()
+  const validateList = ajv.compile(schema)
+
+  // Update token list version
+  // Increment minor version when tokens are added
+  // Increment patch version when tokens already on the list have details changed
+  const newErc1155List = {
+    ...erc1155,
+    timestamp: (new Date()).toISOString(),
+    tokens: newYearnList,
+    version: nextVersion(erc1155.version, newYearnList.length > erc1155.tokens.length ? VersionUpgrade.MINOR : VersionUpgrade.PATCH)
+  }
+
+  // Validate list against schema
+  if (!validateList(newErc1155List)) {
+    console.log("New list has invalid schema: ")
+    console.log(validateList.errors)
+    //throw Error("^^^")
+  }
+
+  // Check whether list changed or not (except version)
+  if (isEqual(newErc1155List.tokens, erc1155.tokens)) {
+    console.log("List is already up-to-date")
+    return
+  } 
+
+  // Store latest erc-1155 tokens list
+  fs.writeFile(
+    ERC1155_LIST_PATH,
+    JSON.stringify(newErc1155List),
+    { flag: "w+" },
+    function (err) {
+      if (err) throw err
+      console.log("ERC-1155 Mainnet List Updated")
+    }
+  )
+}
+
+main()
+  .then(() => {
+    console.log("Finished")
+  })
+  .catch((error) => {
+    console.error(error)
+  })

api/scripts/erc20-mainnet.ts

@@ -0,0 +1,109 @@
+import * as fs from "fs"
+const Ajv = require("ajv")
+import fetch from "node-fetch"
+import * as ethers from 'ethers'
+import { TokenList, schema, nextVersion, VersionUpgrade } from "@uniswap/token-lists"
+import { getEnvConfig } from "../src/utils"
+const isEqual = require("lodash.isequal")
+const cliProgress = require('cli-progress');
+const erc20json = require("@openzeppelin/contracts/build/contracts/ERC20.json")
+const erc20: TokenList = require("../../index/mainnet/erc20.json")
+
+// List to fetch
+const ERC20_LIST_URL = "https://tokens.coingecko.com/uniswap/all.json"
+const ERC20_LIST_PATH = "../index/mainnet/erc20.json"
+const config = getEnvConfig()
+const provider = new ethers.providers.InfuraProvider('mainnet', config['INFURA_API_KEY'])
+
+const main = async () => {
+  // Fetch ERC-20 token list
+  const newList: TokenList = await (await fetch(ERC20_LIST_URL)).json()
+
+  // Enchance coingecko list with description and link
+  console.log('Fetch ERC20 tokens information')
+  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+  progressBar.start(newList.tokens.length, 0)
+  for (let i=0; i < newList.tokens.length; i++) {
+    let resp = await fetch('https://api.coingecko.com/api/v3/coins/ethereum/contract/' + newList.tokens[i].address)
+    while (resp.status == 429) {
+      await new Promise(r => setTimeout(r, 5000));
+      resp = await fetch('https://api.coingecko.com/api/v3/coins/ethereum/contract/' + newList.tokens[i].address)
+    }
+    if (!resp.ok) {
+      console.log('Error: ' + resp.statusText)
+      progressBar.update(i+1)
+      continue
+    }
+    const info = await resp.json()
+
+    let validSymbol
+    if (!info.symbol || info.symbol === "") {
+      const erc20contract = new ethers.Contract(newList.tokens[i].address, erc20json.abi, provider)
+      try {
+        validSymbol = await erc20contract.symbol()
+      } catch {
+        validSymbol = ""
+      }
+    } else {
+      validSymbol = info.symbol
+    }
+
+    validSymbol = validSymbol.length <= 20 ? validSymbol : validSymbol.slice(0,20)
+    const validDescription = !info.description.en || info.description.en.length <= 1000 ? info.description.en : info.description.en.slice(0,997) + '...'
+
+    newList.tokens[i] = {
+      ...newList.tokens[i],
+      //@ts-ignore
+      extensions: {
+        "link": !info.links.homepage[0] || info.links.homepage[0] === "" ? null : info.links.homepage[0],
+        "description": !validDescription || validDescription === "" ? null : validDescription
+      }
+    }
+
+    progressBar.update(i+1)
+  }
+  progressBar.stop()
+
+  const newErc20List = {
+    ...newList,
+    timestamp: (new Date()).toISOString(),
+    tokens: newList.tokens,
+    version: nextVersion(erc20.version, newList.tokens.length > erc20.tokens.length ? VersionUpgrade.MINOR : VersionUpgrade.PATCH)
+  }
+
+  // Validate the list fetched against current TokenList schema1
+  const ajv = new Ajv()
+  const validateList = ajv.compile(schema)
+
+  // Validate list against schema
+  if (!validateList(newErc20List)) {
+    console.log("New list has invalid schema: ")
+    //console.log(validateList.errors)
+    //throw Error("^^^")
+  }
+
+  // Check whether list changed or not
+  if (isEqual(newErc20List.tokens, erc20.tokens)) {
+    console.log("List is already up-to-date")
+    return
+  } 
+
+  // Store latest erc-20 tokens list
+  fs.writeFile(
+    ERC20_LIST_PATH,
+    JSON.stringify(newErc20List),
+    { flag: "w+" },
+    function (err) {
+      if (err) throw err
+      console.log("ERC-20 Mainnet List Updated")
+    }
+  )
+}
+
+main()
+  .then(() => {
+    console.log("Finished")
+  })
+  .catch((error) => {
+    console.error(error)
+  })