Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: increase github rate limit with multiple PATs #58

Merged
merged 2 commits into from
Jul 15, 2020
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
refactor: refactored retryer logic & handled invalid tokens
  • Loading branch information
anuraghazra committed Jul 15, 2020
commit 429d65a52bdd1618ca022b2577e1e080d1fe239a
2 changes: 1 addition & 1 deletion api/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ module.exports = async (req, res) => {
} = req.query;
let stats;

res.setHeader("Cache-Control", "public, max-age=300");
res.setHeader("Cache-Control", "public, max-age=1800");
res.setHeader("Content-Type", "image/svg+xml");

try {
Expand Down
2 changes: 1 addition & 1 deletion api/pin.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ module.exports = async (req, res) => {

let repoData;

res.setHeader("Cache-Control", "public, max-age=300");
res.setHeader("Cache-Control", "public, max-age=1800");
res.setHeader("Content-Type", "image/svg+xml");

try {
Expand Down
24 changes: 13 additions & 11 deletions src/fetchRepo.js
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
const { request } = require("./utils");
const retryer = require("./retryer");

async function fetchRepo(username, reponame) {
if (!username || !reponame) {
throw new Error("Invalid username or reponame");
}

const res = await request(
const fetcher = (variables, token) => {
return request(
{
query: `
fragment RepoInfo on Repository {
Expand Down Expand Up @@ -34,15 +31,20 @@ async function fetchRepo(username, reponame) {
}
}
`,
variables: {
login: username,
repo: reponame,
},
variables,
},
{
Authorization: `bearer ${process.env.PAT_1}`,
Authorization: `bearer ${token}`,
}
);
};

async function fetchRepo(username, reponame) {
if (!username || !reponame) {
throw new Error("Invalid username or reponame");
}

let res = await retryer(fetcher, { login: username, repo: reponame });

const data = res.data.data;

Expand Down
36 changes: 4 additions & 32 deletions src/fetchStats.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
const { request } = require("./utils");
const retryer = require("./retryer");
const calculateRank = require("./calculateRank");
require("dotenv").config();

// creating a fetcher function to reduce duplication
const fetcher = (username, token) => {
const fetcher = (variables, token) => {
return request(
{
query: `
Expand Down Expand Up @@ -37,43 +37,15 @@ const fetcher = (username, token) => {
}
}
`,
variables: { login: username },
variables,
},
{
// set the token
Authorization: `bearer ${token}`,
}
);
};

async function retryer(username, RETRIES) {
try {
console.log(`Trying PAT_${RETRIES + 1}`);

// try to fetch with the first token since RETRIES is 0 index i'm adding +1
let response = await fetcher(username, process.env[`PAT_${RETRIES + 1}`]);

// if rate limit is hit increase the RETRIES and recursively call the retryer
// with username, and current RETRIES
if (
response.data.errors &&
response.data.errors[0].type === "RATE_LIMITED"
) {
console.log(`PAT_${RETRIES} Failed`);
RETRIES++;
// directly return from the function
return await retryer(username, RETRIES);
}

// finally return the response
return response;
} catch (err) {
console.log(err);
}
}

async function fetchStats(username) {
let RETRIES = 0;
if (!username) throw Error("Invalid username");

const stats = {
Expand All @@ -86,7 +58,7 @@ async function fetchStats(username) {
rank: { level: "C", score: 0 },
};

let res = await retryer(username, RETRIES);
let res = await retryer(fetcher, { login: username });

if (res.data.errors) {
console.log(res.data.errors);
Expand Down
43 changes: 43 additions & 0 deletions src/retryer.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
const retryer = async (fetcher, variables, retries = 0) => {
if (retries > 7) {
throw new Error("Maximum retries exceeded");
}
try {
console.log(`Trying PAT_${retries + 1}`);

// try to fetch with the first token since RETRIES is 0 index i'm adding +1
let response = await fetcher(
variables,
process.env[`PAT_${retries + 1}`],
retries
);

// prettier-ignore
const isRateExceeded = response.data.errors && response.data.errors[0].type === "RATE_LIMITED";

// if rate limit is hit increase the RETRIES and recursively call the retryer
// with username, and current RETRIES
if (isRateExceeded) {
console.log(`PAT_${retries + 1} Failed`);
retries++;
// directly return from the function
return retryer(fetcher, variables, retries);
}

// finally return the response
return response;
} catch (err) {
// prettier-ignore
// also checking for bad credentials if any tokens gets invalidated
const isBadCredential = err.response.data && err.response.data.message === "Bad credentials";

if (isBadCredential) {
console.log(`PAT_${retries + 1} Failed`);
retries++;
// directly return from the function
return retryer(fetcher, variables, retries);
}
}
};

module.exports = retryer;
50 changes: 50 additions & 0 deletions tests/retryer.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
require("@testing-library/jest-dom");
const retryer = require("../src/retryer");

const fetcher = jest.fn((variables, token) => {
console.log(variables, token);
return new Promise((res, rej) => res({ data: "ok" }));
});

const fetcherFail = jest.fn(() => {
return new Promise((res, rej) =>
res({ data: { errors: [{ type: "RATE_LIMITED" }] } })
);
});

const fetcherFailOnSecondTry = jest.fn((_vars, _token, retries) => {
return new Promise((res, rej) => {
// faking rate limit
if (retries < 1) {
return res({ data: { errors: [{ type: "RATE_LIMITED" }] } });
}
return res({ data: "ok" });
});
});

describe("Test Retryer", () => {
it("retryer should return value and have zero retries on first try", async () => {
let res = await retryer(fetcher, {});

expect(fetcher).toBeCalledTimes(1);
expect(res).toStrictEqual({ data: "ok" });
});

it("retryer should return value and have 2 retries", async () => {
let res = await retryer(fetcherFailOnSecondTry, {});

expect(fetcherFailOnSecondTry).toBeCalledTimes(2);
expect(res).toStrictEqual({ data: "ok" });
});

it("retryer should throw error if maximum retries reached", async () => {
let res;

try {
res = await retryer(fetcherFail, {});
} catch (err) {
expect(fetcherFail).toBeCalledTimes(8);
expect(err.message).toBe("Maximum retries exceeded");
}
});
});