Skip to content

Fix review scraping if reviewCount is zero #37

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Fix review scraping if reviewCount is zero #37

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6ec9be4
Fix review scraping if reviewCount is zero
Jeshwin Apr 11, 2025
0f673e3
Unfix the fix
Jeshwin Apr 11, 2025
cf13b1e
Add sort options to fetching reviews
Jeshwin Apr 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 97 additions & 91 deletions controllers/reviewsController.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,104 +1,110 @@
const gplay = require("google-play-scraper");
const {jsonToCsv} = require('../utilities/jsonToCsv');
const { jsonToCsv } = require("../utilities/jsonToCsv");

const MAX_REVIEWS_COUNT = 100000; //Fixed value for now, can be updated for future development

const fetchReviews = async (appId, reviewsCount, countryCode) => {
const options = {
appId: appId,
sort: gplay.sort.NEWEST,
lang: 'en',
country: countryCode
};

try {
let reviews = [];
let nextToken;
let totalFetched = 0;
let numReviews = reviewsCount && reviewsCount < MAX_REVIEWS_COUNT ? reviewsCount : MAX_REVIEWS_COUNT;
console.log(`Fetching ${numReviews} Reviews for AppId: ${appId}`);

while (totalFetched < numReviews) {
if (nextToken) {
options.nextPaginationToken = nextToken;
}

const result = await gplay.reviews(options);
const newData = result.data || [];

// Process each review to include the criteria section
const processedReviews = newData.map(review => {
const criterias = review.criterias || []; // Get criterias section
const criteriaData = criterias.map(criteria => `criteria: ${criteria.criteria}: rating: ${criteria.rating}`).join('; '); // Convert criterias to string
return {
...review,
criterias: criteriaData
};
});

reviews = reviews.concat(processedReviews);
totalFetched += newData.length;
console.log(`Total reviews fetched so far: ${totalFetched}`);
nextToken = result.nextPaginationToken;
const fetchReviews = async (appId, reviewsCount, countryCode, sort) => {
const options = {
appId: appId,
lang: "en",
country: countryCode,
sort: sort == "Recency" ? gplay.sort.NEWEST : sort == "Rating" ? gplay.sort.RATING : gplay.sort.HELPFULNESS,
};

try {
let reviews = [];
let nextToken;
let totalFetched = 0;

if (reviewsCount === undefined) {
throw new Error('"reviewsCount is undefined :(" - Jeshwin');
}

let numReviews = reviewsCount < MAX_REVIEWS_COUNT ? reviewsCount : MAX_REVIEWS_COUNT;
console.log(`Fetching ${numReviews} Reviews for AppId: ${appId}`);

while (totalFetched < numReviews) {
if (nextToken) {
options.nextPaginationToken = nextToken;
}

const result = await gplay.reviews(options);
const newData = result.data || [];

// Process each review to include the criteria section
const processedReviews = newData.map((review) => {
const criterias = review.criterias || []; // Get criterias section
const criteriaData = criterias
.map((criteria) => `criteria: ${criteria.criteria}: rating: ${criteria.rating}`)
.join("; "); // Convert criterias to string
return {
...review,
criterias: criteriaData,
};
});

reviews = reviews.concat(processedReviews);
totalFetched += newData.length;
console.log(`Total reviews fetched so far: ${totalFetched}`);
nextToken = result.nextPaginationToken;
}

if (reviews.length > numReviews) {
reviews = reviews.slice(0, numReviews);
}

return reviews;
} catch (error) {
console.error("Error scraping reviews:", error);
throw new Error("An error occurred while scraping reviews.");
}

if(reviews.length > numReviews){
reviews = reviews.slice(0, numReviews);
}

return reviews;
} catch (error) {
console.error("Error scraping reviews:", error);
throw new Error("An error occurred while scraping reviews.");
}
};

const scrapeReviews = async (req, res) => {
// console.log(req);
// console.log("\n");
const appId = req.query.appId;
const countryCode = req.query.countryCode;
// console.log("AppID: %s\n", appId);
// console.log("CountryCode: %s\n", countryCode);

try {
// Get the actual count of reviews
const appDetails = await gplay.app({ appId: appId, country: countryCode });
const reviewsCount = appDetails.reviews;

console.log(`App ${appId} contains ${reviewsCount} reviews`);

// Fetch reviews based on the count or the maximum limit
const reviews = await fetchReviews(appId, reviewsCount, countryCode);
console.log(`Received ${reviews.length} reviews for app: ${appId}`);

console.log("First 10 reviews:");
for (let i = 0; i < Math.min(10, reviews.length); i++) {
console.log(JSON.stringify(reviews[i]));
// console.log(req);
// console.log("\n");
const appId = req.query.appId;
const countryCode = req.query.countryCode;
const sortBy = req.query.sortBy;
// console.log("AppID: %s\n", appId);
// console.log("CountryCode: %s\n", countryCode);

try {
// Get the actual count of reviews
const appDetails = await gplay.app({ appId: appId, country: countryCode });
const reviewsCount = appDetails.reviews;

console.log(`App ${appId} contains ${reviewsCount} reviews`);

// Fetch reviews based on the count or the maximum limit
const reviews = await fetchReviews(appId, reviewsCount, countryCode, sortBy);
console.log(`Received ${reviews.length} reviews for app: ${appId}`);

console.log("First 10 reviews:");
for (let i = 0; i < Math.min(10, reviews.length); i++) {
console.log(JSON.stringify(reviews[i]));
}

res.set("Access-Control-Allow-Origin", "*");
const country_reviews = [...reviews.map((review) => ({ ...review, country: countryCode }))];

// Convert reviews data to CSV format
const csvData = jsonToCsv(country_reviews, "reviews");

res.setHeader("Access-Control-Expose-Headers", "Content-Disposition");

// Set response headers for CSV download
res.setHeader("Content-Disposition", `attachment; filename="${appId}_reviews.csv"`);
res.setHeader("Content-Type", "text/csv");

// Send the CSV data as a response
res.status(200).send(csvData);
} catch (error) {
console.error("Error getting reviews:", error);
res.status(500).json({ error: "An error occurred while getting reviews." });
}

res.set("Access-Control-Allow-Origin", "*");
const country_reviews = [
...reviews.map((review) => ({ ...review, country: countryCode}))];

// Convert reviews data to CSV format
const csvData = jsonToCsv(country_reviews, 'reviews');

res.setHeader('Access-Control-Expose-Headers', 'Content-Disposition');

// Set response headers for CSV download
res.setHeader('Content-Disposition', `attachment; filename="${appId}_reviews.csv"`);
res.setHeader('Content-Type', 'text/csv');

// Send the CSV data as a response
res.status(200).send(csvData);

} catch (error) {
console.error("Error getting reviews:", error);
res.status(500).json({ error: "An error occurred while getting reviews." });
}
};

module.exports = {
scrapeReviews,
scrapeReviews,
};