diff --git a/conf/nginx/sites-available/off b/conf/nginx/sites-available/off index dcbfebeef7666..0abce759dc10f 100644 --- a/conf/nginx/sites-available/off +++ b/conf/nginx/sites-available/off @@ -53,6 +53,29 @@ server { gunzip on; } + # Add an HTTP 302 redirect to AWS S3 bucket for specific dump files + location = /data/openfoodfacts_recent_changes.jsonl.gz { + return 302 https://openfoodfacts-ds.s3.eu-west-3.amazonaws.com/openfoodfacts_recent_changes.jsonl.gz; + } + location = /data/openfoodfacts-mongodbdump.gz { + return 302 https://openfoodfacts-ds.s3.eu-west-3.amazonaws.com/openfoodfacts-mongodbdump.gz; + } + location = /data/openfoodfacts-products.jsonl.gz { + return 302 https://openfoodfacts-ds.s3.eu-west-3.amazonaws.com/openfoodfacts-products.jsonl.gz; + } + location = /data/en.openfoodfacts.org.products.csv { + return 302 https://openfoodfacts-ds.s3.eu-west-3.amazonaws.com/en.openfoodfacts.org.products.csv; + } + location = /data/en.openfoodfacts.org.products.csv.gz { + return 302 https://openfoodfacts-ds.s3.eu-west-3.amazonaws.com/en.openfoodfacts.org.products.csv.gz; + } + location = /data/fr.openfoodfacts.org.products.csv { + return 302 https://openfoodfacts-ds.s3.eu-west-3.amazonaws.com/fr.openfoodfacts.org.products.csv; + } + location = /data/fr.openfoodfacts.org.products.csv.gz { + return 302 https://openfoodfacts-ds.s3.eu-west-3.amazonaws.com/fr.openfoodfacts.org.products.csv.gz; + } + if ($http_referer ~* (jobothoniel.com) ) { return 403; } # blocked since 2021-07-13 # the app requests /1.json to get the product count... diff --git a/scripts/gen_feeds_daily_off.sh b/scripts/gen_feeds_daily_off.sh index b2401a4af01e9..aabd82ba49911 100755 --- a/scripts/gen_feeds_daily_off.sh +++ b/scripts/gen_feeds_daily_off.sh @@ -21,6 +21,16 @@ for export in en.openfoodfacts.org.products.csv fr.openfoodfacts.org.products.cs mv -f new.$export.gz $export.gz done +# Copy CSV and RDF files to AWS S3 using MinIO client +mc cp \ + en.openfoodfacts.org.products.csv \ + en.openfoodfacts.org.products.csv.gz \ + en.openfoodfacts.org.products.rdf \ + fr.openfoodfacts.org.products.csv \ + fr.openfoodfacts.org.products.csv.gz \ + fr.openfoodfacts.org.products.rdf \ + s3/openfoodfacts-ds + # Generate the MongoDB dumps and jsonl export cd /srv/off/scripts diff --git a/scripts/mongodb_dump.sh b/scripts/mongodb_dump.sh index fe206172f08b2..dc7f43afeadf2 100755 --- a/scripts/mongodb_dump.sh +++ b/scripts/mongodb_dump.sh @@ -60,4 +60,11 @@ popd > /dev/null # data/delta mongoexport --collection recent_changes --host $HOST --db $DB --fields=_id,comment,code,userid,rev,countries_tags,t,diffs | gzip -9 > "new.${PREFIX}_recent_changes.jsonl.gz" && \ mv new.${PREFIX}_recent_changes.jsonl.gz ${PREFIX}_recent_changes.jsonl.gz +# Copy files to AWS S3 using MinIO client +mc cp \ + ${PREFIX}-products.jsonl.gz \ + ${PREFIX}_recent_changes.jsonl.gz \ + ${PREFIX}-mongodbdump.gz \ + s3/openfoodfacts-ds + popd > /dev/null # data