CI - data produced by crawler are statically served #406

circle.yml

@@ -18,9 +18,16 @@ jobs:
       - run: npm install .
       - run: gem install bundler
       - run: bundle install
-      - run: which rsync || sudo apt-get install rsync
+      - run: which rsync || sudo apt-get install rsync # rsync is needed for swagger build, keep it
       - run: make include-npm-deps
       - run: ./_buildscripts/build-swagger.sh
+      - run: wget https://developers.italia.it/crawler/softwares.yml -O _data/crawler/softwares.yml
+      - run: wget https://developers.italia.it/crawler/amministrazioni.yml -O _data/crawler/amministrazioni.yml
+      - run: wget https://developers.italia.it/crawler/software_categories.yml -O _data/crawler/software_categories.yml
+      - run: wget https://developers.italia.it/crawler/software-open-source.yml -O _data/crawler/software-open-source.yml
+      - run: wget https://developers.italia.it/crawler/software-riuso.yml -O _data/crawler/software-riuso.yml
+      - run: wget https://developers.italia.it/crawler/software_scopes.yml -O _data/crawler/software_scopes.yml
+      - run: wget https://developers.italia.it/crawler/software_tags.yml -O _data/crawler/software_tags.yml
       - run: bundle exec jekyll build
       - run: bundle exec htmlproofer ./_site --assume-extension --check-html --allow-hash-href --empty-alt-ignore --only-4xx --disable-external --url_ignore "/esQuery\.config\.js/"
 
@@ -42,7 +49,13 @@ jobs:
       - run: mkdir -p ~/.ssh
       - run: echo $SSH_KNOWN_HOSTS | base64 -d >> ~/.ssh/known_hosts
       - run: which rsync || sudo apt-get install rsync
-      - run: rsync --delete -avP --rsync-path="sudo -u www-data rsync" circleci@developers.italia.it:/data/crawler/output/ ./_data/crawler/
+      - run: wget https://developers.italia.it/crawler/softwares.yml -O _data/crawler/softwares.yml
+      - run: wget https://developers.italia.it/crawler/amministrazioni.yml -O _data/crawler/amministrazioni.yml
+      - run: wget https://developers.italia.it/crawler/software_categories.yml -O _data/crawler/software_categories.yml
+      - run: wget https://developers.italia.it/crawler/software-open-source.yml -O _data/crawler/software-open-source.yml
+      - run: wget https://developers.italia.it/crawler/software-riuso.yml -O _data/crawler/software-riuso.yml
+      - run: wget https://developers.italia.it/crawler/software_scopes.yml -O _data/crawler/software_scopes.yml
+      - run: wget https://developers.italia.it/crawler/software_tags.yml -O _data/crawler/software_tags.yml
       - run: ./_buildscripts/build-swagger.sh
       - run: bundle exec jekyll build
       - run: cp -r ./github ./_site/