Merge pulls bitcoin-dot-org#1048 and bitcoin-dot-org#1051

- 1048: Conditionally build some pages differently based on environmental variables
- 1051: Backend: cache events and contributor listings

.gitignore

@@ -9,3 +9,4 @@ ehthumbs.db
 Thumbs.db
 .bundle
 vendor
+_cache

.travis.yml

@@ -3,5 +3,9 @@ rvm:
   - "2.0.0"
 sudo: false
 cache: bundler
+env:
+  # http://docs.travis-ci.com/user/environment-variables/#Global-Variables
+  global:
+    - BITCOINORG_BUILD_TYPE=deployment
 
 script: make travis

Makefile

@@ -102,7 +102,7 @@ build:
 	  | egrep -v 'sha256sums.txt' \
 	  | sort \
 	  | xargs -d '\n' sha256sum > _site/sha256sums.txt
-	$S git show --oneline > _site/commit.txt
+	$S git log -1 --format="%H" > _site/commit.txt
 
 ## Jekyll annoyingly returns success even when it emits errors and
 ## exceptions, so we'll grep its output for error strings

README.md

@@ -297,6 +297,27 @@ run if the API site is running slow.
 
 For a list of languages, look in the `_translations` directory.
 
+#### Publishing Previews
+
+You can publish your previews online to any static hosting service.
+[GitHub pages](https://pages.github.com/) is a free service available to
+all GitHub users that works with Bitcoin.org's site hierarchy.
+
+Before building a preview site, it is recommended that you set the
+environmental variable `BITCOINORG_BUILD_TYPE` to "preview".  This will
+enable some content that would otherwise be hidden and also create a
+robots.txt file that will help prevent the site from being indexed by
+search engines and mistaken for the actual Bitcoin.org website.
+
+In the bash shell, you can do this by running the following command line
+before building you preview:
+
+    export BITCOINORG_BUILD_TYPE=preview
+
+You can also add this line to your `~/.bashrc` file if you frequently
+build site previews so that you don't have to remember to run it for
+each shell.
+
 ## Developer Documentation
 
 Most parts of the documentation can be found in the [_includes](https://github.com/bitcoin-dot-org/bitcoin.org/tree/master/_includes)

_build/update_site.sh

@@ -11,8 +11,10 @@ BUNDLE_DIR='/bitcoin.org/bundle'
 SITEDIR='/bitcoin.org/site'
 DESTDIR='build@bitcoinorgsite:/var/www/site'
 WORKDIR=`mktemp -d`
+BITCOINORG_BUILD_TYPE='deployment'
 
 export BUNDLE_DIR
+export BITCOINORG_BUILD_TYPE
 
 # Stop script in case a single command fails
 set -e

_build/update_txpreview.sh

@@ -11,6 +11,9 @@ WORKDIR=`mktemp -d`
 LIVEDIR=`mktemp -d`
 SITEDIR='/bitcoin.org/txpreview'
 DESTDIR='/var/www/txpreview'
+BITCOINORG_BUILD_TYPE='preview'
+
+export BITCOINORG_BUILD_TYPE
 
 # Stop script in case a single command fails
 set -e

_plugins/contributors.rb

@@ -106,9 +106,44 @@ def site_payload
         return
       end
 
-      # Populate site.corecontributors and site.sitecontributors arrays
-      site.corecontributors = contributors('bitcoin/bitcoin',site.config['aliases'])
-      site.sitecontributors = contributors('bitcoin-dot-org/bitcoin.org',site.config['aliases'])
+      ## Create cache directory if it doesn't exist
+      if !File.exists?('_cache')
+        Dir.mkdir('_cache')
+      end
+
+      # Populate site.corecontributors and site.sitecontributors with
+      # data from GitHub.com. Store data in the cache and only
+      # re-retrieve the data if 86,400 seconds (24 hours) passes from
+      # the retrieval date or if the cache file is deleted.  For
+      # simplicity, updates on the two cache files are linked, so if one
+      # file has to be updated, they both get updated.
+      corecontributors_cache = '_cache/corecontributors.marshall'
+      sitecontributors_cache = '_cache/sitecontributors.marshall'
+      if File.exists?(corecontributors_cache) && File.exists?(sitecontributors_cache)
+        corecontributors_cache_age = (Time.now - File.stat(corecontributors_cache).mtime).to_i
+        sitecontributors_cache_age = (Time.now - File.stat(sitecontributors_cache).mtime).to_i
+      else
+        corecontributors_cache_age = Time.now.to_i
+        sitecontributors_cache_age = Time.now.to_i
+      end
+
+      if corecontributors_cache_age > 86400 || sitecontributors_cache_age > 86400
+        site.corecontributors = contributors('bitcoin/bitcoin',site.config['aliases'])
+        File.open(corecontributors_cache,'w') do |file|
+          Marshal.dump(site.corecontributors, file)
+        end
+        site.sitecontributors = contributors('bitcoin-dot-org/bitcoin.org',site.config['aliases'])
+        File.open(sitecontributors_cache,'w') do |file|
+          Marshal.dump(site.sitecontributors, file)
+        end
+      else
+        File.open(corecontributors_cache,'r') do |file|
+          site.corecontributors = Marshal.load(file)
+        end
+        File.open(sitecontributors_cache,'r') do |file|
+          site.sitecontributors = Marshal.load(file)
+        end
+      end
 
     end
 

_plugins/env.rb

@@ -0,0 +1,20 @@
+# This file is licensed under the MIT License (MIT) available on
+# http://opensource.org/licenses/MIT.
+
+## env.rb takes select environmental variables and makes them available
+## to the site templates. Currently, only variables starting with
+## BITCOINORG_ are exported
+
+module Jekyll
+  class EnvGenerator < Generator
+    def generate(site)
+      ## If necessary, initialize env hash table
+      site.config["env"] = site.config["env"] ? site.config["env"] : {}
+
+      ## Load matching environmental variables in to array
+      ENV.keys.grep /^BITCOINORG_/ do |key|
+        site.config['env'].merge!({ key => ENV[key] })
+      end
+    end
+  end
+end

_plugins/events.rb

@@ -142,9 +142,58 @@ def site_payload
         return
       end
 
-      # Populate site.conferences and site.meetups arrays
-      site.conferences = conferences()
-      site.meetups = meetups()
+      ## Create cache directory if it doesn't exist
+      if !File.exists?('_cache')
+        Dir.mkdir('_cache')
+      end
+
+      ## Populate site.conferences with conferences from _events.yml
+      ## plus geodata from Google. Store data in the cache and only
+      ## re-retrieve the geodata if _events.yml is edited or the cache
+      ## file is deleted.
+      conferences_cache = '_cache/conferences.marshall'
+      events_file = '_events.yml'
+
+      events_file_unix_time = File.stat(events_file).mtime.to_i
+      if File.exists?(conferences_cache)
+        conferences_cache_unix_time = File.stat(conferences_cache).mtime.to_i
+      else
+        conferences_cache_unix_time = 0
+      end
+
+      if events_file_unix_time >= conferences_cache_unix_time
+        site.conferences = conferences()
+        File.open(conferences_cache,'w') do |file|
+          Marshal.dump(site.conferences, file)
+        end
+      else
+        File.open(conferences_cache,'r') do |file|
+          site.conferences = Marshal.load(file)
+        end
+      end
+
+      # Populate site.meetups with data from Meetup.com.  Store data in
+      # the cache and only re-retrieve the data if 86,400 seconds (24
+      # hours) passes from the retrieval date or if the cache file is
+      # deleted.
+      meetups_cache = '_cache/meetups.marshall'
+      if File.exists?(meetups_cache)
+        meetups_cache_age = (Time.now - File.stat(meetups_cache).mtime).to_i
+      else
+        meetups_cache_age = Time.now.to_i
+      end
+
+      if meetups_cache_age > 86400
+        site.meetups = meetups()
+        File.open(meetups_cache,'w') do |file|
+          Marshal.dump(site.meetups, file)
+        end
+      else
+        File.open(meetups_cache,'r') do |file|
+          site.meetups = Marshal.load(file)
+        end
+      end
+
     end
 
   end

_templates/about-us.html

@@ -5,6 +5,7 @@
 layout: base
 id: about-us
 ---
+{% if site.env.BITCOINORG_BUILD_TYPE %}
 <!-- Note: this file is built non-deterministically -->
 <h1>{% translate pagetitle %}</h1>
 <p class="summary">{% translate pagedesc %}</p>
@@ -89,3 +90,19 @@ <h3 id="github">{% translate github %}</h3>
   </div>
   {% endfor %}
 </div>
+{% else %}
+{% comment %}
+<!-- if you've cloned bitcoin.org, feel free to fill in your own About
+Us below. It would be appreciated it if you link back to Bitcoin.org,
+but please make it clear that your site is not affiliated with
+Bitcoin.org.
+
+It would also be appreciated if you would remove our names and email
+addresses from the README.md file in the top-level directory.  -->
+{% endcomment %}
+<h1>About this site</h1>
+
+This site includes content originally published on <a
+href="https://bitcoin.org">Bitcoin.org</a>, but it is not affiliated with
+Bitcoin.org.
+{% endif %}

_templates/download.html

@@ -16,6 +16,7 @@
 lin64: "linux64.tar.gz"
 
 ---
+{% if site.env.BITCOINORG_BUILD_TYPE %}
 <!-- Note: this file exempt from check-for-subheading-anchors check -->
 
 {% capture PATH_PREFIX %}/bin/bitcoin-core-{{ site.DOWNLOAD_VERSION }}{% endcapture %}
@@ -148,3 +149,13 @@ <h2><img src="/img/icons/note.svg" class="warningicon" alt="note">{% translate p
 break;
 }
 </script>
+{% else %}
+{% capture redirect %}https://bitcoin.org/{{page.lang}}/{% translate download url %}{% endcapture %}
+<meta name="robots" content="noindex">
+<script>window.location.href='{{ redirect }}';</script>
+
+<div class="redirectmsg">
+<h1>This page has been moved</h1>
+<p><a href="{{ redirect }}">{{ redirect }}</a></p>
+</div>
+{% endif %}

robots.txt

@@ -1 +1,12 @@
+---
+# This file is licensed under the MIT License (MIT) available on
+# http://opensource.org/licenses/MIT.
+
+layout: null
+---
+{% if site.env.BITCOINORG_BUILD_TYPE == 'preview' %}
+User-agent: *
+Disallow: /
+{% else %}
 Sitemap: https://bitcoin.org/sitemap.xml
+{% endif %}