Merge pull bitcoin-dot-org#823: Various Backend & Site Docs Updates

.travis.yml

@@ -2,4 +2,4 @@ language: ruby
 rvm:
   - "2.0.0"
 
-script: make all
+script: make travis

Gemfile

@@ -1,15 +1,32 @@
 source 'https://rubygems.org'
 
-#ruby '2.0.0'
+## If you update the version here, also update it in .travis.yml and
+## README.md. Then push your branch and make sure Travis supports that
+## version. Then remind one of the site maintainers that they need to
+## run `rvm install <VERSION>` on the build server(s) before they commit
+## to master
+ruby '2.0.0'
 
+## Used on the build server. If you add a package here (like nokogiri)
+## that has non-Gem dependencies (like zlib), please remind the site
+## maintainers that they need to manually update the build server(s)
+## before they commit to master. If `bundle install` can satisfy all
+## your dependencies, then nothing extra needs to be done
 group :development do
   gem 'ffi-icu'
-  gem 'jekyll'
+  ## When we upgrade to Jekyll 3.0.0 or higher, remove
+  ## _plugin/remove-html-extension.rb
+  gem 'jekyll', '~>1.3.0'
   gem 'json'
   gem 'less'
   gem 'kramdown'
   gem 'RedCloth'
   gem 'therubyracer' # required by less
-  gem 'html-proofer'
 end
 
+## Not used on build server. Only used by developers and Travis CI, so
+## you can put whatever you want here and bundler will tell us humans to
+## install the new Gems.
+group :slow_test do
+  gem 'html-proofer'
+end

Gemfile.lock

@@ -80,7 +80,7 @@ DEPENDENCIES
   RedCloth
   ffi-icu
   html-proofer
-  jekyll
+  jekyll (~> 1.3.0)
   json
   kramdown
   less

Makefile

@@ -4,13 +4,6 @@
 S=@  ## Silent: only print errors by default; 
      ## run `make S='' [other args]` to print commands as they're run
 
-## Old versions of jekyll must not call "build".  If you have one of
-## these versions, either run make like this: make JEKYLL_COMMAND=jekyll
-## or create the JEKYLL_COMMAND environmental variable:
-# echo 'export JEKYLL_COMMAND=jekyll' >> ~/.bashrc
-# exec bash
-# make
-JEKYLL_COMMAND ?= "bundle exec jekyll build"
 SITEDIR=_site
 JEKYLL_LOG=._jekyll.log
 
@@ -21,23 +14,46 @@ JEKYLL_LOG=._jekyll.log
 ## `make` (no arguments): just build
 default: build
 
+## `make preview`: start the built-in Jekyll preview
+preview:
+	$S bundle exec jekyll serve
+
 ## `make test`: don't build, but do run all tests
 test: pre-build-tests post-build-tests
 
 ## `make valid`: build and run fast tests
 valid: pre-build-tests-fast build post-build-tests-fast
 
 ## `make all`: build and run all tests
-all: travis-background-keepalive pre-build-tests build post-build-tests
+all: pre-build-tests build post-build-tests
+
+## `make deployment`: for use on build server
+deployment: install-deps-deployment \
+    valid
+
+## `make travis`: for use with Travis CI
+travis: travis-background-keepalive \
+    install-deps-development \
+    all
+
 
 
 
-## Pre-build tests which, aggregated together, take less than 5 seconds to run on a typical PC
+## Install dependencies (development version)
+install-deps-development:
+	bundle install
+
+## Install dependencies (deployment version)
+install-deps-deployment:
+	bundle install --deployment --without :slow_test
+
+## Pre-build tests which, aggregated together, take less than 10 seconds to run on a typical PC
 pre-build-tests-fast: check-for-non-ascii-urls check-for-wrong-filename-assignments \
     check-for-missing-rpc-summaries \
-    check-for-missing-copyright-licenses
+    check-for-missing-copyright-licenses \
+    check-bundle
 
-## Post-build tests which, aggregated together, take less than 5 seconds to run on a typical PC
+## Post-build tests which, aggregated together, take less than 10 seconds to run on a typical PC
 post-build-tests-fast: check-for-build-errors ensure-each-svg-has-a-png check-for-liquid-errors \
     check-for-missing-anchors check-for-broken-markdown-reference-links \
     check-for-broken-kramdown-tables check-for-duplicate-header-ids \
@@ -74,15 +90,20 @@ ERROR_ON_OUTPUT="sed '1s/^/ERROR:\n/' | if grep . ; then sed 1iERROR ; false ; e
 ## Always build using the default locale so log messages can be grepped.
 ## This should not affect webpage output.
 build:
-	$S export LANG=C.UTF-8 ; eval $(JEKYLL_COMMAND) 2>&1 | tee $(JEKYLL_LOG)
+	$S export LANG=C.UTF-8 ; bundle exec jekyll build 2>&1 | tee $(JEKYLL_LOG)
 	$S grep -r -L 'Note: this file is built non-deterministically' _site/ \
 	  | egrep -v 'sha256sums.txt' \
 	  | xargs sha256sum > _site/sha256sums.txt
 
 ## Jekyll annoyingly returns success even when it emits errors and
 ## exceptions, so we'll grep its output for error strings
+#
+## FIXME: temporarily ignoring errors from WEBrick because
+## _plugin/remove-html-extension does something hackish until we upgrade
+## to Jekyll 3.0.0
 check-for-build-errors:
 	$S egrep -i '(error|warn|exception)' $(JEKYLL_LOG) \
+	    | grep -vi webrick.*filehandler \
 	    | eval $(ERROR_ON_OUTPUT)
 
 
@@ -213,6 +234,12 @@ check-for-broken-bitcoin-core-download-links:
 check-html-proofer:
 	$S bundle exec ruby _contrib/bco-htmlproof
 
+check-bundle:
+## Ensure all the dependencies are installed. If you build without this
+## check, you'll get confusing error messages when your deps aren't up
+## to date
+	$S ! bundle check | grep -v "The Gemfile's dependencies are satisfied"
+
 travis-background-keepalive:
 	$S { while ps aux | grep -q '[m]ake' ; do echo "Ignore me: Travis CI keep alive" ; sleep 1m ; done ; } &
 

README.md

@@ -134,73 +134,114 @@ You simply need to push additional commits on the appropriate branch of your Git
 
 Simple text changes can be previewed live on bitcoin.org. You only need to click anywhere on the page and hold your mouse button for one second. You'll then be able to edit the page just like a document. Changes will be lost as soon as the page is refreshed.
 
-#### Build Site With Jekyll From Bundler
+#### Build The Site Locally
 
-Make sure you have ruby 2.0. If you don't, we recommend
-[installing it with RVM](https://www.digitalocean.com/community/articles/how-to-install-ruby-on-rails-on-ubuntu-14-04-using-rvm),
-which can usually be done by running the following three commands:
+For anything more than simple text previews, you will need to build the
+site. If you can't do this yourself using the instructions below, please
+[open a pull request][pull request] with your suggested change and one
+of the site developers will create a preview for you.
+
+To build the site, you need to go through a one-time installation
+procedure that takes 15 to 30 minutes.  After that you can build the
+site an unlimited number of times with no extra work.
+
+##### Install The Dependencies
+
+Before building the site, you need to install the following
+dependencies and tools, which are pretty easy on any modern Linux:
+
+**Install binary libraries and tools**
+
+On recent versions of Ubuntu and Debian, you can run the following
+command to ensure you have the required libraries, headers, and tools:
+
+    sudo apt-get install build-essential git libicu-dev zlib1g-dev
+
+**Install RVM**
+
+Install RVM using either the [easy instructions](https://rvm.io/) or the
+[more secure instructions](https://rvm.io/rvm/security).
+
+Read the instructions printed to your console during setup to enable the
+`rvm` command in your shell.  After installation, you need to run the
+following command:
 
-    \curl -sSL https://get.rvm.io | bash -s stable
     source ~/.rvm/scripts/rvm
+
+**Install Ruby 2.0.0**
+
+To install Ruby 2.0.0, simply run this command:
+
     rvm install ruby-2.0.0
 
-Next, you need to install bundler, and let it install all gems you need
-to build the site. You must run the last command from within your local
-bitcoin.org repository:
+Sometimes this will find a pre-compiled Ruby package for your Linux
+distribution, but sometimes it will need to compile Ruby from scratch
+(which takes about 15 minutes).
 
-    gem install bundler
-    bundle install
+After Ruby 2.0.0 is installed, make it your default Ruby:
+
+    rvm alias create default ruby-2.0.0
 
-Finally, you can build the website in _site/:
+And tell your system to use it:
 
-    bundle exec jekyll build
+    rvm use default
 
-You can then copy the output files from _site/ to the root of your web server.
-If you have no web server, run `bundle exec jekyll serve` and visit
-http://127.0.0.1:4000/. This server requires you to add a trailing ".html"
-by hand in your browser address bar.
+(Note: you can use a different default Ruby, but if you ever change
+your default Ruby, you must re-run the `gem install bundle` command
+described below before you can build the site. If you ever receive a
+"eval: bundle: not found" error, you failed to re-run `gem install
+bundle`.)
 
-#### Build Site With Jekyll From APT
+**Install Bundle**
 
-The instructions in the section above will ensure that you use the same
-versions of the same software we use to build the website, but you can
-also install dependencies from your Linux distribution. For example:
+When you used RVM to install Ruby, it also installed the `gem` program.
+Use that program to install bundle:
 
-Installing dependencies on Ubuntu 12.10:
+    gem install bundle
 
-    sudo apt-get install jekyll node-less ruby1.9.1-dev libicu-dev
-    sudo gem install ffi-icu
+**Install the Ruby dependencies**
 
-Installing dependencies on older Ubuntu and Debian distributions:
+Ensure you checked out the site repository as described in [Working with
+GitHub](#working-with-github). Then change directory to the top-level of
+your local repository (replace `bitcoin.org` with the full path to your local
+repository clone):
 
-    sudo apt-get install rubygems ruby1.9.1-dev build-essential libicu-dev
-    sudo gem install jekyll json less therubyracer ffi-icu
+    cd bitcoin.org
 
-Finally build the website in _site/:
+And install the necessary dependencies using Bundle:
 
-    jekyll
+    bundle install
 
-...Or `jekyll build` on recent versions. You can then copy the output files
-from _site/ to the root of your web server. If you have no web server, run
-`jekyll --server` (or `jekyll serve` on recent versions) and visit
-http://127.0.0.1:4000/. This server requires you to add a trailing ".html"
-by hand in your browser address bar.
+Note that some of the dependencies (particularly nokogiri) can take a
+long time to install on some systems, so be patient.
 
-#### Building With Make
+Once Bundle completes successfully, you can preview or build the site.
 
-After you've installed Jekyll and the other dependencies, you can
-optionally use GNU Make to automatically build the site and run several
-tests. You will first need to install Make using your package manager;
-for example:
+##### Preview The Site
 
-    sudo apt-get install make
+To preview the website in your local browser, make sure you're in the
+`bitcoin.org` directory and run the following command:
 
-Then in your local bitcoin.org repository, run one of the following
-commands:
+    make preview
+
+This will compile the site (takes 5 to 10 minutes; see the [speed up
+instructions](#fast-partial-previews-or-builds)) and then print the a message like this:
+
+    Server address: http://0.0.0.0:4000
+    Server running... press ctrl-c to stop.
+
+Visit the indicated URL in your browser to view the site.
+
+##### Build The Site
+
+To build the site exactly like we do for the deployment server, make
+sure you're in the `bitcoin.org` directory and run:
 
-    ## To just build the site, the equivalent of: bundle exec jekyll build
     make
 
+The resulting HTML for the entire site will be placed in the `_site`
+directory.  The following alternative options are available:
+
     ## After you build the site, you can run all of the tests (may take awhile)
     make test
 
@@ -210,13 +251,45 @@ commands:
     ## Or build the site and run all tests
     make all
 
-#### Partial build for faster preview
+#### Fast Partial Previews Or Builds
 
 In order to preview some changes faster, you can disable all plugins and
-languages but those you need by prefixing the `ENABLED_LANGS` and `ENABLED_PLUGINS`
-environment variables:
+languages except those you need by prefixing the `ENABLED_LANGS` and
+`ENABLED_PLUGINS` environment variables to your command line.  For
+example, do this to disable everything:
+
+    ## Fast preview, takes less than 30 seconds
+    ENABLED_PLUGINS="" ENABLED_LANGS="" make preview
 
-    ENABLED_PLUGINS="events autocrossref" ENABLED_LANGS="en fr" make all
+    ## Fast build and tests, takes less than 50 seconds
+    ## Some tests may fail in fast mode; use -i to continue despite them
+    ENABLED_PLUGINS="" ENABLED_LANGS="" make -i valid
+
+Then to enable some plugins or languages, you can add them back in.
+For example:
+
+    ## Slower (but still pretty fast) build and test
+    ENABLED_PLUGINS="events autocrossref" ENABLED_LANGS="en fr" make -i valid
+
+Plugins include:
+
+| Plugin       | Seconds | Remote APIs    | Used For
+|--------------|---------|----------------|------------------------
+| alerts       | 5       | --             | Network alert pages
+| autocrossref | 90      | --             | Developer documentation
+| contributors | 5       | GitHub.com     | Contributor listings
+| events       | 5       | Meetup.com; Google Maps | Events page
+| glossary     | 30      | --             | Developer glossary
+| redirects    | 20      | --             | Redirects from old URLs
+| releases     | 10      | --             | Bitcoin Core release notes; Download page
+| sitemap      | 10      | --             | /sitemap.xml
+
+Notes: some plugins interact with each other or with translations; for example running
+'autocrossref' and 'glossary' takes longer than running each other
+separately. Also, plugins that use remote APIs may take a long time to
+run if the API site is running slow.
+
+For a list of languages, look in the `_translations` directory.
 
 ## Developer Documentation
 
@@ -284,7 +357,11 @@ Any change in the English text can be done through a pull request on GitHub. If
 
 ### Events
 
-Events should be placed in `_events.yml` and adhere to this format:
+If you're not comfortable with GitHub pull requests, please submit an
+event using the button near the bottom of the [Events
+page](https://bitcoin.org/en/events).
+
+To create an event pull request, place the event in `_events.yml` and adhere to this format:
 
 ```
 - date: 2014-02-21

_build/update_site.sh

@@ -49,7 +49,7 @@ lasttime=`stat -c %Y "$SITEDIR/_buildlock" | cut -d ' ' -f1`
 # Build website in a child process
 (
 cd $WORKDIR
-make valid && touch "$WORKDIR/_builddone" || touch "$WORKDIR/_buildfail"
+make deployment && touch "$WORKDIR/_builddone" || touch "$WORKDIR/_buildfail"
 )&
 
 # Loop every 1 second to check status

_plugins/autocrossref.rb

@@ -33,12 +33,10 @@ def initialize(tag_name, text, tokens)
     def render(context)
       output = super
 
-      ## Workaround for inconsistent relative directory
-      path = File.expand_path(File.dirname(__FILE__)) + "/.."
       ## Load terms from file
       site = context.registers[:site].config
       if !site.has_key?("crossref")
-        site['crossref'] = YAML.load_file(path + "/_autocrossref.yaml")
+        site['crossref'] = YAML.load_file("_autocrossref.yaml")
       end
 
       ## Sort terms by reverse length, so longest matches get linked