QA: Use HTML Proofer To Check Internal Links

Uses Ruby html-proofer to check the links. This commit also fixes the various problems it found, as well as dealing with some of its non-problem complaints (it doesn't like anchor (a) tags without either an href, name, or id). Running HTML proofer takes about 12 minutes on my system (with up to two threads), during which it prints no text. Travis CI times out after 10 minutes of nothing being written to stdout, so this commit also adds a background process the Makefile to print a line every minute while make runs.
learsu · Mar 14, 2015 · d954708 · d954708
1 parent d44692c
commit d954708
Show file tree

Hide file tree

Showing 42 changed files with 229 additions and 145 deletions.
diff --git a/Gemfile b/Gemfile
@@ -10,5 +10,6 @@ group :development do
   gem 'kramdown'
   gem 'RedCloth'
   gem 'therubyracer' # required by less
+  gem 'html-proofer'
 end
 
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -2,17 +2,29 @@ GEM
   remote: https://rubygems.org/
   specs:
     RedCloth (4.2.9)
+    addressable (2.3.7)
     classifier (1.3.3)
       fast-stemmer (>= 1.0.0)
     colorator (0.1)
+    colored (1.2)
     commander (4.1.5)
       highline (~> 1.6.11)
     commonjs (0.2.7)
+    ethon (0.7.3)
+      ffi (>= 1.3.0)
     fast-stemmer (1.0.2)
     ffi (1.9.3)
     ffi-icu (0.1.2)
       ffi (~> 1.0, >= 1.0.9)
     highline (1.6.20)
+    html-proofer (2.1.0)
+      addressable (~> 2.3)
+      colored (~> 1.2)
+      mercenary (~> 0.3.2)
+      nokogiri (~> 1.5)
+      parallel (~> 1.3)
+      typhoeus (~> 0.7)
+      yell (~> 2.0)
     jekyll (1.3.0)
       classifier (~> 1.3)
       colorator (~> 0.1)
@@ -35,6 +47,11 @@ GEM
       rb-kqueue (>= 0.2)
     maruku (0.6.1)
       syntax (>= 1.0.0)
+    mercenary (0.3.5)
+    mini_portile (0.6.2)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    parallel (1.4.1)
     posix-spawn (0.3.6)
     pygments.rb (0.5.4)
       posix-spawn (~> 0.3.6)
@@ -51,14 +68,18 @@ GEM
     therubyracer (0.12.1)
       libv8 (~> 3.16.14.0)
       ref
+    typhoeus (0.7.1)
+      ethon (>= 0.7.1)
     yajl-ruby (1.1.0)
+    yell (2.0.5)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   RedCloth
   ffi-icu
+  html-proofer
   jekyll
   json
   kramdown

diff --git a/Makefile b/Makefile
@@ -28,7 +28,7 @@ test: pre-build-tests post-build-tests
 valid: pre-build-tests-fast build post-build-tests-fast
 
 ## `make all`: build and run all tests
-all: pre-build-tests build post-build-tests
+all: travis-background-keepalive pre-build-tests build post-build-tests
 
 
 
@@ -49,9 +49,8 @@ pre-build-tests: pre-build-tests-fast
 
 ## All post-build tests, including those which might take multiple minutes
 post-build-tests: post-build-tests-fast \
-    check-for-broken-bitcoin-core-download-links
-	@ true ## SOMEDAY: use linkchecker to find broken links 
-	@      ## after this bug is fixed: https://github.com/wummel/linkchecker/issues/513
+    check-for-broken-bitcoin-core-download-links \
+    check-html-proofer
 
 ## All manual updates to content that should be run by a human. This
 ## will create or update files which should then be diffed and commited.
@@ -209,3 +208,9 @@ check-for-broken-bitcoin-core-download-links:
 	      curl -sI "$$url" ; \
 	    fi | grep -q '200 OK' || echo "Error: Could not retrieve $$url" ; \
 	  done | eval $(ERROR_ON_OUTPUT)
+
+check-html-proofer:
+	$S bundle exec ruby _contrib/bco-htmlproof
+
+travis-background-keepalive:
+	$S { while ps aux | grep -q '[m]ake' ; do echo "Ignore me: Travis CI keep alive" ; sleep 1m ; done ; } &
diff --git a/_alerts/2013-03-15-upgrade-deadline.html b/_alerts/2013-03-15-upgrade-deadline.html
@@ -60,7 +60,7 @@ <h2>Miners/mining pool operators</h2>
 </p>
         <h2>Why this is necessary</h2>
 <p>
-A bug caused a <a href="/chainfork.html">temporary block chain fork on 11 March, 2013</a>.
+A bug caused a <a href="/chainfork">temporary block chain fork on 11 March, 2013</a>.
 After investigating
 that bug, we determined that the bug can happen even if the entire network was
 still running old versions of Bitcoin-Qt/bitcoind. Therefore, the only option is to

diff --git a/_build/update_site.sh b/_build/update_site.sh
@@ -49,7 +49,7 @@ lasttime=`stat -c %Y "$SITEDIR/_buildlock" | cut -d ' ' -f1`
 # Build website in a child process
 (
 cd $WORKDIR
-JEKYLL_COMMAND='jekyll' make all
+make all
 touch "$WORKDIR/_builddone"
 )&
 

diff --git a/_build/update_txpreview.sh b/_build/update_txpreview.sh
@@ -91,7 +91,7 @@ done
 # Build website in a child process
 (
 cd $WORKDIR
-ENABLED_PLUGINS='alerts redirects releases' JEKYLL_COMMAND='jekyll' make all
+ENABLED_PLUGINS='alerts redirects releases' JEKYLL_COMMAND='jekyll' make
 touch "$WORKDIR/_builddone"
 )&
 

diff --git a/_contrib/bco-htmlproof b/_contrib/bco-htmlproof
@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+require 'html/proofer'
+
+## Will throw an exception (exiting false) if any internal links don't
+## work. The Makefile will terminate on the failure
+HTML::Proofer.new(
+    ## To test, uncomment the array below and comment out ./_site and :disable_external
+    #[ "/foo/bar#baz", "/foo/bar", "#", "#wallet", "/foo.css", "/bar.png", "/zh_TW/bitcoin-for-businesses" ],
+    "./_site",
+
+    {
+        ## Disable external link checking by default to avoid spurious
+        ## Travis CI failures. TODO: take an argument to optionally
+        ## enable external link checking as part of the Makefile
+        ## manual checks
+        :disable_external => true,
+
+        ## Links to ignore
+        :href_ignore => [
+          '#',        ## hrefs pointing to the current page (htmlproofer fails them)
+          /^\/bin/,   ## /bin dir is not part of repository; holds Bitcoin Core binaries
+          /^\/stats/  ## /stats dir is not part of repository; generated by separate stats script
+        ],
+
+        ## Mangle links. If we enable external link checking, this will
+        ## require updating
+        :href_swap => {
+          ## (Hack) Append '$' to the ends of filenames we don't want to append .html to
+          /(css|png|rss|pdf|jpg|asc)$/ => '\1$',
+
+          ## Append .html to end of most URLs so proofer can find the local files
+          /^(
+           [^#]+   ## Don't match URL containing a hash, we'll deal with them separately
+           [^\/$]  ## Don't match URLs ending in a slash or $
+          )$/x => '\1.html',
+
+          ## Insert .html between page and anchor, but only if there's a
+          ## page name
+          /^(.+)(#.+)/ => '\1.html\2',
+
+          ## (Un-hack) Remove previously-appended '$' from URLs
+          /\$$/ => '',
+        },
+
+        ## It'd be nice if we had a _local_config.yaml file or something
+        ## for settings specific to particular systems, but for now I
+        ## think 2 is a good setting for Travis CI ("1.5 processors")
+        ## and me (usually 2 processors)
+        :parallel => { :in_processes => 2 }
+    }
+).run
diff --git a/_layouts/base.html b/_layouts/base.html
@@ -92,7 +92,12 @@
     {% else %}
     <a href="/{{ page.lang }}/{% translate privacy url %}">{% translate menu-privacy layout %}</a>
     {% endcase %}
+    {% case page.lang %}
+    {% when 'sv' %}
+    <a href="/en/press">Press</a>
+    {% else %}
     <a href="/{{ page.lang }}/{% translate press url %}">{% translate menu-press layout %}</a>
+    {% endcase %}
     <a href="/{{ page.lang }}/{% translate about-us url %}">{% translate menu-about-us layout %}</a>
     </div>
     <div class="footersponsor">

diff --git a/_less/screen.less b/_less/screen.less
@@ -534,7 +534,7 @@ table td,table th{
 .starttitle{
 	text-align:center;
 }
-.starttitle a{
+.starttitle span{
 	padding:10px;
 	border-bottom:1px solid #E2E2E2;
 	font-size:155%;

diff --git a/_templates/about-us.html b/_templates/about-us.html
@@ -33,30 +33,30 @@ <h3>{% translate maintenance %}</h3>
 
 <div class="credit">
   <p><a href="mailto:saivann@gmail.com">Saïvann Carignan</a><span>Website maintenance</span></p>
-  <p><a>Garland William Binns III</a><span>Translation maintenance</span></p>
+  <p>Garland William Binns III<span>Translation maintenance</span></p>
   <p><a href="http://dtrt.org/">David Harding</a><span>Documentation maintenance</span></p>
 </div>
 
 <h3>{% translate documentation %}</h3>
 
 <div class="credit">
   <p><a href="http://dtrt.org/">David Harding</a><span>Coordination and writing</span></p>
-  <p><a>Greg Sanders</a><span>Writing</span></p>
+  <p>Greg Sanders<span>Writing</span></p>
 </div>
 
 <h3>{% translate translation %}</h3>
 
 <div class="credit">
-  <p><a>Garland William Binns III</a><span>Maintenance</span></p>
-  <p><a>@arvicco</a><span>Russian</span></p>
-  <p><a>Simon Alexander Hinterreiter</a><span>German</span></p>
-  <p><a>Jacob Burenstam</a><span>Swedish</span></p>
-  <p><a>Péter Kemenczés</a><span>Hungarian</span></p>
-  <p><a>Matija Mazi</a><span>Slovenian</span></p>
-  <p><a>Mihai Onosie</a><span>Romanian</span></p>
-  <p><a>Boštjan Pirnar</a><span>Slovenian</span></p>
-  <p><a>Luigigiuseppe Prosperi</a><span>Italian</span></p>
-  <p><a>Thomas Pryds</a><span>Danish</span></p>
+  <p>Garland William Binns III<span>Maintenance</span></p>
+  <p>@arvicco<span>Russian</span></p>
+  <p>Simon Alexander Hinterreiter<span">German</span></p>
+  <p>Jacob Burenstam<span>Swedish</span></p>
+  <p>Péter Kemenczés<span>Hungarian</span></p>
+  <p>Matija Mazi<span>Slovenian</span></p>
+  <p>Mihai Onosie<span>Romanian</span></p>
+  <p>Boštjan Pirnar<span>Slovenian</span></p>
+  <p>Luigigiuseppe Prosperi<span>Italian</span></p>
+  <p>Thomas Pryds<span>Danish</span></p>
 </div>
 
 <h3>{% translate github %}</h3>

diff --git a/_templates/choose-your-wallet.html b/_templates/choose-your-wallet.html
@@ -712,23 +712,23 @@ <h1>{% translate pagetitle %}</h1>
 <div class="walletmenu" id="walletmenu" onmouseover="walletListener(event);" onmouseout="walletListener(event);" onclick="walletListener(event);">
   <ul onclick="mobileMenuHover(event);">
     <li class="wallet-mobile">
-    <a data-walletcompat="mobile">{% translate walletcatmobile %}</a>
+    <a id="mobile" data-walletcompat="mobile">{% translate walletcatmobile %}</a>
     <ul>
-      <li class="wallet-android"><a data-walletcompat="android">{% translate platformandroid %}</a></li>
-      <li class="wallet-ios"><a data-walletcompat="ios">{% translate platformios %}</a></li>
-      <li class="wallet-blackberry"><a data-walletcompat="blackberry">{% translate platformblackberry %}</a></li>
+      <li class="wallet-android"><a id="android" data-walletcompat="android">{% translate platformandroid %}</a></li>
+      <li class="wallet-ios"><a id="ios" data-walletcompat="ios">{% translate platformios %}</a></li>
+      <li class="wallet-blackberry"><a id="blackberry" data-walletcompat="blackberry">{% translate platformblackberry %}</a></li>
     </ul>
     </li>
     <li class="wallet-desktop">
-    <a data-walletcompat="desktop">{% translate walletcatdesktop %}</a>
+    <a id="desktop" data-walletcompat="desktop">{% translate walletcatdesktop %}</a>
     <ul>
-      <li class="wallet-windows"><a data-walletcompat="windows">{% translate platformwindows %}</a></li>
-      <li class="wallet-mac"><a data-walletcompat="mac">{% translate platformmac %}</a></li>
-      <li class="wallet-linux"><a data-walletcompat="linux">{% translate platformlinux %}</a></li>
+      <li class="wallet-windows"><a id="windows" data-walletcompat="windows">{% translate platformwindows %}</a></li>
+      <li class="wallet-mac"><a id="mac" data-walletcompat="mac">{% translate platformmac %}</a></li>
+      <li class="wallet-linux"><a id="linux" data-walletcompat="linux">{% translate platformlinux %}</a></li>
     </ul>
     </li>
-    <li class="wallet-hardware"><a data-walletcompat="hardware">{% translate walletcathardware %}</a></li>
-    <li class="wallet-web"><a data-walletcompat="web">{% translate walletcatweb %}</a></li>
+    <li class="wallet-hardware"><a id="hardware" data-walletcompat="hardware">{% translate walletcathardware %}</a></li>
+    <li class="wallet-web"><a id="web" data-walletcompat="web">{% translate walletcatweb %}</a></li>
   </ul>
 </div>