Release #793

Author: Mr0grog

.circleci/config.yml

@@ -4,7 +4,7 @@ executors:
   rails_executor:
     working_directory: ~/web-monitoring-db
     docker:
-      - image: circleci/ruby:2.6.5-node
+      - image: circleci/ruby:2.6.6-node
         environment:
           RAILS_ENV: test
           RACK_ENV: test

.ruby-version

@@ -1 +1 @@
-2.6.5
+2.6.6

Dockerfile

@@ -2,7 +2,7 @@
 # https://docs.docker.com/engine/userguide/eng-image/multistage-build
 
 ### BASE ENVIRONMENT STAGE ###
-FROM ruby:2.6.5-slim as base
+FROM ruby:2.6.6-slim as base
 LABEL maintainer="enviroDGI@gmail.com"
 
 # Install apt based dependencies required to run Rails as

Gemfile

@@ -5,7 +5,7 @@ git_source(:github) do |repo_name|
   "https://github.com/#{repo_name}.git"
 end
 
-ruby '2.6.5'
+ruby '2.6.6'
 
 gem 'aws-sdk-s3', '~> 1.85'
 gem 'concurrent-ruby', '~> 1.1'

Gemfile.lock

@@ -381,7 +381,7 @@ DEPENDENCIES
   webmock (~> 3.10)
 
 RUBY VERSION
-   ruby 2.6.5p114
+   ruby 2.6.6p146
 
 BUNDLED WITH
    2.1.4

README.md

@@ -22,7 +22,7 @@ We maintain a publicly available *staging server* at https://api-staging.monitor
 
 ## Installation
 
-1. Ensure you have Ruby 2.6.5+.
+1. Ensure you have Ruby 2.6.6+.
 
     You can use [rbenv](https://github.com/rbenv/rbenv) to manage multiple Ruby versions
 

app/controllers/api/v0/pages_controller.rb

@@ -71,7 +71,16 @@ def index
   end
 
   def show
-    page = Page.find(params[:id])
+    begin
+      page = Page.find(params[:id])
+    rescue ActiveRecord::RecordNotFound
+      merge = MergedPage.find(params[:id])
+      redirect_to(
+        api_v0_page_url(merge.target_uuid),
+        status: :permanent_redirect
+      ) and return
+    end
+
     data = page.as_json(include: [:maintainers, :tags])
     if should_allow_versions
       data['versions'] = page.versions.where(different: true).as_json
@@ -155,11 +164,12 @@ def page_collection
 
     if params[:url]
       query = params[:url]
+      collection = collection.joins(:urls)
       if query.include? '*'
         query = query.gsub('%', '\%').gsub('_', '\_').tr('*', '%')
-        collection = collection.where('url LIKE ?', query)
+        collection = collection.where('page_urls.url LIKE ?', query)
       else
-        collection = collection.where(url: query)
+        collection = collection.where('page_urls.url = ?', query)
       end
     end
 

app/controllers/api/v0/urls_controller.rb

@@ -0,0 +1,76 @@
+class Api::V0::UrlsController < Api::V0::ApiController
+  def index
+    urls = page.urls.order('page_urls.to_time DESC')
+
+    render json: {
+      links: { page: api_v0_page_url(page) },
+      data: urls
+    }
+  end
+
+  def show
+    @page_url ||= page.urls.find(params[:id])
+    render json: {
+      links: {
+        page: api_v0_page_url(page),
+        page_urls: api_v0_page_urls_url(page)
+      },
+      data: @page_url
+    }
+  end
+
+  def create
+    @page_url = page.urls.create!(url_params)
+    show
+  rescue ActiveRecord::RecordNotUnique
+    raise Api::ResourceExistsError, 'This page already has the given URL and timeframe'
+  end
+
+  def update
+    updates = url_params
+    if updates.key?(:url)
+      raise Api::UnprocessableError, 'You cannot change a URL\'s `url`'
+    end
+
+    @page_url ||= page.urls.find(params[:id])
+    @page_url.update(url_params)
+    show
+  end
+
+  def destroy
+    @page_url ||= page.urls.find(params[:id])
+    # You cannot delete the canonical URL.
+    if @page_url.url == page.url
+      raise Api::UnprocessableError, 'You cannot remove the page\'s canonical URL'
+    else
+      @page_url.destroy
+      redirect_to(api_v0_page_urls_url(page))
+    end
+  end
+
+  protected
+
+  def page
+    @page ||= Page.find(params[:page_id])
+  end
+
+  def url_params
+    result = params
+      .require(:page_url)
+      .permit(:url, :from_time, :to_time, :notes)
+
+    result.slice('from_time', 'to_time').each do |key, value|
+      result[key] = parse_time(key, value)
+    end
+
+    result
+  end
+
+  def parse_time(field, time_input)
+    return if time_input.nil?
+
+    Time.parse(time_input)
+  rescue ArgumentError
+    raise Api::UnprocessableError, "`#{field}` was not a valid time or `null`"
+  end
+end

app/jobs/import_versions_job.rb

@@ -173,10 +173,13 @@ def page_for_record(record, create: true, row:)
     validate_kind!([String], record, 'page_url')
     validate_kind!([Array, NilClass], record, 'page_maintainers')
     validate_kind!([Array, NilClass], record, 'page_tags')
+    validate_present!(record, 'capture_time')
+    validate_kind!([String], record, 'capture_time')
 
     url = record['page_url']
 
-    existing_page = Page.find_by_url(url)
+    capture_time = Time.parse(record['capture_time'])
+    existing_page = Page.find_by_url(url, at_time: capture_time)
     page = if existing_page
              log(object: existing_page, operation: :found, row: row)
              existing_page
@@ -191,6 +194,20 @@ def page_for_record(record, create: true, row:)
     (record['page_maintainers'] || []).each {|name| page.add_maintainer(name)}
     (record['page_tags'] || []).each {|name| page.add_tag(name)}
 
+    # If the page was not an *exact* URL match, add the URL to the page.
+    # (`page.find_by_url` used above will match by `url_key`, too.)
+    page.urls.find_or_create_by(url: url)
+    # TODO: Add URLs from redirects automatically. The main blocker for
+    # this at the moment is the following situation:
+    #
+    #   Two pages, A=https://example.com/about
+    #              B=https://example.com/about/locations
+    #   Page B is removed, but instead of returning a 404 or 403 status
+    #     code, it starts redirecting to Page A.
+    #
+    # This is unfortunately not uncommon, so we need some heuristics to
+    # account for it, e.g. URL does not already belong to another page.
+
     page
   end
 

app/models/concerns/taggable.rb

@@ -2,8 +2,8 @@ module Taggable
   extend ActiveSupport::Concern
 
   included do
-    has_many :taggings, as: :taggable, foreign_key: 'taggable_uuid'
-    has_many :tags, through: :taggings
+    has_many :taggings, as: :taggable, foreign_key: 'taggable_uuid', dependent: :delete_all
+    has_many :tags, through: :taggings, dependent: nil
   end
 
   def add_tag(tag)

app/models/merged_page.rb

@@ -0,0 +1,14 @@
+# MergedPage keeps track of pages that were merged into others so we can
+# support old links by redirecting to the page they were merged into.
+# - The primary key is the ID of the page that was merged and removed
+# - `target_uuid` is the ID of the page it was merged into
+# - `audit_data` is any useful JSON data about the page (usually a frozen
+#   copy of its attributes).
+class MergedPage < ApplicationRecord
+  include UuidPrimaryKey
+
+  belongs_to :target,
+             class_name: 'Page',
+             foreign_key: :target_uuid,
+             required: true
+end