Merge branch 'LandRegistry:master' into feature/valkey-commodity

Author: brooks630

.rubocop.yml

@@ -1,5 +1,5 @@
 AllCops:
-  TargetRubyVersion: 2.5
+  TargetRubyVersion: 3.0
 
   Exclude:
     - "apps/**/*"

CODE_OF_CONDUCT.md

@@ -0,0 +1,50 @@
+# Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+- Using welcoming and inclusive language
+- Being respectful of differing viewpoints and experiences
+- Gracefully accepting constructive criticism
+- Focusing on what is best for the community
+- Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+- The use of sexualized language or imagery and unwelcome sexual attention or advances
+- Trolling, insulting/derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information, such as a physical or electronic address, without explicit permission
+- Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event.
+
+Representation of a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances.
+
+The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 When contributing to this repository, please first discuss the change you wish to make via issue, email or any other method with the owners of this repository before making a change.
 
-Please note we have a code of conduct, please follow it in all your interactions with the project.
+Please note we have a [code of conduct](CODE_OF_CONDUCT.md), please follow it in all your interactions with the project.
 
 ## Making a change
 
@@ -12,55 +12,8 @@ Please use the `develop` branch as starting point for your own branch, and the t
 
 If you're adding a commodity, at the very minimum it will need a `compose-fragment.yml` and a README entry. If there is support for extra functionality such as provisioning snippets, then a working example should be placed in the snippets directory and linked to from the README.
 
-## Code of Conduct
+It is preferred that base images come from container registries other than Docker Hub where possible to avoid users being adversely affected by pull limits (alternatives include [quay.io](https://quay.io) and [Amazon ECR](https://public.ecr.aws)),
+but only where an official image exists in those locations. If an official image only exists in Docker Hub, then use the Docker Hub image. Care should always be taken when referencing third-party base images from any container registry.
 
-### Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
-
-### Our Standards
-
-Examples of behavior that contributes to creating a positive environment include:
-
-- Using welcoming and inclusive language
-- Being respectful of differing viewpoints and experiences
-- Gracefully accepting constructive criticism
-- Focusing on what is best for the community
-- Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-- The use of sexualized language or imagery and unwelcome sexual attention or advances
-- Trolling, insulting/derogatory comments, and personal or political attacks
-- Public or private harassment
-- Publishing others' private information, such as a physical or electronic address, without explicit permission
-- Other conduct which could reasonably be considered inappropriate in a professional setting
-
-### Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
-
-### Scope
-
-This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event.
-
-Representation of a project may be further defined and clarified by project maintainers.
-
-### Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances.
-
-The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
-
-### Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
-
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/
 [how-to-contribute]: https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github
 [first-timers]: http://www.firsttimersonly.com/

README.md

@@ -12,11 +12,10 @@ It provides several hooks for applications to take advantage of, including:
 ### Prerequisites
 
 - **Docker and Docker Compose**. Exactly what toolset you use depends on your OS and personal preferences, but recommended are:
-  - [Docker For Mac](https://docs.docker.com/docker-for-mac/)
-  - [Docker for Windows 10](https://docs.docker.com/docker-for-windows/) (See [the wiki](https://github.com/LandRegistry/common-dev-env/wiki/Windows-setup) for more information on getting a working Windows environment set up)
+  - [Docker Desktop](https://docs.docker.com/desktop/)
   - [Docker CE for Linux](https://docs.docker.com/install/linux/docker-ce/ubuntu/)
 - **Git**
-- **Ruby 2.5+**
+- **Ruby 3.0+**
 
 ### Git/SSH
 
@@ -33,17 +32,19 @@ To begin:
 
 **TIP:** You can add a # onto the end of the configuration repository location followed by a branch, tag or commit you want to check out, if the default branch is not good enough.
 
+**TIP:** You can specify 'local' to manage your application list without relying on an external Git repository. This is useful when you want complete control over your configuration or are working on a new service that does not have a remote config repo yet. A default `dev-env-config/configuration.yml` will be created for you. You can then manually add applications to that list.
+
 Other `run.sh` parameters are:
 
 - `halt` - stops all containers
 - `reload` - stops all containers then rebuilds and restarts them (including running any commodity fragments)
-- `destroy` - stops/removes all containers, removes all built images (i.e. leaving any pulled from Docker Hub) and resets all dev-env configuration files.
+- `destroy` - stops/removes all containers, removes all built images (i.e. leaving any pulled from external container registries) and resets all dev-env configuration files.
 - `repair` - sets the Docker-compose configuration to use the fragments from applications in _this_ dev-env instance (in case you are switching between several or are in a different terminal window to the one you ran `up` in)
 - `quickup` and `quickreload` - as per `up` and `reload` except they do not update anything from git (apps or config), rebuild Docker images or provision any commodity fragments.
 
 #### Extra functionality
 
-- `--nopull` (or `-n`) can go after `up` or `reload` - e.g. `source run.sh up -n`. This will stop images FROMed in Dockerfiles being checked for updates if a copy already exists on the system. Use to avoid Docker Hub pull rate limits.
+- `--nopull` (or `-n`) can go after `up` or `reload` - e.g. `source run.sh up -n`. This will stop images FROMed in Dockerfiles being checked for updates if a copy already exists on the system. Use to avoid container registry pull rate limits.
 
 ## Usage guide
 
@@ -366,10 +367,6 @@ This file contains details of hosts to be forwarded on the host; if it exists th
 
 [Example](snippets/host-fragments.yml)
 
-**`/fragments/docker-compose-<any value>-fragment.yml`**
-
-This file can be used to override the default settings for a docker container such as environment variables. It will not be loaded by default but can be applied using the add-to-docker-compose alias.
-
 ## Logging
 
 Any messages that get forwarded to the logstash\* container on TCP port 25826 will be output both in the logstash container's own stdout (so `livelogs logstash` can be used to monitor all apps) and in ./logs/log.txt.

SECURITY.md

@@ -0,0 +1,7 @@
+# HM Land Registry Security and Vulnerability Reporting Policy
+
+This repository follows HM Land Registry's [reporting policy][policy] for reporting vulnerabilities that affect HM Land Registry services and systems. All persons disclosing vulnerabilities that affect this repository are requested to follow this policy when reporting.
+
+Do not raise public issues or pull requests for any vulnerabilities affecting HM Land Registry services and systems.
+
+[policy]: https://www.gov.uk/guidance/report-a-vulnerability-on-an-hm-land-registry-service-or-system

logfiles/.gitignore

@@ -0,0 +1 @@
+*.log

logic.rb

@@ -75,7 +75,7 @@
 
 # Does a version check and self-update if required
 if options['self_update']
-  this_version = '2.4.0'
+  this_version = '3.0.0'
   puts colorize_lightblue("This is a universal dev env (version #{this_version})")
   # Skip version check if not on master (prevents infinite loops if you're in a branch that isn't up to date with the
   # latest release code yet)
@@ -124,21 +124,34 @@
     File.open(DEV_ENV_CONTEXT_FILE, 'w+') { |file| file.write(config_repo) }
   end
 
+  config_repo = File.read(DEV_ENV_CONTEXT_FILE).strip
   # Check if dev-env-config exists, and if so pull the dev-env configuration. Otherwise clone it.
-  puts colorize_lightblue('Retrieving custom configuration repo files:')
   if Dir.exist?(DEV_ENV_CONFIG_DIR)
     new_project = false
-    command_successful = run_command("git -C #{root_loc}/dev-env-config pull")
+    command_successful = if config_repo == 'local'
+                           0
+                         else
+                           run_command("git -C #{root_loc}/dev-env-config pull")
+                         end
   else
     new_project = true
-    config_repo = File.read(DEV_ENV_CONTEXT_FILE)
     parsed_repo, delimiter, ref = config_repo.rpartition('#')
     # If they didn't specify a #ref, rpartition returns "", "", wholestring
     parsed_repo = ref if delimiter.empty?
-    command_successful = run_command("git clone #{parsed_repo} #{root_loc}/dev-env-config")
-    if command_successful.zero? && !delimiter.empty?
-      puts colorize_lightblue("Checking out configuration repo ref: #{ref}")
-      command_successful = run_command("git -C #{root_loc}/dev-env-config checkout #{ref}")
+    if config_repo == 'local'
+      puts colorize_lightblue('Initializing local config repository.')
+      run_command("mkdir #{DEV_ENV_CONFIG_DIR}")
+      # Create a configuration.yml with an empty services array
+      File.open("#{DEV_ENV_CONFIG_DIR}/configuration.yml", 'w') do |file|
+        file.write("---\nservices: {}\n")
+      end
+      puts colorize_green("You can start adding apps to #{DEV_ENV_CONFIG_DIR}/configuration.yml")
+      exit 1
+    else
+      command_successful = run_command("git clone #{parsed_repo} #{DEV_ENV_CONFIG_DIR}")
+      if command_successful.zero? && !delimiter.empty?
+        command_successful = run_command("git -C #{DEV_ENV_CONFIG_DIR} checkout #{ref}")
+      end
     end
   end
 
@@ -177,11 +190,9 @@
 # TODO bash autocompletion of container names
 if options['prepare_compose']
   # Create a file called .commodities.yml with the list of commodities in it
-  puts colorize_lightblue('Creating list of commodities')
   create_commodities_list(root_loc)
 
   # Call the ruby function to create the docker compose file containing the apps and their commodities
-  puts colorize_lightblue('Creating docker-compose file list')
   prepare_compose(root_loc, DOCKER_COMPOSE_FILE_LIST)
 end
 
@@ -191,24 +202,28 @@
     exit
   end
 
-  puts colorize_lightblue('Building images...')
-  if run_command("#{ENV['DC_CMD']} build " + (options['nopull'] ? '' : '--pull')) != 0
-    puts colorize_red('Something went wrong when building your app images. Check the output above.')
+  puts colorize_lightblue('Building images (might take a while)... (logging to logfiles/imagebuild.log)')
+  if run_command("#{ENV['DC_CMD']} build #{options['nopull'] ? '' : '--pull'} > logfiles/imagebuild.log 2>&1") != 0
+    puts colorize_red('Something went wrong when building the images, check the log file. Here are the last 10 lines:')
+    lines = File.readlines("#{root_loc}/logfiles/imagebuild.log")
+    last_ten = lines.last(10)
+    last_ten.each { |line| puts line }
     exit
   end
-
 end
 
 if options['provision_commodities']
   # Before creating any containers, let's see what already exists (in case we need to override provision status)
   existing_containers = []
-  # v2 --services seems to work differently. it already excludes deleted containers
   run_command("#{ENV['DC_CMD']} ps --services", existing_containers)
 
   # Let's force a recreation of the containers here so we know they're using up-to-date images
-  puts colorize_lightblue('Creating containers...')
-  if run_command("#{ENV['DC_CMD']} up --remove-orphans --force-recreate --no-start") != 0
-    puts colorize_red('Something went wrong when creating your app containers. Check the output above.')
+  puts colorize_lightblue('Recreating containers... (logging to logfiles/containercreate.log)')
+  if run_command("#{ENV['DC_CMD']} up --remove-orphans --force-recreate --no-start > logfiles/containercreate.log 2>&1") != 0
+    puts colorize_red('Something went wrong when creating the containers, check the log file. Here are the last 10 lines:')
+    lines = File.readlines("#{root_loc}/logfiles/containercreate.log")
+    last_ten = lines.last(10)
+    last_ten.each { |line| puts line }
     exit
   end
 
@@ -239,6 +254,7 @@
   # The list of expensive services currently starting
   expensive_inprogress = []
 
+  puts colorize_lightblue('Checking application configurations...')
   config['applications'].each do |appname, appconfig|
     # First, special options check (in the dev-env-config)
     # for any settings that should override what the app wants to do
@@ -274,26 +290,30 @@
     end
   end
 
-  puts colorize_lightblue('Starting log receiver service...')
-  up = run_command("#{ENV['DC_CMD']} up --no-deps --remove-orphans -d logstash")
+  up = run_command("#{ENV['DC_CMD']} up --no-deps --remove-orphans -d logstash", [])
   sleep(3)
   if up != 0
-    puts colorize_red('Something went wrong when initialising logging. Check the output above.')
+    puts colorize_red('Something went wrong when initialising live container logging. Check the output above.')
     exit
   end
 
   # Now we can start inexpensive apps, which should be quick and easy
   if services_to_start.any?
-    puts colorize_lightblue('Starting inexpensive services...')
-    up = run_command("#{ENV['DC_CMD']} up --no-deps --remove-orphans -d " + services_to_start.join(' '))
+    puts colorize_lightblue('Starting inexpensive services... (logging to logfiles/containerstart.log)')
+    up = run_command("#{ENV['DC_CMD']} up --no-deps --remove-orphans -d #{services_to_start.join(' ')}")
     if up != 0
-      puts colorize_red('Something went wrong when creating your app images or containers. Check the output above.')
+      puts colorize_red('Something went wrong when starting the containers, check the log file. Here are the last 10 lines:')
+      lines = File.readlines("#{root_loc}/logfiles/containerstart.log")
+      last_ten = lines.last(10)
+      last_ten.each { |line| puts line }
       exit
     end
   end
 
   # Until we have no more left to start AND we have no more in progress...
-  puts colorize_lightblue('Starting expensive services...') if expensive_todo.length.positive?
+  if expensive_todo.length.positive?
+    puts colorize_lightblue('Starting expensive services... (logging to logfiles/containerstart.log)')
+  end
   expensive_failed = []
   while expensive_todo.length.positive? || expensive_inprogress.length.positive?
     # Wait for a bit before the next round of checks
@@ -405,7 +425,7 @@
       puts colorize_yellow("  #{service['compose_service']}")
     end
   else
-    puts colorize_green('All done, environment is ready for use')
+    puts colorize_green('Environment is ready for use')
   end
 
   post_up_message = config.fetch('post-up-message', nil)

scripts/add-aliases.sh

@@ -168,20 +168,6 @@ function alembic(){
         bash -c 'cd /src && python3 manage.py db '"${@:2}"''
 }
 
-function add-to-docker-compose(){
-  COMPOSE_FILE_LIST=$(printenv COMPOSE_FILE)
-  IFS=':' read -r -a array <<< "$COMPOSE_FILE_LIST"
-  for element in "${array[@]}"
-    do
-      if [ -f ${element%/*}/docker-compose-${1}-fragment.yml ]; then
-        COMPOSE_FILE_LIST="$COMPOSE_FILE_LIST:${element%/*}/docker-compose-${1}-fragment.yml"
-      fi
-    done
-  export COMPOSE_FILE=$COMPOSE_FILE_LIST
-  alias test="$DC_CMD up -d"
-  test
-}
-
 function devenv-help(){
   cat <<EOF
     If typing a docker-compose command you can use the alias dc instead. For example "dc ps" rather than "docker-compose ps".