From e4d83fc6a7d1a62cd77e0f111386f6ffd898b299 Mon Sep 17 00:00:00 2001 From: Faustin Lammler Date: Mon, 13 Jun 2022 14:12:25 +0200 Subject: [PATCH] Re-enable CI for new repo --- .github/workflows/bb_containers.yml | 10 +- README.md | 163 +++++++++++++--------------- ci_build_images/buildbot.tac | 39 +++++++ ci_build_images/common.Dockerfile | 4 +- constants.py | 2 +- master-libvirt/master.cfg | 20 ++-- master-nonlatent/master.cfg | 6 +- master.cfg | 14 +-- utils.py | 4 +- 9 files changed, 147 insertions(+), 115 deletions(-) create mode 100644 ci_build_images/buildbot.tac diff --git a/.github/workflows/bb_containers.yml b/.github/workflows/bb_containers.yml index dba56c9a..a0316282 100644 --- a/.github/workflows/bb_containers.yml +++ b/.github/workflows/bb_containers.yml @@ -5,11 +5,11 @@ on: push: paths: - .github/workflows/bb_containers.yml - - "buildbot.mariadb.org/ci_build_images/**" + - "ci_build_images/**" pull_request: paths: - .github/workflows/bb_containers.yml - - "buildbot.mariadb.org/ci_build_images/**" + - "ci_build_images/**" jobs: build: @@ -97,7 +97,7 @@ jobs: env: BUILD_RHEL: false DEPLOY_IMAGES: false - WORKDIR: buildbot.mariadb.org/ci_build_images + WORKDIR: ci_build_images steps: - uses: actions/checkout@v2 @@ -197,8 +197,8 @@ jobs: done - name: Check for registry credentials if: > - github.ref == 'refs/heads/master' && - github.repository == 'MariaDB/mariadb.org-tools' + github.ref == 'refs/heads/main' && + github.repository == 'MariaDB/buildbot' run: | missing=() [[ -n "${{ secrets.QUAY_USER }}" ]] || missing+=(QUAY_USER) diff --git a/README.md b/README.md index 7478270e..d224ae12 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,17 @@ - ``` - _________________________ -< Build all the things! > - ------------------------- - \ ^__^ - \ (oo)\_______ - (__)\ )\/\ - ||----w | - || || - ``` - -# Overview -========== +# buildbot.mariadb.org + +```console +_________________________ +< Build all the things! > +------------------------- + \ ^__^ + \ (oo)\_______ + (__)\ )\/\ + ||----w | + || || +``` + +## Overview buildbot.mariadb.org is our continuous integration testing platform, based on the Buildbot.net open-source testing framework. @@ -20,7 +21,7 @@ This directory contains all the configuration needed to deploy a new installatio Quick layout of the main components of the current directory structure: -``` +```console buildbot.mariadb.org/ ├── buildbot.tac ├── dockerfiles @@ -55,114 +56,111 @@ buildbot.mariadb.org/ └── nginx.conf ``` -* __buildbot.tac__: the only application configuration file Buildbot needs to get up and running in master mode, this one can remain largely unchanged -* __dockerfiles/buildbot.tac__: worker application configuration file, being sent by the master to the Docker workers -* __dockerfiles/*.dockerfile__: [docker image files](https://docs.docker.com/engine/reference/builder/), also sent from the master to the docker based worker machines -* __master.cfg__: the master configuration file, describes the build scheme and all other site site specific configuration -* __sponsor.py__: custom Buildbot dashboard plugin that adds a menu item named *Sponsors*, used to list all the donated servers that are being used by this instance -* __static__: image logos for the donors HTML presentation of the dashboard plugin -* __templates__: HTML templates for dashboard plugins, currently only *sponsor.html* for the Sponsors plugin -* __util__: various associated config files and tools, like the systemd service file and nginx configuration -* __master-web__: a separate master configuration for handling the UI - -## A word on Docker -=================== +- **buildbot.tac**: the only application configuration file Buildbot needs to get up and running in master mode, this one can remain largely unchanged +- **dockerfiles/buildbot.tac**: worker application configuration file, being sent by the master to the Docker workers +- **dockerfiles/\*.dockerfile**: [docker image files](https://docs.docker.com/engine/reference/builder/), also sent from the master to the docker based worker machines +- **master.cfg**: the master configuration file, describes the build scheme and all other site site specific configuration +- **sponsor.py**: custom Buildbot dashboard plugin that adds a menu item named _Sponsors_, used to list all the donated servers that are being used by this instance +- **static**: image logos for the donors HTML presentation of the dashboard plugin +- **templates**: HTML templates for dashboard plugins, currently only _sponsor.html_ for the Sponsors plugin +- **util**: various associated config files and tools, like the systemd service file and nginx configuration +- **master-web**: a separate master configuration for handling the UI -We are using Docker for targeting Linux builds. This makes it easy (and cheap) to add new distribution flavored builders with just a Dockerfile, while also facilitating environment replication locally, without much effort, making it easy to inspect, reproduce the build environment and debug failures. +### A word on Docker Using Buildbot's DockerLatentWorker to connect to docker instances, the worker runs inside the Docker image, and on a build trigger, master connects to Docker first via docker-py , instantiates the image and then connects using Buildbot usual Worker API to do the heavy lifting. Upon build completion, the image is stopped and no left-overs are left behind. - -## Navigating the new interface -=============================== +### Navigating the new interface Starting with version 1.0, Buildbot interface changed in a significant and positive manner. The new UI is a fresh rewrite with responsive, lazy loading and modern web conveniences that makes it easy to follow running builds as well as convenient to browse build history. Grid View and MTRLogObserver plugin are usable out of the box and behave in a similar fashion as in the current Buildbot instance. -# Debugging build failures -========================== +## Debugging build failures Debugging build failures with Docker is facilitated by the ease of replicating the actual build environment where a particular builder failed. First step is to identify on which platform the failure occurred and what's the associated Dockerfile. If you're not familiar with Docker, start by reading the [Docker overview](https://docs.docker.com/engine/docker-overview/) and [Docker getting started documentation](https://docs.docker.com/get-started/). -To repeat a build on buildbot, first identify the platform and look for the associated dockerfile in the dockerfiles/ directory. To create a container for Ubuntu 18.04 for e.g. you would use the *docker build* command like this: +To repeat a build on buildbot, first identify the platform and look for the associated dockerfile in the dockerfiles/ directory. To create a container for Ubuntu 18.04 for e.g. you would use the _docker build_ command like this: `docker build -t ubuntu-1804 --file ubuntu1804.dockerfile .` -After a successful image build, you can run a container and execute bash using that image with *docker run*: +After a successful image build, you can run a container and execute bash using that image with _docker run_: `docker run -it ubuntu-1804 bash` After this step you can follow the usual MariaDB development setup instructions, i.e. clone the repo and start a build. To specifically reproduce what a particular Buildbot worker did, look into the master.cfg configuration file what addStep() has been defined for that worker, and copy paste the contents of those into the session you are running in the Docker image. -# Deployment -============ +## Deployment Deploying a new master (since Buildbot > 1.0 supports multi-master configuration): 1. Setup your server. This configuration has been tested on Buildbot 2.0.1, Python 3 and Ubuntu so far. -2. Install Buildbot and associated plugins, you have two options here: +1. Install Buildbot and associated plugins, you have two options here: a) install from the official distribution repositories (1.1 in Ubuntu 18.04) b) install from the PyPi repository which should have the latest stable release (2.0.1 in March 2019) Follow the official install instructions here: - http://docs.buildbot.net/latest/manual/installation/installation.html -3. Clone this directory to your desired master destination. A good choice on Ubuntu would be: + +1. Clone this directory to your desired master destination. A good choice on Ubuntu would be: `/srv/buildbot/master` -4. Create the master using: +1. Create the master using: `buildbot create-master -r /srv/buildbot/master` `buildbot upgrade-master /srv/buildbot/master` -5. Start the master using: +1. Start the master using: `buildbot start /srv/buildbot/master` Stop, restart and reconfig are the other most relevant commands. -6. For service persistence and convenience, consider using the systemd service file *util/buildbot-master.service*. -7. For https support and better HTTP performance overall, consider setting up Nginx as a reverse proxy, see *util/nginx.conf*. -8. Some build artefacts are uploaded from workers to the master for archival and debugging purposes. In our setup, they are placed in /srv/buildbot/bb_builds/. This directory is also exposed via nginx index listing at [ci.mariadb.org](https://ci.mariadb.org). +1. For service persistence and convenience, consider using the systemd service file _util/buildbot-master.service_. +1. For https support and better HTTP performance overall, consider setting up Nginx as a reverse proxy, see _util/nginx.conf_. +1. Some build artefacts are uploaded from workers to the master for archival and debugging purposes. In our setup, they are placed in /srv/buildbot/bb_builds/. This directory is also exposed via nginx index listing at [ci.mariadb.org](https://ci.mariadb.org). To upgrade Buildbot to latest version: -```bash -$ sudo systemctl stop buildbot-master -$ sudo pip3 install -U buildbot buildbot-grid-view buildbot-waterfall-view buildbot-console-view buildbot-wsgi-dashboards buildbot-www -$ buildbot upgrade-master /srv/buildbot/master -$ buildbot cleanupdb /srv/buildbot/master -$ sudo systemctl start buildbot-master + +```console +sudo systemctl stop buildbot-master +sudo pip3 install -U buildbot buildbot-grid-view buildbot-waterfall-view buildbot-console-view buildbot-wsgi-dashboards buildbot-www +buildbot upgrade-master /srv/buildbot/master +buildbot cleanupdb /srv/buildbot/master +sudo systemctl start buildbot-master ``` + To upgrade Buildbot-worker to latest version: -```bash -$ sudo pip3 install -U buildbot-worker + +```console +sudo pip3 install -U buildbot-worker ``` -## Master configuration +### Master configuration The master configuration can be site specific and should be customized according to the needs and purpose for a particular master. What follows is a description of our current master configuration. Master.cfg is a Python script and defines the behavior of Buildbot. These are the main sections of interest: -* PROJECT IDENTITY - various site specific definitions -* WORKERS - defines the list of recognized workers by this instance -* CHANGESOURCES - defines list of repositories to watch for changes -* SCHEDULERS - defines what changes we are interested in and triggers appropriate builders based on that -* BUILDERS & FACTORY CODE - defines the steps for any individual builder +- PROJECT IDENTITY - various site specific definitions +- WORKERS - defines the list of recognized workers by this instance +- CHANGESOURCES - defines list of repositories to watch for changes +- SCHEDULERS - defines what changes we are interested in and triggers appropriate builders based on that +- BUILDERS & FACTORY CODE - defines the steps for any individual builder -Also, master.cfg sources a private config file that is not included in this repo, namely *master-private.cfg*. This file should include a Python dictionary with private information like worker passwords, database url and anything else not deemed for public disclosure. +Also, master.cfg sources a private config file that is not included in this repo, namely _master-private.cfg_. This file should include a Python dictionary with private information like worker passwords, database url and anything else not deemed for public disclosure. -### Reloading the master +#### Reloading the master For changes to the master.cfg to take effect, the server needs to be reloaded. While doing this, please keep an eye on the logs with `tail -f /srv/buildbot/master/twistd.log`. The Buildbot master is managed by our own rolled systemd file. To invoke a reload run `sudo systemctl reload buildbot-master`. The other usual systemctl are also available (`status`, `restart`). Before the reload, consider testing the config with `buildbot checkconfig`. -## Adding workers +### Adding workers There's currently two types of workers that we use. Normal Buildbot workers, running bare-bones builds and can be added using `c['workers'].append(mkWorker("bm-bbw1-ubuntu1804"))`. The password entry needs to be defined in `master-private.cfg`. Note that the worker name cannot be a hostname with dots. Only alphanumeric and dashes are allowed in the worker name. -For instructions on setting up a Buildbot Worker see http://docs.buildbot.net/latest/manual/installation/worker.html. +For instructions on setting up a Buildbot Worker see . To add a buildbot worker use the following commands: -```bash -$ useradd buildbot -$ su - buildbot -$ python3 -m venv buildbot-worker -$ source buildbot-worker/bin/activate -$ pip install buildbot-worker -$ curl -o buildbot.tac https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot.mariadb.org/dockerfiles/buildbot.tac + +```console +useradd buildbot +su - buildbot +python3 -m venv buildbot-worker +source buildbot-worker/bin/activate +pip install buildbot-worker +curl -o buildbot.tac https://raw.githubusercontent.com/MariaDB/buildbot/main/dockerfiles/buildbot.tac #set buildmaster, port and worker user and password -$ buildbot-worker start +buildbot-worker start ``` The other type of worker we use is [DockerLatentWorker](https://docs.buildbot.net/current/manual/configuration/workers-docker.html). It is useful for running tests inside single use, disposable container Linux instances. We are currently aiming to use it for most of our Linux builds. @@ -170,13 +168,15 @@ The other type of worker we use is [DockerLatentWorker](https://docs.buildbot.ne Adding a Docker based worker is as simple as providing the IP address to the Docker server and everything else can be handled on the master side. We currently have 2 Docker instances: -* `do-ubuntu-1804-bbw1-docker`: DigitalOcean worker 1 running Debian/Ubuntu (including main tarbake) builds -* `do-ubuntu-1804-bbw2-docker`: DigitalOcean worker 2 running Fedora, CentOS, OpenSuSe docker builds + +- `do-ubuntu-1804-bbw1-docker`: DigitalOcean worker 1 running Debian/Ubuntu (including main tarbake) builds +- `do-ubuntu-1804-bbw2-docker`: DigitalOcean worker 2 running Fedora, CentOS, OpenSuSe docker builds This separation is purely conventional and can be arbitrarily implemented subject to available resources on each Docker host. Sample DockerLatentWorker configuration: -``` + +```python c['workers'].append(worker.DockerLatentWorker("do-bbw1-docker-tarball", None, docker_host='tcp://10.0.0.46:2375', dockerfile=open("dockerfiles/debian.dockerfile").read(), @@ -186,23 +186,16 @@ c['workers'].append(worker.DockerLatentWorker("do-bbw1-docker-tarball", None, properties={ 'jobs':'2' })) ``` -## Current build matrix +### Current build matrix We currently define a build matrix via _supportedPlatforms_ which specified which builders should we run for a particular change after figuring out which branch it is based on. For this particular reason, all branch names should follow the following naming scheme: `-` or `--`. Examples -* `10.4` - main development branch that becomes the next 10.4.x release -* `10.0-galera` - main development branch that becomes the next 10.0.x Galera release -* `bb-10.3-feature` - feature branch targeting next 10.3.x release, tested by Buildbot normally -* `hf-5.5-fixbug` - hotfix branch targeting next 5.5.x release, **not** tested by Buildbot as it lacks bb-* - - -## Adding builders - -http://docs.buildbot.net/latest/manual/cfg-builders.html -http://docs.buildbot.net/latest/manual/cfg-buildsteps.html --TBD- +- `10.4` - main development branch that becomes the next 10.4.x release +- `10.0-galera` - main development branch that becomes the next 10.0.x Galera release +- `bb-10.3-feature` - feature branch targeting next 10.3.x release, tested by Buildbot normally +- `hf-5.5-fixbug` - hotfix branch targeting next 5.5.x release, **not** tested by Buildbot as it lacks bb-\* ## Special Thanks diff --git a/ci_build_images/buildbot.tac b/ci_build_images/buildbot.tac new file mode 100644 index 00000000..7bfce797 --- /dev/null +++ b/ci_build_images/buildbot.tac @@ -0,0 +1,39 @@ +import fnmatch +import os +import sys + +from twisted.application import service +from twisted.python.log import FileLogObserver +from twisted.python.log import ILogObserver + +from buildbot_worker.bot import Worker + +# setup worker +basedir = os.path.abspath(os.path.dirname(__file__)) +application = service.Application('buildbot-worker') + + +application.setComponent(ILogObserver, FileLogObserver(sys.stdout).emit) +# and worker on the same process! +buildmaster_host = os.environ.get("BUILDMASTER", 'localhost') +port = int(os.environ.get("BUILDMASTER_PORT", 9989)) +workername = os.environ.get("WORKERNAME", 'docker') +passwd = os.environ.get("WORKERPASS") + +# delete the password from the environ so that it is not leaked in the log +blacklist = os.environ.get("WORKER_ENVIRONMENT_BLACKLIST", "WORKERPASS").split() +for name in list(os.environ.keys()): + for toremove in blacklist: + if fnmatch.fnmatch(name, toremove): + del os.environ[name] + +keepalive = 600 +umask = None +maxdelay = 300 +allow_shutdown = None +maxretries = 10 + +s = Worker(buildmaster_host, port, workername, passwd, basedir, + keepalive, umask=umask, maxdelay=maxdelay, + allow_shutdown=allow_shutdown, maxRetries=maxretries, unicode_encoding='utf-8') +s.setServiceParent(application) diff --git a/ci_build_images/common.Dockerfile b/ci_build_images/common.Dockerfile index e5cafbfb..e0c9c3ea 100644 --- a/ci_build_images/common.Dockerfile +++ b/ci_build_images/common.Dockerfile @@ -4,7 +4,7 @@ RUN useradd -ms /bin/bash buildbot \ && gosu buildbot curl -so /home/buildbot/buildbot.tac \ # TODO move buildbot.tac to ci_build_images - https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot.mariadb.org/dockerfiles/buildbot.tac \ + https://raw.githubusercontent.com/MariaDB/buildbot/main/dockerfiles/buildbot.tac \ && echo "[[ -d /home/buildbot/.local/bin/ ]] && export PATH=\"/home/buildbot/.local/bin:\$PATH\"" >>/home/buildbot/.bashrc \ # autobake-deb (debian/ubuntu) will need sudo rights \ && if grep -qi "debian" /etc/os-release; then \ @@ -33,7 +33,7 @@ RUN gosu buildbot curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >/tm else \ pip3 install --no-cache-dir -U pip; \ gosu buildbot curl -so /home/buildbot/requirements.txt \ - https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot.mariadb.org/ci_build_images/requirements.txt; \ + https://raw.githubusercontent.com/MariaDB/buildbot/main/ci_build_images/requirements.txt; \ # https://jira.mariadb.org/browse/MDBF-329 \ if grep -q "stretch" /etc/apt/sources.list; then \ gosu buildbot bash -c "pip3 install --no-cache-dir --no-warn-script-location incremental"; \ diff --git a/constants.py b/constants.py index 685540a3..2598ee80 100644 --- a/constants.py +++ b/constants.py @@ -1,6 +1,6 @@ import yaml -with open('/srv/buildbot/mariadb.org-tools/buildbot.mariadb.org/os_info.yaml', 'r') as f: +with open('/srv/buildbot/master/os_info.yaml', 'r') as f: os_info = yaml.safe_load(f) github_status_builders = [ diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index bdf7af0f..1183b8fb 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -14,7 +14,7 @@ from datetime import timedelta sys.setrecursionlimit(10000) -sys.path.insert(0, '/srv/buildbot/mariadb.org-tools/buildbot.mariadb.org') +sys.path.insert(0, '/srv/buildbot/master') from utils import * @@ -247,12 +247,12 @@ for builder_name in builders_install: nextBuild=nextBuild, canStartBuild=canStartBuild, properties={ - 'systemdCapability': 'yes', - 'needsGalera': 'yes', - 'dist_name': os_name, + 'systemdCapability': 'yes', + 'needsGalera': 'yes', + 'dist_name': os_name, 'version_name': os_info[os_info_name]['version_name'], - 'arch': build_arch, - 'test_mode': 'server', + 'arch': build_arch, + 'test_mode': 'server', 'test_type': 'major', 'BB_CI': True, }, @@ -269,12 +269,12 @@ for builder_name in builders_install: nextBuild=nextBuild, canStartBuild=canStartBuild, properties={ - 'systemdCapability': 'yes', - 'needsGalera': 'yes', - 'dist_name': os_name, + 'systemdCapability': 'yes', + 'needsGalera': 'yes', + 'dist_name': os_name, 'version_name': os_info[os_info_name]['version_name'], 'arch': build_arch, - 'test_mode': 'server', + 'test_mode': 'server', 'test_type': "minor", 'BB_CI': True, }, diff --git a/master-nonlatent/master.cfg b/master-nonlatent/master.cfg index 4e2e6372..1bc84f7f 100644 --- a/master-nonlatent/master.cfg +++ b/master-nonlatent/master.cfg @@ -14,7 +14,7 @@ from datetime import timedelta sys.setrecursionlimit(10000) -sys.path.insert(0, '/srv/buildbot/mariadb.org-tools/buildbot.mariadb.org') +sys.path.insert(0, '/srv/buildbot/master') from utils import * from locks import * from common_factories import * @@ -188,7 +188,7 @@ f_windows_msi.addStep(steps.ShellCommand(name="cleanup", command=["dojob", '"', f_dockerlibrary = util.BuildFactory() f_dockerlibrary.addStep(steps.ShellCommand( name="Update Fetch/Test Script", - command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot.mariadb.org/scripts/docker-library-build-and-test.sh -o docker-library-build-and-test.sh && chmod a+x docker-library-build-and-test.sh"])) + command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/buildbot/main/scripts/docker-library-build-and-test.sh -o docker-library-build-and-test.sh && chmod a+x docker-library-build-and-test.sh"])) f_dockerlibrary.addStep(steps.ShellCommand( name="building and test docker library image for MariaDB", command=["bash", "-xc", util.Interpolate("./docker-library-build-and-test.sh \"%(prop:tarbuildnum)s\" \"%(prop:mariadb_version)s\" \"%(prop:parentbuildername)s\" \"%(prop:revision)s\" \"%(prop:branch)s\"")])) @@ -198,7 +198,7 @@ f_aix = util.BuildFactory() f_aix.addStep(downloadSourceTarballAIX()) f_aix.addStep(steps.ShellCommand(command=util.Interpolate("tar -xvzf /mnt/packages/%(prop:tarbuildnum)s_%(prop:mariadb_version)s.tar.gz"))) f_aix.addStep(steps.ShellCommand(name="create html log file", command=['bash', '-c', util.Interpolate(getHTMLLogString(), jobs=util.Property('jobs', default='6'))])) -f_aix.addStep(steps.ShellCommand(name="fetch build script", command=['/opt/freeware/bin/curl', '--etag-compare', 'aix-script-tag', '--etag-save', 'aix-script-tag', 'https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot.mariadb.org/scripts/aix-build-and-test.sh', '--output', 'aix-build-and-test.sh'])) +f_aix.addStep(steps.ShellCommand(name="fetch build script", command=['/opt/freeware/bin/curl', '--etag-compare', 'aix-script-tag', '--etag-save', 'aix-script-tag', 'https://raw.githubusercontent.com/MariaDB/buildbot/main/scripts/aix-build-and-test.sh', '--output', 'aix-build-and-test.sh'])) # build steps f_aix.addStep(steps.Compile(command= ["sh", "-c", util.Interpolate("chmod a+x ./aix-build-and-test.sh && ./aix-build-and-test.sh build %(prop:mariadb_version)s %(kw:build_type)s %(kw:jobs)s", build_type=util.Property('build_type', default='RelWithDebInfo'))], haltOnFailure="true")) diff --git a/master.cfg b/master.cfg index afaeaf10..f5dfe281 100644 --- a/master.cfg +++ b/master.cfg @@ -683,10 +683,10 @@ f_without_server.addStep(steps.ShellCommand(name="cleanup", command="rm -r * .* f_eco_php = util.BuildFactory() f_eco_php.addStep(steps.ShellCommand( name="fetch_install_script", - command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot.mariadb.org/dockerfiles/ecofiles/installdb.sh -o /buildbot/installdb.sh && chmod a+x /buildbot/installdb.sh"])) + command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/buildbot/main/dockerfiles/ecofiles/installdb.sh -o /buildbot/installdb.sh && chmod a+x /buildbot/installdb.sh"])) f_eco_php.addStep(steps.ShellCommand( name="fetch_test_script", - command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot.mariadb.org/dockerfiles/ecofiles/test-php.sh -o /buildbot/test-php.sh && chmod a+x /buildbot/test-php.sh"])) + command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/buildbot/main/dockerfiles/ecofiles/test-php.sh -o /buildbot/test-php.sh && chmod a+x /buildbot/test-php.sh"])) f_eco_php.addStep(steps.ShellCommand( name="fetching and installing database", command=["sh", "-xc", util.Interpolate("/buildbot/installdb.sh \"https://ci.mariadb.org/%(prop:tarbuildnum)s/%(prop:parentbuildername)s/%(prop:mariadb_binary)s\" --plugin-load-add=auth_pam --pam_use_cleartext_plugin")])) @@ -707,7 +707,7 @@ f_eco_php.addStep(steps.ShellCommand( f_eco_dbdeployer = util.BuildFactory() f_eco_dbdeployer.addStep(steps.ShellCommand( name="fetch_test_script", - command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot.mariadb.org/dockerfiles/ecofiles/test-dbdeployer.sh -o /buildbot/test-dbdeployer.sh && chmod a+x /buildbot/test-dbdeployer.sh"])) + command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/buildbot/main/dockerfiles/ecofiles/test-dbdeployer.sh -o /buildbot/test-dbdeployer.sh && chmod a+x /buildbot/test-dbdeployer.sh"])) f_eco_dbdeployer.addStep(steps.ShellCommand( name="download if needed latest dbdeployer", command=["sh", "-xc", "/buildbot/test-dbdeployer.sh dbdeployerfetch"])) @@ -731,10 +731,10 @@ f_eco_dbdeployer.addStep(steps.ShellCommand( f_eco_pymysql = util.BuildFactory() f_eco_pymysql.addStep(steps.ShellCommand( name="fetch_install_script", - command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot.mariadb.org/dockerfiles/ecofiles/installdb.sh -o /buildbot/installdb.sh && chmod a+x /buildbot/installdb.sh"])) + command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/buildbot/main/dockerfiles/ecofiles/installdb.sh -o /buildbot/installdb.sh && chmod a+x /buildbot/installdb.sh"])) f_eco_pymysql.addStep(steps.ShellCommand( name="fetch_test_script", - command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot.mariadb.org/dockerfiles/ecofiles/test-pymysql.sh -o /buildbot/test-pymysql.sh && chmod a+x /buildbot/test-pymysql.sh"])) + command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/buildbot/main/dockerfiles/ecofiles/test-pymysql.sh -o /buildbot/test-pymysql.sh && chmod a+x /buildbot/test-pymysql.sh"])) f_eco_pymysql.addStep(steps.ShellCommand( name="fetching and installing database", command=["sh", "-xc", util.Interpolate("/buildbot/installdb.sh \"https://ci.mariadb.org/%(prop:tarbuildnum)s/%(prop:parentbuildername)s/%(prop:mariadb_binary)s\"")])) @@ -749,10 +749,10 @@ f_eco_pymysql.addStep(steps.ShellCommand( f_eco_mysqljs = util.BuildFactory() f_eco_mysqljs.addStep(steps.ShellCommand( name="fetch_install_script", - command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot.mariadb.org/dockerfiles/ecofiles/installdb.sh -o /buildbot/installdb.sh && chmod a+x /buildbot/installdb.sh"])) + command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/buildbot/main/dockerfiles/ecofiles/installdb.sh -o /buildbot/installdb.sh && chmod a+x /buildbot/installdb.sh"])) f_eco_mysqljs.addStep(steps.ShellCommand( name="fetch_test_script", - command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot.mariadb.org/dockerfiles/ecofiles/test-mysqljs.sh -o /buildbot/test-mysqljs.sh && chmod a+x /buildbot/test-mysqljs.sh"])) + command=["sh", "-xc", "curl https://raw.githubusercontent.com/MariaDB/buildbot/main/dockerfiles/ecofiles/test-mysqljs.sh -o /buildbot/test-mysqljs.sh && chmod a+x /buildbot/test-mysqljs.sh"])) f_eco_mysqljs.addStep(steps.ShellCommand( name="fetching and installing database", command=["sh", "-xc", util.Interpolate("/buildbot/installdb.sh \"https://ci.mariadb.org/%(prop:tarbuildnum)s/%(prop:parentbuildername)s/%(prop:mariadb_binary)s\"")])) diff --git a/utils.py b/utils.py index 32db4a77..a95eda5e 100644 --- a/utils.py +++ b/utils.py @@ -36,7 +36,7 @@ def getScript(scriptname): name=f"fetch_{scriptname}", command=["bash", "-xc", f""" for script in bash_lib.sh {scriptname}; do - [[ ! -f $script ]] && wget "https://raw.githubusercontent.com/MariaDB/mariadb.org-tools/master/buildbot.mariadb.org/scripts/$script" + [[ ! -f $script ]] && wget "https://raw.githubusercontent.com/MariaDB/buildbot/main/scripts/$script" done chmod a+x {scriptname} """]) @@ -182,7 +182,7 @@ def moveMTRLogs(): @util.renderer def dockerfile(props): worker = props.getProperty('workername') - return "https://github.com/MariaDB/mariadb.org-tools/tree/master/buildbot.mariadb.org/dockerfiles/" + "-".join(worker.split('-')[-2:]) + '.dockerfile' + return "https://github.com/MariaDB/buildbot/tree/main/dockerfiles/" + "-".join(worker.split('-')[-2:]) + '.dockerfile' # checks if the list of files is empty def hasFiles(step):