Skip to content

Commit

Permalink
CI: Use Kubevirt VM to run Molecule and Vagrant jobs
Browse files Browse the repository at this point in the history
  • Loading branch information
ant31 committed Jun 25, 2024
1 parent 1a49fe7 commit f3239e7
Show file tree
Hide file tree
Showing 10 changed files with 187 additions and 41 deletions.
2 changes: 1 addition & 1 deletion .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ before_script:
script:
- ./tests/scripts/testcases_run.sh
after_script:
- chronic ./tests/scripts/testcases_cleanup.sh
- ./tests/scripts/testcases_cleanup.sh

# For failfast, at least 1 job must be defined in .gitlab-ci.yml
# Premoderated with manual actions
Expand Down
61 changes: 35 additions & 26 deletions .gitlab-ci/molecule.yml
Original file line number Diff line number Diff line change
@@ -1,33 +1,42 @@
---

.molecule:
tags: [c3.small.x86]
tags: [ffci-vm-med]
only: [/^pr-.*$/]
except: ['triggers']
image: $PIPELINE_IMAGE
image: quay.io/kubespray/vm-kubespray-ci:v6
services: []
stage: deploy-part1
needs:
- pipeline-image
- ci-not-authorized
needs: []
# - ci-not-authorized
variables:
VAGRANT_DEFAULT_PROVIDER: "libvirt"
before_script:
- tests/scripts/rebase.sh
- ./tests/scripts/vagrant_clean.sh
- groups
- python3 -m venv citest
- source citest/bin/activate
- vagrant plugin expunge --reinstall --force --no-tty
- vagrant plugin install vagrant-libvirt
- pip install --no-compile --no-cache-dir pip -U
- pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt
- pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
- ./tests/scripts/rebase.sh
- ./tests/scripts/vagrant_clean.sh
script:
- ./tests/scripts/molecule_run.sh
- ./tests/scripts/molecule_run.sh
after_script:
- chronic ./tests/scripts/molecule_logs.sh
- ./tests/scripts/molecule_logs.sh
artifacts:
when: always
paths:
- molecule_logs/
- molecule_logs/

# CI template for periodic CI jobs
# Enabled when PERIODIC_CI_ENABLED var is set

.molecule_periodic:
only:
variables:
- $PERIODIC_CI_ENABLED
- $PERIODIC_CI_ENABLED
allow_failure: true
extends: .molecule

Expand All @@ -37,50 +46,50 @@ molecule_full:
molecule_no_container_engines:
extends: .molecule
script:
- ./tests/scripts/molecule_run.sh -e container-engine
- ./tests/scripts/molecule_run.sh -e container-engine
when: on_success

molecule_docker:
extends: .molecule
script:
- ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd
- ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd
when: on_success

molecule_containerd:
extends: .molecule
script:
- ./tests/scripts/molecule_run.sh -i container-engine/containerd
- ./tests/scripts/molecule_run.sh -i container-engine/containerd
when: on_success

molecule_cri-o:
extends: .molecule
stage: deploy-part1
script:
- ./tests/scripts/molecule_run.sh -i container-engine/cri-o
- ./tests/scripts/molecule_run.sh -i container-engine/cri-o
allow_failure: true
when: on_success

# Stage 3 container engines don't get as much attention so allow them to fail
molecule_kata:
extends: .molecule
stage: deploy-extended
script:
- ./tests/scripts/molecule_run.sh -i container-engine/kata-containers
when: manual
# FIXME: this test is broken (perma-failing)
# # Stage 3 container engines don't get as much attention so allow them to fail
# molecule_kata:
# extends: .molecule
# stage: deploy-extended
# script:
# - ./tests/scripts/molecule_run.sh -i container-engine/kata-containers
# when: manual
# # FIXME: this test is broken (perma-failing)

molecule_gvisor:
extends: .molecule
stage: deploy-extended
script:
- ./tests/scripts/molecule_run.sh -i container-engine/gvisor
- ./tests/scripts/molecule_run.sh -i container-engine/gvisor
when: manual
# FIXME: this test is broken (perma-failing)

molecule_youki:
extends: .molecule
stage: deploy-extended
script:
- ./tests/scripts/molecule_run.sh -i container-engine/youki
- ./tests/scripts/molecule_run.sh -i container-engine/youki
when: manual
# FIXME: this test is broken (perma-failing)
10 changes: 8 additions & 2 deletions .gitlab-ci/terraform.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
# Tests for contrib/terraform/
.terraform_install:
extends: .job
stage: test
needs:
- ci-not-authorized
stage: deploy-part1
before_script:
- update-alternatives --install /usr/bin/python python /usr/bin/python3 1
- ./tests/scripts/rebase.sh
Expand Down Expand Up @@ -30,6 +32,10 @@
script:
- terraform -chdir="contrib/terraform/$PROVIDER" validate
- terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
stage: test
needs:
- ci-not-authorized
- pipeline-image

.terraform_apply:
extends: .terraform_install
Expand All @@ -51,7 +57,7 @@
- tests/scripts/testcases_run.sh
after_script:
# Cleanup regardless of exit code
- chronic ./tests/scripts/testcases_cleanup.sh
- ./tests/scripts/testcases_cleanup.sh

tf-validate-openstack:
extends: .terraform_validate
Expand Down
22 changes: 14 additions & 8 deletions .gitlab-ci/vagrant.yml
Original file line number Diff line number Diff line change
@@ -1,26 +1,32 @@
---

.vagrant:
extends: .testcases
needs:
- pipeline-image
- ci-not-authorized
variables:
CI_PLATFORM: "vagrant"
SSH_USER: "vagrant"
VAGRANT_DEFAULT_PROVIDER: "libvirt"
KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb
tags: [c3.small.x86]
only: [/^pr-.*$/]
except: ['triggers']
image: $PIPELINE_IMAGE
DOCKER_NAME: vagrant
VAGRANT_ANSIBLE_TAGS: facts
tags: [ffci-vm-large]
# only: [/^pr-.*$/]
# except: ['triggers']
image: quay.io/kubespray/vm-kubespray-ci:v6
services: []
before_script:
- echo $USER
- python3 -m venv citest
- source citest/bin/activate
- vagrant plugin expunge --reinstall --force --no-tty
- vagrant plugin install vagrant-libvirt
- pip install --no-compile --no-cache-dir pip -U
- pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/requirements.txt
- pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
- ./tests/scripts/vagrant_clean.sh
script:
- ./tests/scripts/testcases_run.sh
after_script:
- chronic ./tests/scripts/testcases_cleanup.sh

vagrant_ubuntu20-calico-dual-stack:
stage: deploy-extended
Expand Down
1 change: 1 addition & 0 deletions Vagrantfile
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,7 @@ Vagrant.configure("2") do |config|
"local_path_provisioner_enabled": "#{$local_path_provisioner_enabled}",
"local_path_provisioner_claim_root": "#{$local_path_provisioner_claim_root}",
"ansible_ssh_user": SUPPORTED_OS[$os][:user],
"ansible_ssh_private_key_file": File.join(Dir.home, ".vagrant.d", "insecure_private_key"),
"unsafe_show_logs": "True"
}

Expand Down
5 changes: 4 additions & 1 deletion pipeline.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ FROM ubuntu:jammy-20230308
# Pip needs this as well at the moment to install ansible
# (and potentially other packages)
# See: https://github.com/pypa/pip/issues/10219
ENV VAGRANT_VERSION=2.3.7 \
ENV VAGRANT_VERSION=2.4.1 \
VAGRANT_DEFAULT_PROVIDER=libvirt \
VAGRANT_ANSIBLE_TAGS=facts \
LANG=C.UTF-8 \
Expand All @@ -31,6 +31,8 @@ RUN apt update -q \
unzip \
libvirt-clients \
qemu-utils \
qemu-kvm \
dnsmasq \
&& curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
&& add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \
&& apt update -q \
Expand All @@ -46,6 +48,7 @@ ADD ./roles/kubespray-defaults/defaults/main/main.yml /kubespray/roles/kubespray
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
&& pip install --no-compile --no-cache-dir pip -U \
&& pip install --no-compile --no-cache-dir -r tests/requirements.txt \
&& pip install --no-compile --no-cache-dir -r requirements.txt \
&& KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main/main.yml) \
&& curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
&& echo $(curl -L https://dl.k8s.io/release/$KUBE_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
Expand Down
3 changes: 1 addition & 2 deletions tests/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,7 @@ cleanup-packet:

create-vagrant:
vagrant up
find / -name vagrant_ansible_inventory
cp /builds/kargo-ci/kubernetes-sigs-kubespray/inventory/sample/vagrant_ansible_inventory $(INVENTORY)
cp $(CI_PROJECT_DIR)/inventory/sample/vagrant_ansible_inventory $(INVENTORY)

delete-vagrant:
vagrant destroy -f
1 change: 0 additions & 1 deletion tests/files/vagrant_ubuntu20-flannel.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

# For CI we are not worries about data persistence across reboot
$libvirt_volume_cache = "unsafe"

# Checking for box update can trigger API rate limiting
# https://www.vagrantup.com/docs/vagrant-cloud/request-limits.html
$box_check_update = false
Expand Down
1 change: 1 addition & 0 deletions tests/files/vagrant_ubuntu20-flannel.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
---
# Kubespray settings
kube_network_plugin: flannel
ansible_ssh_private_key: .vagrant.d/insecure_private_key
122 changes: 122 additions & 0 deletions tests/scripts/vagrant-install.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
#!/bin/bash

# install_vagrant() {
# sudo apt install vagrant-libvirt vagrant -y
# sudo vagrant plugin install vagrant-libvirt
# }

# prep(){
# sudo apt-get update -y
# sudo apt-get install ca-certificates curl libvirt-daemon-system\
# libvirt-clients qemu-utils qemu-kvm htop atop -y

# sudo install -m 0755 -d /etc/apt/keyrings
# }
# install_docker() {
# VERSION_STRING=5:26.1.0-1~ubuntu.24.04~noble
# sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
# sudo chmod a+r /etc/apt/keyrings/docker.asc

# # Add the repository to Apt sources:
# echo \
# "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
# $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
# sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
# sudo apt-get update -y

# sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y
# }
# install_docker_auto () {
# curl -fsSL https://get.docker.com -o get-docker.sh
# sudo sh ./get-docker.sh --dry-run
# }



VAGRANT_VERSION=2.4.1
VAGRANT_DEFAULT_PROVIDER=libvirt
VAGRANT_ANSIBLE_TAGS=facts
LANG=C.UTF-8
DEBIAN_FRONTEND=noninteractive
PYTHONDONTWRITEBYTECODE=1
KUBE_VERSION=1.29.5
pipeline_install() {
cp /etc/apt/sources.list /etc/apt/sources.list."$(date +"%F")"
sed -i -e '/^# deb-src.*universe$/s/# //g' /etc/apt/sources.list
sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources

apt update
# libssl-dev \
# python3-dev \
# # jq \
# moreutils \
# libvirt-dev \
# # rsync \
# git \
# # htop \
# gpg \
# atop

# gnupg2 \
# software-properties-common
#
apt install --no-install-recommends -y \
git \
make \
python3-pip \
sshpass \
apt-transport-https \
openssh-client \
ca-certificates \
curl \
libfuse2 \
unzip \
qemu-utils \
libvirt-daemon-system \
libvirt-clients \
qemu-kvm \
ebtables libguestfs-tools \
ruby-fog-libvirt \
libvirt-dev \
gcc \
build-essential \
ruby-libvirt \
libxslt-dev libxml2-dev zlib1g-dev \
python3-venv python3-full \
dnsmasq

apt-get build-dep -y ruby-libvirt ruby-dev
### VAGRANT ###
# apt-get install -y unzip
curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_linux_amd64.zip
unzip vagrant_${VAGRANT_VERSION}_linux_amd64.zip
mv vagrant /usr/local/bin/vagrant
chmod a+x /usr/local/bin/vagrant
# ls -la /usr/local/bin/vagrant
/usr/local/bin/vagrant plugin install vagrant-libvirt
usermod -aG kvm kubespray
usermod -aG libvirt kubespray

### DOCKER ###
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
add-apt-repository -y "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
apt update
apt install --no-install-recommends -y docker-ce
apt autoremove -y --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/*

### KUBECTL ###
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
mv kubectl /usr/local/bin/kubectl
chmod a+x /usr/local/bin/kubectl
systemctl restart libvirtd
# Install Vagrant
# apt update -y
# echo apt-get install -y unzip libfuse2 vagrant vagrant-libvirt
# apt --fix-broken install -y
# dpkg --configure -a -y


}
# wrapped up in a function so that we have some protection against only getting
# half the file during "curl | sh"
pipeline_install

0 comments on commit f3239e7

Please sign in to comment.