Test internet connectivity of the machines

.github/workflows/integration.yaml

@@ -33,7 +33,7 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         # we can remove desired version again after this fix was released: https://github.com/srl-labs/containerlab/pull/2000
-        DESIRED_VERSION: v0.52.0
+        DESIRED_VERSION: v0.56.0
 
     - name: Log in to the container registry
       uses: docker/login-action@v3

Makefile

@@ -19,14 +19,17 @@ MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:
 MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest)
 
 MACHINE_OS=ubuntu-24.04
+MAX_RETRIES := 30
 
 # Machine flavors
 ifeq ($(MINI_LAB_FLAVOR),cumulus)
 LAB_MACHINES=machine01,machine02
 LAB_TOPOLOGY=mini-lab.cumulus.yaml
+VRF=vrf20
 else ifeq ($(MINI_LAB_FLAVOR),sonic)
 LAB_MACHINES=machine01,machine02
 LAB_TOPOLOGY=mini-lab.sonic.yaml
+VRF=Vrf20
 else
 $(error Unknown flavor $(MINI_LAB_FLAVOR))
 endif
@@ -78,7 +81,7 @@ partition: partition-bake
 	docker compose up --remove-orphans --force-recreate partition
 
 .PHONY: partition-bake
-partition-bake:
+partition-bake: external_network
 	docker pull $(MINI_LAB_VM_IMAGE)
 ifeq ($(MINI_LAB_FLAVOR),sonic)
 	docker pull $(MINI_LAB_SONIC_IMAGE)
@@ -87,20 +90,22 @@ endif
 		sudo --preserve-env $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure && \
 		./scripts/deactivate_offloading.sh; fi
 
+.PHONY: external_network
+external_network:
+	@if ! docker network ls | grep -q mini_lab_ext; then \
+  		docker network create mini_lab_ext \
+			--driver=bridge \
+			--gateway=203.0.113.1 \
+			--subnet=203.0.113.0/24 \
+			--opt "com.docker.network.driver.mtu=9000" \
+			--opt "com.docker.network.bridge.name=mini_lab_ext" \
+			--opt "com.docker.network.bridge.enable_ip_masquerade=true" && \
+		sudo ip route add 203.0.113.128/25 via 203.0.113.2 dev mini_lab_ext; fi
+
 .PHONY: env
 env:
 	@./env.sh
 
-.PHONY: _ips
-_ips:
-	$(eval ipL1 = $(shell ${YQ} --unwrapScalar=true '.nodes.leaf01."mgmt-ipv4-address"' clab-mini-lab/topology-data.json))
-	$(eval ipL2 = $(shell ${YQ} --unwrapScalar=true '.nodes.leaf02."mgmt-ipv4-address"' clab-mini-lab/topology-data.json))
-	$(eval staticR = "100.255.254.0/24 nexthop via $(ipL1) dev docker0 nexthop via $(ipL2) dev docker0")
-
-.PHONY: route
-route: _ips
-	eval "sudo ip r a ${staticR}"
-
 .PHONY: cleanup
 cleanup: cleanup-control-plane cleanup-partition
 
@@ -113,20 +118,26 @@ cleanup-control-plane:
 .PHONY: cleanup-partition
 cleanup-partition:
 	mkdir -p clab-mini-lab
-	sudo $(CONTAINERLAB) destroy --topo mini-lab.cumulus.yaml
-	sudo $(CONTAINERLAB) destroy --topo mini-lab.sonic.yaml
+	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.cumulus.yaml
+	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.sonic.yaml
+	docker network rm --force mini_lab_ext
 
 .PHONY: _privatenet
 _privatenet: env
-	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network | grep user-private-network || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network allocate --partition mini-lab --project 00000000-0000-0000-0000-000000000000 --name user-private-network
+	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network | grep user-private-network || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network allocate --partition mini-lab --project 00000000-0000-0000-0000-000000000001 --name user-private-network
+
+.PHONY: _public_ips
+_public_ips: env
+	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip list --name firewall | grep firewall || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip create --network internet-mini-lab --project 00000000-0000-0000-0000-000000000001 --ipaddress 203.0.113.129 --name firewall
+	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip list --name machine | grep machine || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip create --network internet-mini-lab --project 00000000-0000-0000-0000-000000000001 --ipaddress 203.0.113.130 --name machine
 
 .PHONY: machine
-machine: _privatenet
-	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --networks $(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
+machine: _privatenet _public_ips
+	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000001 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --userdata "@/tmp/ignition.json" --ips 203.0.113.130 --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
 
 .PHONY: firewall
-firewall: _ips _privatenet
-	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
+firewall: _privatenet _public_ips
+	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000001 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --userdata "@/tmp/ignition.json" --ips 203.0.113.129 --firewall-rules-file=/tmp/rules.yaml --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
 
 .PHONY: ls
 ls: env
@@ -212,6 +223,41 @@ console-machine02:
 console-machine03:
 	@$(MAKE) --no-print-directory _console-machine	CONSOLE_PORT=4002
 
+## SSH TARGETS FOR MACHINES ##
+# Python code could be replaced by jq, but it is not preinstalled on Cumulus
+.PHONY: ssh-firewall
+ssh-firewall:
+	$(eval fw = $(shell ssh -F files/ssh/config leaf01 "vtysh -c 'show bgp neighbors fw json' | \
+		python3 -c 'import sys, json; data = json.load(sys.stdin); key = next(iter(data)); print(data[key][\"bgpNeighborAddr\"] + \"%\" + key)'" \
+	))
+	ssh -F files/ssh/config $(fw) $(COMMAND)
+
+.PHONY: ssh-machine
+ssh-machine:
+	$(eval machine = $(shell ssh -F files/ssh/config leaf01 "vtysh -c 'show bgp vrf $(VRF) neighbors test json' | \
+		python3 -c 'import sys, json; data = json.load(sys.stdin); key = next(iter(data)); print(data[key][\"bgpNeighborAddr\"] + \"%\" + key)'" \
+	))
+	ssh -F files/ssh/config $(machine) $(COMMAND)
+
+.PHONY: connect-to-cloudflare
+connect-to-cloudflare:
+	@echo "Attempting to connect to Cloudflare..."
+	@for i in $$(seq 1 $(MAX_RETRIES)); do \
+		if $(MAKE) ssh-machine COMMAND="sudo curl --connect-timeout 1 --fail --silent https://1.1.1.1" > /dev/null 2>&1; then \
+			echo "Connected successfully"; \
+			exit 0; \
+		else \
+			echo "Connection failed"; \
+			if [ $$i -lt $(MAX_RETRIES) ]; then \
+				echo "Retrying in 2 seconds..."; \
+				sleep 2; \
+			else \
+				echo "Max retries reached"; \
+				exit 1; \
+			fi; \
+		fi; \
+	done
+
 ## DEV TARGETS ##
 
 .PHONY: dev-env

README.md

@@ -23,8 +23,8 @@ The mini-lab is a small, virtual setup to locally run the metal-stack. It deploy
 - kvm as hypervisor for the VMs (you can check through the `kvm-ok` command)
 - [docker](https://www.docker.com/) >= 24.x.y (for using kind and our deployment base image)
 - [kind](https://github.com/kubernetes-sigs/kind/releases) == v0.23.0 (for hosting the metal control plane)
-- [containerlab](https://containerlab.dev/install/) >= v0.55.0
-- the lab creates a docker network on your host machine (`172.17.0.1`), this hopefully does not overlap with other networks you have
+- [containerlab](https://containerlab.dev/install/) >= v0.56.0
+- the lab creates a docker network on your host machine with the address block `203.0.113.0/24`, designated as TEST-NET-3 for documentation and examples.
 - (recommended) haveged to have enough random entropy (only needed if the PXE process does not work)
 
 Here is some code that should help you to set up most of the requirements:
@@ -167,16 +167,6 @@ Login with user name metal and the console password from
 docker compose run --rm metalctl machine consolepassword e0ab02d2-27cd-5a5e-8efc-080ba80cf258
 ```
 
-If you want to access the firewall with SSH or have internet connectivity from the firewall and machine, you'll need to have a static route configured that points to the leaf switches:
-
-```bash
-# Add the route to the network internet-mini-lab 100.255.254.0/24 via leaf01 and leaf02, whose IPs are dynamically allocated. Make sure there's no old route before execution.
-make route
-
-# Connect to the firewall
-ssh metal@100.255.254.1
-```
-
 To remove the kind cluster, the switches and machines, run:
 
 ```bash

compose.yaml

@@ -79,6 +79,8 @@ services:
       - METALCTL_API_URL=http://api.172.17.0.1.nip.io:8080/metal
     volumes:
       - ./files/ssh:/root/.ssh:ro
+      - ./files/ignition.json:/tmp/ignition.json
+      - ./files/rules.yaml:/tmp/rules.yaml
     network_mode: host
     dns:
       - 172.17.0.1

files/ignition.json

@@ -0,0 +1,25 @@
+{
+  "ignition": {
+    "config": {},
+    "security": {},
+    "timeouts": {},
+    "version": "2.3.0"
+  },
+  "networkd": {},
+  "passwd": {},
+  "storage": {
+    "files": [
+      {
+        "filesystem": "root",
+        "path": "/etc/hosts.allow",
+        "append": true,
+        "contents": {
+          "source": "data:,ALL%3A%20%5Bfe80%3A%3A%5D%2F10%0D%0AALL%3A%20203.0.113.1%0D%0A",
+          "verification": {}
+        },
+        "mode": 644
+      }
+    ]
+  },
+  "systemd": {}
+}

files/inet/frr.conf

@@ -5,17 +5,20 @@ log syslog informational
 !
 vrf vrfInternet
  vni 104009
- ip route 0.0.0.0/0 172.17.0.1
+ ip route 0.0.0.0/0 203.0.113.1
 exit-vrf
 !
-interface eth2
+interface eth1
  ipv6 nd ra-interval 6
  no ipv6 nd suppress-ra
 !
-interface eth3
+interface eth2
  ipv6 nd ra-interval 6
  no ipv6 nd suppress-ra
 !
+interface ext
+ ip address 203.0.113.2/24
+!
 interface lo
  ip address 10.0.0.21/32
 !

files/inet/network.sh

@@ -3,7 +3,7 @@ set -o errexit -o xtrace
 
 ip link add vrfInternet type vrf table 1000
 ip link set dev vrfInternet up
-ip link set dev eth0 master vrfInternet
+ip link set dev ext master vrfInternet
 
 ip link add name bridge type bridge stp_state 0
 ip link set dev bridge type bridge vlan_filtering 1
@@ -18,12 +18,10 @@ bridge vlan add vid 1000 dev bridge self
 ip link set dev vlanInternet up
 
 ip link add vniInternet type vxlan id 104009 dstport 4789 local 10.0.0.21 nolearning
-ip link set dev vlanInternet mtu 9000
+ip link set dev vniInternet mtu 9000
 ip link set dev vniInternet master bridge
 bridge vlan del vid 1 dev vniInternet
 bridge vlan del vid 1 untagged pvid dev vniInternet
 bridge vlan add vid 1000 dev vniInternet
 bridge vlan add vid 1000 untagged pvid dev vniInternet
 ip link set up dev vniInternet
-
-iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE

files/rules.yaml

@@ -0,0 +1,30 @@
+---
+egress:
+  - comment: allow outgoing https
+    ports:
+      - 443
+    protocol: TCP
+    to:
+      - 0.0.0.0/0
+  - comment: allow outgoing dns via tcp
+    ports:
+      - 53
+    protocol: TCP
+    to:
+      - 0.0.0.0/0
+  - comment: allow outgoing dns and ntp via udp
+    ports:
+      - 53
+      - 123
+    protocol: UDP
+    to:
+      - 0.0.0.0/0
+ingress:
+  - comment: allow incoming ssh
+    ports:
+      - 22
+    protocol: TCP
+    from:
+      - 203.0.113.1/25
+    to:
+      - 203.0.113.128/25

files/ssh/config

@@ -0,0 +1,17 @@
+Host leaf01
+    HostName leaf01
+    User root
+    IdentityFile files/ssh/id_rsa
+    PasswordAuthentication no
+    StrictHostKeyChecking no
+    UserKnownHostsFile /dev/null
+    PubkeyAcceptedKeyTypes +ssh-rsa
+
+Host * !leaf01
+    User metal
+    IdentityFile files/ssh/id_rsa
+    PasswordAuthentication no
+    StrictHostKeyChecking no
+    UserKnownHostsFile /dev/null
+    # bash could be replaced with ncat, but it is not preinstalled on Cumulus
+    ProxyCommand ssh -q -F files/ssh/config leaf01 'sudo ip vrf exec default bash -c "exec 3<>/dev/tcp/%h/%p; cat<&0 >&3 & cat<&3 >&1"'

inventories/group_vars/control-plane/metal.yml

@@ -74,7 +74,7 @@ metal_api_networks:
   partitionid: "mini-lab"
   vrf: 104009
   prefixes:
-  - 100.255.254.0/24
+  - 203.0.113.128/25
   labels:
     network.metal-stack.io/default: ""
     network.metal-stack.io/default-external: ""

inventories/group_vars/cumulus/main.yaml

@@ -3,7 +3,8 @@ ports:
   1: 100G
 interfaces:
 - name: swp1
-uplinks: []
+uplinks:
+- name: swp31
 
 # The best practice recommendation is to set an MTU of 9,216 for the inter-switch links,
 # and an MTU of 9,000 for the server-facing ports, which don’t carry the VXLAN header.

mini-lab.cumulus.yaml

@@ -14,10 +14,10 @@ topology:
         ansible-group: cumulus
       binds:
         - files/ssh/id_rsa.pub:/root/.ssh/authorized_keys
-    linux:
-      image: ${MINI_LAB_VM_IMAGE}
 
   nodes:
+    mini_lab_ext:
+      kind: bridge
     leaf01:
       kind: cvx
       binds:
@@ -26,16 +26,31 @@ topology:
       kind: cvx
       binds:
         - apt-transport-https.tar.gz:/root/jessie-apt-transport-fix.tar.gz
+    inet:
+      kind: linux
+      image: quay.io/frrouting/frr:10.0.1
+      binds:
+        - files/inet/daemons:/etc/frr/daemons
+        - files/inet/frr.conf:/etc/frr/frr.conf
+        - files/inet/vtysh.conf:/etc/frr/vtysh.conf
+        - files/inet/network.sh:/root/network.sh
+      exec:
+        - sh /root/network.sh
     vms:
       kind: linux
+      image: ${MINI_LAB_VM_IMAGE}
       binds:
         - /dev:/dev
         - scripts:/mini-lab
 
   links:
+    - endpoints: ["inet:ext", "mini_lab_ext:inet"]
+      mtu: 9000
     - endpoints: ["leaf01:swp1", "vms:lan0"]
     - endpoints: ["leaf02:swp1", "vms:lan1"]
     - endpoints: ["leaf01:swp2", "vms:lan2"]
     - endpoints: ["leaf02:swp2", "vms:lan3"]
     - endpoints: ["leaf01:swp3", "vms:lan4"]
     - endpoints: ["leaf02:swp3", "vms:lan5"]
+    - endpoints: ["leaf01:swp31", "inet:eth1"]
+    - endpoints: ["leaf02:swp31", "inet:eth2"]

mini-lab.sonic.yaml

@@ -6,6 +6,8 @@ mgmt:
 
 topology:
   nodes:
+    mini_lab_ext:
+      kind: bridge
     leaf01:
       kind: linux
       image: ${MINI_LAB_SONIC_IMAGE}
@@ -39,6 +41,8 @@ topology:
         - /dev:/dev
         - scripts:/mini-lab
   links:
+    - endpoints: ["inet:ext", "mini_lab_ext:inet"]
+      mtu: 9000
     - endpoints: ["leaf01:eth1", "inet:eth1"]
     - endpoints: ["leaf02:eth1", "inet:eth2"]
     - endpoints: ["leaf01:eth2", "vms:lan0"]