Merge pull request #217 from ictsc/delete-router

L2 Announcementを導入

ansible/inventory/group_vars/control_plane.yaml

@@ -5,7 +5,7 @@ netplan_configuration:
     ethernets:
       eth1:
         addresses:
-          - 192.168.100.2{{groups["control_plane"].index(inventory_hostname)}}/24
+          - 192.168.100.{{ groups["control_plane"].index(inventory_hostname) + 1 }}/24
 
 # k8s_joinロール用
 k8s_join_flag: --control-plane --certificate-key {{ hostvars[groups['control_plane'][0]]['certs'] }}

ansible/inventory/inventory_handler.py

@@ -40,7 +40,6 @@ def main():
     inventory = {
         "control_plane": {"hosts": []},
         "worker_node": {"hosts": []},
-        "router": {"hosts": []},
         "_meta": {"hostvars": {}},
     }
     workspace = get_workspace()
@@ -50,15 +49,12 @@ def main():
     #
     # print(tfstate["outputs"])
 
-    router = 0
     control_plane = 0
     worker_node = 0
     inventory_gp = {}
 
     for output_key in tfstate["outputs"]:
         match output_key:
-            case "k8s_router_ip_address":
-                inventory_gp = inventory["router"]
             case "k8s_control_plane_ip_address":
                 inventory_gp = inventory["control_plane"]
             case "k8s_worker_node_ip_address":
@@ -69,48 +65,46 @@ def main():
         for ip_address in tfstate["outputs"][output_key]["value"]:
             # not handle private ip address
             if (
-                ip_address[:11] == "192.168.100"
-                and output_key != "k8s_worker_node_ip_address"
+                ip_address[:7] == "192.168"
+                and output_key == "k8s_control_plane_ip_address"
             ):
                 continue
 
             inventory_gp["hosts"].append(ip_address)
 
             match output_key:
-                case "k8s_router_ip_address":
-                    inventory["_meta"]["hostvars"] = inventory["_meta"]["hostvars"] | {
-                        ip_address: {"internal_ip": f"192.168.100.1{str(router)}"}
-                    }
-                    router += 1
                 case "k8s_control_plane_ip_address":
                     inventory["_meta"]["hostvars"] = inventory["_meta"]["hostvars"] | {
                         ip_address: {
-                            "internal_ip": f"192.168.100.2{str(control_plane)}"
+                            "internal_ip": f"192.168.100.{str(control_plane + 1)}"
                         }
                     }
                     control_plane += 1
                 case "k8s_worker_node_ip_address":
                     inventory["_meta"]["hostvars"] = inventory["_meta"]["hostvars"] | {
-                        ip_address: {"internal_ip": f"192.168.100.3{str(worker_node)}"}
+                        ip_address: {
+                            "internal_ip": f"192.168.100.3{str(worker_node + 101)}"
+                        }
                     }
                     worker_node += 1
 
-    inventory["router"]["vars"] = {
-        "bgp_address": tfstate["outputs"]["external_address_range"]["value"]
-    }
     inventory["control_plane"]["vars"] = {
         "VIP": tfstate["outputs"]["vip_address"]["value"]
     }
     inventory["worker_node"]["vars"] = {
         "ansible_ssh_common_args": (
             "-o ProxyCommand='ssh -o ControlMaster=auto -o ControlPersist=60s "
             "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "
-            f"-i ../id_rsa -W %h:%p ubuntu@{inventory['router']['hosts'][0]}'"
+            f"-i ../id_rsa -W %h:%p ubuntu@{inventory['control_plane']['hosts'][0]}'"
         )
     }
     inventory["delegate_plane"] = {
         "hosts": [inventory["control_plane"]["hosts"][0]],
-        "vars": {"workspace": workspace},
+        "vars": {
+            "workspace": workspace,
+            "min_ip_address": tfstate["outputs"]["min_ip_address"]["value"],
+            "max_ip_address": tfstate["outputs"]["max_ip_address"]["value"],
+        },
     }
 
     print(json.dumps(inventory))

ansible/roles/components/tasks/main.yaml

@@ -35,7 +35,8 @@
     - name: Deploy Cilium
       ansible.builtin.shell: |
         helm upgrade --install cilium cilium/cilium --namespace kube-system \
-          --set kubeProxyReplacement=true --set k8sServiceHost={{ VIP }} --set k8sServicePort=8443 --set bgpControlPlane.enabled=true
+          --set kubeProxyReplacement=true --set k8sServiceHost={{ VIP }} --set k8sServicePort=8443 \
+          --set l2announcements.enabled=true --set k8sClientRateLimit.qps=40 --set k8sClientRateLimit.burst=50
       changed_when: false
     - name: Restart pods
       ansible.builtin.shell:
@@ -47,15 +48,15 @@
       changed_when: false
       failed_when: false
 
-- name: Setup BGP
+- name: Setup L2 announcement
   block:
-    - name: Send bgp-config
+    - name: Send announcement config
       ansible.builtin.template:
-        src: bgp-config.yaml.j2
-        dest: /tmp/bgp-config.yaml
+        src: announcement-config.yaml.j2
+        dest: /tmp/announcement-config.yaml
         mode: "0644"
-    - name: Apply bgp-config
-      ansible.builtin.command: kubectl apply -f /tmp/bgp-config.yaml
+    - name: Apply announcement config
+      ansible.builtin.command: kubectl apply -f /tmp/announcement-config.yaml
       register: ret
       until: ret.rc == 0
       retries: 12

ansible/roles/components/templates/announcement-config.yaml.j2

@@ -0,0 +1,22 @@
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: cilium-announcement
+spec:
+  nodeSelector:
+    matchExpressions:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+  interfaces:
+    - ^eth[0-9]+
+  loadBalancerIPs: true
+---
+apiVersion: cilium.io/v2alpha1
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: default-pool
+  namespace: kube-system
+spec:
+  blocks:
+    - start: {{ min_ip_address }}
+      stop: {{ max_ip_address }}