From e64f9ee740441c70af2db365a986474649f1fa8d Mon Sep 17 00:00:00 2001
From: logica0419 <84759675+logica0419@users.noreply.github.com>
Date: Tue, 22 Oct 2024 03:05:13 +0900
Subject: [PATCH 01/12] delete bird task

---
 ansible/roles/bird/handlers/main.yaml     |   8 -
 ansible/roles/bird/tasks/main.yaml        |  47 ------
 ansible/roles/bird/templates/bird.conf.j2 | 170 ----------------------
 ansible/roles/bird/vars/main.yaml         |   4 -
 4 files changed, 229 deletions(-)
 delete mode 100644 ansible/roles/bird/handlers/main.yaml
 delete mode 100644 ansible/roles/bird/tasks/main.yaml
 delete mode 100644 ansible/roles/bird/templates/bird.conf.j2
 delete mode 100644 ansible/roles/bird/vars/main.yaml

diff --git a/ansible/roles/bird/handlers/main.yaml b/ansible/roles/bird/handlers/main.yaml
deleted file mode 100644
index 92a5d7ff..00000000
--- a/ansible/roles/bird/handlers/main.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-- name: Reconfigure BIRD
-  ansible.builtin.shell:
-    cmd: |
-      set -eu
-      /usr/sbin/bird -p -c /etc/bird/bird.conf
-      /usr/sbin/birdc configure
-    executable: /bin/bash
-  changed_when: true
diff --git a/ansible/roles/bird/tasks/main.yaml b/ansible/roles/bird/tasks/main.yaml
deleted file mode 100644
index 841880fc..00000000
--- a/ansible/roles/bird/tasks/main.yaml
+++ /dev/null
@@ -1,47 +0,0 @@
-- name: Set kernel params
-  ansible.posix.sysctl:
-    name: "{{ item.key }}"
-    value: "{{ item.value }}"
-    sysctl_set: true
-    state: present
-    reload: true
-    ignoreerrors: true
-  with_dict: "{{ bird_sysctl_config }}"
-
-- name: Install BIRD
-  ansible.builtin.apt:
-    name: "{{ packages }}"
-    state: present
-  vars:
-    packages:
-      - bird2
-
-- name: Configure BIRD
-  block:
-    - name: Send BIRD configuration file
-      ansible.builtin.template:
-        src: "{{ item.src }}"
-        dest: "{{ item.dest }}"
-        mode: "0644"
-      with_items:
-        - src: bird.conf.j2
-          dest: /tmp/bird.conf
-      register: bird_pre_config_updated
-    - name: Validate pre BIRD configuration file syntax
-      ansible.builtin.shell:
-        cmd: |
-          set -eu
-          /usr/sbin/bird -p -c /tmp/bird.conf
-        executable: /bin/bash
-      changed_when: false
-    - name: Copy BIRD configuration file to the running BIRD system
-      ansible.builtin.copy:
-        src: "{{ item.src }}"
-        dest: "{{ item.dest }}"
-        remote_src: true
-        owner: root
-        mode: "0644"
-      with_items:
-        - src: /tmp/bird.conf
-          dest: /etc/bird/bird.conf
-      notify: Reconfigure BIRD
diff --git a/ansible/roles/bird/templates/bird.conf.j2 b/ansible/roles/bird/templates/bird.conf.j2
deleted file mode 100644
index 493965b6..00000000
--- a/ansible/roles/bird/templates/bird.conf.j2
+++ /dev/null
@@ -1,170 +0,0 @@
-####
-# Basic configuration
-####
-
-router id {{ groups['router'][0] }};
-
-log syslog { info, remote, warning, error, auth, fatal, bug };
-log stderr all;
-debug protocols {events, states, interfaces};
-graceful restart wait 15;
-
-####
-# Definition
-####
-
-define ROUTER_ASN = 65020;
-define WORKER_ASN = 65021;
-
-
-####
-# Routing Table Objects
-####
-
-ipv4 table kernel_table_v4; # kernel table (default)
-ipv4 table egp_table_v4; # BGP table
-
-
-####
-# Kernel Connection
-####
-
-# Device definition
-protocol device device1 {
-    interface "-ethmgmt", "*";
-
-    scan time 100;
-}
-
-# Import routing table: Direct connected
-protocol direct direct1 {
-    ipv4 {
-        table kernel_table_v4;
-    };
-
-    interface "-ethmgmt", "*";
-}
-
-# Connect routing table; Kernel (IPv4)
-protocol kernel kernel_v4 {
-    ipv4 {
-        table kernel_table_v4;
-        preference 10;
-
-        import filter {
-            reject;
-        };
-
-        export filter {
-            accept;
-        };
-    };
-
-    learn off;
-    scan time 10;
-    graceful restart on;
-    kernel table 254;
-}
-
-
-####
-# Define routing table: Static
-####
-
-# Define routing table: Static (IPv4)
-protocol static static_all_v4 {
-    ipv4 {
-        table kernel_table_v4;
-        preference 200;
-
-        import filter {
-            # kernel_table_v4 <- Static Route
-            accept;
-        };
-
-        export filter {
-            # Read Only
-            reject;
-        };
-    };
-}
-
-
-####
-# Generate BGP Route
-####
-
-# Define routing table: BGP
-# Generate Route(IPv4) (BGP Advertise Only)
-protocol static static_bgp_v4 {
-    ipv4 {
-        table egp_table_v4;
-        preference 400;
-
-        import filter {
-            accept;
-        };
-
-        export filter {
-            # Read Only
-            reject;
-        };
-    };
-
-    route {{ hostvars[groups['router'][0]]['bgp_address'] }} blackhole;
-}
-
-
-####
-# eBGP Configuration
-####
-
-# eBGP Peer
-{% for item in groups['worker_node'] %}
-protocol bgp node_peer_{{ item | replace('.', '_') }} {
-    local {{ groups['router'][0] }} as ROUTER_ASN;
-    neighbor {{ item }} as WORKER_ASN;
-
-    hold time 30;
-    startup hold time 30;
-    keepalive time 10;
-    connect retry time 5;
-    connect delay time 5;
-    error wait time 30, 30;
-    error forget time 30;
-    default bgp_local_pref 100;
-
-    ipv4 {
-        table egp_table_v4;
-        preference 100;
-
-        import filter {
-            accept;
-        };
-
-        export filter {
-            accept;
-        };
-    };
-}
-{% endfor %}
-
-####
-# Table Connection
-####
-
-# Kernel (kernel_table_v4) <-> BGP (egp_table_v4)
-protocol pipe pipe_bgp_to_kernel_table_v4 {
-    table egp_table_v4;
-    peer table kernel_table_v4;
-
-    # Kernel -> BGP
-    import filter {
-        reject;
-    };
-
-    # BGP -> Kernel
-    export filter {
-        accept;
-    };
-}
diff --git a/ansible/roles/bird/vars/main.yaml b/ansible/roles/bird/vars/main.yaml
deleted file mode 100644
index ee276c86..00000000
--- a/ansible/roles/bird/vars/main.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-bird_sysctl_config:
-  net.ipv4.ip_forward: 1
-  net.ipv4.conf.all.forwarding: 1
-  net.ipv6.conf.all.forwarding: 1

From 59d5ab08730f746c8855b35581cff24d6d7b9b69 Mon Sep 17 00:00:00 2001
From: logica0419 <84759675+logica0419@users.noreply.github.com>
Date: Tue, 22 Oct 2024 03:05:37 +0900
Subject: [PATCH 02/12] fix cilium deploy args

---
 ansible/roles/components/tasks/main.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ansible/roles/components/tasks/main.yaml b/ansible/roles/components/tasks/main.yaml
index d24a7dc6..f00a2b04 100644
--- a/ansible/roles/components/tasks/main.yaml
+++ b/ansible/roles/components/tasks/main.yaml
@@ -35,7 +35,8 @@
     - name: Deploy Cilium
       ansible.builtin.shell: |
         helm upgrade --install cilium cilium/cilium --namespace kube-system \
-          --set kubeProxyReplacement=true --set k8sServiceHost={{ VIP }} --set k8sServicePort=8443 --set bgpControlPlane.enabled=true
+          --set kubeProxyReplacement=true --set k8sServiceHost={{ VIP }} --set k8sServicePort=8443 \
+          --set l2announcements.enabled=true --set k8sClientRateLimit.qps=40 --set k8sClientRateLimit.burst=50
       changed_when: false
     - name: Restart pods
       ansible.builtin.shell:

From e09e34afda7fbbe2ff5f45c7a08d8d8fe96f3dd8 Mon Sep 17 00:00:00 2001
From: logica0419 <84759675+logica0419@users.noreply.github.com>
Date: Tue, 22 Oct 2024 03:08:55 +0900
Subject: [PATCH 03/12] delete router resource

---
 terraform/output.tf        |  6 +-----
 terraform/router-disk.tf   | 11 -----------
 terraform/router-server.tf | 29 -----------------------------
 terraform/vars.tf          | 29 -----------------------------
 4 files changed, 1 insertion(+), 74 deletions(-)
 delete mode 100644 terraform/router-disk.tf
 delete mode 100644 terraform/router-server.tf

diff --git a/terraform/output.tf b/terraform/output.tf
index 2c343dc3..9a49721e 100644
--- a/terraform/output.tf
+++ b/terraform/output.tf
@@ -1,15 +1,11 @@
 output "vip_address" {
-  value = sakuracloud_internet.k8s_external_switch.ip_addresses[lookup(var.router, terraform.workspace, 0) + lookup(var.control_plane, terraform.workspace, 0)]
+  value = sakuracloud_internet.k8s_external_switch.ip_addresses[lookup(var.control_plane, terraform.workspace, 0)]
 }
 
 output "external_address_range" {
   value = format("%s/%s", sakuracloud_subnet.bgp_subnet.ip_addresses[0], 28)
 }
 
-output "k8s_router_ip_address" {
-  value = sakuracloud_server.k8s_router[*].ip_address
-}
-
 output "k8s_control_plane_ip_address" {
   value = sakuracloud_server.k8s_control_plane[*].ip_address
 }
diff --git a/terraform/router-disk.tf b/terraform/router-disk.tf
deleted file mode 100644
index 8bf5f57e..00000000
--- a/terraform/router-disk.tf
+++ /dev/null
@@ -1,11 +0,0 @@
-resource "sakuracloud_disk" "k8s_router_disk" {
-  count             = lookup(var.router, terraform.workspace, 0)
-  name              = "k8s-${terraform.workspace}-router-${count.index + 1}"
-  source_archive_id = data.sakuracloud_archive.ubuntu_archive.id
-  size              = lookup(var.router_disk, terraform.workspace, 0)
-  tags              = ["k8s", terraform.workspace]
-  timeouts {
-    create = "1h"
-    delete = "1h"
-  }
-}
diff --git a/terraform/router-server.tf b/terraform/router-server.tf
deleted file mode 100644
index b52d0b1d..00000000
--- a/terraform/router-server.tf
+++ /dev/null
@@ -1,29 +0,0 @@
-resource "sakuracloud_server" "k8s_router" {
-  count  = lookup(var.router, terraform.workspace, 0)
-  name   = "k8s-${terraform.workspace}-router-${count.index + 1}"
-  core   = lookup(var.router_cpu, terraform.workspace, 0)
-  memory = lookup(var.router_mem, terraform.workspace, 0)
-  disks  = [sakuracloud_disk.k8s_router_disk[count.index].id]
-  tags   = ["k8s", terraform.workspace]
-
-  network_interface {
-    upstream = sakuracloud_internet.k8s_external_switch.switch_id
-  }
-  network_interface {
-    upstream        = sakuracloud_switch.k8s_internal_switch.id
-    user_ip_address = "192.168.100.1${count.index}"
-  }
-  disk_edit_parameter {
-    hostname        = "k8s-${terraform.workspace}-router-${count.index + 1}"
-    password        = var.cluster_pass
-    disable_pw_auth = "true"
-    ssh_key_ids     = [sakuracloud_ssh_key_gen.gen_key.id]
-    ip_address      = sakuracloud_internet.k8s_external_switch.ip_addresses[count.index]
-    gateway         = sakuracloud_internet.k8s_external_switch.gateway
-    netmask         = lookup(var.external_subnet, terraform.workspace, 0)
-  }
-  timeouts {
-    create = "1h"
-    delete = "1h"
-  }
-}
diff --git a/terraform/vars.tf b/terraform/vars.tf
index cf17b928..7be01c06 100644
--- a/terraform/vars.tf
+++ b/terraform/vars.tf
@@ -75,32 +75,3 @@ variable "worker_node_rook_disk" {
     prd = 20
   }
 }
-
-variable "router" {
-  type = map(any)
-  default = {
-    dev = 1
-    prd = 1
-  }
-}
-variable "router_cpu" {
-  type = map(any)
-  default = {
-    dev = 1
-    prd = 1
-  }
-}
-variable "router_mem" {
-  type = map(any)
-  default = {
-    dev = 1
-    prd = 1
-  }
-}
-variable "router_disk" {
-  type = map(any)
-  default = {
-    dev = 20
-    prd = 20
-  }
-}

From 4787d44b6febbdf293737ef6b2d347cfcb96f8ad Mon Sep 17 00:00:00 2001
From: logica0419 <84759675+logica0419@users.noreply.github.com>
Date: Tue, 22 Oct 2024 03:11:41 +0900
Subject: [PATCH 04/12] delete subnet

---
 terraform/network-external.tf | 9 ---------
 terraform/output.tf           | 4 ----
 2 files changed, 13 deletions(-)

diff --git a/terraform/network-external.tf b/terraform/network-external.tf
index 217742ac..bb955fde 100644
--- a/terraform/network-external.tf
+++ b/terraform/network-external.tf
@@ -9,12 +9,3 @@ resource "sakuracloud_internet" "k8s_external_switch" {
   }
 }
 
-resource "sakuracloud_subnet" "bgp_subnet" {
-  internet_id = sakuracloud_internet.k8s_external_switch.id
-  next_hop    = sakuracloud_server.k8s_router[0].ip_address
-  netmask     = 28
-  timeouts {
-    create = "1h"
-    delete = "1h"
-  }
-}
diff --git a/terraform/output.tf b/terraform/output.tf
index 9a49721e..e4b732d1 100644
--- a/terraform/output.tf
+++ b/terraform/output.tf
@@ -2,10 +2,6 @@ output "vip_address" {
   value = sakuracloud_internet.k8s_external_switch.ip_addresses[lookup(var.control_plane, terraform.workspace, 0)]
 }
 
-output "external_address_range" {
-  value = format("%s/%s", sakuracloud_subnet.bgp_subnet.ip_addresses[0], 28)
-}
-
 output "k8s_control_plane_ip_address" {
   value = sakuracloud_server.k8s_control_plane[*].ip_address
 }

From 2dde23c59c204399ade8d01f01faa54c2b2f8391 Mon Sep 17 00:00:00 2001
From: logica0419 <84759675+logica0419@users.noreply.github.com>
Date: Tue, 22 Oct 2024 03:13:45 +0900
Subject: [PATCH 05/12] reconsider ip assign

---
 terraform/control-plane-server.tf | 4 ++--
 terraform/worker-node-server.tf   | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/terraform/control-plane-server.tf b/terraform/control-plane-server.tf
index 5ddf486f..a3d91339 100644
--- a/terraform/control-plane-server.tf
+++ b/terraform/control-plane-server.tf
@@ -11,14 +11,14 @@ resource "sakuracloud_server" "k8s_control_plane" {
   }
   network_interface {
     upstream        = sakuracloud_switch.k8s_internal_switch.id
-    user_ip_address = "192.168.100.2${count.index}"
+    user_ip_address = "192.168.100.${count.index + 1}"
   }
   disk_edit_parameter {
     hostname        = "k8s-${terraform.workspace}-control-plane-${count.index + 1}"
     password        = var.cluster_pass
     disable_pw_auth = "true"
     ssh_key_ids     = [sakuracloud_ssh_key_gen.gen_key.id]
-    ip_address      = sakuracloud_internet.k8s_external_switch.ip_addresses[count.index + lookup(var.router, terraform.workspace, 0)]
+    ip_address      = sakuracloud_internet.k8s_external_switch.ip_addresses[count.index]
     gateway         = sakuracloud_internet.k8s_external_switch.gateway
     netmask         = lookup(var.external_subnet, terraform.workspace, 0)
   }
diff --git a/terraform/worker-node-server.tf b/terraform/worker-node-server.tf
index 6a8c0d54..b54a39d3 100644
--- a/terraform/worker-node-server.tf
+++ b/terraform/worker-node-server.tf
@@ -8,14 +8,14 @@ resource "sakuracloud_server" "k8s_worker_node" {
 
   network_interface {
     upstream        = sakuracloud_switch.k8s_internal_switch.id
-    user_ip_address = "192.168.100.3${count.index}"
+    user_ip_address = "192.168.100.${count.index + 101}"
   }
   disk_edit_parameter {
     hostname        = "k8s-${terraform.workspace}-worker-node-${count.index + 1}"
     password        = var.cluster_pass
     disable_pw_auth = "true"
     ssh_key_ids     = [sakuracloud_ssh_key_gen.gen_key.id]
-    ip_address      = "192.168.100.3${count.index}"
+    ip_address      = "192.168.100.${count.index + 101}"
     gateway         = "192.168.100.254"
     netmask         = 24
   }

From 80e91cb83bbca6cadcea48ca6cd000c1d18283cd Mon Sep 17 00:00:00 2001
From: logica0419 <84759675+logica0419@users.noreply.github.com>
Date: Tue, 22 Oct 2024 03:14:22 +0900
Subject: [PATCH 06/12] add assignable min & max address

---
 terraform/output.tf | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/terraform/output.tf b/terraform/output.tf
index e4b732d1..bb8eb5d7 100644
--- a/terraform/output.tf
+++ b/terraform/output.tf
@@ -2,6 +2,14 @@ output "vip_address" {
   value = sakuracloud_internet.k8s_external_switch.ip_addresses[lookup(var.control_plane, terraform.workspace, 0)]
 }
 
+output "min_ip_address" {
+  value = sakuracloud_internet.k8s_external_switch.ip_addresses[lookup(var.control_plane, terraform.workspace, 0) + 1]
+}
+
+output "max_ip_address" {
+  value = sakuracloud_internet.k8s_external_switch.max_ip_address
+}
+
 output "k8s_control_plane_ip_address" {
   value = sakuracloud_server.k8s_control_plane[*].ip_address
 }

From bfd2fca3fe3a49f3559b984273854c483c92cf95 Mon Sep 17 00:00:00 2001
From: logica0419 <84759675+logica0419@users.noreply.github.com>
Date: Tue, 22 Oct 2024 03:19:58 +0900
Subject: [PATCH 07/12] delete router group_vars

---
 ansible/inventory/group_vars/router.yaml     | 12 ------------
 ansible/roles/k8s_install/defaults/main.yaml |  1 -
 ansible/roles/k8s_install/tasks/main.yaml    |  2 +-
 3 files changed, 1 insertion(+), 14 deletions(-)
 delete mode 100644 ansible/inventory/group_vars/router.yaml
 delete mode 100644 ansible/roles/k8s_install/defaults/main.yaml

diff --git a/ansible/inventory/group_vars/router.yaml b/ansible/inventory/group_vars/router.yaml
deleted file mode 100644
index be8d3568..00000000
--- a/ansible/inventory/group_vars/router.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-# netplanロール用
-netplan_configuration:
-  network:
-    version: 2
-    ethernets:
-      eth1:
-        addresses:
-          - 192.168.100.1{{groups["router"].index(inventory_hostname)}}/24
-
-# k8s_installロール用
-# join時にtaintを付与するため、kubeletのインストール時に設定
-k8s_install_kubelet_extra_args: --register-with-taints unusableKey=:NoSchedule
diff --git a/ansible/roles/k8s_install/defaults/main.yaml b/ansible/roles/k8s_install/defaults/main.yaml
deleted file mode 100644
index 5ef58629..00000000
--- a/ansible/roles/k8s_install/defaults/main.yaml
+++ /dev/null
@@ -1 +0,0 @@
-k8s_install_kubelet_extra_args:
diff --git a/ansible/roles/k8s_install/tasks/main.yaml b/ansible/roles/k8s_install/tasks/main.yaml
index 6cb2c84d..4d12d22d 100644
--- a/ansible/roles/k8s_install/tasks/main.yaml
+++ b/ansible/roles/k8s_install/tasks/main.yaml
@@ -37,7 +37,7 @@
       ansible.builtin.lineinfile:
         path: /etc/default/kubelet
         regexp: "^KUBELET_EXTRA_ARGS="
-        line: "KUBELET_EXTRA_ARGS=--node-ip={{ internal_ip }} {{ k8s_install_kubelet_extra_args }}"
+        line: "KUBELET_EXTRA_ARGS=--node-ip={{ internal_ip }}"
         state: present
         create: true
         mode: "0644"

From 46a24d5e2bd6dfb07ac1ce38189bc019180d8130 Mon Sep 17 00:00:00 2001
From: logica0419 <84759675+logica0419@users.noreply.github.com>
Date: Tue, 22 Oct 2024 03:22:07 +0900
Subject: [PATCH 08/12] update inventory handler & control plane group_vars

---
 .../inventory/group_vars/control_plane.yaml   |  2 +-
 ansible/inventory/inventory_handler.py        | 30 ++++++++-----------
 2 files changed, 13 insertions(+), 19 deletions(-)

diff --git a/ansible/inventory/group_vars/control_plane.yaml b/ansible/inventory/group_vars/control_plane.yaml
index c2c8cefe..72317e61 100644
--- a/ansible/inventory/group_vars/control_plane.yaml
+++ b/ansible/inventory/group_vars/control_plane.yaml
@@ -5,7 +5,7 @@ netplan_configuration:
     ethernets:
       eth1:
         addresses:
-          - 192.168.100.2{{groups["control_plane"].index(inventory_hostname)}}/24
+          - 192.168.100.{{ groups["control_plane"].index(inventory_hostname) + 1 }}/24
 
 # k8s_joinロール用
 k8s_join_flag: --control-plane --certificate-key {{ hostvars[groups['control_plane'][0]]['certs'] }}
diff --git a/ansible/inventory/inventory_handler.py b/ansible/inventory/inventory_handler.py
index 1c6b7b1b..8ed4ee47 100755
--- a/ansible/inventory/inventory_handler.py
+++ b/ansible/inventory/inventory_handler.py
@@ -40,7 +40,6 @@ def main():
     inventory = {
         "control_plane": {"hosts": []},
         "worker_node": {"hosts": []},
-        "router": {"hosts": []},
         "_meta": {"hostvars": {}},
     }
     workspace = get_workspace()
@@ -50,15 +49,12 @@ def main():
     #
     # print(tfstate["outputs"])
 
-    router = 0
     control_plane = 0
     worker_node = 0
     inventory_gp = {}
 
     for output_key in tfstate["outputs"]:
         match output_key:
-            case "k8s_router_ip_address":
-                inventory_gp = inventory["router"]
             case "k8s_control_plane_ip_address":
                 inventory_gp = inventory["control_plane"]
             case "k8s_worker_node_ip_address":
@@ -69,35 +65,29 @@ def main():
         for ip_address in tfstate["outputs"][output_key]["value"]:
             # not handle private ip address
             if (
-                ip_address[:11] == "192.168.100"
-                and output_key != "k8s_worker_node_ip_address"
+                ip_address[:7] == "192.168"
+                and output_key == "k8s_control_plane_ip_address"
             ):
                 continue
 
             inventory_gp["hosts"].append(ip_address)
 
             match output_key:
-                case "k8s_router_ip_address":
-                    inventory["_meta"]["hostvars"] = inventory["_meta"]["hostvars"] | {
-                        ip_address: {"internal_ip": f"192.168.100.1{str(router)}"}
-                    }
-                    router += 1
                 case "k8s_control_plane_ip_address":
                     inventory["_meta"]["hostvars"] = inventory["_meta"]["hostvars"] | {
                         ip_address: {
-                            "internal_ip": f"192.168.100.2{str(control_plane)}"
+                            "internal_ip": f"192.168.100.{str(control_plane+1)}"
                         }
                     }
                     control_plane += 1
                 case "k8s_worker_node_ip_address":
                     inventory["_meta"]["hostvars"] = inventory["_meta"]["hostvars"] | {
-                        ip_address: {"internal_ip": f"192.168.100.3{str(worker_node)}"}
+                        ip_address: {
+                            "internal_ip": f"192.168.100.3{str(worker_node+101)}"
+                        }
                     }
                     worker_node += 1
 
-    inventory["router"]["vars"] = {
-        "bgp_address": tfstate["outputs"]["external_address_range"]["value"]
-    }
     inventory["control_plane"]["vars"] = {
         "VIP": tfstate["outputs"]["vip_address"]["value"]
     }
@@ -105,12 +95,16 @@ def main():
         "ansible_ssh_common_args": (
             "-o ProxyCommand='ssh -o ControlMaster=auto -o ControlPersist=60s "
             "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "
-            f"-i ../id_rsa -W %h:%p ubuntu@{inventory['router']['hosts'][0]}'"
+            f"-i ../id_rsa -W %h:%p ubuntu@{inventory['control_plane']['hosts'][0]}'"
         )
     }
     inventory["delegate_plane"] = {
         "hosts": [inventory["control_plane"]["hosts"][0]],
-        "vars": {"workspace": workspace},
+        "vars": {
+            "workspace": workspace,
+            "min_ip_address": tfstate["outputs"]["min_ip_address"]["value"],
+            "max_ip_address": tfstate["outputs"]["max_ip_address"]["value"],
+        },
     }
 
     print(json.dumps(inventory))

From fd3ceff60a764d9ff684e1a26580be7d4289436a Mon Sep 17 00:00:00 2001
From: logica0419 <84759675+logica0419@users.noreply.github.com>
Date: Tue, 22 Oct 2024 03:22:28 +0900
Subject: [PATCH 09/12] put out router from setup.yaml

---
 ansible/setup.yaml | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/ansible/setup.yaml b/ansible/setup.yaml
index c77e8cc8..c13b703f 100644
--- a/ansible/setup.yaml
+++ b/ansible/setup.yaml
@@ -1,5 +1,5 @@
 - name: Private address setup
-  hosts: router:control_plane
+  hosts: control_plane
   any_errors_fatal: true
   gather_facts: false
   roles:
@@ -16,13 +16,6 @@
     - containerd
     - k8s_install
 
-- name: BIRD setup
-  hosts: router
-  any_errors_fatal: true
-  become: true
-  roles:
-    - bird
-
 - name: Kubernetes API loadbalancer setup
   hosts: control_plane
   any_errors_fatal: true

From 69bea4c4377d0c2afc85b7a1ce2b9b02777e1345 Mon Sep 17 00:00:00 2001
From: logica0419 <84759675+logica0419@users.noreply.github.com>
Date: Tue, 22 Oct 2024 03:32:54 +0900
Subject: [PATCH 10/12] rewrite bgp-config to announcement-config

---
 ansible/roles/components/tasks/main.yaml      | 12 ++---
 .../templates/announcement-config.yaml.j2     | 23 +++++++++
 .../components/templates/bgp-config.yaml.j2   | 49 -------------------
 3 files changed, 29 insertions(+), 55 deletions(-)
 create mode 100644 ansible/roles/components/templates/announcement-config.yaml.j2
 delete mode 100644 ansible/roles/components/templates/bgp-config.yaml.j2

diff --git a/ansible/roles/components/tasks/main.yaml b/ansible/roles/components/tasks/main.yaml
index f00a2b04..a8bc47a5 100644
--- a/ansible/roles/components/tasks/main.yaml
+++ b/ansible/roles/components/tasks/main.yaml
@@ -48,15 +48,15 @@
       changed_when: false
       failed_when: false
 
-- name: Setup BGP
+- name: Setup L2 announcement
   block:
-    - name: Send bgp-config
+    - name: Send announcement config
       ansible.builtin.template:
-        src: bgp-config.yaml.j2
-        dest: /tmp/bgp-config.yaml
+        src: announcement-config.yaml.j2
+        dest: /tmp/announcement-config.yaml
         mode: "0644"
-    - name: Apply bgp-config
-      ansible.builtin.command: kubectl apply -f /tmp/bgp-config.yaml
+    - name: Apply announcement config
+      ansible.builtin.command: kubectl apply -f /tmp/announcement-config.yaml
       register: ret
       until: ret.rc == 0
       retries: 12
diff --git a/ansible/roles/components/templates/announcement-config.yaml.j2 b/ansible/roles/components/templates/announcement-config.yaml.j2
new file mode 100644
index 00000000..c91d452b
--- /dev/null
+++ b/ansible/roles/components/templates/announcement-config.yaml.j2
@@ -0,0 +1,23 @@
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: cilium-announcement
+spec:
+  nodeSelector:
+    matchExpressions:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+  interfaces:
+    - ^eth[0-9]+
+  externalIPs: true
+  loadBalancerIPs: true
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: default-pool
+  namespace: kube-system
+spec:
+  blocks:
+    - start: {{ min_ip_address }}
+      end: {{ max_ip_address }}
diff --git a/ansible/roles/components/templates/bgp-config.yaml.j2 b/ansible/roles/components/templates/bgp-config.yaml.j2
deleted file mode 100644
index 90297f67..00000000
--- a/ansible/roles/components/templates/bgp-config.yaml.j2
+++ /dev/null
@@ -1,49 +0,0 @@
-apiVersion: cilium.io/v2alpha1
-kind: CiliumBGPClusterConfig
-metadata:
-  name: cilium-bgp
-spec:
-  bgpInstances:
-    - name: "default"
-      localASN: 65021
-      peers:
-{% for address in groups['router'] %}
-        - name: {{ address }}
-          peerASN: 65020
-          peerAddress: {{ address }}
-          peerConfigRef:
-            name: "cilium-peer"
-{% endfor %}
----
-apiVersion: cilium.io/v2alpha1
-kind: CiliumBGPPeerConfig
-metadata:
-  name: cilium-peer
-spec:
-  gracefulRestart:
-    enabled: true
-    restartTimeSeconds: 15
----
-apiVersion: cilium.io/v2alpha1
-kind: CiliumBGPAdvertisement
-metadata:
-  name: bgp-advertisements
-  labels:
-    advertise: bgp
-spec:
-  advertisements:
-    - advertisementType: "Service"
-      service:
-        addresses:
-          - LoadBalancerIP
----
-apiVersion: "cilium.io/v2alpha1"
-kind: CiliumLoadBalancerIPPool
-metadata:
-  name: default-pool
-  namespace: kube-system
-spec:
-  blocks:
-{% for item in groups['router'] %}
-    - cidr: {{ hostvars[item]['bgp_address'] }}
-{% endfor %}

From 246e559ca1755ec906ea96905fe7f61500570905 Mon Sep 17 00:00:00 2001
From: logica0419 <84759675+logica0419@users.noreply.github.com>
Date: Tue, 22 Oct 2024 03:57:14 +0900
Subject: [PATCH 11/12] fix LoadBalancerIPPool setting

---
 .../roles/components/templates/announcement-config.yaml.j2 | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/ansible/roles/components/templates/announcement-config.yaml.j2 b/ansible/roles/components/templates/announcement-config.yaml.j2
index c91d452b..d0c35cb2 100644
--- a/ansible/roles/components/templates/announcement-config.yaml.j2
+++ b/ansible/roles/components/templates/announcement-config.yaml.j2
@@ -1,4 +1,4 @@
-apiVersion: "cilium.io/v2alpha1"
+apiVersion: cilium.io/v2alpha1
 kind: CiliumL2AnnouncementPolicy
 metadata:
   name: cilium-announcement
@@ -9,10 +9,9 @@ spec:
         operator: Exists
   interfaces:
     - ^eth[0-9]+
-  externalIPs: true
   loadBalancerIPs: true
 ---
-apiVersion: "cilium.io/v2alpha1"
+apiVersion: cilium.io/v2alpha1
 kind: CiliumLoadBalancerIPPool
 metadata:
   name: default-pool
@@ -20,4 +19,4 @@ metadata:
 spec:
   blocks:
     - start: {{ min_ip_address }}
-      end: {{ max_ip_address }}
+      stop: {{ max_ip_address }}

From 715f909fb34cddd61c199d1d5eff92da867bb23f Mon Sep 17 00:00:00 2001
From: logica0419 <84759675+logica0419@users.noreply.github.com>
Date: Tue, 22 Oct 2024 04:02:16 +0900
Subject: [PATCH 12/12] fix inventory handler lint

---
 ansible/inventory/inventory_handler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ansible/inventory/inventory_handler.py b/ansible/inventory/inventory_handler.py
index 8ed4ee47..b1bc2c96 100755
--- a/ansible/inventory/inventory_handler.py
+++ b/ansible/inventory/inventory_handler.py
@@ -76,14 +76,14 @@ def main():
                 case "k8s_control_plane_ip_address":
                     inventory["_meta"]["hostvars"] = inventory["_meta"]["hostvars"] | {
                         ip_address: {
-                            "internal_ip": f"192.168.100.{str(control_plane+1)}"
+                            "internal_ip": f"192.168.100.{str(control_plane + 1)}"
                         }
                     }
                     control_plane += 1
                 case "k8s_worker_node_ip_address":
                     inventory["_meta"]["hostvars"] = inventory["_meta"]["hostvars"] | {
                         ip_address: {
-                            "internal_ip": f"192.168.100.3{str(worker_node+101)}"
+                            "internal_ip": f"192.168.100.3{str(worker_node + 101)}"
                         }
                     }
                     worker_node += 1