feat: add k8s_gateway for accessing ingresses locally over DNS (#287)

.config.sample.env

@@ -16,8 +16,11 @@ export BOOTSTRAP_CLOUDFLARE_APIKEY=""
 # You don't need many IPs, just choose 10 IPs to start with
 # e.g. 192.168.1.220-192.168.1.230
 export BOOTSTRAP_METALLB_LB_RANGE=""
-# The load balancer IP for traefik, choose from one of the available IPs above
+# The load balancer IP for k8s_gateway, choose from one of the available IPs above
 # e.g. 192.168.1.220
+export BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR=""
+# The load balancer IP for traefik, choose from one of the available IPs above
+# e.g. 192.168.1.221
 export BOOTSTRAP_METALLB_TRAEFIK_ADDR=""
 # Age Public Key - string should start with age
 # e.g. age15uzrw396e67z9wdzsxzdk7ka0g2gr3l460e0slaea563zll3hdfqwqxdta

.github/ISSUE_TEMPLATE/config.yml

@@ -1,3 +1,4 @@
+---
 blank_issues_enabled: false
 contact_links:
   - name: Discuss on Discord

.github/linters/.yamllint.yaml

@@ -1,3 +1,4 @@
+---
 ignore: |
   *.sops.*
   gotk-components.yaml

README.md

@@ -24,6 +24,7 @@ Feel free to read up on any of these technologies before you get started to be m
 - [echo-server](https://github.com/Ealenn/Echo-Server) - REST Server Tests (Echo-Server) API (useful for debugging HTTP issues)
 - [flux](https://toolkit.fluxcd.io/) - GitOps tool for deploying manifests from the `cluster` directory
 - [hajimari](https://github.com/toboshii/hajimari) - start page with ingress discovery
+- [k8s_gateway](https://github.com/ori-edge/k8s_gateway) - DNS resolver for all types of external Kubernetes resources
 - [kube-vip](https://kube-vip.io/) - layer 2 load balancer for the Kubernetes control plane
 - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) - default storage class provided by k3s
 - [metallb](https://metallb.universe.tf/) - bare metal load balancer
@@ -306,19 +307,31 @@ kubectl --kubeconfig=./provision/kubeconfig get pods -n flux-system
 # source-controller-7d6875bcb4-zqw9f         1/1     Running   0          1h
 ```
 
-If all goes well and you have port forwarded `80` and `443` in your router to the `${BOOTSTRAP_METALLB_TRAEFIK_ADDR}` IP, in a few moments head over to your browser and you _should_ be able to access `https://hajimari.${BOOTSTRAP_CLOUDFLARE_DOMAIN}`
+🎉 **Congratulations** if all goes smooth you'll have a Kubernetes cluster managed by Flux, your Git repository is driving the state of your cluster.
 
-🎉 **Congratulations** you have a Kubernetes cluster managed by Flux, your Git repository is driving the state of your cluster.
+Now it's time to pause and go get some coffee ☕
 
 ## 📣 Post installation
 
+### 🌐 DNS
+
+`k8s_gateway` is deployed on the IP choosen for `${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}`. Inorder to test DNS you can point your clients DNS to the `${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}` IP address and load `https://hajimari.${BOOTSTRAP_CLOUDFLARE_DOMAIN}` in your browser.
+
+You can also try debugging with the command `dig`, e.g. `dig @${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR} hajimari.${BOOTSTRAP_CLOUDFLARE_DOMAIN}` and you should get a valid answer containing your `${BOOTSTRAP_METALLB_TRAEFIK_ADDR}` IP address.
+
+If your router (or Pi-Hole, Adguard Home or whatever) supports conditional DNS forwarding (also know as split-horizon DNS) you may have DNS requests for `${SECRET_DOMAIN}` only point to the  `${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}` IP address. This will ensure only DNS requests for `${SECRET_DOMAIN}` will get routed to your `k8s_gateway` service.
+
+To access services from the outside world port forwarded `80` and `443` in your router to the `${BOOTSTRAP_METALLB_TRAEFIK_ADDR}` IP, in a few moments head over to your browser and you _should_ be able to access `https://hajimari.${BOOTSTRAP_CLOUDFLARE_DOMAIN}` from a device outside your LAN.
+
 ### 👉 Troubleshooting
 
-Our [wiki](https://github.com/k8s-at-home/template-cluster-k3s/wiki) is a good place to start troubleshooting issues. If that doesn't cover your issue, start a new thread in the #support channel on our [Discord](https://discord.gg/k8s-at-home).
+Our [wiki](https://github.com/k8s-at-home/template-cluster-k3s/wiki) (WIP, contributions welcome) is a good place to start troubleshooting issues. If that doesn't cover your issue, come join and say Hi in our [Discord](https://discord.gg/k8s-at-home) server by starting a new thread in the #kubernetes support channel.
+
+You may also open a issue on this GitHub repo or open a [discussion on GitHub](https://github.com/k8s-at-home/organization/discussions).
 
 ### 🤖 Integrations
 
-Our Check out our [wiki](https://github.com/k8s-at-home/template-cluster-k3s/wiki) (WIP) for more integrations!
+Our Check out our [wiki](https://github.com/k8s-at-home/template-cluster-k3s/wiki) (WIP, contributions welcome) for more integrations!
 
 ## ❔ What's next
 

cluster/apps/default/echo-server/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/apps/default/hajimari/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/apps/default/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/apps/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/apps/networking/k8s-gateway/helm-release.yaml

@@ -0,0 +1,41 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: k8s-gateway
+  namespace: networking
+spec:
+  interval: 5m
+  chart:
+    spec:
+      chart: k8s-gateway
+      version: 1.1.6
+      sourceRef:
+        kind: HelmRepository
+        name: k8s-gateway
+        namespace: flux-system
+      interval: 5m
+  values:
+    fullnameOverride: k8s-gateway
+    domain: ${SECRET_DOMAIN}
+    ttl: 1
+    service:
+      type: LoadBalancer
+      port: 53
+      annotations:
+        metallb.universe.tf/loadBalancerIPs: "${METALLB_K8S_GATEWAY_ADDR}"
+      externalTrafficPolicy: Local
+    # Increase replicaCount and set an anti-affinity if you wish to scale this
+    # replicaCount: 1
+    # affinity:
+    #   podAntiAffinity:
+    #     preferredDuringSchedulingIgnoredDuringExecution:
+    #       - weight: 100
+    #         podAffinityTerm:
+    #           labelSelector:
+    #             matchExpressions:
+    #               - key: app.kubernetes.io/name
+    #                 operator: In
+    #                 values:
+    #                   - k8s-gateway
+    #           topologyKey: kubernetes.io/hostname

cluster/apps/networking/k8s-gateway/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - helm-release.yaml

cluster/apps/networking/kustomization.yaml

@@ -1,5 +1,7 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - cloudflare-ddns
+  - k8s-gateway
   - traefik

cluster/apps/networking/traefik/dashboard/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/apps/networking/traefik/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/base/flux-system/charts/helm/k8s-gateway-charts.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: k8s-gateway
+  namespace: flux-system
+spec:
+  interval: 30m
+  url: https://ori-edge.github.io/k8s_gateway/
+  timeout: 3m

cluster/base/flux-system/charts/helm/kustomization.yaml

@@ -13,6 +13,7 @@ resources:
   - infracloudio-charts.yaml
   - jetstack-charts.yaml
   - k8s-at-home-charts.yaml
+  - k8s-gateway-charts.yaml
   - metallb-charts.yaml
   - metrics-server-charts.yaml
   - minecraft-server-charts.yaml

cluster/base/flux-system/charts/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/base/flux-system/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/core/kube-system/kube-vip/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/core/kube-system/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/core/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/core/metallb-system/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/core/namespaces/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/crds/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

cluster/crds/traefik/kustomization.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:

configure.sh

@@ -173,13 +173,16 @@ verify_metallb() {
     local ip_floor=
     local ip_ceil=
     _has_envar "BOOTSTRAP_METALLB_LB_RANGE"
+    _has_envar "BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR"
     _has_envar "BOOTSTRAP_METALLB_TRAEFIK_ADDR"
 
     ip_floor=$(echo "${BOOTSTRAP_METALLB_LB_RANGE}" | cut -d- -f1)
     ip_ceil=$(echo "${BOOTSTRAP_METALLB_LB_RANGE}" | cut -d- -f2)
 
+    # TODO(configure.sh): More checks on valid IP addressing
     _has_valid_ip "${ip_floor}" "BOOTSTRAP_METALLB_LB_RANGE"
     _has_valid_ip "${ip_ceil}" "BOOTSTRAP_METALLB_LB_RANGE"
+    _has_valid_ip "${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}" "BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR"
     _has_valid_ip "${BOOTSTRAP_METALLB_TRAEFIK_ADDR}" "BOOTSTRAP_METALLB_TRAEFIK_ADDR"
 }
 

tmpl/cluster/cluster-settings.yaml

@@ -6,4 +6,5 @@ metadata:
   namespace: flux-system
 data:
   METALLB_LB_RANGE: ${BOOTSTRAP_METALLB_LB_RANGE}
+  METALLB_K8S_GATEWAY_ADDR: ${BOOTSTRAP_METALLB_K8S_GATEWAY_ADDR}
   METALLB_TRAEFIK_ADDR: ${BOOTSTRAP_METALLB_TRAEFIK_ADDR}

tmpl/cluster/kube-vip-daemonset.yaml

@@ -57,11 +57,11 @@ spec:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
               - matchExpressions:
-                - key: node-role.kubernetes.io/master
-                  operator: Exists
+                  - key: node-role.kubernetes.io/master
+                    operator: Exists
               - matchExpressions:
-                - key: node-role.kubernetes.io/control-plane
-                  operator: Exists
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: Exists
       tolerations:
         - effect: NoSchedule
           operator: Exists