feat(ansible): Deploy coredns with HelmChart instead of builtin metho…

…d and other small QoL changes

Signed-off-by: Devin Buhl <devin@buhl.casa>

ansible/inventory/group_vars/kubernetes/k3s.yml → ansible/inventory/group_vars/kubernetes/kubernetes.yml

@@ -8,7 +8,6 @@
 k3s_release_version: "v1.27.3+k3s1"
 k3s_install_hard_links: true
 k3s_become: true
-k3s_debug: false
 k3s_etcd_datastore: true
 k3s_use_unsupported_config: true
 k3s_registration_address: "{{ kubevip_address }}"
@@ -26,6 +25,7 @@ k3s_server_manifests_urls:
 # /var/lib/rancher/k3s/server/manifests
 k3s_server_manifests_templates:
   - custom-cilium-helmchart.yaml.j2
+  - custom-coredns-helmchart.yaml.j2
 # /var/lib/rancher/k3s/agent/pod-manifests
 k3s_server_pod_manifests_templates:
   - kube-vip-static-pod.yaml.j2

ansible/inventory/group_vars/master/k3s.yml → ansible/inventory/group_vars/master/kubernetes.yml

@@ -10,6 +10,7 @@ k3s_server:
   docker: false
   flannel-backend: "none"             # This needs to be in quotes
   disable:
+    - coredns                         # Disable coredns                 - replaced with Helm Chart
     - flannel                         # Disable flannel                 - replaced with Cilium
     - local-storage                   # Disable local-path-provisioner  - installed with Flux
     - metrics-server                  # Disable metrics-server          - installed with Flux
@@ -21,10 +22,10 @@ k3s_server:
   write-kubeconfig-mode: "644"
   cluster-cidr: "{{ cluster_cidr }}"
   service-cidr: "{{ service_cidr }}"
+  etcd-expose-metrics: true           # Required to monitor etcd with kube-prometheus-stack
   kube-controller-manager-arg:
     - "bind-address=0.0.0.0"          # Required to monitor kube-controller-manager with kube-prometheus-stack
   kube-scheduler-arg:
     - "bind-address=0.0.0.0"          # Required to monitor kube-scheduler with kube-prometheus-stack
-  etcd-expose-metrics: true           # Required to monitor etcd with kube-prometheus-stack
   kube-apiserver-arg:
     - "anonymous-auth=true"           # Required for HAProxy health-checks

ansible/playbooks/cluster-installation.yml

@@ -1,7 +1,6 @@
 ---
-- hosts:
-    - master
-    - worker
+- name: Cluster Installation
+  hosts: all
   become: true
   gather_facts: true
   any_errors_fatal: true
@@ -14,10 +13,10 @@
       check_mode: false
       ansible.builtin.stat:
         path: /etc/rancher/k3s/config.yaml
-      register: k3s_check_installed
+      register: k3s_installed
 
     - name: Ignore manifests templates and urls if the cluster is already installed
-      when: k3s_check_installed.stat.exists
+      when: k3s_installed.stat.exists
       ansible.builtin.set_fact:
         k3s_server_manifests_templates: []
         k3s_server_manifests_urls: []
@@ -29,168 +28,48 @@
       vars:
         k3s_state: installed
 
-    - name: Get absolute path to this Git repository
-      delegate_to: localhost
-      become: false
-      run_once: true
-      check_mode: false
-      ansible.builtin.command: git rev-parse --show-toplevel
-      register: repository_path
+    - name: Kubeconfig
+      ansible.builtin.include_tasks: tasks/kubeconfig.yml
 
-    - name: Copy kubeconfig to the project directory
-      when: k3s_primary_control_node
-      ansible.builtin.fetch:
-        src: /etc/rancher/k3s/k3s.yaml
-        dest: "{{ repository_path.stdout }}/kubeconfig"
-        flat: true
+    - name: Wait for custom manifests to rollout
+      when:
+        - k3s_primary_control_node
+        - (k3s_server_manifests_templates | length > 0
+            or k3s_server_manifests_urls | length > 0)
+      kubernetes.core.k8s_info:
+        kubeconfig: /etc/rancher/k3s/k3s.yaml
+        kind: "{{ item.kind }}"
+        name: "{{ item.name }}"
+        namespace: "{{ item.namespace | default('') }}"
+        wait: true
+        wait_sleep: 10
+        wait_timeout: 360
+      loop:
+        - { name: cilium, kind: HelmChart, namespace: kube-system }
+        - { name: coredns, kind: HelmChart, namespace: kube-system }
+        - { name: podmonitors.monitoring.coreos.com, kind: CustomResourceDefinition }
+        - { name: prometheusrules.monitoring.coreos.com, kind: CustomResourceDefinition }
+        - { name: servicemonitors.monitoring.coreos.com, kind: CustomResourceDefinition }
 
-    - name: Update kubeconfig with the correct load balancer address
-      delegate_to: localhost
-      become: false
-      run_once: true
-      ansible.builtin.replace:
-        path: "{{ repository_path.stdout }}/kubeconfig"
-        regexp: https://127.0.0.1:6443
-        replace: "https://{{ k3s_registration_address }}:6443"
+    - name: Coredns
+      when:
+        - k3s_primary_control_node
+        - (k3s_server_manifests_templates | length > 0
+            or k3s_server_manifests_urls | length > 0)
+      ansible.builtin.include_tasks: tasks/coredns.yml
 
-    - name: Custom manifests (1)
+    - name: Cilium
       when:
         - k3s_primary_control_node
         - (k3s_server_manifests_templates | length > 0
             or k3s_server_manifests_urls | length > 0)
-      block:
-        - name: Custom manifests (1) | Wait for custom manifests to rollout
-          kubernetes.core.k8s_info:
-            kubeconfig: /etc/rancher/k3s/k3s.yaml
-            kind: "{{ item.kind }}"
-            name: "{{ item.name }}"
-            namespace: "{{ item.namespace | default('') }}"
-            wait: true
-            wait_sleep: 10
-            wait_timeout: 360
-          loop:
-            - name: cilium
-              kind: HelmChart
-              namespace: kube-system
-            - name: podmonitors.monitoring.coreos.com
-              kind: CustomResourceDefinition
-            - name: prometheusrules.monitoring.coreos.com
-              kind: CustomResourceDefinition
-            - name: servicemonitors.monitoring.coreos.com
-              kind: CustomResourceDefinition
-        - name: Custom manifests (1) | Wait for Cilium to rollout
-          kubernetes.core.k8s_info:
-            kubeconfig: /etc/rancher/k3s/k3s.yaml
-            kind: Job
-            name: helm-install-cilium
-            namespace: kube-system
-            wait: true
-            wait_condition:
-              type: Complete
-              status: true
-            wait_timeout: 360
-        # Unmanage and remove the Cilium HelmChart in-order for
-        # flux to take over managing the lifecycle of Cilium
-        - name: Custom manifests (1) | Patch the Cilium HelmChart to unmanage it
-          kubernetes.core.k8s_json_patch:
-            kubeconfig: /etc/rancher/k3s/k3s.yaml
-            name: cilium
-            kind: HelmChart
-            namespace: kube-system
-            patch:
-              - op: add
-                path: /metadata/annotations/helmcharts.helm.cattle.io~1unmanaged
-                value: "true"
-        - name: Custom manifests (1) | Delete the Cilium HelmChart CR
-          kubernetes.core.k8s:
-            kubeconfig: /etc/rancher/k3s/k3s.yaml
-            name: cilium
-            kind: HelmChart
-            namespace: kube-system
-            state: absent
-        - name: Custom manifests (1) | Check if Cilium HelmChart was deleted
-          kubernetes.core.k8s_info:
-            kubeconfig: /etc/rancher/k3s/k3s.yaml
-            name: cilium
-            kind: HelmChart
-            namespace: kube-system
-          register: cilium_helmchart
-        - name: Custom manifests (1) | Force delete the Cilium HelmChart
-          when: cilium_helmchart.resources | count > 0
-          kubernetes.core.k8s:
-            kubeconfig: /etc/rancher/k3s/k3s.yaml
-            name: cilium
-            kind: HelmChart
-            namespace: kube-system
-            state: patched
-            definition:
-              metadata:
-                finalizers: []
+      ansible.builtin.include_tasks: tasks/cilium.yml
 
-    # Cleaning up certain manifests from the /var/lib/rancher/k3s/server/manifests directory
-    # is needed because k3s has an awesome "feature" to always re-deploy them when the k3s
-    # service is restarted. Removing them does not uninstall the manifests from your cluster.
-    - name: Custom manifests (2)
+    - name: Cruft
       when: k3s_primary_control_node
-      block:
-        - name: Custom manifests (2) | Get list of custom mantifests
-          ansible.builtin.find:
-            paths: "{{ k3s_server_manifests_dir }}"
-            file_type: file
-            use_regex: true
-            patterns: ["^custom-.*"]
-          register: custom_manifest
-        - name: Custom manifests (2) | Delete custom mantifests
-          ansible.builtin.file:
-            path: "{{ item.path }}"
-            state: absent
-          loop: "{{ custom_manifest.files }}"
-        - name: Custom manifests (2) | Get list of custom addons
-          kubernetes.core.k8s_info:
-            kubeconfig: /etc/rancher/k3s/k3s.yaml
-            kind: Addon
-          register: addons_list
-        - name: Custom manifests (2) | Delete addons
-          kubernetes.core.k8s:
-            kubeconfig: /etc/rancher/k3s/k3s.yaml
-            name: "{{ item.metadata.name }}"
-            kind: Addon
-            namespace: kube-system
-            state: absent
-          loop: "{{ addons_list.resources | selectattr('metadata.name', 'match', '^custom-.*') | list }}"
+      ansible.builtin.include_tasks: tasks/cruft.yml
 
-    # https://github.com/k3s-io/k3s/issues/1900
-    - name: Stale containers
-      block:
-        - name: Stale containers | Create systemd unit
-          ansible.builtin.blockinfile:
-            path: /etc/systemd/system/stale-containers.service
-            create: true
-            mode: "0644"
-            block: |
-              [Unit]
-              Description=Clean up stale containers
-              [Service]
-              Type=oneshot
-              ExecStart=/usr/local/bin/k3s crictl rmi --prune > /dev/null 2>&1
-        - name: Stale containers | Create systemd timer
-          ansible.builtin.blockinfile:
-            path: /etc/systemd/system/stale-containers.timer
-            create: true
-            mode: "0644"
-            block: |
-              [Unit]
-              Description=Clean up stale containers
-              [Timer]
-              OnCalendar=weekly
-              AccuracySec=1h
-              Persistent=true
-              RandomizedDelaySec=6000
-              [Install]
-              WantedBy=timers.target
-        - name: Stale containers | Start the systemd timer
-          ansible.builtin.systemd:
-            name: stale-containers.timer
-            enabled: true
-            daemon_reload: true
-            state: started
+    - name: Stale Containers
+      ansible.builtin.include_tasks: tasks/stale_containers.yml
+      vars:
+        stale_containers_state: enabled

ansible/playbooks/cluster-nuke.yml

@@ -1,7 +1,6 @@
 ---
-- hosts:
-    - master
-    - worker
+- name: Cluster Nuke
+  hosts: all
   become: true
   gather_facts: true
   any_errors_fatal: true
@@ -22,12 +21,15 @@
       ansible.builtin.pause:
         seconds: 5
   tasks:
-    - name: Stop Kubernetes
-      ansible.builtin.include_role:
-        name: xanmanning.k3s
-        public: true
-      vars:
-        k3s_state: stopped
+    - name: Stop Kubernetes # noqa: ignore-errors
+      ignore_errors: true
+      block:
+        - name: Stop Kubernetes
+          ansible.builtin.include_role:
+            name: xanmanning.k3s
+            public: true
+          vars:
+            k3s_state: stopped
 
     # https://github.com/k3s-io/docs/blob/main/docs/installation/network-options.md
     - name: Networking
@@ -60,6 +62,11 @@
       vars:
         k3s_state: uninstalled
 
+    - name: Stale Containers
+      ansible.builtin.include_tasks: tasks/stale_containers.yml
+      vars:
+        stale_containers_state: disabled
+
     - name: Reboot
       ansible.builtin.reboot:
         msg: Rebooting nodes