test: add a test for inline machine config trusted roots

Run SideroLink API server via TLS with self-signed certificate, inject
that certificate into Talos via `talos.config.inline=`.

Fix a couple of place where our special TLS root CA provider supporting
reloading on the fly was not used.

Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>

.github/workflows/ci.yaml

@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2024-09-09T13:58:35Z by kres 8be5fa7.
+# Generated on 2024-09-12T16:43:46Z by kres 8be5fa7.
 
 name: default
 concurrency:
@@ -2385,6 +2385,14 @@ jobs:
           WITH_SIDEROLINK_AGENT: tunnel
         run: |
           sudo -E make e2e-qemu
+      - name: e2e-siderolink-tls
+        env:
+          IMAGE_REGISTRY: registry.dev.siderolabs.io
+          SHORT_INTEGRATION_TEST: "yes"
+          VIA_MAINTENANCE_MODE: "true"
+          WITH_SIDEROLINK_AGENT: wireguard+tls
+        run: |
+          sudo -E make e2e-qemu
       - name: e2e-apparmor
         env:
           IMAGE_REGISTRY: registry.dev.siderolabs.io

.github/workflows/integration-misc-4-cron.yaml

@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2024-09-09T13:58:35Z by kres 8be5fa7.
+# Generated on 2024-09-12T16:43:46Z by kres 8be5fa7.
 
 name: integration-misc-4-cron
 concurrency:
@@ -94,6 +94,14 @@ jobs:
           WITH_SIDEROLINK_AGENT: tunnel
         run: |
           sudo -E make e2e-qemu
+      - name: e2e-siderolink-tls
+        env:
+          IMAGE_REGISTRY: registry.dev.siderolabs.io
+          SHORT_INTEGRATION_TEST: "yes"
+          VIA_MAINTENANCE_MODE: "true"
+          WITH_SIDEROLINK_AGENT: wireguard+tls
+        run: |
+          sudo -E make e2e-qemu
       - name: e2e-apparmor
         env:
           IMAGE_REGISTRY: registry.dev.siderolabs.io

.kres.yaml

@@ -926,6 +926,14 @@ spec:
             WITH_SIDEROLINK_AGENT: tunnel
             VIA_MAINTENANCE_MODE: true
             IMAGE_REGISTRY: registry.dev.siderolabs.io
+        - name: e2e-siderolink-tls
+          command: e2e-qemu
+          withSudo: true
+          environment:
+            SHORT_INTEGRATION_TEST: yes
+            WITH_SIDEROLINK_AGENT: wireguard+tls
+            VIA_MAINTENANCE_MODE: true
+            IMAGE_REGISTRY: registry.dev.siderolabs.io
         - name: e2e-apparmor
           command: e2e-qemu
           withSudo: true

cmd/talosctl/cmd/mgmt/cluster/create.go

@@ -5,7 +5,9 @@
 package cluster
 
 import (
+	"bytes"
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"math/big"
@@ -23,6 +25,8 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/google/uuid"
 	"github.com/hashicorp/go-getter/v2"
+	"github.com/klauspost/compress/zstd"
+	"github.com/siderolabs/crypto/x509"
 	"github.com/siderolabs/gen/maps"
 	"github.com/siderolabs/go-blockdevice/v2/encryption"
 	"github.com/siderolabs/go-kubeconfig"
@@ -40,10 +44,12 @@ import (
 	clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config"
 	"github.com/siderolabs/talos/pkg/machinery/config"
 	"github.com/siderolabs/talos/pkg/machinery/config/bundle"
+	"github.com/siderolabs/talos/pkg/machinery/config/configloader"
 	"github.com/siderolabs/talos/pkg/machinery/config/configpatcher"
 	"github.com/siderolabs/talos/pkg/machinery/config/encoder"
 	"github.com/siderolabs/talos/pkg/machinery/config/generate"
 	"github.com/siderolabs/talos/pkg/machinery/config/machine"
+	"github.com/siderolabs/talos/pkg/machinery/config/types/security"
 	"github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1"
 	"github.com/siderolabs/talos/pkg/machinery/constants"
 	"github.com/siderolabs/talos/pkg/machinery/nethelpers"
@@ -752,6 +758,24 @@ func create(ctx context.Context) error {
 		)
 	}
 
+	var slb *siderolinkBuilder
+
+	if withSiderolinkAgent.IsEnabled() {
+		slb, err = newSiderolinkBuilder(gatewayIPs[0].String(), withSiderolinkAgent.IsTLS())
+		if err != nil {
+			return err
+		}
+	}
+
+	if trustedRootsConfig := slb.TrustedRootsConfig(); trustedRootsConfig != nil {
+		trustedRootsPatch, err := configloader.NewFromBytes(trustedRootsConfig)
+		if err != nil {
+			return fmt.Errorf("error loading trusted roots config: %w", err)
+		}
+
+		configBundleOpts = append(configBundleOpts, bundle.WithPatch([]configpatcher.Patch{configpatcher.NewStrategicMergePatch(trustedRootsPatch)}))
+	}
+
 	configBundle, err := bundle.NewBundle(configBundleOpts...)
 	if err != nil {
 		return err
@@ -795,15 +819,6 @@ func create(ctx context.Context) error {
 		extraKernelArgs = procfs.NewCmdline(extraBootKernelArgs)
 	}
 
-	var slb *siderolinkBuilder
-
-	if withSiderolinkAgent.IsEnabled() {
-		slb, err = newSiderolinkBuilder(gatewayIPs[0].String())
-		if err != nil {
-			return err
-		}
-	}
-
 	err = slb.SetKernelArgs(extraKernelArgs, withSiderolinkAgent.IsTunnel())
 	if err != nil {
 		return err
@@ -1255,7 +1270,7 @@ func init() {
 	Cmd.AddCommand(createCmd)
 }
 
-func newSiderolinkBuilder(wgHost string) (*siderolinkBuilder, error) {
+func newSiderolinkBuilder(wgHost string, useTLS bool) (*siderolinkBuilder, error) {
 	prefix, err := networkPrefix("")
 	if err != nil {
 		return nil, err
@@ -1268,6 +1283,16 @@ func newSiderolinkBuilder(wgHost string) (*siderolinkBuilder, error) {
 		nodeIPv6Addr: prefix.Addr().Next().String(),
 	}
 
+	if useTLS {
+		ca, err := x509.NewSelfSignedCertificateAuthority(x509.ECDSA(true), x509.IPAddresses([]net.IP{net.ParseIP(wgHost)}))
+		if err != nil {
+			return nil, err
+		}
+
+		result.apiCert = ca.CrtPEM
+		result.apiKey = ca.KeyPEM
+	}
+
 	var resultErr error
 
 	for range 10 {
@@ -1312,6 +1337,9 @@ type siderolinkBuilder struct {
 	apiPort      int
 	sinkPort     int
 	logPort      int
+
+	apiCert []byte
+	apiKey  []byte
 }
 
 // DefineIPv6ForUUID defines an IPv6 address for a given UUID. It is safe to call this method on a nil pointer.
@@ -1340,6 +1368,8 @@ func (slb *siderolinkBuilder) SiderolinkRequest() provision.SiderolinkRequest {
 	return provision.SiderolinkRequest{
 		WireguardEndpoint: net.JoinHostPort(slb.wgHost, strconv.Itoa(slb.wgPort)),
 		APIEndpoint:       ":" + strconv.Itoa(slb.apiPort),
+		APICertificate:    slb.apiCert,
+		APIKey:            slb.apiKey,
 		SinkEndpoint:      ":" + strconv.Itoa(slb.sinkPort),
 		LogEndpoint:       ":" + strconv.Itoa(slb.logPort),
 		SiderolinkBind: maps.ToSlice(slb.binds, func(k uuid.UUID, v netip.Addr) provision.SiderolinkBind {
@@ -1351,6 +1381,24 @@ func (slb *siderolinkBuilder) SiderolinkRequest() provision.SiderolinkRequest {
 	}
 }
 
+// TrustedRootsConfig returns the trusted roots config for the current builder.
+func (slb *siderolinkBuilder) TrustedRootsConfig() []byte {
+	if slb == nil || slb.apiCert == nil {
+		return nil
+	}
+
+	trustedRootsConfig := security.NewTrustedRootsConfigV1Alpha1()
+	trustedRootsConfig.MetaName = "siderolink-ca"
+	trustedRootsConfig.Certificates = string(slb.apiCert)
+
+	marshaled, err := encoder.NewEncoder(trustedRootsConfig, encoder.WithComments(encoder.CommentsDisabled)).Encode()
+	if err != nil {
+		panic(fmt.Sprintf("failed to marshal trusted roots config: %s", err))
+	}
+
+	return marshaled
+}
+
 // SetKernelArgs sets the kernel arguments for the current builder. It is safe to call this method on a nil pointer.
 func (slb *siderolinkBuilder) SetKernelArgs(extraKernelArgs *procfs.Cmdline, tunnel bool) error {
 	switch {
@@ -1361,7 +1409,13 @@ func (slb *siderolinkBuilder) SetKernelArgs(extraKernelArgs *procfs.Cmdline, tun
 		extraKernelArgs.Get("talos.logging.kernel") != nil:
 		return errors.New("siderolink kernel arguments are already set, cannot run with --with-siderolink")
 	default:
-		apiLink := "grpc://" + net.JoinHostPort(slb.wgHost, strconv.Itoa(slb.apiPort)) + "?jointoken=foo"
+		scheme := "grpc://"
+
+		if slb.apiCert != nil {
+			scheme = "https://"
+		}
+
+		apiLink := scheme + net.JoinHostPort(slb.wgHost, strconv.Itoa(slb.apiPort)) + "?jointoken=foo"
 
 		if tunnel {
 			apiLink += "&grpc_tunnel=true"
@@ -1371,6 +1425,26 @@ func (slb *siderolinkBuilder) SetKernelArgs(extraKernelArgs *procfs.Cmdline, tun
 		extraKernelArgs.Append("talos.events.sink", net.JoinHostPort(slb.nodeIPv6Addr, strconv.Itoa(slb.sinkPort)))
 		extraKernelArgs.Append("talos.logging.kernel", "tcp://"+net.JoinHostPort(slb.nodeIPv6Addr, strconv.Itoa(slb.logPort)))
 
+		if trustedRootsConfig := slb.TrustedRootsConfig(); trustedRootsConfig != nil {
+			var buf bytes.Buffer
+
+			zencoder, err := zstd.NewWriter(&buf)
+			if err != nil {
+				return fmt.Errorf("failed to create zstd encoder: %w", err)
+			}
+
+			_, err = zencoder.Write(trustedRootsConfig)
+			if err != nil {
+				return fmt.Errorf("failed to write zstd data: %w", err)
+			}
+
+			if err = zencoder.Close(); err != nil {
+				return fmt.Errorf("failed to close zstd encoder: %w", err)
+			}
+
+			extraKernelArgs.Append(constants.KernelParamConfigInline, base64.StdEncoding.EncodeToString(buf.Bytes()))
+		}
+
 		return nil
 	}
 }
@@ -1444,6 +1518,10 @@ func (a *agentFlag) String() string {
 		return "wireguard"
 	case 2:
 		return "grpc-tunnel"
+	case 3:
+		return "wireguard+tls"
+	case 4:
+		return "grpc-tunnel+tls"
 	default:
 		return "none"
 	}
@@ -1455,13 +1533,18 @@ func (a *agentFlag) Set(s string) error {
 		*a = 1
 	case "tunnel":
 		*a = 2
+	case "wireguard+tls":
+		*a = 3
+	case "grpc-tunnel+tls":
+		*a = 4
 	default:
-		return fmt.Errorf("unknown type: %s, possible values: 'true', 'wireguard' for the usual WG; 'tunnel' for WG over GRPC", s)
+		return fmt.Errorf("unknown type: %s, possible values: 'true', 'wireguard' for the usual WG; 'tunnel' for WG over GRPC, add '+tls' to enable TLS for API", s)
 	}
 
 	return nil
 }
 
 func (a *agentFlag) Type() string    { return "agent" }
 func (a *agentFlag) IsEnabled() bool { return *a != 0 }
-func (a *agentFlag) IsTunnel() bool  { return *a == 2 }
+func (a *agentFlag) IsTunnel() bool  { return *a == 2 || *a == 4 }
+func (a *agentFlag) IsTLS() bool     { return *a == 3 || *a == 4 }

cmd/talosctl/cmd/mgmt/siderolink_launch_linux.go

@@ -6,6 +6,7 @@ package mgmt
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"os"
 	"os/signal"
@@ -23,6 +24,8 @@ var siderolinkFlags struct {
 	wireguardEndpoint string
 	sinkEndpoint      string
 	apiEndpoint       string
+	apiCertPath       string
+	apiKeyPath        string
 	logEndpoint       string
 	predefinedPairs   []string
 }
@@ -46,6 +49,8 @@ func init() {
 	siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.wireguardEndpoint, "sidero-link-wireguard-endpoint", "", "advertised Wireguard endpoint")
 	siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.sinkEndpoint, "event-sink-endpoint", "", "gRPC API endpoint for the Event Sink")
 	siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.apiEndpoint, "sidero-link-api-endpoint", "", "gRPC API endpoint for the SideroLink")
+	siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.apiCertPath, "sidero-link-api-cert", "", "path to the API server certificate (optional)")
+	siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.apiKeyPath, "sidero-link-api-key", "", "path to the API server key (optional)")
 	siderolinkCmd.PersistentFlags().StringVar(&siderolinkFlags.logEndpoint, "log-receiver-endpoint", "", "TCP log receiver endpoint")
 	siderolinkCmd.PersistentFlags().StringArrayVar(&siderolinkFlags.predefinedPairs, "predefined-pair", nil, "predefined pairs of UUID=IPv6 addrs for the nodes")
 
@@ -68,11 +73,25 @@ func run(ctx context.Context) error {
 	logger.Info("starting embedded siderolink agent")
 	defer logger.Info("stopping embedded siderolink agent")
 
+	var apiTLSConfig *tls.Config
+
+	if siderolinkFlags.apiCertPath != "" && siderolinkFlags.apiKeyPath != "" {
+		apiCert, err := tls.LoadX509KeyPair(siderolinkFlags.apiCertPath, siderolinkFlags.apiKeyPath)
+		if err != nil {
+			return fmt.Errorf("failed to load API server certificate: %w", err)
+		}
+
+		apiTLSConfig = &tls.Config{
+			Certificates: []tls.Certificate{apiCert},
+		}
+	}
+
 	err = agent.Run(
 		ctx,
 		agent.Config{
 			WireguardEndpoint: siderolinkFlags.wireguardEndpoint,
 			APIEndpoint:       siderolinkFlags.apiEndpoint,
+			APITLSConfig:      apiTLSConfig,
 			JoinToken:         siderolinkFlags.joinToken,
 			SinkEndpoint:      siderolinkFlags.sinkEndpoint,
 			LogEndpoint:       siderolinkFlags.logEndpoint,

go.mod

@@ -152,7 +152,7 @@ require (
 	github.com/siderolabs/kms-client v0.1.0
 	github.com/siderolabs/net v0.4.0
 	github.com/siderolabs/protoenc v0.2.1
-	github.com/siderolabs/siderolink v0.3.9
+	github.com/siderolabs/siderolink v0.3.10
 	github.com/siderolabs/talos/pkg/machinery v1.8.0-alpha.2
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -629,8 +629,8 @@ github.com/siderolabs/net v0.4.0 h1:1bOgVay/ijPkJz4qct98nHsiB/ysLQU0KLoBC4qLm7I=
 github.com/siderolabs/net v0.4.0/go.mod h1:/ibG+Hm9HU27agp5r9Q3eZicEfjquzNzQNux5uEk0kM=
 github.com/siderolabs/protoenc v0.2.1 h1:BqxEmeWQeMpNP3R6WrPqDatX8sM/r4t97OP8mFmg6GA=
 github.com/siderolabs/protoenc v0.2.1/go.mod h1:StTHxjet1g11GpNAWiATgc8K0HMKiFSEVVFOa/H0otc=
-github.com/siderolabs/siderolink v0.3.9 h1:lvHFCu+CdfUyMk90g1Zt5r7n1Dw3jhXMxyzXmQ0776o=
-github.com/siderolabs/siderolink v0.3.9/go.mod h1:QbGnXpHI5MDq6qMZkCFnxYOOw5eE+lkLx53L5ZgjLMQ=
+github.com/siderolabs/siderolink v0.3.10 h1:M8OrRyfzmyyGksHalOqvRSxvb1Fwi7S3AFQx6ERap44=
+github.com/siderolabs/siderolink v0.3.10/go.mod h1:QbGnXpHI5MDq6qMZkCFnxYOOw5eE+lkLx53L5ZgjLMQ=
 github.com/siderolabs/tcpproxy v0.1.0 h1:IbkS9vRhjMOscc1US3M5P1RnsGKFgB6U5IzUk+4WkKA=
 github.com/siderolabs/tcpproxy v0.1.0/go.mod h1:onn6CPPj/w1UNqQ0U97oRPF0CqbrgEApYCw4P9IiCW8=
 github.com/siderolabs/wgctrl-go v0.0.0-20240401105613-579af3342774 h1:wLhs5zMQVjA6LN9WpF2owOdtcoRp40zL8AaQSle+9EE=

internal/app/machined/pkg/controllers/siderolink/manager.go

@@ -30,6 +30,7 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 
 	networkutils "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/network/utils"
+	"github.com/siderolabs/talos/pkg/httpdefaults"
 	"github.com/siderolabs/talos/pkg/machinery/constants"
 	"github.com/siderolabs/talos/pkg/machinery/nethelpers"
 	"github.com/siderolabs/talos/pkg/machinery/resources/config"
@@ -487,7 +488,9 @@ func withTransportCredentials(insec bool) grpc.DialOption {
 	if insec {
 		transportCredentials = insecure.NewCredentials()
 	} else {
-		transportCredentials = credentials.NewTLS(&tls.Config{})
+		transportCredentials = credentials.NewTLS(&tls.Config{
+			RootCAs: httpdefaults.RootCAs(),
+		})
 	}
 
 	return grpc.WithTransportCredentials(transportCredentials)

internal/pkg/encryption/keys/kms.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/siderolabs/talos/internal/pkg/encryption/helpers"
 	"github.com/siderolabs/talos/internal/pkg/endpoint"
+	"github.com/siderolabs/talos/pkg/httpdefaults"
 )
 
 // KMSToken is the userdata stored in the partition token metadata.
@@ -130,7 +131,9 @@ func (h *KMSKeyHandler) getConn() (*grpc.ClientConn, error) {
 	if endpoint.Insecure {
 		transportCredentials = insecure.NewCredentials()
 	} else {
-		transportCredentials = credentials.NewTLS(&tls.Config{})
+		transportCredentials = credentials.NewTLS(&tls.Config{
+			RootCAs: httpdefaults.RootCAs(),
+		})
 	}
 
 	return grpc.NewClient(