From 2c50feac6688499b29ca00de421db25485ae45b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Maintrot?=
<3097030+ImpSy@users.noreply.github.com>
Date: Wed, 20 Jan 2021 01:45:26 +0100
Subject: [PATCH] add hostAliases for SparkPodSpec (#1133)
* add hostAliases for SparkPodSpec and update codegen files
* add webhook patcher for HostAliases + add tests
* update chart CRDs
* update api doc
---
...tor.k8s.io_scheduledsparkapplications.yaml | 22 +++++
...parkoperator.k8s.io_sparkapplications.yaml | 22 +++++
docs/api-docs.md | 14 +++
.../sparkoperator.k8s.io/v1beta2/types.go | 3 +
.../v1beta2/zz_generated.deepcopy.go | 7 ++
pkg/webhook/patch.go | 26 ++++++
pkg/webhook/patch_test.go | 91 +++++++++++++++++++
7 files changed, 185 insertions(+)
diff --git a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
index bd5b4f6a1..279b6e09c 100644
--- a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
+++ b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
@@ -508,6 +508,17 @@ spec:
- name
- quantity
type: object
+ hostAliases:
+ items:
+ properties:
+ hostnames:
+ items:
+ type: string
+ type: array
+ ip:
+ type: string
+ type: object
+ type: array
hostNetwork:
type: boolean
image:
@@ -2307,6 +2318,17 @@ spec:
- name
- quantity
type: object
+ hostAliases:
+ items:
+ properties:
+ hostnames:
+ items:
+ type: string
+ type: array
+ ip:
+ type: string
+ type: object
+ type: array
hostNetwork:
type: boolean
image:
diff --git a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
index e3cc7043b..45f97135b 100644
--- a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
+++ b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
@@ -494,6 +494,17 @@ spec:
- name
- quantity
type: object
+ hostAliases:
+ items:
+ properties:
+ hostnames:
+ items:
+ type: string
+ type: array
+ ip:
+ type: string
+ type: object
+ type: array
hostNetwork:
type: boolean
image:
@@ -2293,6 +2304,17 @@ spec:
- name
- quantity
type: object
+ hostAliases:
+ items:
+ properties:
+ hostnames:
+ items:
+ type: string
+ type: array
+ ip:
+ type: string
+ type: object
+ type: array
hostNetwork:
type: boolean
image:
diff --git a/docs/api-docs.md b/docs/api-docs.md
index 9f2faf677..7a26539ca 100644
--- a/docs/api-docs.md
+++ b/docs/api-docs.md
@@ -2780,6 +2780,20 @@ string
ServiceAccount is the name of the custom Kubernetes service account used by the pod.
+
+
+hostAliases
+
+
+[]Kubernetes core/v1.HostAlias
+
+
+ |
+
+(Optional)
+ HostAliases settings for the pod, following the Kubernetes specifications.
+ |
+
SparkUIConfiguration
diff --git a/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go b/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go
index 88009f48c..16474c671 100644
--- a/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go
+++ b/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go
@@ -515,6 +515,9 @@ type SparkPodSpec struct {
// ServiceAccount is the name of the custom Kubernetes service account used by the pod.
// +optional
ServiceAccount *string `json:"serviceAccount,omitempty"`
+ // HostAliases settings for the pod, following the Kubernetes specifications.
+ // +optional
+ HostAliases []apiv1.HostAlias `json:"hostAliases,omitempty"`
}
// DriverSpec is specification of the driver.
diff --git a/pkg/apis/sparkoperator.k8s.io/v1beta2/zz_generated.deepcopy.go b/pkg/apis/sparkoperator.k8s.io/v1beta2/zz_generated.deepcopy.go
index ce8ef614e..662b2766d 100644
--- a/pkg/apis/sparkoperator.k8s.io/v1beta2/zz_generated.deepcopy.go
+++ b/pkg/apis/sparkoperator.k8s.io/v1beta2/zz_generated.deepcopy.go
@@ -940,6 +940,13 @@ func (in *SparkPodSpec) DeepCopyInto(out *SparkPodSpec) {
*out = new(string)
**out = **in
}
+ if in.HostAliases != nil {
+ in, out := &in.HostAliases, &out.HostAliases
+ *out = make([]v1.HostAlias, len(*in))
+ for i := range *in {
+ (*in)[i].DeepCopyInto(&(*out)[i])
+ }
+ }
return
}
diff --git a/pkg/webhook/patch.go b/pkg/webhook/patch.go
index 18c72a725..d482c4310 100644
--- a/pkg/webhook/patch.go
+++ b/pkg/webhook/patch.go
@@ -62,6 +62,7 @@ func patchSparkPod(pod *corev1.Pod, app *v1beta2.SparkApplication) []patchOperat
patchOps = append(patchOps, addDNSConfig(pod, app)...)
patchOps = append(patchOps, addEnvVars(pod, app)...)
patchOps = append(patchOps, addEnvFrom(pod, app)...)
+ patchOps = append(patchOps, addHostAliases(pod, app)...)
op := addSchedulerName(pod, app)
if op != nil {
@@ -783,3 +784,28 @@ func findContainer(pod *corev1.Pod) int {
}
return -1
}
+
+func addHostAliases(pod *corev1.Pod, app *v1beta2.SparkApplication) []patchOperation {
+ var hostAliases []corev1.HostAlias
+ if util.IsDriverPod(pod) {
+ hostAliases = app.Spec.Driver.HostAliases
+ } else if util.IsExecutorPod(pod) {
+ hostAliases = app.Spec.Executor.HostAliases
+ }
+
+ first := false
+ if len(pod.Spec.HostAliases) == 0 {
+ first = true
+ }
+
+ var ops []patchOperation
+ for _, v := range hostAliases {
+ if first {
+ ops = append(ops, patchOperation{Op: "add", Path: "/spec/hostAliases", Value: []corev1.HostAlias{v}})
+ first = false
+ } else {
+ ops = append(ops, patchOperation{Op: "add", Path: "/spec/hostAliases/-", Value: &v})
+ }
+ }
+ return ops
+}
diff --git a/pkg/webhook/patch_test.go b/pkg/webhook/patch_test.go
index da329fcec..545c33e1a 100644
--- a/pkg/webhook/patch_test.go
+++ b/pkg/webhook/patch_test.go
@@ -1811,3 +1811,94 @@ func getModifiedPod(pod *corev1.Pod, app *v1beta2.SparkApplication) (*corev1.Pod
return modifiedPod, nil
}
+
+func TestPatchSparkPod_HostAliases(t *testing.T) {
+ app := &v1beta2.SparkApplication{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "spark-test",
+ UID: "spark-test-1",
+ },
+ Spec: v1beta2.SparkApplicationSpec{
+ Driver: v1beta2.DriverSpec{
+ SparkPodSpec: v1beta2.SparkPodSpec{
+ HostAliases: []corev1.HostAlias{
+ {
+ IP: "127.0.0.1",
+ Hostnames: []string{"localhost"},
+ },
+ {
+ IP: "192.168.0.1",
+ Hostnames: []string{"test.com", "test2.com"},
+ },
+ },
+ },
+ },
+ Executor: v1beta2.ExecutorSpec{
+ SparkPodSpec: v1beta2.SparkPodSpec{
+ HostAliases: []corev1.HostAlias{
+ {
+ IP: "127.0.0.1",
+ Hostnames: []string{"localhost"},
+ },
+ {
+ IP: "192.168.0.1",
+ Hostnames: []string{"test.com", "test2.com"},
+ },
+ },
+ },
+ },
+ },
+ }
+
+ driverPod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "spark-driver",
+ Labels: map[string]string{
+ config.SparkRoleLabel: config.SparkDriverRole,
+ config.LaunchedBySparkOperatorLabel: "true",
+ },
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: config.SparkDriverContainerName,
+ Image: "spark-driver:latest",
+ },
+ },
+ },
+ }
+
+ modifiedDriverPod, err := getModifiedPod(driverPod, app)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, 2, len(modifiedDriverPod.Spec.HostAliases))
+ assert.Equal(t, "127.0.0.1", modifiedDriverPod.Spec.HostAliases[0].IP)
+ assert.Equal(t, "192.168.0.1", modifiedDriverPod.Spec.HostAliases[1].IP)
+
+ executorPod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "spark-executor",
+ Labels: map[string]string{
+ config.SparkRoleLabel: config.SparkExecutorRole,
+ config.LaunchedBySparkOperatorLabel: "true",
+ },
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: config.SparkExecutorContainerName,
+ Image: "spark-executor:latest",
+ },
+ },
+ },
+ }
+
+ modifiedExecutorPod, err := getModifiedPod(executorPod, app)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, 2, len(modifiedExecutorPod.Spec.HostAliases))
+ assert.Equal(t, "127.0.0.1", modifiedExecutorPod.Spec.HostAliases[0].IP)
+ assert.Equal(t, "192.168.0.1", modifiedExecutorPod.Spec.HostAliases[1].IP)
+}