Enable testing in OCP 4.3 (#198)

* Switch e2e tests to use the 4.3 cluster

* Disable Websphere image testing temporarily to avoid strain on cluster

.travis.yml

@@ -34,14 +34,14 @@ jobs:
     if: branch != master OR fork = true OR type = pull_request
   - name: E2E testing on Minikube
     script: make test-minikube
-  - name: E2E testing on OCP 3.11 with Open Liberty
+  - name: E2E testing on OCP 4.3 with Open Liberty
     script: make test-e2e
     env: LIBERTY_IMAGE="openliberty/open-liberty:kernel-java8-openj9-ubi"
     if: fork = false
-  - name: E2E testing on OCP 3.11 with WebSphere Liberty
-    script: make test-e2e
-    env: LIBERTY_IMAGE="ibmcom/websphere-liberty:full-java8-ibmjava-ubi"
-    if: fork = false
+  # - name: E2E testing on OCP 4.3 with WebSphere Liberty
+  #   script: make test-e2e
+  #   env: LIBERTY_IMAGE="ibmcom/websphere-liberty:full-java8-ibmjava-ubi"
+  #   if: fork = false
   ## if master branch build and push image for amd64,ppc64le,s390 to DH
   - name: Build image on amd64
     stage: build

Makefile

@@ -35,8 +35,11 @@ build: ## Compile the operator
 unit-test: ## Run unit tests
 	go test -v -mod=vendor -tags=unit github.com/OpenLiberty/open-liberty-operator/pkg/...
 
-test-e2e: setup ## Run end-to-end tests
-	./scripts/e2e.sh
+test-e2e: setup ## Run end-to-end tests on 4.3
+	./scripts/e2e.sh --cluster-url ${CLUSTER_43_URL} --cluster-token ${CLUSTER_43_TOKEN} --registry-name image-registry --registry-namespace openshift-image-registry
+
+test-e2e-legacy: setup ## Run end-to-end tests on 3.11
+	./scripts/e2e.sh --cluster-url ${CLUSTER_311_URL} --cluster-token ${CLUSTER_311_TOKEN} --registry-name docker-registry --registry-namespace default
 
 test-minikube: setup setup-minikube
 	CLUSTER_ENV=minikube operator-sdk test local github.com/OpenLiberty/open-liberty-operator/test/e2e --verbose --debug --up-local --namespace ${WATCH_NAMESPACE}

scripts/e2e.sh

@@ -10,9 +10,9 @@ login_cluster(){
     sudo mv oc kubectl /usr/local/bin/
     cd ..
     # Start a cluster and login
-    oc login $CLUSTER_URL --token=$CLUSTER_TOKEN
+    oc login $OC_URL --token=$OC_TOKEN
     # Set variables for rest of script to use
-    readonly DEFAULT_REGISTRY=$(oc get route docker-registry -o jsonpath="{ .spec.host }" -n default)
+    readonly DEFAULT_REGISTRY=$(oc get route "${REGISTRY_NAME}" -o jsonpath="{ .spec.host }" -n "${REGISTRY_NAMESPACE}")
     readonly BUILD_IMAGE=$DEFAULT_REGISTRY/openshift/operator:${TRAVIS_BUILD_NUMBER}
 }
 
@@ -23,6 +23,7 @@ cleanup() {
 }
 
 main() {
+    parse_args "$@"
     echo "****** Logging into remote cluster..."
     login_cluster
     echo "****** Logging into private registry..."
@@ -52,4 +53,34 @@ main() {
     return $result
 }
 
-main
+parse_args() {
+    while [ $# -gt 0 ]; do
+    case "$1" in
+    --cluster-url)
+      shift
+      readonly OC_URL="${1}"
+      ;;
+    --cluster-token)
+      shift
+      readonly OC_TOKEN="${1}"
+      ;;
+    --registry-name)
+      shift
+      readonly REGISTRY_NAME="${1}"
+      ;;
+    --registry-namespace)
+      shift
+      readonly REGISTRY_NAMESPACE="${1}"
+      ;;
+    *)
+      echo "Error: Invalid argument - $1"
+      echo "$usage"
+      exit 1
+      ;;
+    esac
+    shift
+  done
+}
+
+
+main "$@"

test/e2e/openliberty_certmanager.go

@@ -54,7 +54,7 @@ func OpenLibertyCertManagerTest(t *testing.T) {
 }
 
 func libertyPodCertificateTest(t *testing.T, f *framework.Framework, ctx *framework.TestCtx) error {
-	const name = "example-liberty-pod-cert"
+	const name = "l-pod-cert"
 
 	ns, err := ctx.GetNamespace()
 	if err != nil {
@@ -87,7 +87,7 @@ func libertyPodCertificateTest(t *testing.T, f *framework.Framework, ctx *framew
 }
 
 func libertyRouteCertificateTest(t *testing.T, f *framework.Framework, ctx *framework.TestCtx) error {
-	const name = "example-liberty-route-cert"
+	const name = "l-route-cert"
 
 	ns, err := ctx.GetNamespace()
 	if err != nil {
@@ -123,7 +123,7 @@ func libertyRouteCertificateTest(t *testing.T, f *framework.Framework, ctx *fram
 }
 
 func libertyAdvancedCertificateTest(t *testing.T, f *framework.Framework, ctx *framework.TestCtx) error {
-	const name = "example-liberty-advanced-cert"
+	const name = "l-advanced-cert"
 	ns, err := ctx.GetNamespace()
 	if err != nil {
 		return fmt.Errorf("could not get namespace %v", err)

test/e2e/openliberty_knative.go

@@ -12,7 +12,6 @@ import (
 	e2eutil "github.com/operator-framework/operator-sdk/pkg/test/e2eutil"
 
 	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 )
@@ -37,7 +36,7 @@ func OpenLibertyKnativeTest(t *testing.T) {
 	f := framework.Global
 
 	// catch cases where running tests locally with a cluster that does not have knative
-	if !isKnativeInstalled(t, f) {
+	if util.IsKnativeInstalled(t, f) != nil {
 		t.Log("Knative is not installed on this cluster, skipping RuntimeKnativeTest...")
 		return
 	}
@@ -53,20 +52,6 @@ func OpenLibertyKnativeTest(t *testing.T) {
 	testKnIsTrueAndTurnOff(t, f, ctx, namespace)
 }
 
-func isKnativeInstalled(t *testing.T, f *framework.Framework) bool {
-	ns := &corev1.NamespaceList{}
-	err := f.Client.List(goctx.TODO(), ns)
-	if err != nil {
-		t.Fatalf("Error occurred while trying to find knative-serving %v", err)
-	}
-	for _, val := range ns.Items {
-		if val.Name == "knative-serving" {
-			return true
-		}
-	}
-	return false
-}
-
 func testKnIsFalse(t *testing.T, f *framework.Framework, ctx *framework.TestCtx, namespace string) {
 	knativeBool := false
 	applicationName := "example-liberty-knative-0"

test/e2e/openliberty_sso.go

@@ -58,7 +58,7 @@ func testSocialLogin(t *testing.T, f *framework.Framework, ctx *framework.TestCt
 		return err
 	}
 
-	const name string = "openliberty-sso"
+	const name string = "ol-sso"
 
 	// create Secret for Github Login
 	secretTarget := types.NamespacedName{Name: name + "-olapp-sso", Namespace: ns}
@@ -211,7 +211,7 @@ func testProviderLogins(t *testing.T, f *framework.Framework, ctx *framework.Tes
 		return err
 	}
 
-	const name string = "openliberty-sso-1"
+	const name string = "ol-sso-1"
 
 	// create Secret for Github Login
 	secretTarget := types.NamespacedName{Name: name + "-olapp-sso", Namespace: ns}

test/e2e/openliberty_test.go

@@ -23,18 +23,18 @@ var (
 		{"OpenLibertyBasicTest", OpenLibertyBasicTest},
 		{"OpenLibertyProbeTest", OpenLibertyProbeTest},
 		{"OpenLibertyAutoScalingTest", OpenLibertyAutoScalingTest},
-		{"OpenLibertyStorageTest", OpenLibertyBasicStorageTest},
-		{"OpenLibertyPersistenceTest", OpenLibertyPersistenceTest},
-		{"OpenLibertyTraceTest", OpenLibertyTraceTest},
 	}
 	advancedTests = []Test{
 		{"OpenLibertyServiceMonitorTest", OpenLibertyServiceMonitorTest},
 		{"OpenLibertyKnativeTest", OpenLibertyKnativeTest},
-		{"OpenLibertyServiceBindingTest", OpenLibertyServiceBindingTest},
-		{"OpenLibertyCertManagerTest", OpenLibertyCertManagerTest},
+		{"OpenLibertyStorageTest", OpenLibertyBasicStorageTest},
+		{"OpenLibertyPersistenceTest", OpenLibertyPersistenceTest},
+		{"OpenLibertyTraceTest", OpenLibertyTraceTest},
 		{"OpenLibertyDumpsTest", OpenLibertyDumpsTest},
 		{"OpenLibertyKappNavTest", OpenLibertyKappNavTest},
-		{"OpenLibertySSOTest", OpenLibertySSOTest},
+		{"OpenLibertyServiceBindingTest", OpenLibertyServiceBindingTest},
+		// {"OpenLibertySSOTest", OpenLibertySSOTest},
+		{"OpenLibertyCertManagerTest", OpenLibertyCertManagerTest},
 	}
 	ocpTests = []Test{
 		{"OpenLibertyImageStreamTest", OpenLibertyImageStreamTest},
@@ -69,18 +69,22 @@ func TestOpenLibertyApplication(t *testing.T) {
 
 	var wg sync.WaitGroup
 
+	if cluster != "minikube" {
+		t.Run("OpenLibertySSOTest", OpenLibertySSOTest)
+	}
+
 	// basic tests that are runnable locally in minishift/kube
 	for _, test := range basicTests {
 		wg.Add(1)
-		RuntimeTestRunner(&wg, t, test)
+		go RuntimeTestRunner(&wg, t, test)
 	}
 
 	// tests for features that will require cluster configuration
 	// i.e. knative requires installations
 	if cluster != "minikube" {
 		for _, test := range advancedTests {
 			wg.Add(1)
-			go RuntimeTestRunner(&wg, t, test)
+			RuntimeTestRunner(&wg, t, test)
 		}
 	}
 
@@ -89,7 +93,7 @@ func TestOpenLibertyApplication(t *testing.T) {
 	if cluster == "minikube" || cluster == "kubernetes" {
 		for _, test := range independantTests {
 			wg.Add(1)
-			go RuntimeTestRunner(&wg, t, test)
+			RuntimeTestRunner(&wg, t, test)
 		}
 	}
 
@@ -104,28 +108,6 @@ func TestOpenLibertyApplication(t *testing.T) {
 	wg.Wait()
 }
 
-func testAdvancedFeatures(t *testing.T) {
-	// These features require a bit of configuration
-	// which makes them less ideal for quick minikube tests
-	t.Run("OpenLibertyServiceMonitorTest", OpenLibertyServiceMonitorTest)
-	t.Run("OpenLibertyKnativeTest", OpenLibertyKnativeTest)
-	t.Run("OpenLibertyServiceBindingTest", OpenLibertyServiceBindingTest)
-	t.Run("OpenLibertyCertManagerTest", OpenLibertyCertManagerTest)
-	t.Run("OpenLibertyDumpsTest", OpenLibertyDumpsTest)
-	t.Run("OpenLibertyKappNavTest", OpenLibertyKappNavTest)
-	t.Run("OpenLibertySSOTest", OpenLibertySSOTest)
-}
-
-// Verify functionality that is tied to OCP
-func testOCPFeatures(t *testing.T) {
-	t.Run("OpenLibertyImageStreamTest", OpenLibertyImageStreamTest)
-}
-
-// Verify functionality that is not expected to run on OCP
-func testIndependantFeatures(t *testing.T) {
-	// TODO: implement test for ingress
-}
-
 func RuntimeTestRunner(wg *sync.WaitGroup, t *testing.T, test Test) {
 	defer wg.Done()
 	t.Run(test.Name, test.Test)

test/util/util.go

@@ -559,9 +559,9 @@ func CreateSecretForSSO(f *framework.Framework, ctx *framework.TestCtx, target t
 
 func WaitForPodUpdates(t *testing.T, f *framework.Framework, ctx *framework.TestCtx, target types.NamespacedName, replicas int) error {
 	retryInterval := time.Second * 2
-	timeout := time.Second * 30
+	timeout := time.Second * 80
 	// give operator a moment to kick off NEW pod
-	time.Sleep(time.Second * 4)
+	time.Sleep(time.Second * 6)
 	// wait for only the new pod to be running
 	err := wait.Poll(retryInterval, timeout, func() (done bool, err error) {
 		podList, err := GetPods(f, ctx, target.Name, target.Namespace)
@@ -673,3 +673,20 @@ func CreateApplicationTarget(f *framework.Framework, ctx *framework.TestCtx, tar
 
 	return nil
 }
+
+// IsKnativeInstalled : check for knative support before running the e2e test
+func IsKnativeInstalled(t *testing.T, f *framework.Framework) error {
+	ksvc := servingv1alpha1.Service{}
+	err := servingv1alpha1.AddToScheme(f.Scheme)
+	if err != nil {
+		return err
+	}
+
+	err = f.Client.List(goctx.TODO(), &ksvc)
+	if err != nil {
+		t.Log(err)
+		return err
+	}
+
+	return nil
+}