refactor: e2e infras (#407)

Signed-off-by: bitliu <bitliu@tencent.com>

test/e2e/conformance/utils/kubernetes/apply.go

@@ -0,0 +1,218 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubernetes
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	ingress "github.com/alibaba/higress/test/e2e/conformance"
+	"github.com/stretchr/testify/require"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/yaml"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/alibaba/higress/test/e2e/conformance/utils/config"
+)
+
+// Applier prepares manifests depending on the available options and applies
+// them to the Kubernetes cluster.
+type Applier struct {
+	NamespaceLabels map[string]string
+
+	// IngressClass will be used as the spec.ingressClassName when applying Ingress resources
+	IngressClass string
+}
+
+// prepareNamespace adjusts the Namespace labels.
+func prepareNamespace(t *testing.T, uObj *unstructured.Unstructured, namespaceLabels map[string]string) {
+	labels, _, err := unstructured.NestedStringMap(uObj.Object, "metadata", "labels")
+	require.NoErrorf(t, err, "error getting labels on Namespace %s", uObj.GetName())
+
+	for k, v := range namespaceLabels {
+		if labels == nil {
+			labels = map[string]string{}
+		}
+
+		labels[k] = v
+	}
+
+	// SetNestedStringMap converts nil to an empty map
+	if labels != nil {
+		err = unstructured.SetNestedStringMap(uObj.Object, labels, "metadata", "labels")
+	}
+	require.NoErrorf(t, err, "error setting labels on Namespace %s", uObj.GetName())
+}
+
+// prepareResources uses the options from an Applier to tweak resources given by
+// a set of manifests.
+func (a Applier) prepareResources(t *testing.T, decoder *yaml.YAMLOrJSONDecoder) ([]unstructured.Unstructured, error) {
+	var resources []unstructured.Unstructured
+
+	for {
+		uObj := unstructured.Unstructured{}
+		if err := decoder.Decode(&uObj); err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return nil, err
+		}
+		if len(uObj.Object) == 0 {
+			continue
+		}
+
+		if uObj.GetKind() == "Namespace" && uObj.GetObjectKind().GroupVersionKind().Group == "" {
+			prepareNamespace(t, &uObj, a.NamespaceLabels)
+		}
+
+		resources = append(resources, uObj)
+	}
+
+	return resources, nil
+}
+
+func (a Applier) MustApplyObjectsWithCleanup(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, resources []client.Object, cleanup bool) {
+	for _, resource := range resources {
+		resource := resource
+
+		ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.CreateTimeout)
+		defer cancel()
+
+		t.Logf("Creating %s %s", resource.GetName(), resource.GetObjectKind().GroupVersionKind().Kind)
+
+		err := c.Create(ctx, resource)
+		if err != nil {
+			if !apierrors.IsAlreadyExists(err) {
+				require.NoError(t, err, "error creating resource")
+			}
+		}
+
+		if cleanup {
+			t.Cleanup(func() {
+				ctx, cancel = context.WithTimeout(context.Background(), timeoutConfig.DeleteTimeout)
+				defer cancel()
+				t.Logf("Deleting %s %s", resource.GetName(), resource.GetObjectKind().GroupVersionKind().Kind)
+				err = c.Delete(ctx, resource)
+				require.NoErrorf(t, err, "error deleting resource")
+			})
+		}
+	}
+}
+
+// MustApplyWithCleanup creates or updates Kubernetes resources defined with the
+// provided YAML file and registers a cleanup function for resources it created.
+// Note that this does not remove resources that already existed in the cluster.
+func (a Applier) MustApplyWithCleanup(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, location string, cleanup bool) {
+	data, err := getContentsFromPathOrURL(location, timeoutConfig)
+	require.NoError(t, err)
+
+	decoder := yaml.NewYAMLOrJSONDecoder(data, 4096)
+
+	resources, err := a.prepareResources(t, decoder)
+	if err != nil {
+		t.Logf("manifest: %s", data.String())
+		require.NoErrorf(t, err, "error parsing manifest")
+	}
+
+	for i := range resources {
+		uObj := &resources[i]
+
+		ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.CreateTimeout)
+		defer cancel()
+
+		namespacedName := types.NamespacedName{Namespace: uObj.GetNamespace(), Name: uObj.GetName()}
+		fetchedObj := uObj.DeepCopy()
+		err := c.Get(ctx, namespacedName, fetchedObj)
+		if err != nil {
+			if !apierrors.IsNotFound(err) {
+				require.NoErrorf(t, err, "error getting resource")
+			}
+			t.Logf("Creating %s %s", uObj.GetName(), uObj.GetKind())
+			err = c.Create(ctx, uObj)
+			require.NoErrorf(t, err, "error creating resource")
+
+			if cleanup {
+				t.Cleanup(func() {
+					ctx, cancel = context.WithTimeout(context.Background(), timeoutConfig.DeleteTimeout)
+					defer cancel()
+					t.Logf("Deleting %s %s", uObj.GetName(), uObj.GetKind())
+					err = c.Delete(ctx, uObj)
+					require.NoErrorf(t, err, "error deleting resource")
+				})
+			}
+			continue
+		}
+
+		uObj.SetResourceVersion(fetchedObj.GetResourceVersion())
+		t.Logf("Updating %s %s", uObj.GetName(), uObj.GetKind())
+		err = c.Update(ctx, uObj)
+
+		if cleanup {
+			t.Cleanup(func() {
+				ctx, cancel = context.WithTimeout(context.Background(), timeoutConfig.DeleteTimeout)
+				defer cancel()
+				t.Logf("Deleting %s %s", uObj.GetName(), uObj.GetKind())
+				err = c.Delete(ctx, uObj)
+				require.NoErrorf(t, err, "error deleting resource")
+			})
+		}
+		require.NoErrorf(t, err, "error updating resource")
+	}
+}
+
+// getContentsFromPathOrURL takes a string that can either be a local file
+// path or an https:// URL to YAML manifests and provides the contents.
+func getContentsFromPathOrURL(location string, timeoutConfig config.TimeoutConfig) (*bytes.Buffer, error) {
+	if strings.HasPrefix(location, "http://") {
+		return nil, fmt.Errorf("data can't be retrieved from %s: http is not supported, use https", location)
+	} else if strings.HasPrefix(location, "https://") {
+		ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.ManifestFetchTimeout)
+		defer cancel()
+
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, location, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+
+		manifests := new(bytes.Buffer)
+		count, err := manifests.ReadFrom(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+
+		if resp.ContentLength != -1 && count != resp.ContentLength {
+			return nil, fmt.Errorf("received %d bytes from %s, expected %d", count, location, resp.ContentLength)
+		}
+		return manifests, nil
+	}
+	b, err := ingress.Manifests.ReadFile(location)
+	if err != nil {
+		return nil, err
+	}
+	return bytes.NewBuffer(b), nil
+}

test/e2e/conformance/utils/kubernetes/apply_test.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubernetes
+
+import (
+	_ "github.com/alibaba/higress/test/e2e/conformance/utils/flags"
+)

test/e2e/conformance/utils/kubernetes/cert.go

@@ -0,0 +1,123 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubernetes
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"fmt"
+	"io"
+	"math/big"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	// ensure auth plugins are loaded
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
+
+	"github.com/alibaba/higress/test/e2e/conformance/utils/cert"
+)
+
+// MustCreateSelfSignedCertSecret creates a self-signed SSL certificate and stores it in a secret
+func MustCreateSelfSignedCertSecret(t *testing.T, namespace, secretName string, hosts []string) *corev1.Secret {
+	require.Greater(t, len(hosts), 0, "require a non-empty hosts for Subject Alternate Name values")
+
+	var serverKey, serverCert bytes.Buffer
+	host := strings.Join(hosts, ",")
+	require.NoError(t, generateRSACert(host, &serverKey, &serverCert), "failed to generate RSA certificate")
+
+	return ConstructTLSSecret(namespace, secretName, serverCert.Bytes(), serverKey.Bytes())
+}
+
+// ConstructTLSSecret constructs a secret of type "kubernetes.io/tls"
+func ConstructTLSSecret(namespace, secretName string, cert, key []byte) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      secretName,
+		},
+		Type: corev1.SecretTypeTLS,
+		Data: map[string][]byte{
+			corev1.TLSCertKey:       cert,
+			corev1.TLSPrivateKeyKey: key,
+		},
+	}
+}
+
+// ConstructCASecret construct a CA secret of type "Opaque"
+func ConstructCASecret(namespace, secretName string, cert []byte) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      secretName,
+		},
+		Type: corev1.SecretTypeOpaque,
+		Data: map[string][]byte{
+			corev1.ServiceAccountRootCAKey: cert,
+		},
+	}
+}
+
+// generateRSACert generates a basic self signed certificate valid for a year
+func generateRSACert(host string, keyOut, certOut io.Writer) error {
+	notBefore := time.Now()
+	notAfter := notBefore.Add(cert.ValidFor)
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	if err != nil {
+		return fmt.Errorf("failed to generate serial number: %w", err)
+	}
+
+	template := &x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			CommonName:   "default",
+			Organization: []string{"Higress E2E Test"},
+		},
+		NotBefore: notBefore,
+		NotAfter:  notAfter,
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+
+	hosts := strings.Split(host, ",")
+	for _, h := range hosts {
+		if ip := net.ParseIP(h); ip != nil {
+			template.IPAddresses = append(template.IPAddresses, ip)
+		} else {
+			template.DNSNames = append(template.DNSNames, h)
+		}
+	}
+
+	priv, err := rsa.GenerateKey(rand.Reader, cert.RSABits)
+	if err != nil {
+		return fmt.Errorf("failed to generate key: %w", err)
+	}
+	certOut, keyOut, err = cert.GenerateCert(template, priv, template, nil)
+	if err != nil {
+		return fmt.Errorf("failed to generate rsa certificate: %w", err)
+	}
+
+	return nil
+}

test/e2e/conformance/utils/kubernetes/helpers.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubernetes
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/alibaba/higress/test/e2e/conformance/utils/config"
+)
+
+// FilterStaleConditions returns the list of status condition whos observedGeneration does not
+// match the objects metadata.Generation
+func FilterStaleConditions(obj metav1.Object, conditions []metav1.Condition) []metav1.Condition {
+	stale := make([]metav1.Condition, 0, len(conditions))
+	for _, condition := range conditions {
+		if obj.GetGeneration() != condition.ObservedGeneration {
+			stale = append(stale, condition)
+		}
+	}
+	return stale
+}
+
+// NamespacesMustBeAccepted waits until all Pods are marked ready
+// in the provided namespaces. This will cause the test to
+// halt if the specified timeout is exceeded.
+func NamespacesMustBeAccepted(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, namespaces []string) {
+	t.Helper()
+
+	waitErr := wait.PollImmediate(1*time.Second, timeoutConfig.NamespacesMustBeReady, func() (bool, error) {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+
+		for _, ns := range namespaces {
+			podList := &v1.PodList{}
+			err := c.List(ctx, podList, client.InNamespace(ns))
+			if err != nil {
+				t.Errorf("Error listing Pods: %v", err)
+			}
+			for _, pod := range podList.Items {
+				if !FindPodConditionInList(t, pod.Status.Conditions, "Ready", "True") &&
+					pod.Status.Phase != v1.PodSucceeded {
+					t.Logf("%s/%s Pod not ready yet", ns, pod.Name)
+					return false, nil
+				}
+			}
+		}
+		t.Logf("Gateways and Pods in %s namespaces ready", strings.Join(namespaces, ", "))
+		return true, nil
+	})
+	require.NoErrorf(t, waitErr, "error waiting for %s namespaces to be ready", strings.Join(namespaces, ", "))
+}
+
+func ConditionsMatch(t *testing.T, expected, actual []metav1.Condition) bool {
+	if len(actual) < len(expected) {
+		t.Logf("Expected more conditions to be present")
+		return false
+	}
+	for _, condition := range expected {
+		if !FindConditionInList(t, actual, condition.Type, string(condition.Status), condition.Reason) {
+			return false
+		}
+	}
+
+	t.Logf("Conditions matched expectations")
+	return true
+}
+
+// findConditionInList finds a condition in a list of Conditions, checking
+// the Name, Value, and Reason. If an empty reason is passed, any Reason will match.
+func FindConditionInList(t *testing.T, conditions []metav1.Condition, condName, expectedStatus, expectedReason string) bool {
+	for _, cond := range conditions {
+		if cond.Type == condName {
+			if cond.Status == metav1.ConditionStatus(expectedStatus) {
+				// an empty Reason string means "Match any reason".
+				if expectedReason == "" || cond.Reason == expectedReason {
+					return true
+				}
+				t.Logf("%s condition Reason set to %s, expected %s", condName, cond.Reason, expectedReason)
+			}
+
+			t.Logf("%s condition set to Status %s with Reason %v, expected Status %s", condName, cond.Status, cond.Reason, expectedStatus)
+		}
+	}
+
+	t.Logf("%s was not in conditions list", condName)
+	return false
+}
+
+func FindPodConditionInList(t *testing.T, conditions []v1.PodCondition, condName, condValue string) bool {
+	for _, cond := range conditions {
+		if cond.Type == v1.PodConditionType(condName) {
+			if cond.Status == v1.ConditionStatus(condValue) {
+				return true
+			}
+			t.Logf("%s condition set to %s, expected %s", condName, cond.Status, condValue)
+		}
+	}
+
+	t.Logf("%s was not in conditions list", condName)
+	return false
+}