diff --git a/.licenserc.yaml b/.licenserc.yaml index 0ff6521e2..10b7bac45 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -25,6 +25,7 @@ header: - 'CODEOWNERS' - 'VERSION' - 'tools/' + - 'test/README.md' comment: on-failure dependency: diff --git a/Makefile.core.mk b/Makefile.core.mk index a3bdf7a91..25929acca 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -196,14 +196,11 @@ kube-load-image: docker-build $(tools/kind) ## Install the EG image to a kind cl .PHONY: run-e2e-test run-e2e-test: - @echo -e "\n\033[36mRunning higress e2e tests\033[0m\n" - kubectl apply -f samples/hello-world/quickstart.yaml + @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" @echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n" kubectl wait --timeout=5m -n higress-system deployment/higress-controller --for=condition=Available @echo -e "\n\033[36mWaiting istiod to be ready...\033[0m\n" kubectl wait --timeout=5m -n istio-system deployment/istiod --for=condition=Available @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" kubectl wait --timeout=5m -n higress-system deployment/higress-gateway --for=condition=Available - - @echo -e "\n\033[36mSend request to call backend...\033[0m\n" - curl -i -v http://localhost/hello-world + go test -v -tags conformance ./test/ingress/e2e_test.go --ingress-class=higress --debug=true --use-unique-ports=true diff --git a/test/README.md b/test/README.md new file mode 100644 index 000000000..f84e7492f --- /dev/null +++ b/test/README.md @@ -0,0 +1,42 @@ +# Higress E2E Tests + +Higress e2e tests are mainly focusing on two parts for now: + ++ Conformance Test for Ingress API ++ Conformance Test for Gateway API + +## Ingress APIs Conformance Tests + +Higress provides make target to run ingress api conformance tests: `make e2e-test`. It can be divided into below steps: + +1. delete-cluster: checks if we have undeleted kind cluster. +2. create-cluster: create a new kind cluster. +3. kube-load-image: build a dev image of higress, and load it into kind cluster. +4. install-dev: install higress-controller with dev image, and latest higress-gateway, istiod with helm. +5. run-e2e-test: + 1. Setup conformance suite, like define what conformance tests we want to run, in `e2e_test.go` / `higressTests Slice`. Each case we choose to open is difined in `test/ingress/conformance/tests`. + 2. Prepare resources and install them into cluster, like backend services/deployments. + 3. Load conformance tests we choose to open in `e2e_test.go` / `higressTests Slice`, and run them one by one, fail if it is not expected. + +### How to write a test case + +To add a new test case, you firstly need to add `xxx.go` and `xxx.yaml` in `test/ingress/conformance/tests`. `xxx.yaml` is the Ingress resource you need to apply in the cluster, `xxx.go` defines the HigressConformanceTest. + +And after that, you should add your defined HigressConformanceTest to `e2e_test.go` / `higressTests Slice`. + +You can understand it quickly just by looking at codes in `test/ingress/conformance/tests/httproute-simple-same-namespace.go` and `test/ingress/conformance/tests/httproute-simple-same-namespace.yaml`, and try to write one. + +## Gateway APIs Conformance Tests + +Gateway API Conformance tests are based on the suite provided by `kubernetes-sig/gateway-api`, we can reuse that, +and descide what conformance tests we need to open. Conformance tests of Gateway API. + +This API covers a broad set of features and use cases and has been implemented widely. +This combination of both a large feature set and variety of implementations requires +clear conformance definitions and tests to ensure the API provides a consistent experience wherever it is used. + +Gateway API includes a set of conformance tests. These create a series of Gateways and Routes with the specified +GatewayClass, and test that the implementation matches the API specification. + +Each release contains a set of conformance tests, these will continue to expand as the API evolves. +Currently conformance tests cover the majority of Core capabilities in the standard channel, in addition to some Extended capabilities. diff --git a/test/gateway/e2e.go b/test/gateway/e2e.go new file mode 100644 index 000000000..11c3103d1 --- /dev/null +++ b/test/gateway/e2e.go @@ -0,0 +1,15 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gateway diff --git a/test/gateway/e2e_test.go b/test/gateway/e2e_test.go new file mode 100644 index 000000000..11c3103d1 --- /dev/null +++ b/test/gateway/e2e_test.go @@ -0,0 +1,15 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gateway diff --git a/test/ingress/conformance/base/manifests.yaml b/test/ingress/conformance/base/manifests.yaml new file mode 100644 index 000000000..9b0a7ae9d --- /dev/null +++ b/test/ingress/conformance/base/manifests.yaml @@ -0,0 +1,320 @@ +# Copyright (c) 2022 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file contains the base resources that most conformance tests will rely +# on. This includes 3 namespaces along with Gateways, Services and Deployments +# that can be used as backends for routing traffic. The most important +# resources included are the Gateways (all in the higress-conformance-infra +# namespace): +# - same-namespace (only supports route in same ns) +# - all-namespaces (supports routes in all ns) +# - backend-namespaces (supports routes in ns with backend label) + +apiVersion: v1 +kind: Namespace +metadata: + name: higress-conformance-infra + labels: + higress-conformance: infra +--- +apiVersion: v1 +kind: Service +metadata: + name: infra-backend-v1 + namespace: higress-conformance-infra +spec: + selector: + app: infra-backend-v1 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: infra-backend-v1 + namespace: higress-conformance-infra + labels: + app: infra-backend-v1 +spec: + replicas: 2 + selector: + matchLabels: + app: infra-backend-v1 + template: + metadata: + labels: + app: infra-backend-v1 + spec: + containers: + - name: infra-backend-v1 + # From https://github.com/kubernetes-sigs/ingress-controller-conformance/tree/master/images/echoserver + image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echoserver:v20221109-7ee2f3e + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 10m +--- +apiVersion: v1 +kind: Service +metadata: + name: infra-backend-v2 + namespace: higress-conformance-infra +spec: + selector: + app: infra-backend-v2 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: infra-backend-v2 + namespace: higress-conformance-infra + labels: + app: infra-backend-v2 +spec: + replicas: 2 + selector: + matchLabels: + app: infra-backend-v2 + template: + metadata: + labels: + app: infra-backend-v2 + spec: + containers: + - name: infra-backend-v2 + image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echoserver:v20221109-7ee2f3e + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 10m +--- +apiVersion: v1 +kind: Service +metadata: + name: infra-backend-v3 + namespace: higress-conformance-infra +spec: + selector: + app: infra-backend-v3 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: infra-backend-v3 + namespace: higress-conformance-infra + labels: + app: infra-backend-v3 +spec: + replicas: 2 + selector: + matchLabels: + app: infra-backend-v3 + template: + metadata: + labels: + app: infra-backend-v3 + spec: + containers: + - name: infra-backend-v3 + image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echoserver:v20221109-7ee2f3e + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 10m +--- +apiVersion: v1 +kind: Namespace +metadata: + name: higress-conformance-app-backend + labels: + higress-conformance: backend +--- +apiVersion: v1 +kind: Service +metadata: + name: app-backend-v1 + namespace: higress-conformance-app-backend +spec: + selector: + app: app-backend-v1 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app-backend-v1 + namespace: higress-conformance-app-backend + labels: + app: app-backend-v1 +spec: + replicas: 2 + selector: + matchLabels: + app: app-backend-v1 + template: + metadata: + labels: + app: app-backend-v1 + spec: + containers: + - name: app-backend-v1 + image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echoserver:v20221109-7ee2f3e + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 10m +--- +apiVersion: v1 +kind: Service +metadata: + name: app-backend-v2 + namespace: higress-conformance-app-backend +spec: + selector: + app: app-backend-v2 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app-backend-v2 + namespace: higress-conformance-app-backend + labels: + app: app-backend-v2 +spec: + replicas: 2 + selector: + matchLabels: + app: app-backend-v2 + template: + metadata: + labels: + app: app-backend-v2 + spec: + containers: + - name: app-backend-v2 + image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echoserver:v20221109-7ee2f3e + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 10m +--- +apiVersion: v1 +kind: Namespace +metadata: + name: higress-conformance-web-backend + labels: + higress-conformance: backend +--- +apiVersion: v1 +kind: Service +metadata: + name: web-backend + namespace: higress-conformance-web-backend +spec: + selector: + app: web-backend + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web-backend + namespace: higress-conformance-web-backend + labels: + app: web-backend +spec: + replicas: 2 + selector: + matchLabels: + app: web-backend + template: + metadata: + labels: + app: web-backend + spec: + containers: + - name: web-backend + image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echoserver:v20221109-7ee2f3e + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: + requests: + cpu: 10m diff --git a/test/ingress/conformance/embed.go b/test/ingress/conformance/embed.go new file mode 100644 index 000000000..b1c8bc54d --- /dev/null +++ b/test/ingress/conformance/embed.go @@ -0,0 +1,20 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingress + +import "embed" + +//go:embed tests/* base/* +var Manifests embed.FS diff --git a/test/ingress/conformance/tests/httproute-simple-same-namespace.go b/test/ingress/conformance/tests/httproute-simple-same-namespace.go new file mode 100644 index 000000000..c8f4dff86 --- /dev/null +++ b/test/ingress/conformance/tests/httproute-simple-same-namespace.go @@ -0,0 +1,43 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "testing" + + "github.com/alibaba/higress/test/ingress/conformance/utils/http" + "github.com/alibaba/higress/test/ingress/conformance/utils/suite" +) + +func init() { + HigressConformanceTests = append(HigressConformanceTests, HTTPRouteSimpleSameNamespace) +} + +var HTTPRouteSimpleSameNamespace = suite.ConformanceTest{ + ShortName: "HTTPRouteSimpleSameNamespace", + Description: "A single Ingress in the higress-conformance-infra namespace attaches to a Gateway in the same namespace", + Manifests: []string{"tests/httproute-simple-same-namespace.yaml"}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + + t.Run("Simple HTTP request should reach infra-backend", func(t *testing.T) { + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, suite.GatewayAddress, http.ExpectedResponse{ + Request: http.Request{Path: "/hello-world"}, + Response: http.Response{StatusCode: 200}, + Backend: "infra-backend-v1", + Namespace: "higress-conformance-infra", + }) + }) + }, +} diff --git a/test/ingress/conformance/tests/httproute-simple-same-namespace.yaml b/test/ingress/conformance/tests/httproute-simple-same-namespace.yaml new file mode 100644 index 000000000..8734775a6 --- /dev/null +++ b/test/ingress/conformance/tests/httproute-simple-same-namespace.yaml @@ -0,0 +1,30 @@ +# Copyright (c) 2022 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: higress-conformance-infra-test + namespace: higress-conformance-infra +spec: + rules: + - http: + paths: + - pathType: Prefix + path: "/hello-world" + backend: + service: + name: infra-backend-v1 + port: + number: 8080 diff --git a/test/ingress/conformance/tests/main.go b/test/ingress/conformance/tests/main.go new file mode 100644 index 000000000..43fce7faa --- /dev/null +++ b/test/ingress/conformance/tests/main.go @@ -0,0 +1,19 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import "github.com/alibaba/higress/test/ingress/conformance/utils/suite" + +var HigressConformanceTests []suite.ConformanceTest diff --git a/test/ingress/conformance/utils/config/timeout.go b/test/ingress/conformance/utils/config/timeout.go new file mode 100644 index 000000000..8483096b2 --- /dev/null +++ b/test/ingress/conformance/utils/config/timeout.go @@ -0,0 +1,133 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import "time" + +type TimeoutConfig struct { + // CreateTimeout represents the maximum time for a Kubernetes object to be created. + // Max value for conformant implementation: None + CreateTimeout time.Duration + + // DeleteTimeout represents the maximum time for a Kubernetes object to be deleted. + // Max value for conformant implementation: None + DeleteTimeout time.Duration + + // GetTimeout represents the maximum time to get a Kubernetes object. + // Max value for conformant implementation: None + GetTimeout time.Duration + + // GatewayMustHaveAddress represents the maximum time for at least one IP Address has been set in the status of a Gateway. + // Max value for conformant implementation: None + GatewayMustHaveAddress time.Duration + + // GatewayStatusMustHaveListeners represents the maximum time for a Gateway to have listeners in status that match the expected listeners. + // Max value for conformant implementation: None + GatewayStatusMustHaveListeners time.Duration + + // GWCMustBeAccepted represents the maximum time for a GatewayClass to have an Accepted condition set to true. + // Max value for conformant implementation: None + GWCMustBeAccepted time.Duration + + // HTTPRouteMustNotHaveParents represents the maximum time for an HTTPRoute to have either no parents or a single parent that is not accepted. + // Max value for conformant implementation: None + HTTPRouteMustNotHaveParents time.Duration + + // HTTPRouteMustHaveCondition represents the maximum time for an HTTPRoute to have the supplied Condition. + // Max value for conformant implementation: None + HTTPRouteMustHaveCondition time.Duration + + // HTTPRouteMustHaveParents represents the maximum time for an HTTPRoute to have parents in status that match the expected parents. + // Max value for conformant implementation: None + HTTPRouteMustHaveParents time.Duration + + // ManifestFetchTimeout represents the maximum time for getting content from a https:// URL. + // Max value for conformant implementation: None + ManifestFetchTimeout time.Duration + + // MaxTimeToConsistency is the maximum time for requiredConsecutiveSuccesses (default 3) requests to succeed in a row before failing the test. + // Max value for conformant implementation: 30 seconds + MaxTimeToConsistency time.Duration + + // NamespacesMustBeReady represents the maximum time for all Pods and Gateways in a namespaces to be marked as ready. + // Max value for conformant implementation: None + NamespacesMustBeReady time.Duration + + // RequestTimeout represents the maximum time for making an HTTP Request with the roundtripper. + // Max value for conformant implementation: None + RequestTimeout time.Duration +} + +// DefaultTimeoutConfig populates a TimeoutConfig with the default values. +func DefaultTimeoutConfig() TimeoutConfig { + return TimeoutConfig{ + CreateTimeout: 60 * time.Second, + DeleteTimeout: 10 * time.Second, + GetTimeout: 10 * time.Second, + GatewayMustHaveAddress: 180 * time.Second, + GatewayStatusMustHaveListeners: 60 * time.Second, + GWCMustBeAccepted: 180 * time.Second, + HTTPRouteMustNotHaveParents: 60 * time.Second, + HTTPRouteMustHaveCondition: 60 * time.Second, + HTTPRouteMustHaveParents: 60 * time.Second, + ManifestFetchTimeout: 10 * time.Second, + MaxTimeToConsistency: 30 * time.Second, + NamespacesMustBeReady: 300 * time.Second, + RequestTimeout: 10 * time.Second, + } +} + +func SetupTimeoutConfig(timeoutConfig *TimeoutConfig) { + defaultTimeoutConfig := DefaultTimeoutConfig() + if timeoutConfig.CreateTimeout == 0 { + timeoutConfig.CreateTimeout = defaultTimeoutConfig.CreateTimeout + } + if timeoutConfig.DeleteTimeout == 0 { + timeoutConfig.DeleteTimeout = defaultTimeoutConfig.DeleteTimeout + } + if timeoutConfig.GetTimeout == 0 { + timeoutConfig.GetTimeout = defaultTimeoutConfig.GetTimeout + } + if timeoutConfig.GatewayMustHaveAddress == 0 { + timeoutConfig.GatewayMustHaveAddress = defaultTimeoutConfig.GatewayMustHaveAddress + } + if timeoutConfig.GatewayStatusMustHaveListeners == 0 { + timeoutConfig.GatewayStatusMustHaveListeners = defaultTimeoutConfig.GatewayStatusMustHaveListeners + } + if timeoutConfig.GWCMustBeAccepted == 0 { + timeoutConfig.GWCMustBeAccepted = defaultTimeoutConfig.GWCMustBeAccepted + } + if timeoutConfig.HTTPRouteMustNotHaveParents == 0 { + timeoutConfig.HTTPRouteMustNotHaveParents = defaultTimeoutConfig.HTTPRouteMustNotHaveParents + } + if timeoutConfig.HTTPRouteMustHaveCondition == 0 { + timeoutConfig.HTTPRouteMustHaveCondition = defaultTimeoutConfig.HTTPRouteMustHaveCondition + } + if timeoutConfig.HTTPRouteMustHaveParents == 0 { + timeoutConfig.HTTPRouteMustHaveParents = defaultTimeoutConfig.HTTPRouteMustHaveParents + } + if timeoutConfig.ManifestFetchTimeout == 0 { + timeoutConfig.ManifestFetchTimeout = defaultTimeoutConfig.ManifestFetchTimeout + } + if timeoutConfig.MaxTimeToConsistency == 0 { + timeoutConfig.MaxTimeToConsistency = defaultTimeoutConfig.MaxTimeToConsistency + } + if timeoutConfig.NamespacesMustBeReady == 0 { + timeoutConfig.NamespacesMustBeReady = defaultTimeoutConfig.NamespacesMustBeReady + } + if timeoutConfig.RequestTimeout == 0 { + timeoutConfig.RequestTimeout = defaultTimeoutConfig.RequestTimeout + } +} diff --git a/test/ingress/conformance/utils/flags/flags.go b/test/ingress/conformance/utils/flags/flags.go new file mode 100644 index 000000000..ef0c1748d --- /dev/null +++ b/test/ingress/conformance/utils/flags/flags.go @@ -0,0 +1,27 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flags + +import ( + "flag" +) + +var ( + IngressClassName = flag.String("ingress-class", "higress", "Name of IngressClass to use for tests") + ShowDebug = flag.Bool("debug", false, "Whether to print debug logs") + CleanupBaseResources = flag.Bool("cleanup-base-resources", true, "Whether to cleanup base test resources after the run") + SupportedFeatures = flag.String("supported-features", "", "Supported features included in conformance tests suites") + ExemptFeatures = flag.String("exempt-features", "", "Exempt Features excluded from conformance tests suites") +) diff --git a/test/ingress/conformance/utils/http/http.go b/test/ingress/conformance/utils/http/http.go new file mode 100644 index 000000000..96fb79311 --- /dev/null +++ b/test/ingress/conformance/utils/http/http.go @@ -0,0 +1,354 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "fmt" + "net/url" + "strings" + "testing" + "time" + + "github.com/alibaba/higress/test/ingress/conformance/utils/config" + "github.com/alibaba/higress/test/ingress/conformance/utils/roundtripper" +) + +// ExpectedResponse defines the response expected for a given request. +type ExpectedResponse struct { + // Request defines the request to make. + Request Request + + // ExpectedRequest defines the request that + // is expected to arrive at the backend. If + // not specified, the backend request will be + // expected to match Request. + ExpectedRequest *ExpectedRequest + + RedirectRequest *roundtripper.RedirectRequest + + // BackendSetResponseHeaders is a set of headers + // the echoserver should set in its response. + BackendSetResponseHeaders map[string]string + + // Response defines what response the test case + // should receive. + Response Response + + Backend string + Namespace string + + // User Given TestCase name + TestCaseName string +} + +// Request can be used as both the request to make and a means to verify +// that echoserver received the expected request. Note that multiple header +// values can be provided, as a comma-separated value. +type Request struct { + Host string + Method string + Path string + Headers map[string]string + UnfollowRedirect bool +} + +// ExpectedRequest defines expected properties of a request that reaches a backend. +type ExpectedRequest struct { + Request + + // AbsentHeaders are names of headers that are expected + // *not* to be present on the request. + AbsentHeaders []string +} + +// Response defines expected properties of a response from a backend. +type Response struct { + StatusCode int + Headers map[string]string + AbsentHeaders []string +} + +// requiredConsecutiveSuccesses is the number of requests that must succeed in a row +// for MakeRequestAndExpectEventuallyConsistentResponse to consider the response "consistent" +// before making additional assertions on the response body. If this number is not reached within +// maxTimeToConsistency, the test will fail. +const requiredConsecutiveSuccesses = 3 + +// MakeRequestAndExpectEventuallyConsistentResponse makes a request with the given parameters, +// understanding that the request may fail for some amount of time. +// +// Once the request succeeds consistently with the response having the expected status code, make +// additional assertions on the response body using the provided ExpectedResponse. +func MakeRequestAndExpectEventuallyConsistentResponse(t *testing.T, r roundtripper.RoundTripper, timeoutConfig config.TimeoutConfig, gwAddr string, expected ExpectedResponse) { + t.Helper() + + if expected.Request.Method == "" { + expected.Request.Method = "GET" + } + + if expected.Response.StatusCode == 0 { + expected.Response.StatusCode = 200 + } + + t.Logf("Making %s request to http://%s%s", expected.Request.Method, gwAddr, expected.Request.Path) + + path, query, _ := strings.Cut(expected.Request.Path, "?") + + req := roundtripper.Request{ + Method: expected.Request.Method, + Host: expected.Request.Host, + URL: url.URL{Scheme: "http", Host: gwAddr, Path: path, RawQuery: query}, + Protocol: "HTTP", + Headers: map[string][]string{}, + UnfollowRedirect: expected.Request.UnfollowRedirect, + } + + if expected.Request.Headers != nil { + for name, value := range expected.Request.Headers { + req.Headers[name] = []string{value} + } + } + + backendSetHeaders := []string{} + for name, val := range expected.BackendSetResponseHeaders { + backendSetHeaders = append(backendSetHeaders, name+":"+val) + } + req.Headers["X-Echo-Set-Header"] = []string{strings.Join(backendSetHeaders, ",")} + + WaitForConsistentResponse(t, r, req, expected, requiredConsecutiveSuccesses, timeoutConfig.MaxTimeToConsistency) +} + +// awaitConvergence runs the given function until it returns 'true' `threshold` times in a row. +// Each failed attempt has a 1s delay; successful attempts have no delay. +func awaitConvergence(t *testing.T, threshold int, maxTimeToConsistency time.Duration, fn func(elapsed time.Duration) bool) { + successes := 0 + attempts := 0 + start := time.Now() + to := time.After(maxTimeToConsistency) + delay := time.Second + for { + select { + case <-to: + t.Fatalf("timeout while waiting after %d attempts", attempts) + default: + } + + completed := fn(time.Now().Sub(start)) + attempts++ + if completed { + successes++ + if successes >= threshold { + return + } + // Skip delay if we have a success + continue + } + + successes = 0 + select { + // Capture the overall timeout + case <-to: + t.Fatalf("timeout while waiting after %d attempts, %d/%d sucessess", attempts, successes, threshold) + // And the per-try delay + case <-time.After(delay): + } + } +} + +// WaitForConsistentResponse repeats the provided request until it completes with a response having +// the expected response consistently. The provided threshold determines how many times in +// a row this must occur to be considered "consistent". +func WaitForConsistentResponse(t *testing.T, r roundtripper.RoundTripper, req roundtripper.Request, expected ExpectedResponse, threshold int, maxTimeToConsistency time.Duration) { + awaitConvergence(t, threshold, maxTimeToConsistency, func(elapsed time.Duration) bool { + cReq, cRes, err := r.CaptureRoundTrip(req) + if err != nil { + t.Logf("Request failed, not ready yet: %v (after %v)", err.Error(), elapsed) + return false + } + + if err := CompareRequest(&req, cReq, cRes, expected); err != nil { + t.Logf("Response expectation failed for request: %v not ready yet: %v (after %v)", req, err, elapsed) + return false + } + + return true + }) + t.Logf("Request passed") +} + +func CompareRequest(req *roundtripper.Request, cReq *roundtripper.CapturedRequest, cRes *roundtripper.CapturedResponse, expected ExpectedResponse) error { + if expected.Response.StatusCode != cRes.StatusCode { + return fmt.Errorf("expected status code to be %d, got %d", expected.Response.StatusCode, cRes.StatusCode) + } + if cRes.StatusCode == 200 { + // The request expected to arrive at the backend is + // the same as the request made, unless otherwise + // specified. + if expected.ExpectedRequest == nil { + expected.ExpectedRequest = &ExpectedRequest{Request: expected.Request} + } + + if expected.ExpectedRequest.Method == "" { + expected.ExpectedRequest.Method = "GET" + } + + if expected.ExpectedRequest.Host != "" && expected.ExpectedRequest.Host != cReq.Host { + return fmt.Errorf("expected host to be %s, got %s", expected.ExpectedRequest.Host, cReq.Host) + } + + if expected.ExpectedRequest.Path != cReq.Path { + return fmt.Errorf("expected path to be %s, got %s", expected.ExpectedRequest.Path, cReq.Path) + } + if expected.ExpectedRequest.Method != cReq.Method { + return fmt.Errorf("expected method to be %s, got %s", expected.ExpectedRequest.Method, cReq.Method) + } + if expected.Namespace != cReq.Namespace { + return fmt.Errorf("expected namespace to be %s, got %s", expected.Namespace, cReq.Namespace) + } + if expected.ExpectedRequest.Headers != nil { + if cReq.Headers == nil { + return fmt.Errorf("no headers captured, expected %v", len(expected.ExpectedRequest.Headers)) + } + for name, val := range cReq.Headers { + cReq.Headers[strings.ToLower(name)] = val + } + for name, expectedVal := range expected.ExpectedRequest.Headers { + actualVal, ok := cReq.Headers[strings.ToLower(name)] + if !ok { + return fmt.Errorf("expected %s header to be set, actual headers: %v", name, cReq.Headers) + } else if strings.Join(actualVal, ",") != expectedVal { + return fmt.Errorf("expected %s header to be set to %s, got %s", name, expectedVal, strings.Join(actualVal, ",")) + } + + } + } + + if expected.Response.Headers != nil { + if cRes.Headers == nil { + return fmt.Errorf("no headers captured, expected %v", len(expected.ExpectedRequest.Headers)) + } + for name, val := range cRes.Headers { + cRes.Headers[strings.ToLower(name)] = val + } + + for name, expectedVal := range expected.Response.Headers { + actualVal, ok := cRes.Headers[strings.ToLower(name)] + if !ok { + return fmt.Errorf("expected %s header to be set, actual headers: %v", name, cRes.Headers) + } else if strings.Join(actualVal, ",") != expectedVal { + return fmt.Errorf("expected %s header to be set to %s, got %s", name, expectedVal, strings.Join(actualVal, ",")) + } + } + } + + if len(expected.Response.AbsentHeaders) > 0 { + for name, val := range cRes.Headers { + cRes.Headers[strings.ToLower(name)] = val + } + + for _, name := range expected.Response.AbsentHeaders { + val, ok := cRes.Headers[strings.ToLower(name)] + if ok { + return fmt.Errorf("expected %s header to not be set, got %s", name, val) + } + } + } + + // Verify that headers expected *not* to be present on the + // request are actually not present. + if len(expected.ExpectedRequest.AbsentHeaders) > 0 { + for name, val := range cReq.Headers { + cReq.Headers[strings.ToLower(name)] = val + } + + for _, name := range expected.ExpectedRequest.AbsentHeaders { + val, ok := cReq.Headers[strings.ToLower(name)] + if ok { + return fmt.Errorf("expected %s header to not be set, got %s", name, val) + } + } + } + + if !strings.HasPrefix(cReq.Pod, expected.Backend) { + return fmt.Errorf("expected pod name to start with %s, got %s", expected.Backend, cReq.Pod) + } + } else if roundtripper.IsRedirect(cRes.StatusCode) { + if expected.RedirectRequest == nil { + return nil + } + + setRedirectRequestDefaults(req, cRes, &expected) + + if expected.RedirectRequest.Host != cRes.RedirectRequest.Host { + return fmt.Errorf("expected redirected hostname to be %s, got %s", expected.RedirectRequest.Host, cRes.RedirectRequest.Host) + } + + if expected.RedirectRequest.Port != cRes.RedirectRequest.Port { + return fmt.Errorf("expected redirected port to be %s, got %s", expected.RedirectRequest.Port, cRes.RedirectRequest.Port) + } + + if expected.RedirectRequest.Scheme != cRes.RedirectRequest.Scheme { + return fmt.Errorf("expected redirected scheme to be %s, got %s", expected.RedirectRequest.Scheme, cRes.RedirectRequest.Scheme) + } + + if expected.RedirectRequest.Path != cRes.RedirectRequest.Path { + return fmt.Errorf("expected redirected path to be %s, got %s", expected.RedirectRequest.Path, cRes.RedirectRequest.Path) + } + } + return nil +} + +// Get User-defined test case name or generate from expected response to a given request. +func (er *ExpectedResponse) GetTestCaseName(i int) string { + + // If TestCase name is provided then use that or else generate one. + if er.TestCaseName != "" { + return er.TestCaseName + } + + headerStr := "" + reqStr := "" + + if er.Request.Headers != nil { + headerStr = " with headers" + } + + reqStr = fmt.Sprintf("%d request to '%s%s'%s", i, er.Request.Host, er.Request.Path, headerStr) + + if er.Backend != "" { + return fmt.Sprintf("%s should go to %s", reqStr, er.Backend) + } + return fmt.Sprintf("%s should receive a %d", reqStr, er.Response.StatusCode) +} + +func setRedirectRequestDefaults(req *roundtripper.Request, cRes *roundtripper.CapturedResponse, expected *ExpectedResponse) { + // If the expected host is nil it means we do not test host redirect. + // In that case we are setting it to the one we got from the response because we do not know the ip/host of the gateway. + if expected.RedirectRequest.Host == "" { + expected.RedirectRequest.Host = cRes.RedirectRequest.Host + } + + if expected.RedirectRequest.Port == "" { + expected.RedirectRequest.Port = req.URL.Port() + } + + if expected.RedirectRequest.Scheme == "" { + expected.RedirectRequest.Scheme = req.URL.Scheme + } + + if expected.RedirectRequest.Path == "" { + expected.RedirectRequest.Path = req.URL.Path + } +} diff --git a/test/ingress/conformance/utils/kubernetes/apply.go b/test/ingress/conformance/utils/kubernetes/apply.go new file mode 100644 index 000000000..25a13fce0 --- /dev/null +++ b/test/ingress/conformance/utils/kubernetes/apply.go @@ -0,0 +1,229 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "testing" + + ingress "github.com/alibaba/higress/test/ingress/conformance" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/alibaba/higress/test/ingress/conformance/utils/config" +) + +// Applier prepares manifests depending on the available options and applies +// them to the Kubernetes cluster. +type Applier struct { + NamespaceLabels map[string]string + // ValidUniqueListenerPorts maps each listener port of each Gateway in the + // manifests to a valid, unique port. There must be as many + // ValidUniqueListenerPorts as there are listeners in the set of manifests. + // For example, given two Gateways, each with 2 listeners, there should be + // four ValidUniqueListenerPorts. + // If empty or nil, ports are not modified. + ValidUniqueListenerPorts []int + + // IngressClass will be used as the spec.gatewayClassName when applying Gateway resources + IngressClass string + + // ControllerName will be used as the spec.controllerName when applying GatewayClass resources + ControllerName string +} + +// prepareNamespace adjusts the Namespace labels. +func prepareNamespace(t *testing.T, uObj *unstructured.Unstructured, namespaceLabels map[string]string) { + labels, _, err := unstructured.NestedStringMap(uObj.Object, "metadata", "labels") + require.NoErrorf(t, err, "error getting labels on Namespace %s", uObj.GetName()) + + for k, v := range namespaceLabels { + if labels == nil { + labels = map[string]string{} + } + + labels[k] = v + } + + // SetNestedStringMap converts nil to an empty map + if labels != nil { + err = unstructured.SetNestedStringMap(uObj.Object, labels, "metadata", "labels") + } + require.NoErrorf(t, err, "error setting labels on Namespace %s", uObj.GetName()) +} + +// prepareResources uses the options from an Applier to tweak resources given by +// a set of manifests. +func (a Applier) prepareResources(t *testing.T, decoder *yaml.YAMLOrJSONDecoder) ([]unstructured.Unstructured, error) { + var resources []unstructured.Unstructured + + for { + uObj := unstructured.Unstructured{} + if err := decoder.Decode(&uObj); err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, err + } + if len(uObj.Object) == 0 { + continue + } + + if uObj.GetKind() == "Namespace" && uObj.GetObjectKind().GroupVersionKind().Group == "" { + prepareNamespace(t, &uObj, a.NamespaceLabels) + } + + resources = append(resources, uObj) + } + + return resources, nil +} + +func (a Applier) MustApplyObjectsWithCleanup(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, resources []client.Object, cleanup bool) { + for _, resource := range resources { + resource := resource + + ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.CreateTimeout) + defer cancel() + + t.Logf("Creating %s %s", resource.GetName(), resource.GetObjectKind().GroupVersionKind().Kind) + + err := c.Create(ctx, resource) + if err != nil { + if !apierrors.IsAlreadyExists(err) { + require.NoError(t, err, "error creating resource") + } + } + + if cleanup { + t.Cleanup(func() { + ctx, cancel = context.WithTimeout(context.Background(), timeoutConfig.DeleteTimeout) + defer cancel() + t.Logf("Deleting %s %s", resource.GetName(), resource.GetObjectKind().GroupVersionKind().Kind) + err = c.Delete(ctx, resource) + require.NoErrorf(t, err, "error deleting resource") + }) + } + } +} + +// MustApplyWithCleanup creates or updates Kubernetes resources defined with the +// provided YAML file and registers a cleanup function for resources it created. +// Note that this does not remove resources that already existed in the cluster. +func (a Applier) MustApplyWithCleanup(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, location string, cleanup bool) { + data, err := getContentsFromPathOrURL(location, timeoutConfig) + require.NoError(t, err) + + decoder := yaml.NewYAMLOrJSONDecoder(data, 4096) + + resources, err := a.prepareResources(t, decoder) + if err != nil { + t.Logf("manifest: %s", data.String()) + require.NoErrorf(t, err, "error parsing manifest") + } + + for i := range resources { + uObj := &resources[i] + + ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.CreateTimeout) + defer cancel() + + namespacedName := types.NamespacedName{Namespace: uObj.GetNamespace(), Name: uObj.GetName()} + fetchedObj := uObj.DeepCopy() + err := c.Get(ctx, namespacedName, fetchedObj) + if err != nil { + if !apierrors.IsNotFound(err) { + require.NoErrorf(t, err, "error getting resource") + } + t.Logf("Creating %s %s", uObj.GetName(), uObj.GetKind()) + err = c.Create(ctx, uObj) + require.NoErrorf(t, err, "error creating resource") + + if cleanup { + t.Cleanup(func() { + ctx, cancel = context.WithTimeout(context.Background(), timeoutConfig.DeleteTimeout) + defer cancel() + t.Logf("Deleting %s %s", uObj.GetName(), uObj.GetKind()) + err = c.Delete(ctx, uObj) + require.NoErrorf(t, err, "error deleting resource") + }) + } + continue + } + + uObj.SetResourceVersion(fetchedObj.GetResourceVersion()) + t.Logf("Updating %s %s", uObj.GetName(), uObj.GetKind()) + err = c.Update(ctx, uObj) + + if cleanup { + t.Cleanup(func() { + ctx, cancel = context.WithTimeout(context.Background(), timeoutConfig.DeleteTimeout) + defer cancel() + t.Logf("Deleting %s %s", uObj.GetName(), uObj.GetKind()) + err = c.Delete(ctx, uObj) + require.NoErrorf(t, err, "error deleting resource") + }) + } + require.NoErrorf(t, err, "error updating resource") + } +} + +// getContentsFromPathOrURL takes a string that can either be a local file +// path or an https:// URL to YAML manifests and provides the contents. +func getContentsFromPathOrURL(location string, timeoutConfig config.TimeoutConfig) (*bytes.Buffer, error) { + if strings.HasPrefix(location, "http://") { + return nil, fmt.Errorf("data can't be retrieved from %s: http is not supported, use https", location) + } else if strings.HasPrefix(location, "https://") { + ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.ManifestFetchTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, location, nil) + if err != nil { + return nil, err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + manifests := new(bytes.Buffer) + count, err := manifests.ReadFrom(resp.Body) + if err != nil { + return nil, err + } + + if resp.ContentLength != -1 && count != resp.ContentLength { + return nil, fmt.Errorf("received %d bytes from %s, expected %d", count, location, resp.ContentLength) + } + return manifests, nil + } + b, err := ingress.Manifests.ReadFile(location) + if err != nil { + return nil, err + } + return bytes.NewBuffer(b), nil +} diff --git a/test/ingress/conformance/utils/kubernetes/apply_test.go b/test/ingress/conformance/utils/kubernetes/apply_test.go new file mode 100644 index 000000000..a7d33373d --- /dev/null +++ b/test/ingress/conformance/utils/kubernetes/apply_test.go @@ -0,0 +1,19 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + _ "github.com/alibaba/higress/test/ingress/conformance/utils/flags" +) diff --git a/test/ingress/conformance/utils/kubernetes/cert.go b/test/ingress/conformance/utils/kubernetes/cert.go new file mode 100644 index 000000000..76a60174a --- /dev/null +++ b/test/ingress/conformance/utils/kubernetes/cert.go @@ -0,0 +1,126 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "io" + "math/big" + "net" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + // ensure auth plugins are loaded + _ "k8s.io/client-go/plugin/pkg/client/auth" +) + +const ( + rsaBits = 2048 + validFor = 365 * 24 * time.Hour +) + +// MustCreateSelfSignedCertSecret creates a self-signed SSL certificate and stores it in a secret +func MustCreateSelfSignedCertSecret(t *testing.T, namespace, secretName string, hosts []string) *corev1.Secret { + require.Greater(t, len(hosts), 0, "require a non-empty hosts for Subject Alternate Name values") + + var serverKey, serverCert bytes.Buffer + + host := strings.Join(hosts, ",") + + require.NoError(t, generateRSACert(host, &serverKey, &serverCert), "failed to generate RSA certificate") + + data := map[string][]byte{ + corev1.TLSCertKey: serverCert.Bytes(), + corev1.TLSPrivateKeyKey: serverKey.Bytes(), + } + + newSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: secretName, + }, + Type: corev1.SecretTypeTLS, + Data: data, + } + + return newSecret +} + +// generateRSACert generates a basic self signed certificate valir for a year +func generateRSACert(host string, keyOut, certOut io.Writer) error { + priv, err := rsa.GenerateKey(rand.Reader, rsaBits) + if err != nil { + return fmt.Errorf("failed to generate key: %w", err) + } + notBefore := time.Now() + notAfter := notBefore.Add(validFor) + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + + if err != nil { + return fmt.Errorf("failed to generate serial number: %w", err) + } + + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: "default", + Organization: []string{"Acme Co"}, + }, + NotBefore: notBefore, + NotAfter: notAfter, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + hosts := strings.Split(host, ",") + for _, h := range hosts { + if ip := net.ParseIP(h); ip != nil { + template.IPAddresses = append(template.IPAddresses, ip) + } else { + template.DNSNames = append(template.DNSNames, h) + } + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + + if err != nil { + return fmt.Errorf("failed to create certificate: %w", err) + } + + if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { + return fmt.Errorf("failed creating cert: %w", err) + } + + if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil { + return fmt.Errorf("failed creating key: %w", err) + } + + return nil +} diff --git a/test/ingress/conformance/utils/kubernetes/helpers.go b/test/ingress/conformance/utils/kubernetes/helpers.go new file mode 100644 index 000000000..4d5f5d48b --- /dev/null +++ b/test/ingress/conformance/utils/kubernetes/helpers.go @@ -0,0 +1,122 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/alibaba/higress/test/ingress/conformance/utils/config" +) + +// FilterStaleConditions returns the list of status condition whos observedGeneration does not +// match the objects metadata.Generation +func FilterStaleConditions(obj metav1.Object, conditions []metav1.Condition) []metav1.Condition { + stale := make([]metav1.Condition, 0, len(conditions)) + for _, condition := range conditions { + if obj.GetGeneration() != condition.ObservedGeneration { + stale = append(stale, condition) + } + } + return stale +} + +// NamespacesMustBeAccepted waits until all Pods are marked ready +// in the provided namespaces. This will cause the test to +// halt if the specified timeout is exceeded. +func NamespacesMustBeAccepted(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, namespaces []string) { + t.Helper() + + waitErr := wait.PollImmediate(1*time.Second, timeoutConfig.NamespacesMustBeReady, func() (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + for _, ns := range namespaces { + podList := &v1.PodList{} + err := c.List(ctx, podList, client.InNamespace(ns)) + if err != nil { + t.Errorf("Error listing Pods: %v", err) + } + for _, pod := range podList.Items { + if !FindPodConditionInList(t, pod.Status.Conditions, "Ready", "True") && + pod.Status.Phase != v1.PodSucceeded { + t.Logf("%s/%s Pod not ready yet", ns, pod.Name) + return false, nil + } + } + } + t.Logf("Gateways and Pods in %s namespaces ready", strings.Join(namespaces, ", ")) + return true, nil + }) + require.NoErrorf(t, waitErr, "error waiting for %s namespaces to be ready", strings.Join(namespaces, ", ")) +} + +func ConditionsMatch(t *testing.T, expected, actual []metav1.Condition) bool { + if len(actual) < len(expected) { + t.Logf("Expected more conditions to be present") + return false + } + for _, condition := range expected { + if !FindConditionInList(t, actual, condition.Type, string(condition.Status), condition.Reason) { + return false + } + } + + t.Logf("Conditions matched expectations") + return true +} + +// findConditionInList finds a condition in a list of Conditions, checking +// the Name, Value, and Reason. If an empty reason is passed, any Reason will match. +func FindConditionInList(t *testing.T, conditions []metav1.Condition, condName, expectedStatus, expectedReason string) bool { + for _, cond := range conditions { + if cond.Type == condName { + if cond.Status == metav1.ConditionStatus(expectedStatus) { + // an empty Reason string means "Match any reason". + if expectedReason == "" || cond.Reason == expectedReason { + return true + } + t.Logf("%s condition Reason set to %s, expected %s", condName, cond.Reason, expectedReason) + } + + t.Logf("%s condition set to Status %s with Reason %v, expected Status %s", condName, cond.Status, cond.Reason, expectedStatus) + } + } + + t.Logf("%s was not in conditions list", condName) + return false +} + +func FindPodConditionInList(t *testing.T, conditions []v1.PodCondition, condName, condValue string) bool { + for _, cond := range conditions { + if cond.Type == v1.PodConditionType(condName) { + if cond.Status == v1.ConditionStatus(condValue) { + return true + } + t.Logf("%s condition set to %s, expected %s", condName, cond.Status, condValue) + } + } + + t.Logf("%s was not in conditions list", condName) + return false +} diff --git a/test/ingress/conformance/utils/roundtripper/roundtripper.go b/test/ingress/conformance/utils/roundtripper/roundtripper.go new file mode 100644 index 000000000..d6d49191d --- /dev/null +++ b/test/ingress/conformance/utils/roundtripper/roundtripper.go @@ -0,0 +1,199 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package roundtripper + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httputil" + "net/url" + "regexp" + + "github.com/alibaba/higress/test/ingress/conformance/utils/config" +) + +// RoundTripper is an interface used to make requests within conformance tests. +// This can be overridden with custom implementations whenever necessary. +type RoundTripper interface { + CaptureRoundTrip(Request) (*CapturedRequest, *CapturedResponse, error) +} + +// Request is the primary input for making a request. +type Request struct { + URL url.URL + Host string + Protocol string + Method string + Headers map[string][]string + UnfollowRedirect bool +} + +// CapturedRequest contains request metadata captured from an echoserver +// response. +type CapturedRequest struct { + Path string `json:"path"` + Host string `json:"host"` + Method string `json:"method"` + Protocol string `json:"proto"` + Headers map[string][]string `json:"headers"` + + Namespace string `json:"namespace"` + Pod string `json:"pod"` +} + +// RedirectRequest contains a follow up request metadata captured from a redirect +// response. +type RedirectRequest struct { + Scheme string + Host string + Port string + Path string +} + +// CapturedResponse contains response metadata. +type CapturedResponse struct { + StatusCode int + ContentLength int64 + Protocol string + Headers map[string][]string + RedirectRequest *RedirectRequest +} + +// DefaultRoundTripper is the default implementation of a RoundTripper. It will +// be used if a custom implementation is not specified. +type DefaultRoundTripper struct { + Debug bool + TimeoutConfig config.TimeoutConfig +} + +// CaptureRoundTrip makes a request with the provided parameters and returns the +// captured request and response from echoserver. An error will be returned if +// there is an error running the function but not if an HTTP error status code +// is received. +func (d *DefaultRoundTripper) CaptureRoundTrip(request Request) (*CapturedRequest, *CapturedResponse, error) { + cReq := &CapturedRequest{} + client := &http.Client{} + + if request.UnfollowRedirect { + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + } + + method := "GET" + if request.Method != "" { + method = request.Method + } + ctx, cancel := context.WithTimeout(context.Background(), d.TimeoutConfig.RequestTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, method, request.URL.String(), nil) + if err != nil { + return nil, nil, err + } + + if request.Host != "" { + req.Host = request.Host + } + + if request.Headers != nil { + for name, value := range request.Headers { + req.Header.Set(name, value[0]) + } + } + + if d.Debug { + var dump []byte + dump, err = httputil.DumpRequestOut(req, true) + if err != nil { + return nil, nil, err + } + + fmt.Printf("Sending Request:\n%s\n\n", formatDump(dump, "< ")) + } + + resp, err := client.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + if d.Debug { + var dump []byte + dump, err = httputil.DumpResponse(resp, true) + if err != nil { + return nil, nil, err + } + + fmt.Printf("Received Response:\n%s\n\n", formatDump(dump, "< ")) + } + + body, _ := io.ReadAll(resp.Body) + + // we cannot assume the response is JSON + if resp.Header.Get("Content-type") == "application/json" { + err = json.Unmarshal(body, cReq) + if err != nil { + return nil, nil, fmt.Errorf("unexpected error reading response: %w", err) + } + } + + cRes := &CapturedResponse{ + StatusCode: resp.StatusCode, + ContentLength: resp.ContentLength, + Protocol: resp.Proto, + Headers: resp.Header, + } + + if IsRedirect(resp.StatusCode) { + redirectURL, err := resp.Location() + if err != nil { + return nil, nil, err + } + cRes.RedirectRequest = &RedirectRequest{ + Scheme: redirectURL.Scheme, + Host: redirectURL.Hostname(), + Port: redirectURL.Port(), + Path: redirectURL.Path, + } + } + + return cReq, cRes, nil +} + +// IsRedirect returns true if a given status code is a redirect code. +func IsRedirect(statusCode int) bool { + switch statusCode { + case http.StatusMultipleChoices, + http.StatusMovedPermanently, + http.StatusFound, + http.StatusSeeOther, + http.StatusNotModified, + http.StatusUseProxy, + http.StatusTemporaryRedirect, + http.StatusPermanentRedirect: + return true + } + return false +} + +var startLineRegex = regexp.MustCompile(`(?m)^`) + +func formatDump(data []byte, prefix string) string { + data = startLineRegex.ReplaceAllLiteral(data, []byte(prefix)) + return string(data) +} diff --git a/test/ingress/conformance/utils/suite/suite.go b/test/ingress/conformance/utils/suite/suite.go new file mode 100644 index 000000000..a7d4eb157 --- /dev/null +++ b/test/ingress/conformance/utils/suite/suite.go @@ -0,0 +1,217 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package suite + +import ( + "testing" + + "github.com/alibaba/higress/test/ingress/conformance/utils/config" + "github.com/alibaba/higress/test/ingress/conformance/utils/kubernetes" + "github.com/alibaba/higress/test/ingress/conformance/utils/roundtripper" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// SupportedFeature allows opting in to additional conformance tests at an +// individual feature granularity. +type SupportedFeature string + +const ( + // This option indicates support for TLSRoute (extended conformance). + SupportTLSRoute SupportedFeature = "TLSRoute" + + // This option indicates support for HTTPRoute query param matching (extended conformance). + SupportHTTPRouteQueryParamMatching SupportedFeature = "HTTPRouteQueryParamMatching" + + // This option indicates support for HTTPRoute method matching (extended conformance). + SupportHTTPRouteMethodMatching SupportedFeature = "HTTPRouteMethodMatching" + + // This option indicates support for HTTPRoute response header modification (extended conformance). + SupportHTTPResponseHeaderModification SupportedFeature = "HTTPResponseHeaderModification" + + // This option indicates support for Destination Port matching (extended conformance). + SupportRouteDestinationPortMatching SupportedFeature = "RouteDestinationPortMatching" + + // This option indicates support for HTTPRoute port redirect (extended conformance). + SupportHTTPRoutePortRedirect SupportedFeature = "HTTPRoutePortRedirect" + + // This option indicates support for HTTPRoute scheme redirect (extended conformance). + SupportHTTPRouteSchemeRedirect SupportedFeature = "HTTPRouteSchemeRedirect" + + // This option indicates support for HTTPRoute path redirect (experimental conformance). + SupportHTTPRoutePathRedirect SupportedFeature = "HTTPRoutePathRedirect" + + // This option indicates support for HTTPRoute host rewrite (experimental conformance) + SupportHTTPRouteHostRewrite SupportedFeature = "HTTPRouteHostRewrite" + + // This option indicates support for HTTPRoute path rewrite (experimental conformance) + SupportHTTPRoutePathRewrite SupportedFeature = "HTTPRoutePathRewrite" +) + +// StandardCoreFeatures are the features that are required to be conformant with +// the Core API features that are part of the Standard release channel. +var StandardCoreFeatures = map[SupportedFeature]bool{} + +// ConformanceTestSuite defines the test suite used to run Gateway API +// conformance tests. +type ConformanceTestSuite struct { + Client client.Client + RoundTripper roundtripper.RoundTripper + GatewayAddress string + IngressClassName string + ControllerName string + Debug bool + Cleanup bool + BaseManifests string + Applier kubernetes.Applier + SupportedFeatures map[SupportedFeature]bool + TimeoutConfig config.TimeoutConfig +} + +// Options can be used to initialize a ConformanceTestSuite. +type Options struct { + Client client.Client + GatewayAddress string + IngressClassName string + Debug bool + RoundTripper roundtripper.RoundTripper + BaseManifests string + NamespaceLabels map[string]string + // ValidUniqueListenerPorts maps each listener port of each Gateway in the + // manifests to a valid, unique port. There must be as many + // ValidUniqueListenerPorts as there are listeners in the set of manifests. + // For example, given two Gateways, each with 2 listeners, there should be + // four ValidUniqueListenerPorts. + // If empty or nil, ports are not modified. + ValidUniqueListenerPorts []int + + // CleanupBaseResources indicates whether or not the base test + // resources such as Gateways should be cleaned up after the run. + CleanupBaseResources bool + SupportedFeatures map[SupportedFeature]bool + TimeoutConfig config.TimeoutConfig +} + +// New returns a new ConformanceTestSuite. +func New(s Options) *ConformanceTestSuite { + config.SetupTimeoutConfig(&s.TimeoutConfig) + + roundTripper := s.RoundTripper + if roundTripper == nil { + roundTripper = &roundtripper.DefaultRoundTripper{Debug: s.Debug, TimeoutConfig: s.TimeoutConfig} + } + + if s.SupportedFeatures == nil { + s.SupportedFeatures = StandardCoreFeatures + } else { + for feature, val := range StandardCoreFeatures { + if _, ok := s.SupportedFeatures[feature]; !ok { + s.SupportedFeatures[feature] = val + } + } + } + + suite := &ConformanceTestSuite{ + Client: s.Client, + RoundTripper: roundTripper, + IngressClassName: s.IngressClassName, + Debug: s.Debug, + Cleanup: s.CleanupBaseResources, + BaseManifests: s.BaseManifests, + GatewayAddress: s.GatewayAddress, + Applier: kubernetes.Applier{ + NamespaceLabels: s.NamespaceLabels, + ValidUniqueListenerPorts: s.ValidUniqueListenerPorts, + }, + SupportedFeatures: s.SupportedFeatures, + TimeoutConfig: s.TimeoutConfig, + } + + // apply defaults + if suite.BaseManifests == "" { + suite.BaseManifests = "base/manifests.yaml" + } + + return suite +} + +// Setup ensures the base resources required for conformance tests are installed +// in the cluster. It also ensures that all relevant resources are ready. +func (suite *ConformanceTestSuite) Setup(t *testing.T) { + t.Logf("Test Setup: Ensuring IngressClass has been accepted") + suite.ControllerName = suite.IngressClassName + + suite.Applier.IngressClass = suite.IngressClassName + suite.Applier.ControllerName = suite.ControllerName + + t.Logf("Test Setup: Applying base manifests") + suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, suite.BaseManifests, suite.Cleanup) + + t.Logf("Test Setup: Applying programmatic resources") + secret := kubernetes.MustCreateSelfSignedCertSecret(t, "higress-conformance-web-backend", "certificate", []string{"*"}) + suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup) + secret = kubernetes.MustCreateSelfSignedCertSecret(t, "higress-conformance-infra", "tls-validity-checks-certificate", []string{"*"}) + suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup) + + t.Logf("Test Setup: Ensuring Pods from base manifests are ready") + namespaces := []string{ + "higress-conformance-infra", + "higress-conformance-app-backend", + "higress-conformance-web-backend", + } + kubernetes.NamespacesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, namespaces) +} + +// Run runs the provided set of conformance tests. +func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) { + for _, test := range tests { + t.Run(test.ShortName, func(t *testing.T) { + test.Run(t, suite) + }) + } +} + +// ConformanceTest is used to define each individual conformance test. +type ConformanceTest struct { + ShortName string + Description string + Features []SupportedFeature + Manifests []string + Slow bool + Parallel bool + Test func(*testing.T, *ConformanceTestSuite) +} + +// Run runs an individual tests, applying and cleaning up the required manifests +// before calling the Test function. +func (test *ConformanceTest) Run(t *testing.T, suite *ConformanceTestSuite) { + if test.Parallel { + t.Parallel() + } + + // Check that all features exercised by the test have been opted into by + // the suite. + for _, feature := range test.Features { + if supported, ok := suite.SupportedFeatures[feature]; !ok || !supported { + t.Skipf("Skipping %s: suite does not support %s", test.ShortName, feature) + } + } + + for _, manifestLocation := range test.Manifests { + t.Logf("Applying %s", manifestLocation) + suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, true) + } + + test.Test(t, suite) +} diff --git a/test/ingress/e2e_test.go b/test/ingress/e2e_test.go new file mode 100644 index 000000000..fcf5b4d03 --- /dev/null +++ b/test/ingress/e2e_test.go @@ -0,0 +1,69 @@ +//go:build conformance +// +build conformance + +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "flag" + "testing" + + "github.com/alibaba/higress/test/ingress/conformance/tests" + "github.com/alibaba/higress/test/ingress/conformance/utils/flags" + "github.com/alibaba/higress/test/ingress/conformance/utils/suite" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/networking/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +var useUniquePorts = flag.Bool("use-unique-ports", true, "whether to use unique ports") + +func TestHigressConformanceTests(t *testing.T) { + flag.Parse() + + cfg, err := config.GetConfig() + require.NoError(t, err) + + client, err := client.New(cfg, client.Options{}) + require.NoError(t, err) + + require.NoError(t, v1.AddToScheme(client.Scheme())) + + validUniqueListenerPorts := []int{ + 80, + 443, + } + + if !*useUniquePorts { + validUniqueListenerPorts = []int{} + } + + cSuite := suite.New(suite.Options{ + Client: client, + IngressClassName: *flags.IngressClassName, + Debug: *flags.ShowDebug, + CleanupBaseResources: *flags.CleanupBaseResources, + ValidUniqueListenerPorts: validUniqueListenerPorts, + SupportedFeatures: map[suite.SupportedFeature]bool{}, + GatewayAddress: "localhost", + }) + cSuite.Setup(t) + higressTests := []suite.ConformanceTest{ + tests.HTTPRouteSimpleSameNamespace, + } + cSuite.Run(t, higressTests) +}