feat: Add e2e testcases for 'auth-tls-secret' and 'ssl-cipher' (#354)

test/ingress/conformance/utils/cert/cert.go

@@ -0,0 +1,145 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cert
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"math/big"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+type CertType int
+
+const (
+	CACertType CertType = iota
+	ServerCertType
+	ClientCertType
+)
+
+const (
+	// RSABits defines the bit length of the RSA private key
+	RSABits = 2048
+	// ValidFor defines the certificate validity period
+	ValidFor = 365 * 24 * time.Hour
+)
+
+// MustGenerateCaCert must generate a CA certificate and private key.
+// `certOut` and `keyOut` are PEM format buffers for certificate and private key, respectively.
+// `caCert` and `caKey` are the corresponding structures.
+func MustGenerateCaCert(t *testing.T) (certOut, keyOut *bytes.Buffer, caCert *x509.Certificate, caKey *rsa.PrivateKey) {
+	notBefore := time.Now()
+	notAfter := notBefore.Add(ValidFor)
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1+int64(CACertType)), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	require.NoError(t, err, "failed to generate serial number")
+
+	caCert = &x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			CommonName:   "default",
+			Organization: []string{"Higress E2E Test"},
+		},
+		NotBefore:             notBefore,
+		NotAfter:              notAfter,
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}
+	caKey, err = rsa.GenerateKey(rand.Reader, RSABits)
+	certOut, keyOut, err = GenerateCert(caCert, caKey, caCert, caKey)
+	return
+}
+
+// MustGenerateCertWithCA must generate a self-signed client/server certificate and private key
+// using CA certificate and private key.
+// `hosts` is used when CertType == ServerCertType
+func MustGenerateCertWithCA(t *testing.T, certType CertType, caCert *x509.Certificate, caKey *rsa.PrivateKey, hosts []string) (certOut, keyOut *bytes.Buffer) {
+	notBefore := time.Now()
+	notAfter := notBefore.Add(ValidFor)
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1+int64(certType)), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	require.NoError(t, err, "failed to generate serial number")
+
+	template := &x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			CommonName:   "default",
+			Organization: []string{"Higress E2E Test"},
+		},
+		NotBefore:   notBefore,
+		NotAfter:    notAfter,
+		KeyUsage:    x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+	}
+
+	if certType == ServerCertType && hosts != nil {
+		for _, h := range hosts {
+			if ip := net.ParseIP(h); ip != nil {
+				template.IPAddresses = append(template.IPAddresses, ip)
+			} else {
+				template.DNSNames = append(template.DNSNames, h)
+			}
+		}
+	}
+
+	privateKey, err := rsa.GenerateKey(rand.Reader, RSABits)
+	require.NoError(t, err, "failed to generate ras key")
+	certOut, keyOut, err = GenerateCert(template, privateKey, caCert, caKey)
+	return
+}
+
+// GenerateCert obtains the corresponding certificate and private key buffers
+// using the certificate template and private key.
+func GenerateCert(cert *x509.Certificate, key *rsa.PrivateKey, caCert *x509.Certificate, caKey *rsa.PrivateKey) (
+	certOut, keyOut *bytes.Buffer, err error) {
+	var (
+		priv   = key
+		pub    = &priv.PublicKey
+		privPm = priv
+	)
+	if caKey != nil {
+		privPm = caKey
+	}
+	certDER, err := x509.CreateCertificate(rand.Reader, cert, caCert, pub, privPm)
+	if err != nil {
+		err = fmt.Errorf("failed to create certificate: %w", err)
+		return
+	}
+	certOut = new(bytes.Buffer)
+	err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+	if err != nil {
+		err = fmt.Errorf("failed creating cert: %w", err)
+		return
+	}
+	keyOut = new(bytes.Buffer)
+	privDER := x509.MarshalPKCS1PrivateKey(priv)
+	err = pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: privDER})
+	if err != nil {
+		err = fmt.Errorf("failed creating key: %w", err)
+		return
+	}
+	return
+}

test/ingress/conformance/utils/config/timeout.go

@@ -68,6 +68,10 @@ type TimeoutConfig struct {
 	// RequestTimeout represents the maximum time for making an HTTP Request with the roundtripper.
 	// Max value for conformant implementation: None
 	RequestTimeout time.Duration
+
+	// TLSHandshakeTimeout represents the maximum time for waiting for a TLS handshake. Zero means no timeout.
+	// Max value for conformant implementation: None
+	TLSHandshakeTimeout time.Duration
 }
 
 // DefaultTimeoutConfig populates a TimeoutConfig with the default values.
@@ -86,6 +90,7 @@ func DefaultTimeoutConfig() TimeoutConfig {
 		MaxTimeToConsistency:           30 * time.Second,
 		NamespacesMustBeReady:          300 * time.Second,
 		RequestTimeout:                 10 * time.Second,
+		TLSHandshakeTimeout:            10 * time.Second,
 	}
 }
 
@@ -130,4 +135,7 @@ func SetupTimeoutConfig(timeoutConfig *TimeoutConfig) {
 	if timeoutConfig.RequestTimeout == 0 {
 		timeoutConfig.RequestTimeout = defaultTimeoutConfig.RequestTimeout
 	}
+	if timeoutConfig.TLSHandshakeTimeout == 0 {
+		timeoutConfig.TLSHandshakeTimeout = defaultTimeoutConfig.TLSHandshakeTimeout
+	}
 }

test/ingress/conformance/utils/http/http.go

@@ -70,6 +70,38 @@ type Request struct {
 	Path             string
 	Headers          map[string]string
 	UnfollowRedirect bool
+	TLSConfig        *TLSConfig
+}
+
+// TLSConfig defines the TLS configuration for the client.
+// When this field is set, the HTTPS protocol is used.
+type TLSConfig struct {
+	// MinVersion specifies the minimum TLS version,
+	// e.g. tls.VersionTLS12.
+	MinVersion uint16
+	// MinVersion specifies the maximum TLS version,
+	// e.g. tls.VersionTLS13.
+	MaxVersion uint16
+	// SNI is short for Server Name Indication.
+	// If this field is not specified, the value will be equal to `Host`.
+	SNI string
+	// CipherSuites can specify multiple client cipher suites,
+	// e.g. tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA.
+	CipherSuites []uint16
+	// Certificates defines the certificate chain
+	Certificates Certificates
+}
+
+// Certificates contains CA and client certificate chain
+type Certificates struct {
+	CACerts        [][]byte
+	ClientKeyPairs []ClientKeyPair
+}
+
+// ClientKeyPair is a pair of client certificate and private key.
+type ClientKeyPair struct {
+	ClientCert []byte
+	ClientKey  []byte
 }
 
 // ExpectedRequest defines expected properties of a request that reaches a backend.
@@ -102,6 +134,36 @@ const requiredConsecutiveSuccesses = 3
 func MakeRequestAndExpectEventuallyConsistentResponse(t *testing.T, r roundtripper.RoundTripper, timeoutConfig config.TimeoutConfig, gwAddr string, expected Assertion) {
 	t.Helper()
 
+	var (
+		scheme    = "http"
+		protocol  = "HTTP"
+		tlsConfig *roundtripper.TLSConfig
+	)
+	if expected.Request.ActualRequest.TLSConfig != nil {
+		scheme = "https"
+		protocol = "HTTPS"
+		clientKeyPairs := make([]roundtripper.ClientKeyPair, 0, len(expected.Request.ActualRequest.TLSConfig.Certificates.ClientKeyPairs))
+		for _, keyPair := range expected.Request.ActualRequest.TLSConfig.Certificates.ClientKeyPairs {
+			clientKeyPairs = append(clientKeyPairs, roundtripper.ClientKeyPair{
+				ClientCert: keyPair.ClientCert,
+				ClientKey:  keyPair.ClientKey,
+			})
+		}
+		tlsConfig = &roundtripper.TLSConfig{
+			MinVersion:   expected.Request.ActualRequest.TLSConfig.MinVersion,
+			MaxVersion:   expected.Request.ActualRequest.TLSConfig.MaxVersion,
+			SNI:          expected.Request.ActualRequest.TLSConfig.SNI,
+			CipherSuites: expected.Request.ActualRequest.TLSConfig.CipherSuites,
+			Certificates: roundtripper.Certificates{
+				CACert:         expected.Request.ActualRequest.TLSConfig.Certificates.CACerts,
+				ClientKeyPairs: clientKeyPairs,
+			},
+		}
+		if tlsConfig.SNI == "" {
+			tlsConfig.SNI = expected.Request.ActualRequest.Host
+		}
+	}
+
 	if expected.Request.ActualRequest.Method == "" {
 		expected.Request.ActualRequest.Method = "GET"
 	}
@@ -110,17 +172,18 @@ func MakeRequestAndExpectEventuallyConsistentResponse(t *testing.T, r roundtripp
 		expected.Response.ExpectedResponse.StatusCode = 200
 	}
 
-	t.Logf("Making %s request to http://%s%s", expected.Request.ActualRequest.Method, gwAddr, expected.Request.ActualRequest.Path)
+	t.Logf("Making %s request to %s://%s%s", expected.Request.ActualRequest.Method, scheme, gwAddr, expected.Request.ActualRequest.Path)
 
 	path, query, _ := strings.Cut(expected.Request.ActualRequest.Path, "?")
 
 	req := roundtripper.Request{
 		Method:           expected.Request.ActualRequest.Method,
 		Host:             expected.Request.ActualRequest.Host,
-		URL:              url.URL{Scheme: "http", Host: gwAddr, Path: path, RawQuery: query},
-		Protocol:         "HTTP",
+		URL:              url.URL{Scheme: scheme, Host: gwAddr, Path: path, RawQuery: query},
+		Protocol:         protocol,
 		Headers:          map[string][]string{},
 		UnfollowRedirect: expected.Request.ActualRequest.UnfollowRedirect,
+		TLSConfig:        tlsConfig,
 	}
 
 	if expected.Request.ActualRequest.Headers != nil {

test/ingress/conformance/utils/kubernetes/cert.go

@@ -20,7 +20,6 @@ import (
 	"crypto/rsa"
 	"crypto/x509"
 	"crypto/x509/pkix"
-	"encoding/pem"
 	"fmt"
 	"io"
 	"math/big"
@@ -35,11 +34,8 @@ import (
 
 	// ensure auth plugins are loaded
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
-)
 
-const (
-	rsaBits  = 2048
-	validFor = 365 * 24 * time.Hour
+	"github.com/alibaba/higress/test/ingress/conformance/utils/cert"
 )
 
 // MustCreateSelfSignedCertSecret creates a self-signed SSL certificate and stores it in a secret
@@ -47,49 +43,56 @@ func MustCreateSelfSignedCertSecret(t *testing.T, namespace, secretName string,
 	require.Greater(t, len(hosts), 0, "require a non-empty hosts for Subject Alternate Name values")
 
 	var serverKey, serverCert bytes.Buffer
-
 	host := strings.Join(hosts, ",")
-
 	require.NoError(t, generateRSACert(host, &serverKey, &serverCert), "failed to generate RSA certificate")
 
-	data := map[string][]byte{
-		corev1.TLSCertKey:       serverCert.Bytes(),
-		corev1.TLSPrivateKeyKey: serverKey.Bytes(),
-	}
+	return ConstructTLSSecret(namespace, secretName, serverCert.Bytes(), serverKey.Bytes())
+}
 
-	newSecret := &corev1.Secret{
+// ConstructTLSSecret constructs a secret of type "kubernetes.io/tls"
+func ConstructTLSSecret(namespace, secretName string, cert, key []byte) *corev1.Secret {
+	return &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: namespace,
 			Name:      secretName,
 		},
 		Type: corev1.SecretTypeTLS,
-		Data: data,
+		Data: map[string][]byte{
+			corev1.TLSCertKey:       cert,
+			corev1.TLSPrivateKeyKey: key,
+		},
 	}
-
-	return newSecret
 }
 
-// generateRSACert generates a basic self signed certificate valir for a year
-func generateRSACert(host string, keyOut, certOut io.Writer) error {
-	priv, err := rsa.GenerateKey(rand.Reader, rsaBits)
-	if err != nil {
-		return fmt.Errorf("failed to generate key: %w", err)
+// ConstructCASecret construct a CA secret of type "Opaque"
+func ConstructCASecret(namespace, secretName string, cert []byte) *corev1.Secret {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      secretName,
+		},
+		Type: corev1.SecretTypeOpaque,
+		Data: map[string][]byte{
+			corev1.ServiceAccountRootCAKey: cert,
+		},
 	}
-	notBefore := time.Now()
-	notAfter := notBefore.Add(validFor)
+}
 
+// generateRSACert generates a basic self signed certificate valid for a year
+func generateRSACert(host string, keyOut, certOut io.Writer) error {
+	notBefore := time.Now()
+	notAfter := notBefore.Add(cert.ValidFor)
 	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
 	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
-
 	if err != nil {
 		return fmt.Errorf("failed to generate serial number: %w", err)
 	}
 
-	template := x509.Certificate{
+	template := &x509.Certificate{
 		SerialNumber: serialNumber,
 		Subject: pkix.Name{
 			CommonName:   "default",
-			Organization: []string{"Acme Co"},
+			Organization: []string{"Higress E2E Test"},
 		},
 		NotBefore: notBefore,
 		NotAfter:  notAfter,
@@ -108,18 +111,13 @@ func generateRSACert(host string, keyOut, certOut io.Writer) error {
 		}
 	}
 
-	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
-
+	priv, err := rsa.GenerateKey(rand.Reader, cert.RSABits)
 	if err != nil {
-		return fmt.Errorf("failed to create certificate: %w", err)
+		return fmt.Errorf("failed to generate key: %w", err)
 	}
-
-	if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
-		return fmt.Errorf("failed creating cert: %w", err)
-	}
-
-	if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
-		return fmt.Errorf("failed creating key: %w", err)
+	certOut, keyOut, err = cert.GenerateCert(template, priv, template, nil)
+	if err != nil {
+		return fmt.Errorf("failed to generate rsa certificate: %w", err)
 	}
 
 	return nil

test/ingress/conformance/utils/roundtripper/roundtripper.go

@@ -16,6 +16,8 @@ package roundtripper
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -41,6 +43,29 @@ type Request struct {
 	Method           string
 	Headers          map[string][]string
 	UnfollowRedirect bool
+	TLSConfig        *TLSConfig
+}
+
+// TLSConfig defines the TLS configuration for the client.
+// When this field is set, the HTTPS protocol is used.
+type TLSConfig struct {
+	MinVersion   uint16
+	MaxVersion   uint16
+	SNI          string
+	CipherSuites []uint16
+	Certificates Certificates
+}
+
+// Certificates defines the self-signed client and CA certificate chain
+type Certificates struct {
+	CACert         [][]byte
+	ClientKeyPairs []ClientKeyPair
+}
+
+// ClientKeyPair is a pair of client certificate and private key.
+type ClientKeyPair struct {
+	ClientCert []byte
+	ClientKey  []byte
 }
 
 // CapturedRequest contains request metadata captured from an echoserver
@@ -95,6 +120,35 @@ func (d *DefaultRoundTripper) CaptureRoundTrip(request Request) (*CapturedReques
 		}
 	}
 
+	if request.TLSConfig != nil {
+		pool := x509.NewCertPool()
+		for _, caCert := range request.TLSConfig.Certificates.CACert {
+			pool.AppendCertsFromPEM(caCert)
+		}
+		var clientCerts []tls.Certificate
+		for _, keyPair := range request.TLSConfig.Certificates.ClientKeyPairs {
+			newClientCert, err := tls.X509KeyPair(keyPair.ClientCert, keyPair.ClientKey)
+			if err != nil {
+				return nil, nil, fmt.Errorf("failed to load client key pair: %w", err)
+			}
+			clientCerts = append(clientCerts, newClientCert)
+		}
+
+		client.Transport = &http.Transport{
+			TLSHandshakeTimeout: d.TimeoutConfig.TLSHandshakeTimeout,
+			DisableKeepAlives:   true,
+			TLSClientConfig: &tls.Config{
+				MinVersion:         request.TLSConfig.MinVersion,
+				MaxVersion:         request.TLSConfig.MaxVersion,
+				ServerName:         request.TLSConfig.SNI,
+				CipherSuites:       request.TLSConfig.CipherSuites,
+				RootCAs:            pool,
+				Certificates:       clientCerts,
+				InsecureSkipVerify: true,
+			},
+		}
+	}
+
 	method := "GET"
 	if request.Method != "" {
 		method = request.Method
@@ -130,6 +184,7 @@ func (d *DefaultRoundTripper) CaptureRoundTrip(request Request) (*CapturedReques
 	if err != nil {
 		return nil, nil, err
 	}
+	defer client.CloseIdleConnections()
 	defer resp.Body.Close()
 
 	if d.Debug {