feat:add higress automatic https (#854)

Co-authored-by: 澄潭 <zty98751@alibaba-inc.com>
This commit is contained in:
Jun
2024-05-09 10:56:28 +08:00
committed by GitHub
parent 6577ae8822
commit 03d2f01274
21 changed files with 2239 additions and 37 deletions

View File

@@ -20,6 +20,7 @@ import (
"net/http"
"time"
"github.com/alibaba/higress/pkg/cert"
"github.com/alibaba/higress/pkg/ingress/kube/common"
"github.com/alibaba/higress/pkg/ingress/mcp"
"github.com/alibaba/higress/pkg/ingress/translation"
@@ -112,6 +113,9 @@ type ServerArgs struct {
GatewaySelectorValue string
GatewayHttpPort uint32
GatewayHttpsPort uint32
EnableAutomaticHttps bool
AutomaticHttpsEmail string
CertHttpAddress string
}
type readinessProbe func() (bool, error)
@@ -133,6 +137,7 @@ type Server struct {
xdsServer *xds.DiscoveryServer
server server.Instance
readinessProbes map[string]readinessProbe
certServer *cert.Server
}
var (
@@ -168,6 +173,7 @@ func NewServer(args *ServerArgs) (*Server, error) {
s.initConfigController,
s.initRegistryEventHandlers,
s.initAuthenticators,
s.initAutomaticHttps,
}
for _, f := range initFuncList {
@@ -287,6 +293,15 @@ func (s *Server) Start(stop <-chan struct{}) error {
}
}()
if s.EnableAutomaticHttps {
go func() {
log.Infof("starting Automatic Cert HTTP service at %s", s.CertHttpAddress)
if err := s.certServer.Run(stop); err != nil {
log.Errorf("error serving Automatic Cert HTTP server: %v", err)
}
}()
}
s.waitForShutDown(stop)
return nil
}
@@ -370,6 +385,26 @@ func (s *Server) initAuthenticators() error {
return nil
}
func (s *Server) initAutomaticHttps() error {
certOption := &cert.Option{
Namespace: PodNamespace,
ServerAddress: s.CertHttpAddress,
Email: s.AutomaticHttpsEmail,
}
certServer, err := cert.NewServer(s.kubeClient.Kube(), certOption)
if err != nil {
return err
}
s.certServer = certServer
log.Infof("init cert default config")
s.certServer.InitDefaultConfig()
if !s.EnableAutomaticHttps {
log.Info("automatic https is disabled")
return nil
}
return s.certServer.InitServer()
}
func (s *Server) initKubeClient() error {
if s.kubeClient != nil {
// Already initialized by startup arguments

219
pkg/cert/certmgr.go Normal file
View File

@@ -0,0 +1,219 @@
// Copyright (c) 2022 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cert
import (
"context"
"fmt"
"sync"
"github.com/caddyserver/certmagic"
"github.com/mholt/acmez"
"k8s.io/client-go/kubernetes"
)
const (
EventCertObtained = "cert_obtained"
)
type CertMgr struct {
cfg *certmagic.Config
client kubernetes.Interface
namespace string
mux sync.RWMutex
storage certmagic.Storage
cache *certmagic.Cache
myACME *certmagic.ACMEIssuer
ingressSolver acmez.Solver
configMgr *ConfigMgr
secretMgr *SecretMgr
}
func InitCertMgr(opts *Option, clientSet kubernetes.Interface, config *Config) (*CertMgr, error) {
CertLog.Infof("certmgr init config: %+v", config)
// Init certmagic config
// First make a pointer to a Cache as we need to reference the same Cache in
// GetConfigForCert below.
var cache *certmagic.Cache
var storage certmagic.Storage
storage, _ = NewConfigmapStorage(opts.Namespace, clientSet)
renewalWindowRatio := float64(config.RenewBeforeDays / RenewMaxDays)
magicConfig := certmagic.Config{
RenewalWindowRatio: renewalWindowRatio,
Storage: storage,
}
cache = certmagic.NewCache(certmagic.CacheOptions{
GetConfigForCert: func(cert certmagic.Certificate) (*certmagic.Config, error) {
// Here we use New to get a valid Config associated with the same cache.
// The provided Config is used as a template and will be completed with
// any defaults that are set in the Default config.
return certmagic.New(cache, magicConfig), nil
},
})
// init certmagic
cfg := certmagic.New(cache, magicConfig)
// Init certmagic acme
issuer := config.GetIssuer(IssuerTypeLetsencrypt)
if issuer == nil {
// should never happen here
return nil, fmt.Errorf("there is no Letsencrypt Issuer found in config")
}
myACME := certmagic.NewACMEIssuer(cfg, certmagic.ACMEIssuer{
//CA: certmagic.LetsEncryptStagingCA,
CA: certmagic.LetsEncryptProductionCA,
Email: issuer.Email,
Agreed: true,
DisableHTTPChallenge: false,
DisableTLSALPNChallenge: true,
})
// inject http01 solver
ingressSolver, _ := NewIngressSolver(opts.Namespace, clientSet, myACME)
myACME.Http01Solver = ingressSolver
// init issuers
cfg.Issuers = []certmagic.Issuer{myACME}
configMgr, _ := NewConfigMgr(opts.Namespace, clientSet)
secretMgr, _ := NewSecretMgr(opts.Namespace, clientSet)
certMgr := &CertMgr{
cfg: cfg,
client: clientSet,
namespace: opts.Namespace,
myACME: myACME,
ingressSolver: ingressSolver,
configMgr: configMgr,
secretMgr: secretMgr,
cache: cache,
}
certMgr.cfg.OnEvent = certMgr.OnEvent
return certMgr, nil
}
func (s *CertMgr) Reconcile(ctx context.Context, oldConfig *Config, newConfig *Config) error {
CertLog.Infof("cermgr reconcile old config:%+v to new config:%+v", oldConfig, newConfig)
// sync email
if oldConfig != nil && newConfig != nil {
oldIssuer := oldConfig.GetIssuer(IssuerTypeLetsencrypt)
newIssuer := newConfig.GetIssuer(IssuerTypeLetsencrypt)
if oldIssuer.Email != newIssuer.Email {
// TODO before sync email, maybe need to clean up cache and account
}
}
// sync domains
newDomains := make([]string, 0)
newDomainsMap := make(map[string]string, 0)
removeDomains := make([]string, 0)
if newConfig != nil {
for _, config := range newConfig.CredentialConfig {
if config.TLSIssuer == IssuerTypeLetsencrypt {
for _, newDomain := range config.Domains {
newDomains = append(newDomains, newDomain)
newDomainsMap[newDomain] = newDomain
}
}
}
}
if oldConfig != nil {
for _, config := range oldConfig.CredentialConfig {
if config.TLSIssuer == IssuerTypeLetsencrypt {
for _, oldDomain := range config.Domains {
if _, ok := newDomainsMap[oldDomain]; !ok {
removeDomains = append(removeDomains, oldDomain)
}
}
}
}
}
if newConfig.AutomaticHttps == true {
newIssuer := newConfig.GetIssuer(IssuerTypeLetsencrypt)
// clean up unused domains
s.cleanSync(context.Background(), removeDomains)
// sync email
s.myACME.Email = newIssuer.Email
// sync RenewalWindowRatio
s.cfg.RenewalWindowRatio = float64(newConfig.RenewBeforeDays / RenewMaxDays)
// start cache
s.cache.Start()
// sync domains
s.manageSync(context.Background(), newDomains)
s.configMgr.SetConfig(newConfig)
} else {
// stop cache maintainAssets
s.cache.Stop()
s.configMgr.SetConfig(newConfig)
}
return nil
}
func (s *CertMgr) manageSync(ctx context.Context, domainNames []string) error {
CertLog.Infof("cert manage sync domains:%v", domainNames)
return s.cfg.ManageSync(ctx, domainNames)
}
func (s *CertMgr) cleanSync(ctx context.Context, domainNames []string) error {
//TODO implement clean up domains
CertLog.Infof("cert clean sync domains:%v", domainNames)
return nil
}
func (s *CertMgr) OnEvent(ctx context.Context, event string, data map[string]any) error {
CertLog.Infof("certmgr receive event:% data:%+v", event, data)
/**
event: cert_obtained
cfg.emit(ctx, "cert_obtained", map[string]any{
"renewal": true,
"remaining": timeLeft,
"identifier": name,
"issuer": issuerKey,
"storage_path": StorageKeys.CertsSitePrefix(issuerKey, certKey),
"private_key_path": StorageKeys.SitePrivateKey(issuerKey, certKey),
"certificate_path": StorageKeys.SiteCert(issuerKey, certKey),
"metadata_path": StorageKeys.SiteMeta(issuerKey, certKey),
})
*/
if event == EventCertObtained {
// obtain certificate and update secret
domain := data["identifier"].(string)
isRenew := data["renewal"].(bool)
privateKeyPath := data["private_key_path"].(string)
certificatePath := data["certificate_path"].(string)
privateKey, err := s.cfg.Storage.Load(context.Background(), privateKeyPath)
certificate, err := s.cfg.Storage.Load(context.Background(), certificatePath)
certChain, err := parseCertsFromPEMBundle(certificate)
if err != nil {
return err
}
notAfterTime := notAfter(certChain[0])
notBeforeTime := notBefore(certChain[0])
secretName := s.configMgr.GetConfig().GetSecretNameByDomain(IssuerTypeLetsencrypt, domain)
if len(secretName) == 0 {
CertLog.Errorf("can not find secret name for domain % in config", domain)
return nil
}
err2 := s.secretMgr.Update(domain, secretName, privateKey, certificate, notBeforeTime, notAfterTime, isRenew)
if err2 != nil {
CertLog.Errorf("update secretName %s for domain %s error: %v", secretName, domain, err2)
}
return err
}
return nil
}

290
pkg/cert/config.go Normal file
View File

@@ -0,0 +1,290 @@
// Copyright (c) 2022 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cert
import (
"context"
"fmt"
"strings"
"sync/atomic"
"time"
"istio.io/istio/pkg/config/host"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/yaml"
)
const (
ConfigmapCertName = "higress-https"
ConfigmapCertConfigKey = "cert"
DefaultRenewBeforeDays = 30
RenewMaxDays = 90
)
type IssuerName string
const (
IssuerTypeAliyunSSL IssuerName = "aliyunssl"
IssuerTypeLetsencrypt IssuerName = "letsencrypt"
)
// Config is the configuration of automatic https.
type Config struct {
AutomaticHttps bool `json:"automaticHttps"`
RenewBeforeDays int `json:"renewBeforeDays"`
CredentialConfig []CredentialEntry `json:"credentialConfig"`
ACMEIssuer []ACMEIssuerEntry `json:"acmeIssuer"`
Version string `json:"version"`
}
func (c *Config) GetIssuer(issuerName IssuerName) *ACMEIssuerEntry {
for _, issuer := range c.ACMEIssuer {
if issuer.Name == issuerName {
return &issuer
}
}
return nil
}
func (c *Config) MatchSecretNameByDomain(domain string) string {
for _, credential := range c.CredentialConfig {
for _, credDomain := range credential.Domains {
if host.Name(strings.ToLower(domain)).SubsetOf(host.Name(strings.ToLower(credDomain))) {
return credential.TLSSecret
}
}
}
return ""
}
func (c *Config) GetSecretNameByDomain(issuerName IssuerName, domain string) string {
for _, credential := range c.CredentialConfig {
if credential.TLSIssuer == issuerName {
for _, credDomain := range credential.Domains {
if host.Name(strings.ToLower(domain)).SubsetOf(host.Name(strings.ToLower(credDomain))) {
return credential.TLSSecret
}
}
}
}
return ""
}
func (c *Config) Validate() error {
// check acmeIssuer
if len(c.ACMEIssuer) == 0 {
return fmt.Errorf("acmeIssuer is empty")
}
for _, issuer := range c.ACMEIssuer {
switch issuer.Name {
case IssuerTypeLetsencrypt:
if issuer.Email == "" {
return fmt.Errorf("acmeIssuer %s email is empty", issuer.Name)
}
if !ValidateEmail(issuer.Email) {
return fmt.Errorf("acmeIssuer %s email %s is invalid", issuer.Name, issuer.Email)
}
default:
return fmt.Errorf("acmeIssuer name %s is not supported", issuer.Name)
}
}
// check credentialConfig
for _, credential := range c.CredentialConfig {
if len(credential.Domains) == 0 {
return fmt.Errorf("credentialConfig domains is empty")
}
if credential.TLSSecret == "" {
return fmt.Errorf("credentialConfig tlsSecret is empty")
}
if credential.TLSIssuer == IssuerTypeLetsencrypt {
if len(credential.Domains) > 1 {
return fmt.Errorf("credentialConfig tlsIssuer %s only support one domain", credential.TLSIssuer)
}
}
if credential.TLSIssuer != IssuerTypeLetsencrypt && len(credential.TLSIssuer) > 0 {
return fmt.Errorf("credential tls issuer %s is not support", credential.TLSIssuer)
}
}
if c.RenewBeforeDays <= 0 {
return fmt.Errorf("RenewBeforeDays should be large than zero")
}
if c.RenewBeforeDays >= RenewMaxDays {
return fmt.Errorf("RenewBeforeDays should be less than %d", RenewMaxDays)
}
return nil
}
type CredentialEntry struct {
Domains []string `json:"domains"`
TLSIssuer IssuerName `json:"tlsIssuer,omitempty"`
TLSSecret string `json:"tlsSecret,omitempty"`
CACertSecret string `json:"cacertSecret,omitempty"`
}
type ACMEIssuerEntry struct {
Name IssuerName `json:"name"`
Email string `json:"email"`
AK string `json:"ak"` // Only applicable for certain issuers like 'aliyunssl'
SK string `json:"sk"` // Only applicable for certain issuers like 'aliyunssl'
}
type ConfigMgr struct {
client kubernetes.Interface
config atomic.Value
namespace string
}
func (c *ConfigMgr) SetConfig(config *Config) {
c.config.Store(config)
}
func (c *ConfigMgr) GetConfig() *Config {
value := c.config.Load()
if value != nil {
if config, ok := value.(*Config); ok {
return config
}
}
return nil
}
func (c *ConfigMgr) InitConfig(email string) (*Config, error) {
var defaultConfig *Config
cm, err := c.GetConfigmap()
if err != nil {
if errors.IsNotFound(err) {
if len(strings.TrimSpace(email)) == 0 {
email = getRandEmail()
}
defaultConfig = newDefaultConfig(email)
err2 := c.ApplyConfigmap(defaultConfig)
if err2 != nil {
return nil, err2
}
}
return nil, err
} else {
defaultConfig, err = c.ParseConfigFromConfigmap(cm)
if err != nil {
return nil, err
}
}
return defaultConfig, nil
}
func (c *ConfigMgr) ParseConfigFromConfigmap(configmap *v1.ConfigMap) (*Config, error) {
if _, ok := configmap.Data[ConfigmapCertConfigKey]; !ok {
return nil, fmt.Errorf("no cert key %s in configmap %s", ConfigmapCertConfigKey, configmap.Name)
}
config := newDefaultConfig("")
if err := yaml.Unmarshal([]byte(configmap.Data[ConfigmapCertConfigKey]), config); err != nil {
return nil, fmt.Errorf("data:%s, convert to higress config error, error: %+v", configmap.Data[ConfigmapCertConfigKey], err)
}
// validate config
if err := config.Validate(); err != nil {
return nil, err
}
return config, nil
}
func (c *ConfigMgr) GetConfigFromConfigmap() (*Config, error) {
var config *Config
cm, err := c.GetConfigmap()
if err != nil {
return nil, err
} else {
config, err = c.ParseConfigFromConfigmap(cm)
if err != nil {
return nil, err
}
}
return config, nil
}
func (c *ConfigMgr) GetConfigmap() (configmap *v1.ConfigMap, err error) {
configmapName := ConfigmapCertName
cm, err := c.client.CoreV1().ConfigMaps(c.namespace).Get(context.Background(), configmapName, metav1.GetOptions{})
return cm, err
}
func (c *ConfigMgr) ApplyConfigmap(config *Config) error {
configmapName := ConfigmapCertName
cm := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Namespace: c.namespace,
Name: configmapName,
},
}
bytes, err := yaml.Marshal(config)
if err != nil {
return err
}
cm.Data = make(map[string]string, 0)
cm.Data[ConfigmapCertConfigKey] = string(bytes)
_, err = c.client.CoreV1().ConfigMaps(c.namespace).Get(context.Background(), configmapName, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
if _, err = c.client.CoreV1().ConfigMaps(c.namespace).Create(context.Background(), cm, metav1.CreateOptions{}); err != nil {
return err
}
} else {
return err
}
} else {
if _, err = c.client.CoreV1().ConfigMaps(c.namespace).Update(context.Background(), cm, metav1.UpdateOptions{}); err != nil {
return err
}
}
return nil
}
func NewConfigMgr(namespace string, client kubernetes.Interface) (*ConfigMgr, error) {
configMgr := &ConfigMgr{
client: client,
namespace: namespace,
}
return configMgr, nil
}
func newDefaultConfig(email string) *Config {
defaultIssuer := []ACMEIssuerEntry{
{
Name: IssuerTypeLetsencrypt,
Email: email,
},
}
defaultCredentialConfig := make([]CredentialEntry, 0)
config := &Config{
AutomaticHttps: true,
RenewBeforeDays: DefaultRenewBeforeDays,
ACMEIssuer: defaultIssuer,
CredentialConfig: defaultCredentialConfig,
Version: time.Now().Format("20060102030405"),
}
return config
}
func getRandEmail() string {
num1 := rangeRandom(100, 100000)
num2 := rangeRandom(100, 100000)
return fmt.Sprintf("your%d@yours%d.com", num1, num2)
}

122
pkg/cert/config_test.go Normal file
View File

@@ -0,0 +1,122 @@
// Copyright (c) 2022 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cert
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestMatchSecretNameByDomain(t *testing.T) {
tests := []struct {
name string
domain string
credentialCfg []CredentialEntry
expected string
}{
{
name: "Exact match",
domain: "example.com",
credentialCfg: []CredentialEntry{
{
Domains: []string{"example.com"},
TLSSecret: "example-com-tls",
},
},
expected: "example-com-tls",
},
{
name: "Exact match ignore case ",
domain: "eXample.com",
credentialCfg: []CredentialEntry{
{
Domains: []string{"example.com"},
TLSSecret: "example-com-tls",
},
},
expected: "example-com-tls",
},
{
name: "Wildcard match",
domain: "sub.example.com",
credentialCfg: []CredentialEntry{
{
Domains: []string{"*.example.com"},
TLSSecret: "wildcard-example-com-tls",
},
},
expected: "wildcard-example-com-tls",
},
{
name: "Wildcard match ignore case",
domain: "sub.Example.com",
credentialCfg: []CredentialEntry{
{
Domains: []string{"*.example.com"},
TLSSecret: "wildcard-example-com-tls",
},
},
expected: "wildcard-example-com-tls",
},
{
name: "* match",
domain: "blog.example.co.uk",
credentialCfg: []CredentialEntry{
{
Domains: []string{"*"},
TLSSecret: "blog-co-uk-tls",
},
},
expected: "blog-co-uk-tls",
},
{
name: "No match",
domain: "unknown.com",
credentialCfg: []CredentialEntry{
{
Domains: []string{"example.com"},
TLSSecret: "example-com-tls",
},
},
expected: "",
},
{
name: "Multiple matches - first match wins",
domain: "example.com",
credentialCfg: []CredentialEntry{
{
Domains: []string{"example.com"},
TLSSecret: "example-com-tls",
},
{
Domains: []string{"*.example.com"},
TLSSecret: "wildcard-example-com-tls",
},
},
expected: "example-com-tls",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := Config{CredentialConfig: tt.credentialCfg}
result := cfg.MatchSecretNameByDomain(tt.domain)
assert.Equal(t, tt.expected, result)
})
}
}

165
pkg/cert/controller.go Normal file
View File

@@ -0,0 +1,165 @@
// Copyright (c) 2022 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cert
import (
"context"
"fmt"
"reflect"
"time"
"k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/informers"
v1informer "k8s.io/client-go/informers/core/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
)
const (
workNum = 1
maxRetry = 2
configMapName = "higress-https"
)
type Controller struct {
namespace string
ConfigMapInformer v1informer.ConfigMapInformer
client kubernetes.Interface
queue workqueue.RateLimitingInterface
configMgr *ConfigMgr
server *Server
certMgr *CertMgr
factory informers.SharedInformerFactory
}
func (c *Controller) addConfigmap(obj interface{}) {
key, err := cache.MetaNamespaceKeyFunc(obj)
if err != nil {
return
}
namespace, name, _ := cache.SplitMetaNamespaceKey(key)
if namespace != c.namespace || name != configMapName {
return
}
c.enqueue(name)
}
func (c *Controller) updateConfigmap(oldObj interface{}, newObj interface{}) {
key, err := cache.MetaNamespaceKeyFunc(oldObj)
if err != nil {
return
}
namespace, name, _ := cache.SplitMetaNamespaceKey(key)
if namespace != c.namespace || name != configMapName {
return
}
if reflect.DeepEqual(oldObj, newObj) {
return
}
c.enqueue(name)
}
func (c *Controller) enqueue(name string) {
c.queue.Add(name)
}
func (c *Controller) cachesSynced() bool {
return c.ConfigMapInformer.Informer().HasSynced()
}
func (c *Controller) Run(stopCh <-chan struct{}) error {
defer runtime.HandleCrash()
defer c.queue.ShutDown()
CertLog.Info("Waiting for informer caches to sync")
c.factory.Start(stopCh)
if ok := cache.WaitForCacheSync(stopCh, c.cachesSynced); !ok {
return fmt.Errorf("failed to wait for caches to sync")
}
CertLog.Info("Starting controller")
// Launch one workers to process configmap resources
for i := 0; i < workNum; i++ {
go wait.Until(c.worker, time.Minute, stopCh)
}
CertLog.Info("Started workers")
<-stopCh
CertLog.Info("Shutting down workers")
return nil
}
func (c *Controller) worker() {
for c.processNextItem() {
}
}
func (c *Controller) processNextItem() bool {
item, shutdown := c.queue.Get()
if shutdown {
return false
}
defer c.queue.Done(item)
key := item.(string)
CertLog.Infof("controller process item:%s", key)
err := c.syncConfigmap(key)
if err != nil {
c.handleError(key, err)
}
return true
}
func (c *Controller) syncConfigmap(key string) error {
configmap, err := c.ConfigMapInformer.Lister().ConfigMaps(c.namespace).Get(key)
if err != nil {
return err
}
newConfig, err := c.configMgr.ParseConfigFromConfigmap(configmap)
if err != nil {
return err
}
oldConfig := c.configMgr.GetConfig()
// reconcile old config and new config
return c.certMgr.Reconcile(context.Background(), oldConfig, newConfig)
}
func (c *Controller) handleError(key string, err error) {
runtime.HandleError(err)
CertLog.Errorf("%+v", err)
c.queue.Forget(key)
}
func NewController(client kubernetes.Interface, namespace string, certMgr *CertMgr, configMgr *ConfigMgr) (*Controller, error) {
kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(client, 0, informers.WithNamespace(namespace))
configmapInformer := kubeInformerFactory.Core().V1().ConfigMaps()
c := &Controller{
certMgr: certMgr,
configMgr: configMgr,
client: client,
namespace: namespace,
factory: kubeInformerFactory,
ConfigMapInformer: configmapInformer,
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ingressManage"),
}
CertLog.Info("Setting up configmap informer event handlers")
configmapInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: c.addConfigmap,
UpdateFunc: c.updateConfigmap,
})
return c, nil
}

158
pkg/cert/ingress.go Normal file
View File

@@ -0,0 +1,158 @@
// Copyright (c) 2022 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cert
import (
"context"
"strings"
"sync"
"time"
"github.com/caddyserver/certmagic"
"github.com/mholt/acmez"
"github.com/mholt/acmez/acme"
v1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
const (
IngressClassName = "higress"
IngressServiceName = "higress-controller"
IngressNamePefix = "higress-http-solver-"
IngressPathPrefix = "/.well-known/acme-challenge/"
IngressServicePort = 8889
)
type IngressSolver struct {
client kubernetes.Interface
acmeIssuer *certmagic.ACMEIssuer
solversMu sync.Mutex
namespace string
ingressDelay time.Duration
}
func NewIngressSolver(namespace string, client kubernetes.Interface, acmeIssuer *certmagic.ACMEIssuer) (acmez.Solver, error) {
solver := &IngressSolver{
namespace: namespace,
client: client,
acmeIssuer: acmeIssuer,
ingressDelay: 5 * time.Second,
}
return solver, nil
}
func (s *IngressSolver) Present(_ context.Context, challenge acme.Challenge) error {
CertLog.Infof("ingress solver present challenge:%+v", challenge)
s.solversMu.Lock()
defer s.solversMu.Unlock()
ingressName := s.getIngressName(challenge)
ingress := s.constructIngress(challenge)
CertLog.Infof("update ingress name:%s, ingress:%v", ingressName, ingress)
_, err := s.client.NetworkingV1().Ingresses(s.namespace).Get(context.Background(), ingressName, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
// create ingress
_, err2 := s.client.NetworkingV1().Ingresses(s.namespace).Create(context.Background(), ingress, metav1.CreateOptions{})
return err2
}
return err
}
_, err1 := s.client.NetworkingV1().Ingresses(s.namespace).Update(context.Background(), ingress, metav1.UpdateOptions{})
if err1 != nil {
return err1
}
return nil
}
func (s *IngressSolver) Wait(ctx context.Context, challenge acme.Challenge) error {
CertLog.Infof("ingress solver wait challenge:%+v", challenge)
// wait for ingress ready
if s.ingressDelay > 0 {
select {
case <-time.After(s.ingressDelay):
case <-ctx.Done():
return ctx.Err()
}
}
CertLog.Infof("ingress solver wait challenge done")
return nil
}
func (s *IngressSolver) CleanUp(_ context.Context, challenge acme.Challenge) error {
CertLog.Infof("ingress solver cleanup challenge:%+v", challenge)
s.solversMu.Lock()
defer s.solversMu.Unlock()
ingressName := s.getIngressName(challenge)
CertLog.Infof("cleanup ingress name:%s", ingressName)
err := s.client.NetworkingV1().Ingresses(s.namespace).Delete(context.Background(), ingressName, metav1.DeleteOptions{})
if err != nil {
return err
}
return nil
}
func (s *IngressSolver) Delete(_ context.Context, challenge acme.Challenge) error {
s.solversMu.Lock()
defer s.solversMu.Unlock()
err := s.client.NetworkingV1().Ingresses(s.namespace).Delete(context.Background(), s.getIngressName(challenge), metav1.DeleteOptions{})
if err != nil {
return err
}
return nil
}
func (s *IngressSolver) getIngressName(challenge acme.Challenge) string {
return IngressNamePefix + strings.ReplaceAll(challenge.Identifier.Value, ".", "-")
}
func (s *IngressSolver) constructIngress(challenge acme.Challenge) *v1.Ingress {
ingressClassName := IngressClassName
ingressDomain := challenge.Identifier.Value
ingressPath := IngressPathPrefix + challenge.Token
ingress := v1.Ingress{}
ingress.Name = s.getIngressName(challenge)
ingress.Namespace = s.namespace
pathType := v1.PathTypePrefix
ingress.Spec = v1.IngressSpec{
IngressClassName: &ingressClassName,
Rules: []v1.IngressRule{
{
Host: ingressDomain,
IngressRuleValue: v1.IngressRuleValue{
HTTP: &v1.HTTPIngressRuleValue{
Paths: []v1.HTTPIngressPath{
{
Path: ingressPath,
PathType: &pathType,
Backend: v1.IngressBackend{
Service: &v1.IngressServiceBackend{
Name: IngressServiceName,
Port: v1.ServiceBackendPort{
Number: IngressServicePort,
},
},
},
},
},
},
},
},
},
}
return &ingress
}

19
pkg/cert/log.go Normal file
View File

@@ -0,0 +1,19 @@
// Copyright (c) 2022 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cert
import "istio.io/pkg/log"
var CertLog = log.RegisterScope("cert", "Higress Cert process.", 0)

108
pkg/cert/secret.go Normal file
View File

@@ -0,0 +1,108 @@
// Copyright (c) 2022 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cert
import (
"context"
"fmt"
"strconv"
"strings"
"time"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
const (
SecretNamePrefix = "higress-secret-"
)
type SecretMgr struct {
client kubernetes.Interface
namespace string
}
func NewSecretMgr(namespace string, client kubernetes.Interface) (*SecretMgr, error) {
secretMgr := &SecretMgr{
namespace: namespace,
client: client,
}
return secretMgr, nil
}
func (s *SecretMgr) Update(domain string, secretName string, privateKey []byte, certificate []byte, notBefore time.Time, notAfter time.Time, isRenew bool) error {
//secretName := s.getSecretName(domain)
secret := s.constructSecret(domain, privateKey, certificate, notBefore, notAfter, isRenew)
_, err := s.client.CoreV1().Secrets(s.namespace).Get(context.Background(), secretName, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
// create secret
_, err2 := s.client.CoreV1().Secrets(s.namespace).Create(context.Background(), secret, metav1.CreateOptions{})
return err2
}
return err
}
// check secret annotations
if _, ok := secret.Annotations["higress.io/cert-domain"]; !ok {
return fmt.Errorf("the secret name %s is not automatic https secret name for the domain:%s, please rename it in config", secretName, domain)
}
_, err1 := s.client.CoreV1().Secrets(s.namespace).Update(context.Background(), secret, metav1.UpdateOptions{})
if err1 != nil {
return err1
}
return nil
}
func (s *SecretMgr) Delete(domain string) error {
secretName := s.getSecretName(domain)
err := s.client.CoreV1().Secrets(s.namespace).Delete(context.Background(), secretName, metav1.DeleteOptions{})
return err
}
func (s *SecretMgr) getSecretName(domain string) string {
return SecretNamePrefix + strings.ReplaceAll(strings.TrimSpace(domain), ".", "-")
}
func (s *SecretMgr) constructSecret(domain string, privateKey []byte, certificate []byte, notBefore time.Time, notAfter time.Time, isRenew bool) *v1.Secret {
secretName := s.getSecretName(domain)
annotationMap := make(map[string]string, 0)
annotationMap["higress.io/cert-domain"] = domain
annotationMap["higress.io/cert-notAfter"] = notAfter.Format("2006-01-02 15:04:05")
annotationMap["higress.io/cert-notBefore"] = notBefore.Format("2006-01-02 15:04:05")
annotationMap["higress.io/cert-renew"] = strconv.FormatBool(isRenew)
if isRenew {
annotationMap["higress.io/cert-renew-time"] = time.Now().Format("2006-01-02 15:04:05")
}
// Required fields:
// - Secret.Data["tls.key"] - TLS private key.
// Secret.Data["tls.crt"] - TLS certificate.
dataMap := make(map[string][]byte, 0)
dataMap["tls.key"] = privateKey
dataMap["tls.crt"] = certificate
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: s.namespace,
Annotations: annotationMap,
},
Type: v1.SecretTypeTLS,
Data: dataMap,
}
return secret
}

115
pkg/cert/server.go Normal file
View File

@@ -0,0 +1,115 @@
// Copyright (c) 2022 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cert
import (
"context"
"fmt"
"net"
"net/http"
"time"
"github.com/caddyserver/certmagic"
"k8s.io/client-go/kubernetes"
)
type Option struct {
Namespace string
ServerAddress string
Email string
}
type Server struct {
httpServer *http.Server
opts *Option
clientSet kubernetes.Interface
controller *Controller
certMgr *CertMgr
}
func NewServer(clientSet kubernetes.Interface, opts *Option) (*Server, error) {
server := &Server{
clientSet: clientSet,
opts: opts,
}
return server, nil
}
func (s *Server) InitDefaultConfig() error {
configMgr, _ := NewConfigMgr(s.opts.Namespace, s.clientSet)
// init config if there is not existed
_, err := configMgr.InitConfig(s.opts.Email)
if err != nil {
return err
}
return nil
}
func (s *Server) InitServer() error {
configMgr, _ := NewConfigMgr(s.opts.Namespace, s.clientSet)
// init config if there is not existed
defaultConfig, err := configMgr.InitConfig(s.opts.Email)
if err != nil {
return err
}
// init certmgr
certMgr, err := InitCertMgr(s.opts, s.clientSet, defaultConfig) // config and start
s.certMgr = certMgr
// init controller
controller, err := NewController(s.clientSet, s.opts.Namespace, certMgr, configMgr)
s.controller = controller
// init http server
s.initHttpServer()
return nil
}
func (s *Server) initHttpServer() error {
CertLog.Infof("server init http server")
ctx := context.Background()
mux := http.NewServeMux()
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "Lookit my cool website over HTTPS!")
})
httpServer := &http.Server{
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 5 * time.Second,
WriteTimeout: 5 * time.Second,
IdleTimeout: 5 * time.Second,
Addr: s.opts.ServerAddress,
BaseContext: func(listener net.Listener) context.Context { return ctx },
}
cfg := s.certMgr.cfg
if len(cfg.Issuers) > 0 {
if am, ok := cfg.Issuers[0].(*certmagic.ACMEIssuer); ok {
httpServer.Handler = am.HTTPChallengeHandler(mux)
}
} else {
httpServer.Handler = mux
}
s.httpServer = httpServer
return nil
}
func (s *Server) Run(stopCh <-chan struct{}) error {
go s.controller.Run(stopCh)
CertLog.Infof("server run")
go func() {
<-stopCh
CertLog.Infof("server http server shutdown now...")
s.httpServer.Shutdown(context.Background())
}()
err := s.httpServer.ListenAndServe()
return err
}

337
pkg/cert/storage.go Normal file
View File

@@ -0,0 +1,337 @@
// Copyright (c) 2022 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cert
import (
"context"
"encoding/json"
"fmt"
"io/fs"
"path"
"strings"
"sync"
"time"
"github.com/caddyserver/certmagic"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
const (
CertificatesPrefix = "/certificates"
ConfigmapStoreCertficatesPrefix = "higress-cert-store-certificates-"
ConfigmapStoreDefaultName = "higress-cert-store-default"
)
var _ certmagic.Storage = (*ConfigmapStorage)(nil)
type ConfigmapStorage struct {
namespace string
client kubernetes.Interface
mux sync.RWMutex
}
type HashValue struct {
K string `json:"k,omitempty"`
V []byte `json:"v,omitempty"`
}
func NewConfigmapStorage(namespace string, client kubernetes.Interface) (certmagic.Storage, error) {
storage := &ConfigmapStorage{
namespace: namespace,
client: client,
}
return storage, nil
}
// Exists returns true if key exists in s.
func (s *ConfigmapStorage) Exists(_ context.Context, key string) bool {
s.mux.RLock()
defer s.mux.RUnlock()
cm, err := s.getConfigmapStoreByKey(key)
if err != nil {
return false
}
if cm.Data == nil {
return false
}
hashKey := fastHash([]byte(key))
if _, ok := cm.Data[hashKey]; ok {
return true
}
return false
}
// Store saves value at key.
func (s *ConfigmapStorage) Store(_ context.Context, key string, value []byte) error {
s.mux.Lock()
defer s.mux.Unlock()
cm, err := s.getConfigmapStoreByKey(key)
if err != nil {
return err
}
if cm.Data == nil {
cm.Data = make(map[string]string, 0)
}
hashKey := fastHash([]byte(key))
hashV := &HashValue{
K: key,
V: value,
}
bytes, err := json.Marshal(hashV)
if err != nil {
return err
}
cm.Data[hashKey] = string(bytes)
return s.updateConfigmap(cm)
}
// Load retrieves the value at key.
func (s *ConfigmapStorage) Load(_ context.Context, key string) ([]byte, error) {
s.mux.RLock()
defer s.mux.RUnlock()
var value []byte
cm, err := s.getConfigmapStoreByKey(key)
if err != nil {
return value, err
}
if cm.Data == nil {
return value, fs.ErrNotExist
}
hashKey := fastHash([]byte(key))
if v, ok := cm.Data[hashKey]; ok {
hV := &HashValue{}
err = json.Unmarshal([]byte(v), hV)
if err != nil {
return value, err
}
return hV.V, nil
}
return value, fs.ErrNotExist
}
// Delete deletes the value at key.
func (s *ConfigmapStorage) Delete(_ context.Context, key string) error {
s.mux.Lock()
defer s.mux.Unlock()
cm, err := s.getConfigmapStoreByKey(key)
if err != nil {
return err
}
if cm.Data == nil {
cm.Data = make(map[string]string, 0)
}
hashKey := fastHash([]byte(key))
delete(cm.Data, hashKey)
return s.updateConfigmap(cm)
}
// List returns all keys that match the prefix.
// If the prefix is "/certificates", it retrieves all ConfigMaps, otherwise only one.
func (s *ConfigmapStorage) List(ctx context.Context, prefix string, recursive bool) ([]string, error) {
s.mux.RLock()
defer s.mux.RUnlock()
var keys []string
var configmapKeys []string
visitedDirs := make(map[string]struct{})
// Check if the prefix corresponds to a specific key
hashPrefix := fastHash([]byte(prefix))
if strings.HasPrefix(prefix, CertificatesPrefix) {
// If the prefix is "/certificates", get all ConfigMaps and traverse each one
// List all ConfigMaps in the namespace with label higress.io/cert-https=true
configmaps, err := s.client.CoreV1().ConfigMaps(s.namespace).List(ctx, metav1.ListOptions{FieldSelector: "metadata.annotations['higress.io/cert-https'] == 'true'"})
if err != nil {
return keys, err
}
for _, cm := range configmaps.Items {
// Check if the ConfigMap name starts with the expected prefix
if strings.HasPrefix(cm.Name, ConfigmapStoreCertficatesPrefix) {
// Add the keys from Data field to the list
for _, v := range cm.Data {
// Unmarshal the value into hashValue struct
var hv HashValue
if err := json.Unmarshal([]byte(v), &hv); err != nil {
return nil, err
}
// Check if the key starts with the specified prefix
if strings.HasPrefix(hv.K, prefix) {
// Add the key to the list
configmapKeys = append(configmapKeys, hv.K)
}
}
}
}
} else {
// If not starting with "/certificates", get the specific ConfigMap
cm, err := s.getConfigmapStoreByKey(prefix)
if err != nil {
return keys, err
}
if _, ok := cm.Data[hashPrefix]; ok {
// The prefix corresponds to a specific key, add it to the list
configmapKeys = append(configmapKeys, prefix)
} else {
// The prefix is considered a directory
for _, v := range cm.Data {
// Unmarshal the value into hashValue struct
var hv HashValue
if err := json.Unmarshal([]byte(v), &hv); err != nil {
return nil, err
}
// Check if the key starts with the specified prefix
if strings.HasPrefix(hv.K, prefix) {
// Add the key to the list
configmapKeys = append(configmapKeys, hv.K)
}
}
}
}
// return all
if recursive {
return configmapKeys, nil
}
// only return sub dirs
for _, key := range configmapKeys {
subPath := strings.TrimPrefix(strings.ReplaceAll(key, prefix, ""), "/")
paths := strings.Split(subPath, "/")
if len(paths) > 0 {
subDir := path.Join(prefix, paths[0])
if _, ok := visitedDirs[subDir]; !ok {
keys = append(keys, subDir)
}
visitedDirs[subDir] = struct{}{}
}
}
return keys, nil
}
// Stat returns information about key. only support for no certificates path
func (s *ConfigmapStorage) Stat(_ context.Context, key string) (certmagic.KeyInfo, error) {
s.mux.RLock()
defer s.mux.RUnlock()
// Create a new KeyInfo struct
info := certmagic.KeyInfo{}
// Get the ConfigMap containing the keys
cm, err := s.getConfigmapStoreByKey(key)
if err != nil {
return info, err
}
// Check if the key exists in the ConfigMap
hashKey := fastHash([]byte(key))
if data, ok := cm.Data[hashKey]; ok {
// The key exists, populate the KeyInfo struct
info.Key = key
info.Modified = time.Now() // Since we're not tracking modification time in ConfigMap
info.Size = int64(len(data))
info.IsTerminal = true
} else {
// Check if there are other keys with the same prefix
prefixKeys := make([]string, 0)
for _, v := range cm.Data {
var hv HashValue
if err := json.Unmarshal([]byte(v), &hv); err != nil {
return info, err
}
// Check if the key starts with the specified prefix
if strings.HasPrefix(hv.K, key) {
// Add the key to the list
prefixKeys = append(prefixKeys, hv.K)
}
}
// If there are multiple keys with the same prefix, then it's not a terminal node
if len(prefixKeys) > 0 {
info.Key = key
info.IsTerminal = false
} else {
return info, fmt.Errorf("prefix '%s' is not existed", key)
}
}
return info, nil
}
// Lock obtains a lock named by the given name. It blocks
// until the lock can be obtained or an error is returned.
func (s *ConfigmapStorage) Lock(ctx context.Context, name string) error {
return nil
}
// Unlock releases the lock for name.
func (s *ConfigmapStorage) Unlock(_ context.Context, name string) error {
return nil
}
func (s *ConfigmapStorage) String() string {
return "ConfigmapStorage"
}
func (s *ConfigmapStorage) getConfigmapStoreNameByKey(key string) string {
parts := strings.SplitN(key, "/", 10)
if len(parts) >= 4 && parts[1] == "certificates" {
domain := strings.TrimSuffix(parts[3], ".crt")
domain = strings.TrimSuffix(domain, ".key")
domain = strings.TrimSuffix(domain, ".json")
issuerKey := parts[2]
return ConfigmapStoreCertficatesPrefix + fastHash([]byte(issuerKey+domain))
}
return ConfigmapStoreDefaultName
}
func (s *ConfigmapStorage) getConfigmapStoreByKey(key string) (*v1.ConfigMap, error) {
configmapName := s.getConfigmapStoreNameByKey(key)
cm, err := s.client.CoreV1().ConfigMaps(s.namespace).Get(context.Background(), configmapName, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
// Save default ConfigMap
cm = &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Namespace: s.namespace,
Name: configmapName,
Annotations: map[string]string{"higress.io/cert-https": "true"},
},
}
_, err = s.client.CoreV1().ConfigMaps(s.namespace).Create(context.Background(), cm, metav1.CreateOptions{})
if err != nil {
return nil, err
}
} else {
return nil, err
}
}
return cm, nil
}
// updateConfigmap adds or updates the annotation higress.io/cert-https to true.
func (s *ConfigmapStorage) updateConfigmap(configmap *v1.ConfigMap) error {
if configmap.ObjectMeta.Annotations == nil {
configmap.ObjectMeta.Annotations = make(map[string]string)
}
configmap.ObjectMeta.Annotations["higress.io/cert-https"] = "true"
_, err := s.client.CoreV1().ConfigMaps(configmap.Namespace).Update(context.Background(), configmap, metav1.UpdateOptions{})
return err
}

325
pkg/cert/storage_test.go Normal file
View File

@@ -0,0 +1,325 @@
// Copyright (c) 2022 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cert
import (
"context"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"testing"
"github.com/stretchr/testify/assert"
"k8s.io/client-go/kubernetes/fake"
)
func TestGetConfigmapStoreNameByKey(t *testing.T) {
// Create a fake client for testing
fakeClient := fake.NewSimpleClientset()
// Create a new ConfigmapStorage instance for testing
namespace := "your-namespace"
storage := &ConfigmapStorage{
namespace: namespace,
client: fakeClient,
}
tests := []struct {
name string
key string
expected string
}{
{
name: "certificate crt",
key: "/certificates/issuerKey/domain.crt",
expected: "higress-cert-store-certificates-" + fastHash([]byte("issuerKey"+"domain")),
},
{
name: "certificate meta",
key: "/certificates/issuerKey/domain.json",
expected: "higress-cert-store-certificates-" + fastHash([]byte("issuerKey"+"domain")),
},
{
name: "certificate key",
key: "/certificates/issuerKey/domain.key",
expected: "higress-cert-store-certificates-" + fastHash([]byte("issuerKey"+"domain")),
},
{
name: "user key",
key: "/users/hello/2",
expected: "higress-cert-store-default",
},
{
name: "Empty Key",
key: "",
expected: "higress-cert-store-default",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
storageName := storage.getConfigmapStoreNameByKey(test.key)
assert.Equal(t, test.expected, storageName)
})
}
}
func TestExists(t *testing.T) {
// Create a fake client for testing
fakeClient := fake.NewSimpleClientset()
// Create a new ConfigmapStorage instance for testing
namespace := "your-namespace"
storage, err := NewConfigmapStorage(namespace, fakeClient)
assert.NoError(t, err)
// Store a test key
testKey := "/certificates/issuer1/domain1.crt"
err = storage.Store(context.Background(), testKey, []byte("test-data"))
assert.NoError(t, err)
// Define test cases
tests := []struct {
name string
key string
shouldExist bool
}{
{
name: "Existing Key",
key: "/certificates/issuer1/domain1.crt",
shouldExist: true,
},
{
name: "Non-Existent Key1",
key: "/certificates/issuer2/domain2.crt",
shouldExist: false,
},
{
name: "Non-Existent Key2",
key: "/users/hello/a",
shouldExist: false,
},
// Add more test cases as needed
}
// Run tests
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
exists := storage.Exists(context.Background(), test.key)
assert.Equal(t, test.shouldExist, exists)
})
}
}
func TestLoad(t *testing.T) {
// Create a fake client for testing
fakeClient := fake.NewSimpleClientset()
// Create a new ConfigmapStorage instance for testing
namespace := "your-namespace"
storage, err := NewConfigmapStorage(namespace, fakeClient)
assert.NoError(t, err)
// Store a test key
testKey := "/certificates/issuer1/domain1.crt"
testValue := []byte("test-data")
err = storage.Store(context.Background(), testKey, testValue)
assert.NoError(t, err)
// Define test cases
tests := []struct {
name string
key string
expected []byte
shouldError bool
}{
{
name: "Existing Key",
key: "/certificates/issuer1/domain1.crt",
expected: testValue,
shouldError: false,
},
{
name: "Non-Existent Key",
key: "/certificates/issuer2/domain2.crt",
expected: nil,
shouldError: true,
},
// Add more test cases as needed
}
// Run tests
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
value, err := storage.Load(context.Background(), test.key)
if test.shouldError {
assert.Error(t, err)
assert.Nil(t, value)
} else {
assert.NoError(t, err)
assert.Equal(t, test.expected, value)
}
})
}
}
func TestStore(t *testing.T) {
// Create a fake client for testing
fakeClient := fake.NewSimpleClientset()
// Create a new ConfigmapStorage instance for testing
namespace := "your-namespace"
storage := ConfigmapStorage{
namespace: namespace,
client: fakeClient,
}
// Define test cases
tests := []struct {
name string
key string
value []byte
expected map[string]string
expectedConfigmapName string
shouldError bool
}{
{
name: "Store Key with /certificates prefix",
key: "/certificates/issuer1/domain1.crt",
value: []byte("test-data1"),
expected: map[string]string{fastHash([]byte("/certificates/issuer1/domain1.crt")): `{"k":"/certificates/issuer1/domain1.crt","v":"dGVzdC1kYXRhMQ=="}`},
expectedConfigmapName: "higress-cert-store-certificates-" + fastHash([]byte("issuer1"+"domain1")),
shouldError: false,
},
{
name: "Store Key with /certificates prefix (additional data)",
key: "/certificates/issuer2/domain2.crt",
value: []byte("test-data2"),
expected: map[string]string{
fastHash([]byte("/certificates/issuer2/domain2.crt")): `{"k":"/certificates/issuer2/domain2.crt","v":"dGVzdC1kYXRhMg=="}`,
},
expectedConfigmapName: "higress-cert-store-certificates-" + fastHash([]byte("issuer2"+"domain2")),
shouldError: false,
},
{
name: "Store Key without /certificates prefix",
key: "/other/path/data.txt",
value: []byte("test-data3"),
expected: map[string]string{fastHash([]byte("/other/path/data.txt")): `{"k":"/other/path/data.txt","v":"dGVzdC1kYXRhMw=="}`},
expectedConfigmapName: "higress-cert-store-default",
shouldError: false,
},
// Add more test cases as needed
}
// Run tests
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
err := storage.Store(context.Background(), test.key, test.value)
if test.shouldError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
// Check the contents of the ConfigMap after storing
configmapName := storage.getConfigmapStoreNameByKey(test.key)
cm, err := fakeClient.CoreV1().ConfigMaps(namespace).Get(context.Background(), configmapName, metav1.GetOptions{})
assert.NoError(t, err)
// Check if the data is as expected
assert.Equal(t, test.expected, cm.Data)
// Check if the configmapName is correct
assert.Equal(t, test.expectedConfigmapName, configmapName)
}
})
}
}
func TestList(t *testing.T) {
// Create a fake client for testing
fakeClient := fake.NewSimpleClientset()
// Create a new ConfigmapStorage instance for testing
namespace := "your-namespace"
storage, err := NewConfigmapStorage(namespace, fakeClient)
assert.NoError(t, err)
// Store some test data
// Store some test data
testKeys := []string{
"/certificates/issuer1/domain1.crt",
"/certificates/issuer1/domain2.crt",
"/certificates/issuer1/domain3.crt", // Added another domain for issuer1
"/certificates/issuer2/domain4.crt",
"/certificates/issuer2/domain5.crt",
"/certificates/issuer3/subdomain1/domain6.crt", // Two-level subdirectory under issuer3
"/certificates/issuer3/subdomain1/subdomain2/domain7.crt", // Two more levels under issuer3
"/other-prefix/key1/file1",
"/other-prefix/key1/file2",
"/other-prefix/key2/file3",
"/other-prefix/key2/file4",
}
for _, key := range testKeys {
err := storage.Store(context.Background(), key, []byte("test-data"))
assert.NoError(t, err)
}
// Define test cases
tests := []struct {
name string
prefix string
recursive bool
expected []string
}{
{
name: "List Certificates (Non-Recursive)",
prefix: "/certificates",
recursive: false,
expected: []string{"/certificates/issuer1", "/certificates/issuer2", "/certificates/issuer3"},
},
{
name: "List Certificates (Recursive)",
prefix: "/certificates",
recursive: true,
expected: []string{"/certificates/issuer1/domain1.crt", "/certificates/issuer1/domain2.crt", "/certificates/issuer1/domain3.crt", "/certificates/issuer2/domain4.crt", "/certificates/issuer2/domain5.crt", "/certificates/issuer3/subdomain1/domain6.crt", "/certificates/issuer3/subdomain1/subdomain2/domain7.crt"},
},
{
name: "List Other Prefix (Non-Recursive)",
prefix: "/other-prefix",
recursive: false,
expected: []string{"/other-prefix/key1", "/other-prefix/key2"},
},
{
name: "List Other Prefix (Non-Recursive)",
prefix: "/other-prefix/key1",
recursive: false,
expected: []string{"/other-prefix/key1/file1", "/other-prefix/key1/file2"},
},
{
name: "List Other Prefix (Recursive)",
prefix: "/other-prefix",
recursive: true,
expected: []string{"/other-prefix/key1/file1", "/other-prefix/key1/file2", "/other-prefix/key2/file3", "/other-prefix/key2/file4"},
},
}
// Run tests
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
keys, err := storage.List(context.Background(), test.prefix, test.recursive)
assert.NoError(t, err)
assert.ElementsMatch(t, test.expected, keys)
})
}
}

97
pkg/cert/util.go Normal file
View File

@@ -0,0 +1,97 @@
// Copyright (c) 2022 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cert
import (
"crypto/x509"
"encoding/pem"
"fmt"
"hash/fnv"
"math/rand"
"net"
"regexp"
"time"
)
// parseCertsFromPEMBundle parses a certificate bundle from top to bottom and returns
// a slice of x509 certificates. This function will error if no certificates are found.
func parseCertsFromPEMBundle(bundle []byte) ([]*x509.Certificate, error) {
var certificates []*x509.Certificate
var certDERBlock *pem.Block
for {
certDERBlock, bundle = pem.Decode(bundle)
if certDERBlock == nil {
break
}
if certDERBlock.Type == "CERTIFICATE" {
cert, err := x509.ParseCertificate(certDERBlock.Bytes)
if err != nil {
return nil, err
}
certificates = append(certificates, cert)
}
}
if len(certificates) == 0 {
return nil, fmt.Errorf("no certificates found in bundle")
}
return certificates, nil
}
func notAfter(cert *x509.Certificate) time.Time {
if cert == nil {
return time.Time{}
}
return cert.NotAfter.Truncate(time.Second).Add(1 * time.Second)
}
func notBefore(cert *x509.Certificate) time.Time {
if cert == nil {
return time.Time{}
}
return cert.NotBefore.Truncate(time.Second).Add(1 * time.Second)
}
// hostOnly returns only the host portion of hostport.
// If there is no port or if there is an error splitting
// the port off, the whole input string is returned.
func hostOnly(hostport string) string {
host, _, err := net.SplitHostPort(hostport)
if err != nil {
return hostport // OK; probably had no port to begin with
}
return host
}
func rangeRandom(min, max int) (number int) {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
number = r.Intn(max-min) + min
return number
}
func ValidateEmail(email string) bool {
pattern := `^[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}$`
regExp := regexp.MustCompile(pattern)
if regExp.MatchString(email) {
return true
} else {
return false
}
}
func fastHash(input []byte) string {
h := fnv.New32a()
h.Write(input)
return fmt.Sprintf("%x", h.Sum32())
}

View File

@@ -76,6 +76,7 @@ func getServerCommand() *cobra.Command {
Debug: true,
NativeIstio: true,
HttpAddress: ":8888",
CertHttpAddress: ":8889",
GrpcAddress: ":15051",
GrpcKeepAliveOptions: keepalive.DefaultOption(),
XdsOptions: bootstrap.XdsOptions{
@@ -117,6 +118,10 @@ func getServerCommand() *cobra.Command {
serveCmd.PersistentFlags().Uint32Var(&serverArgs.GatewayHttpsPort, "gatewayHttpsPort", 443,
"Https listening port of gateway pod")
serveCmd.PersistentFlags().BoolVar(&serverArgs.EnableAutomaticHttps, "enableAutomaticHttps", false, "if true, enables automatic https")
serveCmd.PersistentFlags().StringVar(&serverArgs.AutomaticHttpsEmail, "automaticHttpsEmail", "", "email for automatic https")
serveCmd.PersistentFlags().StringVar(&serverArgs.CertHttpAddress, "certHttpAddress", serverArgs.CertHttpAddress, "the cert http address")
loggingOptions.AttachCobraFlags(serveCmd)
serverArgs.GrpcKeepAliveOptions.AttachCobraFlags(serveCmd)

View File

@@ -17,6 +17,7 @@ package ingress
import (
"errors"
"fmt"
"github.com/alibaba/higress/pkg/cert"
"path"
"reflect"
"sort"
@@ -85,6 +86,8 @@ type controller struct {
secretController secret.SecretController
statusSyncer *statusSyncer
configMgr *cert.ConfigMgr
}
// NewController creates a new Kubernetes controller
@@ -103,6 +106,7 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt
IngressLog.Infof("Skipping IngressClass, resource not supported for cluster %s", options.ClusterId)
}
configMgr, _ := cert.NewConfigMgr(options.SystemNamespace, client.Kube())
c := &controller{
options: options,
queue: q,
@@ -113,6 +117,7 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt
serviceInformer: serviceInformer.Informer(),
serviceLister: serviceInformer.Lister(),
secretController: secretController,
configMgr: configMgr,
}
handler := controllers.LatestVersionHandlerFuncs(controllers.EnqueueForSelf(q))
@@ -371,7 +376,7 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
common.IncrementInvalidIngress(c.options.ClusterId, common.EmptyRule)
return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId)
}
httpsCredentialConfig, _ := c.configMgr.GetConfigFromConfigmap()
for _, rule := range ingressV1Beta.Rules {
// Need create builder for every rule.
domainBuilder := &common.IngressDomainBuilder{
@@ -422,13 +427,19 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
// Get tls secret matching the rule host
secretName := extractTLSSecretName(rule.Host, ingressV1Beta.TLS)
secretNamespace := cfg.Namespace
// If there is no matching secret, try to get it from configmap.
if secretName == "" && httpsCredentialConfig != nil {
secretName = httpsCredentialConfig.MatchSecretNameByDomain(rule.Host)
secretNamespace = c.options.SystemNamespace
}
if secretName == "" {
// There no matching secret, so just skip.
continue
}
domainBuilder.Protocol = common.HTTPS
domainBuilder.SecretName = path.Join(c.options.ClusterId, cfg.Namespace, secretName)
domainBuilder.SecretName = path.Join(c.options.ClusterId, secretNamespace, secretName)
// There is a matching secret and the gateway has already a tls secret.
// We should report the duplicated tls secret event.
@@ -450,7 +461,7 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
Hosts: []string{rule.Host},
Tls: &networking.ServerTLSSettings{
Mode: networking.ServerTLSSettings_SIMPLE,
CredentialName: credentials.ToKubernetesIngressResource(c.options.RawClusterId, cfg.Namespace, secretName),
CredentialName: credentials.ToKubernetesIngressResource(c.options.RawClusterId, secretNamespace, secretName),
},
})

View File

@@ -25,6 +25,7 @@ import (
"sync"
"time"
"github.com/alibaba/higress/pkg/cert"
"github.com/hashicorp/go-multierror"
networking "istio.io/api/networking/v1alpha3"
"istio.io/istio/pilot/pkg/model"
@@ -84,6 +85,8 @@ type controller struct {
secretController secret.SecretController
statusSyncer *statusSyncer
configMgr *cert.ConfigMgr
}
// NewController creates a new Kubernetes controller
@@ -96,6 +99,7 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt
classes := client.KubeInformer().Networking().V1().IngressClasses()
classes.Informer()
configMgr, _ := cert.NewConfigMgr(options.SystemNamespace, client.Kube())
c := &controller{
options: options,
queue: q,
@@ -106,6 +110,7 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt
serviceInformer: serviceInformer.Informer(),
serviceLister: serviceInformer.Lister(),
secretController: secretController,
configMgr: configMgr,
}
handler := controllers.LatestVersionHandlerFuncs(controllers.EnqueueForSelf(q))
@@ -358,7 +363,7 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId)
}
httpsCredentialConfig, _ := c.configMgr.GetConfigFromConfigmap()
for _, rule := range ingressV1.Rules {
// Need create builder for every rule.
domainBuilder := &common.IngressDomainBuilder{
@@ -409,13 +414,19 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
// Get tls secret matching the rule host
secretName := extractTLSSecretName(rule.Host, ingressV1.TLS)
secretNamespace := cfg.Namespace
// If there is no matching secret, try to get it from configmap.
if secretName == "" && httpsCredentialConfig != nil {
secretName = httpsCredentialConfig.MatchSecretNameByDomain(rule.Host)
secretNamespace = c.options.SystemNamespace
}
if secretName == "" {
// There no matching secret, so just skip.
continue
}
domainBuilder.Protocol = common.HTTPS
domainBuilder.SecretName = path.Join(c.options.ClusterId, cfg.Namespace, secretName)
domainBuilder.SecretName = path.Join(c.options.ClusterId, secretNamespace, secretName)
// There is a matching secret and the gateway has already a tls secret.
// We should report the duplicated tls secret event.
@@ -437,7 +448,7 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
Hosts: []string{rule.Host},
Tls: &networking.ServerTLSSettings{
Mode: networking.ServerTLSSettings_SIMPLE,
CredentialName: credentials.ToKubernetesIngressResource(c.options.RawClusterId, cfg.Namespace, secretName),
CredentialName: credentials.ToKubernetesIngressResource(c.options.RawClusterId, secretNamespace, secretName),
},
})