mirror of
https://github.com/alibaba/higress.git
synced 2026-03-07 01:50:51 +08:00
feat:add higress automatic https (#854)
Co-authored-by: 澄潭 <zty98751@alibaba-inc.com>
This commit is contained in:
@@ -20,6 +20,7 @@ import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/alibaba/higress/pkg/cert"
|
||||
"github.com/alibaba/higress/pkg/ingress/kube/common"
|
||||
"github.com/alibaba/higress/pkg/ingress/mcp"
|
||||
"github.com/alibaba/higress/pkg/ingress/translation"
|
||||
@@ -112,6 +113,9 @@ type ServerArgs struct {
|
||||
GatewaySelectorValue string
|
||||
GatewayHttpPort uint32
|
||||
GatewayHttpsPort uint32
|
||||
EnableAutomaticHttps bool
|
||||
AutomaticHttpsEmail string
|
||||
CertHttpAddress string
|
||||
}
|
||||
|
||||
type readinessProbe func() (bool, error)
|
||||
@@ -133,6 +137,7 @@ type Server struct {
|
||||
xdsServer *xds.DiscoveryServer
|
||||
server server.Instance
|
||||
readinessProbes map[string]readinessProbe
|
||||
certServer *cert.Server
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -168,6 +173,7 @@ func NewServer(args *ServerArgs) (*Server, error) {
|
||||
s.initConfigController,
|
||||
s.initRegistryEventHandlers,
|
||||
s.initAuthenticators,
|
||||
s.initAutomaticHttps,
|
||||
}
|
||||
|
||||
for _, f := range initFuncList {
|
||||
@@ -287,6 +293,15 @@ func (s *Server) Start(stop <-chan struct{}) error {
|
||||
}
|
||||
}()
|
||||
|
||||
if s.EnableAutomaticHttps {
|
||||
go func() {
|
||||
log.Infof("starting Automatic Cert HTTP service at %s", s.CertHttpAddress)
|
||||
if err := s.certServer.Run(stop); err != nil {
|
||||
log.Errorf("error serving Automatic Cert HTTP server: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
s.waitForShutDown(stop)
|
||||
return nil
|
||||
}
|
||||
@@ -370,6 +385,26 @@ func (s *Server) initAuthenticators() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) initAutomaticHttps() error {
|
||||
certOption := &cert.Option{
|
||||
Namespace: PodNamespace,
|
||||
ServerAddress: s.CertHttpAddress,
|
||||
Email: s.AutomaticHttpsEmail,
|
||||
}
|
||||
certServer, err := cert.NewServer(s.kubeClient.Kube(), certOption)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.certServer = certServer
|
||||
log.Infof("init cert default config")
|
||||
s.certServer.InitDefaultConfig()
|
||||
if !s.EnableAutomaticHttps {
|
||||
log.Info("automatic https is disabled")
|
||||
return nil
|
||||
}
|
||||
return s.certServer.InitServer()
|
||||
}
|
||||
|
||||
func (s *Server) initKubeClient() error {
|
||||
if s.kubeClient != nil {
|
||||
// Already initialized by startup arguments
|
||||
|
||||
219
pkg/cert/certmgr.go
Normal file
219
pkg/cert/certmgr.go
Normal file
@@ -0,0 +1,219 @@
|
||||
// Copyright (c) 2022 Alibaba Group Holding Ltd.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cert
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/caddyserver/certmagic"
|
||||
"github.com/mholt/acmez"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
const (
|
||||
EventCertObtained = "cert_obtained"
|
||||
)
|
||||
|
||||
type CertMgr struct {
|
||||
cfg *certmagic.Config
|
||||
client kubernetes.Interface
|
||||
namespace string
|
||||
mux sync.RWMutex
|
||||
storage certmagic.Storage
|
||||
cache *certmagic.Cache
|
||||
myACME *certmagic.ACMEIssuer
|
||||
ingressSolver acmez.Solver
|
||||
configMgr *ConfigMgr
|
||||
secretMgr *SecretMgr
|
||||
}
|
||||
|
||||
func InitCertMgr(opts *Option, clientSet kubernetes.Interface, config *Config) (*CertMgr, error) {
|
||||
CertLog.Infof("certmgr init config: %+v", config)
|
||||
// Init certmagic config
|
||||
// First make a pointer to a Cache as we need to reference the same Cache in
|
||||
// GetConfigForCert below.
|
||||
var cache *certmagic.Cache
|
||||
var storage certmagic.Storage
|
||||
storage, _ = NewConfigmapStorage(opts.Namespace, clientSet)
|
||||
renewalWindowRatio := float64(config.RenewBeforeDays / RenewMaxDays)
|
||||
magicConfig := certmagic.Config{
|
||||
RenewalWindowRatio: renewalWindowRatio,
|
||||
Storage: storage,
|
||||
}
|
||||
cache = certmagic.NewCache(certmagic.CacheOptions{
|
||||
GetConfigForCert: func(cert certmagic.Certificate) (*certmagic.Config, error) {
|
||||
// Here we use New to get a valid Config associated with the same cache.
|
||||
// The provided Config is used as a template and will be completed with
|
||||
// any defaults that are set in the Default config.
|
||||
return certmagic.New(cache, magicConfig), nil
|
||||
},
|
||||
})
|
||||
// init certmagic
|
||||
cfg := certmagic.New(cache, magicConfig)
|
||||
// Init certmagic acme
|
||||
issuer := config.GetIssuer(IssuerTypeLetsencrypt)
|
||||
if issuer == nil {
|
||||
// should never happen here
|
||||
return nil, fmt.Errorf("there is no Letsencrypt Issuer found in config")
|
||||
}
|
||||
|
||||
myACME := certmagic.NewACMEIssuer(cfg, certmagic.ACMEIssuer{
|
||||
//CA: certmagic.LetsEncryptStagingCA,
|
||||
CA: certmagic.LetsEncryptProductionCA,
|
||||
Email: issuer.Email,
|
||||
Agreed: true,
|
||||
DisableHTTPChallenge: false,
|
||||
DisableTLSALPNChallenge: true,
|
||||
})
|
||||
// inject http01 solver
|
||||
ingressSolver, _ := NewIngressSolver(opts.Namespace, clientSet, myACME)
|
||||
myACME.Http01Solver = ingressSolver
|
||||
// init issuers
|
||||
cfg.Issuers = []certmagic.Issuer{myACME}
|
||||
|
||||
configMgr, _ := NewConfigMgr(opts.Namespace, clientSet)
|
||||
secretMgr, _ := NewSecretMgr(opts.Namespace, clientSet)
|
||||
|
||||
certMgr := &CertMgr{
|
||||
cfg: cfg,
|
||||
client: clientSet,
|
||||
namespace: opts.Namespace,
|
||||
myACME: myACME,
|
||||
ingressSolver: ingressSolver,
|
||||
configMgr: configMgr,
|
||||
secretMgr: secretMgr,
|
||||
cache: cache,
|
||||
}
|
||||
certMgr.cfg.OnEvent = certMgr.OnEvent
|
||||
return certMgr, nil
|
||||
}
|
||||
func (s *CertMgr) Reconcile(ctx context.Context, oldConfig *Config, newConfig *Config) error {
|
||||
CertLog.Infof("cermgr reconcile old config:%+v to new config:%+v", oldConfig, newConfig)
|
||||
// sync email
|
||||
if oldConfig != nil && newConfig != nil {
|
||||
oldIssuer := oldConfig.GetIssuer(IssuerTypeLetsencrypt)
|
||||
newIssuer := newConfig.GetIssuer(IssuerTypeLetsencrypt)
|
||||
if oldIssuer.Email != newIssuer.Email {
|
||||
// TODO before sync email, maybe need to clean up cache and account
|
||||
}
|
||||
}
|
||||
|
||||
// sync domains
|
||||
newDomains := make([]string, 0)
|
||||
newDomainsMap := make(map[string]string, 0)
|
||||
removeDomains := make([]string, 0)
|
||||
|
||||
if newConfig != nil {
|
||||
for _, config := range newConfig.CredentialConfig {
|
||||
if config.TLSIssuer == IssuerTypeLetsencrypt {
|
||||
for _, newDomain := range config.Domains {
|
||||
newDomains = append(newDomains, newDomain)
|
||||
newDomainsMap[newDomain] = newDomain
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if oldConfig != nil {
|
||||
for _, config := range oldConfig.CredentialConfig {
|
||||
if config.TLSIssuer == IssuerTypeLetsencrypt {
|
||||
for _, oldDomain := range config.Domains {
|
||||
if _, ok := newDomainsMap[oldDomain]; !ok {
|
||||
removeDomains = append(removeDomains, oldDomain)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if newConfig.AutomaticHttps == true {
|
||||
newIssuer := newConfig.GetIssuer(IssuerTypeLetsencrypt)
|
||||
// clean up unused domains
|
||||
s.cleanSync(context.Background(), removeDomains)
|
||||
// sync email
|
||||
s.myACME.Email = newIssuer.Email
|
||||
// sync RenewalWindowRatio
|
||||
s.cfg.RenewalWindowRatio = float64(newConfig.RenewBeforeDays / RenewMaxDays)
|
||||
// start cache
|
||||
s.cache.Start()
|
||||
// sync domains
|
||||
s.manageSync(context.Background(), newDomains)
|
||||
s.configMgr.SetConfig(newConfig)
|
||||
} else {
|
||||
// stop cache maintainAssets
|
||||
s.cache.Stop()
|
||||
s.configMgr.SetConfig(newConfig)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *CertMgr) manageSync(ctx context.Context, domainNames []string) error {
|
||||
CertLog.Infof("cert manage sync domains:%v", domainNames)
|
||||
return s.cfg.ManageSync(ctx, domainNames)
|
||||
}
|
||||
|
||||
func (s *CertMgr) cleanSync(ctx context.Context, domainNames []string) error {
|
||||
//TODO implement clean up domains
|
||||
CertLog.Infof("cert clean sync domains:%v", domainNames)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *CertMgr) OnEvent(ctx context.Context, event string, data map[string]any) error {
|
||||
CertLog.Infof("certmgr receive event:% data:%+v", event, data)
|
||||
/**
|
||||
event: cert_obtained
|
||||
cfg.emit(ctx, "cert_obtained", map[string]any{
|
||||
"renewal": true,
|
||||
"remaining": timeLeft,
|
||||
"identifier": name,
|
||||
"issuer": issuerKey,
|
||||
"storage_path": StorageKeys.CertsSitePrefix(issuerKey, certKey),
|
||||
"private_key_path": StorageKeys.SitePrivateKey(issuerKey, certKey),
|
||||
"certificate_path": StorageKeys.SiteCert(issuerKey, certKey),
|
||||
"metadata_path": StorageKeys.SiteMeta(issuerKey, certKey),
|
||||
})
|
||||
*/
|
||||
if event == EventCertObtained {
|
||||
// obtain certificate and update secret
|
||||
domain := data["identifier"].(string)
|
||||
isRenew := data["renewal"].(bool)
|
||||
privateKeyPath := data["private_key_path"].(string)
|
||||
certificatePath := data["certificate_path"].(string)
|
||||
privateKey, err := s.cfg.Storage.Load(context.Background(), privateKeyPath)
|
||||
certificate, err := s.cfg.Storage.Load(context.Background(), certificatePath)
|
||||
certChain, err := parseCertsFromPEMBundle(certificate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
notAfterTime := notAfter(certChain[0])
|
||||
notBeforeTime := notBefore(certChain[0])
|
||||
secretName := s.configMgr.GetConfig().GetSecretNameByDomain(IssuerTypeLetsencrypt, domain)
|
||||
if len(secretName) == 0 {
|
||||
CertLog.Errorf("can not find secret name for domain % in config", domain)
|
||||
return nil
|
||||
}
|
||||
err2 := s.secretMgr.Update(domain, secretName, privateKey, certificate, notBeforeTime, notAfterTime, isRenew)
|
||||
if err2 != nil {
|
||||
CertLog.Errorf("update secretName %s for domain %s error: %v", secretName, domain, err2)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
290
pkg/cert/config.go
Normal file
290
pkg/cert/config.go
Normal file
@@ -0,0 +1,290 @@
|
||||
// Copyright (c) 2022 Alibaba Group Holding Ltd.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cert
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"istio.io/istio/pkg/config/host"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"sigs.k8s.io/yaml"
|
||||
)
|
||||
|
||||
const (
|
||||
ConfigmapCertName = "higress-https"
|
||||
ConfigmapCertConfigKey = "cert"
|
||||
DefaultRenewBeforeDays = 30
|
||||
RenewMaxDays = 90
|
||||
)
|
||||
|
||||
type IssuerName string
|
||||
|
||||
const (
|
||||
IssuerTypeAliyunSSL IssuerName = "aliyunssl"
|
||||
IssuerTypeLetsencrypt IssuerName = "letsencrypt"
|
||||
)
|
||||
|
||||
// Config is the configuration of automatic https.
|
||||
type Config struct {
|
||||
AutomaticHttps bool `json:"automaticHttps"`
|
||||
RenewBeforeDays int `json:"renewBeforeDays"`
|
||||
CredentialConfig []CredentialEntry `json:"credentialConfig"`
|
||||
ACMEIssuer []ACMEIssuerEntry `json:"acmeIssuer"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
func (c *Config) GetIssuer(issuerName IssuerName) *ACMEIssuerEntry {
|
||||
for _, issuer := range c.ACMEIssuer {
|
||||
if issuer.Name == issuerName {
|
||||
return &issuer
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Config) MatchSecretNameByDomain(domain string) string {
|
||||
for _, credential := range c.CredentialConfig {
|
||||
for _, credDomain := range credential.Domains {
|
||||
if host.Name(strings.ToLower(domain)).SubsetOf(host.Name(strings.ToLower(credDomain))) {
|
||||
return credential.TLSSecret
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (c *Config) GetSecretNameByDomain(issuerName IssuerName, domain string) string {
|
||||
for _, credential := range c.CredentialConfig {
|
||||
if credential.TLSIssuer == issuerName {
|
||||
for _, credDomain := range credential.Domains {
|
||||
if host.Name(strings.ToLower(domain)).SubsetOf(host.Name(strings.ToLower(credDomain))) {
|
||||
return credential.TLSSecret
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (c *Config) Validate() error {
|
||||
// check acmeIssuer
|
||||
if len(c.ACMEIssuer) == 0 {
|
||||
return fmt.Errorf("acmeIssuer is empty")
|
||||
}
|
||||
for _, issuer := range c.ACMEIssuer {
|
||||
switch issuer.Name {
|
||||
case IssuerTypeLetsencrypt:
|
||||
if issuer.Email == "" {
|
||||
return fmt.Errorf("acmeIssuer %s email is empty", issuer.Name)
|
||||
}
|
||||
if !ValidateEmail(issuer.Email) {
|
||||
return fmt.Errorf("acmeIssuer %s email %s is invalid", issuer.Name, issuer.Email)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("acmeIssuer name %s is not supported", issuer.Name)
|
||||
}
|
||||
}
|
||||
// check credentialConfig
|
||||
for _, credential := range c.CredentialConfig {
|
||||
if len(credential.Domains) == 0 {
|
||||
return fmt.Errorf("credentialConfig domains is empty")
|
||||
}
|
||||
if credential.TLSSecret == "" {
|
||||
return fmt.Errorf("credentialConfig tlsSecret is empty")
|
||||
}
|
||||
if credential.TLSIssuer == IssuerTypeLetsencrypt {
|
||||
if len(credential.Domains) > 1 {
|
||||
return fmt.Errorf("credentialConfig tlsIssuer %s only support one domain", credential.TLSIssuer)
|
||||
}
|
||||
}
|
||||
if credential.TLSIssuer != IssuerTypeLetsencrypt && len(credential.TLSIssuer) > 0 {
|
||||
return fmt.Errorf("credential tls issuer %s is not support", credential.TLSIssuer)
|
||||
}
|
||||
}
|
||||
|
||||
if c.RenewBeforeDays <= 0 {
|
||||
return fmt.Errorf("RenewBeforeDays should be large than zero")
|
||||
}
|
||||
|
||||
if c.RenewBeforeDays >= RenewMaxDays {
|
||||
return fmt.Errorf("RenewBeforeDays should be less than %d", RenewMaxDays)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type CredentialEntry struct {
|
||||
Domains []string `json:"domains"`
|
||||
TLSIssuer IssuerName `json:"tlsIssuer,omitempty"`
|
||||
TLSSecret string `json:"tlsSecret,omitempty"`
|
||||
CACertSecret string `json:"cacertSecret,omitempty"`
|
||||
}
|
||||
|
||||
type ACMEIssuerEntry struct {
|
||||
Name IssuerName `json:"name"`
|
||||
Email string `json:"email"`
|
||||
AK string `json:"ak"` // Only applicable for certain issuers like 'aliyunssl'
|
||||
SK string `json:"sk"` // Only applicable for certain issuers like 'aliyunssl'
|
||||
}
|
||||
type ConfigMgr struct {
|
||||
client kubernetes.Interface
|
||||
config atomic.Value
|
||||
namespace string
|
||||
}
|
||||
|
||||
func (c *ConfigMgr) SetConfig(config *Config) {
|
||||
c.config.Store(config)
|
||||
}
|
||||
|
||||
func (c *ConfigMgr) GetConfig() *Config {
|
||||
value := c.config.Load()
|
||||
if value != nil {
|
||||
if config, ok := value.(*Config); ok {
|
||||
return config
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ConfigMgr) InitConfig(email string) (*Config, error) {
|
||||
var defaultConfig *Config
|
||||
cm, err := c.GetConfigmap()
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
if len(strings.TrimSpace(email)) == 0 {
|
||||
email = getRandEmail()
|
||||
}
|
||||
defaultConfig = newDefaultConfig(email)
|
||||
err2 := c.ApplyConfigmap(defaultConfig)
|
||||
if err2 != nil {
|
||||
return nil, err2
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
} else {
|
||||
defaultConfig, err = c.ParseConfigFromConfigmap(cm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return defaultConfig, nil
|
||||
}
|
||||
|
||||
func (c *ConfigMgr) ParseConfigFromConfigmap(configmap *v1.ConfigMap) (*Config, error) {
|
||||
if _, ok := configmap.Data[ConfigmapCertConfigKey]; !ok {
|
||||
return nil, fmt.Errorf("no cert key %s in configmap %s", ConfigmapCertConfigKey, configmap.Name)
|
||||
}
|
||||
|
||||
config := newDefaultConfig("")
|
||||
if err := yaml.Unmarshal([]byte(configmap.Data[ConfigmapCertConfigKey]), config); err != nil {
|
||||
return nil, fmt.Errorf("data:%s, convert to higress config error, error: %+v", configmap.Data[ConfigmapCertConfigKey], err)
|
||||
}
|
||||
// validate config
|
||||
if err := config.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func (c *ConfigMgr) GetConfigFromConfigmap() (*Config, error) {
|
||||
var config *Config
|
||||
cm, err := c.GetConfigmap()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
config, err = c.ParseConfigFromConfigmap(cm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func (c *ConfigMgr) GetConfigmap() (configmap *v1.ConfigMap, err error) {
|
||||
configmapName := ConfigmapCertName
|
||||
cm, err := c.client.CoreV1().ConfigMaps(c.namespace).Get(context.Background(), configmapName, metav1.GetOptions{})
|
||||
return cm, err
|
||||
}
|
||||
|
||||
func (c *ConfigMgr) ApplyConfigmap(config *Config) error {
|
||||
configmapName := ConfigmapCertName
|
||||
cm := &v1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: c.namespace,
|
||||
Name: configmapName,
|
||||
},
|
||||
}
|
||||
bytes, err := yaml.Marshal(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cm.Data = make(map[string]string, 0)
|
||||
cm.Data[ConfigmapCertConfigKey] = string(bytes)
|
||||
|
||||
_, err = c.client.CoreV1().ConfigMaps(c.namespace).Get(context.Background(), configmapName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
if _, err = c.client.CoreV1().ConfigMaps(c.namespace).Create(context.Background(), cm, metav1.CreateOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if _, err = c.client.CoreV1().ConfigMaps(c.namespace).Update(context.Background(), cm, metav1.UpdateOptions{}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewConfigMgr(namespace string, client kubernetes.Interface) (*ConfigMgr, error) {
|
||||
configMgr := &ConfigMgr{
|
||||
client: client,
|
||||
namespace: namespace,
|
||||
}
|
||||
return configMgr, nil
|
||||
}
|
||||
|
||||
func newDefaultConfig(email string) *Config {
|
||||
|
||||
defaultIssuer := []ACMEIssuerEntry{
|
||||
{
|
||||
Name: IssuerTypeLetsencrypt,
|
||||
Email: email,
|
||||
},
|
||||
}
|
||||
defaultCredentialConfig := make([]CredentialEntry, 0)
|
||||
config := &Config{
|
||||
AutomaticHttps: true,
|
||||
RenewBeforeDays: DefaultRenewBeforeDays,
|
||||
ACMEIssuer: defaultIssuer,
|
||||
CredentialConfig: defaultCredentialConfig,
|
||||
Version: time.Now().Format("20060102030405"),
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
func getRandEmail() string {
|
||||
num1 := rangeRandom(100, 100000)
|
||||
num2 := rangeRandom(100, 100000)
|
||||
return fmt.Sprintf("your%d@yours%d.com", num1, num2)
|
||||
}
|
||||
122
pkg/cert/config_test.go
Normal file
122
pkg/cert/config_test.go
Normal file
@@ -0,0 +1,122 @@
|
||||
// Copyright (c) 2022 Alibaba Group Holding Ltd.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cert
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestMatchSecretNameByDomain(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
domain string
|
||||
credentialCfg []CredentialEntry
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "Exact match",
|
||||
domain: "example.com",
|
||||
credentialCfg: []CredentialEntry{
|
||||
{
|
||||
Domains: []string{"example.com"},
|
||||
TLSSecret: "example-com-tls",
|
||||
},
|
||||
},
|
||||
expected: "example-com-tls",
|
||||
},
|
||||
|
||||
{
|
||||
name: "Exact match ignore case ",
|
||||
domain: "eXample.com",
|
||||
credentialCfg: []CredentialEntry{
|
||||
{
|
||||
Domains: []string{"example.com"},
|
||||
TLSSecret: "example-com-tls",
|
||||
},
|
||||
},
|
||||
expected: "example-com-tls",
|
||||
},
|
||||
{
|
||||
name: "Wildcard match",
|
||||
domain: "sub.example.com",
|
||||
credentialCfg: []CredentialEntry{
|
||||
{
|
||||
Domains: []string{"*.example.com"},
|
||||
TLSSecret: "wildcard-example-com-tls",
|
||||
},
|
||||
},
|
||||
expected: "wildcard-example-com-tls",
|
||||
},
|
||||
|
||||
{
|
||||
name: "Wildcard match ignore case",
|
||||
domain: "sub.Example.com",
|
||||
credentialCfg: []CredentialEntry{
|
||||
{
|
||||
Domains: []string{"*.example.com"},
|
||||
TLSSecret: "wildcard-example-com-tls",
|
||||
},
|
||||
},
|
||||
expected: "wildcard-example-com-tls",
|
||||
},
|
||||
{
|
||||
name: "* match",
|
||||
domain: "blog.example.co.uk",
|
||||
credentialCfg: []CredentialEntry{
|
||||
{
|
||||
Domains: []string{"*"},
|
||||
TLSSecret: "blog-co-uk-tls",
|
||||
},
|
||||
},
|
||||
expected: "blog-co-uk-tls",
|
||||
},
|
||||
{
|
||||
name: "No match",
|
||||
domain: "unknown.com",
|
||||
credentialCfg: []CredentialEntry{
|
||||
{
|
||||
Domains: []string{"example.com"},
|
||||
TLSSecret: "example-com-tls",
|
||||
},
|
||||
},
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "Multiple matches - first match wins",
|
||||
domain: "example.com",
|
||||
credentialCfg: []CredentialEntry{
|
||||
{
|
||||
Domains: []string{"example.com"},
|
||||
TLSSecret: "example-com-tls",
|
||||
},
|
||||
{
|
||||
Domains: []string{"*.example.com"},
|
||||
TLSSecret: "wildcard-example-com-tls",
|
||||
},
|
||||
},
|
||||
expected: "example-com-tls",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := Config{CredentialConfig: tt.credentialCfg}
|
||||
result := cfg.MatchSecretNameByDomain(tt.domain)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
165
pkg/cert/controller.go
Normal file
165
pkg/cert/controller.go
Normal file
@@ -0,0 +1,165 @@
|
||||
// Copyright (c) 2022 Alibaba Group Holding Ltd.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cert
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/client-go/informers"
|
||||
v1informer "k8s.io/client-go/informers/core/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
)
|
||||
|
||||
const (
|
||||
workNum = 1
|
||||
maxRetry = 2
|
||||
configMapName = "higress-https"
|
||||
)
|
||||
|
||||
type Controller struct {
|
||||
namespace string
|
||||
ConfigMapInformer v1informer.ConfigMapInformer
|
||||
client kubernetes.Interface
|
||||
queue workqueue.RateLimitingInterface
|
||||
configMgr *ConfigMgr
|
||||
server *Server
|
||||
certMgr *CertMgr
|
||||
factory informers.SharedInformerFactory
|
||||
}
|
||||
|
||||
func (c *Controller) addConfigmap(obj interface{}) {
|
||||
key, err := cache.MetaNamespaceKeyFunc(obj)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
namespace, name, _ := cache.SplitMetaNamespaceKey(key)
|
||||
if namespace != c.namespace || name != configMapName {
|
||||
return
|
||||
}
|
||||
c.enqueue(name)
|
||||
|
||||
}
|
||||
func (c *Controller) updateConfigmap(oldObj interface{}, newObj interface{}) {
|
||||
key, err := cache.MetaNamespaceKeyFunc(oldObj)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
namespace, name, _ := cache.SplitMetaNamespaceKey(key)
|
||||
if namespace != c.namespace || name != configMapName {
|
||||
return
|
||||
}
|
||||
if reflect.DeepEqual(oldObj, newObj) {
|
||||
return
|
||||
}
|
||||
c.enqueue(name)
|
||||
}
|
||||
|
||||
func (c *Controller) enqueue(name string) {
|
||||
c.queue.Add(name)
|
||||
}
|
||||
|
||||
func (c *Controller) cachesSynced() bool {
|
||||
return c.ConfigMapInformer.Informer().HasSynced()
|
||||
}
|
||||
|
||||
func (c *Controller) Run(stopCh <-chan struct{}) error {
|
||||
defer runtime.HandleCrash()
|
||||
defer c.queue.ShutDown()
|
||||
CertLog.Info("Waiting for informer caches to sync")
|
||||
c.factory.Start(stopCh)
|
||||
if ok := cache.WaitForCacheSync(stopCh, c.cachesSynced); !ok {
|
||||
return fmt.Errorf("failed to wait for caches to sync")
|
||||
}
|
||||
CertLog.Info("Starting controller")
|
||||
// Launch one workers to process configmap resources
|
||||
for i := 0; i < workNum; i++ {
|
||||
go wait.Until(c.worker, time.Minute, stopCh)
|
||||
}
|
||||
CertLog.Info("Started workers")
|
||||
<-stopCh
|
||||
CertLog.Info("Shutting down workers")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Controller) worker() {
|
||||
for c.processNextItem() {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) processNextItem() bool {
|
||||
item, shutdown := c.queue.Get()
|
||||
if shutdown {
|
||||
return false
|
||||
}
|
||||
defer c.queue.Done(item)
|
||||
key := item.(string)
|
||||
CertLog.Infof("controller process item:%s", key)
|
||||
err := c.syncConfigmap(key)
|
||||
if err != nil {
|
||||
c.handleError(key, err)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *Controller) syncConfigmap(key string) error {
|
||||
configmap, err := c.ConfigMapInformer.Lister().ConfigMaps(c.namespace).Get(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newConfig, err := c.configMgr.ParseConfigFromConfigmap(configmap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
oldConfig := c.configMgr.GetConfig()
|
||||
// reconcile old config and new config
|
||||
return c.certMgr.Reconcile(context.Background(), oldConfig, newConfig)
|
||||
}
|
||||
|
||||
func (c *Controller) handleError(key string, err error) {
|
||||
runtime.HandleError(err)
|
||||
CertLog.Errorf("%+v", err)
|
||||
c.queue.Forget(key)
|
||||
}
|
||||
|
||||
func NewController(client kubernetes.Interface, namespace string, certMgr *CertMgr, configMgr *ConfigMgr) (*Controller, error) {
|
||||
kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(client, 0, informers.WithNamespace(namespace))
|
||||
configmapInformer := kubeInformerFactory.Core().V1().ConfigMaps()
|
||||
c := &Controller{
|
||||
certMgr: certMgr,
|
||||
configMgr: configMgr,
|
||||
client: client,
|
||||
namespace: namespace,
|
||||
factory: kubeInformerFactory,
|
||||
ConfigMapInformer: configmapInformer,
|
||||
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ingressManage"),
|
||||
}
|
||||
|
||||
CertLog.Info("Setting up configmap informer event handlers")
|
||||
configmapInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: c.addConfigmap,
|
||||
UpdateFunc: c.updateConfigmap,
|
||||
})
|
||||
|
||||
return c, nil
|
||||
}
|
||||
158
pkg/cert/ingress.go
Normal file
158
pkg/cert/ingress.go
Normal file
@@ -0,0 +1,158 @@
|
||||
// Copyright (c) 2022 Alibaba Group Holding Ltd.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cert
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/caddyserver/certmagic"
|
||||
"github.com/mholt/acmez"
|
||||
"github.com/mholt/acmez/acme"
|
||||
v1 "k8s.io/api/networking/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
const (
|
||||
IngressClassName = "higress"
|
||||
IngressServiceName = "higress-controller"
|
||||
IngressNamePefix = "higress-http-solver-"
|
||||
IngressPathPrefix = "/.well-known/acme-challenge/"
|
||||
IngressServicePort = 8889
|
||||
)
|
||||
|
||||
type IngressSolver struct {
|
||||
client kubernetes.Interface
|
||||
acmeIssuer *certmagic.ACMEIssuer
|
||||
solversMu sync.Mutex
|
||||
namespace string
|
||||
ingressDelay time.Duration
|
||||
}
|
||||
|
||||
func NewIngressSolver(namespace string, client kubernetes.Interface, acmeIssuer *certmagic.ACMEIssuer) (acmez.Solver, error) {
|
||||
solver := &IngressSolver{
|
||||
namespace: namespace,
|
||||
client: client,
|
||||
acmeIssuer: acmeIssuer,
|
||||
ingressDelay: 5 * time.Second,
|
||||
}
|
||||
return solver, nil
|
||||
}
|
||||
|
||||
func (s *IngressSolver) Present(_ context.Context, challenge acme.Challenge) error {
|
||||
CertLog.Infof("ingress solver present challenge:%+v", challenge)
|
||||
s.solversMu.Lock()
|
||||
defer s.solversMu.Unlock()
|
||||
ingressName := s.getIngressName(challenge)
|
||||
ingress := s.constructIngress(challenge)
|
||||
CertLog.Infof("update ingress name:%s, ingress:%v", ingressName, ingress)
|
||||
_, err := s.client.NetworkingV1().Ingresses(s.namespace).Get(context.Background(), ingressName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
// create ingress
|
||||
_, err2 := s.client.NetworkingV1().Ingresses(s.namespace).Create(context.Background(), ingress, metav1.CreateOptions{})
|
||||
return err2
|
||||
}
|
||||
return err
|
||||
}
|
||||
_, err1 := s.client.NetworkingV1().Ingresses(s.namespace).Update(context.Background(), ingress, metav1.UpdateOptions{})
|
||||
if err1 != nil {
|
||||
return err1
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *IngressSolver) Wait(ctx context.Context, challenge acme.Challenge) error {
|
||||
CertLog.Infof("ingress solver wait challenge:%+v", challenge)
|
||||
// wait for ingress ready
|
||||
if s.ingressDelay > 0 {
|
||||
select {
|
||||
case <-time.After(s.ingressDelay):
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
CertLog.Infof("ingress solver wait challenge done")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *IngressSolver) CleanUp(_ context.Context, challenge acme.Challenge) error {
|
||||
CertLog.Infof("ingress solver cleanup challenge:%+v", challenge)
|
||||
s.solversMu.Lock()
|
||||
defer s.solversMu.Unlock()
|
||||
ingressName := s.getIngressName(challenge)
|
||||
CertLog.Infof("cleanup ingress name:%s", ingressName)
|
||||
err := s.client.NetworkingV1().Ingresses(s.namespace).Delete(context.Background(), ingressName, metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *IngressSolver) Delete(_ context.Context, challenge acme.Challenge) error {
|
||||
s.solversMu.Lock()
|
||||
defer s.solversMu.Unlock()
|
||||
err := s.client.NetworkingV1().Ingresses(s.namespace).Delete(context.Background(), s.getIngressName(challenge), metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *IngressSolver) getIngressName(challenge acme.Challenge) string {
|
||||
return IngressNamePefix + strings.ReplaceAll(challenge.Identifier.Value, ".", "-")
|
||||
}
|
||||
|
||||
func (s *IngressSolver) constructIngress(challenge acme.Challenge) *v1.Ingress {
|
||||
ingressClassName := IngressClassName
|
||||
ingressDomain := challenge.Identifier.Value
|
||||
ingressPath := IngressPathPrefix + challenge.Token
|
||||
ingress := v1.Ingress{}
|
||||
ingress.Name = s.getIngressName(challenge)
|
||||
ingress.Namespace = s.namespace
|
||||
pathType := v1.PathTypePrefix
|
||||
ingress.Spec = v1.IngressSpec{
|
||||
IngressClassName: &ingressClassName,
|
||||
Rules: []v1.IngressRule{
|
||||
{
|
||||
Host: ingressDomain,
|
||||
IngressRuleValue: v1.IngressRuleValue{
|
||||
HTTP: &v1.HTTPIngressRuleValue{
|
||||
Paths: []v1.HTTPIngressPath{
|
||||
{
|
||||
Path: ingressPath,
|
||||
PathType: &pathType,
|
||||
Backend: v1.IngressBackend{
|
||||
Service: &v1.IngressServiceBackend{
|
||||
Name: IngressServiceName,
|
||||
Port: v1.ServiceBackendPort{
|
||||
Number: IngressServicePort,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return &ingress
|
||||
}
|
||||
19
pkg/cert/log.go
Normal file
19
pkg/cert/log.go
Normal file
@@ -0,0 +1,19 @@
|
||||
// Copyright (c) 2022 Alibaba Group Holding Ltd.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cert
|
||||
|
||||
import "istio.io/pkg/log"
|
||||
|
||||
var CertLog = log.RegisterScope("cert", "Higress Cert process.", 0)
|
||||
108
pkg/cert/secret.go
Normal file
108
pkg/cert/secret.go
Normal file
@@ -0,0 +1,108 @@
|
||||
// Copyright (c) 2022 Alibaba Group Holding Ltd.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cert
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
const (
|
||||
SecretNamePrefix = "higress-secret-"
|
||||
)
|
||||
|
||||
type SecretMgr struct {
|
||||
client kubernetes.Interface
|
||||
namespace string
|
||||
}
|
||||
|
||||
func NewSecretMgr(namespace string, client kubernetes.Interface) (*SecretMgr, error) {
|
||||
secretMgr := &SecretMgr{
|
||||
namespace: namespace,
|
||||
client: client,
|
||||
}
|
||||
|
||||
return secretMgr, nil
|
||||
}
|
||||
|
||||
func (s *SecretMgr) Update(domain string, secretName string, privateKey []byte, certificate []byte, notBefore time.Time, notAfter time.Time, isRenew bool) error {
|
||||
//secretName := s.getSecretName(domain)
|
||||
secret := s.constructSecret(domain, privateKey, certificate, notBefore, notAfter, isRenew)
|
||||
_, err := s.client.CoreV1().Secrets(s.namespace).Get(context.Background(), secretName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
// create secret
|
||||
_, err2 := s.client.CoreV1().Secrets(s.namespace).Create(context.Background(), secret, metav1.CreateOptions{})
|
||||
return err2
|
||||
}
|
||||
return err
|
||||
}
|
||||
// check secret annotations
|
||||
if _, ok := secret.Annotations["higress.io/cert-domain"]; !ok {
|
||||
return fmt.Errorf("the secret name %s is not automatic https secret name for the domain:%s, please rename it in config", secretName, domain)
|
||||
}
|
||||
_, err1 := s.client.CoreV1().Secrets(s.namespace).Update(context.Background(), secret, metav1.UpdateOptions{})
|
||||
if err1 != nil {
|
||||
return err1
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *SecretMgr) Delete(domain string) error {
|
||||
secretName := s.getSecretName(domain)
|
||||
err := s.client.CoreV1().Secrets(s.namespace).Delete(context.Background(), secretName, metav1.DeleteOptions{})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *SecretMgr) getSecretName(domain string) string {
|
||||
return SecretNamePrefix + strings.ReplaceAll(strings.TrimSpace(domain), ".", "-")
|
||||
}
|
||||
|
||||
func (s *SecretMgr) constructSecret(domain string, privateKey []byte, certificate []byte, notBefore time.Time, notAfter time.Time, isRenew bool) *v1.Secret {
|
||||
secretName := s.getSecretName(domain)
|
||||
annotationMap := make(map[string]string, 0)
|
||||
annotationMap["higress.io/cert-domain"] = domain
|
||||
annotationMap["higress.io/cert-notAfter"] = notAfter.Format("2006-01-02 15:04:05")
|
||||
annotationMap["higress.io/cert-notBefore"] = notBefore.Format("2006-01-02 15:04:05")
|
||||
annotationMap["higress.io/cert-renew"] = strconv.FormatBool(isRenew)
|
||||
if isRenew {
|
||||
annotationMap["higress.io/cert-renew-time"] = time.Now().Format("2006-01-02 15:04:05")
|
||||
}
|
||||
// Required fields:
|
||||
// - Secret.Data["tls.key"] - TLS private key.
|
||||
// Secret.Data["tls.crt"] - TLS certificate.
|
||||
dataMap := make(map[string][]byte, 0)
|
||||
dataMap["tls.key"] = privateKey
|
||||
dataMap["tls.crt"] = certificate
|
||||
secret := &v1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: secretName,
|
||||
Namespace: s.namespace,
|
||||
Annotations: annotationMap,
|
||||
},
|
||||
Type: v1.SecretTypeTLS,
|
||||
Data: dataMap,
|
||||
}
|
||||
return secret
|
||||
}
|
||||
115
pkg/cert/server.go
Normal file
115
pkg/cert/server.go
Normal file
@@ -0,0 +1,115 @@
|
||||
// Copyright (c) 2022 Alibaba Group Holding Ltd.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cert
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/caddyserver/certmagic"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
type Option struct {
|
||||
Namespace string
|
||||
ServerAddress string
|
||||
Email string
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
httpServer *http.Server
|
||||
opts *Option
|
||||
clientSet kubernetes.Interface
|
||||
controller *Controller
|
||||
certMgr *CertMgr
|
||||
}
|
||||
|
||||
func NewServer(clientSet kubernetes.Interface, opts *Option) (*Server, error) {
|
||||
server := &Server{
|
||||
clientSet: clientSet,
|
||||
opts: opts,
|
||||
}
|
||||
return server, nil
|
||||
}
|
||||
|
||||
func (s *Server) InitDefaultConfig() error {
|
||||
configMgr, _ := NewConfigMgr(s.opts.Namespace, s.clientSet)
|
||||
// init config if there is not existed
|
||||
_, err := configMgr.InitConfig(s.opts.Email)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) InitServer() error {
|
||||
configMgr, _ := NewConfigMgr(s.opts.Namespace, s.clientSet)
|
||||
// init config if there is not existed
|
||||
defaultConfig, err := configMgr.InitConfig(s.opts.Email)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// init certmgr
|
||||
certMgr, err := InitCertMgr(s.opts, s.clientSet, defaultConfig) // config and start
|
||||
s.certMgr = certMgr
|
||||
// init controller
|
||||
controller, err := NewController(s.clientSet, s.opts.Namespace, certMgr, configMgr)
|
||||
s.controller = controller
|
||||
// init http server
|
||||
s.initHttpServer()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) initHttpServer() error {
|
||||
CertLog.Infof("server init http server")
|
||||
ctx := context.Background()
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, "Lookit my cool website over HTTPS!")
|
||||
})
|
||||
httpServer := &http.Server{
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 5 * time.Second,
|
||||
WriteTimeout: 5 * time.Second,
|
||||
IdleTimeout: 5 * time.Second,
|
||||
Addr: s.opts.ServerAddress,
|
||||
BaseContext: func(listener net.Listener) context.Context { return ctx },
|
||||
}
|
||||
cfg := s.certMgr.cfg
|
||||
if len(cfg.Issuers) > 0 {
|
||||
if am, ok := cfg.Issuers[0].(*certmagic.ACMEIssuer); ok {
|
||||
httpServer.Handler = am.HTTPChallengeHandler(mux)
|
||||
}
|
||||
} else {
|
||||
httpServer.Handler = mux
|
||||
}
|
||||
s.httpServer = httpServer
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) Run(stopCh <-chan struct{}) error {
|
||||
go s.controller.Run(stopCh)
|
||||
CertLog.Infof("server run")
|
||||
go func() {
|
||||
<-stopCh
|
||||
CertLog.Infof("server http server shutdown now...")
|
||||
s.httpServer.Shutdown(context.Background())
|
||||
}()
|
||||
err := s.httpServer.ListenAndServe()
|
||||
return err
|
||||
}
|
||||
337
pkg/cert/storage.go
Normal file
337
pkg/cert/storage.go
Normal file
@@ -0,0 +1,337 @@
|
||||
// Copyright (c) 2022 Alibaba Group Holding Ltd.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cert
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/caddyserver/certmagic"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
const (
|
||||
CertificatesPrefix = "/certificates"
|
||||
ConfigmapStoreCertficatesPrefix = "higress-cert-store-certificates-"
|
||||
ConfigmapStoreDefaultName = "higress-cert-store-default"
|
||||
)
|
||||
|
||||
var _ certmagic.Storage = (*ConfigmapStorage)(nil)
|
||||
|
||||
type ConfigmapStorage struct {
|
||||
namespace string
|
||||
client kubernetes.Interface
|
||||
mux sync.RWMutex
|
||||
}
|
||||
|
||||
type HashValue struct {
|
||||
K string `json:"k,omitempty"`
|
||||
V []byte `json:"v,omitempty"`
|
||||
}
|
||||
|
||||
func NewConfigmapStorage(namespace string, client kubernetes.Interface) (certmagic.Storage, error) {
|
||||
storage := &ConfigmapStorage{
|
||||
namespace: namespace,
|
||||
client: client,
|
||||
}
|
||||
return storage, nil
|
||||
}
|
||||
|
||||
// Exists returns true if key exists in s.
|
||||
func (s *ConfigmapStorage) Exists(_ context.Context, key string) bool {
|
||||
s.mux.RLock()
|
||||
defer s.mux.RUnlock()
|
||||
cm, err := s.getConfigmapStoreByKey(key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if cm.Data == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
hashKey := fastHash([]byte(key))
|
||||
if _, ok := cm.Data[hashKey]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Store saves value at key.
|
||||
func (s *ConfigmapStorage) Store(_ context.Context, key string, value []byte) error {
|
||||
s.mux.Lock()
|
||||
defer s.mux.Unlock()
|
||||
cm, err := s.getConfigmapStoreByKey(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cm.Data == nil {
|
||||
cm.Data = make(map[string]string, 0)
|
||||
}
|
||||
|
||||
hashKey := fastHash([]byte(key))
|
||||
hashV := &HashValue{
|
||||
K: key,
|
||||
V: value,
|
||||
}
|
||||
bytes, err := json.Marshal(hashV)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cm.Data[hashKey] = string(bytes)
|
||||
return s.updateConfigmap(cm)
|
||||
}
|
||||
|
||||
// Load retrieves the value at key.
|
||||
func (s *ConfigmapStorage) Load(_ context.Context, key string) ([]byte, error) {
|
||||
s.mux.RLock()
|
||||
defer s.mux.RUnlock()
|
||||
var value []byte
|
||||
cm, err := s.getConfigmapStoreByKey(key)
|
||||
if err != nil {
|
||||
return value, err
|
||||
}
|
||||
if cm.Data == nil {
|
||||
return value, fs.ErrNotExist
|
||||
}
|
||||
|
||||
hashKey := fastHash([]byte(key))
|
||||
if v, ok := cm.Data[hashKey]; ok {
|
||||
hV := &HashValue{}
|
||||
err = json.Unmarshal([]byte(v), hV)
|
||||
if err != nil {
|
||||
return value, err
|
||||
}
|
||||
return hV.V, nil
|
||||
}
|
||||
return value, fs.ErrNotExist
|
||||
}
|
||||
|
||||
// Delete deletes the value at key.
|
||||
func (s *ConfigmapStorage) Delete(_ context.Context, key string) error {
|
||||
s.mux.Lock()
|
||||
defer s.mux.Unlock()
|
||||
cm, err := s.getConfigmapStoreByKey(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cm.Data == nil {
|
||||
cm.Data = make(map[string]string, 0)
|
||||
}
|
||||
hashKey := fastHash([]byte(key))
|
||||
delete(cm.Data, hashKey)
|
||||
return s.updateConfigmap(cm)
|
||||
}
|
||||
|
||||
// List returns all keys that match the prefix.
|
||||
// If the prefix is "/certificates", it retrieves all ConfigMaps, otherwise only one.
|
||||
func (s *ConfigmapStorage) List(ctx context.Context, prefix string, recursive bool) ([]string, error) {
|
||||
s.mux.RLock()
|
||||
defer s.mux.RUnlock()
|
||||
var keys []string
|
||||
var configmapKeys []string
|
||||
visitedDirs := make(map[string]struct{})
|
||||
|
||||
// Check if the prefix corresponds to a specific key
|
||||
hashPrefix := fastHash([]byte(prefix))
|
||||
if strings.HasPrefix(prefix, CertificatesPrefix) {
|
||||
// If the prefix is "/certificates", get all ConfigMaps and traverse each one
|
||||
// List all ConfigMaps in the namespace with label higress.io/cert-https=true
|
||||
configmaps, err := s.client.CoreV1().ConfigMaps(s.namespace).List(ctx, metav1.ListOptions{FieldSelector: "metadata.annotations['higress.io/cert-https'] == 'true'"})
|
||||
if err != nil {
|
||||
return keys, err
|
||||
}
|
||||
|
||||
for _, cm := range configmaps.Items {
|
||||
// Check if the ConfigMap name starts with the expected prefix
|
||||
if strings.HasPrefix(cm.Name, ConfigmapStoreCertficatesPrefix) {
|
||||
// Add the keys from Data field to the list
|
||||
for _, v := range cm.Data {
|
||||
// Unmarshal the value into hashValue struct
|
||||
var hv HashValue
|
||||
if err := json.Unmarshal([]byte(v), &hv); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Check if the key starts with the specified prefix
|
||||
if strings.HasPrefix(hv.K, prefix) {
|
||||
// Add the key to the list
|
||||
configmapKeys = append(configmapKeys, hv.K)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// If not starting with "/certificates", get the specific ConfigMap
|
||||
cm, err := s.getConfigmapStoreByKey(prefix)
|
||||
if err != nil {
|
||||
return keys, err
|
||||
}
|
||||
|
||||
if _, ok := cm.Data[hashPrefix]; ok {
|
||||
// The prefix corresponds to a specific key, add it to the list
|
||||
configmapKeys = append(configmapKeys, prefix)
|
||||
} else {
|
||||
// The prefix is considered a directory
|
||||
for _, v := range cm.Data {
|
||||
// Unmarshal the value into hashValue struct
|
||||
var hv HashValue
|
||||
if err := json.Unmarshal([]byte(v), &hv); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Check if the key starts with the specified prefix
|
||||
if strings.HasPrefix(hv.K, prefix) {
|
||||
// Add the key to the list
|
||||
configmapKeys = append(configmapKeys, hv.K)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// return all
|
||||
if recursive {
|
||||
return configmapKeys, nil
|
||||
}
|
||||
|
||||
// only return sub dirs
|
||||
for _, key := range configmapKeys {
|
||||
subPath := strings.TrimPrefix(strings.ReplaceAll(key, prefix, ""), "/")
|
||||
paths := strings.Split(subPath, "/")
|
||||
if len(paths) > 0 {
|
||||
subDir := path.Join(prefix, paths[0])
|
||||
if _, ok := visitedDirs[subDir]; !ok {
|
||||
keys = append(keys, subDir)
|
||||
}
|
||||
visitedDirs[subDir] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// Stat returns information about key. only support for no certificates path
|
||||
func (s *ConfigmapStorage) Stat(_ context.Context, key string) (certmagic.KeyInfo, error) {
|
||||
s.mux.RLock()
|
||||
defer s.mux.RUnlock()
|
||||
// Create a new KeyInfo struct
|
||||
info := certmagic.KeyInfo{}
|
||||
|
||||
// Get the ConfigMap containing the keys
|
||||
cm, err := s.getConfigmapStoreByKey(key)
|
||||
if err != nil {
|
||||
return info, err
|
||||
}
|
||||
|
||||
// Check if the key exists in the ConfigMap
|
||||
hashKey := fastHash([]byte(key))
|
||||
if data, ok := cm.Data[hashKey]; ok {
|
||||
// The key exists, populate the KeyInfo struct
|
||||
info.Key = key
|
||||
info.Modified = time.Now() // Since we're not tracking modification time in ConfigMap
|
||||
info.Size = int64(len(data))
|
||||
info.IsTerminal = true
|
||||
} else {
|
||||
// Check if there are other keys with the same prefix
|
||||
prefixKeys := make([]string, 0)
|
||||
for _, v := range cm.Data {
|
||||
var hv HashValue
|
||||
if err := json.Unmarshal([]byte(v), &hv); err != nil {
|
||||
return info, err
|
||||
}
|
||||
// Check if the key starts with the specified prefix
|
||||
if strings.HasPrefix(hv.K, key) {
|
||||
// Add the key to the list
|
||||
prefixKeys = append(prefixKeys, hv.K)
|
||||
}
|
||||
}
|
||||
// If there are multiple keys with the same prefix, then it's not a terminal node
|
||||
if len(prefixKeys) > 0 {
|
||||
info.Key = key
|
||||
info.IsTerminal = false
|
||||
} else {
|
||||
return info, fmt.Errorf("prefix '%s' is not existed", key)
|
||||
}
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// Lock obtains a lock named by the given name. It blocks
|
||||
// until the lock can be obtained or an error is returned.
|
||||
func (s *ConfigmapStorage) Lock(ctx context.Context, name string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unlock releases the lock for name.
|
||||
func (s *ConfigmapStorage) Unlock(_ context.Context, name string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *ConfigmapStorage) String() string {
|
||||
return "ConfigmapStorage"
|
||||
}
|
||||
|
||||
func (s *ConfigmapStorage) getConfigmapStoreNameByKey(key string) string {
|
||||
parts := strings.SplitN(key, "/", 10)
|
||||
if len(parts) >= 4 && parts[1] == "certificates" {
|
||||
domain := strings.TrimSuffix(parts[3], ".crt")
|
||||
domain = strings.TrimSuffix(domain, ".key")
|
||||
domain = strings.TrimSuffix(domain, ".json")
|
||||
issuerKey := parts[2]
|
||||
return ConfigmapStoreCertficatesPrefix + fastHash([]byte(issuerKey+domain))
|
||||
}
|
||||
return ConfigmapStoreDefaultName
|
||||
}
|
||||
|
||||
func (s *ConfigmapStorage) getConfigmapStoreByKey(key string) (*v1.ConfigMap, error) {
|
||||
configmapName := s.getConfigmapStoreNameByKey(key)
|
||||
cm, err := s.client.CoreV1().ConfigMaps(s.namespace).Get(context.Background(), configmapName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
// Save default ConfigMap
|
||||
cm = &v1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: s.namespace,
|
||||
Name: configmapName,
|
||||
Annotations: map[string]string{"higress.io/cert-https": "true"},
|
||||
},
|
||||
}
|
||||
_, err = s.client.CoreV1().ConfigMaps(s.namespace).Create(context.Background(), cm, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return cm, nil
|
||||
}
|
||||
|
||||
// updateConfigmap adds or updates the annotation higress.io/cert-https to true.
|
||||
func (s *ConfigmapStorage) updateConfigmap(configmap *v1.ConfigMap) error {
|
||||
if configmap.ObjectMeta.Annotations == nil {
|
||||
configmap.ObjectMeta.Annotations = make(map[string]string)
|
||||
}
|
||||
configmap.ObjectMeta.Annotations["higress.io/cert-https"] = "true"
|
||||
|
||||
_, err := s.client.CoreV1().ConfigMaps(configmap.Namespace).Update(context.Background(), configmap, metav1.UpdateOptions{})
|
||||
return err
|
||||
}
|
||||
325
pkg/cert/storage_test.go
Normal file
325
pkg/cert/storage_test.go
Normal file
@@ -0,0 +1,325 @@
|
||||
// Copyright (c) 2022 Alibaba Group Holding Ltd.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cert
|
||||
|
||||
import (
|
||||
"context"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
)
|
||||
|
||||
func TestGetConfigmapStoreNameByKey(t *testing.T) {
|
||||
// Create a fake client for testing
|
||||
fakeClient := fake.NewSimpleClientset()
|
||||
// Create a new ConfigmapStorage instance for testing
|
||||
namespace := "your-namespace"
|
||||
storage := &ConfigmapStorage{
|
||||
namespace: namespace,
|
||||
client: fakeClient,
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "certificate crt",
|
||||
key: "/certificates/issuerKey/domain.crt",
|
||||
expected: "higress-cert-store-certificates-" + fastHash([]byte("issuerKey"+"domain")),
|
||||
},
|
||||
{
|
||||
name: "certificate meta",
|
||||
key: "/certificates/issuerKey/domain.json",
|
||||
expected: "higress-cert-store-certificates-" + fastHash([]byte("issuerKey"+"domain")),
|
||||
},
|
||||
{
|
||||
name: "certificate key",
|
||||
key: "/certificates/issuerKey/domain.key",
|
||||
expected: "higress-cert-store-certificates-" + fastHash([]byte("issuerKey"+"domain")),
|
||||
},
|
||||
{
|
||||
name: "user key",
|
||||
key: "/users/hello/2",
|
||||
expected: "higress-cert-store-default",
|
||||
},
|
||||
{
|
||||
name: "Empty Key",
|
||||
key: "",
|
||||
expected: "higress-cert-store-default",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
storageName := storage.getConfigmapStoreNameByKey(test.key)
|
||||
assert.Equal(t, test.expected, storageName)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExists(t *testing.T) {
|
||||
// Create a fake client for testing
|
||||
fakeClient := fake.NewSimpleClientset()
|
||||
|
||||
// Create a new ConfigmapStorage instance for testing
|
||||
namespace := "your-namespace"
|
||||
storage, err := NewConfigmapStorage(namespace, fakeClient)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Store a test key
|
||||
testKey := "/certificates/issuer1/domain1.crt"
|
||||
err = storage.Store(context.Background(), testKey, []byte("test-data"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Define test cases
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
shouldExist bool
|
||||
}{
|
||||
{
|
||||
name: "Existing Key",
|
||||
key: "/certificates/issuer1/domain1.crt",
|
||||
shouldExist: true,
|
||||
},
|
||||
{
|
||||
name: "Non-Existent Key1",
|
||||
key: "/certificates/issuer2/domain2.crt",
|
||||
shouldExist: false,
|
||||
},
|
||||
{
|
||||
name: "Non-Existent Key2",
|
||||
key: "/users/hello/a",
|
||||
shouldExist: false,
|
||||
},
|
||||
// Add more test cases as needed
|
||||
}
|
||||
|
||||
// Run tests
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
exists := storage.Exists(context.Background(), test.key)
|
||||
assert.Equal(t, test.shouldExist, exists)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad(t *testing.T) {
|
||||
// Create a fake client for testing
|
||||
fakeClient := fake.NewSimpleClientset()
|
||||
|
||||
// Create a new ConfigmapStorage instance for testing
|
||||
namespace := "your-namespace"
|
||||
storage, err := NewConfigmapStorage(namespace, fakeClient)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Store a test key
|
||||
testKey := "/certificates/issuer1/domain1.crt"
|
||||
testValue := []byte("test-data")
|
||||
err = storage.Store(context.Background(), testKey, testValue)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Define test cases
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
expected []byte
|
||||
shouldError bool
|
||||
}{
|
||||
{
|
||||
name: "Existing Key",
|
||||
key: "/certificates/issuer1/domain1.crt",
|
||||
expected: testValue,
|
||||
shouldError: false,
|
||||
},
|
||||
{
|
||||
name: "Non-Existent Key",
|
||||
key: "/certificates/issuer2/domain2.crt",
|
||||
expected: nil,
|
||||
shouldError: true,
|
||||
},
|
||||
// Add more test cases as needed
|
||||
}
|
||||
|
||||
// Run tests
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
value, err := storage.Load(context.Background(), test.key)
|
||||
if test.shouldError {
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, value)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, test.expected, value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStore(t *testing.T) {
|
||||
// Create a fake client for testing
|
||||
fakeClient := fake.NewSimpleClientset()
|
||||
|
||||
// Create a new ConfigmapStorage instance for testing
|
||||
namespace := "your-namespace"
|
||||
storage := ConfigmapStorage{
|
||||
namespace: namespace,
|
||||
client: fakeClient,
|
||||
}
|
||||
|
||||
// Define test cases
|
||||
tests := []struct {
|
||||
name string
|
||||
key string
|
||||
value []byte
|
||||
expected map[string]string
|
||||
expectedConfigmapName string
|
||||
shouldError bool
|
||||
}{
|
||||
{
|
||||
name: "Store Key with /certificates prefix",
|
||||
key: "/certificates/issuer1/domain1.crt",
|
||||
value: []byte("test-data1"),
|
||||
expected: map[string]string{fastHash([]byte("/certificates/issuer1/domain1.crt")): `{"k":"/certificates/issuer1/domain1.crt","v":"dGVzdC1kYXRhMQ=="}`},
|
||||
expectedConfigmapName: "higress-cert-store-certificates-" + fastHash([]byte("issuer1"+"domain1")),
|
||||
shouldError: false,
|
||||
},
|
||||
{
|
||||
name: "Store Key with /certificates prefix (additional data)",
|
||||
key: "/certificates/issuer2/domain2.crt",
|
||||
value: []byte("test-data2"),
|
||||
expected: map[string]string{
|
||||
fastHash([]byte("/certificates/issuer2/domain2.crt")): `{"k":"/certificates/issuer2/domain2.crt","v":"dGVzdC1kYXRhMg=="}`,
|
||||
},
|
||||
expectedConfigmapName: "higress-cert-store-certificates-" + fastHash([]byte("issuer2"+"domain2")),
|
||||
shouldError: false,
|
||||
},
|
||||
{
|
||||
name: "Store Key without /certificates prefix",
|
||||
key: "/other/path/data.txt",
|
||||
value: []byte("test-data3"),
|
||||
expected: map[string]string{fastHash([]byte("/other/path/data.txt")): `{"k":"/other/path/data.txt","v":"dGVzdC1kYXRhMw=="}`},
|
||||
expectedConfigmapName: "higress-cert-store-default",
|
||||
shouldError: false,
|
||||
},
|
||||
// Add more test cases as needed
|
||||
}
|
||||
|
||||
// Run tests
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
err := storage.Store(context.Background(), test.key, test.value)
|
||||
if test.shouldError {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Check the contents of the ConfigMap after storing
|
||||
configmapName := storage.getConfigmapStoreNameByKey(test.key)
|
||||
cm, err := fakeClient.CoreV1().ConfigMaps(namespace).Get(context.Background(), configmapName, metav1.GetOptions{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Check if the data is as expected
|
||||
assert.Equal(t, test.expected, cm.Data)
|
||||
|
||||
// Check if the configmapName is correct
|
||||
assert.Equal(t, test.expectedConfigmapName, configmapName)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestList(t *testing.T) {
|
||||
// Create a fake client for testing
|
||||
fakeClient := fake.NewSimpleClientset()
|
||||
|
||||
// Create a new ConfigmapStorage instance for testing
|
||||
namespace := "your-namespace"
|
||||
storage, err := NewConfigmapStorage(namespace, fakeClient)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Store some test data
|
||||
// Store some test data
|
||||
testKeys := []string{
|
||||
"/certificates/issuer1/domain1.crt",
|
||||
"/certificates/issuer1/domain2.crt",
|
||||
"/certificates/issuer1/domain3.crt", // Added another domain for issuer1
|
||||
"/certificates/issuer2/domain4.crt",
|
||||
"/certificates/issuer2/domain5.crt",
|
||||
"/certificates/issuer3/subdomain1/domain6.crt", // Two-level subdirectory under issuer3
|
||||
"/certificates/issuer3/subdomain1/subdomain2/domain7.crt", // Two more levels under issuer3
|
||||
"/other-prefix/key1/file1",
|
||||
"/other-prefix/key1/file2",
|
||||
"/other-prefix/key2/file3",
|
||||
"/other-prefix/key2/file4",
|
||||
}
|
||||
|
||||
for _, key := range testKeys {
|
||||
err := storage.Store(context.Background(), key, []byte("test-data"))
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
// Define test cases
|
||||
tests := []struct {
|
||||
name string
|
||||
prefix string
|
||||
recursive bool
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "List Certificates (Non-Recursive)",
|
||||
prefix: "/certificates",
|
||||
recursive: false,
|
||||
expected: []string{"/certificates/issuer1", "/certificates/issuer2", "/certificates/issuer3"},
|
||||
},
|
||||
{
|
||||
name: "List Certificates (Recursive)",
|
||||
prefix: "/certificates",
|
||||
recursive: true,
|
||||
expected: []string{"/certificates/issuer1/domain1.crt", "/certificates/issuer1/domain2.crt", "/certificates/issuer1/domain3.crt", "/certificates/issuer2/domain4.crt", "/certificates/issuer2/domain5.crt", "/certificates/issuer3/subdomain1/domain6.crt", "/certificates/issuer3/subdomain1/subdomain2/domain7.crt"},
|
||||
},
|
||||
{
|
||||
name: "List Other Prefix (Non-Recursive)",
|
||||
prefix: "/other-prefix",
|
||||
recursive: false,
|
||||
expected: []string{"/other-prefix/key1", "/other-prefix/key2"},
|
||||
},
|
||||
|
||||
{
|
||||
name: "List Other Prefix (Non-Recursive)",
|
||||
prefix: "/other-prefix/key1",
|
||||
recursive: false,
|
||||
expected: []string{"/other-prefix/key1/file1", "/other-prefix/key1/file2"},
|
||||
},
|
||||
{
|
||||
name: "List Other Prefix (Recursive)",
|
||||
prefix: "/other-prefix",
|
||||
recursive: true,
|
||||
expected: []string{"/other-prefix/key1/file1", "/other-prefix/key1/file2", "/other-prefix/key2/file3", "/other-prefix/key2/file4"},
|
||||
},
|
||||
}
|
||||
|
||||
// Run tests
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
keys, err := storage.List(context.Background(), test.prefix, test.recursive)
|
||||
assert.NoError(t, err)
|
||||
assert.ElementsMatch(t, test.expected, keys)
|
||||
})
|
||||
}
|
||||
}
|
||||
97
pkg/cert/util.go
Normal file
97
pkg/cert/util.go
Normal file
@@ -0,0 +1,97 @@
|
||||
// Copyright (c) 2022 Alibaba Group Holding Ltd.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cert
|
||||
|
||||
import (
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"math/rand"
|
||||
"net"
|
||||
"regexp"
|
||||
"time"
|
||||
)
|
||||
|
||||
// parseCertsFromPEMBundle parses a certificate bundle from top to bottom and returns
|
||||
// a slice of x509 certificates. This function will error if no certificates are found.
|
||||
func parseCertsFromPEMBundle(bundle []byte) ([]*x509.Certificate, error) {
|
||||
var certificates []*x509.Certificate
|
||||
var certDERBlock *pem.Block
|
||||
for {
|
||||
certDERBlock, bundle = pem.Decode(bundle)
|
||||
if certDERBlock == nil {
|
||||
break
|
||||
}
|
||||
if certDERBlock.Type == "CERTIFICATE" {
|
||||
cert, err := x509.ParseCertificate(certDERBlock.Bytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
certificates = append(certificates, cert)
|
||||
}
|
||||
}
|
||||
if len(certificates) == 0 {
|
||||
return nil, fmt.Errorf("no certificates found in bundle")
|
||||
}
|
||||
return certificates, nil
|
||||
}
|
||||
|
||||
func notAfter(cert *x509.Certificate) time.Time {
|
||||
if cert == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return cert.NotAfter.Truncate(time.Second).Add(1 * time.Second)
|
||||
}
|
||||
|
||||
func notBefore(cert *x509.Certificate) time.Time {
|
||||
if cert == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return cert.NotBefore.Truncate(time.Second).Add(1 * time.Second)
|
||||
}
|
||||
|
||||
// hostOnly returns only the host portion of hostport.
|
||||
// If there is no port or if there is an error splitting
|
||||
// the port off, the whole input string is returned.
|
||||
func hostOnly(hostport string) string {
|
||||
host, _, err := net.SplitHostPort(hostport)
|
||||
if err != nil {
|
||||
return hostport // OK; probably had no port to begin with
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
func rangeRandom(min, max int) (number int) {
|
||||
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
number = r.Intn(max-min) + min
|
||||
return number
|
||||
}
|
||||
|
||||
func ValidateEmail(email string) bool {
|
||||
pattern := `^[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}$`
|
||||
regExp := regexp.MustCompile(pattern)
|
||||
if regExp.MatchString(email) {
|
||||
return true
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func fastHash(input []byte) string {
|
||||
h := fnv.New32a()
|
||||
h.Write(input)
|
||||
return fmt.Sprintf("%x", h.Sum32())
|
||||
}
|
||||
@@ -76,6 +76,7 @@ func getServerCommand() *cobra.Command {
|
||||
Debug: true,
|
||||
NativeIstio: true,
|
||||
HttpAddress: ":8888",
|
||||
CertHttpAddress: ":8889",
|
||||
GrpcAddress: ":15051",
|
||||
GrpcKeepAliveOptions: keepalive.DefaultOption(),
|
||||
XdsOptions: bootstrap.XdsOptions{
|
||||
@@ -117,6 +118,10 @@ func getServerCommand() *cobra.Command {
|
||||
serveCmd.PersistentFlags().Uint32Var(&serverArgs.GatewayHttpsPort, "gatewayHttpsPort", 443,
|
||||
"Https listening port of gateway pod")
|
||||
|
||||
serveCmd.PersistentFlags().BoolVar(&serverArgs.EnableAutomaticHttps, "enableAutomaticHttps", false, "if true, enables automatic https")
|
||||
serveCmd.PersistentFlags().StringVar(&serverArgs.AutomaticHttpsEmail, "automaticHttpsEmail", "", "email for automatic https")
|
||||
serveCmd.PersistentFlags().StringVar(&serverArgs.CertHttpAddress, "certHttpAddress", serverArgs.CertHttpAddress, "the cert http address")
|
||||
|
||||
loggingOptions.AttachCobraFlags(serveCmd)
|
||||
serverArgs.GrpcKeepAliveOptions.AttachCobraFlags(serveCmd)
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ package ingress
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/alibaba/higress/pkg/cert"
|
||||
"path"
|
||||
"reflect"
|
||||
"sort"
|
||||
@@ -85,6 +86,8 @@ type controller struct {
|
||||
secretController secret.SecretController
|
||||
|
||||
statusSyncer *statusSyncer
|
||||
|
||||
configMgr *cert.ConfigMgr
|
||||
}
|
||||
|
||||
// NewController creates a new Kubernetes controller
|
||||
@@ -103,6 +106,7 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt
|
||||
IngressLog.Infof("Skipping IngressClass, resource not supported for cluster %s", options.ClusterId)
|
||||
}
|
||||
|
||||
configMgr, _ := cert.NewConfigMgr(options.SystemNamespace, client.Kube())
|
||||
c := &controller{
|
||||
options: options,
|
||||
queue: q,
|
||||
@@ -113,6 +117,7 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt
|
||||
serviceInformer: serviceInformer.Informer(),
|
||||
serviceLister: serviceInformer.Lister(),
|
||||
secretController: secretController,
|
||||
configMgr: configMgr,
|
||||
}
|
||||
|
||||
handler := controllers.LatestVersionHandlerFuncs(controllers.EnqueueForSelf(q))
|
||||
@@ -371,7 +376,7 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
|
||||
common.IncrementInvalidIngress(c.options.ClusterId, common.EmptyRule)
|
||||
return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId)
|
||||
}
|
||||
|
||||
httpsCredentialConfig, _ := c.configMgr.GetConfigFromConfigmap()
|
||||
for _, rule := range ingressV1Beta.Rules {
|
||||
// Need create builder for every rule.
|
||||
domainBuilder := &common.IngressDomainBuilder{
|
||||
@@ -422,13 +427,19 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
|
||||
|
||||
// Get tls secret matching the rule host
|
||||
secretName := extractTLSSecretName(rule.Host, ingressV1Beta.TLS)
|
||||
secretNamespace := cfg.Namespace
|
||||
// If there is no matching secret, try to get it from configmap.
|
||||
if secretName == "" && httpsCredentialConfig != nil {
|
||||
secretName = httpsCredentialConfig.MatchSecretNameByDomain(rule.Host)
|
||||
secretNamespace = c.options.SystemNamespace
|
||||
}
|
||||
if secretName == "" {
|
||||
// There no matching secret, so just skip.
|
||||
continue
|
||||
}
|
||||
|
||||
domainBuilder.Protocol = common.HTTPS
|
||||
domainBuilder.SecretName = path.Join(c.options.ClusterId, cfg.Namespace, secretName)
|
||||
domainBuilder.SecretName = path.Join(c.options.ClusterId, secretNamespace, secretName)
|
||||
|
||||
// There is a matching secret and the gateway has already a tls secret.
|
||||
// We should report the duplicated tls secret event.
|
||||
@@ -450,7 +461,7 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
|
||||
Hosts: []string{rule.Host},
|
||||
Tls: &networking.ServerTLSSettings{
|
||||
Mode: networking.ServerTLSSettings_SIMPLE,
|
||||
CredentialName: credentials.ToKubernetesIngressResource(c.options.RawClusterId, cfg.Namespace, secretName),
|
||||
CredentialName: credentials.ToKubernetesIngressResource(c.options.RawClusterId, secretNamespace, secretName),
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/alibaba/higress/pkg/cert"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
networking "istio.io/api/networking/v1alpha3"
|
||||
"istio.io/istio/pilot/pkg/model"
|
||||
@@ -84,6 +85,8 @@ type controller struct {
|
||||
secretController secret.SecretController
|
||||
|
||||
statusSyncer *statusSyncer
|
||||
|
||||
configMgr *cert.ConfigMgr
|
||||
}
|
||||
|
||||
// NewController creates a new Kubernetes controller
|
||||
@@ -96,6 +99,7 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt
|
||||
classes := client.KubeInformer().Networking().V1().IngressClasses()
|
||||
classes.Informer()
|
||||
|
||||
configMgr, _ := cert.NewConfigMgr(options.SystemNamespace, client.Kube())
|
||||
c := &controller{
|
||||
options: options,
|
||||
queue: q,
|
||||
@@ -106,6 +110,7 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt
|
||||
serviceInformer: serviceInformer.Informer(),
|
||||
serviceLister: serviceInformer.Lister(),
|
||||
secretController: secretController,
|
||||
configMgr: configMgr,
|
||||
}
|
||||
|
||||
handler := controllers.LatestVersionHandlerFuncs(controllers.EnqueueForSelf(q))
|
||||
@@ -358,7 +363,7 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
|
||||
return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId)
|
||||
}
|
||||
|
||||
|
||||
httpsCredentialConfig, _ := c.configMgr.GetConfigFromConfigmap()
|
||||
for _, rule := range ingressV1.Rules {
|
||||
// Need create builder for every rule.
|
||||
domainBuilder := &common.IngressDomainBuilder{
|
||||
@@ -409,13 +414,19 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
|
||||
|
||||
// Get tls secret matching the rule host
|
||||
secretName := extractTLSSecretName(rule.Host, ingressV1.TLS)
|
||||
secretNamespace := cfg.Namespace
|
||||
// If there is no matching secret, try to get it from configmap.
|
||||
if secretName == "" && httpsCredentialConfig != nil {
|
||||
secretName = httpsCredentialConfig.MatchSecretNameByDomain(rule.Host)
|
||||
secretNamespace = c.options.SystemNamespace
|
||||
}
|
||||
if secretName == "" {
|
||||
// There no matching secret, so just skip.
|
||||
continue
|
||||
}
|
||||
|
||||
domainBuilder.Protocol = common.HTTPS
|
||||
domainBuilder.SecretName = path.Join(c.options.ClusterId, cfg.Namespace, secretName)
|
||||
domainBuilder.SecretName = path.Join(c.options.ClusterId, secretNamespace, secretName)
|
||||
|
||||
// There is a matching secret and the gateway has already a tls secret.
|
||||
// We should report the duplicated tls secret event.
|
||||
@@ -437,7 +448,7 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp
|
||||
Hosts: []string{rule.Host},
|
||||
Tls: &networking.ServerTLSSettings{
|
||||
Mode: networking.ServerTLSSettings_SIMPLE,
|
||||
CredentialName: credentials.ToKubernetesIngressResource(c.options.RawClusterId, cfg.Namespace, secretName),
|
||||
CredentialName: credentials.ToKubernetesIngressResource(c.options.RawClusterId, secretNamespace, secretName),
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user