From 03d2f01274fd01cfc0d320bbfe5cca19c42e1446 Mon Sep 17 00:00:00 2001 From: Jun <108045855+2456868764@users.noreply.github.com> Date: Thu, 9 May 2024 10:56:28 +0800 Subject: [PATCH] feat:add higress automatic https (#854) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 澄潭 --- go.mod | 35 +- go.sum | 58 ++- .../core/templates/controller-deployment.yaml | 2 + helm/core/values.yaml | 9 + pkg/bootstrap/server.go | 35 ++ pkg/cert/certmgr.go | 219 ++++++++++++ pkg/cert/config.go | 290 +++++++++++++++ pkg/cert/config_test.go | 122 +++++++ pkg/cert/controller.go | 165 +++++++++ pkg/cert/ingress.go | 158 ++++++++ pkg/cert/log.go | 19 + pkg/cert/secret.go | 108 ++++++ pkg/cert/server.go | 115 ++++++ pkg/cert/storage.go | 337 ++++++++++++++++++ pkg/cert/storage_test.go | 325 +++++++++++++++++ pkg/cert/util.go | 97 +++++ pkg/cmd/server.go | 5 + pkg/ingress/kube/ingress/controller.go | 17 +- pkg/ingress/kube/ingressv1/controller.go | 17 +- test/e2e/conformance/tests/configmap-https.go | 82 +++++ .../conformance/tests/configmap-https.yaml | 61 ++++ 21 files changed, 2239 insertions(+), 37 deletions(-) create mode 100644 pkg/cert/certmgr.go create mode 100644 pkg/cert/config.go create mode 100644 pkg/cert/config_test.go create mode 100644 pkg/cert/controller.go create mode 100644 pkg/cert/ingress.go create mode 100644 pkg/cert/log.go create mode 100644 pkg/cert/secret.go create mode 100644 pkg/cert/server.go create mode 100644 pkg/cert/storage.go create mode 100644 pkg/cert/storage_test.go create mode 100644 pkg/cert/util.go create mode 100644 test/e2e/conformance/tests/configmap-https.go create mode 100644 test/e2e/conformance/tests/configmap-https.yaml diff --git a/go.mod b/go.mod index e3e2ac8d8..8a3170da2 100644 --- a/go.mod +++ b/go.mod @@ -44,13 +44,13 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.8.3 - go.uber.org/atomic v1.9.0 + go.uber.org/atomic v1.11.0 google.golang.org/grpc v1.48.0 google.golang.org/protobuf v1.28.1 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 istio.io/api v0.0.0-20211122181927-8da52c66ff23 - istio.io/client-go v1.12.0-rc.1.0.20211118171212-b744b6f111e4 + istio.io/client-go v1.12.0-rc.1.0.20211118171212-b744b6f111e4 // indirect istio.io/gogo-genproto v0.0.0-20211115195057-0e34bdd2be67 istio.io/istio v0.0.0 istio.io/pkg v0.0.0-20211115195056-e379f31ee62a @@ -172,6 +172,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.15.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect @@ -185,6 +186,7 @@ require ( github.com/lestrrat/go-file-rotatelogs v0.0.0-20180223000712-d3151e2a480f // indirect github.com/lestrrat/go-strftime v0.0.0-20180220042222-ba3bf9c1d042 // indirect github.com/lib/pq v1.10.0 // indirect + github.com/libdns/libdns v0.2.1 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -194,7 +196,7 @@ require ( github.com/mattn/go-shellwords v1.0.12 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect - github.com/miekg/dns v1.1.43 // indirect + github.com/miekg/dns v1.1.55 // indirect github.com/miekg/pkcs11 v1.1.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-wordwrap v1.0.0 // indirect @@ -248,20 +250,23 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect github.com/yl2chen/cidranger v1.0.2 // indirect + github.com/zeebo/blake3 v0.2.3 // indirect go.opencensus.io v0.23.0 // indirect go.opentelemetry.io/proto/otlp v0.12.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - go.uber.org/multierr v1.7.0 // indirect - go.uber.org/zap v1.21.0 // indirect - golang.org/x/crypto v0.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.24.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect - golang.org/x/net v0.12.0 // indirect + golang.org/x/mod v0.11.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.6.0 // indirect - golang.org/x/sync v0.2.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/term v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect + golang.org/x/tools v0.10.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect gomodules.xyz/jsonpatch/v3 v3.0.1 // indirect gomodules.xyz/orderedmap v0.1.0 // indirect @@ -276,8 +281,6 @@ require ( gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect k8s.io/apiserver v0.22.5 // indirect - k8s.io/component-base v0.22.5 // indirect - k8s.io/klog/v2 v2.60.1 // indirect k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c // indirect k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect oras.land/oras-go v0.4.0 // indirect @@ -300,13 +303,19 @@ replace istio.io/client-go => ./external/client-go replace istio.io/istio => ./external/istio +replace github.com/caddyserver/certmagic => github.com/2456868764/certmagic v1.0.1 + require ( + github.com/caddyserver/certmagic v0.20.0 github.com/evanphx/json-patch/v5 v5.6.0 github.com/google/yamlfmt v0.10.0 github.com/kylelemons/godebug v1.1.0 + github.com/mholt/acmez v1.2.0 github.com/tidwall/gjson v1.17.0 helm.sh/helm/v3 v3.7.1 k8s.io/apiextensions-apiserver v0.25.4 + k8s.io/component-base v0.22.5 + k8s.io/klog/v2 v2.60.1 knative.dev/networking v0.0.0-20220302134042-e8b2eb995165 knative.dev/pkg v0.0.0-20220301181942-2fdd5f232e77 ) diff --git a/go.sum b/go.sum index fdfad8ad4..f9c7046ae 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,8 @@ dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBr dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/2456868764/certmagic v1.0.1 h1:dRzow2Npe9llFTBhNVl0fVe8Yi/Q14ygNonlaZUyDZQ= +github.com/2456868764/certmagic v1.0.1/go.mod h1:LOn81EQYMPajdew6Ln6SVdHPxPqPv6jwsUg92kiNlcQ= github.com/AdaLogics/go-fuzz-headers v0.0.0-20210929163055-e81b3f25be97/go.mod h1:WpB7kf89yJUETZxQnP1kgYPNwlT2jjdDYUCoxVggM3g= github.com/AlecAivazis/survey/v2 v2.3.6 h1:NvTuVHISgTHEHeBFqt6BHOe4Ny/NwGZr7w+F8S9ziyw= github.com/AlecAivazis/survey/v2 v2.3.6/go.mod h1:4AuI9b7RjAR+G7v9+C4YSlX/YL3K3cWNXgWXOhllqvI= @@ -1006,6 +1008,9 @@ github.com/klauspost/compress v1.13.0/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1055,6 +1060,8 @@ github.com/lib/pq v0.0.0-20150723085316-0dad96c0b94f/go.mod h1:5WUZQaWbwv1U+lTRe github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis= +github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= @@ -1145,13 +1152,16 @@ github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqA github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mholt/acmez v1.2.0 h1:1hhLxSgY5FvH5HCnGUuwbKY2VQVo8IU7rxXKSnZ7F30= +github.com/mholt/acmez v1.2.0/go.mod h1:VT9YwH1xgNX1kmYY89gY8xPJC84BFAisjo8Egigt4kE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.17/go.mod h1:WgzbA6oji13JREwiNsRDNfl7jYdPnmz+VEuLrA+/48M= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= +github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= @@ -1658,6 +1668,12 @@ github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= github.com/zclconf/go-cty v1.4.0/go.mod h1:nHzOclRkoj++EU9ZjSrZvRG0BXIWt8c7loYc0qXAFGQ= github.com/zclconf/go-cty v1.7.1/go.mod h1:VDR4+I79ubFBGm1uJac1226K5yANQFHeauxPBoP54+o= +github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= +github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ= +github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= +github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -1711,8 +1727,9 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= @@ -1722,8 +1739,9 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= @@ -1733,8 +1751,9 @@ go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1775,8 +1794,8 @@ golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1817,7 +1836,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1894,8 +1914,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1932,8 +1952,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2074,15 +2094,16 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2092,8 +2113,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2182,7 +2203,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/helm/core/templates/controller-deployment.yaml b/helm/core/templates/controller-deployment.yaml index 0c37811f7..785549f5a 100644 --- a/helm/core/templates/controller-deployment.yaml +++ b/helm/core/templates/controller-deployment.yaml @@ -206,6 +206,8 @@ spec: {{- if .Values.global.watchNamespace }} - --watchNamespace={{ .Values.global.watchNamespace }} {{- end }} + - --enableAutomaticHttps={{ .Values.controller.automaticHttps.enabled }} + - --automaticHttpsEmail={{ .Values.controller.automaticHttps.email }} env: - name: POD_NAME valueFrom: diff --git a/helm/core/values.yaml b/helm/core/values.yaml index 9ce784e9a..ff2f5d81e 100644 --- a/helm/core/values.yaml +++ b/helm/core/values.yaml @@ -529,6 +529,12 @@ controller: "port": 8888, "targetPort": 8888, }, + { + "name": "http-solver", + "protocol": "TCP", + "port": 8889, + "targetPort": 8889, + }, { "name": "grpc", "protocol": "TCP", @@ -567,6 +573,9 @@ controller: minReplicas: 1 maxReplicas: 5 targetCPUUtilizationPercentage: 80 + automaticHttps: + enabled: false + email: "" ## Discovery Settings pilot: diff --git a/pkg/bootstrap/server.go b/pkg/bootstrap/server.go index dce553247..65843721a 100644 --- a/pkg/bootstrap/server.go +++ b/pkg/bootstrap/server.go @@ -20,6 +20,7 @@ import ( "net/http" "time" + "github.com/alibaba/higress/pkg/cert" "github.com/alibaba/higress/pkg/ingress/kube/common" "github.com/alibaba/higress/pkg/ingress/mcp" "github.com/alibaba/higress/pkg/ingress/translation" @@ -112,6 +113,9 @@ type ServerArgs struct { GatewaySelectorValue string GatewayHttpPort uint32 GatewayHttpsPort uint32 + EnableAutomaticHttps bool + AutomaticHttpsEmail string + CertHttpAddress string } type readinessProbe func() (bool, error) @@ -133,6 +137,7 @@ type Server struct { xdsServer *xds.DiscoveryServer server server.Instance readinessProbes map[string]readinessProbe + certServer *cert.Server } var ( @@ -168,6 +173,7 @@ func NewServer(args *ServerArgs) (*Server, error) { s.initConfigController, s.initRegistryEventHandlers, s.initAuthenticators, + s.initAutomaticHttps, } for _, f := range initFuncList { @@ -287,6 +293,15 @@ func (s *Server) Start(stop <-chan struct{}) error { } }() + if s.EnableAutomaticHttps { + go func() { + log.Infof("starting Automatic Cert HTTP service at %s", s.CertHttpAddress) + if err := s.certServer.Run(stop); err != nil { + log.Errorf("error serving Automatic Cert HTTP server: %v", err) + } + }() + } + s.waitForShutDown(stop) return nil } @@ -370,6 +385,26 @@ func (s *Server) initAuthenticators() error { return nil } +func (s *Server) initAutomaticHttps() error { + certOption := &cert.Option{ + Namespace: PodNamespace, + ServerAddress: s.CertHttpAddress, + Email: s.AutomaticHttpsEmail, + } + certServer, err := cert.NewServer(s.kubeClient.Kube(), certOption) + if err != nil { + return err + } + s.certServer = certServer + log.Infof("init cert default config") + s.certServer.InitDefaultConfig() + if !s.EnableAutomaticHttps { + log.Info("automatic https is disabled") + return nil + } + return s.certServer.InitServer() +} + func (s *Server) initKubeClient() error { if s.kubeClient != nil { // Already initialized by startup arguments diff --git a/pkg/cert/certmgr.go b/pkg/cert/certmgr.go new file mode 100644 index 000000000..dcb8d9670 --- /dev/null +++ b/pkg/cert/certmgr.go @@ -0,0 +1,219 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cert + +import ( + "context" + "fmt" + "sync" + + "github.com/caddyserver/certmagic" + "github.com/mholt/acmez" + "k8s.io/client-go/kubernetes" +) + +const ( + EventCertObtained = "cert_obtained" +) + +type CertMgr struct { + cfg *certmagic.Config + client kubernetes.Interface + namespace string + mux sync.RWMutex + storage certmagic.Storage + cache *certmagic.Cache + myACME *certmagic.ACMEIssuer + ingressSolver acmez.Solver + configMgr *ConfigMgr + secretMgr *SecretMgr +} + +func InitCertMgr(opts *Option, clientSet kubernetes.Interface, config *Config) (*CertMgr, error) { + CertLog.Infof("certmgr init config: %+v", config) + // Init certmagic config + // First make a pointer to a Cache as we need to reference the same Cache in + // GetConfigForCert below. + var cache *certmagic.Cache + var storage certmagic.Storage + storage, _ = NewConfigmapStorage(opts.Namespace, clientSet) + renewalWindowRatio := float64(config.RenewBeforeDays / RenewMaxDays) + magicConfig := certmagic.Config{ + RenewalWindowRatio: renewalWindowRatio, + Storage: storage, + } + cache = certmagic.NewCache(certmagic.CacheOptions{ + GetConfigForCert: func(cert certmagic.Certificate) (*certmagic.Config, error) { + // Here we use New to get a valid Config associated with the same cache. + // The provided Config is used as a template and will be completed with + // any defaults that are set in the Default config. + return certmagic.New(cache, magicConfig), nil + }, + }) + // init certmagic + cfg := certmagic.New(cache, magicConfig) + // Init certmagic acme + issuer := config.GetIssuer(IssuerTypeLetsencrypt) + if issuer == nil { + // should never happen here + return nil, fmt.Errorf("there is no Letsencrypt Issuer found in config") + } + + myACME := certmagic.NewACMEIssuer(cfg, certmagic.ACMEIssuer{ + //CA: certmagic.LetsEncryptStagingCA, + CA: certmagic.LetsEncryptProductionCA, + Email: issuer.Email, + Agreed: true, + DisableHTTPChallenge: false, + DisableTLSALPNChallenge: true, + }) + // inject http01 solver + ingressSolver, _ := NewIngressSolver(opts.Namespace, clientSet, myACME) + myACME.Http01Solver = ingressSolver + // init issuers + cfg.Issuers = []certmagic.Issuer{myACME} + + configMgr, _ := NewConfigMgr(opts.Namespace, clientSet) + secretMgr, _ := NewSecretMgr(opts.Namespace, clientSet) + + certMgr := &CertMgr{ + cfg: cfg, + client: clientSet, + namespace: opts.Namespace, + myACME: myACME, + ingressSolver: ingressSolver, + configMgr: configMgr, + secretMgr: secretMgr, + cache: cache, + } + certMgr.cfg.OnEvent = certMgr.OnEvent + return certMgr, nil +} +func (s *CertMgr) Reconcile(ctx context.Context, oldConfig *Config, newConfig *Config) error { + CertLog.Infof("cermgr reconcile old config:%+v to new config:%+v", oldConfig, newConfig) + // sync email + if oldConfig != nil && newConfig != nil { + oldIssuer := oldConfig.GetIssuer(IssuerTypeLetsencrypt) + newIssuer := newConfig.GetIssuer(IssuerTypeLetsencrypt) + if oldIssuer.Email != newIssuer.Email { + // TODO before sync email, maybe need to clean up cache and account + } + } + + // sync domains + newDomains := make([]string, 0) + newDomainsMap := make(map[string]string, 0) + removeDomains := make([]string, 0) + + if newConfig != nil { + for _, config := range newConfig.CredentialConfig { + if config.TLSIssuer == IssuerTypeLetsencrypt { + for _, newDomain := range config.Domains { + newDomains = append(newDomains, newDomain) + newDomainsMap[newDomain] = newDomain + } + + } + } + } + + if oldConfig != nil { + for _, config := range oldConfig.CredentialConfig { + if config.TLSIssuer == IssuerTypeLetsencrypt { + for _, oldDomain := range config.Domains { + if _, ok := newDomainsMap[oldDomain]; !ok { + removeDomains = append(removeDomains, oldDomain) + } + } + + } + } + } + + if newConfig.AutomaticHttps == true { + newIssuer := newConfig.GetIssuer(IssuerTypeLetsencrypt) + // clean up unused domains + s.cleanSync(context.Background(), removeDomains) + // sync email + s.myACME.Email = newIssuer.Email + // sync RenewalWindowRatio + s.cfg.RenewalWindowRatio = float64(newConfig.RenewBeforeDays / RenewMaxDays) + // start cache + s.cache.Start() + // sync domains + s.manageSync(context.Background(), newDomains) + s.configMgr.SetConfig(newConfig) + } else { + // stop cache maintainAssets + s.cache.Stop() + s.configMgr.SetConfig(newConfig) + } + + return nil +} + +func (s *CertMgr) manageSync(ctx context.Context, domainNames []string) error { + CertLog.Infof("cert manage sync domains:%v", domainNames) + return s.cfg.ManageSync(ctx, domainNames) +} + +func (s *CertMgr) cleanSync(ctx context.Context, domainNames []string) error { + //TODO implement clean up domains + CertLog.Infof("cert clean sync domains:%v", domainNames) + return nil +} + +func (s *CertMgr) OnEvent(ctx context.Context, event string, data map[string]any) error { + CertLog.Infof("certmgr receive event:% data:%+v", event, data) + /** + event: cert_obtained + cfg.emit(ctx, "cert_obtained", map[string]any{ + "renewal": true, + "remaining": timeLeft, + "identifier": name, + "issuer": issuerKey, + "storage_path": StorageKeys.CertsSitePrefix(issuerKey, certKey), + "private_key_path": StorageKeys.SitePrivateKey(issuerKey, certKey), + "certificate_path": StorageKeys.SiteCert(issuerKey, certKey), + "metadata_path": StorageKeys.SiteMeta(issuerKey, certKey), + }) + */ + if event == EventCertObtained { + // obtain certificate and update secret + domain := data["identifier"].(string) + isRenew := data["renewal"].(bool) + privateKeyPath := data["private_key_path"].(string) + certificatePath := data["certificate_path"].(string) + privateKey, err := s.cfg.Storage.Load(context.Background(), privateKeyPath) + certificate, err := s.cfg.Storage.Load(context.Background(), certificatePath) + certChain, err := parseCertsFromPEMBundle(certificate) + if err != nil { + return err + } + notAfterTime := notAfter(certChain[0]) + notBeforeTime := notBefore(certChain[0]) + secretName := s.configMgr.GetConfig().GetSecretNameByDomain(IssuerTypeLetsencrypt, domain) + if len(secretName) == 0 { + CertLog.Errorf("can not find secret name for domain % in config", domain) + return nil + } + err2 := s.secretMgr.Update(domain, secretName, privateKey, certificate, notBeforeTime, notAfterTime, isRenew) + if err2 != nil { + CertLog.Errorf("update secretName %s for domain %s error: %v", secretName, domain, err2) + } + return err + } + return nil +} diff --git a/pkg/cert/config.go b/pkg/cert/config.go new file mode 100644 index 000000000..be95c4231 --- /dev/null +++ b/pkg/cert/config.go @@ -0,0 +1,290 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cert + +import ( + "context" + "fmt" + "strings" + "sync/atomic" + "time" + + "istio.io/istio/pkg/config/host" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/yaml" +) + +const ( + ConfigmapCertName = "higress-https" + ConfigmapCertConfigKey = "cert" + DefaultRenewBeforeDays = 30 + RenewMaxDays = 90 +) + +type IssuerName string + +const ( + IssuerTypeAliyunSSL IssuerName = "aliyunssl" + IssuerTypeLetsencrypt IssuerName = "letsencrypt" +) + +// Config is the configuration of automatic https. +type Config struct { + AutomaticHttps bool `json:"automaticHttps"` + RenewBeforeDays int `json:"renewBeforeDays"` + CredentialConfig []CredentialEntry `json:"credentialConfig"` + ACMEIssuer []ACMEIssuerEntry `json:"acmeIssuer"` + Version string `json:"version"` +} + +func (c *Config) GetIssuer(issuerName IssuerName) *ACMEIssuerEntry { + for _, issuer := range c.ACMEIssuer { + if issuer.Name == issuerName { + return &issuer + } + } + return nil +} + +func (c *Config) MatchSecretNameByDomain(domain string) string { + for _, credential := range c.CredentialConfig { + for _, credDomain := range credential.Domains { + if host.Name(strings.ToLower(domain)).SubsetOf(host.Name(strings.ToLower(credDomain))) { + return credential.TLSSecret + } + } + } + return "" +} + +func (c *Config) GetSecretNameByDomain(issuerName IssuerName, domain string) string { + for _, credential := range c.CredentialConfig { + if credential.TLSIssuer == issuerName { + for _, credDomain := range credential.Domains { + if host.Name(strings.ToLower(domain)).SubsetOf(host.Name(strings.ToLower(credDomain))) { + return credential.TLSSecret + } + } + } + } + return "" +} + +func (c *Config) Validate() error { + // check acmeIssuer + if len(c.ACMEIssuer) == 0 { + return fmt.Errorf("acmeIssuer is empty") + } + for _, issuer := range c.ACMEIssuer { + switch issuer.Name { + case IssuerTypeLetsencrypt: + if issuer.Email == "" { + return fmt.Errorf("acmeIssuer %s email is empty", issuer.Name) + } + if !ValidateEmail(issuer.Email) { + return fmt.Errorf("acmeIssuer %s email %s is invalid", issuer.Name, issuer.Email) + } + default: + return fmt.Errorf("acmeIssuer name %s is not supported", issuer.Name) + } + } + // check credentialConfig + for _, credential := range c.CredentialConfig { + if len(credential.Domains) == 0 { + return fmt.Errorf("credentialConfig domains is empty") + } + if credential.TLSSecret == "" { + return fmt.Errorf("credentialConfig tlsSecret is empty") + } + if credential.TLSIssuer == IssuerTypeLetsencrypt { + if len(credential.Domains) > 1 { + return fmt.Errorf("credentialConfig tlsIssuer %s only support one domain", credential.TLSIssuer) + } + } + if credential.TLSIssuer != IssuerTypeLetsencrypt && len(credential.TLSIssuer) > 0 { + return fmt.Errorf("credential tls issuer %s is not support", credential.TLSIssuer) + } + } + + if c.RenewBeforeDays <= 0 { + return fmt.Errorf("RenewBeforeDays should be large than zero") + } + + if c.RenewBeforeDays >= RenewMaxDays { + return fmt.Errorf("RenewBeforeDays should be less than %d", RenewMaxDays) + } + return nil +} + +type CredentialEntry struct { + Domains []string `json:"domains"` + TLSIssuer IssuerName `json:"tlsIssuer,omitempty"` + TLSSecret string `json:"tlsSecret,omitempty"` + CACertSecret string `json:"cacertSecret,omitempty"` +} + +type ACMEIssuerEntry struct { + Name IssuerName `json:"name"` + Email string `json:"email"` + AK string `json:"ak"` // Only applicable for certain issuers like 'aliyunssl' + SK string `json:"sk"` // Only applicable for certain issuers like 'aliyunssl' +} +type ConfigMgr struct { + client kubernetes.Interface + config atomic.Value + namespace string +} + +func (c *ConfigMgr) SetConfig(config *Config) { + c.config.Store(config) +} + +func (c *ConfigMgr) GetConfig() *Config { + value := c.config.Load() + if value != nil { + if config, ok := value.(*Config); ok { + return config + } + } + return nil +} + +func (c *ConfigMgr) InitConfig(email string) (*Config, error) { + var defaultConfig *Config + cm, err := c.GetConfigmap() + if err != nil { + if errors.IsNotFound(err) { + if len(strings.TrimSpace(email)) == 0 { + email = getRandEmail() + } + defaultConfig = newDefaultConfig(email) + err2 := c.ApplyConfigmap(defaultConfig) + if err2 != nil { + return nil, err2 + } + } + return nil, err + } else { + defaultConfig, err = c.ParseConfigFromConfigmap(cm) + if err != nil { + return nil, err + } + } + return defaultConfig, nil +} + +func (c *ConfigMgr) ParseConfigFromConfigmap(configmap *v1.ConfigMap) (*Config, error) { + if _, ok := configmap.Data[ConfigmapCertConfigKey]; !ok { + return nil, fmt.Errorf("no cert key %s in configmap %s", ConfigmapCertConfigKey, configmap.Name) + } + + config := newDefaultConfig("") + if err := yaml.Unmarshal([]byte(configmap.Data[ConfigmapCertConfigKey]), config); err != nil { + return nil, fmt.Errorf("data:%s, convert to higress config error, error: %+v", configmap.Data[ConfigmapCertConfigKey], err) + } + // validate config + if err := config.Validate(); err != nil { + return nil, err + } + return config, nil +} + +func (c *ConfigMgr) GetConfigFromConfigmap() (*Config, error) { + var config *Config + cm, err := c.GetConfigmap() + if err != nil { + return nil, err + } else { + config, err = c.ParseConfigFromConfigmap(cm) + if err != nil { + return nil, err + } + } + return config, nil +} + +func (c *ConfigMgr) GetConfigmap() (configmap *v1.ConfigMap, err error) { + configmapName := ConfigmapCertName + cm, err := c.client.CoreV1().ConfigMaps(c.namespace).Get(context.Background(), configmapName, metav1.GetOptions{}) + return cm, err +} + +func (c *ConfigMgr) ApplyConfigmap(config *Config) error { + configmapName := ConfigmapCertName + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: c.namespace, + Name: configmapName, + }, + } + bytes, err := yaml.Marshal(config) + if err != nil { + return err + } + cm.Data = make(map[string]string, 0) + cm.Data[ConfigmapCertConfigKey] = string(bytes) + + _, err = c.client.CoreV1().ConfigMaps(c.namespace).Get(context.Background(), configmapName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + if _, err = c.client.CoreV1().ConfigMaps(c.namespace).Create(context.Background(), cm, metav1.CreateOptions{}); err != nil { + return err + } + } else { + return err + } + } else { + if _, err = c.client.CoreV1().ConfigMaps(c.namespace).Update(context.Background(), cm, metav1.UpdateOptions{}); err != nil { + return err + } + } + return nil +} + +func NewConfigMgr(namespace string, client kubernetes.Interface) (*ConfigMgr, error) { + configMgr := &ConfigMgr{ + client: client, + namespace: namespace, + } + return configMgr, nil +} + +func newDefaultConfig(email string) *Config { + + defaultIssuer := []ACMEIssuerEntry{ + { + Name: IssuerTypeLetsencrypt, + Email: email, + }, + } + defaultCredentialConfig := make([]CredentialEntry, 0) + config := &Config{ + AutomaticHttps: true, + RenewBeforeDays: DefaultRenewBeforeDays, + ACMEIssuer: defaultIssuer, + CredentialConfig: defaultCredentialConfig, + Version: time.Now().Format("20060102030405"), + } + return config +} + +func getRandEmail() string { + num1 := rangeRandom(100, 100000) + num2 := rangeRandom(100, 100000) + return fmt.Sprintf("your%d@yours%d.com", num1, num2) +} diff --git a/pkg/cert/config_test.go b/pkg/cert/config_test.go new file mode 100644 index 000000000..0a802e9e8 --- /dev/null +++ b/pkg/cert/config_test.go @@ -0,0 +1,122 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cert + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMatchSecretNameByDomain(t *testing.T) { + tests := []struct { + name string + domain string + credentialCfg []CredentialEntry + expected string + }{ + { + name: "Exact match", + domain: "example.com", + credentialCfg: []CredentialEntry{ + { + Domains: []string{"example.com"}, + TLSSecret: "example-com-tls", + }, + }, + expected: "example-com-tls", + }, + + { + name: "Exact match ignore case ", + domain: "eXample.com", + credentialCfg: []CredentialEntry{ + { + Domains: []string{"example.com"}, + TLSSecret: "example-com-tls", + }, + }, + expected: "example-com-tls", + }, + { + name: "Wildcard match", + domain: "sub.example.com", + credentialCfg: []CredentialEntry{ + { + Domains: []string{"*.example.com"}, + TLSSecret: "wildcard-example-com-tls", + }, + }, + expected: "wildcard-example-com-tls", + }, + + { + name: "Wildcard match ignore case", + domain: "sub.Example.com", + credentialCfg: []CredentialEntry{ + { + Domains: []string{"*.example.com"}, + TLSSecret: "wildcard-example-com-tls", + }, + }, + expected: "wildcard-example-com-tls", + }, + { + name: "* match", + domain: "blog.example.co.uk", + credentialCfg: []CredentialEntry{ + { + Domains: []string{"*"}, + TLSSecret: "blog-co-uk-tls", + }, + }, + expected: "blog-co-uk-tls", + }, + { + name: "No match", + domain: "unknown.com", + credentialCfg: []CredentialEntry{ + { + Domains: []string{"example.com"}, + TLSSecret: "example-com-tls", + }, + }, + expected: "", + }, + { + name: "Multiple matches - first match wins", + domain: "example.com", + credentialCfg: []CredentialEntry{ + { + Domains: []string{"example.com"}, + TLSSecret: "example-com-tls", + }, + { + Domains: []string{"*.example.com"}, + TLSSecret: "wildcard-example-com-tls", + }, + }, + expected: "example-com-tls", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Config{CredentialConfig: tt.credentialCfg} + result := cfg.MatchSecretNameByDomain(tt.domain) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/pkg/cert/controller.go b/pkg/cert/controller.go new file mode 100644 index 000000000..6a095826b --- /dev/null +++ b/pkg/cert/controller.go @@ -0,0 +1,165 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cert + +import ( + "context" + "fmt" + "reflect" + "time" + + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/informers" + v1informer "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" +) + +const ( + workNum = 1 + maxRetry = 2 + configMapName = "higress-https" +) + +type Controller struct { + namespace string + ConfigMapInformer v1informer.ConfigMapInformer + client kubernetes.Interface + queue workqueue.RateLimitingInterface + configMgr *ConfigMgr + server *Server + certMgr *CertMgr + factory informers.SharedInformerFactory +} + +func (c *Controller) addConfigmap(obj interface{}) { + key, err := cache.MetaNamespaceKeyFunc(obj) + if err != nil { + return + } + namespace, name, _ := cache.SplitMetaNamespaceKey(key) + if namespace != c.namespace || name != configMapName { + return + } + c.enqueue(name) + +} +func (c *Controller) updateConfigmap(oldObj interface{}, newObj interface{}) { + key, err := cache.MetaNamespaceKeyFunc(oldObj) + if err != nil { + return + } + namespace, name, _ := cache.SplitMetaNamespaceKey(key) + if namespace != c.namespace || name != configMapName { + return + } + if reflect.DeepEqual(oldObj, newObj) { + return + } + c.enqueue(name) +} + +func (c *Controller) enqueue(name string) { + c.queue.Add(name) +} + +func (c *Controller) cachesSynced() bool { + return c.ConfigMapInformer.Informer().HasSynced() +} + +func (c *Controller) Run(stopCh <-chan struct{}) error { + defer runtime.HandleCrash() + defer c.queue.ShutDown() + CertLog.Info("Waiting for informer caches to sync") + c.factory.Start(stopCh) + if ok := cache.WaitForCacheSync(stopCh, c.cachesSynced); !ok { + return fmt.Errorf("failed to wait for caches to sync") + } + CertLog.Info("Starting controller") + // Launch one workers to process configmap resources + for i := 0; i < workNum; i++ { + go wait.Until(c.worker, time.Minute, stopCh) + } + CertLog.Info("Started workers") + <-stopCh + CertLog.Info("Shutting down workers") + + return nil +} + +func (c *Controller) worker() { + for c.processNextItem() { + + } +} + +func (c *Controller) processNextItem() bool { + item, shutdown := c.queue.Get() + if shutdown { + return false + } + defer c.queue.Done(item) + key := item.(string) + CertLog.Infof("controller process item:%s", key) + err := c.syncConfigmap(key) + if err != nil { + c.handleError(key, err) + } + return true +} + +func (c *Controller) syncConfigmap(key string) error { + configmap, err := c.ConfigMapInformer.Lister().ConfigMaps(c.namespace).Get(key) + if err != nil { + return err + } + newConfig, err := c.configMgr.ParseConfigFromConfigmap(configmap) + if err != nil { + return err + } + oldConfig := c.configMgr.GetConfig() + // reconcile old config and new config + return c.certMgr.Reconcile(context.Background(), oldConfig, newConfig) +} + +func (c *Controller) handleError(key string, err error) { + runtime.HandleError(err) + CertLog.Errorf("%+v", err) + c.queue.Forget(key) +} + +func NewController(client kubernetes.Interface, namespace string, certMgr *CertMgr, configMgr *ConfigMgr) (*Controller, error) { + kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(client, 0, informers.WithNamespace(namespace)) + configmapInformer := kubeInformerFactory.Core().V1().ConfigMaps() + c := &Controller{ + certMgr: certMgr, + configMgr: configMgr, + client: client, + namespace: namespace, + factory: kubeInformerFactory, + ConfigMapInformer: configmapInformer, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ingressManage"), + } + + CertLog.Info("Setting up configmap informer event handlers") + configmapInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: c.addConfigmap, + UpdateFunc: c.updateConfigmap, + }) + + return c, nil +} diff --git a/pkg/cert/ingress.go b/pkg/cert/ingress.go new file mode 100644 index 000000000..668626637 --- /dev/null +++ b/pkg/cert/ingress.go @@ -0,0 +1,158 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cert + +import ( + "context" + "strings" + "sync" + "time" + + "github.com/caddyserver/certmagic" + "github.com/mholt/acmez" + "github.com/mholt/acmez/acme" + v1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + IngressClassName = "higress" + IngressServiceName = "higress-controller" + IngressNamePefix = "higress-http-solver-" + IngressPathPrefix = "/.well-known/acme-challenge/" + IngressServicePort = 8889 +) + +type IngressSolver struct { + client kubernetes.Interface + acmeIssuer *certmagic.ACMEIssuer + solversMu sync.Mutex + namespace string + ingressDelay time.Duration +} + +func NewIngressSolver(namespace string, client kubernetes.Interface, acmeIssuer *certmagic.ACMEIssuer) (acmez.Solver, error) { + solver := &IngressSolver{ + namespace: namespace, + client: client, + acmeIssuer: acmeIssuer, + ingressDelay: 5 * time.Second, + } + return solver, nil +} + +func (s *IngressSolver) Present(_ context.Context, challenge acme.Challenge) error { + CertLog.Infof("ingress solver present challenge:%+v", challenge) + s.solversMu.Lock() + defer s.solversMu.Unlock() + ingressName := s.getIngressName(challenge) + ingress := s.constructIngress(challenge) + CertLog.Infof("update ingress name:%s, ingress:%v", ingressName, ingress) + _, err := s.client.NetworkingV1().Ingresses(s.namespace).Get(context.Background(), ingressName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + // create ingress + _, err2 := s.client.NetworkingV1().Ingresses(s.namespace).Create(context.Background(), ingress, metav1.CreateOptions{}) + return err2 + } + return err + } + _, err1 := s.client.NetworkingV1().Ingresses(s.namespace).Update(context.Background(), ingress, metav1.UpdateOptions{}) + if err1 != nil { + return err1 + } + return nil +} + +func (s *IngressSolver) Wait(ctx context.Context, challenge acme.Challenge) error { + CertLog.Infof("ingress solver wait challenge:%+v", challenge) + // wait for ingress ready + if s.ingressDelay > 0 { + select { + case <-time.After(s.ingressDelay): + case <-ctx.Done(): + return ctx.Err() + } + } + CertLog.Infof("ingress solver wait challenge done") + return nil +} + +func (s *IngressSolver) CleanUp(_ context.Context, challenge acme.Challenge) error { + CertLog.Infof("ingress solver cleanup challenge:%+v", challenge) + s.solversMu.Lock() + defer s.solversMu.Unlock() + ingressName := s.getIngressName(challenge) + CertLog.Infof("cleanup ingress name:%s", ingressName) + err := s.client.NetworkingV1().Ingresses(s.namespace).Delete(context.Background(), ingressName, metav1.DeleteOptions{}) + if err != nil { + return err + } + return nil +} + +func (s *IngressSolver) Delete(_ context.Context, challenge acme.Challenge) error { + s.solversMu.Lock() + defer s.solversMu.Unlock() + err := s.client.NetworkingV1().Ingresses(s.namespace).Delete(context.Background(), s.getIngressName(challenge), metav1.DeleteOptions{}) + if err != nil { + return err + } + return nil +} + +func (s *IngressSolver) getIngressName(challenge acme.Challenge) string { + return IngressNamePefix + strings.ReplaceAll(challenge.Identifier.Value, ".", "-") +} + +func (s *IngressSolver) constructIngress(challenge acme.Challenge) *v1.Ingress { + ingressClassName := IngressClassName + ingressDomain := challenge.Identifier.Value + ingressPath := IngressPathPrefix + challenge.Token + ingress := v1.Ingress{} + ingress.Name = s.getIngressName(challenge) + ingress.Namespace = s.namespace + pathType := v1.PathTypePrefix + ingress.Spec = v1.IngressSpec{ + IngressClassName: &ingressClassName, + Rules: []v1.IngressRule{ + { + Host: ingressDomain, + IngressRuleValue: v1.IngressRuleValue{ + HTTP: &v1.HTTPIngressRuleValue{ + Paths: []v1.HTTPIngressPath{ + { + Path: ingressPath, + PathType: &pathType, + Backend: v1.IngressBackend{ + Service: &v1.IngressServiceBackend{ + Name: IngressServiceName, + Port: v1.ServiceBackendPort{ + Number: IngressServicePort, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + return &ingress +} diff --git a/pkg/cert/log.go b/pkg/cert/log.go new file mode 100644 index 000000000..15017c8f6 --- /dev/null +++ b/pkg/cert/log.go @@ -0,0 +1,19 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cert + +import "istio.io/pkg/log" + +var CertLog = log.RegisterScope("cert", "Higress Cert process.", 0) diff --git a/pkg/cert/secret.go b/pkg/cert/secret.go new file mode 100644 index 000000000..0bfac2640 --- /dev/null +++ b/pkg/cert/secret.go @@ -0,0 +1,108 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cert + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + SecretNamePrefix = "higress-secret-" +) + +type SecretMgr struct { + client kubernetes.Interface + namespace string +} + +func NewSecretMgr(namespace string, client kubernetes.Interface) (*SecretMgr, error) { + secretMgr := &SecretMgr{ + namespace: namespace, + client: client, + } + + return secretMgr, nil +} + +func (s *SecretMgr) Update(domain string, secretName string, privateKey []byte, certificate []byte, notBefore time.Time, notAfter time.Time, isRenew bool) error { + //secretName := s.getSecretName(domain) + secret := s.constructSecret(domain, privateKey, certificate, notBefore, notAfter, isRenew) + _, err := s.client.CoreV1().Secrets(s.namespace).Get(context.Background(), secretName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + // create secret + _, err2 := s.client.CoreV1().Secrets(s.namespace).Create(context.Background(), secret, metav1.CreateOptions{}) + return err2 + } + return err + } + // check secret annotations + if _, ok := secret.Annotations["higress.io/cert-domain"]; !ok { + return fmt.Errorf("the secret name %s is not automatic https secret name for the domain:%s, please rename it in config", secretName, domain) + } + _, err1 := s.client.CoreV1().Secrets(s.namespace).Update(context.Background(), secret, metav1.UpdateOptions{}) + if err1 != nil { + return err1 + } + + return nil +} + +func (s *SecretMgr) Delete(domain string) error { + secretName := s.getSecretName(domain) + err := s.client.CoreV1().Secrets(s.namespace).Delete(context.Background(), secretName, metav1.DeleteOptions{}) + return err +} + +func (s *SecretMgr) getSecretName(domain string) string { + return SecretNamePrefix + strings.ReplaceAll(strings.TrimSpace(domain), ".", "-") +} + +func (s *SecretMgr) constructSecret(domain string, privateKey []byte, certificate []byte, notBefore time.Time, notAfter time.Time, isRenew bool) *v1.Secret { + secretName := s.getSecretName(domain) + annotationMap := make(map[string]string, 0) + annotationMap["higress.io/cert-domain"] = domain + annotationMap["higress.io/cert-notAfter"] = notAfter.Format("2006-01-02 15:04:05") + annotationMap["higress.io/cert-notBefore"] = notBefore.Format("2006-01-02 15:04:05") + annotationMap["higress.io/cert-renew"] = strconv.FormatBool(isRenew) + if isRenew { + annotationMap["higress.io/cert-renew-time"] = time.Now().Format("2006-01-02 15:04:05") + } + // Required fields: + // - Secret.Data["tls.key"] - TLS private key. + // Secret.Data["tls.crt"] - TLS certificate. + dataMap := make(map[string][]byte, 0) + dataMap["tls.key"] = privateKey + dataMap["tls.crt"] = certificate + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: s.namespace, + Annotations: annotationMap, + }, + Type: v1.SecretTypeTLS, + Data: dataMap, + } + return secret +} diff --git a/pkg/cert/server.go b/pkg/cert/server.go new file mode 100644 index 000000000..a0e36f037 --- /dev/null +++ b/pkg/cert/server.go @@ -0,0 +1,115 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cert + +import ( + "context" + "fmt" + "net" + "net/http" + "time" + + "github.com/caddyserver/certmagic" + "k8s.io/client-go/kubernetes" +) + +type Option struct { + Namespace string + ServerAddress string + Email string +} + +type Server struct { + httpServer *http.Server + opts *Option + clientSet kubernetes.Interface + controller *Controller + certMgr *CertMgr +} + +func NewServer(clientSet kubernetes.Interface, opts *Option) (*Server, error) { + server := &Server{ + clientSet: clientSet, + opts: opts, + } + return server, nil +} + +func (s *Server) InitDefaultConfig() error { + configMgr, _ := NewConfigMgr(s.opts.Namespace, s.clientSet) + // init config if there is not existed + _, err := configMgr.InitConfig(s.opts.Email) + if err != nil { + return err + } + return nil +} + +func (s *Server) InitServer() error { + configMgr, _ := NewConfigMgr(s.opts.Namespace, s.clientSet) + // init config if there is not existed + defaultConfig, err := configMgr.InitConfig(s.opts.Email) + if err != nil { + return err + } + // init certmgr + certMgr, err := InitCertMgr(s.opts, s.clientSet, defaultConfig) // config and start + s.certMgr = certMgr + // init controller + controller, err := NewController(s.clientSet, s.opts.Namespace, certMgr, configMgr) + s.controller = controller + // init http server + s.initHttpServer() + return nil +} + +func (s *Server) initHttpServer() error { + CertLog.Infof("server init http server") + ctx := context.Background() + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Lookit my cool website over HTTPS!") + }) + httpServer := &http.Server{ + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 5 * time.Second, + WriteTimeout: 5 * time.Second, + IdleTimeout: 5 * time.Second, + Addr: s.opts.ServerAddress, + BaseContext: func(listener net.Listener) context.Context { return ctx }, + } + cfg := s.certMgr.cfg + if len(cfg.Issuers) > 0 { + if am, ok := cfg.Issuers[0].(*certmagic.ACMEIssuer); ok { + httpServer.Handler = am.HTTPChallengeHandler(mux) + } + } else { + httpServer.Handler = mux + } + s.httpServer = httpServer + return nil +} + +func (s *Server) Run(stopCh <-chan struct{}) error { + go s.controller.Run(stopCh) + CertLog.Infof("server run") + go func() { + <-stopCh + CertLog.Infof("server http server shutdown now...") + s.httpServer.Shutdown(context.Background()) + }() + err := s.httpServer.ListenAndServe() + return err +} diff --git a/pkg/cert/storage.go b/pkg/cert/storage.go new file mode 100644 index 000000000..bbc3803cc --- /dev/null +++ b/pkg/cert/storage.go @@ -0,0 +1,337 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cert + +import ( + "context" + "encoding/json" + "fmt" + "io/fs" + "path" + "strings" + "sync" + "time" + + "github.com/caddyserver/certmagic" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + CertificatesPrefix = "/certificates" + ConfigmapStoreCertficatesPrefix = "higress-cert-store-certificates-" + ConfigmapStoreDefaultName = "higress-cert-store-default" +) + +var _ certmagic.Storage = (*ConfigmapStorage)(nil) + +type ConfigmapStorage struct { + namespace string + client kubernetes.Interface + mux sync.RWMutex +} + +type HashValue struct { + K string `json:"k,omitempty"` + V []byte `json:"v,omitempty"` +} + +func NewConfigmapStorage(namespace string, client kubernetes.Interface) (certmagic.Storage, error) { + storage := &ConfigmapStorage{ + namespace: namespace, + client: client, + } + return storage, nil +} + +// Exists returns true if key exists in s. +func (s *ConfigmapStorage) Exists(_ context.Context, key string) bool { + s.mux.RLock() + defer s.mux.RUnlock() + cm, err := s.getConfigmapStoreByKey(key) + if err != nil { + return false + } + if cm.Data == nil { + return false + } + + hashKey := fastHash([]byte(key)) + if _, ok := cm.Data[hashKey]; ok { + return true + } + return false +} + +// Store saves value at key. +func (s *ConfigmapStorage) Store(_ context.Context, key string, value []byte) error { + s.mux.Lock() + defer s.mux.Unlock() + cm, err := s.getConfigmapStoreByKey(key) + if err != nil { + return err + } + if cm.Data == nil { + cm.Data = make(map[string]string, 0) + } + + hashKey := fastHash([]byte(key)) + hashV := &HashValue{ + K: key, + V: value, + } + bytes, err := json.Marshal(hashV) + if err != nil { + return err + } + cm.Data[hashKey] = string(bytes) + return s.updateConfigmap(cm) +} + +// Load retrieves the value at key. +func (s *ConfigmapStorage) Load(_ context.Context, key string) ([]byte, error) { + s.mux.RLock() + defer s.mux.RUnlock() + var value []byte + cm, err := s.getConfigmapStoreByKey(key) + if err != nil { + return value, err + } + if cm.Data == nil { + return value, fs.ErrNotExist + } + + hashKey := fastHash([]byte(key)) + if v, ok := cm.Data[hashKey]; ok { + hV := &HashValue{} + err = json.Unmarshal([]byte(v), hV) + if err != nil { + return value, err + } + return hV.V, nil + } + return value, fs.ErrNotExist +} + +// Delete deletes the value at key. +func (s *ConfigmapStorage) Delete(_ context.Context, key string) error { + s.mux.Lock() + defer s.mux.Unlock() + cm, err := s.getConfigmapStoreByKey(key) + if err != nil { + return err + } + if cm.Data == nil { + cm.Data = make(map[string]string, 0) + } + hashKey := fastHash([]byte(key)) + delete(cm.Data, hashKey) + return s.updateConfigmap(cm) +} + +// List returns all keys that match the prefix. +// If the prefix is "/certificates", it retrieves all ConfigMaps, otherwise only one. +func (s *ConfigmapStorage) List(ctx context.Context, prefix string, recursive bool) ([]string, error) { + s.mux.RLock() + defer s.mux.RUnlock() + var keys []string + var configmapKeys []string + visitedDirs := make(map[string]struct{}) + + // Check if the prefix corresponds to a specific key + hashPrefix := fastHash([]byte(prefix)) + if strings.HasPrefix(prefix, CertificatesPrefix) { + // If the prefix is "/certificates", get all ConfigMaps and traverse each one + // List all ConfigMaps in the namespace with label higress.io/cert-https=true + configmaps, err := s.client.CoreV1().ConfigMaps(s.namespace).List(ctx, metav1.ListOptions{FieldSelector: "metadata.annotations['higress.io/cert-https'] == 'true'"}) + if err != nil { + return keys, err + } + + for _, cm := range configmaps.Items { + // Check if the ConfigMap name starts with the expected prefix + if strings.HasPrefix(cm.Name, ConfigmapStoreCertficatesPrefix) { + // Add the keys from Data field to the list + for _, v := range cm.Data { + // Unmarshal the value into hashValue struct + var hv HashValue + if err := json.Unmarshal([]byte(v), &hv); err != nil { + return nil, err + } + // Check if the key starts with the specified prefix + if strings.HasPrefix(hv.K, prefix) { + // Add the key to the list + configmapKeys = append(configmapKeys, hv.K) + } + } + } + } + } else { + // If not starting with "/certificates", get the specific ConfigMap + cm, err := s.getConfigmapStoreByKey(prefix) + if err != nil { + return keys, err + } + + if _, ok := cm.Data[hashPrefix]; ok { + // The prefix corresponds to a specific key, add it to the list + configmapKeys = append(configmapKeys, prefix) + } else { + // The prefix is considered a directory + for _, v := range cm.Data { + // Unmarshal the value into hashValue struct + var hv HashValue + if err := json.Unmarshal([]byte(v), &hv); err != nil { + return nil, err + } + // Check if the key starts with the specified prefix + if strings.HasPrefix(hv.K, prefix) { + // Add the key to the list + configmapKeys = append(configmapKeys, hv.K) + } + } + } + } + + // return all + if recursive { + return configmapKeys, nil + } + + // only return sub dirs + for _, key := range configmapKeys { + subPath := strings.TrimPrefix(strings.ReplaceAll(key, prefix, ""), "/") + paths := strings.Split(subPath, "/") + if len(paths) > 0 { + subDir := path.Join(prefix, paths[0]) + if _, ok := visitedDirs[subDir]; !ok { + keys = append(keys, subDir) + } + visitedDirs[subDir] = struct{}{} + } + } + + return keys, nil +} + +// Stat returns information about key. only support for no certificates path +func (s *ConfigmapStorage) Stat(_ context.Context, key string) (certmagic.KeyInfo, error) { + s.mux.RLock() + defer s.mux.RUnlock() + // Create a new KeyInfo struct + info := certmagic.KeyInfo{} + + // Get the ConfigMap containing the keys + cm, err := s.getConfigmapStoreByKey(key) + if err != nil { + return info, err + } + + // Check if the key exists in the ConfigMap + hashKey := fastHash([]byte(key)) + if data, ok := cm.Data[hashKey]; ok { + // The key exists, populate the KeyInfo struct + info.Key = key + info.Modified = time.Now() // Since we're not tracking modification time in ConfigMap + info.Size = int64(len(data)) + info.IsTerminal = true + } else { + // Check if there are other keys with the same prefix + prefixKeys := make([]string, 0) + for _, v := range cm.Data { + var hv HashValue + if err := json.Unmarshal([]byte(v), &hv); err != nil { + return info, err + } + // Check if the key starts with the specified prefix + if strings.HasPrefix(hv.K, key) { + // Add the key to the list + prefixKeys = append(prefixKeys, hv.K) + } + } + // If there are multiple keys with the same prefix, then it's not a terminal node + if len(prefixKeys) > 0 { + info.Key = key + info.IsTerminal = false + } else { + return info, fmt.Errorf("prefix '%s' is not existed", key) + } + } + return info, nil +} + +// Lock obtains a lock named by the given name. It blocks +// until the lock can be obtained or an error is returned. +func (s *ConfigmapStorage) Lock(ctx context.Context, name string) error { + return nil +} + +// Unlock releases the lock for name. +func (s *ConfigmapStorage) Unlock(_ context.Context, name string) error { + return nil +} + +func (s *ConfigmapStorage) String() string { + return "ConfigmapStorage" +} + +func (s *ConfigmapStorage) getConfigmapStoreNameByKey(key string) string { + parts := strings.SplitN(key, "/", 10) + if len(parts) >= 4 && parts[1] == "certificates" { + domain := strings.TrimSuffix(parts[3], ".crt") + domain = strings.TrimSuffix(domain, ".key") + domain = strings.TrimSuffix(domain, ".json") + issuerKey := parts[2] + return ConfigmapStoreCertficatesPrefix + fastHash([]byte(issuerKey+domain)) + } + return ConfigmapStoreDefaultName +} + +func (s *ConfigmapStorage) getConfigmapStoreByKey(key string) (*v1.ConfigMap, error) { + configmapName := s.getConfigmapStoreNameByKey(key) + cm, err := s.client.CoreV1().ConfigMaps(s.namespace).Get(context.Background(), configmapName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + // Save default ConfigMap + cm = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: s.namespace, + Name: configmapName, + Annotations: map[string]string{"higress.io/cert-https": "true"}, + }, + } + _, err = s.client.CoreV1().ConfigMaps(s.namespace).Create(context.Background(), cm, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + } else { + return nil, err + } + } + return cm, nil +} + +// updateConfigmap adds or updates the annotation higress.io/cert-https to true. +func (s *ConfigmapStorage) updateConfigmap(configmap *v1.ConfigMap) error { + if configmap.ObjectMeta.Annotations == nil { + configmap.ObjectMeta.Annotations = make(map[string]string) + } + configmap.ObjectMeta.Annotations["higress.io/cert-https"] = "true" + + _, err := s.client.CoreV1().ConfigMaps(configmap.Namespace).Update(context.Background(), configmap, metav1.UpdateOptions{}) + return err +} diff --git a/pkg/cert/storage_test.go b/pkg/cert/storage_test.go new file mode 100644 index 000000000..f4ddfffd1 --- /dev/null +++ b/pkg/cert/storage_test.go @@ -0,0 +1,325 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cert + +import ( + "context" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/client-go/kubernetes/fake" +) + +func TestGetConfigmapStoreNameByKey(t *testing.T) { + // Create a fake client for testing + fakeClient := fake.NewSimpleClientset() + // Create a new ConfigmapStorage instance for testing + namespace := "your-namespace" + storage := &ConfigmapStorage{ + namespace: namespace, + client: fakeClient, + } + tests := []struct { + name string + key string + expected string + }{ + { + name: "certificate crt", + key: "/certificates/issuerKey/domain.crt", + expected: "higress-cert-store-certificates-" + fastHash([]byte("issuerKey"+"domain")), + }, + { + name: "certificate meta", + key: "/certificates/issuerKey/domain.json", + expected: "higress-cert-store-certificates-" + fastHash([]byte("issuerKey"+"domain")), + }, + { + name: "certificate key", + key: "/certificates/issuerKey/domain.key", + expected: "higress-cert-store-certificates-" + fastHash([]byte("issuerKey"+"domain")), + }, + { + name: "user key", + key: "/users/hello/2", + expected: "higress-cert-store-default", + }, + { + name: "Empty Key", + key: "", + expected: "higress-cert-store-default", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + storageName := storage.getConfigmapStoreNameByKey(test.key) + assert.Equal(t, test.expected, storageName) + }) + } +} + +func TestExists(t *testing.T) { + // Create a fake client for testing + fakeClient := fake.NewSimpleClientset() + + // Create a new ConfigmapStorage instance for testing + namespace := "your-namespace" + storage, err := NewConfigmapStorage(namespace, fakeClient) + assert.NoError(t, err) + + // Store a test key + testKey := "/certificates/issuer1/domain1.crt" + err = storage.Store(context.Background(), testKey, []byte("test-data")) + assert.NoError(t, err) + + // Define test cases + tests := []struct { + name string + key string + shouldExist bool + }{ + { + name: "Existing Key", + key: "/certificates/issuer1/domain1.crt", + shouldExist: true, + }, + { + name: "Non-Existent Key1", + key: "/certificates/issuer2/domain2.crt", + shouldExist: false, + }, + { + name: "Non-Existent Key2", + key: "/users/hello/a", + shouldExist: false, + }, + // Add more test cases as needed + } + + // Run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + exists := storage.Exists(context.Background(), test.key) + assert.Equal(t, test.shouldExist, exists) + }) + } +} + +func TestLoad(t *testing.T) { + // Create a fake client for testing + fakeClient := fake.NewSimpleClientset() + + // Create a new ConfigmapStorage instance for testing + namespace := "your-namespace" + storage, err := NewConfigmapStorage(namespace, fakeClient) + assert.NoError(t, err) + + // Store a test key + testKey := "/certificates/issuer1/domain1.crt" + testValue := []byte("test-data") + err = storage.Store(context.Background(), testKey, testValue) + assert.NoError(t, err) + + // Define test cases + tests := []struct { + name string + key string + expected []byte + shouldError bool + }{ + { + name: "Existing Key", + key: "/certificates/issuer1/domain1.crt", + expected: testValue, + shouldError: false, + }, + { + name: "Non-Existent Key", + key: "/certificates/issuer2/domain2.crt", + expected: nil, + shouldError: true, + }, + // Add more test cases as needed + } + + // Run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + value, err := storage.Load(context.Background(), test.key) + if test.shouldError { + assert.Error(t, err) + assert.Nil(t, value) + } else { + assert.NoError(t, err) + assert.Equal(t, test.expected, value) + } + }) + } +} + +func TestStore(t *testing.T) { + // Create a fake client for testing + fakeClient := fake.NewSimpleClientset() + + // Create a new ConfigmapStorage instance for testing + namespace := "your-namespace" + storage := ConfigmapStorage{ + namespace: namespace, + client: fakeClient, + } + + // Define test cases + tests := []struct { + name string + key string + value []byte + expected map[string]string + expectedConfigmapName string + shouldError bool + }{ + { + name: "Store Key with /certificates prefix", + key: "/certificates/issuer1/domain1.crt", + value: []byte("test-data1"), + expected: map[string]string{fastHash([]byte("/certificates/issuer1/domain1.crt")): `{"k":"/certificates/issuer1/domain1.crt","v":"dGVzdC1kYXRhMQ=="}`}, + expectedConfigmapName: "higress-cert-store-certificates-" + fastHash([]byte("issuer1"+"domain1")), + shouldError: false, + }, + { + name: "Store Key with /certificates prefix (additional data)", + key: "/certificates/issuer2/domain2.crt", + value: []byte("test-data2"), + expected: map[string]string{ + fastHash([]byte("/certificates/issuer2/domain2.crt")): `{"k":"/certificates/issuer2/domain2.crt","v":"dGVzdC1kYXRhMg=="}`, + }, + expectedConfigmapName: "higress-cert-store-certificates-" + fastHash([]byte("issuer2"+"domain2")), + shouldError: false, + }, + { + name: "Store Key without /certificates prefix", + key: "/other/path/data.txt", + value: []byte("test-data3"), + expected: map[string]string{fastHash([]byte("/other/path/data.txt")): `{"k":"/other/path/data.txt","v":"dGVzdC1kYXRhMw=="}`}, + expectedConfigmapName: "higress-cert-store-default", + shouldError: false, + }, + // Add more test cases as needed + } + + // Run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := storage.Store(context.Background(), test.key, test.value) + if test.shouldError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + + // Check the contents of the ConfigMap after storing + configmapName := storage.getConfigmapStoreNameByKey(test.key) + cm, err := fakeClient.CoreV1().ConfigMaps(namespace).Get(context.Background(), configmapName, metav1.GetOptions{}) + assert.NoError(t, err) + + // Check if the data is as expected + assert.Equal(t, test.expected, cm.Data) + + // Check if the configmapName is correct + assert.Equal(t, test.expectedConfigmapName, configmapName) + } + }) + } +} + +func TestList(t *testing.T) { + // Create a fake client for testing + fakeClient := fake.NewSimpleClientset() + + // Create a new ConfigmapStorage instance for testing + namespace := "your-namespace" + storage, err := NewConfigmapStorage(namespace, fakeClient) + assert.NoError(t, err) + + // Store some test data + // Store some test data + testKeys := []string{ + "/certificates/issuer1/domain1.crt", + "/certificates/issuer1/domain2.crt", + "/certificates/issuer1/domain3.crt", // Added another domain for issuer1 + "/certificates/issuer2/domain4.crt", + "/certificates/issuer2/domain5.crt", + "/certificates/issuer3/subdomain1/domain6.crt", // Two-level subdirectory under issuer3 + "/certificates/issuer3/subdomain1/subdomain2/domain7.crt", // Two more levels under issuer3 + "/other-prefix/key1/file1", + "/other-prefix/key1/file2", + "/other-prefix/key2/file3", + "/other-prefix/key2/file4", + } + + for _, key := range testKeys { + err := storage.Store(context.Background(), key, []byte("test-data")) + assert.NoError(t, err) + } + + // Define test cases + tests := []struct { + name string + prefix string + recursive bool + expected []string + }{ + { + name: "List Certificates (Non-Recursive)", + prefix: "/certificates", + recursive: false, + expected: []string{"/certificates/issuer1", "/certificates/issuer2", "/certificates/issuer3"}, + }, + { + name: "List Certificates (Recursive)", + prefix: "/certificates", + recursive: true, + expected: []string{"/certificates/issuer1/domain1.crt", "/certificates/issuer1/domain2.crt", "/certificates/issuer1/domain3.crt", "/certificates/issuer2/domain4.crt", "/certificates/issuer2/domain5.crt", "/certificates/issuer3/subdomain1/domain6.crt", "/certificates/issuer3/subdomain1/subdomain2/domain7.crt"}, + }, + { + name: "List Other Prefix (Non-Recursive)", + prefix: "/other-prefix", + recursive: false, + expected: []string{"/other-prefix/key1", "/other-prefix/key2"}, + }, + + { + name: "List Other Prefix (Non-Recursive)", + prefix: "/other-prefix/key1", + recursive: false, + expected: []string{"/other-prefix/key1/file1", "/other-prefix/key1/file2"}, + }, + { + name: "List Other Prefix (Recursive)", + prefix: "/other-prefix", + recursive: true, + expected: []string{"/other-prefix/key1/file1", "/other-prefix/key1/file2", "/other-prefix/key2/file3", "/other-prefix/key2/file4"}, + }, + } + + // Run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + keys, err := storage.List(context.Background(), test.prefix, test.recursive) + assert.NoError(t, err) + assert.ElementsMatch(t, test.expected, keys) + }) + } +} diff --git a/pkg/cert/util.go b/pkg/cert/util.go new file mode 100644 index 000000000..4fbd93f9f --- /dev/null +++ b/pkg/cert/util.go @@ -0,0 +1,97 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cert + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "hash/fnv" + "math/rand" + "net" + "regexp" + "time" +) + +// parseCertsFromPEMBundle parses a certificate bundle from top to bottom and returns +// a slice of x509 certificates. This function will error if no certificates are found. +func parseCertsFromPEMBundle(bundle []byte) ([]*x509.Certificate, error) { + var certificates []*x509.Certificate + var certDERBlock *pem.Block + for { + certDERBlock, bundle = pem.Decode(bundle) + if certDERBlock == nil { + break + } + if certDERBlock.Type == "CERTIFICATE" { + cert, err := x509.ParseCertificate(certDERBlock.Bytes) + if err != nil { + return nil, err + } + certificates = append(certificates, cert) + } + } + if len(certificates) == 0 { + return nil, fmt.Errorf("no certificates found in bundle") + } + return certificates, nil +} + +func notAfter(cert *x509.Certificate) time.Time { + if cert == nil { + return time.Time{} + } + return cert.NotAfter.Truncate(time.Second).Add(1 * time.Second) +} + +func notBefore(cert *x509.Certificate) time.Time { + if cert == nil { + return time.Time{} + } + return cert.NotBefore.Truncate(time.Second).Add(1 * time.Second) +} + +// hostOnly returns only the host portion of hostport. +// If there is no port or if there is an error splitting +// the port off, the whole input string is returned. +func hostOnly(hostport string) string { + host, _, err := net.SplitHostPort(hostport) + if err != nil { + return hostport // OK; probably had no port to begin with + } + return host +} + +func rangeRandom(min, max int) (number int) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + number = r.Intn(max-min) + min + return number +} + +func ValidateEmail(email string) bool { + pattern := `^[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}$` + regExp := regexp.MustCompile(pattern) + if regExp.MatchString(email) { + return true + } else { + return false + } +} + +func fastHash(input []byte) string { + h := fnv.New32a() + h.Write(input) + return fmt.Sprintf("%x", h.Sum32()) +} diff --git a/pkg/cmd/server.go b/pkg/cmd/server.go index 391910ba7..99b53d45d 100644 --- a/pkg/cmd/server.go +++ b/pkg/cmd/server.go @@ -76,6 +76,7 @@ func getServerCommand() *cobra.Command { Debug: true, NativeIstio: true, HttpAddress: ":8888", + CertHttpAddress: ":8889", GrpcAddress: ":15051", GrpcKeepAliveOptions: keepalive.DefaultOption(), XdsOptions: bootstrap.XdsOptions{ @@ -117,6 +118,10 @@ func getServerCommand() *cobra.Command { serveCmd.PersistentFlags().Uint32Var(&serverArgs.GatewayHttpsPort, "gatewayHttpsPort", 443, "Https listening port of gateway pod") + serveCmd.PersistentFlags().BoolVar(&serverArgs.EnableAutomaticHttps, "enableAutomaticHttps", false, "if true, enables automatic https") + serveCmd.PersistentFlags().StringVar(&serverArgs.AutomaticHttpsEmail, "automaticHttpsEmail", "", "email for automatic https") + serveCmd.PersistentFlags().StringVar(&serverArgs.CertHttpAddress, "certHttpAddress", serverArgs.CertHttpAddress, "the cert http address") + loggingOptions.AttachCobraFlags(serveCmd) serverArgs.GrpcKeepAliveOptions.AttachCobraFlags(serveCmd) diff --git a/pkg/ingress/kube/ingress/controller.go b/pkg/ingress/kube/ingress/controller.go index a71115dc9..dfeb6c675 100644 --- a/pkg/ingress/kube/ingress/controller.go +++ b/pkg/ingress/kube/ingress/controller.go @@ -17,6 +17,7 @@ package ingress import ( "errors" "fmt" + "github.com/alibaba/higress/pkg/cert" "path" "reflect" "sort" @@ -85,6 +86,8 @@ type controller struct { secretController secret.SecretController statusSyncer *statusSyncer + + configMgr *cert.ConfigMgr } // NewController creates a new Kubernetes controller @@ -103,6 +106,7 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt IngressLog.Infof("Skipping IngressClass, resource not supported for cluster %s", options.ClusterId) } + configMgr, _ := cert.NewConfigMgr(options.SystemNamespace, client.Kube()) c := &controller{ options: options, queue: q, @@ -113,6 +117,7 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt serviceInformer: serviceInformer.Informer(), serviceLister: serviceInformer.Lister(), secretController: secretController, + configMgr: configMgr, } handler := controllers.LatestVersionHandlerFuncs(controllers.EnqueueForSelf(q)) @@ -371,7 +376,7 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp common.IncrementInvalidIngress(c.options.ClusterId, common.EmptyRule) return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId) } - + httpsCredentialConfig, _ := c.configMgr.GetConfigFromConfigmap() for _, rule := range ingressV1Beta.Rules { // Need create builder for every rule. domainBuilder := &common.IngressDomainBuilder{ @@ -422,13 +427,19 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp // Get tls secret matching the rule host secretName := extractTLSSecretName(rule.Host, ingressV1Beta.TLS) + secretNamespace := cfg.Namespace + // If there is no matching secret, try to get it from configmap. + if secretName == "" && httpsCredentialConfig != nil { + secretName = httpsCredentialConfig.MatchSecretNameByDomain(rule.Host) + secretNamespace = c.options.SystemNamespace + } if secretName == "" { // There no matching secret, so just skip. continue } domainBuilder.Protocol = common.HTTPS - domainBuilder.SecretName = path.Join(c.options.ClusterId, cfg.Namespace, secretName) + domainBuilder.SecretName = path.Join(c.options.ClusterId, secretNamespace, secretName) // There is a matching secret and the gateway has already a tls secret. // We should report the duplicated tls secret event. @@ -450,7 +461,7 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp Hosts: []string{rule.Host}, Tls: &networking.ServerTLSSettings{ Mode: networking.ServerTLSSettings_SIMPLE, - CredentialName: credentials.ToKubernetesIngressResource(c.options.RawClusterId, cfg.Namespace, secretName), + CredentialName: credentials.ToKubernetesIngressResource(c.options.RawClusterId, secretNamespace, secretName), }, }) diff --git a/pkg/ingress/kube/ingressv1/controller.go b/pkg/ingress/kube/ingressv1/controller.go index f6afb80f5..d96b95e59 100644 --- a/pkg/ingress/kube/ingressv1/controller.go +++ b/pkg/ingress/kube/ingressv1/controller.go @@ -25,6 +25,7 @@ import ( "sync" "time" + "github.com/alibaba/higress/pkg/cert" "github.com/hashicorp/go-multierror" networking "istio.io/api/networking/v1alpha3" "istio.io/istio/pilot/pkg/model" @@ -84,6 +85,8 @@ type controller struct { secretController secret.SecretController statusSyncer *statusSyncer + + configMgr *cert.ConfigMgr } // NewController creates a new Kubernetes controller @@ -96,6 +99,7 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt classes := client.KubeInformer().Networking().V1().IngressClasses() classes.Informer() + configMgr, _ := cert.NewConfigMgr(options.SystemNamespace, client.Kube()) c := &controller{ options: options, queue: q, @@ -106,6 +110,7 @@ func NewController(localKubeClient, client kubeclient.Client, options common.Opt serviceInformer: serviceInformer.Informer(), serviceLister: serviceInformer.Lister(), secretController: secretController, + configMgr: configMgr, } handler := controllers.LatestVersionHandlerFuncs(controllers.EnqueueForSelf(q)) @@ -358,7 +363,7 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId) } - + httpsCredentialConfig, _ := c.configMgr.GetConfigFromConfigmap() for _, rule := range ingressV1.Rules { // Need create builder for every rule. domainBuilder := &common.IngressDomainBuilder{ @@ -409,13 +414,19 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp // Get tls secret matching the rule host secretName := extractTLSSecretName(rule.Host, ingressV1.TLS) + secretNamespace := cfg.Namespace + // If there is no matching secret, try to get it from configmap. + if secretName == "" && httpsCredentialConfig != nil { + secretName = httpsCredentialConfig.MatchSecretNameByDomain(rule.Host) + secretNamespace = c.options.SystemNamespace + } if secretName == "" { // There no matching secret, so just skip. continue } domainBuilder.Protocol = common.HTTPS - domainBuilder.SecretName = path.Join(c.options.ClusterId, cfg.Namespace, secretName) + domainBuilder.SecretName = path.Join(c.options.ClusterId, secretNamespace, secretName) // There is a matching secret and the gateway has already a tls secret. // We should report the duplicated tls secret event. @@ -437,7 +448,7 @@ func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapp Hosts: []string{rule.Host}, Tls: &networking.ServerTLSSettings{ Mode: networking.ServerTLSSettings_SIMPLE, - CredentialName: credentials.ToKubernetesIngressResource(c.options.RawClusterId, cfg.Namespace, secretName), + CredentialName: credentials.ToKubernetesIngressResource(c.options.RawClusterId, secretNamespace, secretName), }, }) diff --git a/test/e2e/conformance/tests/configmap-https.go b/test/e2e/conformance/tests/configmap-https.go new file mode 100644 index 000000000..28bef9905 --- /dev/null +++ b/test/e2e/conformance/tests/configmap-https.go @@ -0,0 +1,82 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "testing" + + "github.com/alibaba/higress/test/e2e/conformance/utils/cert" + "github.com/alibaba/higress/test/e2e/conformance/utils/http" + "github.com/alibaba/higress/test/e2e/conformance/utils/kubernetes" + "github.com/alibaba/higress/test/e2e/conformance/utils/suite" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func init() { + Register(ConfigmapHttps) +} + +var ConfigmapHttps = suite.ConformanceTest{ + ShortName: "ConfigmapHttps", + Description: "The Ingress in the higress-conformance-infra namespace uses the configmap https.", + Manifests: []string{"tests/configmap-https.yaml"}, + Features: []suite.SupportedFeature{suite.HTTPConformanceFeature}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + // Prepare secrets for testcases + _, _, caCert, caKey := cert.MustGenerateCaCert(t) + svcCertOut, svcKeyOut := cert.MustGenerateCertWithCA(t, cert.ServerCertType, caCert, caKey, []string{"foo.com"}) + fooSecret := kubernetes.ConstructTLSSecret("higress-system", "foo-com-secret", svcCertOut.Bytes(), svcKeyOut.Bytes()) + suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{fooSecret}, suite.Cleanup) + + testCases := []struct { + httpAssert http.Assertion + }{ + { + httpAssert: http.Assertion{ + Meta: http.AssertionMeta{ + TestCaseName: "test configmap https", + TargetBackend: "infra-backend-v2", + TargetNamespace: "higress-conformance-infra", + }, + Request: http.AssertionRequest{ + ActualRequest: http.Request{ + Path: "/foohttps", + Host: "foo.com", + TLSConfig: &http.TLSConfig{ + SNI: "foo.com", + }, + }, + ExpectedRequest: &http.ExpectedRequest{ + Request: http.Request{ + Path: "/foohttps", + Host: "foo.com", + }, + }, + }, + Response: http.AssertionResponse{ + ExpectedResponse: http.Response{ + StatusCode: 200, + }, + }, + }, + }, + } + t.Run("Configmap Https", func(t *testing.T) { + for _, testcase := range testCases { + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, suite.GatewayAddress, testcase.httpAssert) + } + }) + }, +} diff --git a/test/e2e/conformance/tests/configmap-https.yaml b/test/e2e/conformance/tests/configmap-https.yaml new file mode 100644 index 000000000..f1135c81f --- /dev/null +++ b/test/e2e/conformance/tests/configmap-https.yaml @@ -0,0 +1,61 @@ +# Copyright (c) 2022 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ConfigMap +metadata: + name: higress-https + namespace: higress-system +data: + cert: | + automaticHttps: true + renewBeforeDays: 30 + acmeIssuer: + - ak: test + sk: test + email: test@example.com + name: letsencrypt + credentialConfig: + - cacertSecret: foo-com-ca-secret + domains: + - foo.com + - '*.foo.com' + tlsSecret: foo-com-secret + version: test + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: httproute-configmap-https + namespace: higress-conformance-infra +spec: + ingressClassName: higress + tls: + - hosts: + - "foo.com" + rules: + - host: "foo.com" + http: + paths: + - pathType: Exact + path: "/foohttps" + backend: + service: + name: infra-backend-v2 + port: + number: 8080 + + + +