diff --git a/ingress/config/ingress_config.go b/ingress/config/ingress_config.go new file mode 100644 index 000000000..e56a71e63 --- /dev/null +++ b/ingress/config/ingress_config.go @@ -0,0 +1,846 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "encoding/json" + "strings" + "sync" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + wasm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/wasm/v3" + httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/wasm/v3" + "github.com/golang/protobuf/ptypes/wrappers" + "google.golang.org/protobuf/types/known/anypb" + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/model" + networkingutil "istio.io/istio/pilot/pkg/networking/util" + "istio.io/istio/pilot/pkg/util/sets" + "istio.io/istio/pkg/config" + "istio.io/istio/pkg/config/constants" + "istio.io/istio/pkg/config/schema/collection" + "istio.io/istio/pkg/config/schema/gvk" + "istio.io/istio/pkg/kube" + listersv1 "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + + "github.com/alibaba/higress/ingress/kube/annotations" + "github.com/alibaba/higress/ingress/kube/common" + "github.com/alibaba/higress/ingress/kube/ingress" + "github.com/alibaba/higress/ingress/kube/ingressv1" + secretkube "github.com/alibaba/higress/ingress/kube/secret/kube" + "github.com/alibaba/higress/ingress/kube/util" + . "github.com/alibaba/higress/ingress/log" +) + +var ( + _ model.ConfigStoreCache = &IngressConfig{} + _ model.IngressStore = &IngressConfig{} +) + +type IngressConfig struct { + // key: cluster id + remoteIngressControllers map[string]common.IngressController + mutex sync.RWMutex + + ingressRouteCache model.IngressRouteCollection + ingressDomainCache model.IngressDomainCollection + + localKubeClient kube.Client + + virtualServiceHandlers []model.EventHandler + gatewayHandlers []model.EventHandler + destinationRuleHandlers []model.EventHandler + envoyFilterHandlers []model.EventHandler + watchErrorHandler cache.WatchErrorHandler + + cachedEnvoyFilters []config.Config + + watchedSecretSet sets.Set + + XDSUpdater model.XDSUpdater + + annotationHandler annotations.AnnotationHandler + + globalGatewayName string + + namespace string + + clusterId string +} + +func NewIngressConfig(localKubeClient kube.Client, XDSUpdater model.XDSUpdater, namespace, clusterId string) *IngressConfig { + if clusterId == "Kubernetes" { + clusterId = "" + } + return &IngressConfig{ + remoteIngressControllers: make(map[string]common.IngressController), + localKubeClient: localKubeClient, + XDSUpdater: XDSUpdater, + annotationHandler: annotations.NewAnnotationHandlerManager(), + clusterId: clusterId, + globalGatewayName: namespace + "/" + + common.CreateConvertedName(clusterId, "global"), + watchedSecretSet: sets.NewSet(), + namespace: namespace, + } +} + +func (m *IngressConfig) RegisterEventHandler(kind config.GroupVersionKind, f model.EventHandler) { + IngressLog.Infof("register resource %v", kind) + if kind != gvk.VirtualService && kind != gvk.Gateway && + kind != gvk.DestinationRule && kind != gvk.EnvoyFilter { + return + } + + switch kind { + case gvk.VirtualService: + m.virtualServiceHandlers = append(m.virtualServiceHandlers, f) + + case gvk.Gateway: + m.gatewayHandlers = append(m.gatewayHandlers, f) + + case gvk.DestinationRule: + m.destinationRuleHandlers = append(m.destinationRuleHandlers, f) + + case gvk.EnvoyFilter: + m.envoyFilterHandlers = append(m.envoyFilterHandlers, f) + } + + for _, remoteIngressController := range m.remoteIngressControllers { + remoteIngressController.RegisterEventHandler(kind, f) + } +} + +func (m *IngressConfig) AddLocalCluster(options common.Options) common.IngressController { + secretController := secretkube.NewController(m.localKubeClient, options) + secretController.AddEventHandler(m.ReflectSecretChanges) + + var ingressController common.IngressController + v1 := common.V1Available(m.localKubeClient) + if !v1 { + ingressController = ingress.NewController(m.localKubeClient, m.localKubeClient, options, secretController) + } else { + ingressController = ingressv1.NewController(m.localKubeClient, m.localKubeClient, options, secretController) + } + + m.remoteIngressControllers[options.ClusterId] = ingressController + return ingressController +} + +func (m *IngressConfig) InitializeCluster(ingressController common.IngressController, stop <-chan struct{}) error { + for _, handler := range m.virtualServiceHandlers { + ingressController.RegisterEventHandler(gvk.VirtualService, handler) + } + for _, handler := range m.gatewayHandlers { + ingressController.RegisterEventHandler(gvk.Gateway, handler) + } + for _, handler := range m.destinationRuleHandlers { + ingressController.RegisterEventHandler(gvk.DestinationRule, handler) + } + for _, handler := range m.envoyFilterHandlers { + ingressController.RegisterEventHandler(gvk.EnvoyFilter, handler) + } + + _ = ingressController.SetWatchErrorHandler(m.watchErrorHandler) + + go ingressController.Run(stop) + return nil +} + +func (m *IngressConfig) List(typ config.GroupVersionKind, namespace string) ([]config.Config, error) { + if typ != gvk.Gateway && + typ != gvk.VirtualService && + typ != gvk.DestinationRule && + typ != gvk.EnvoyFilter { + return nil, common.ErrUnsupportedOp + } + + // Currently, only support list all namespaces gateways or virtualservices. + if namespace != "" { + IngressLog.Warnf("ingress store only support type %s of all namespace.", typ) + return nil, common.ErrUnsupportedOp + } + + if typ == gvk.EnvoyFilter { + m.mutex.RLock() + defer m.mutex.RUnlock() + IngressLog.Infof("resource type %s, configs number %d", typ, len(m.cachedEnvoyFilters)) + return m.cachedEnvoyFilters, nil + } + + var configs []config.Config + m.mutex.RLock() + for _, ingressController := range m.remoteIngressControllers { + configs = append(configs, ingressController.List()...) + } + m.mutex.RUnlock() + + common.SortIngressByCreationTime(configs) + wrapperConfigs := m.createWrapperConfigs(configs) + + IngressLog.Infof("resource type %s, configs number %d", typ, len(wrapperConfigs)) + switch typ { + case gvk.Gateway: + return m.convertGateways(wrapperConfigs), nil + case gvk.VirtualService: + return m.convertVirtualService(wrapperConfigs), nil + case gvk.DestinationRule: + return m.convertDestinationRule(wrapperConfigs), nil + } + + return nil, nil +} + +func (m *IngressConfig) createWrapperConfigs(configs []config.Config) []common.WrapperConfig { + var wrapperConfigs []common.WrapperConfig + + // Init global context + clusterSecretListers := map[string]listersv1.SecretLister{} + clusterServiceListers := map[string]listersv1.ServiceLister{} + m.mutex.RLock() + for clusterId, controller := range m.remoteIngressControllers { + clusterSecretListers[clusterId] = controller.SecretLister() + clusterServiceListers[clusterId] = controller.ServiceLister() + } + m.mutex.RUnlock() + globalContext := &annotations.GlobalContext{ + WatchedSecrets: sets.NewSet(), + ClusterSecretLister: clusterSecretListers, + ClusterServiceList: clusterServiceListers, + } + + for idx := range configs { + rawConfig := configs[idx] + annotationsConfig := &annotations.Ingress{ + Meta: annotations.Meta{ + Namespace: rawConfig.Namespace, + Name: rawConfig.Name, + RawClusterId: common.GetRawClusterId(rawConfig.Annotations), + ClusterId: common.GetClusterId(rawConfig.Annotations), + }, + } + _ = m.annotationHandler.Parse(rawConfig.Annotations, annotationsConfig, globalContext) + wrapperConfigs = append(wrapperConfigs, common.WrapperConfig{ + Config: &rawConfig, + AnnotationsConfig: annotationsConfig, + }) + } + + m.mutex.Lock() + m.watchedSecretSet = globalContext.WatchedSecrets + m.mutex.Unlock() + + return wrapperConfigs +} + +func (m *IngressConfig) convertGateways(configs []common.WrapperConfig) []config.Config { + convertOptions := common.ConvertOptions{ + IngressDomainCache: common.NewIngressDomainCache(), + Gateways: map[string]*common.WrapperGateway{}, + } + + for idx := range configs { + cfg := configs[idx] + clusterId := common.GetClusterId(cfg.Config.Annotations) + m.mutex.RLock() + ingressController := m.remoteIngressControllers[clusterId] + m.mutex.RUnlock() + if ingressController == nil { + continue + } + if err := ingressController.ConvertGateway(&convertOptions, &cfg); err != nil { + IngressLog.Errorf("Convert ingress %s/%s to gateway fail in cluster %s, err %v", cfg.Config.Namespace, cfg.Config.Name, clusterId, err) + } + } + + // apply annotation + for _, wrapperGateway := range convertOptions.Gateways { + m.annotationHandler.ApplyGateway(wrapperGateway.Gateway, wrapperGateway.WrapperConfig.AnnotationsConfig) + } + + m.mutex.Lock() + m.ingressDomainCache = convertOptions.IngressDomainCache.Extract() + m.mutex.Unlock() + + out := make([]config.Config, 0, len(convertOptions.Gateways)) + for _, gateway := range convertOptions.Gateways { + cleanHost := common.CleanHost(gateway.Host) + out = append(out, config.Config{ + Meta: config.Meta{ + GroupVersionKind: gvk.Gateway, + Name: common.CreateConvertedName(constants.IstioIngressGatewayName, cleanHost), + Namespace: m.namespace, + Annotations: map[string]string{ + common.ClusterIdAnnotation: gateway.ClusterId, + common.HostAnnotation: gateway.Host, + }, + }, + Spec: gateway.Gateway, + }) + } + return out +} + +func (m *IngressConfig) convertVirtualService(configs []common.WrapperConfig) []config.Config { + convertOptions := common.ConvertOptions{ + HostAndPath2Ingress: map[string]*config.Config{}, + IngressRouteCache: common.NewIngressRouteCache(), + VirtualServices: map[string]*common.WrapperVirtualService{}, + HTTPRoutes: map[string][]*common.WrapperHTTPRoute{}, + } + + // convert http route + for idx := range configs { + cfg := configs[idx] + clusterId := common.GetClusterId(cfg.Config.Annotations) + m.mutex.RLock() + ingressController := m.remoteIngressControllers[clusterId] + m.mutex.RUnlock() + if ingressController == nil { + continue + } + if err := ingressController.ConvertHTTPRoute(&convertOptions, &cfg); err != nil { + IngressLog.Errorf("Convert ingress %s/%s to HTTP route fail in cluster %s, err %v", cfg.Config.Namespace, cfg.Config.Name, clusterId, err) + } + } + + // Apply annotation on routes + for _, routes := range convertOptions.HTTPRoutes { + for _, route := range routes { + m.annotationHandler.ApplyRoute(route.HTTPRoute, route.WrapperConfig.AnnotationsConfig) + } + } + + // Apply canary ingress + if len(configs) > len(convertOptions.CanaryIngresses) { + m.applyCanaryIngresses(&convertOptions) + } + + // Normalize weighted cluster to make sure the sum of weight is 100. + for _, host := range convertOptions.HTTPRoutes { + for _, route := range host { + normalizeWeightedCluster(convertOptions.IngressRouteCache, route) + } + } + + // Apply spec default backend. + if convertOptions.HasDefaultBackend { + for idx := range configs { + cfg := configs[idx] + clusterId := common.GetClusterId(cfg.Config.Annotations) + m.mutex.RLock() + ingressController := m.remoteIngressControllers[clusterId] + m.mutex.RUnlock() + if ingressController == nil { + continue + } + if err := ingressController.ApplyDefaultBackend(&convertOptions, &cfg); err != nil { + IngressLog.Errorf("Apply default backend on ingress %s/%s fail in cluster %s, err %v", cfg.Config.Namespace, cfg.Config.Name, clusterId, err) + } + } + } + + // Apply annotation on virtual services + for _, virtualService := range convertOptions.VirtualServices { + m.annotationHandler.ApplyVirtualServiceHandler(virtualService.VirtualService, virtualService.WrapperConfig.AnnotationsConfig) + } + + // Apply app root for per host. + m.applyAppRoot(&convertOptions) + + // Apply internal active redirect for error page. + m.applyInternalActiveRedirect(&convertOptions) + + m.mutex.Lock() + m.ingressRouteCache = convertOptions.IngressRouteCache.Extract() + m.mutex.Unlock() + + // Convert http route to virtual service + out := make([]config.Config, 0, len(convertOptions.HTTPRoutes)) + for host, routes := range convertOptions.HTTPRoutes { + if len(routes) == 0 { + continue + } + + cleanHost := common.CleanHost(host) + // namespace/name, name format: (istio cluster id)-host + gateways := []string{m.namespace + "/" + + common.CreateConvertedName(m.clusterId, cleanHost), + common.CreateConvertedName(constants.IstioIngressGatewayName, cleanHost)} + if host != "*" { + gateways = append(gateways, m.globalGatewayName) + } + + wrapperVS, exist := convertOptions.VirtualServices[host] + if !exist { + IngressLog.Warnf("virtual service for host %s does not exist.", host) + } + vs := wrapperVS.VirtualService + vs.Gateways = gateways + + for _, route := range routes { + vs.Http = append(vs.Http, route.HTTPRoute) + } + + firstRoute := routes[0] + out = append(out, config.Config{ + Meta: config.Meta{ + GroupVersionKind: gvk.VirtualService, + Name: common.CreateConvertedName(constants.IstioIngressGatewayName, firstRoute.WrapperConfig.Config.Namespace, firstRoute.WrapperConfig.Config.Name, cleanHost), + Namespace: m.namespace, + Annotations: map[string]string{ + common.ClusterIdAnnotation: firstRoute.ClusterId, + }, + }, + Spec: vs, + }) + } + + // We generate some specific envoy filter here to avoid duplicated computation. + m.convertEnvoyFilter(&convertOptions) + + return out +} + +func (m *IngressConfig) convertEnvoyFilter(convertOptions *common.ConvertOptions) { + var envoyFilters []config.Config + mappings := map[string]*common.Rule{} + + for _, routes := range convertOptions.HTTPRoutes { + for _, route := range routes { + if strings.HasSuffix(route.HTTPRoute.Name, "app-root") { + continue + } + + auth := route.WrapperConfig.AnnotationsConfig.Auth + if auth == nil { + continue + } + + key := auth.AuthSecret.String() + "/" + auth.AuthRealm + if rule, exist := mappings[key]; !exist { + mappings[key] = &common.Rule{ + Realm: auth.AuthRealm, + MatchRoute: []string{route.HTTPRoute.Name}, + Credentials: auth.Credentials, + Encrypted: true, + } + } else { + rule.MatchRoute = append(rule.MatchRoute, route.HTTPRoute.Name) + } + } + } + + IngressLog.Infof("Found %d number of basic auth", len(mappings)) + if len(mappings) > 0 { + rules := &common.BasicAuthRules{} + for _, rule := range mappings { + rules.Rules = append(rules.Rules, rule) + } + + basicAuth, err := constructBasicAuthEnvoyFilter(rules, m.namespace) + if err != nil { + IngressLog.Errorf("Construct basic auth filter error %v", err) + } else { + envoyFilters = append(envoyFilters, *basicAuth) + } + } + + // TODO Support other envoy filters + + m.mutex.Lock() + m.cachedEnvoyFilters = envoyFilters + m.mutex.Unlock() +} + +func (m *IngressConfig) convertDestinationRule(configs []common.WrapperConfig) []config.Config { + convertOptions := common.ConvertOptions{ + Service2TrafficPolicy: map[common.ServiceKey]*common.WrapperTrafficPolicy{}, + } + + // Convert destination from service within ingress rule. + for idx := range configs { + cfg := configs[idx] + clusterId := common.GetClusterId(cfg.Config.Annotations) + m.mutex.RLock() + ingressController := m.remoteIngressControllers[clusterId] + m.mutex.RUnlock() + if ingressController == nil { + continue + } + if err := ingressController.ConvertTrafficPolicy(&convertOptions, &cfg); err != nil { + IngressLog.Errorf("Convert ingress %s/%s to destination rule fail in cluster %s, err %v", cfg.Config.Namespace, cfg.Config.Name, clusterId, err) + } + } + + IngressLog.Debugf("traffic policy number %d", len(convertOptions.Service2TrafficPolicy)) + + for _, wrapperTrafficPolicy := range convertOptions.Service2TrafficPolicy { + m.annotationHandler.ApplyTrafficPolicy(wrapperTrafficPolicy.TrafficPolicy, wrapperTrafficPolicy.WrapperConfig.AnnotationsConfig) + } + + // Merge multi-port traffic policy per service into one destination rule. + destinationRules := map[string]*common.WrapperDestinationRule{} + for key, wrapperTrafficPolicy := range convertOptions.Service2TrafficPolicy { + serviceName := util.CreateServiceFQDN(key.Namespace, key.Name) + dr, exist := destinationRules[serviceName] + if !exist { + dr = &common.WrapperDestinationRule{ + DestinationRule: &networking.DestinationRule{ + Host: serviceName, + TrafficPolicy: &networking.TrafficPolicy{ + PortLevelSettings: []*networking.TrafficPolicy_PortTrafficPolicy{wrapperTrafficPolicy.TrafficPolicy}, + }, + }, + WrapperConfig: wrapperTrafficPolicy.WrapperConfig, + ServiceKey: key, + } + } else { + dr.DestinationRule.TrafficPolicy.PortLevelSettings = append(dr.DestinationRule.TrafficPolicy.PortLevelSettings, wrapperTrafficPolicy.TrafficPolicy) + } + + destinationRules[serviceName] = dr + } + + out := make([]config.Config, 0, len(destinationRules)) + for _, dr := range destinationRules { + drName := util.CreateDestinationRuleName(m.clusterId, dr.ServiceKey.Namespace, dr.ServiceKey.Name) + out = append(out, config.Config{ + Meta: config.Meta{ + GroupVersionKind: gvk.DestinationRule, + Name: common.CreateConvertedName(constants.IstioIngressGatewayName, drName), + Namespace: m.namespace, + }, + Spec: dr.DestinationRule, + }) + } + return out +} + +func (m *IngressConfig) applyAppRoot(convertOptions *common.ConvertOptions) { + for host, wrapVS := range convertOptions.VirtualServices { + if wrapVS.AppRoot != "" { + route := &common.WrapperHTTPRoute{ + HTTPRoute: &networking.HTTPRoute{ + Name: common.CreateConvertedName(host, "app-root"), + Match: []*networking.HTTPMatchRequest{ + { + Uri: &networking.StringMatch{ + MatchType: &networking.StringMatch_Exact{ + Exact: "/", + }, + }, + }, + }, + Redirect: &networking.HTTPRedirect{ + RedirectCode: 302, + Uri: wrapVS.AppRoot, + }, + }, + WrapperConfig: wrapVS.WrapperConfig, + ClusterId: wrapVS.WrapperConfig.AnnotationsConfig.ClusterId, + } + convertOptions.HTTPRoutes[host] = append([]*common.WrapperHTTPRoute{route}, convertOptions.HTTPRoutes[host]...) + } + } +} + +func (m *IngressConfig) applyInternalActiveRedirect(convertOptions *common.ConvertOptions) { + for host, routes := range convertOptions.HTTPRoutes { + var tempRoutes []*common.WrapperHTTPRoute + for _, route := range routes { + tempRoutes = append(tempRoutes, route) + if route.HTTPRoute.InternalActiveRedirect != nil { + fallbackConfig := route.WrapperConfig.AnnotationsConfig.Fallback + if fallbackConfig == nil { + continue + } + + typedNamespace := fallbackConfig.DefaultBackend + internalRedirectRoute := route.HTTPRoute.DeepCopy() + internalRedirectRoute.Name = internalRedirectRoute.Name + annotations.FallbackRouteNameSuffix + internalRedirectRoute.InternalActiveRedirect = nil + internalRedirectRoute.Match = []*networking.HTTPMatchRequest{ + { + Uri: &networking.StringMatch{ + MatchType: &networking.StringMatch_Exact{ + Exact: "/", + }, + }, + Headers: map[string]*networking.StringMatch{ + annotations.FallbackInjectHeaderRouteName: { + MatchType: &networking.StringMatch_Exact{ + Exact: internalRedirectRoute.Name, + }, + }, + annotations.FallbackInjectFallbackService: { + MatchType: &networking.StringMatch_Exact{ + Exact: typedNamespace.String(), + }, + }, + }, + }, + } + internalRedirectRoute.Route = []*networking.HTTPRouteDestination{ + { + Destination: &networking.Destination{ + Host: util.CreateServiceFQDN(typedNamespace.Namespace, typedNamespace.Name), + Port: &networking.PortSelector{ + Number: fallbackConfig.Port, + }, + }, + Weight: 100, + }, + } + + tempRoutes = append([]*common.WrapperHTTPRoute{{ + HTTPRoute: internalRedirectRoute, + WrapperConfig: route.WrapperConfig, + ClusterId: route.ClusterId, + }}, tempRoutes...) + } + } + convertOptions.HTTPRoutes[host] = tempRoutes + } +} + +func (m *IngressConfig) ReflectSecretChanges(clusterNamespacedName util.ClusterNamespacedName) { + var hit bool + m.mutex.RLock() + if m.watchedSecretSet.Contains(clusterNamespacedName.String()) { + hit = true + } + m.mutex.RUnlock() + + if hit { + push := func(kind config.GroupVersionKind) { + m.XDSUpdater.ConfigUpdate(&model.PushRequest{ + Full: true, + ConfigsUpdated: map[model.ConfigKey]struct{}{{ + Kind: kind, + Name: clusterNamespacedName.Name, + Namespace: clusterNamespacedName.Namespace, + }: {}}, + Reason: []model.TriggerReason{"auth-secret-change"}, + }) + } + push(gvk.VirtualService) + push(gvk.EnvoyFilter) + } +} + +func normalizeWeightedCluster(cache *common.IngressRouteCache, route *common.WrapperHTTPRoute) { + if len(route.HTTPRoute.Route) == 1 { + route.HTTPRoute.Route[0].Weight = 100 + return + } + + var weightTotal int32 = 0 + for idx, routeDestination := range route.HTTPRoute.Route { + if idx == 0 { + continue + } + + weightTotal += routeDestination.Weight + } + + if weightTotal < route.WeightTotal { + weightTotal = route.WeightTotal + } + + var sum int32 + for idx, routeDestination := range route.HTTPRoute.Route { + if idx == 0 { + continue + } + + weight := float32(routeDestination.Weight) / float32(weightTotal) + routeDestination.Weight = int32(weight * 100) + + sum += routeDestination.Weight + } + + route.HTTPRoute.Route[0].Weight = 100 - sum + + // Update the recorded status in ingress builder + if cache != nil { + cache.Update(route) + } +} + +func (m *IngressConfig) applyCanaryIngresses(convertOptions *common.ConvertOptions) { + if len(convertOptions.CanaryIngresses) == 0 { + return + } + + IngressLog.Infof("Found %d number of canary ingresses.", len(convertOptions.CanaryIngresses)) + for _, cfg := range convertOptions.CanaryIngresses { + clusterId := common.GetClusterId(cfg.Config.Annotations) + m.mutex.RLock() + ingressController := m.remoteIngressControllers[clusterId] + m.mutex.RUnlock() + if ingressController == nil { + continue + } + if err := ingressController.ApplyCanaryIngress(convertOptions, cfg); err != nil { + IngressLog.Errorf("Apply canary ingress %s/%s fail in cluster %s, err %v", cfg.Config.Namespace, cfg.Config.Name, clusterId, err) + } + } +} + +func constructBasicAuthEnvoyFilter(rules *common.BasicAuthRules, namespace string) (*config.Config, error) { + rulesStr, err := json.Marshal(rules) + if err != nil { + return nil, err + } + configuration := &wrappers.StringValue{ + Value: string(rulesStr), + } + + wasm := &wasm.Wasm{ + Config: &v3.PluginConfig{ + Name: "basic-auth", + FailOpen: true, + Vm: &v3.PluginConfig_VmConfig{ + VmConfig: &v3.VmConfig{ + Runtime: "envoy.wasm.runtime.null", + Code: &corev3.AsyncDataSource{ + Specifier: &corev3.AsyncDataSource_Local{ + Local: &corev3.DataSource{ + Specifier: &corev3.DataSource_InlineString{ + InlineString: "envoy.wasm.basic_auth", + }, + }, + }, + }, + }, + }, + Configuration: networkingutil.MessageToAny(configuration), + }, + } + + wasmAny, err := anypb.New(wasm) + if err != nil { + return nil, err + } + + typedConfig := &httppb.HttpFilter{ + Name: "basic-auth", + ConfigType: &httppb.HttpFilter_TypedConfig{ + TypedConfig: wasmAny, + }, + } + + gogoTypedConfig, err := util.MessageToGoGoStruct(typedConfig) + if err != nil { + return nil, err + } + + return &config.Config{ + Meta: config.Meta{ + GroupVersionKind: gvk.EnvoyFilter, + Name: common.CreateConvertedName(constants.IstioIngressGatewayName, "basic-auth"), + Namespace: namespace, + }, + Spec: &networking.EnvoyFilter{ + ConfigPatches: []*networking.EnvoyFilter_EnvoyConfigObjectPatch{ + { + ApplyTo: networking.EnvoyFilter_HTTP_FILTER, + Match: &networking.EnvoyFilter_EnvoyConfigObjectMatch{ + Context: networking.EnvoyFilter_GATEWAY, + ObjectTypes: &networking.EnvoyFilter_EnvoyConfigObjectMatch_Listener{ + Listener: &networking.EnvoyFilter_ListenerMatch{ + FilterChain: &networking.EnvoyFilter_ListenerMatch_FilterChainMatch{ + Filter: &networking.EnvoyFilter_ListenerMatch_FilterMatch{ + Name: "envoy.filters.network.http_connection_manager", + SubFilter: &networking.EnvoyFilter_ListenerMatch_SubFilterMatch{ + Name: "envoy.filters.http.cors", + }, + }, + }, + }, + }, + }, + Patch: &networking.EnvoyFilter_Patch{ + Operation: networking.EnvoyFilter_Patch_INSERT_AFTER, + Value: gogoTypedConfig, + }, + }, + }, + }, + }, nil +} + +func (m *IngressConfig) Run(<-chan struct{}) {} + +func (m *IngressConfig) HasSynced() bool { + m.mutex.RLock() + defer m.mutex.RUnlock() + for _, remoteIngressController := range m.remoteIngressControllers { + if !remoteIngressController.HasSynced() { + return false + } + } + + IngressLog.Info("Ingress config controller synced.") + return true +} + +func (m *IngressConfig) SetWatchErrorHandler(f func(r *cache.Reflector, err error)) error { + m.watchErrorHandler = f + return nil +} + +func (m *IngressConfig) GetIngressRoutes() model.IngressRouteCollection { + m.mutex.RLock() + defer m.mutex.RUnlock() + return m.ingressRouteCache +} + +func (m *IngressConfig) GetIngressDomains() model.IngressDomainCollection { + m.mutex.RLock() + defer m.mutex.RUnlock() + return m.ingressDomainCache +} + +func (m *IngressConfig) Schemas() collection.Schemas { + return common.Schemas +} + +func (m *IngressConfig) Get(config.GroupVersionKind, string, string) *config.Config { + return nil +} + +func (m *IngressConfig) Create(config.Config) (revision string, err error) { + return "", common.ErrUnsupportedOp +} + +func (m *IngressConfig) Update(config.Config) (newRevision string, err error) { + return "", common.ErrUnsupportedOp +} + +func (m *IngressConfig) UpdateStatus(config.Config) (newRevision string, err error) { + return "", common.ErrUnsupportedOp +} + +func (m *IngressConfig) Patch(config.Config, config.PatchFunc) (string, error) { + return "", common.ErrUnsupportedOp +} + +func (m *IngressConfig) Delete(config.GroupVersionKind, string, string, *string) error { + return common.ErrUnsupportedOp +} diff --git a/ingress/config/ingress_config_test.go b/ingress/config/ingress_config_test.go new file mode 100644 index 000000000..f0ffb1a85 --- /dev/null +++ b/ingress/config/ingress_config_test.go @@ -0,0 +1,613 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "testing" + + httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/proto" + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pkg/config" + "istio.io/istio/pkg/config/schema/gvk" + "istio.io/istio/pkg/config/xds" + "istio.io/istio/pkg/kube" + ingress "k8s.io/api/networking/v1" + ingressv1beta1 "k8s.io/api/networking/v1beta1" + + "github.com/alibaba/higress/ingress/kube/annotations" + "github.com/alibaba/higress/ingress/kube/common" + controllerv1beta1 "github.com/alibaba/higress/ingress/kube/ingress" + controllerv1 "github.com/alibaba/higress/ingress/kube/ingressv1" +) + +func TestNormalizeWeightedCluster(t *testing.T) { + validate := func(route *common.WrapperHTTPRoute) int32 { + var total int32 + for _, routeDestination := range route.HTTPRoute.Route { + total += routeDestination.Weight + } + + return total + } + + var testCases []*common.WrapperHTTPRoute + testCases = append(testCases, &common.WrapperHTTPRoute{ + HTTPRoute: &networking.HTTPRoute{ + Route: []*networking.HTTPRouteDestination{ + { + Weight: 100, + }, + }, + }, + }) + testCases = append(testCases, &common.WrapperHTTPRoute{ + HTTPRoute: &networking.HTTPRoute{ + Route: []*networking.HTTPRouteDestination{ + { + Weight: 98, + }, + }, + }, + }) + + testCases = append(testCases, &common.WrapperHTTPRoute{ + HTTPRoute: &networking.HTTPRoute{ + Route: []*networking.HTTPRouteDestination{ + { + Weight: 0, + }, + { + Weight: 48, + }, + { + Weight: 48, + }, + }, + }, + WeightTotal: 100, + }) + + testCases = append(testCases, &common.WrapperHTTPRoute{ + HTTPRoute: &networking.HTTPRoute{ + Route: []*networking.HTTPRouteDestination{ + { + Weight: 0, + }, + { + Weight: 48, + }, + { + Weight: 48, + }, + }, + }, + WeightTotal: 80, + }) + + for _, route := range testCases { + t.Run("", func(t *testing.T) { + normalizeWeightedCluster(nil, route) + if validate(route) != 100 { + t.Fatalf("Weight sum should be 100, but actual is %d", validate(route)) + } + }) + } +} + +func TestConvertGatewaysForIngress(t *testing.T) { + fake := kube.NewFakeClient() + v1Beta1Options := common.Options{ + Enable: true, + ClusterId: "ingress-v1beta1", + RawClusterId: "ingress-v1beta1__", + } + v1Options := common.Options{ + Enable: true, + ClusterId: "ingress-v1", + RawClusterId: "ingress-v1__", + } + ingressV1Beta1Controller := controllerv1beta1.NewController(fake, fake, v1Beta1Options, nil) + ingressV1Controller := controllerv1.NewController(fake, fake, v1Options, nil) + m := NewIngressConfig(fake, nil, "wakanda", "gw-123-istio") + m.remoteIngressControllers = map[string]common.IngressController{ + "ingress-v1beta1": ingressV1Beta1Controller, + "ingress-v1": ingressV1Controller, + } + + testCases := []struct { + name string + inputConfig []common.WrapperConfig + expect map[string]config.Config + }{ + { + name: "ingress v1beta1", + inputConfig: []common.WrapperConfig{ + { + Config: &config.Config{ + Meta: config.Meta{ + Name: "test-1", + Namespace: "wakanda", + Annotations: map[string]string{ + common.ClusterIdAnnotation: "ingress-v1beta1", + }, + }, + Spec: ingressv1beta1.IngressSpec{ + TLS: []ingressv1beta1.IngressTLS{ + { + Hosts: []string{"test.com"}, + SecretName: "test-com", + }, + }, + Rules: []ingressv1beta1.IngressRule{ + { + Host: "foo.com", + IngressRuleValue: ingressv1beta1.IngressRuleValue{ + HTTP: &ingressv1beta1.HTTPIngressRuleValue{ + Paths: []ingressv1beta1.HTTPIngressPath{ + { + Path: "/test", + }, + }, + }, + }, + }, + { + Host: "test.com", + IngressRuleValue: ingressv1beta1.IngressRuleValue{ + HTTP: &ingressv1beta1.HTTPIngressRuleValue{ + Paths: []ingressv1beta1.HTTPIngressPath{ + { + Path: "/test", + }, + }, + }, + }, + }, + }, + }, + }, + AnnotationsConfig: &annotations.Ingress{ + DownstreamTLS: &annotations.DownstreamTLSConfig{ + TlsMinVersion: annotations.TLSProtocolVersion("TLSv1.1"), + CipherSuites: []string{"ECDHE-RSA-AES128-GCM-SHA256", "AES256-SHA"}, + }, + }, + }, + { + Config: &config.Config{ + Meta: config.Meta{ + Name: "test-2", + Namespace: "wakanda", + Annotations: map[string]string{ + common.ClusterIdAnnotation: "ingress-v1beta1", + }, + }, + Spec: ingressv1beta1.IngressSpec{ + TLS: []ingressv1beta1.IngressTLS{ + { + Hosts: []string{"foo.com"}, + SecretName: "foo-com", + }, + { + Hosts: []string{"test.com"}, + SecretName: "test-com-2", + }, + }, + Rules: []ingressv1beta1.IngressRule{ + { + Host: "foo.com", + IngressRuleValue: ingressv1beta1.IngressRuleValue{ + HTTP: &ingressv1beta1.HTTPIngressRuleValue{ + Paths: []ingressv1beta1.HTTPIngressPath{ + { + Path: "/test", + }, + }, + }, + }, + }, + { + Host: "bar.com", + IngressRuleValue: ingressv1beta1.IngressRuleValue{ + HTTP: &ingressv1beta1.HTTPIngressRuleValue{ + Paths: []ingressv1beta1.HTTPIngressPath{ + { + Path: "/test", + }, + }, + }, + }, + }, + { + Host: "test.com", + IngressRuleValue: ingressv1beta1.IngressRuleValue{ + HTTP: &ingressv1beta1.HTTPIngressRuleValue{ + Paths: []ingressv1beta1.HTTPIngressPath{ + { + Path: "/test", + }, + }, + }, + }, + }, + }, + }, + }, + AnnotationsConfig: &annotations.Ingress{ + DownstreamTLS: &annotations.DownstreamTLSConfig{ + TlsMinVersion: annotations.TLSProtocolVersion("TLSv1.2"), + CipherSuites: []string{"ECDHE-RSA-AES128-GCM-SHA256"}, + }, + }, + }, + }, + expect: map[string]config.Config{ + "foo.com": { + Meta: config.Meta{ + GroupVersionKind: gvk.Gateway, + Name: "istio-autogenerated-k8s-ingress-foo-com", + Namespace: "wakanda", + Annotations: map[string]string{ + common.ClusterIdAnnotation: "ingress-v1beta1", + common.HostAnnotation: "foo.com", + }, + }, + Spec: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Number: 80, + Protocol: "HTTP", + Name: "http-80-ingress-ingress-v1beta1-wakanda-test-1-foo-com", + }, + Hosts: []string{"foo.com"}, + }, + { + Port: &networking.Port{ + Number: 443, + Protocol: "HTTPS", + Name: "https-443-ingress-ingress-v1beta1-wakanda-test-2-foo-com", + }, + Hosts: []string{"foo.com"}, + Tls: &networking.ServerTLSSettings{ + Mode: networking.ServerTLSSettings_SIMPLE, + CredentialName: "kubernetes-ingress://ingress-v1beta1__/wakanda/foo-com", + MinProtocolVersion: networking.ServerTLSSettings_TLSV1_1, + CipherSuites: []string{"ECDHE-RSA-AES128-GCM-SHA256", "AES256-SHA"}, + }, + }, + }, + }, + }, + "test.com": { + Meta: config.Meta{ + GroupVersionKind: gvk.Gateway, + Name: "istio-autogenerated-k8s-ingress-test-com", + Namespace: "wakanda", + Annotations: map[string]string{ + common.ClusterIdAnnotation: "ingress-v1beta1", + common.HostAnnotation: "test.com", + }, + }, + Spec: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Number: 80, + Protocol: "HTTP", + Name: "http-80-ingress-ingress-v1beta1-wakanda-test-1-test-com", + }, + Hosts: []string{"test.com"}, + }, + { + Port: &networking.Port{ + Number: 443, + Protocol: "HTTPS", + Name: "https-443-ingress-ingress-v1beta1-wakanda-test-1-test-com", + }, + Hosts: []string{"test.com"}, + Tls: &networking.ServerTLSSettings{ + Mode: networking.ServerTLSSettings_SIMPLE, + CredentialName: "kubernetes-ingress://ingress-v1beta1__/wakanda/test-com", + MinProtocolVersion: networking.ServerTLSSettings_TLSV1_1, + CipherSuites: []string{"ECDHE-RSA-AES128-GCM-SHA256", "AES256-SHA"}, + }, + }, + }, + }, + }, + "bar.com": { + Meta: config.Meta{ + GroupVersionKind: gvk.Gateway, + Name: "istio-autogenerated-k8s-ingress-bar-com", + Namespace: "wakanda", + Annotations: map[string]string{ + common.ClusterIdAnnotation: "ingress-v1beta1", + common.HostAnnotation: "bar.com", + }, + }, + Spec: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Number: 80, + Protocol: "HTTP", + Name: "http-80-ingress-ingress-v1beta1-wakanda-test-2-bar-com", + }, + Hosts: []string{"bar.com"}, + }, + }, + }, + }, + }, + }, + { + name: "ingress v1", + inputConfig: []common.WrapperConfig{ + { + Config: &config.Config{ + Meta: config.Meta{ + Name: "test-1", + Namespace: "wakanda", + Annotations: map[string]string{ + common.ClusterIdAnnotation: "ingress-v1", + }, + }, + Spec: ingress.IngressSpec{ + TLS: []ingress.IngressTLS{ + { + Hosts: []string{"test.com"}, + SecretName: "test-com", + }, + }, + Rules: []ingress.IngressRule{ + { + Host: "foo.com", + IngressRuleValue: ingress.IngressRuleValue{ + HTTP: &ingress.HTTPIngressRuleValue{ + Paths: []ingress.HTTPIngressPath{ + { + Path: "/test", + }, + }, + }, + }, + }, + { + Host: "test.com", + IngressRuleValue: ingress.IngressRuleValue{ + HTTP: &ingress.HTTPIngressRuleValue{ + Paths: []ingress.HTTPIngressPath{ + { + Path: "/test", + }, + }, + }, + }, + }, + }, + }, + }, + AnnotationsConfig: &annotations.Ingress{}, + }, + { + Config: &config.Config{ + Meta: config.Meta{ + Name: "test-2", + Namespace: "wakanda", + Annotations: map[string]string{ + common.ClusterIdAnnotation: "ingress-v1", + }, + }, + Spec: ingress.IngressSpec{ + TLS: []ingress.IngressTLS{ + { + Hosts: []string{"foo.com"}, + SecretName: "foo-com", + }, + { + Hosts: []string{"test.com"}, + SecretName: "test-com-2", + }, + }, + Rules: []ingress.IngressRule{ + { + Host: "foo.com", + IngressRuleValue: ingress.IngressRuleValue{ + HTTP: &ingress.HTTPIngressRuleValue{ + Paths: []ingress.HTTPIngressPath{ + { + Path: "/test", + }, + }, + }, + }, + }, + { + Host: "bar.com", + IngressRuleValue: ingress.IngressRuleValue{ + HTTP: &ingress.HTTPIngressRuleValue{ + Paths: []ingress.HTTPIngressPath{ + { + Path: "/test", + }, + }, + }, + }, + }, + { + Host: "test.com", + IngressRuleValue: ingress.IngressRuleValue{ + HTTP: &ingress.HTTPIngressRuleValue{ + Paths: []ingress.HTTPIngressPath{ + { + Path: "/test", + }, + }, + }, + }, + }, + }, + }, + }, + AnnotationsConfig: &annotations.Ingress{ + DownstreamTLS: &annotations.DownstreamTLSConfig{ + TlsMinVersion: annotations.TLSProtocolVersion("TLSv1.2"), + CipherSuites: []string{"ECDHE-RSA-AES128-GCM-SHA256"}, + }, + }, + }, + }, + expect: map[string]config.Config{ + "foo.com": { + Meta: config.Meta{ + GroupVersionKind: gvk.Gateway, + Name: "istio-autogenerated-k8s-ingress-foo-com", + Namespace: "wakanda", + Annotations: map[string]string{ + common.ClusterIdAnnotation: "ingress-v1", + common.HostAnnotation: "foo.com", + }, + }, + Spec: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Number: 80, + Protocol: "HTTP", + Name: "http-80-ingress-ingress-v1-wakanda-test-1-foo-com", + }, + Hosts: []string{"foo.com"}, + }, + { + Port: &networking.Port{ + Number: 443, + Protocol: "HTTPS", + Name: "https-443-ingress-ingress-v1-wakanda-test-2-foo-com", + }, + Hosts: []string{"foo.com"}, + Tls: &networking.ServerTLSSettings{ + Mode: networking.ServerTLSSettings_SIMPLE, + CredentialName: "kubernetes-ingress://ingress-v1__/wakanda/foo-com", + MinProtocolVersion: networking.ServerTLSSettings_TLSV1_2, + CipherSuites: []string{"ECDHE-RSA-AES128-GCM-SHA256"}, + }, + }, + }, + }, + }, + "test.com": { + Meta: config.Meta{ + GroupVersionKind: gvk.Gateway, + Name: "istio-autogenerated-k8s-ingress-test-com", + Namespace: "wakanda", + Annotations: map[string]string{ + common.ClusterIdAnnotation: "ingress-v1", + common.HostAnnotation: "test.com", + }, + }, + Spec: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Number: 80, + Protocol: "HTTP", + Name: "http-80-ingress-ingress-v1-wakanda-test-1-test-com", + }, + Hosts: []string{"test.com"}, + }, + { + Port: &networking.Port{ + Number: 443, + Protocol: "HTTPS", + Name: "https-443-ingress-ingress-v1-wakanda-test-1-test-com", + }, + Hosts: []string{"test.com"}, + Tls: &networking.ServerTLSSettings{ + Mode: networking.ServerTLSSettings_SIMPLE, + CredentialName: "kubernetes-ingress://ingress-v1__/wakanda/test-com", + MinProtocolVersion: networking.ServerTLSSettings_TLSV1_2, + CipherSuites: []string{"ECDHE-RSA-AES128-GCM-SHA256"}, + }, + }, + }, + }, + }, + "bar.com": { + Meta: config.Meta{ + GroupVersionKind: gvk.Gateway, + Name: "istio-autogenerated-k8s-ingress-bar-com", + Namespace: "wakanda", + Annotations: map[string]string{ + common.ClusterIdAnnotation: "ingress-v1", + common.HostAnnotation: "bar.com", + }, + }, + Spec: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Number: 80, + Protocol: "HTTP", + Name: "http-80-ingress-ingress-v1-wakanda-test-2-bar-com", + }, + Hosts: []string{"bar.com"}, + }, + }, + }, + }, + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + result := m.convertGateways(testCase.inputConfig) + target := map[string]config.Config{} + for _, item := range result { + host := common.GetHost(item.Annotations) + target[host] = item + } + assert.Equal(t, testCase.expect, target) + }) + } +} + +func TestConstructBasicAuthEnvoyFilter(t *testing.T) { + rules := &common.BasicAuthRules{ + Rules: []*common.Rule{ + { + Realm: "test", + MatchRoute: []string{"route"}, + Credentials: []string{"user:password"}, + Encrypted: true, + }, + }, + } + + config, err := constructBasicAuthEnvoyFilter(rules, "") + if err != nil { + t.Fatalf("construct error %v", err) + } + envoyFilter := config.Spec.(*networking.EnvoyFilter) + pb, err := xds.BuildXDSObjectFromStruct(networking.EnvoyFilter_HTTP_FILTER, envoyFilter.ConfigPatches[0].Patch.Value, false) + if err != nil { + t.Fatalf("build object error %v", err) + } + target := proto.Clone(pb).(*httppb.HttpFilter) + t.Log(target) +} diff --git a/ingress/kube/annotations/annotations.go b/ingress/kube/annotations/annotations.go new file mode 100644 index 000000000..b40a9eb34 --- /dev/null +++ b/ingress/kube/annotations/annotations.go @@ -0,0 +1,212 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/util/sets" + listersv1 "k8s.io/client-go/listers/core/v1" +) + +type GlobalContext struct { + // secret key is cluster/namespace/name + WatchedSecrets sets.Set + + ClusterSecretLister map[string]listersv1.SecretLister + + ClusterServiceList map[string]listersv1.ServiceLister +} + +type Meta struct { + Namespace string + Name string + RawClusterId string + ClusterId string +} + +// Ingress defines the valid annotations present in one NGINX Ingress. +type Ingress struct { + Meta + + Cors *CorsConfig + + Rewrite *RewriteConfig + + Redirect *RedirectConfig + + UpstreamTLS *UpstreamTLSConfig + + DownstreamTLS *DownstreamTLSConfig + + Canary *CanaryConfig + + IPAccessControl *IPAccessControlConfig + + HeaderControl *HeaderControlConfig + + Timeout *TimeoutConfig + + Retry *RetryConfig + + LoadBalance *LoadBalanceConfig + + localRateLimit *localRateLimitConfig + + Fallback *FallbackConfig + + Auth *AuthConfig +} + +func (i *Ingress) NeedRegexMatch() bool { + if i.Rewrite == nil { + return false + } + + return i.Rewrite.RewriteTarget != "" || i.Rewrite.UseRegex +} + +func (i *Ingress) IsCanary() bool { + if i.Canary == nil { + return false + } + + return i.Canary.Enabled +} + +// CanaryKind return byHeader, byWeight +func (i *Ingress) CanaryKind() (bool, bool) { + if !i.IsCanary() { + return false, false + } + + // first header, cookie + if i.Canary.Header != "" || i.Canary.Cookie != "" { + return true, false + } + + // then weight + return false, true +} + +func (i *Ingress) NeedTrafficPolicy() bool { + return i.UpstreamTLS != nil || + i.LoadBalance != nil +} + +func (i *Ingress) MergeHostIPAccessControlIfNotExist(ac *IPAccessControlConfig) { + if i.IPAccessControl != nil && i.IPAccessControl.Domain != nil { + return + } + + if ac != nil && ac.Domain != nil { + if i.IPAccessControl == nil { + i.IPAccessControl = &IPAccessControlConfig{ + Domain: ac.Domain, + } + } else { + i.IPAccessControl.Domain = ac.Domain + } + } +} + +type AnnotationHandler interface { + Parser + GatewayHandler + VirtualServiceHandler + RouteHandler + TrafficPolicyHandler +} + +type AnnotationHandlerManager struct { + parsers []Parser + gatewayHandlers []GatewayHandler + virtualServiceHandlers []VirtualServiceHandler + routeHandlers []RouteHandler + trafficPolicyHandlers []TrafficPolicyHandler +} + +func NewAnnotationHandlerManager() AnnotationHandler { + return &AnnotationHandlerManager{ + parsers: []Parser{ + canary{}, + cors{}, + downstreamTLS{}, + redirect{}, + rewrite{}, + upstreamTLS{}, + ipAccessControl{}, + headerControl{}, + timeout{}, + retry{}, + loadBalance{}, + localRateLimit{}, + fallback{}, + auth{}, + }, + gatewayHandlers: []GatewayHandler{ + downstreamTLS{}, + }, + virtualServiceHandlers: []VirtualServiceHandler{ + ipAccessControl{}, + }, + routeHandlers: []RouteHandler{ + cors{}, + redirect{}, + rewrite{}, + ipAccessControl{}, + headerControl{}, + timeout{}, + retry{}, + localRateLimit{}, + fallback{}, + }, + trafficPolicyHandlers: []TrafficPolicyHandler{ + upstreamTLS{}, + loadBalance{}, + }, + } +} + +func (h *AnnotationHandlerManager) Parse(annotations Annotations, config *Ingress, globalContext *GlobalContext) error { + for _, parser := range h.parsers { + _ = parser.Parse(annotations, config, globalContext) + } + + return nil +} + +func (h *AnnotationHandlerManager) ApplyGateway(gateway *networking.Gateway, config *Ingress) { + for _, handler := range h.gatewayHandlers { + handler.ApplyGateway(gateway, config) + } +} + +func (h *AnnotationHandlerManager) ApplyVirtualServiceHandler(virtualService *networking.VirtualService, config *Ingress) { + for _, handler := range h.virtualServiceHandlers { + handler.ApplyVirtualServiceHandler(virtualService, config) + } +} + +func (h *AnnotationHandlerManager) ApplyRoute(route *networking.HTTPRoute, config *Ingress) { + for _, handler := range h.routeHandlers { + handler.ApplyRoute(route, config) + } +} + +func (h *AnnotationHandlerManager) ApplyTrafficPolicy(trafficPolicy *networking.TrafficPolicy_PortTrafficPolicy, config *Ingress) { + for _, handler := range h.trafficPolicyHandlers { + handler.ApplyTrafficPolicy(trafficPolicy, config) + } +} diff --git a/ingress/kube/annotations/annotations_test.go b/ingress/kube/annotations/annotations_test.go new file mode 100644 index 000000000..9195e918b --- /dev/null +++ b/ingress/kube/annotations/annotations_test.go @@ -0,0 +1,182 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import "testing" + +func TestNeedRegexMatch(t *testing.T) { + testCases := []struct { + input *Ingress + expect bool + }{ + { + input: &Ingress{}, + expect: false, + }, + { + input: &Ingress{ + Rewrite: &RewriteConfig{}, + }, + expect: false, + }, + { + input: &Ingress{ + Rewrite: &RewriteConfig{ + RewriteTarget: "/test", + }, + }, + expect: true, + }, + { + input: &Ingress{ + Rewrite: &RewriteConfig{ + UseRegex: true, + }, + }, + expect: true, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + if testCase.input.NeedRegexMatch() != testCase.expect { + t.Fatalf("Should be %t, but actual is %t", testCase.expect, testCase.input.NeedRegexMatch()) + } + }) + } +} + +func TestIsCanary(t *testing.T) { + testCases := []struct { + input *Ingress + expect bool + }{ + { + input: &Ingress{}, + expect: false, + }, + { + input: &Ingress{ + Canary: &CanaryConfig{}, + }, + expect: false, + }, + { + input: &Ingress{ + Canary: &CanaryConfig{ + Enabled: true, + }, + }, + expect: true, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + if testCase.input.IsCanary() != testCase.expect { + t.Fatalf("Should be %t, but actual is %t", testCase.expect, testCase.input.IsCanary()) + } + }) + } +} + +func TestCanaryKind(t *testing.T) { + testCases := []struct { + input *Ingress + byHeader bool + byWeight bool + }{ + { + input: &Ingress{}, + byHeader: false, + byWeight: false, + }, + { + input: &Ingress{ + Canary: &CanaryConfig{}, + }, + byHeader: false, + byWeight: false, + }, + { + input: &Ingress{ + Canary: &CanaryConfig{ + Enabled: true, + }, + }, + byHeader: false, + byWeight: true, + }, + { + input: &Ingress{ + Canary: &CanaryConfig{ + Enabled: true, + Header: "test", + }, + }, + byHeader: true, + byWeight: false, + }, + { + input: &Ingress{ + Canary: &CanaryConfig{ + Enabled: true, + Cookie: "test", + }, + }, + byHeader: true, + byWeight: false, + }, + { + input: &Ingress{ + Canary: &CanaryConfig{ + Enabled: true, + Weight: 2, + }, + }, + byHeader: false, + byWeight: true, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + byHeader, byWeight := testCase.input.CanaryKind() + if byHeader != testCase.byHeader { + t.Fatalf("Should be %t, but actual is %t", testCase.byHeader, byHeader) + } + + if byWeight != testCase.byWeight { + t.Fatalf("Should be %t, but actual is %t", testCase.byWeight, byWeight) + } + }) + } +} + +func TestNeedTrafficPolicy(t *testing.T) { + config1 := &Ingress{} + if config1.NeedTrafficPolicy() { + t.Fatal("should be false") + } + + config2 := &Ingress{ + UpstreamTLS: &UpstreamTLSConfig{ + BackendProtocol: defaultBackendProtocol, + }, + } + if !config2.NeedTrafficPolicy() { + t.Fatal("should be true") + } +} diff --git a/ingress/kube/annotations/auth.go b/ingress/kube/annotations/auth.go new file mode 100644 index 000000000..20b7909f7 --- /dev/null +++ b/ingress/kube/annotations/auth.go @@ -0,0 +1,155 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "errors" + "sort" + "strings" + + corev1 "k8s.io/api/core/v1" + + "github.com/alibaba/higress/ingress/kube/util" + . "github.com/alibaba/higress/ingress/log" +) + +const ( + authType = "auth-type" + authRealm = "auth-realm" + authSecretAnn = "auth-secret" + authSecretTypeAnn = "auth-secret-type" + + defaultAuthType = "basic" + authFileKey = "auth" +) + +type authSecretType string + +const ( + authFileAuthSecretType authSecretType = "auth-file" + authMapAuthSecretType authSecretType = "auth-map" +) + +var _ Parser = auth{} + +type AuthConfig struct { + AuthType string + AuthRealm string + Credentials []string + AuthSecret util.ClusterNamespacedName +} + +type auth struct{} + +func (a auth) Parse(annotations Annotations, config *Ingress, globalContext *GlobalContext) error { + if !needAuthConfig(annotations) { + return nil + } + + authConfig := &AuthConfig{ + AuthType: defaultAuthType, + } + + // Check auth type + authType, err := annotations.ParseStringASAP(authType) + if err != nil { + IngressLog.Errorf("Parse auth type error %v within ingress %/%s", err, config.Namespace, config.Name) + return nil + } + if authType != defaultAuthType { + IngressLog.Errorf("Auth type %s within ingress %/%s is not supported yet.", authType, config.Namespace, config.Name) + return nil + } + + secretName, _ := annotations.ParseStringASAP(authSecretAnn) + namespaced := util.SplitNamespacedName(secretName) + if namespaced.Name == "" { + IngressLog.Errorf("Auth secret name within ingress %s/%s is invalid", config.Namespace, config.Name) + return nil + } + if namespaced.Namespace == "" { + namespaced.Namespace = config.Namespace + } + + configKey := util.ClusterNamespacedName{ + NamespacedName: namespaced, + ClusterId: config.ClusterId, + } + authConfig.AuthSecret = configKey + + // Subscribe secret + globalContext.WatchedSecrets.Insert(configKey.String()) + + secretType := authFileAuthSecretType + if rawSecretType, err := annotations.ParseStringASAP(authSecretTypeAnn); err == nil { + resultAuthSecretType := authSecretType(rawSecretType) + if resultAuthSecretType == authFileAuthSecretType || resultAuthSecretType == authMapAuthSecretType { + secretType = resultAuthSecretType + } + } + + authConfig.AuthRealm, _ = annotations.ParseStringASAP(authRealm) + + // Process credentials. + secretLister, exist := globalContext.ClusterSecretLister[config.ClusterId] + if !exist { + IngressLog.Errorf("secret lister of cluster %s doesn't exist", config.ClusterId) + return nil + } + authSecret, err := secretLister.Secrets(namespaced.Namespace).Get(namespaced.Name) + if err != nil { + IngressLog.Errorf("Secret %s within ingress %s/%s is not found", + namespaced.String(), config.Namespace, config.Name) + return nil + } + credentials, err := convertCredentials(secretType, authSecret) + if err != nil { + IngressLog.Errorf("Parse auth secret fail, err %v", err) + return nil + } + authConfig.Credentials = credentials + + config.Auth = authConfig + return nil +} + +func convertCredentials(secretType authSecretType, secret *corev1.Secret) ([]string, error) { + var result []string + switch secretType { + case authFileAuthSecretType: + users, exist := secret.Data[authFileKey] + if !exist { + return nil, errors.New("the auth file type must has auth key in secret data") + } + userList := strings.Split(string(users), "\n") + for _, item := range userList { + result = append(result, item) + } + case authMapAuthSecretType: + for name, password := range secret.Data { + result = append(result, name+":"+string(password)) + } + } + sort.SliceStable(result, func(i, j int) bool { + return result[i] < result[j] + }) + + return result, nil +} + +func needAuthConfig(annotations Annotations) bool { + return annotations.HasASAP(authType) && + annotations.HasASAP(authSecretAnn) +} diff --git a/ingress/kube/annotations/auth_test.go b/ingress/kube/annotations/auth_test.go new file mode 100644 index 000000000..c60986e8f --- /dev/null +++ b/ingress/kube/annotations/auth_test.go @@ -0,0 +1,196 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "context" + "reflect" + "testing" + "time" + + "istio.io/istio/pilot/pkg/model" + "istio.io/istio/pilot/pkg/util/sets" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + listerv1 "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + + "github.com/alibaba/higress/ingress/kube/util" +) + +func TestAuthParse(t *testing.T) { + auth := auth{} + inputCases := []struct { + input map[string]string + secret *v1.Secret + expect *AuthConfig + watchedSecret string + }{ + { + secret: &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Namespace: "foo", + }, + Data: map[string][]byte{ + "auth": []byte("A:a\nB:b"), + }, + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(authType): "digest", + }, + expect: nil, + secret: &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Namespace: "foo", + }, + Data: map[string][]byte{ + "auth": []byte("A:a\nB:b"), + }, + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(authType): defaultAuthType, + buildMSEAnnotationKey(authSecretAnn): "foo/bar", + }, + secret: &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Namespace: "foo", + }, + Data: map[string][]byte{ + "auth": []byte("A:a\nB:b"), + }, + }, + expect: &AuthConfig{ + AuthType: defaultAuthType, + AuthSecret: util.ClusterNamespacedName{ + NamespacedName: model.NamespacedName{ + Namespace: "foo", + Name: "bar", + }, + ClusterId: "cluster", + }, + Credentials: []string{"A:a", "B:b"}, + }, + watchedSecret: "cluster/foo/bar", + }, + { + input: map[string]string{ + buildNginxAnnotationKey(authType): defaultAuthType, + buildMSEAnnotationKey(authSecretAnn): "foo/bar", + buildNginxAnnotationKey(authSecretTypeAnn): string(authMapAuthSecretType), + }, + secret: &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Namespace: "foo", + }, + Data: map[string][]byte{ + "A": []byte("a"), + "B": []byte("b"), + }, + }, + expect: &AuthConfig{ + AuthType: defaultAuthType, + AuthSecret: util.ClusterNamespacedName{ + NamespacedName: model.NamespacedName{ + Namespace: "foo", + Name: "bar", + }, + ClusterId: "cluster", + }, + Credentials: []string{"A:a", "B:b"}, + }, + watchedSecret: "cluster/foo/bar", + }, + { + input: map[string]string{ + buildNginxAnnotationKey(authType): defaultAuthType, + buildMSEAnnotationKey(authSecretAnn): "bar", + buildNginxAnnotationKey(authSecretTypeAnn): string(authFileAuthSecretType), + }, + secret: &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Namespace: "default", + }, + Data: map[string][]byte{ + "auth": []byte("A:a\nB:b"), + }, + }, + expect: &AuthConfig{ + AuthType: defaultAuthType, + AuthSecret: util.ClusterNamespacedName{ + NamespacedName: model.NamespacedName{ + Namespace: "default", + Name: "bar", + }, + ClusterId: "cluster", + }, + Credentials: []string{"A:a", "B:b"}, + }, + watchedSecret: "cluster/default/bar", + }, + } + + for _, inputCase := range inputCases { + t.Run("", func(t *testing.T) { + config := &Ingress{ + Meta: Meta{ + Namespace: "default", + ClusterId: "cluster", + }, + } + + globalContext, cancel := initGlobalContext(inputCase.secret) + defer cancel() + + _ = auth.Parse(inputCase.input, config, globalContext) + if !reflect.DeepEqual(inputCase.expect, config.Auth) { + t.Fatal("Should be equal") + } + + if inputCase.watchedSecret != "" { + if !globalContext.WatchedSecrets.Contains(inputCase.watchedSecret) { + t.Fatalf("Should watch secret %s", inputCase.watchedSecret) + } + } + }) + } +} + +func initGlobalContext(secret *v1.Secret) (*GlobalContext, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + + client := fake.NewSimpleClientset(secret) + informerFactory := informers.NewSharedInformerFactory(client, time.Hour) + secretInformer := informerFactory.Core().V1().Secrets() + go secretInformer.Informer().Run(ctx.Done()) + cache.WaitForCacheSync(ctx.Done(), secretInformer.Informer().HasSynced) + + return &GlobalContext{ + WatchedSecrets: sets.NewSet(), + ClusterSecretLister: map[string]listerv1.SecretLister{ + "cluster": secretInformer.Lister(), + }, + }, cancel +} diff --git a/ingress/kube/annotations/canary.go b/ingress/kube/annotations/canary.go new file mode 100644 index 000000000..7b6f4b5ef --- /dev/null +++ b/ingress/kube/annotations/canary.go @@ -0,0 +1,185 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + networking "istio.io/api/networking/v1alpha3" +) + +const ( + enableCanary = "canary" + canaryByHeader = "canary-by-header" + canaryByHeaderValue = "canary-by-header-value" + canaryByHeaderPattern = "canary-by-header-pattern" + canaryByCookie = "canary-by-cookie" + canaryWeight = "canary-weight" + canaryWeightTotal = "canary-weight-total" + + defaultCanaryWeightTotal = 100 +) + +var _ Parser = &canary{} + +type CanaryConfig struct { + Enabled bool + Header string + HeaderValue string + HeaderPattern string + Cookie string + Weight int + WeightTotal int +} + +type canary struct{} + +func (c canary) Parse(annotations Annotations, config *Ingress, _ *GlobalContext) error { + if !needCanaryConfig(annotations) { + return nil + } + + canaryConfig := &CanaryConfig{ + WeightTotal: defaultCanaryWeightTotal, + } + + defer func() { + config.Canary = canaryConfig + }() + + canaryConfig.Enabled, _ = annotations.ParseBoolASAP(enableCanary) + if !canaryConfig.Enabled { + return nil + } + + if header, err := annotations.ParseStringASAP(canaryByHeader); err == nil { + canaryConfig.Header = header + } + + if headerValue, err := annotations.ParseStringASAP(canaryByHeaderValue); err == nil && + headerValue != "" { + canaryConfig.HeaderValue = headerValue + return nil + } + + if headerPattern, err := annotations.ParseStringASAP(canaryByHeaderPattern); err == nil && + headerPattern != "" { + canaryConfig.HeaderPattern = headerPattern + return nil + } + + if cookie, err := annotations.ParseStringASAP(canaryByCookie); err == nil && + cookie != "" { + canaryConfig.Cookie = cookie + return nil + } + + canaryConfig.Weight, _ = annotations.ParseIntASAP(canaryWeight) + if weightTotal, err := annotations.ParseIntASAP(canaryWeightTotal); err == nil && weightTotal > 0 { + canaryConfig.WeightTotal = weightTotal + } + + return nil +} + +func ApplyByWeight(canary, route *networking.HTTPRoute, canaryIngress *Ingress) { + if len(route.Route) == 1 { + // Move route level to destination level + route.Route[0].Headers = route.Headers + route.Headers = nil + } + + // Modify canary weighted cluster + canary.Route[0].Weight = int32(canaryIngress.Canary.Weight) + + // Append canary weight upstream service. + // We will process total weight in the end. + route.Route = append(route.Route, canary.Route[0]) + + // canary route use the header control applied on itself. + headerControl{}.ApplyRoute(canary, canaryIngress) + // Move route level to destination level + canary.Route[0].Headers = canary.Headers + + // First add normal route cluster + canary.Route[0].FallbackClusters = append(canary.Route[0].FallbackClusters, + route.Route[0].Destination.DeepCopy()) + // Second add fallback cluster of normal route cluster + canary.Route[0].FallbackClusters = append(canary.Route[0].FallbackClusters, + route.Route[0].FallbackClusters...) +} + +func ApplyByHeader(canary, route *networking.HTTPRoute, canaryIngress *Ingress) { + canaryConfig := canaryIngress.Canary + + // Copy canary http route + temp := canary.DeepCopy() + + // Inherit configuration from non-canary rule + route.DeepCopyInto(canary) + // Assign temp copied canary route match + canary.Match = temp.Match + // Assign temp copied canary route destination + canary.Route = temp.Route + + // Modified match base on by header + if canaryConfig.Header != "" { + canary.Match[0].Headers = map[string]*networking.StringMatch{ + canaryConfig.Header: { + MatchType: &networking.StringMatch_Exact{ + Exact: "always", + }, + }, + } + if canaryConfig.HeaderValue != "" { + canary.Match[0].Headers = map[string]*networking.StringMatch{ + canaryConfig.Header: { + MatchType: &networking.StringMatch_Regex{ + Regex: "always|" + canaryConfig.HeaderValue, + }, + }, + } + } else if canaryConfig.HeaderPattern != "" { + canary.Match[0].Headers = map[string]*networking.StringMatch{ + canaryConfig.Header: { + MatchType: &networking.StringMatch_Regex{ + Regex: canaryConfig.HeaderPattern, + }, + }, + } + } + } else if canaryConfig.Cookie != "" { + canary.Match[0].Headers = map[string]*networking.StringMatch{ + "cookie": { + MatchType: &networking.StringMatch_Regex{ + Regex: "^(.\\*?;)?(" + canaryConfig.Cookie + "=always)(;.\\*)?$", + }, + }, + } + } + + canary.Headers = nil + // canary route use the header control applied on itself. + headerControl{}.ApplyRoute(canary, canaryIngress) + + // First add normal route cluster + canary.Route[0].FallbackClusters = append(canary.Route[0].FallbackClusters, + route.Route[0].Destination.DeepCopy()) + // Second add fallback cluster of normal route cluster + canary.Route[0].FallbackClusters = append(canary.Route[0].FallbackClusters, + route.Route[0].FallbackClusters...) +} + +func needCanaryConfig(annotations Annotations) bool { + return annotations.HasASAP(enableCanary) +} diff --git a/ingress/kube/annotations/canary_test.go b/ingress/kube/annotations/canary_test.go new file mode 100644 index 000000000..1b24154a3 --- /dev/null +++ b/ingress/kube/annotations/canary_test.go @@ -0,0 +1,254 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "reflect" + "testing" + + networking "istio.io/api/networking/v1alpha3" +) + +func TestApplyWeight(t *testing.T) { + route := &networking.HTTPRoute{ + Headers: &networking.Headers{ + Request: &networking.Headers_HeaderOperations{ + Add: map[string]string{ + "normal": "true", + }, + }, + }, + Route: []*networking.HTTPRouteDestination{ + { + Destination: &networking.Destination{ + Host: "normal", + Port: &networking.PortSelector{ + Number: 80, + }, + }, + }, + }, + } + + canary1 := &networking.HTTPRoute{ + Headers: &networking.Headers{ + Request: &networking.Headers_HeaderOperations{ + Add: map[string]string{ + "canary1": "true", + }, + }, + }, + Route: []*networking.HTTPRouteDestination{ + { + Destination: &networking.Destination{ + Host: "canary1", + Port: &networking.PortSelector{ + Number: 80, + }, + }, + }, + }, + } + + canary2 := &networking.HTTPRoute{ + Headers: &networking.Headers{ + Request: &networking.Headers_HeaderOperations{ + Add: map[string]string{ + "canary2": "true", + }, + }, + }, + Route: []*networking.HTTPRouteDestination{ + { + Destination: &networking.Destination{ + Host: "canary2", + Port: &networking.PortSelector{ + Number: 80, + }, + }, + }, + }, + } + + ApplyByWeight(canary1, route, &Ingress{ + Canary: &CanaryConfig{ + Weight: 30, + }, + }) + + ApplyByWeight(canary2, route, &Ingress{ + Canary: &CanaryConfig{ + Weight: 20, + }, + }) + + expect := &networking.HTTPRoute{ + Route: []*networking.HTTPRouteDestination{ + { + Destination: &networking.Destination{ + Host: "normal", + Port: &networking.PortSelector{ + Number: 80, + }, + }, + Headers: &networking.Headers{ + Request: &networking.Headers_HeaderOperations{ + Add: map[string]string{ + "normal": "true", + }, + }, + }, + }, + { + Destination: &networking.Destination{ + Host: "canary1", + Port: &networking.PortSelector{ + Number: 80, + }, + }, + Headers: &networking.Headers{ + Request: &networking.Headers_HeaderOperations{ + Add: map[string]string{ + "canary1": "true", + }, + }, + }, + Weight: 30, + FallbackClusters: []*networking.Destination{ + { + Host: "normal", + Port: &networking.PortSelector{ + Number: 80, + }, + }, + }, + }, + { + Destination: &networking.Destination{ + Host: "canary2", + Port: &networking.PortSelector{ + Number: 80, + }, + }, + Headers: &networking.Headers{ + Request: &networking.Headers_HeaderOperations{ + Add: map[string]string{ + "canary2": "true", + }, + }, + }, + Weight: 20, + FallbackClusters: []*networking.Destination{ + { + Host: "normal", + Port: &networking.PortSelector{ + Number: 80, + }, + }, + }, + }, + }, + } + + if !reflect.DeepEqual(route, expect) { + t.Fatal("Should be equal") + } +} + +func TestApplyHeader(t *testing.T) { + route := &networking.HTTPRoute{ + Headers: &networking.Headers{ + Request: &networking.Headers_HeaderOperations{ + Add: map[string]string{ + "normal": "true", + }, + }, + }, + Route: []*networking.HTTPRouteDestination{ + { + Destination: &networking.Destination{ + Host: "normal", + Port: &networking.PortSelector{ + Number: 80, + }, + }, + }, + }, + } + + canary := &networking.HTTPRoute{ + Headers: &networking.Headers{ + Request: &networking.Headers_HeaderOperations{ + Add: map[string]string{ + "canary": "true", + }, + }, + }, + Route: []*networking.HTTPRouteDestination{ + { + Destination: &networking.Destination{ + Host: "canary", + Port: &networking.PortSelector{ + Number: 80, + }, + }, + }, + }, + } + + ApplyByHeader(canary, route, &Ingress{ + Canary: &CanaryConfig{}, + HeaderControl: &HeaderControlConfig{ + Request: &HeaderOperation{ + Add: map[string]string{ + "canary": "true", + }, + }, + }, + }) + + expect := &networking.HTTPRoute{ + Headers: &networking.Headers{ + Request: &networking.Headers_HeaderOperations{ + Add: map[string]string{ + "canary": "true", + }, + }, + Response: &networking.Headers_HeaderOperations{}, + }, + Route: []*networking.HTTPRouteDestination{ + { + Destination: &networking.Destination{ + Host: "canary", + Port: &networking.PortSelector{ + Number: 80, + }, + }, + FallbackClusters: []*networking.Destination{ + { + Host: "normal", + Port: &networking.PortSelector{ + Number: 80, + }, + }, + }, + }, + }, + } + + if !reflect.DeepEqual(canary, expect) { + t.Fatal("Should be equal") + } +} diff --git a/ingress/kube/annotations/cors.go b/ingress/kube/annotations/cors.go new file mode 100644 index 000000000..effde1427 --- /dev/null +++ b/ingress/kube/annotations/cors.go @@ -0,0 +1,200 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "net/url" + "strings" + + "github.com/gogo/protobuf/types" + networking "istio.io/api/networking/v1alpha3" +) + +const ( + // annotation key + enableCors = "enable-cors" + allowOrigin = "cors-allow-origin" + allowMethods = "cors-allow-methods" + allowHeaders = "cors-allow-headers" + exposeHeaders = "cors-expose-headers" + allowCredentials = "cors-allow-credentials" + maxAge = "cors-max-age" + + // default annotation value + defaultAllowOrigin = "*" + defaultAllowMethods = "GET, PUT, POST, DELETE, PATCH, OPTIONS" + defaultAllowHeaders = "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With," + + "If-Modified-Since,Cache-Control,Content-Type,Authorization" + defaultAllowCredentials = true + defaultMaxAge = 1728000 +) + +var ( + _ Parser = &cors{} + _ RouteHandler = &cors{} +) + +type CorsConfig struct { + Enabled bool + AllowOrigin []string + AllowMethods []string + AllowHeaders []string + ExposeHeaders []string + AllowCredentials bool + MaxAge int +} + +type cors struct{} + +func (c cors) Parse(annotations Annotations, config *Ingress, _ *GlobalContext) error { + if !needCorsConfig(annotations) { + return nil + } + + // cors enable + enable, _ := annotations.ParseBoolASAP(enableCors) + if !enable { + return nil + } + + corsConfig := &CorsConfig{ + Enabled: enable, + AllowOrigin: []string{defaultAllowOrigin}, + AllowMethods: splitStringWithSpaceTrim(defaultAllowMethods), + AllowHeaders: splitStringWithSpaceTrim(defaultAllowHeaders), + AllowCredentials: defaultAllowCredentials, + MaxAge: defaultMaxAge, + } + + defer func() { + config.Cors = corsConfig + }() + + // allow origin + if origin, err := annotations.ParseStringASAP(allowOrigin); err == nil { + corsConfig.AllowOrigin = splitStringWithSpaceTrim(origin) + } + + // allow methods + if methods, err := annotations.ParseStringASAP(allowMethods); err == nil { + corsConfig.AllowMethods = splitStringWithSpaceTrim(methods) + } + + // allow headers + if headers, err := annotations.ParseStringASAP(allowHeaders); err == nil { + corsConfig.AllowHeaders = splitStringWithSpaceTrim(headers) + } + + // expose headers + if exposeHeaders, err := annotations.ParseStringASAP(exposeHeaders); err == nil { + corsConfig.ExposeHeaders = splitStringWithSpaceTrim(exposeHeaders) + } + + // allow credentials + if allowCredentials, err := annotations.ParseBoolASAP(allowCredentials); err == nil { + corsConfig.AllowCredentials = allowCredentials + } + + // max age + if age, err := annotations.ParseIntASAP(maxAge); err == nil { + corsConfig.MaxAge = age + } + + return nil +} + +func (c cors) ApplyRoute(route *networking.HTTPRoute, config *Ingress) { + corsConfig := config.Cors + if corsConfig == nil || !corsConfig.Enabled { + return + } + + corsPolicy := &networking.CorsPolicy{ + AllowMethods: corsConfig.AllowMethods, + AllowHeaders: corsConfig.AllowHeaders, + ExposeHeaders: corsConfig.ExposeHeaders, + AllowCredentials: &types.BoolValue{ + Value: corsConfig.AllowCredentials, + }, + MaxAge: &types.Duration{ + Seconds: int64(corsConfig.MaxAge), + }, + } + + var allowOrigins []*networking.StringMatch + for _, origin := range corsConfig.AllowOrigin { + if origin == "*" { + allowOrigins = append(allowOrigins, &networking.StringMatch{ + MatchType: &networking.StringMatch_Regex{ + Regex: ".*", + }, + }) + break + } + if strings.Contains(origin, "*") { + parsedURL, err := url.Parse(origin) + if err != nil { + continue + } + if strings.HasPrefix(parsedURL.Host, "*") { + var sb strings.Builder + sb.WriteString(".*") + for idx, char := range parsedURL.Host { + if idx == 0 { + continue + } + + if char == '.' { + sb.WriteString("\\") + } + + sb.WriteString(string(char)) + } + + allowOrigins = append(allowOrigins, &networking.StringMatch{ + MatchType: &networking.StringMatch_Regex{ + Regex: sb.String(), + }, + }) + } + continue + } + + allowOrigins = append(allowOrigins, &networking.StringMatch{ + MatchType: &networking.StringMatch_Exact{ + Exact: origin, + }, + }) + } + corsPolicy.AllowOrigins = allowOrigins + + route.CorsPolicy = corsPolicy +} + +func needCorsConfig(annotations Annotations) bool { + return annotations.HasASAP(enableCors) +} + +func splitStringWithSpaceTrim(input string) []string { + out := strings.Split(input, ",") + for i, item := range out { + converted := strings.TrimSpace(item) + if converted == "*" { + return []string{"*"} + } + out[i] = converted + } + return out +} diff --git a/ingress/kube/annotations/cors_test.go b/ingress/kube/annotations/cors_test.go new file mode 100644 index 000000000..9dff49950 --- /dev/null +++ b/ingress/kube/annotations/cors_test.go @@ -0,0 +1,282 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "reflect" + "testing" + + "github.com/gogo/protobuf/proto" + "github.com/gogo/protobuf/types" + networking "istio.io/api/networking/v1alpha3" +) + +func TestSplitStringWithSpaceTrim(t *testing.T) { + testCases := []struct { + input string + expect []string + }{ + { + input: "*", + expect: []string{"*"}, + }, + { + input: "a, b, c", + expect: []string{"a", "b", "c"}, + }, + { + input: "a, *, c", + expect: []string{"*"}, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + result := splitStringWithSpaceTrim(testCase.input) + if !reflect.DeepEqual(testCase.expect, result) { + t.Fatalf("Must be equal, but got %s", result) + } + }) + } +} + +func TestCorsParse(t *testing.T) { + cors := cors{} + testCases := []struct { + input Annotations + expect *CorsConfig + }{ + { + input: Annotations{}, + expect: nil, + }, + { + input: Annotations{ + buildNginxAnnotationKey(enableCors): "false", + }, + expect: nil, + }, + { + input: Annotations{ + buildNginxAnnotationKey(enableCors): "true", + }, + expect: &CorsConfig{ + Enabled: true, + AllowOrigin: []string{defaultAllowOrigin}, + AllowMethods: splitStringWithSpaceTrim(defaultAllowMethods), + AllowHeaders: splitStringWithSpaceTrim(defaultAllowHeaders), + AllowCredentials: defaultAllowCredentials, + MaxAge: defaultMaxAge, + }, + }, + { + input: Annotations{ + buildNginxAnnotationKey(enableCors): "true", + buildNginxAnnotationKey(allowOrigin): "https://origin-site.com:4443, http://origin-site.com, https://example.org:1199", + }, + expect: &CorsConfig{ + Enabled: true, + AllowOrigin: []string{"https://origin-site.com:4443", "http://origin-site.com", "https://example.org:1199"}, + AllowMethods: splitStringWithSpaceTrim(defaultAllowMethods), + AllowHeaders: splitStringWithSpaceTrim(defaultAllowHeaders), + AllowCredentials: defaultAllowCredentials, + MaxAge: defaultMaxAge, + }, + }, + { + input: Annotations{ + buildNginxAnnotationKey(enableCors): "true", + buildNginxAnnotationKey(allowOrigin): "https://origin-site.com:4443, http://origin-site.com, https://example.org:1199", + buildNginxAnnotationKey(allowMethods): "GET, PUT", + buildNginxAnnotationKey(allowHeaders): "foo,bar", + }, + expect: &CorsConfig{ + Enabled: true, + AllowOrigin: []string{"https://origin-site.com:4443", "http://origin-site.com", "https://example.org:1199"}, + AllowMethods: []string{"GET", "PUT"}, + AllowHeaders: []string{"foo", "bar"}, + AllowCredentials: defaultAllowCredentials, + MaxAge: defaultMaxAge, + }, + }, + { + input: Annotations{ + buildNginxAnnotationKey(enableCors): "true", + buildNginxAnnotationKey(allowOrigin): "https://origin-site.com:4443, http://origin-site.com, https://example.org:1199", + buildNginxAnnotationKey(allowMethods): "GET, PUT", + buildNginxAnnotationKey(allowHeaders): "foo,bar", + buildNginxAnnotationKey(allowCredentials): "false", + buildNginxAnnotationKey(maxAge): "100", + }, + expect: &CorsConfig{ + Enabled: true, + AllowOrigin: []string{"https://origin-site.com:4443", "http://origin-site.com", "https://example.org:1199"}, + AllowMethods: []string{"GET", "PUT"}, + AllowHeaders: []string{"foo", "bar"}, + AllowCredentials: false, + MaxAge: 100, + }, + }, + { + input: Annotations{ + buildMSEAnnotationKey(enableCors): "true", + buildNginxAnnotationKey(allowOrigin): "https://origin-site.com:4443, http://origin-site.com, https://example.org:1199", + buildMSEAnnotationKey(allowMethods): "GET, PUT", + buildNginxAnnotationKey(allowHeaders): "foo,bar", + buildNginxAnnotationKey(allowCredentials): "false", + buildNginxAnnotationKey(maxAge): "100", + }, + expect: &CorsConfig{ + Enabled: true, + AllowOrigin: []string{"https://origin-site.com:4443", "http://origin-site.com", "https://example.org:1199"}, + AllowMethods: []string{"GET", "PUT"}, + AllowHeaders: []string{"foo", "bar"}, + AllowCredentials: false, + MaxAge: 100, + }, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + config := &Ingress{} + _ = cors.Parse(testCase.input, config, nil) + if !reflect.DeepEqual(config.Cors, testCase.expect) { + t.Fatalf("Must be equal.") + } + }) + } +} + +func TestCorsApplyRoute(t *testing.T) { + cors := cors{} + testCases := []struct { + route *networking.HTTPRoute + config *Ingress + expect *networking.HTTPRoute + }{ + { + route: &networking.HTTPRoute{}, + config: &Ingress{}, + expect: &networking.HTTPRoute{}, + }, + { + route: &networking.HTTPRoute{}, + config: &Ingress{ + Cors: &CorsConfig{ + Enabled: false, + }, + }, + expect: &networking.HTTPRoute{}, + }, + { + route: &networking.HTTPRoute{}, + config: &Ingress{ + Cors: &CorsConfig{ + Enabled: true, + AllowOrigin: []string{"https://origin-site.com:4443", "http://origin-site.com", "https://example.org:1199"}, + AllowMethods: []string{"GET", "POST"}, + AllowHeaders: []string{"test", "unique"}, + ExposeHeaders: []string{"hello", "bye"}, + AllowCredentials: defaultAllowCredentials, + MaxAge: defaultMaxAge, + }, + }, + expect: &networking.HTTPRoute{ + CorsPolicy: &networking.CorsPolicy{ + AllowOrigins: []*networking.StringMatch{ + { + MatchType: &networking.StringMatch_Exact{ + Exact: "https://origin-site.com:4443", + }, + }, + { + MatchType: &networking.StringMatch_Exact{ + Exact: "http://origin-site.com", + }, + }, + { + MatchType: &networking.StringMatch_Exact{ + Exact: "https://example.org:1199", + }, + }, + }, + AllowMethods: []string{"GET", "POST"}, + AllowHeaders: []string{"test", "unique"}, + ExposeHeaders: []string{"hello", "bye"}, + AllowCredentials: &types.BoolValue{ + Value: true, + }, + MaxAge: &types.Duration{ + Seconds: defaultMaxAge, + }, + }, + }, + }, + { + route: &networking.HTTPRoute{}, + config: &Ingress{ + Cors: &CorsConfig{ + Enabled: true, + AllowOrigin: []string{"https://*.origin-site.com:4443", "http://*.origin-site.com", "https://example.org:1199"}, + AllowMethods: []string{"GET", "POST"}, + AllowHeaders: []string{"test", "unique"}, + ExposeHeaders: []string{"hello", "bye"}, + AllowCredentials: defaultAllowCredentials, + MaxAge: defaultMaxAge, + }, + }, + expect: &networking.HTTPRoute{ + CorsPolicy: &networking.CorsPolicy{ + AllowOrigins: []*networking.StringMatch{ + { + MatchType: &networking.StringMatch_Regex{ + Regex: ".*\\.origin-site\\.com:4443", + }, + }, + { + MatchType: &networking.StringMatch_Regex{ + Regex: ".*\\.origin-site\\.com", + }, + }, + { + MatchType: &networking.StringMatch_Exact{ + Exact: "https://example.org:1199", + }, + }, + }, + AllowMethods: []string{"GET", "POST"}, + AllowHeaders: []string{"test", "unique"}, + ExposeHeaders: []string{"hello", "bye"}, + AllowCredentials: &types.BoolValue{ + Value: true, + }, + MaxAge: &types.Duration{ + Seconds: defaultMaxAge, + }, + }, + }, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + cors.ApplyRoute(testCase.route, testCase.config) + if !proto.Equal(testCase.route, testCase.expect) { + t.Fatal("Must be equal.") + } + }) + } +} diff --git a/ingress/kube/annotations/default_backend.go b/ingress/kube/annotations/default_backend.go new file mode 100644 index 000000000..1c3b248aa --- /dev/null +++ b/ingress/kube/annotations/default_backend.go @@ -0,0 +1,151 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "strconv" + + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/model" + + "github.com/alibaba/higress/ingress/kube/util" + . "github.com/alibaba/higress/ingress/log" +) + +const ( + annDefaultBackend = "default-backend" + customHTTPError = "custom-http-errors" + + defaultRedirectUrl = "http://example.com/" + FallbackRouteNameSuffix = "-fallback" + FallbackInjectHeaderRouteName = "x-envoy-route-name" + FallbackInjectFallbackService = "x-envoy-fallback-service" +) + +var ( + _ Parser = fallback{} + _ RouteHandler = fallback{} +) + +type FallbackConfig struct { + DefaultBackend model.NamespacedName + Port uint32 + customHTTPErrors []uint32 +} + +type fallback struct{} + +func (f fallback) Parse(annotations Annotations, config *Ingress, globalContext *GlobalContext) error { + if !needFallback(annotations) { + return nil + } + + fallBackConfig := &FallbackConfig{} + svcName, err := annotations.ParseStringASAP(annDefaultBackend) + if err != nil { + IngressLog.Errorf("Parse annotation default backend err: %v", err) + return nil + } + + fallBackConfig.DefaultBackend = util.SplitNamespacedName(svcName) + if fallBackConfig.DefaultBackend.Name == "" { + IngressLog.Errorf("Annotation default backend within ingress %s/%s is invalid", config.Namespace, config.Name) + return nil + } + // Use ingress namespace instead, if user don't specify the namespace for default backend svc. + if fallBackConfig.DefaultBackend.Namespace == "" { + fallBackConfig.DefaultBackend.Namespace = config.Namespace + } + + serviceLister, exist := globalContext.ClusterServiceList[config.ClusterId] + if !exist { + IngressLog.Errorf("service lister of cluster %s doesn't exist", config.ClusterId) + return nil + } + + fallbackSvc, err := serviceLister.Services(fallBackConfig.DefaultBackend.Namespace).Get(fallBackConfig.DefaultBackend.Name) + if err != nil { + IngressLog.Errorf("Fallback service %s/%s within ingress %s/%s is not found", + fallBackConfig.DefaultBackend.Namespace, fallBackConfig.DefaultBackend.Name, config.Namespace, config.Name) + return nil + } + if len(fallbackSvc.Spec.Ports) == 0 { + IngressLog.Errorf("Fallback service %s/%s within ingress %s/%s haven't ports", + fallBackConfig.DefaultBackend.Namespace, fallBackConfig.DefaultBackend.Name, config.Namespace, config.Name) + return nil + } + // Use the first port like nginx ingress. + fallBackConfig.Port = uint32(fallbackSvc.Spec.Ports[0].Port) + + config.Fallback = fallBackConfig + + if codes, err := annotations.ParseStringASAP(customHTTPError); err == nil { + codesStr := splitBySeparator(codes, ",") + var codesUint32 []uint32 + for _, rawCode := range codesStr { + code, err := strconv.Atoi(rawCode) + if err != nil { + IngressLog.Errorf("Custom HTTP code %s within ingress %s/%s is invalid", rawCode, config.Namespace, config.Name) + continue + } + codesUint32 = append(codesUint32, uint32(code)) + } + fallBackConfig.customHTTPErrors = codesUint32 + } + + return nil +} + +func (f fallback) ApplyRoute(route *networking.HTTPRoute, config *Ingress) { + fallback := config.Fallback + if fallback == nil { + return + } + + // Add fallback svc + route.Route[0].FallbackClusters = []*networking.Destination{ + { + Host: util.CreateServiceFQDN(fallback.DefaultBackend.Namespace, fallback.DefaultBackend.Name), + Port: &networking.PortSelector{ + Number: fallback.Port, + }, + }, + } + + if len(fallback.customHTTPErrors) > 0 { + route.InternalActiveRedirect = &networking.HTTPInternalActiveRedirect{ + MaxInternalRedirects: 1, + RedirectResponseCodes: fallback.customHTTPErrors, + AllowCrossScheme: true, + Headers: &networking.Headers{ + Request: &networking.Headers_HeaderOperations{ + Add: map[string]string{ + FallbackInjectHeaderRouteName: route.Name + FallbackRouteNameSuffix, + FallbackInjectFallbackService: fallback.DefaultBackend.String(), + }, + }, + }, + RedirectUrlRewriteSpecifier: &networking.HTTPInternalActiveRedirect_RedirectUrl{ + RedirectUrl: defaultRedirectUrl, + }, + ForcedUseOriginalHost: true, + ForcedAddHeaderBeforeRouteMatcher: true, + } + } +} + +func needFallback(annotations Annotations) bool { + return annotations.HasASAP(annDefaultBackend) +} diff --git a/ingress/kube/annotations/default_backend_test.go b/ingress/kube/annotations/default_backend_test.go new file mode 100644 index 000000000..f619fb81a --- /dev/null +++ b/ingress/kube/annotations/default_backend_test.go @@ -0,0 +1,229 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "context" + "reflect" + "testing" + "time" + + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/model" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + listerv1 "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" +) + +var ( + normalService = &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "app", + Namespace: "test", + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{{ + Name: "http", + Port: 80, + }}, + }, + } + + abnormalService = &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "app", + Namespace: "foo", + }, + } +) + +func TestFallbackParse(t *testing.T) { + fallback := fallback{} + inputCases := []struct { + input map[string]string + expect *FallbackConfig + }{ + {}, + { + input: map[string]string{ + buildNginxAnnotationKey(annDefaultBackend): "test/app", + }, + expect: &FallbackConfig{ + DefaultBackend: model.NamespacedName{ + Namespace: "test", + Name: "app", + }, + Port: 80, + }, + }, + { + input: map[string]string{ + buildMSEAnnotationKey(annDefaultBackend): "app", + }, + expect: &FallbackConfig{ + DefaultBackend: model.NamespacedName{ + Namespace: "test", + Name: "app", + }, + Port: 80, + }, + }, + { + input: map[string]string{ + buildMSEAnnotationKey(annDefaultBackend): "foo/app", + }, + }, + { + input: map[string]string{ + buildMSEAnnotationKey(annDefaultBackend): "test/app", + buildNginxAnnotationKey(customHTTPError): "404,503", + }, + expect: &FallbackConfig{ + DefaultBackend: model.NamespacedName{ + Namespace: "test", + Name: "app", + }, + Port: 80, + customHTTPErrors: []uint32{404, 503}, + }, + }, + { + input: map[string]string{ + buildMSEAnnotationKey(annDefaultBackend): "test/app", + buildNginxAnnotationKey(customHTTPError): "404,5ac", + }, + expect: &FallbackConfig{ + DefaultBackend: model.NamespacedName{ + Namespace: "test", + Name: "app", + }, + Port: 80, + customHTTPErrors: []uint32{404}, + }, + }, + } + + for _, inputCase := range inputCases { + t.Run("", func(t *testing.T) { + config := &Ingress{ + Meta: Meta{ + Namespace: "test", + ClusterId: "cluster", + }, + } + globalContext, cancel := initGlobalContextForService() + defer cancel() + + _ = fallback.Parse(inputCase.input, config, globalContext) + if !reflect.DeepEqual(inputCase.expect, config.Fallback) { + t.Fatal("Should be equal") + } + }) + } +} + +func TestFallbackApplyRoute(t *testing.T) { + fallback := fallback{} + inputCases := []struct { + config *Ingress + input *networking.HTTPRoute + expect *networking.HTTPRoute + }{ + { + config: &Ingress{}, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{}, + }, + { + config: &Ingress{ + Fallback: &FallbackConfig{ + DefaultBackend: model.NamespacedName{ + Namespace: "test", + Name: "app", + }, + Port: 80, + customHTTPErrors: []uint32{404, 503}, + }, + }, + input: &networking.HTTPRoute{ + Name: "route", + Route: []*networking.HTTPRouteDestination{ + {}, + }, + }, + expect: &networking.HTTPRoute{ + Name: "route", + InternalActiveRedirect: &networking.HTTPInternalActiveRedirect{ + MaxInternalRedirects: 1, + RedirectResponseCodes: []uint32{404, 503}, + AllowCrossScheme: true, + Headers: &networking.Headers{ + Request: &networking.Headers_HeaderOperations{ + Add: map[string]string{ + FallbackInjectHeaderRouteName: "route" + FallbackRouteNameSuffix, + FallbackInjectFallbackService: "test/app", + }, + }, + }, + RedirectUrlRewriteSpecifier: &networking.HTTPInternalActiveRedirect_RedirectUrl{ + RedirectUrl: defaultRedirectUrl, + }, + ForcedUseOriginalHost: true, + ForcedAddHeaderBeforeRouteMatcher: true, + }, + Route: []*networking.HTTPRouteDestination{ + { + FallbackClusters: []*networking.Destination{ + { + Host: "app.test.svc.cluster.local", + Port: &networking.PortSelector{ + Number: 80, + }, + }, + }, + }, + }, + }, + }, + } + + for _, inputCase := range inputCases { + t.Run("", func(t *testing.T) { + fallback.ApplyRoute(inputCase.input, inputCase.config) + if !reflect.DeepEqual(inputCase.input, inputCase.expect) { + t.Fatal("Should be equal") + } + }) + } +} + +func initGlobalContextForService() (*GlobalContext, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + + client := fake.NewSimpleClientset(normalService, abnormalService) + informerFactory := informers.NewSharedInformerFactory(client, time.Hour) + serviceInformer := informerFactory.Core().V1().Services() + go serviceInformer.Informer().Run(ctx.Done()) + cache.WaitForCacheSync(ctx.Done(), serviceInformer.Informer().HasSynced) + + return &GlobalContext{ + ClusterServiceList: map[string]listerv1.ServiceLister{ + "cluster": serviceInformer.Lister(), + }, + }, cancel +} diff --git a/ingress/kube/annotations/downstreamtls.go b/ingress/kube/annotations/downstreamtls.go new file mode 100644 index 000000000..4ba46e8c4 --- /dev/null +++ b/ingress/kube/annotations/downstreamtls.go @@ -0,0 +1,164 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "strings" + + networking "istio.io/api/networking/v1alpha3" + "github.com/alibaba/higress/ingress/kube/util" + . "github.com/alibaba/higress/ingress/log" + "istio.io/istio/pilot/pkg/credentials/kube" + "istio.io/istio/pilot/pkg/model" + gatewaytool "istio.io/istio/pkg/config/gateway" + "istio.io/istio/pkg/config/security" +) + +const ( + authTLSSecret = "auth-tls-secret" + tlsMinVersion = "tls-min-protocol-version" + tlsMaxVersion = "tls-max-protocol-version" + sslCipher = "ssl-cipher" +) + +type TLSProtocolVersion string + +const ( + tlsV10 TLSProtocolVersion = "TLSv1.0" + tlsV11 TLSProtocolVersion = "TLSv1.1" + tlsV12 TLSProtocolVersion = "TLSv1.2" + tlsV13 TLSProtocolVersion = "TLSv1.3" +) + +var ( + _ Parser = &downstreamTLS{} + _ GatewayHandler = &downstreamTLS{} + + tlsProtocol = map[TLSProtocolVersion]networking.ServerTLSSettings_TLSProtocol{ + tlsV10: networking.ServerTLSSettings_TLSV1_0, + tlsV11: networking.ServerTLSSettings_TLSV1_1, + tlsV12: networking.ServerTLSSettings_TLSV1_2, + tlsV13: networking.ServerTLSSettings_TLSV1_3, + } +) + +func isValidTLSProtocolVersion(protocol string) bool { + tls := TLSProtocolVersion(protocol) + _, exist := tlsProtocol[tls] + return exist +} + +func Convert(protocol string) networking.ServerTLSSettings_TLSProtocol { + return tlsProtocol[TLSProtocolVersion(protocol)] +} + +type DownstreamTLSConfig struct { + TlsMinVersion TLSProtocolVersion + TlsMaxVersion TLSProtocolVersion + CipherSuites []string + Mode networking.ServerTLSSettings_TLSmode + CASecretName model.NamespacedName +} + +type downstreamTLS struct{} + +func (d downstreamTLS) Parse(annotations Annotations, config *Ingress, _ *GlobalContext) error { + if !needDownstreamTLS(annotations) { + return nil + } + + downstreamTLSConfig := &DownstreamTLSConfig{ + Mode: networking.ServerTLSSettings_SIMPLE, + } + defer func() { + config.DownstreamTLS = downstreamTLSConfig + }() + + if secretName, err := annotations.ParseStringASAP(authTLSSecret); err == nil { + namespacedName := util.SplitNamespacedName(secretName) + if namespacedName.Name == "" { + IngressLog.Errorf("CA secret name %s format is invalid.", secretName) + } else { + if namespacedName.Namespace == "" { + namespacedName.Namespace = config.Namespace + } + downstreamTLSConfig.CASecretName = namespacedName + downstreamTLSConfig.Mode = networking.ServerTLSSettings_MUTUAL + } + } + + if minVersion, err := annotations.ParseStringForMSE(tlsMinVersion); err == nil && + isValidTLSProtocolVersion(minVersion) { + downstreamTLSConfig.TlsMinVersion = TLSProtocolVersion(minVersion) + } + + if maxVersion, err := annotations.ParseStringForMSE(tlsMaxVersion); err == nil && + isValidTLSProtocolVersion(maxVersion) { + downstreamTLSConfig.TlsMaxVersion = TLSProtocolVersion(maxVersion) + } + + if rawTlsCipherSuite, err := annotations.ParseStringASAP(sslCipher); err == nil { + var validCipherSuite []string + cipherList := strings.Split(rawTlsCipherSuite, ":") + for _, cipher := range cipherList { + if security.IsValidCipherSuite(cipher) { + validCipherSuite = append(validCipherSuite, cipher) + } + } + + downstreamTLSConfig.CipherSuites = validCipherSuite + } + + return nil +} + +func (d downstreamTLS) ApplyGateway(gateway *networking.Gateway, config *Ingress) { + if config.DownstreamTLS == nil { + return + } + + downstreamTLSConfig := config.DownstreamTLS + for _, server := range gateway.Servers { + if gatewaytool.IsTLSServer(server) { + if downstreamTLSConfig.CASecretName.Name != "" { + serverCert := extraSecret(server.Tls.CredentialName) + if downstreamTLSConfig.CASecretName.Namespace != serverCert.Namespace || + (downstreamTLSConfig.CASecretName.Name != serverCert.Name && + downstreamTLSConfig.CASecretName.Name != serverCert.Name+kube.GatewaySdsCaSuffix) { + IngressLog.Errorf("CA secret %s is invalid", downstreamTLSConfig.CASecretName.String()) + } else { + server.Tls.Mode = downstreamTLSConfig.Mode + } + } + + if downstreamTLSConfig.TlsMinVersion != "" { + server.Tls.MinProtocolVersion = tlsProtocol[downstreamTLSConfig.TlsMinVersion] + } + if downstreamTLSConfig.TlsMaxVersion != "" { + server.Tls.MaxProtocolVersion = tlsProtocol[downstreamTLSConfig.TlsMaxVersion] + } + if len(downstreamTLSConfig.CipherSuites) != 0 { + server.Tls.CipherSuites = downstreamTLSConfig.CipherSuites + } + } + } +} + +func needDownstreamTLS(annotations Annotations) bool { + return annotations.HasMSE(tlsMinVersion) || + annotations.HasMSE(tlsMaxVersion) || + annotations.HasASAP(sslCipher) || + annotations.HasASAP(authTLSSecret) +} diff --git a/ingress/kube/annotations/downstreamtls_test.go b/ingress/kube/annotations/downstreamtls_test.go new file mode 100644 index 000000000..082e424b9 --- /dev/null +++ b/ingress/kube/annotations/downstreamtls_test.go @@ -0,0 +1,351 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "reflect" + "testing" + + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/model" +) + +var parser = downstreamTLS{} + +func TestParse(t *testing.T) { + testCases := []struct { + input map[string]string + expect *DownstreamTLSConfig + }{ + {}, + { + input: map[string]string{ + MSEAnnotationsPrefix + "/" + tlsMinVersion: "TLSv1.0", + }, + expect: &DownstreamTLSConfig{ + Mode: networking.ServerTLSSettings_SIMPLE, + TlsMinVersion: tlsV10, + }, + }, + { + input: map[string]string{ + MSEAnnotationsPrefix + "/" + tlsMinVersion: "TLSv1.3", + DefaultAnnotationsPrefix + "/" + sslCipher: "ECDHE-RSA-AES256-GCM-SHA384:AES128-SHA", + }, + expect: &DownstreamTLSConfig{ + Mode: networking.ServerTLSSettings_SIMPLE, + TlsMinVersion: tlsV13, + CipherSuites: []string{"ECDHE-RSA-AES256-GCM-SHA384", "AES128-SHA"}, + }, + }, + { + input: map[string]string{ + MSEAnnotationsPrefix + "/" + tlsMinVersion: "xxx", + DefaultAnnotationsPrefix + "/" + sslCipher: "ECDHE-RSA-AES256-GCM-SHA384:AES128-SHA", + }, + expect: &DownstreamTLSConfig{ + Mode: networking.ServerTLSSettings_SIMPLE, + CipherSuites: []string{"ECDHE-RSA-AES256-GCM-SHA384", "AES128-SHA"}, + }, + }, + { + input: map[string]string{ + MSEAnnotationsPrefix + "/" + tlsMinVersion: "xxx", + MSEAnnotationsPrefix + "/" + sslCipher: "ECDHE-RSA-AES256-GCM-SHA384:AES128-SHA", + }, + expect: &DownstreamTLSConfig{ + Mode: networking.ServerTLSSettings_SIMPLE, + CipherSuites: []string{"ECDHE-RSA-AES256-GCM-SHA384", "AES128-SHA"}, + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(authTLSSecret): "test", + MSEAnnotationsPrefix + "/" + tlsMinVersion: "xxx", + MSEAnnotationsPrefix + "/" + sslCipher: "ECDHE-RSA-AES256-GCM-SHA384:AES128-SHA", + }, + expect: &DownstreamTLSConfig{ + CASecretName: model.NamespacedName{ + Namespace: "foo", + Name: "test", + }, + Mode: networking.ServerTLSSettings_MUTUAL, + CipherSuites: []string{"ECDHE-RSA-AES256-GCM-SHA384", "AES128-SHA"}, + }, + }, + { + input: map[string]string{ + buildMSEAnnotationKey(authTLSSecret): "test/foo", + MSEAnnotationsPrefix + "/" + tlsMinVersion: "TLSv1.3", + DefaultAnnotationsPrefix + "/" + sslCipher: "ECDHE-RSA-AES256-GCM-SHA384:AES128-SHA", + }, + expect: &DownstreamTLSConfig{ + CASecretName: model.NamespacedName{ + Namespace: "test", + Name: "foo", + }, + Mode: networking.ServerTLSSettings_MUTUAL, + TlsMinVersion: tlsV13, + CipherSuites: []string{"ECDHE-RSA-AES256-GCM-SHA384", "AES128-SHA"}, + }, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + config := &Ingress{ + Meta: Meta{ + Namespace: "foo", + }, + } + _ = parser.Parse(testCase.input, config, nil) + if !reflect.DeepEqual(testCase.expect, config.DownstreamTLS) { + t.Fatalf("Should be equal") + } + }) + } +} + +func TestApplyGateway(t *testing.T) { + testCases := []struct { + input *networking.Gateway + config *Ingress + expect *networking.Gateway + }{ + { + input: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Protocol: "HTTP", + }, + }, + }, + }, + config: &Ingress{ + DownstreamTLS: &DownstreamTLSConfig{ + TlsMinVersion: tlsV10, + }, + }, + expect: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Protocol: "HTTP", + }, + }, + }, + }, + }, + { + input: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Protocol: "HTTPS", + }, + Tls: &networking.ServerTLSSettings{ + Mode: networking.ServerTLSSettings_SIMPLE, + }, + }, + }, + }, + config: &Ingress{ + DownstreamTLS: &DownstreamTLSConfig{ + TlsMinVersion: tlsV12, + }, + }, + expect: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Protocol: "HTTPS", + }, + Tls: &networking.ServerTLSSettings{ + Mode: networking.ServerTLSSettings_SIMPLE, + MinProtocolVersion: networking.ServerTLSSettings_TLSV1_2, + }, + }, + }, + }, + }, + { + input: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Protocol: "HTTPS", + }, + Tls: &networking.ServerTLSSettings{ + Mode: networking.ServerTLSSettings_SIMPLE, + }, + }, + }, + }, + config: &Ingress{ + DownstreamTLS: &DownstreamTLSConfig{ + TlsMaxVersion: tlsV13, + CipherSuites: []string{"ECDHE-RSA-AES256-GCM-SHA384"}, + }, + }, + expect: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Protocol: "HTTPS", + }, + Tls: &networking.ServerTLSSettings{ + Mode: networking.ServerTLSSettings_SIMPLE, + MaxProtocolVersion: networking.ServerTLSSettings_TLSV1_3, + CipherSuites: []string{"ECDHE-RSA-AES256-GCM-SHA384"}, + }, + }, + }, + }, + }, + { + input: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Protocol: "HTTPS", + }, + Tls: &networking.ServerTLSSettings{ + Mode: networking.ServerTLSSettings_SIMPLE, + CredentialName: "kubernetes-ingress://cluster/foo/bar", + }, + }, + }, + }, + config: &Ingress{ + DownstreamTLS: &DownstreamTLSConfig{ + CASecretName: model.NamespacedName{ + Namespace: "foo", + Name: "bar", + }, + Mode: networking.ServerTLSSettings_MUTUAL, + TlsMaxVersion: tlsV13, + CipherSuites: []string{"ECDHE-RSA-AES256-GCM-SHA384"}, + }, + }, + expect: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Protocol: "HTTPS", + }, + Tls: &networking.ServerTLSSettings{ + CredentialName: "kubernetes-ingress://cluster/foo/bar", + Mode: networking.ServerTLSSettings_MUTUAL, + MaxProtocolVersion: networking.ServerTLSSettings_TLSV1_3, + CipherSuites: []string{"ECDHE-RSA-AES256-GCM-SHA384"}, + }, + }, + }, + }, + }, + { + input: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Protocol: "HTTPS", + }, + Tls: &networking.ServerTLSSettings{ + Mode: networking.ServerTLSSettings_SIMPLE, + CredentialName: "kubernetes-ingress://cluster/foo/bar", + }, + }, + }, + }, + config: &Ingress{ + DownstreamTLS: &DownstreamTLSConfig{ + CASecretName: model.NamespacedName{ + Namespace: "foo", + Name: "bar-cacert", + }, + Mode: networking.ServerTLSSettings_MUTUAL, + TlsMaxVersion: tlsV13, + CipherSuites: []string{"ECDHE-RSA-AES256-GCM-SHA384"}, + }, + }, + expect: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Protocol: "HTTPS", + }, + Tls: &networking.ServerTLSSettings{ + CredentialName: "kubernetes-ingress://cluster/foo/bar", + Mode: networking.ServerTLSSettings_MUTUAL, + MaxProtocolVersion: networking.ServerTLSSettings_TLSV1_3, + CipherSuites: []string{"ECDHE-RSA-AES256-GCM-SHA384"}, + }, + }, + }, + }, + }, + { + input: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Protocol: "HTTPS", + }, + Tls: &networking.ServerTLSSettings{ + Mode: networking.ServerTLSSettings_SIMPLE, + CredentialName: "kubernetes-ingress://cluster/foo/bar", + }, + }, + }, + }, + config: &Ingress{ + DownstreamTLS: &DownstreamTLSConfig{ + CASecretName: model.NamespacedName{ + Namespace: "bar", + Name: "foo", + }, + Mode: networking.ServerTLSSettings_MUTUAL, + TlsMaxVersion: tlsV13, + CipherSuites: []string{"ECDHE-RSA-AES256-GCM-SHA384"}, + }, + }, + expect: &networking.Gateway{ + Servers: []*networking.Server{ + { + Port: &networking.Port{ + Protocol: "HTTPS", + }, + Tls: &networking.ServerTLSSettings{ + CredentialName: "kubernetes-ingress://cluster/foo/bar", + Mode: networking.ServerTLSSettings_SIMPLE, + MaxProtocolVersion: networking.ServerTLSSettings_TLSV1_3, + CipherSuites: []string{"ECDHE-RSA-AES256-GCM-SHA384"}, + }, + }, + }, + }, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + parser.ApplyGateway(testCase.input, testCase.config) + if !reflect.DeepEqual(testCase.input, testCase.expect) { + t.Fatalf("Should be equal") + } + }) + } +} diff --git a/ingress/kube/annotations/header_control.go b/ingress/kube/annotations/header_control.go new file mode 100644 index 000000000..635df65b2 --- /dev/null +++ b/ingress/kube/annotations/header_control.go @@ -0,0 +1,160 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "strings" + + networking "istio.io/api/networking/v1alpha3" + + . "github.com/alibaba/higress/ingress/log" +) + +const ( + // request + requestHeaderAdd = "request-header-control-add" + requestHeaderUpdate = "request-header-control-update" + requestHeaderRemove = "request-header-control-remove" + + // response + responseHeaderAdd = "response-header-control-add" + responseHeaderUpdate = "response-header-control-update" + responseHeaderRemove = "response-header-control-remove" +) + +var ( + _ Parser = headerControl{} + _ RouteHandler = headerControl{} +) + +type HeaderOperation struct { + Add map[string]string + Update map[string]string + Remove []string +} + +// HeaderControlConfig enforces header operations on route level. +// Note: Canary route don't use header control applied on the normal route. +type HeaderControlConfig struct { + Request *HeaderOperation + Response *HeaderOperation +} + +type headerControl struct{} + +func (h headerControl) Parse(annotations Annotations, config *Ingress, _ *GlobalContext) error { + if !needHeaderControlConfig(annotations) { + return nil + } + + config.HeaderControl = &HeaderControlConfig{} + + var requestAdd map[string]string + var requestUpdate map[string]string + var requestRemove []string + if add, err := annotations.ParseStringForMSE(requestHeaderAdd); err == nil { + requestAdd = convertAddOrUpdate(add) + } + if update, err := annotations.ParseStringForMSE(requestHeaderUpdate); err == nil { + requestUpdate = convertAddOrUpdate(update) + } + if remove, err := annotations.ParseStringForMSE(requestHeaderRemove); err == nil { + requestRemove = splitBySeparator(remove, ",") + } + if len(requestAdd) > 0 || len(requestUpdate) > 0 || len(requestRemove) > 0 { + config.HeaderControl.Request = &HeaderOperation{ + Add: requestAdd, + Update: requestUpdate, + Remove: requestRemove, + } + } + + var responseAdd map[string]string + var responseUpdate map[string]string + var responseRemove []string + if add, err := annotations.ParseStringForMSE(responseHeaderAdd); err == nil { + responseAdd = convertAddOrUpdate(add) + } + if update, err := annotations.ParseStringForMSE(responseHeaderUpdate); err == nil { + responseUpdate = convertAddOrUpdate(update) + } + if remove, err := annotations.ParseStringForMSE(responseHeaderRemove); err == nil { + responseRemove = splitBySeparator(remove, ",") + } + if len(responseAdd) > 0 || len(responseUpdate) > 0 || len(responseRemove) > 0 { + config.HeaderControl.Response = &HeaderOperation{ + Add: responseAdd, + Update: responseUpdate, + Remove: responseRemove, + } + } + + return nil +} + +func (h headerControl) ApplyRoute(route *networking.HTTPRoute, config *Ingress) { + headerControlConfig := config.HeaderControl + if headerControlConfig == nil { + return + } + + headers := &networking.Headers{ + Request: &networking.Headers_HeaderOperations{}, + Response: &networking.Headers_HeaderOperations{}, + } + if headerControlConfig.Request != nil { + headers.Request.Add = headerControlConfig.Request.Add + headers.Request.Set = headerControlConfig.Request.Update + headers.Request.Remove = headerControlConfig.Request.Remove + } + + if headerControlConfig.Response != nil { + headers.Response.Add = headerControlConfig.Response.Add + headers.Response.Set = headerControlConfig.Response.Update + headers.Response.Remove = headerControlConfig.Response.Remove + } + + route.Headers = headers +} + +func needHeaderControlConfig(annotations Annotations) bool { + return annotations.HasMSE(requestHeaderAdd) || + annotations.HasMSE(requestHeaderUpdate) || + annotations.HasMSE(requestHeaderRemove) || + annotations.HasMSE(responseHeaderAdd) || + annotations.HasMSE(responseHeaderUpdate) || + annotations.HasMSE(responseHeaderRemove) +} + +func convertAddOrUpdate(headers string) map[string]string { + result := map[string]string{} + parts := strings.Split(headers, "\n") + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + + keyValue := strings.Fields(part) + if len(keyValue) != 2 { + IngressLog.Infof("Header format %s is invalid.", keyValue) + continue + } + key := strings.TrimSpace(keyValue[0]) + value := strings.TrimSpace(keyValue[1]) + result[key] = value + } + return result +} diff --git a/ingress/kube/annotations/header_control_test.go b/ingress/kube/annotations/header_control_test.go new file mode 100644 index 000000000..363a6e5cc --- /dev/null +++ b/ingress/kube/annotations/header_control_test.go @@ -0,0 +1,235 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "reflect" + "testing" + + networking "istio.io/api/networking/v1alpha3" +) + +func TestHeaderControlParse(t *testing.T) { + headerControl := &headerControl{} + inputCases := []struct { + input map[string]string + expect *HeaderControlConfig + }{ + {}, + { + input: map[string]string{ + buildMSEAnnotationKey(requestHeaderAdd): "one 1", + buildMSEAnnotationKey(responseHeaderAdd): "A a", + }, + expect: &HeaderControlConfig{ + Request: &HeaderOperation{ + Add: map[string]string{ + "one": "1", + }, + }, + Response: &HeaderOperation{ + Add: map[string]string{ + "A": "a", + }, + }, + }, + }, + { + input: map[string]string{ + buildMSEAnnotationKey(requestHeaderAdd): "one 1\n two 2\nthree 3 \n", + buildMSEAnnotationKey(requestHeaderUpdate): "two 2", + buildMSEAnnotationKey(requestHeaderRemove): "one, two,three\n", + buildMSEAnnotationKey(responseHeaderAdd): "A a\nB b\n", + buildMSEAnnotationKey(responseHeaderUpdate): "X x\nY y\n", + buildMSEAnnotationKey(responseHeaderRemove): "x", + }, + expect: &HeaderControlConfig{ + Request: &HeaderOperation{ + Add: map[string]string{ + "one": "1", + "two": "2", + "three": "3", + }, + Update: map[string]string{ + "two": "2", + }, + Remove: []string{"one", "two", "three"}, + }, + Response: &HeaderOperation{ + Add: map[string]string{ + "A": "a", + "B": "b", + }, + Update: map[string]string{ + "X": "x", + "Y": "y", + }, + Remove: []string{"x"}, + }, + }, + }, + } + + for _, inputCase := range inputCases { + t.Run("", func(t *testing.T) { + config := &Ingress{} + _ = headerControl.Parse(inputCase.input, config, nil) + if !reflect.DeepEqual(inputCase.expect, config.HeaderControl) { + t.Fatal("Should be equal") + } + }) + } +} + +func TestHeaderControlApplyRoute(t *testing.T) { + headerControl := headerControl{} + inputCases := []struct { + config *Ingress + input *networking.HTTPRoute + expect *networking.HTTPRoute + }{ + { + config: &Ingress{}, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{}, + }, + { + config: &Ingress{ + HeaderControl: &HeaderControlConfig{}, + }, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{ + Headers: &networking.Headers{ + Request: &networking.Headers_HeaderOperations{}, + Response: &networking.Headers_HeaderOperations{}, + }, + }, + }, + { + config: &Ingress{ + HeaderControl: &HeaderControlConfig{ + Request: &HeaderOperation{ + Add: map[string]string{ + "one": "1", + "two": "2", + "three": "3", + }, + Update: map[string]string{ + "two": "2", + }, + Remove: []string{"one", "two", "three"}, + }, + }, + }, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{ + Headers: &networking.Headers{ + Request: &networking.Headers_HeaderOperations{ + Add: map[string]string{ + "one": "1", + "two": "2", + "three": "3", + }, + Set: map[string]string{ + "two": "2", + }, + Remove: []string{"one", "two", "three"}, + }, + Response: &networking.Headers_HeaderOperations{}, + }, + }, + }, + { + config: &Ingress{ + HeaderControl: &HeaderControlConfig{ + Response: &HeaderOperation{ + Add: map[string]string{ + "A": "a", + "B": "b", + }, + Update: map[string]string{ + "X": "x", + "Y": "y", + }, + Remove: []string{"x"}, + }, + }, + }, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{ + Headers: &networking.Headers{ + Request: &networking.Headers_HeaderOperations{}, + Response: &networking.Headers_HeaderOperations{ + Add: map[string]string{ + "A": "a", + "B": "b", + }, + Set: map[string]string{ + "X": "x", + "Y": "y", + }, + Remove: []string{"x"}, + }, + }, + }, + }, + { + config: &Ingress{ + HeaderControl: &HeaderControlConfig{ + Request: &HeaderOperation{ + Update: map[string]string{ + "two": "2", + }, + Remove: []string{"one", "two", "three"}, + }, + Response: &HeaderOperation{ + Add: map[string]string{ + "A": "a", + "B": "b", + }, + Remove: []string{"x"}, + }, + }, + }, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{ + Headers: &networking.Headers{ + Request: &networking.Headers_HeaderOperations{ + Set: map[string]string{ + "two": "2", + }, + Remove: []string{"one", "two", "three"}, + }, + Response: &networking.Headers_HeaderOperations{ + Add: map[string]string{ + "A": "a", + "B": "b", + }, + Remove: []string{"x"}, + }, + }, + }, + }, + } + + for _, inputCase := range inputCases { + t.Run("", func(t *testing.T) { + headerControl.ApplyRoute(inputCase.input, inputCase.config) + if !reflect.DeepEqual(inputCase.input, inputCase.expect) { + t.Fatal("Should be equal") + } + }) + } +} diff --git a/ingress/kube/annotations/interface.go b/ingress/kube/annotations/interface.go new file mode 100644 index 000000000..e5dfbc6f2 --- /dev/null +++ b/ingress/kube/annotations/interface.go @@ -0,0 +1,42 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import networking "istio.io/api/networking/v1alpha3" + +type Parser interface { + // Parse parses ingress annotations and puts result on config + Parse(annotations Annotations, config *Ingress, globalContext *GlobalContext) error +} + +type GatewayHandler interface { + // ApplyGateway parsed ingress annotation config reflected on gateway + ApplyGateway(gateway *networking.Gateway, config *Ingress) +} + +type VirtualServiceHandler interface { + // ApplyVirtualServiceHandler parsed ingress annotation config reflected on virtual host + ApplyVirtualServiceHandler(virtualService *networking.VirtualService, config *Ingress) +} + +type RouteHandler interface { + // ApplyRoute parsed ingress annotation config reflected on route + ApplyRoute(route *networking.HTTPRoute, config *Ingress) +} + +type TrafficPolicyHandler interface { + // ApplyTrafficPolicy parsed ingress annotation config reflected on traffic policy + ApplyTrafficPolicy(trafficPolicy *networking.TrafficPolicy_PortTrafficPolicy, config *Ingress) +} diff --git a/ingress/kube/annotations/ip_access_control.go b/ingress/kube/annotations/ip_access_control.go new file mode 100644 index 000000000..5c22c7607 --- /dev/null +++ b/ingress/kube/annotations/ip_access_control.go @@ -0,0 +1,144 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/networking/core/v1alpha3/mseingress" +) + +const ( + domainWhitelist = "domain-whitelist-source-range" + domainBlacklist = "domain-blacklist-source-range" + whitelist = "whitelist-source-range" + blacklist = "blacklist-source-range" +) + +var ( + _ Parser = &ipAccessControl{} + _ RouteHandler = &ipAccessControl{} +) + +type IPAccessControl struct { + isWhite bool + remoteIp []string +} + +type IPAccessControlConfig struct { + Domain *IPAccessControl + Route *IPAccessControl +} + +type ipAccessControl struct{} + +func (i ipAccessControl) Parse(annotations Annotations, config *Ingress, _ *GlobalContext) error { + if !needIPAccessControlConfig(annotations) { + return nil + } + + ipConfig := &IPAccessControlConfig{} + defer func() { + config.IPAccessControl = ipConfig + }() + + var domain *IPAccessControl + rawWhitelist, err := annotations.ParseStringForMSE(domainWhitelist) + if err == nil { + domain = &IPAccessControl{ + isWhite: true, + remoteIp: splitStringWithSpaceTrim(rawWhitelist), + } + } else { + if rawBlacklist, err := annotations.ParseStringForMSE(domainBlacklist); err == nil { + domain = &IPAccessControl{ + isWhite: false, + remoteIp: splitStringWithSpaceTrim(rawBlacklist), + } + } + } + if domain != nil { + ipConfig.Domain = domain + } + + var route *IPAccessControl + rawWhitelist, err = annotations.ParseStringASAP(whitelist) + if err == nil { + route = &IPAccessControl{ + isWhite: true, + remoteIp: splitStringWithSpaceTrim(rawWhitelist), + } + } else { + if rawBlacklist, err := annotations.ParseStringForMSE(blacklist); err == nil { + route = &IPAccessControl{ + isWhite: false, + remoteIp: splitStringWithSpaceTrim(rawBlacklist), + } + } + } + if route != nil { + ipConfig.Route = route + } + + return nil +} + +func (i ipAccessControl) ApplyVirtualServiceHandler(virtualService *networking.VirtualService, config *Ingress) { + ac := config.IPAccessControl + if ac == nil || ac.Domain == nil { + return + } + + filter := &networking.IPAccessControl{} + if ac.Domain.isWhite { + filter.RemoteIpBlocks = ac.Domain.remoteIp + } else { + filter.NotRemoteIpBlocks = ac.Domain.remoteIp + } + + virtualService.HostHTTPFilters = append(virtualService.HostHTTPFilters, &networking.HTTPFilter{ + Name: mseingress.IPAccessControl, + Filter: &networking.HTTPFilter_IpAccessControl{ + IpAccessControl: filter, + }, + }) +} + +func (i ipAccessControl) ApplyRoute(route *networking.HTTPRoute, config *Ingress) { + ac := config.IPAccessControl + if ac == nil || ac.Route == nil { + return + } + + filter := &networking.IPAccessControl{} + if ac.Route.isWhite { + filter.RemoteIpBlocks = ac.Route.remoteIp + } else { + filter.NotRemoteIpBlocks = ac.Route.remoteIp + } + + route.RouteHTTPFilters = append(route.RouteHTTPFilters, &networking.HTTPFilter{ + Name: mseingress.IPAccessControl, + Filter: &networking.HTTPFilter_IpAccessControl{ + IpAccessControl: filter, + }, + }) +} + +func needIPAccessControlConfig(annotations Annotations) bool { + return annotations.HasMSE(domainWhitelist) || + annotations.HasMSE(domainBlacklist) || + annotations.HasASAP(whitelist) || + annotations.HasMSE(blacklist) +} diff --git a/ingress/kube/annotations/ip_access_control_test.go b/ingress/kube/annotations/ip_access_control_test.go new file mode 100644 index 000000000..a08ba5cb2 --- /dev/null +++ b/ingress/kube/annotations/ip_access_control_test.go @@ -0,0 +1,241 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "reflect" + "testing" + + networking "istio.io/api/networking/v1alpha3" +) + +func TestIPAccessControlParse(t *testing.T) { + parser := ipAccessControl{} + + testCases := []struct { + input map[string]string + expect *IPAccessControlConfig + }{ + {}, + { + input: map[string]string{ + DefaultAnnotationsPrefix + "/" + whitelist: "1.1.1.1", + MSEAnnotationsPrefix + "/" + blacklist: "2.2.2.2", + }, + expect: &IPAccessControlConfig{ + Route: &IPAccessControl{ + isWhite: true, + remoteIp: []string{"1.1.1.1"}, + }, + }, + }, + { + input: map[string]string{ + MSEAnnotationsPrefix + "/" + blacklist: "2.2.2.2", + }, + expect: &IPAccessControlConfig{ + Route: &IPAccessControl{ + isWhite: false, + remoteIp: []string{"2.2.2.2"}, + }, + }, + }, + { + input: map[string]string{ + MSEAnnotationsPrefix + "/" + domainWhitelist: "1.1.1.1", + }, + expect: &IPAccessControlConfig{ + Domain: &IPAccessControl{ + isWhite: true, + remoteIp: []string{"1.1.1.1"}, + }, + }, + }, + { + input: map[string]string{ + MSEAnnotationsPrefix + "/" + whitelist: "1.1.1.1, 3.3.3.3", + MSEAnnotationsPrefix + "/" + domainBlacklist: "2.2.2.2", + }, + expect: &IPAccessControlConfig{ + Route: &IPAccessControl{ + isWhite: true, + remoteIp: []string{"1.1.1.1", "3.3.3.3"}, + }, + Domain: &IPAccessControl{ + isWhite: false, + remoteIp: []string{"2.2.2.2"}, + }, + }, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + config := &Ingress{} + _ = parser.Parse(testCase.input, config, nil) + if !reflect.DeepEqual(testCase.expect, config.IPAccessControl) { + t.Fatalf("Should be equal") + } + }) + } +} + +func TestIpAccessControl_ApplyVirtualServiceHandler(t *testing.T) { + parser := ipAccessControl{} + + testCases := []struct { + config *Ingress + input *networking.VirtualService + expect *networking.HTTPFilter + }{ + { + config: &Ingress{}, + input: &networking.VirtualService{}, + expect: nil, + }, + { + config: &Ingress{ + IPAccessControl: &IPAccessControlConfig{ + Domain: &IPAccessControl{ + isWhite: true, + remoteIp: []string{"1.1.1.1"}, + }, + }, + }, + input: &networking.VirtualService{}, + expect: &networking.HTTPFilter{ + Name: "ip-access-control", + Disable: false, + Filter: &networking.HTTPFilter_IpAccessControl{ + IpAccessControl: &networking.IPAccessControl{ + RemoteIpBlocks: []string{"1.1.1.1"}, + }, + }, + }, + }, + { + config: &Ingress{ + IPAccessControl: &IPAccessControlConfig{ + Domain: &IPAccessControl{ + isWhite: false, + remoteIp: []string{"2.2.2.2"}, + }, + }, + }, + input: &networking.VirtualService{}, + expect: &networking.HTTPFilter{ + Name: "ip-access-control", + Disable: false, + Filter: &networking.HTTPFilter_IpAccessControl{ + IpAccessControl: &networking.IPAccessControl{ + NotRemoteIpBlocks: []string{"2.2.2.2"}, + }, + }, + }, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + parser.ApplyVirtualServiceHandler(testCase.input, testCase.config) + if testCase.config.IPAccessControl == nil { + if len(testCase.input.HostHTTPFilters) != 0 { + t.Fatalf("Should be empty") + } + } else { + if len(testCase.input.HostHTTPFilters) == 0 { + t.Fatalf("Should be not empty") + } + if !reflect.DeepEqual(testCase.expect, testCase.input.HostHTTPFilters[0]) { + t.Fatalf("Should be equal") + } + } + }) + } +} + +func TestIpAccessControl_ApplyRoute(t *testing.T) { + parser := ipAccessControl{} + + testCases := []struct { + config *Ingress + input *networking.HTTPRoute + expect *networking.HTTPFilter + }{ + { + config: &Ingress{}, + input: &networking.HTTPRoute{}, + expect: nil, + }, + { + config: &Ingress{ + IPAccessControl: &IPAccessControlConfig{ + Route: &IPAccessControl{ + isWhite: true, + remoteIp: []string{"1.1.1.1"}, + }, + }, + }, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPFilter{ + Name: "ip-access-control", + Disable: false, + Filter: &networking.HTTPFilter_IpAccessControl{ + IpAccessControl: &networking.IPAccessControl{ + RemoteIpBlocks: []string{"1.1.1.1"}, + }, + }, + }, + }, + { + config: &Ingress{ + IPAccessControl: &IPAccessControlConfig{ + Route: &IPAccessControl{ + isWhite: false, + remoteIp: []string{"2.2.2.2"}, + }, + }, + }, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPFilter{ + Name: "ip-access-control", + Disable: false, + Filter: &networking.HTTPFilter_IpAccessControl{ + IpAccessControl: &networking.IPAccessControl{ + NotRemoteIpBlocks: []string{"2.2.2.2"}, + }, + }, + }, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + parser.ApplyRoute(testCase.input, testCase.config) + if testCase.config.IPAccessControl == nil { + if len(testCase.input.RouteHTTPFilters) != 0 { + t.Fatalf("Should be empty") + } + } else { + if len(testCase.input.RouteHTTPFilters) == 0 { + t.Fatalf("Should be not empty") + } + if !reflect.DeepEqual(testCase.expect, testCase.input.RouteHTTPFilters[0]) { + t.Fatalf("Should be equal") + } + } + }) + } +} diff --git a/ingress/kube/annotations/loadbalance.go b/ingress/kube/annotations/loadbalance.go new file mode 100644 index 000000000..0b9600b5a --- /dev/null +++ b/ingress/kube/annotations/loadbalance.go @@ -0,0 +1,212 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "strings" + + "github.com/gogo/protobuf/types" + networking "istio.io/api/networking/v1alpha3" +) + +const ( + loadBalanceAnnotation = "load-balance" + upstreamHashBy = "upstream-hash-by" + // affinity in nginx/mse ingress always be cookie + affinity = "affinity" + // affinityMode in mse ingress always be balanced + affinityMode = "affinity-mode" + // affinityCanaryBehavior in mse ingress always be legacy + affinityCanaryBehavior = "affinity-canary-behavior" + sessionCookieName = "session-cookie-name" + sessionCookiePath = "session-cookie-path" + sessionCookieMaxAge = "session-cookie-max-age" + sessionCookieExpires = "session-cookie-expires" + warmup = "warmup" + + varIndicator = "$" + headerIndicator = "$http_" + queryParamIndicator = "$arg_" + + defaultAffinityCookieName = "INGRESSCOOKIE" + defaultAffinityCookiePath = "/" +) + +var ( + _ Parser = loadBalance{} + _ TrafficPolicyHandler = loadBalance{} + + headersMapping = map[string]string{ + "$request_uri": ":path", + "$host": ":authority", + "$remote_addr": "x-envoy-external-address", + } +) + +type consistentHashByOther struct { + header string + queryParam string +} + +type consistentHashByCookie struct { + name string + path string + age *types.Duration +} + +type LoadBalanceConfig struct { + simple networking.LoadBalancerSettings_SimpleLB + warmup *types.Duration + other *consistentHashByOther + cookie *consistentHashByCookie +} + +type loadBalance struct{} + +func (l loadBalance) Parse(annotations Annotations, config *Ingress, _ *GlobalContext) error { + if !needLoadBalanceConfig(annotations) { + return nil + } + + loadBalanceConfig := &LoadBalanceConfig{ + simple: networking.LoadBalancerSettings_ROUND_ROBIN, + } + defer func() { + config.LoadBalance = loadBalanceConfig + }() + + if isCookieAffinity(annotations) { + loadBalanceConfig.cookie = &consistentHashByCookie{ + name: defaultAffinityCookieName, + path: defaultAffinityCookiePath, + age: &types.Duration{}, + } + if name, err := annotations.ParseStringASAP(sessionCookieName); err == nil { + loadBalanceConfig.cookie.name = name + } + if path, err := annotations.ParseStringASAP(sessionCookiePath); err == nil { + loadBalanceConfig.cookie.path = path + } + if age, err := annotations.ParseIntASAP(sessionCookieMaxAge); err == nil { + loadBalanceConfig.cookie.age = &types.Duration{ + Seconds: int64(age), + } + } else if age, err = annotations.ParseIntASAP(sessionCookieExpires); err == nil { + loadBalanceConfig.cookie.age = &types.Duration{ + Seconds: int64(age), + } + } + } else if isOtherAffinity(annotations) { + if key, err := annotations.ParseStringASAP(upstreamHashBy); err == nil && + strings.HasPrefix(key, varIndicator) { + value, exist := headersMapping[key] + if exist { + loadBalanceConfig.other = &consistentHashByOther{ + header: value, + } + } else { + if strings.HasPrefix(key, headerIndicator) { + loadBalanceConfig.other = &consistentHashByOther{ + header: strings.TrimPrefix(key, headerIndicator), + } + } else if strings.HasPrefix(key, queryParamIndicator) { + loadBalanceConfig.other = &consistentHashByOther{ + queryParam: strings.TrimPrefix(key, queryParamIndicator), + } + } + } + } + } else { + if lb, err := annotations.ParseStringASAP(loadBalanceAnnotation); err == nil { + lb = strings.ToUpper(lb) + loadBalanceConfig.simple = networking.LoadBalancerSettings_SimpleLB(networking.LoadBalancerSettings_SimpleLB_value[lb]) + } + + if warmup, err := annotations.ParseIntForMSE(warmup); err == nil && warmup != 0 { + loadBalanceConfig.warmup = &types.Duration{ + Seconds: int64(warmup), + } + } + } + + return nil +} + +func (l loadBalance) ApplyTrafficPolicy(trafficPolicy *networking.TrafficPolicy_PortTrafficPolicy, config *Ingress) { + loadBalanceConfig := config.LoadBalance + if loadBalanceConfig == nil { + return + } + + if loadBalanceConfig.cookie != nil { + trafficPolicy.LoadBalancer = &networking.LoadBalancerSettings{ + LbPolicy: &networking.LoadBalancerSettings_ConsistentHash{ + ConsistentHash: &networking.LoadBalancerSettings_ConsistentHashLB{ + HashKey: &networking.LoadBalancerSettings_ConsistentHashLB_HttpCookie{ + HttpCookie: &networking.LoadBalancerSettings_ConsistentHashLB_HTTPCookie{ + Name: loadBalanceConfig.cookie.name, + Path: loadBalanceConfig.cookie.path, + Ttl: loadBalanceConfig.cookie.age, + }, + }, + }, + }, + } + } else if loadBalanceConfig.other != nil { + var consistentHash *networking.LoadBalancerSettings_ConsistentHashLB + if loadBalanceConfig.other.header != "" { + consistentHash = &networking.LoadBalancerSettings_ConsistentHashLB{ + HashKey: &networking.LoadBalancerSettings_ConsistentHashLB_HttpHeaderName{ + HttpHeaderName: loadBalanceConfig.other.header, + }, + } + } else { + consistentHash = &networking.LoadBalancerSettings_ConsistentHashLB{ + HashKey: &networking.LoadBalancerSettings_ConsistentHashLB_HttpQueryParameterName{ + HttpQueryParameterName: loadBalanceConfig.other.queryParam, + }, + } + } + trafficPolicy.LoadBalancer = &networking.LoadBalancerSettings{ + LbPolicy: &networking.LoadBalancerSettings_ConsistentHash{ + ConsistentHash: consistentHash, + }, + } + } else { + trafficPolicy.LoadBalancer = &networking.LoadBalancerSettings{ + LbPolicy: &networking.LoadBalancerSettings_Simple{ + Simple: loadBalanceConfig.simple, + }, + } + trafficPolicy.LoadBalancer.WarmupDurationSecs = loadBalanceConfig.warmup + } +} + +func isCookieAffinity(annotations Annotations) bool { + return annotations.HasASAP(affinity) || + annotations.HasASAP(sessionCookieName) || + annotations.HasASAP(sessionCookiePath) +} + +func isOtherAffinity(annotations Annotations) bool { + return annotations.HasASAP(upstreamHashBy) +} + +func needLoadBalanceConfig(annotations Annotations) bool { + return annotations.HasASAP(loadBalanceAnnotation) || + annotations.HasMSE(warmup) || + isCookieAffinity(annotations) || + isOtherAffinity(annotations) +} diff --git a/ingress/kube/annotations/loadbalance_test.go b/ingress/kube/annotations/loadbalance_test.go new file mode 100644 index 000000000..c77f98fcd --- /dev/null +++ b/ingress/kube/annotations/loadbalance_test.go @@ -0,0 +1,294 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "reflect" + "testing" + + "github.com/gogo/protobuf/types" + networking "istio.io/api/networking/v1alpha3" +) + +func TestLoadBalanceParse(t *testing.T) { + loadBalance := loadBalance{} + inputCases := []struct { + input map[string]string + expect *LoadBalanceConfig + }{ + {}, + { + input: map[string]string{ + buildNginxAnnotationKey(affinity): "cookie", + buildNginxAnnotationKey(affinityMode): "balanced", + }, + expect: &LoadBalanceConfig{ + cookie: &consistentHashByCookie{ + name: defaultAffinityCookieName, + path: defaultAffinityCookiePath, + age: &types.Duration{}, + }, + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(affinity): "cookie", + buildNginxAnnotationKey(affinityMode): "balanced", + buildNginxAnnotationKey(sessionCookieName): "test", + buildNginxAnnotationKey(sessionCookiePath): "/test", + buildNginxAnnotationKey(sessionCookieMaxAge): "100", + }, + expect: &LoadBalanceConfig{ + cookie: &consistentHashByCookie{ + name: "test", + path: "/test", + age: &types.Duration{ + Seconds: 100, + }, + }, + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(affinity): "cookie", + buildNginxAnnotationKey(affinityMode): "balanced", + buildNginxAnnotationKey(sessionCookieName): "test", + buildNginxAnnotationKey(sessionCookieExpires): "10", + }, + expect: &LoadBalanceConfig{ + cookie: &consistentHashByCookie{ + name: "test", + path: defaultAffinityCookiePath, + age: &types.Duration{ + Seconds: 10, + }, + }, + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(upstreamHashBy): "$request_uri", + }, + expect: &LoadBalanceConfig{ + other: &consistentHashByOther{ + header: ":path", + }, + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(upstreamHashBy): "$host", + }, + expect: &LoadBalanceConfig{ + other: &consistentHashByOther{ + header: ":authority", + }, + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(upstreamHashBy): "$remote_addr", + }, + expect: &LoadBalanceConfig{ + other: &consistentHashByOther{ + header: "x-envoy-external-address", + }, + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(upstreamHashBy): "$http_test", + }, + expect: &LoadBalanceConfig{ + other: &consistentHashByOther{ + header: "test", + }, + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(upstreamHashBy): "$arg_query", + }, + expect: &LoadBalanceConfig{ + other: &consistentHashByOther{ + queryParam: "query", + }, + }, + }, + { + input: map[string]string{ + buildMSEAnnotationKey(warmup): "100", + }, + expect: &LoadBalanceConfig{ + simple: networking.LoadBalancerSettings_ROUND_ROBIN, + warmup: &types.Duration{ + Seconds: 100, + }, + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(loadBalanceAnnotation): "LEAST_CONN", + buildMSEAnnotationKey(warmup): "100", + }, + expect: &LoadBalanceConfig{ + simple: networking.LoadBalancerSettings_LEAST_CONN, + warmup: &types.Duration{ + Seconds: 100, + }, + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(loadBalanceAnnotation): "random", + buildMSEAnnotationKey(warmup): "100", + }, + expect: &LoadBalanceConfig{ + simple: networking.LoadBalancerSettings_RANDOM, + warmup: &types.Duration{ + Seconds: 100, + }, + }, + }, + } + + for _, inputCase := range inputCases { + t.Run("", func(t *testing.T) { + config := &Ingress{} + _ = loadBalance.Parse(inputCase.input, config, nil) + if !reflect.DeepEqual(inputCase.expect, config.LoadBalance) { + t.Fatal("Should be equal") + } + }) + } +} + +func TestLoadBalanceApplyTrafficPolicy(t *testing.T) { + loadBalance := loadBalance{} + inputCases := []struct { + config *Ingress + input *networking.TrafficPolicy_PortTrafficPolicy + expect *networking.TrafficPolicy_PortTrafficPolicy + }{ + { + config: &Ingress{}, + input: &networking.TrafficPolicy_PortTrafficPolicy{}, + expect: &networking.TrafficPolicy_PortTrafficPolicy{}, + }, + { + config: &Ingress{ + LoadBalance: &LoadBalanceConfig{ + cookie: &consistentHashByCookie{ + name: "test", + path: "/", + age: &types.Duration{ + Seconds: 100, + }, + }, + }, + }, + input: &networking.TrafficPolicy_PortTrafficPolicy{}, + expect: &networking.TrafficPolicy_PortTrafficPolicy{ + LoadBalancer: &networking.LoadBalancerSettings{ + LbPolicy: &networking.LoadBalancerSettings_ConsistentHash{ + ConsistentHash: &networking.LoadBalancerSettings_ConsistentHashLB{ + HashKey: &networking.LoadBalancerSettings_ConsistentHashLB_HttpCookie{ + HttpCookie: &networking.LoadBalancerSettings_ConsistentHashLB_HTTPCookie{ + Name: "test", + Path: "/", + Ttl: &types.Duration{ + Seconds: 100, + }, + }, + }, + }, + }, + }, + }, + }, + { + config: &Ingress{ + LoadBalance: &LoadBalanceConfig{ + other: &consistentHashByOther{ + header: ":authority", + }, + }, + }, + input: &networking.TrafficPolicy_PortTrafficPolicy{}, + expect: &networking.TrafficPolicy_PortTrafficPolicy{ + LoadBalancer: &networking.LoadBalancerSettings{ + LbPolicy: &networking.LoadBalancerSettings_ConsistentHash{ + ConsistentHash: &networking.LoadBalancerSettings_ConsistentHashLB{ + HashKey: &networking.LoadBalancerSettings_ConsistentHashLB_HttpHeaderName{ + HttpHeaderName: ":authority", + }, + }, + }, + }, + }, + }, + { + config: &Ingress{ + LoadBalance: &LoadBalanceConfig{ + other: &consistentHashByOther{ + queryParam: "query", + }, + }, + }, + input: &networking.TrafficPolicy_PortTrafficPolicy{}, + expect: &networking.TrafficPolicy_PortTrafficPolicy{ + LoadBalancer: &networking.LoadBalancerSettings{ + LbPolicy: &networking.LoadBalancerSettings_ConsistentHash{ + ConsistentHash: &networking.LoadBalancerSettings_ConsistentHashLB{ + HashKey: &networking.LoadBalancerSettings_ConsistentHashLB_HttpQueryParameterName{ + HttpQueryParameterName: "query", + }, + }, + }, + }, + }, + }, + { + config: &Ingress{ + LoadBalance: &LoadBalanceConfig{ + simple: networking.LoadBalancerSettings_ROUND_ROBIN, + warmup: &types.Duration{ + Seconds: 100, + }, + }, + }, + input: &networking.TrafficPolicy_PortTrafficPolicy{}, + expect: &networking.TrafficPolicy_PortTrafficPolicy{ + LoadBalancer: &networking.LoadBalancerSettings{ + LbPolicy: &networking.LoadBalancerSettings_Simple{ + Simple: networking.LoadBalancerSettings_ROUND_ROBIN, + }, + WarmupDurationSecs: &types.Duration{ + Seconds: 100, + }, + }, + }, + }, + } + + for _, inputCase := range inputCases { + t.Run("", func(t *testing.T) { + loadBalance.ApplyTrafficPolicy(inputCase.input, inputCase.config) + if !reflect.DeepEqual(inputCase.input, inputCase.expect) { + t.Fatal("Should be equal") + } + }) + } +} diff --git a/ingress/kube/annotations/local_rate_limit.go b/ingress/kube/annotations/local_rate_limit.go new file mode 100644 index 000000000..2069217ef --- /dev/null +++ b/ingress/kube/annotations/local_rate_limit.go @@ -0,0 +1,110 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "github.com/gogo/protobuf/types" + + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/networking/core/v1alpha3/mseingress" +) + +const ( + limitRPM = "route-limit-rpm" + limitRPS = "route-limit-rps" + limitBurstMultiplier = "route-limit-burst-multiplier" + + defaultBurstMultiplier = 5 + defaultStatusCode = 503 +) + +var ( + _ Parser = localRateLimit{} + _ RouteHandler = localRateLimit{} + + second = &types.Duration{ + Seconds: 1, + } + + minute = &types.Duration{ + Seconds: 60, + } +) + +type localRateLimitConfig struct { + TokensPerFill uint32 + MaxTokens uint32 + FillInterval *types.Duration +} + +type localRateLimit struct{} + +func (l localRateLimit) Parse(annotations Annotations, config *Ingress, _ *GlobalContext) error { + if !needLocalRateLimitConfig(annotations) { + return nil + } + + var local *localRateLimitConfig + defer func() { + config.localRateLimit = local + }() + + multiplier := defaultBurstMultiplier + if m, err := annotations.ParseIntForMSE(limitBurstMultiplier); err == nil { + multiplier = m + } + + if rpm, err := annotations.ParseIntForMSE(limitRPM); err == nil { + local = &localRateLimitConfig{ + MaxTokens: uint32(rpm * multiplier), + TokensPerFill: uint32(rpm), + FillInterval: minute, + } + } else if rps, err := annotations.ParseIntForMSE(limitRPS); err == nil { + local = &localRateLimitConfig{ + MaxTokens: uint32(rps * multiplier), + TokensPerFill: uint32(rps), + FillInterval: second, + } + } + + return nil +} + +func (l localRateLimit) ApplyRoute(route *networking.HTTPRoute, config *Ingress) { + localRateLimitConfig := config.localRateLimit + if localRateLimitConfig == nil { + return + } + + route.RouteHTTPFilters = append(route.RouteHTTPFilters, &networking.HTTPFilter{ + Name: mseingress.LocalRateLimit, + Filter: &networking.HTTPFilter_LocalRateLimit{ + LocalRateLimit: &networking.LocalRateLimit{ + TokenBucket: &networking.TokenBucket{ + MaxTokens: localRateLimitConfig.MaxTokens, + TokensPefFill: localRateLimitConfig.TokensPerFill, + FillInterval: localRateLimitConfig.FillInterval, + }, + StatusCode: defaultStatusCode, + }, + }, + }) +} + +func needLocalRateLimitConfig(annotations Annotations) bool { + return annotations.HasMSE(limitRPM) || + annotations.HasMSE(limitRPS) +} diff --git a/ingress/kube/annotations/local_rate_limit_test.go b/ingress/kube/annotations/local_rate_limit_test.go new file mode 100644 index 000000000..49e16b689 --- /dev/null +++ b/ingress/kube/annotations/local_rate_limit_test.go @@ -0,0 +1,127 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "reflect" + "testing" + + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/networking/core/v1alpha3/mseingress" +) + +func TestLocalRateLimitParse(t *testing.T) { + localRateLimit := localRateLimit{} + inputCases := []struct { + input map[string]string + expect *localRateLimitConfig + }{ + {}, + { + input: map[string]string{ + buildMSEAnnotationKey(limitRPM): "2", + }, + expect: &localRateLimitConfig{ + MaxTokens: 10, + TokensPerFill: 2, + FillInterval: minute, + }, + }, + { + input: map[string]string{ + buildMSEAnnotationKey(limitRPM): "2", + buildMSEAnnotationKey(limitRPS): "3", + buildMSEAnnotationKey(limitBurstMultiplier): "10", + }, + expect: &localRateLimitConfig{ + MaxTokens: 20, + TokensPerFill: 2, + FillInterval: minute, + }, + }, + { + input: map[string]string{ + buildMSEAnnotationKey(limitRPS): "3", + buildMSEAnnotationKey(limitBurstMultiplier): "10", + }, + expect: &localRateLimitConfig{ + MaxTokens: 30, + TokensPerFill: 3, + FillInterval: second, + }, + }, + } + + for _, inputCase := range inputCases { + t.Run("", func(t *testing.T) { + config := &Ingress{} + _ = localRateLimit.Parse(inputCase.input, config, nil) + if !reflect.DeepEqual(inputCase.expect, config.localRateLimit) { + t.Fatal("Should be equal") + } + }) + } +} + +func TestLocalRateLimitApplyRoute(t *testing.T) { + localRateLimit := localRateLimit{} + inputCases := []struct { + config *Ingress + input *networking.HTTPRoute + expect *networking.HTTPRoute + }{ + { + config: &Ingress{}, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{}, + }, + { + config: &Ingress{ + localRateLimit: &localRateLimitConfig{ + MaxTokens: 60, + TokensPerFill: 20, + FillInterval: second, + }, + }, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{ + RouteHTTPFilters: []*networking.HTTPFilter{ + { + Name: mseingress.LocalRateLimit, + Filter: &networking.HTTPFilter_LocalRateLimit{ + LocalRateLimit: &networking.LocalRateLimit{ + TokenBucket: &networking.TokenBucket{ + MaxTokens: 60, + TokensPefFill: 20, + FillInterval: second, + }, + StatusCode: defaultStatusCode, + }, + }, + }, + }, + }, + }, + } + + for _, inputCase := range inputCases { + t.Run("", func(t *testing.T) { + localRateLimit.ApplyRoute(inputCase.input, inputCase.config) + if !reflect.DeepEqual(inputCase.input, inputCase.expect) { + t.Fatal("Should be equal") + } + }) + } +} diff --git a/ingress/kube/annotations/parser.go b/ingress/kube/annotations/parser.go new file mode 100644 index 000000000..b29678b6c --- /dev/null +++ b/ingress/kube/annotations/parser.go @@ -0,0 +1,216 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "errors" + "strconv" + "strings" +) + +const ( + // DefaultAnnotationsPrefix defines the common prefix used in the nginx ingress controller + DefaultAnnotationsPrefix = "nginx.ingress.kubernetes.io" + + // MSEAnnotationsPrefix defines the common prefix used in the mse ingress controller + MSEAnnotationsPrefix = "mse.ingress.kubernetes.io" +) + +var ( + // ErrMissingAnnotations the ingress rule does not contain annotations + // This is an error only when annotations are being parsed + ErrMissingAnnotations = errors.New("ingress rule without annotations") + + // ErrInvalidAnnotationName the ingress rule does contain an invalid + // annotation name + ErrInvalidAnnotationName = errors.New("invalid annotation name") + + // ErrInvalidAnnotationValue the ingress rule does contain an invalid + // annotation value + ErrInvalidAnnotationValue = errors.New("invalid annotation value") +) + +// IsMissingAnnotations checks if the error is an error which +// indicates the ingress does not contain annotations +func IsMissingAnnotations(e error) bool { + return e == ErrMissingAnnotations +} + +type Annotations map[string]string + +func (a Annotations) ParseBool(key string) (bool, error) { + if len(a) == 0 { + return false, ErrMissingAnnotations + } + + val, ok := a[buildNginxAnnotationKey(key)] + if ok { + b, err := strconv.ParseBool(val) + if err != nil { + return false, ErrInvalidAnnotationValue + } + return b, nil + } + + return false, ErrMissingAnnotations +} + +func (a Annotations) ParseBoolForMSE(key string) (bool, error) { + if len(a) == 0 { + return false, ErrMissingAnnotations + } + + val, ok := a[buildMSEAnnotationKey(key)] + if ok { + b, err := strconv.ParseBool(val) + if err != nil { + return false, ErrInvalidAnnotationValue + } + return b, nil + } + + return false, ErrMissingAnnotations +} + +func (a Annotations) ParseBoolASAP(key string) (bool, error) { + if result, err := a.ParseBool(key); err == nil { + return result, nil + } + return a.ParseBoolForMSE(key) +} + +func (a Annotations) ParseString(key string) (string, error) { + if len(a) == 0 { + return "", ErrMissingAnnotations + } + + val, ok := a[buildNginxAnnotationKey(key)] + if ok { + s := normalizeString(val) + if s == "" { + return "", ErrInvalidAnnotationValue + } + return s, nil + } + + return "", ErrMissingAnnotations +} + +func (a Annotations) ParseStringForMSE(key string) (string, error) { + if len(a) == 0 { + return "", ErrMissingAnnotations + } + + val, ok := a[buildMSEAnnotationKey(key)] + if ok { + s := normalizeString(val) + if s == "" { + return "", ErrInvalidAnnotationValue + } + return s, nil + } + + return "", ErrMissingAnnotations +} + +// ParseStringASAP will first extra config from nginx annotation, then will +// try to extra config from mse annotation if the first step fails. +func (a Annotations) ParseStringASAP(key string) (string, error) { + if result, err := a.ParseString(key); err == nil { + return result, nil + } + return a.ParseStringForMSE(key) +} + +func (a Annotations) ParseInt(key string) (int, error) { + if len(a) == 0 { + return 0, ErrMissingAnnotations + } + + val, ok := a[buildNginxAnnotationKey(key)] + if ok { + i, err := strconv.Atoi(val) + if err != nil { + return 0, ErrInvalidAnnotationValue + } + return i, nil + } + return 0, ErrMissingAnnotations +} + +func (a Annotations) ParseIntForMSE(key string) (int, error) { + if len(a) == 0 { + return 0, ErrMissingAnnotations + } + + val, ok := a[buildMSEAnnotationKey(key)] + if ok { + i, err := strconv.Atoi(val) + if err != nil { + return 0, ErrInvalidAnnotationValue + } + return i, nil + } + return 0, ErrMissingAnnotations +} + +func (a Annotations) ParseIntASAP(key string) (int, error) { + if result, err := a.ParseInt(key); err == nil { + return result, nil + } + return a.ParseIntForMSE(key) +} + +func (a Annotations) Has(key string) bool { + if len(a) == 0 { + return false + } + + _, exist := a[buildNginxAnnotationKey(key)] + return exist +} + +func (a Annotations) HasMSE(key string) bool { + if len(a) == 0 { + return false + } + + _, exist := a[buildMSEAnnotationKey(key)] + return exist +} + +func (a Annotations) HasASAP(key string) bool { + if a.Has(key) { + return true + } + return a.HasMSE(key) +} + +func buildNginxAnnotationKey(key string) string { + return DefaultAnnotationsPrefix + "/" + key +} + +func buildMSEAnnotationKey(key string) string { + return MSEAnnotationsPrefix + "/" + key +} + +func normalizeString(input string) string { + var trimmedContent []string + for _, line := range strings.Split(input, "\n") { + trimmedContent = append(trimmedContent, strings.TrimSpace(line)) + } + + return strings.Join(trimmedContent, "\n") +} diff --git a/ingress/kube/annotations/redirect.go b/ingress/kube/annotations/redirect.go new file mode 100644 index 000000000..13215903b --- /dev/null +++ b/ingress/kube/annotations/redirect.go @@ -0,0 +1,152 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "fmt" + "net/http" + "net/url" + "strings" + + networking "istio.io/api/networking/v1alpha3" +) + +const ( + appRoot = "app-root" + temporalRedirect = "temporal-redirect" + permanentRedirect = "permanent-redirect" + permanentRedirectCode = "permanent-redirect-code" + sslRedirect = "ssl-redirect" + forceSSLRedirect = "force-ssl-redirect" + + defaultPermanentRedirectCode = 301 + defaultTemporalRedirectCode = 302 +) + +var ( + _ Parser = &redirect{} + _ RouteHandler = &redirect{} +) + +type RedirectConfig struct { + AppRoot string + + URL string + + Code int + + httpsRedirect bool +} + +type redirect struct{} + +func (r redirect) Parse(annotations Annotations, config *Ingress, _ *GlobalContext) error { + if !needRedirectConfig(annotations) { + return nil + } + + redirectConfig := &RedirectConfig{ + Code: defaultPermanentRedirectCode, + } + config.Redirect = redirectConfig + + redirectConfig.AppRoot, _ = annotations.ParseStringASAP(appRoot) + + httpsRedirect, _ := annotations.ParseBoolASAP(sslRedirect) + forceHTTPSRedirect, _ := annotations.ParseBoolASAP(forceSSLRedirect) + if httpsRedirect || forceHTTPSRedirect { + redirectConfig.httpsRedirect = true + } + + // temporal redirect is firstly applied. + tr, err := annotations.ParseStringASAP(temporalRedirect) + if err != nil && !IsMissingAnnotations(err) { + return nil + } + if tr != "" && isValidURL(tr) == nil { + redirectConfig.URL = tr + redirectConfig.Code = defaultTemporalRedirectCode + return nil + } + + // permanent redirect + // url + pr, err := annotations.ParseStringASAP(permanentRedirect) + if err != nil && !IsMissingAnnotations(err) { + return nil + } + if pr != "" && isValidURL(pr) == nil { + redirectConfig.URL = pr + } + // code + if prc, err := annotations.ParseIntASAP(permanentRedirectCode); err == nil { + if prc < http.StatusMultipleChoices || prc > http.StatusPermanentRedirect { + prc = defaultPermanentRedirectCode + } + redirectConfig.Code = prc + } + + return nil +} + +func (r redirect) ApplyRoute(route *networking.HTTPRoute, config *Ingress) { + redirectConfig := config.Redirect + if redirectConfig == nil { + return + } + + var redirectPolicy *networking.HTTPRedirect + if redirectConfig.URL != "" { + parseURL, err := url.Parse(redirectConfig.URL) + if err != nil { + return + } + redirectPolicy = &networking.HTTPRedirect{ + Scheme: parseURL.Scheme, + Authority: parseURL.Host, + Uri: parseURL.Path, + RedirectCode: uint32(redirectConfig.Code), + } + } else if redirectConfig.httpsRedirect { + redirectPolicy = &networking.HTTPRedirect{ + Scheme: "https", + // 308 is the default code for ssl redirect + RedirectCode: 308, + } + } + + route.Redirect = redirectPolicy +} + +func needRedirectConfig(annotations Annotations) bool { + return annotations.HasASAP(temporalRedirect) || + annotations.HasASAP(permanentRedirect) || + annotations.HasASAP(sslRedirect) || + annotations.HasASAP(forceSSLRedirect) || + annotations.HasASAP(appRoot) +} + +func isValidURL(s string) error { + u, err := url.Parse(s) + if err != nil { + return err + } + + if !strings.HasPrefix(u.Scheme, "http") { + return fmt.Errorf("only http and https are valid protocols (%v)", u.Scheme) + } + + return nil +} diff --git a/ingress/kube/annotations/retry.go b/ingress/kube/annotations/retry.go new file mode 100644 index 000000000..bff194f35 --- /dev/null +++ b/ingress/kube/annotations/retry.go @@ -0,0 +1,134 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "strings" + + "github.com/gogo/protobuf/types" + + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/util/sets" +) + +const ( + retryCount = "proxy-next-upstream-tries" + perRetryTimeout = "proxy-next-upstream-timeout" + retryOn = "proxy-next-upstream" + + defaultRetryCount = 3 + defaultRetryOn = "5xx" + retryStatusCode = "retriable-status-codes" +) + +var ( + _ Parser = retry{} + _ RouteHandler = retry{} +) + +type RetryConfig struct { + retryCount int32 + perRetryTimeout *types.Duration + retryOn string +} + +type retry struct{} + +func (r retry) Parse(annotations Annotations, config *Ingress, _ *GlobalContext) error { + if !needRetryConfig(annotations) { + return nil + } + + retryConfig := &RetryConfig{ + retryCount: defaultRetryCount, + perRetryTimeout: &types.Duration{}, + retryOn: defaultRetryOn, + } + defer func() { + config.Retry = retryConfig + }() + + if count, err := annotations.ParseIntASAP(retryCount); err == nil { + retryConfig.retryCount = int32(count) + } + + if timeout, err := annotations.ParseIntASAP(perRetryTimeout); err == nil { + retryConfig.perRetryTimeout = &types.Duration{ + Seconds: int64(timeout), + } + } + + if retryOn, err := annotations.ParseStringASAP(retryOn); err == nil { + conditions := toSet(splitBySeparator(retryOn, ",")) + if len(conditions) > 0 { + if conditions.Contains("off") { + retryConfig.retryCount = 0 + } else { + var stringBuilder strings.Builder + // Convert error, timeout, invalid_header to 5xx + if conditions.Contains("error") || + conditions.Contains("timeout") || + conditions.Contains("invalid_header") { + stringBuilder.WriteString(defaultRetryOn + ",") + } + // Just use the raw. + if conditions.Contains("non_idempotent") { + stringBuilder.WriteString("non_idempotent,") + } + // Append the status codes. + statusCodes := convertStatusCodes(conditions) + if len(statusCodes) > 0 { + stringBuilder.WriteString(retryStatusCode + ",") + for _, code := range statusCodes { + stringBuilder.WriteString(code + ",") + } + } + + retryConfig.retryOn = strings.TrimSuffix(stringBuilder.String(), ",") + } + } + } + + return nil +} + +func (r retry) ApplyRoute(route *networking.HTTPRoute, config *Ingress) { + retryConfig := config.Retry + if retryConfig == nil { + return + } + + route.Retries = &networking.HTTPRetry{ + Attempts: retryConfig.retryCount, + PerTryTimeout: retryConfig.perRetryTimeout, + RetryOn: retryConfig.retryOn, + } +} + +func needRetryConfig(annotations Annotations) bool { + return annotations.HasASAP(retryCount) || + annotations.HasASAP(perRetryTimeout) || + annotations.HasASAP(retryOn) +} + +func convertStatusCodes(set sets.Set) []string { + var result []string + for condition := range set { + if strings.HasPrefix(condition, "http_") { + result = append(result, strings.TrimPrefix(condition, "http_")) + } + } + return result +} diff --git a/ingress/kube/annotations/retry_test.go b/ingress/kube/annotations/retry_test.go new file mode 100644 index 000000000..0552a0768 --- /dev/null +++ b/ingress/kube/annotations/retry_test.go @@ -0,0 +1,147 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "reflect" + "testing" + + "github.com/gogo/protobuf/types" + + networking "istio.io/api/networking/v1alpha3" +) + +func TestRetryParse(t *testing.T) { + retry := retry{} + inputCases := []struct { + input map[string]string + expect *RetryConfig + }{ + {}, + { + input: map[string]string{ + buildNginxAnnotationKey(retryCount): "1", + }, + expect: &RetryConfig{ + retryCount: 1, + perRetryTimeout: &types.Duration{}, + retryOn: "5xx", + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(perRetryTimeout): "10", + }, + expect: &RetryConfig{ + retryCount: 3, + perRetryTimeout: &types.Duration{ + Seconds: 10, + }, + retryOn: "5xx", + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(retryCount): "2", + buildNginxAnnotationKey(retryOn): "off", + }, + expect: &RetryConfig{ + retryCount: 0, + perRetryTimeout: &types.Duration{}, + retryOn: "5xx", + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(retryCount): "2", + buildNginxAnnotationKey(retryOn): "error,timeout", + }, + expect: &RetryConfig{ + retryCount: 2, + perRetryTimeout: &types.Duration{}, + retryOn: "5xx", + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(retryOn): "timeout,non_idempotent", + }, + expect: &RetryConfig{ + retryCount: 3, + perRetryTimeout: &types.Duration{}, + retryOn: "5xx,non_idempotent", + }, + }, + { + input: map[string]string{ + buildNginxAnnotationKey(retryOn): "timeout,http_503,http_502,http_404", + }, + expect: &RetryConfig{ + retryCount: 3, + perRetryTimeout: &types.Duration{}, + retryOn: "5xx,retriable-status-codes,503,502,404", + }, + }, + } + + for _, inputCase := range inputCases { + t.Run("", func(t *testing.T) { + config := &Ingress{} + _ = retry.Parse(inputCase.input, config, nil) + if !reflect.DeepEqual(inputCase.expect, config.Retry) { + t.Fatalf("Should be equal") + } + }) + } +} + +func TestRetryApplyRoute(t *testing.T) { + retry := retry{} + inputCases := []struct { + config *Ingress + input *networking.HTTPRoute + expect *networking.HTTPRoute + }{ + { + config: &Ingress{}, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{}, + }, + { + config: &Ingress{ + Retry: &RetryConfig{ + retryCount: 3, + retryOn: "test", + }, + }, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{ + Retries: &networking.HTTPRetry{ + Attempts: 3, + RetryOn: "test", + }, + }, + }, + } + + for _, inputCase := range inputCases { + t.Run("", func(t *testing.T) { + retry.ApplyRoute(inputCase.input, inputCase.config) + if !reflect.DeepEqual(inputCase.input, inputCase.expect) { + t.Fatalf("Should be equal") + } + }) + } +} diff --git a/ingress/kube/annotations/rewrite.go b/ingress/kube/annotations/rewrite.go new file mode 100644 index 000000000..8ddeedc72 --- /dev/null +++ b/ingress/kube/annotations/rewrite.go @@ -0,0 +1,106 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "regexp" + "strings" + + networking "istio.io/api/networking/v1alpha3" +) + +const ( + rewriteTarget = "rewrite-target" + useRegex = "use-regex" + upstreamVhost = "upstream-vhost" + + re2Regex = "\\$[0-9]" +) + +var ( + _ Parser = &rewrite{} + _ RouteHandler = &rewrite{} +) + +type RewriteConfig struct { + RewriteTarget string + UseRegex bool + RewriteHost string +} + +type rewrite struct{} + +func (r rewrite) Parse(annotations Annotations, config *Ingress, _ *GlobalContext) error { + if !needRewriteConfig(annotations) { + return nil + } + + rewriteConfig := &RewriteConfig{} + rewriteConfig.RewriteTarget, _ = annotations.ParseStringASAP(rewriteTarget) + rewriteConfig.UseRegex, _ = annotations.ParseBoolASAP(useRegex) + rewriteConfig.RewriteHost, _ = annotations.ParseStringASAP(upstreamVhost) + + if rewriteConfig.RewriteTarget != "" { + // When rewrite target is present and not empty, + // we will enforce regex match on all rules in this ingress. + rewriteConfig.UseRegex = true + + // We should convert nginx regex rule to envoy regex rule. + rewriteConfig.RewriteTarget = convertToRE2(rewriteConfig.RewriteTarget) + } + + config.Rewrite = rewriteConfig + return nil +} + +func (r rewrite) ApplyRoute(route *networking.HTTPRoute, config *Ingress) { + rewriteConfig := config.Rewrite + if rewriteConfig == nil || (rewriteConfig.RewriteTarget == "" && + rewriteConfig.RewriteHost == "") { + return + } + + route.Rewrite = &networking.HTTPRewrite{} + if rewriteConfig.RewriteTarget != "" { + route.Rewrite.UriRegex = &networking.RegexMatchAndSubstitute{ + Pattern: route.Match[0].Uri.GetRegex(), + Substitution: rewriteConfig.RewriteTarget, + } + } + + if rewriteConfig.RewriteHost != "" { + route.Rewrite.Authority = rewriteConfig.RewriteHost + } +} + +func convertToRE2(target string) string { + if match, err := regexp.MatchString(re2Regex, target); err != nil || !match { + return target + } + + return strings.ReplaceAll(target, "$", "\\") +} + +func NeedRegexMatch(annotations map[string]string) bool { + target, _ := Annotations(annotations).ParseStringASAP(rewriteTarget) + regex, _ := Annotations(annotations).ParseBoolASAP(useRegex) + + return regex || target != "" +} + +func needRewriteConfig(annotations Annotations) bool { + return annotations.HasASAP(rewriteTarget) || annotations.HasASAP(useRegex) || + annotations.HasASAP(upstreamVhost) +} diff --git a/ingress/kube/annotations/rewrite_test.go b/ingress/kube/annotations/rewrite_test.go new file mode 100644 index 000000000..6be98a93c --- /dev/null +++ b/ingress/kube/annotations/rewrite_test.go @@ -0,0 +1,254 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "reflect" + "testing" + + networking "istio.io/api/networking/v1alpha3" +) + +func TestConvertToRE2(t *testing.T) { + useCases := []struct { + input string + except string + }{ + { + input: "/test", + except: "/test", + }, + { + input: "/test/app", + except: "/test/app", + }, + { + input: "/$1", + except: "/\\1", + }, + { + input: "/$2/$1", + except: "/\\2/\\1", + }, + { + input: "/$test/$a", + except: "/$test/$a", + }, + } + + for _, c := range useCases { + t.Run("", func(t *testing.T) { + if convertToRE2(c.input) != c.except { + t.Fatalf("input %s is not equal to except %s.", c.input, c.except) + } + }) + } +} + +func TestRewriteParse(t *testing.T) { + rewrite := rewrite{} + testCases := []struct { + input Annotations + expect *RewriteConfig + }{ + { + input: nil, + expect: nil, + }, + { + input: Annotations{}, + expect: nil, + }, + { + input: Annotations{ + buildNginxAnnotationKey(rewriteTarget): "/test", + }, + expect: &RewriteConfig{ + RewriteTarget: "/test", + UseRegex: true, + }, + }, + { + input: Annotations{ + buildNginxAnnotationKey(rewriteTarget): "", + }, + expect: &RewriteConfig{ + UseRegex: false, + }, + }, + { + input: Annotations{ + buildNginxAnnotationKey(rewriteTarget): "/$2/$1", + }, + expect: &RewriteConfig{ + RewriteTarget: "/\\2/\\1", + UseRegex: true, + }, + }, + { + input: Annotations{ + buildNginxAnnotationKey(useRegex): "true", + }, + expect: &RewriteConfig{ + UseRegex: true, + }, + }, + { + input: Annotations{ + buildNginxAnnotationKey(upstreamVhost): "test.com", + }, + expect: &RewriteConfig{ + RewriteHost: "test.com", + }, + }, + { + input: Annotations{ + buildNginxAnnotationKey(rewriteTarget): "/$2/$1", + buildNginxAnnotationKey(upstreamVhost): "test.com", + }, + expect: &RewriteConfig{ + RewriteTarget: "/\\2/\\1", + UseRegex: true, + RewriteHost: "test.com", + }, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + config := &Ingress{} + _ = rewrite.Parse(testCase.input, config, nil) + if !reflect.DeepEqual(config.Rewrite, testCase.expect) { + t.Fatalf("Must be equal.") + } + }) + } +} + +func TestRewriteApplyRoute(t *testing.T) { + rewrite := rewrite{} + inputCases := []struct { + config *Ingress + input *networking.HTTPRoute + expect *networking.HTTPRoute + }{ + { + config: &Ingress{}, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{}, + }, + { + config: &Ingress{ + Rewrite: &RewriteConfig{}, + }, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{}, + }, + { + config: &Ingress{ + Rewrite: &RewriteConfig{ + RewriteTarget: "/test", + }, + }, + input: &networking.HTTPRoute{ + Match: []*networking.HTTPMatchRequest{ + { + Uri: &networking.StringMatch{ + MatchType: &networking.StringMatch_Regex{ + Regex: "/hello", + }, + }, + }, + }, + }, + expect: &networking.HTTPRoute{ + Match: []*networking.HTTPMatchRequest{ + { + Uri: &networking.StringMatch{ + MatchType: &networking.StringMatch_Regex{ + Regex: "/hello", + }, + }, + }, + }, + Rewrite: &networking.HTTPRewrite{ + UriRegex: &networking.RegexMatchAndSubstitute{ + Pattern: "/hello", + Substitution: "/test", + }, + }, + }, + }, + { + config: &Ingress{ + Rewrite: &RewriteConfig{ + RewriteHost: "test.com", + }, + }, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{ + Rewrite: &networking.HTTPRewrite{ + Authority: "test.com", + }, + }, + }, + { + config: &Ingress{ + Rewrite: &RewriteConfig{ + RewriteTarget: "/test", + RewriteHost: "test.com", + }, + }, + input: &networking.HTTPRoute{ + Match: []*networking.HTTPMatchRequest{ + { + Uri: &networking.StringMatch{ + MatchType: &networking.StringMatch_Regex{ + Regex: "/hello", + }, + }, + }, + }, + }, + expect: &networking.HTTPRoute{ + Match: []*networking.HTTPMatchRequest{ + { + Uri: &networking.StringMatch{ + MatchType: &networking.StringMatch_Regex{ + Regex: "/hello", + }, + }, + }, + }, + Rewrite: &networking.HTTPRewrite{ + UriRegex: &networking.RegexMatchAndSubstitute{ + Pattern: "/hello", + Substitution: "/test", + }, + Authority: "test.com", + }, + }, + }, + } + + for _, inputCase := range inputCases { + t.Run("", func(t *testing.T) { + rewrite.ApplyRoute(inputCase.input, inputCase.config) + if !reflect.DeepEqual(inputCase.input, inputCase.expect) { + t.Fatal("Should be equal") + } + }) + } +} diff --git a/ingress/kube/annotations/timeout.go b/ingress/kube/annotations/timeout.go new file mode 100644 index 000000000..cd65a4a73 --- /dev/null +++ b/ingress/kube/annotations/timeout.go @@ -0,0 +1,62 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "github.com/gogo/protobuf/types" + + networking "istio.io/api/networking/v1alpha3" +) + +const timeoutAnnotation = "timeout" + +var ( + _ Parser = timeout{} + _ RouteHandler = timeout{} +) + +type TimeoutConfig struct { + time *types.Duration +} + +type timeout struct{} + +func (t timeout) Parse(annotations Annotations, config *Ingress, _ *GlobalContext) error { + if !needTimeoutConfig(annotations) { + return nil + } + + if time, err := annotations.ParseIntForMSE(timeoutAnnotation); err == nil { + config.Timeout = &TimeoutConfig{ + time: &types.Duration{ + Seconds: int64(time), + }, + } + } + return nil +} + +func (t timeout) ApplyRoute(route *networking.HTTPRoute, config *Ingress) { + timeout := config.Timeout + if timeout == nil || timeout.time == nil || timeout.time.Seconds == 0 { + return + } + + route.Timeout = timeout.time +} + +func needTimeoutConfig(annotations Annotations) bool { + return annotations.HasMSE(timeoutAnnotation) +} diff --git a/ingress/kube/annotations/timeout_test.go b/ingress/kube/annotations/timeout_test.go new file mode 100644 index 000000000..fd589d63d --- /dev/null +++ b/ingress/kube/annotations/timeout_test.go @@ -0,0 +1,121 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "reflect" + "testing" + + "github.com/gogo/protobuf/types" + networking "istio.io/api/networking/v1alpha3" +) + +func TestTimeoutParse(t *testing.T) { + timeout := timeout{} + inputCases := []struct { + input map[string]string + expect *TimeoutConfig + }{ + {}, + { + input: map[string]string{ + MSEAnnotationsPrefix + "/" + timeoutAnnotation: "", + }, + }, + { + input: map[string]string{ + MSEAnnotationsPrefix + "/" + timeoutAnnotation: "0", + }, + expect: &TimeoutConfig{ + time: &types.Duration{}, + }, + }, + { + input: map[string]string{ + MSEAnnotationsPrefix + "/" + timeoutAnnotation: "10", + }, + expect: &TimeoutConfig{ + time: &types.Duration{ + Seconds: 10, + }, + }, + }, + } + + for _, c := range inputCases { + t.Run("", func(t *testing.T) { + config := &Ingress{} + _ = timeout.Parse(c.input, config, nil) + if !reflect.DeepEqual(c.expect, config.Timeout) { + t.Fatalf("Should be equal.") + } + }) + } +} + +func TestTimeoutApplyRoute(t *testing.T) { + timeout := timeout{} + inputCases := []struct { + config *Ingress + input *networking.HTTPRoute + expect *networking.HTTPRoute + }{ + { + config: &Ingress{}, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{}, + }, + { + config: &Ingress{ + Timeout: &TimeoutConfig{}, + }, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{}, + }, + { + config: &Ingress{ + Timeout: &TimeoutConfig{ + time: &types.Duration{}, + }, + }, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{}, + }, + { + config: &Ingress{ + Timeout: &TimeoutConfig{ + time: &types.Duration{ + Seconds: 10, + }, + }, + }, + input: &networking.HTTPRoute{}, + expect: &networking.HTTPRoute{ + Timeout: &types.Duration{ + Seconds: 10, + }, + }, + }, + } + + for _, inputCase := range inputCases { + t.Run("", func(t *testing.T) { + timeout.ApplyRoute(inputCase.input, inputCase.config) + if !reflect.DeepEqual(inputCase.input, inputCase.expect) { + t.Fatalf("Should be equal") + } + }) + } +} diff --git a/ingress/kube/annotations/upstreamtls.go b/ingress/kube/annotations/upstreamtls.go new file mode 100644 index 000000000..98941c508 --- /dev/null +++ b/ingress/kube/annotations/upstreamtls.go @@ -0,0 +1,188 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "regexp" + "strings" + + "github.com/gogo/protobuf/types" + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/model/credentials" + + "github.com/alibaba/higress/ingress/kube/util" +) + +const ( + backendProtocol = "backend-protocol" + proxySSLSecret = "proxy-ssl-secret" + proxySSLVerify = "proxy-ssl-verify" + proxySSLName = "proxy-ssl-name" + proxySSLServerName = "proxy-ssl-server-name" + + defaultBackendProtocol = "HTTP" +) + +var ( + _ Parser = &upstreamTLS{} + _ TrafficPolicyHandler = &upstreamTLS{} + + validProtocols = regexp.MustCompile(`^(HTTP|HTTP2|HTTPS|GRPC|GRPCS)$`) + + OnOffRegex = regexp.MustCompile(`^(on|off)$`) +) + +type UpstreamTLSConfig struct { + BackendProtocol string + + SecretName string + SSLVerify bool + SNI string + EnableSNI bool +} + +type upstreamTLS struct{} + +func (u upstreamTLS) Parse(annotations Annotations, config *Ingress, _ *GlobalContext) error { + if !needUpstreamTLSConfig(annotations) { + return nil + } + + upstreamTLSConfig := &UpstreamTLSConfig{ + BackendProtocol: defaultBackendProtocol, + } + + defer func() { + config.UpstreamTLS = upstreamTLSConfig + }() + + if proto, err := annotations.ParseStringASAP(backendProtocol); err == nil { + proto = strings.TrimSpace(strings.ToUpper(proto)) + if validProtocols.MatchString(proto) { + upstreamTLSConfig.BackendProtocol = proto + } + } + + secretName, _ := annotations.ParseStringASAP(proxySSLSecret) + namespacedName := util.SplitNamespacedName(secretName) + if namespacedName.Name == "" { + return nil + } + + if namespacedName.Namespace == "" { + namespacedName.Namespace = config.Namespace + } + upstreamTLSConfig.SecretName = namespacedName.String() + + if sslVerify, err := annotations.ParseStringASAP(proxySSLVerify); err == nil { + if OnOffRegex.MatchString(sslVerify) { + upstreamTLSConfig.SSLVerify = onOffToBool(sslVerify) + } + } + + upstreamTLSConfig.SNI, _ = annotations.ParseStringASAP(proxySSLName) + + if enableSNI, err := annotations.ParseStringASAP(proxySSLServerName); err == nil { + if OnOffRegex.MatchString(enableSNI) { + upstreamTLSConfig.SSLVerify = onOffToBool(enableSNI) + } + } + + return nil +} + +func (u upstreamTLS) ApplyTrafficPolicy(trafficPolicy *networking.TrafficPolicy_PortTrafficPolicy, config *Ingress) { + if config.UpstreamTLS == nil { + return + } + + upstreamTLSConfig := config.UpstreamTLS + + if isH2(upstreamTLSConfig.BackendProtocol) { + trafficPolicy.ConnectionPool = &networking.ConnectionPoolSettings{ + Http: &networking.ConnectionPoolSettings_HTTPSettings{ + H2UpgradePolicy: networking.ConnectionPoolSettings_HTTPSettings_UPGRADE, + }, + } + } + + var tls *networking.ClientTLSSettings + if upstreamTLSConfig.SecretName != "" { + // MTLS + tls = processMTLS(config) + } else if isHTTPS(upstreamTLSConfig.BackendProtocol) { + tls = processSimple(config) + } + + trafficPolicy.Tls = tls +} + +func processMTLS(config *Ingress) *networking.ClientTLSSettings { + namespacedName := util.SplitNamespacedName(config.UpstreamTLS.SecretName) + if namespacedName.Name == "" { + return nil + } + + tls := &networking.ClientTLSSettings{ + Mode: networking.ClientTLSSettings_MUTUAL, + CredentialName: credentials.ToKubernetesIngressResource(config.RawClusterId, namespacedName.Namespace, namespacedName.Name), + } + + if !config.UpstreamTLS.SSLVerify { + // This api InsecureSkipVerify hasn't been support yet. + // Until this pr https://github.com/istio/istio/pull/35357. + tls.InsecureSkipVerify = &types.BoolValue{ + Value: false, + } + } + + if config.UpstreamTLS.EnableSNI && config.UpstreamTLS.SNI != "" { + tls.Sni = config.UpstreamTLS.SNI + } + + return tls +} + +func processSimple(config *Ingress) *networking.ClientTLSSettings { + tls := &networking.ClientTLSSettings{ + Mode: networking.ClientTLSSettings_SIMPLE, + } + + if config.UpstreamTLS.EnableSNI && config.UpstreamTLS.SNI != "" { + tls.Sni = config.UpstreamTLS.SNI + } + + return tls +} + +func needUpstreamTLSConfig(annotations Annotations) bool { + return annotations.HasASAP(backendProtocol) || + annotations.HasASAP(proxySSLSecret) +} + +func onOffToBool(onOff string) bool { + return onOff == "on" +} + +func isH2(protocol string) bool { + return protocol == "HTTP2" || + protocol == "GRPC" || + protocol == "GRPCS" +} + +func isHTTPS(protocol string) bool { + return protocol == "HTTPS" || + protocol == "GRPCS" +} diff --git a/ingress/kube/annotations/util.go b/ingress/kube/annotations/util.go new file mode 100644 index 000000000..10b36a52a --- /dev/null +++ b/ingress/kube/annotations/util.go @@ -0,0 +1,54 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "strings" + + "istio.io/istio/pilot/pkg/model" + "istio.io/istio/pilot/pkg/model/credentials" + "istio.io/istio/pilot/pkg/util/sets" +) + +func extraSecret(name string) model.NamespacedName { + result := model.NamespacedName{} + res := strings.TrimPrefix(name, credentials.KubernetesIngressSecretTypeURI) + split := strings.Split(res, "/") + if len(split) != 3 { + return result + } + + return model.NamespacedName{ + Namespace: split[1], + Name: split[2], + } +} + +func splitBySeparator(content, separator string) []string { + var result []string + parts := strings.Split(content, separator) + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + result = append(result, part) + } + return result +} + +func toSet(slice []string) sets.Set { + return sets.NewSet(slice...) +} diff --git a/ingress/kube/annotations/util_test.go b/ingress/kube/annotations/util_test.go new file mode 100644 index 000000000..a862cab59 --- /dev/null +++ b/ingress/kube/annotations/util_test.go @@ -0,0 +1,53 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package annotations + +import ( + "reflect" + "testing" + + "istio.io/istio/pilot/pkg/model" +) + +func TestExtraSecret(t *testing.T) { + inputCases := []struct { + input string + expect model.NamespacedName + }{ + { + input: "test/test", + expect: model.NamespacedName{}, + }, + { + input: "kubernetes-ingress://test/test", + expect: model.NamespacedName{}, + }, + { + input: "kubernetes-ingress://cluster/foo/bar", + expect: model.NamespacedName{ + Namespace: "foo", + Name: "bar", + }, + }, + } + + for _, inputCase := range inputCases { + t.Run("", func(t *testing.T) { + if !reflect.DeepEqual(inputCase.expect, extraSecret(inputCase.input)) { + t.Fatal("Should be equal") + } + }) + } +} diff --git a/ingress/kube/common/controller.go b/ingress/kube/common/controller.go new file mode 100644 index 000000000..74d43eccf --- /dev/null +++ b/ingress/kube/common/controller.go @@ -0,0 +1,133 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "strings" + + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/model" + "istio.io/istio/pkg/config" + gatewaytool "istio.io/istio/pkg/config/gateway" + listerv1 "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + + "github.com/alibaba/higress/ingress/kube/annotations" +) + +type ServiceKey struct { + Namespace string + Name string + Port int32 +} + +type WrapperConfig struct { + Config *config.Config + AnnotationsConfig *annotations.Ingress +} + +type WrapperGateway struct { + Gateway *networking.Gateway + WrapperConfig *WrapperConfig + ClusterId string + Host string +} + +func (w *WrapperGateway) IsHTTPS() bool { + if w.Gateway == nil || len(w.Gateway.Servers) == 0 { + return false + } + + for _, server := range w.Gateway.Servers { + if gatewaytool.IsTLSServer(server) { + return true + } + } + + return false +} + +type WrapperHTTPRoute struct { + HTTPRoute *networking.HTTPRoute + WrapperConfig *WrapperConfig + RawClusterId string + ClusterId string + ClusterName string + Host string + OriginPath string + OriginPathType PathType + WeightTotal int32 + IsDefaultBackend bool +} + +func (w *WrapperHTTPRoute) Meta() string { + return strings.Join([]string{w.WrapperConfig.Config.Namespace, w.WrapperConfig.Config.Name}, "-") +} + +func (w *WrapperHTTPRoute) BasePathFormat() string { + return strings.Join([]string{w.Host, w.OriginPath}, "-") +} + +func (w *WrapperHTTPRoute) PathFormat() string { + return strings.Join([]string{w.Host, string(w.OriginPathType), w.OriginPath}, "-") +} + +type WrapperVirtualService struct { + VirtualService *networking.VirtualService + WrapperConfig *WrapperConfig + ConfiguredDefaultBackend bool + AppRoot string +} + +type WrapperTrafficPolicy struct { + TrafficPolicy *networking.TrafficPolicy_PortTrafficPolicy + WrapperConfig *WrapperConfig +} + +type WrapperDestinationRule struct { + DestinationRule *networking.DestinationRule + WrapperConfig *WrapperConfig + ServiceKey ServiceKey +} + +type IngressController interface { + // RegisterEventHandler adds a handler to receive config update events for a + // configuration type + RegisterEventHandler(kind config.GroupVersionKind, handler model.EventHandler) + + List() []config.Config + + ServiceLister() listerv1.ServiceLister + + SecretLister() listerv1.SecretLister + + ConvertGateway(convertOptions *ConvertOptions, wrapper *WrapperConfig) error + + ConvertHTTPRoute(convertOptions *ConvertOptions, wrapper *WrapperConfig) error + + ApplyDefaultBackend(convertOptions *ConvertOptions, wrapper *WrapperConfig) error + + ApplyCanaryIngress(convertOptions *ConvertOptions, wrapper *WrapperConfig) error + + ConvertTrafficPolicy(convertOptions *ConvertOptions, wrapper *WrapperConfig) error + + // Run until a signal is received + Run(stop <-chan struct{}) + + SetWatchErrorHandler(func(r *cache.Reflector, err error)) error + + // HasSynced returns true after initial cache synchronization is complete + HasSynced() bool +} diff --git a/ingress/kube/common/metrics.go b/ingress/kube/common/metrics.go new file mode 100644 index 000000000..a8795796d --- /dev/null +++ b/ingress/kube/common/metrics.go @@ -0,0 +1,68 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import "istio.io/pkg/monitoring" + +type Event string + +const ( + Normal Event = "normal" + + Unknown Event = "unknown" + + EmptyRule Event = "empty-rule" + + MissingSecret Event = "missing-secret" + + InvalidBackendService Event = "invalid-backend-service" + + DuplicatedRoute Event = "duplicated-route" + + DuplicatedTls Event = "duplicated-tls" + + PortNameResolveError Event = "port-name-resolve-error" +) + +var ( + clusterTag = monitoring.MustCreateLabel("cluster") + invalidType = monitoring.MustCreateLabel("type") + + // totalIngresses tracks the total number of ingress + totalIngresses = monitoring.NewGauge( + "pilot_total_ingresses", + "Total ingresses known to pilot.", + monitoring.WithLabels(clusterTag), + ) + + totalInvalidIngress = monitoring.NewSum( + "pilot_total_invalid_ingresses", + "Total invalid ingresses known to pilot.", + monitoring.WithLabels(clusterTag, invalidType), + ) +) + +func init() { + monitoring.MustRegister(totalIngresses) + monitoring.MustRegister(totalInvalidIngress) +} + +func RecordIngressNumber(cluster string, number int) { + totalIngresses.With(clusterTag.Value(cluster)).Record(float64(number)) +} + +func IncrementInvalidIngress(cluster string, event Event) { + totalInvalidIngress.With(clusterTag.Value(cluster), invalidType.Value(string(event))).Increment() +} diff --git a/ingress/kube/common/model.go b/ingress/kube/common/model.go new file mode 100644 index 000000000..0c8c65b6c --- /dev/null +++ b/ingress/kube/common/model.go @@ -0,0 +1,411 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "errors" + "fmt" + "strconv" + "strings" + "time" + + "istio.io/istio/pilot/pkg/model" + "istio.io/istio/pkg/cluster" + "istio.io/istio/pkg/config" + "istio.io/istio/pkg/config/schema/collection" + "istio.io/istio/pkg/config/schema/collections" + "k8s.io/apimachinery/pkg/labels" + + . "github.com/alibaba/higress/ingress/log" +) + +type PathType string + +const ( + prefixAnnotation = "internal.higress.io/" + + ClusterIdAnnotation = prefixAnnotation + "cluster-id" + + RawClusterIdAnnotation = prefixAnnotation + "raw-cluster-id" + + HostAnnotation = prefixAnnotation + "host" + + // PrefixMatchRegex optionally matches "/..." at the end of a path. + // regex taken from https://github.com/projectcontour/contour/blob/2b3376449bedfea7b8cea5fbade99fb64009c0f6/internal/envoy/v3/route.go#L59 + PrefixMatchRegex = `((\/).*)?` + + DefaultHost = "*" + + DefaultPath = "/" + + // DefaultIngressClass defines the default class used in the nginx ingress controller. + // For compatible ingress nginx case, istio controller will watch ingresses whose ingressClass is + // nginx, empty string or unset. + DefaultIngressClass = "nginx" + + Exact PathType = "exact" + + Prefix PathType = "prefix" + + Regex PathType = "regex" + + DefaultStatusUpdateInterval = 10 * time.Second + + AppKey = "app" + AppValue = "higress-gateway" + SvcHostNameSuffix = ".multiplenic" +) + +var ( + ErrUnsupportedOp = errors.New("unsupported operation: the ingress config store is a read-only view") + + ErrNotFound = errors.New("item not found") + + Schemas = collection.SchemasFor( + collections.IstioNetworkingV1Alpha3Virtualservices, + collections.IstioNetworkingV1Alpha3Gateways, + collections.IstioNetworkingV1Alpha3Destinationrules, + collections.IstioNetworkingV1Alpha3Envoyfilters, + ) + + clusterPrefix string + SvcLabelSelector labels.Selector +) + +func init() { + set := labels.Set{ + AppKey: AppValue, + } + SvcLabelSelector = labels.SelectorFromSet(set) +} + +type Options struct { + Enable bool + ClusterId string + IngressClass string + WatchNamespace string + RawClusterId string + EnableStatus bool + SystemNamespace string + GatewaySelectorKey string + GatewaySelectorValue string +} + +type BasicAuthRules struct { + Rules []*Rule `json:"_rules_"` +} + +type Rule struct { + Realm string `json:"realm"` + MatchRoute []string `json:"_match_route_"` + Credentials []string `json:"credentials"` + Encrypted bool `json:"encrypted"` +} + +type IngressDomainCache struct { + // host as key + Valid map[string]*IngressDomainBuilder + + Invalid []model.IngressDomain +} + +func NewIngressDomainCache() *IngressDomainCache { + return &IngressDomainCache{ + Valid: map[string]*IngressDomainBuilder{}, + } +} + +func (i *IngressDomainCache) Extract() model.IngressDomainCollection { + var valid []model.IngressDomain + + for _, builder := range i.Valid { + valid = append(valid, builder.Build()) + } + + return model.IngressDomainCollection{ + Valid: valid, + Invalid: i.Invalid, + } +} + +type ConvertOptions struct { + HostWithRule2Ingress map[string]*config.Config + + HostWithTls2Ingress map[string]*config.Config + + Gateways map[string]*WrapperGateway + + IngressDomainCache *IngressDomainCache + + HostAndPath2Ingress map[string]*config.Config + + // Record valid/invalid routes from ingress + IngressRouteCache *IngressRouteCache + + VirtualServices map[string]*WrapperVirtualService + + // host -> routes + HTTPRoutes map[string][]*WrapperHTTPRoute + + CanaryIngresses []*WrapperConfig + + Service2TrafficPolicy map[ServiceKey]*WrapperTrafficPolicy + + HasDefaultBackend bool +} + +// CreateOptions obtain options from cluster id. +// The cluster id format is k8sClusterId ingressClass watchNamespace EnableStatus, delimited by _. +func CreateOptions(clusterId cluster.ID) Options { + parts := strings.Split(clusterId.String(), "_") + // Old cluster key + if len(parts) < 3 { + out := Options{ + RawClusterId: clusterId.String(), + } + if len(parts) > 0 { + out.ClusterId = parts[0] + } + return out + } + + options := Options{ + Enable: true, + ClusterId: parts[0], + IngressClass: parts[1], + WatchNamespace: parts[2], + RawClusterId: clusterId.String(), + // The status switch is enabled by default. + EnableStatus: true, + } + + if len(parts) == 4 { + if enable, err := strconv.ParseBool(parts[3]); err == nil { + options.EnableStatus = enable + } + } + return options +} + +type IngressRouteCache struct { + routes map[string]*IngressRouteBuilder + invalid []model.IngressRoute +} + +func NewIngressRouteCache() *IngressRouteCache { + return &IngressRouteCache{ + routes: map[string]*IngressRouteBuilder{}, + } +} + +func (i *IngressRouteCache) New(route *WrapperHTTPRoute) *IngressRouteBuilder { + return &IngressRouteBuilder{ + ClusterId: route.ClusterId, + RouteName: route.HTTPRoute.Name, + Path: route.OriginPath, + PathType: string(route.OriginPathType), + Host: route.Host, + Event: Normal, + Ingress: route.WrapperConfig.Config, + } +} + +func (i *IngressRouteCache) NewAndAdd(route *WrapperHTTPRoute) { + routeBuilder := &IngressRouteBuilder{ + ClusterId: route.ClusterId, + RouteName: route.HTTPRoute.Name, + Path: route.OriginPath, + PathType: string(route.OriginPathType), + Host: route.Host, + Event: Normal, + Ingress: route.WrapperConfig.Config, + } + + // Only care about the first destination + destination := route.HTTPRoute.Route[0].Destination + svcName, namespace, _ := SplitServiceFQDN(destination.Host) + routeBuilder.ServiceList = []model.BackendService{ + { + Name: svcName, + Namespace: namespace, + Port: destination.Port.Number, + Weight: route.HTTPRoute.Route[0].Weight, + }, + } + + i.routes[route.HTTPRoute.Name] = routeBuilder +} + +func (i *IngressRouteCache) Add(builder *IngressRouteBuilder) { + if builder.Event != Normal { + builder.RouteName = "invalid-route" + i.invalid = append(i.invalid, builder.Build()) + return + } + + i.routes[builder.RouteName] = builder +} + +func (i *IngressRouteCache) Update(route *WrapperHTTPRoute) { + oldBuilder, exist := i.routes[route.HTTPRoute.Name] + if !exist { + // Never happen + IngressLog.Errorf("ingress route builder %s not found.", route.HTTPRoute.Name) + return + } + + var serviceList []model.BackendService + for _, routeDestination := range route.HTTPRoute.Route { + serviceList = append(serviceList, ConvertBackendService(routeDestination)) + } + + oldBuilder.ServiceList = serviceList +} + +func (i *IngressRouteCache) Delete(route *WrapperHTTPRoute) { + delete(i.routes, route.HTTPRoute.Name) +} + +func (i *IngressRouteCache) Extract() model.IngressRouteCollection { + var valid []model.IngressRoute + + for _, builder := range i.routes { + valid = append(valid, builder.Build()) + } + + return model.IngressRouteCollection{ + Valid: valid, + Invalid: i.invalid, + } +} + +type IngressRouteBuilder struct { + ClusterId string + RouteName string + Host string + PathType string + Path string + ServiceList []model.BackendService + PortName string + Event Event + Ingress *config.Config + PreIngress *config.Config +} + +func (i *IngressRouteBuilder) Build() model.IngressRoute { + errorMsg := "" + switch i.Event { + case DuplicatedRoute: + preClusterId := GetClusterId(i.PreIngress.Annotations) + errorMsg = fmt.Sprintf("host %s and path %s in ingress %s/%s within cluster %s is already defined in ingress %s/%s within cluster %s", + i.Host, + i.Path, + i.Ingress.Namespace, + i.Ingress.Name, + i.ClusterId, + i.PreIngress.Namespace, + i.PreIngress.Name, + preClusterId) + case InvalidBackendService: + errorMsg = fmt.Sprintf("backend service of host %s and path %s is invalid defined in ingress %s/%s within cluster %s", + i.Host, + i.Path, + i.Ingress.Namespace, + i.Ingress.Name, + i.ClusterId, + ) + case PortNameResolveError: + errorMsg = fmt.Sprintf("service port name %s of host %s and path %s resolves error defined in ingress %s/%s within cluster %s", + i.PortName, + i.Host, + i.Path, + i.Ingress.Namespace, + i.Ingress.Name, + i.ClusterId, + ) + } + + ingressRoute := model.IngressRoute{ + Name: i.RouteName, + Host: i.Host, + Path: i.Path, + PathType: i.PathType, + DestinationType: model.Single, + ServiceList: i.ServiceList, + Error: errorMsg, + } + + // backward compatibility + if len(ingressRoute.ServiceList) > 0 { + ingressRoute.ServiceName = i.ServiceList[0].Name + } + + if len(ingressRoute.ServiceList) > 1 { + ingressRoute.DestinationType = model.Multiple + } + + return ingressRoute +} + +type Protocol string + +const ( + HTTP Protocol = "HTTP" + HTTPS Protocol = "HTTPS" +) + +type IngressDomainBuilder struct { + ClusterId string + Host string + Protocol Protocol + Event Event + // format is cluster id/namespace/name + SecretName string + Ingress *config.Config + PreIngress *config.Config +} + +func (i *IngressDomainBuilder) Build() model.IngressDomain { + errorMsg := "" + switch i.Event { + case MissingSecret: + errorMsg = fmt.Sprintf("tls field of host %s defined in ingress %s/%s within cluster %s misses secret", + i.Host, + i.Ingress.Namespace, + i.Ingress.Name, + i.ClusterId, + ) + case DuplicatedTls: + preClusterId := GetClusterId(i.PreIngress.Annotations) + errorMsg = fmt.Sprintf("tls field of host %s defined in ingress %s/%s within cluster %s "+ + "is conflicted with ingress %s/%s within cluster %s", + i.Host, + i.Ingress.Namespace, + i.Ingress.Name, + i.ClusterId, + i.PreIngress.Namespace, + i.PreIngress.Name, + preClusterId, + ) + } + + return model.IngressDomain{ + Host: i.Host, + Protocol: string(i.Protocol), + SecretName: i.SecretName, + CreationTime: i.Ingress.CreationTimestamp, + Error: errorMsg, + } +} diff --git a/ingress/kube/common/tool.go b/ingress/kube/common/tool.go new file mode 100644 index 000000000..49731cf4a --- /dev/null +++ b/ingress/kube/common/tool.go @@ -0,0 +1,365 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "crypto/md5" + "encoding/hex" + "net" + "sort" + "strings" + + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/model" + "istio.io/istio/pkg/config" + "istio.io/istio/pkg/kube" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/version" + + . "github.com/alibaba/higress/ingress/log" +) + +// V1Available check if the "networking/v1" Ingress is available. +func V1Available(client kube.Client) bool { + // check kubernetes version to use new ingress package or not + version119, _ := version.ParseGeneric("v1.19.0") + + serverVersion, err := client.GetKubernetesVersion() + if err != nil { + return false + } + + runningVersion, err := version.ParseGeneric(serverVersion.String()) + if err != nil { + IngressLog.Errorf("unexpected error parsing running Kubernetes version: %v", err) + return false + } + + return runningVersion.AtLeast(version119) +} + +// NetworkingIngressAvailable check if the "networking" group Ingress is available. +func NetworkingIngressAvailable(client kube.Client) bool { + // check kubernetes version to use new ingress package or not + version118, _ := version.ParseGeneric("v1.18.0") + + serverVersion, err := client.GetKubernetesVersion() + if err != nil { + return false + } + + runningVersion, err := version.ParseGeneric(serverVersion.String()) + if err != nil { + IngressLog.Errorf("unexpected error parsing running Kubernetes version: %v", err) + return false + } + + return runningVersion.AtLeast(version118) +} + +// SortIngressByCreationTime sorts the list of config objects in ascending order by their creation time (if available). +func SortIngressByCreationTime(configs []config.Config) { + sort.Slice(configs, func(i, j int) bool { + if configs[i].CreationTimestamp == configs[j].CreationTimestamp { + in := configs[i].Name + "." + configs[i].Namespace + jn := configs[j].Name + "." + configs[j].Namespace + return in < jn + } + return configs[i].CreationTimestamp.Before(configs[j].CreationTimestamp) + }) +} + +func CreateOrUpdateAnnotations(annotations map[string]string, options Options) map[string]string { + out := make(map[string]string, len(annotations)) + + for key, value := range annotations { + out[key] = value + } + + out[ClusterIdAnnotation] = options.ClusterId + out[RawClusterIdAnnotation] = options.RawClusterId + return out +} + +func GetClusterId(annotations map[string]string) string { + if len(annotations) == 0 { + return "" + } + + if value, exist := annotations[ClusterIdAnnotation]; exist { + return value + } + + return "" +} + +func GetRawClusterId(annotations map[string]string) string { + if len(annotations) == 0 { + return "" + } + + if value, exist := annotations[RawClusterIdAnnotation]; exist { + return value + } + + return "" +} + +func GetHost(annotations map[string]string) string { + if len(annotations) == 0 { + return "" + } + + if value, exist := annotations[HostAnnotation]; exist { + return value + } + + return "" +} + +// CleanHost follow the format of mse-ops for host. +func CleanHost(host string) string { + if host == "*" { + return "global" + } + + if strings.HasPrefix(host, "*") { + host = strings.ReplaceAll(host, "*", "global-") + } + + return strings.ReplaceAll(host, ".", "-") +} + +func CreateConvertedName(items ...string) string { + for i := len(items) - 1; i >= 0; i-- { + if items[i] == "" { + items = append(items[:i], items[i+1:]...) + } + } + return strings.Join(items, "-") +} + +// SortHTTPRoutes sort routes base on path type and path length +func SortHTTPRoutes(routes []*WrapperHTTPRoute) { + isDefaultBackend := func(route *WrapperHTTPRoute) bool { + return route.IsDefaultBackend + } + + isAllCatch := func(route *WrapperHTTPRoute) bool { + if route.OriginPathType == Prefix && route.OriginPath == "/" { + return true + } + return false + } + + sort.SliceStable(routes, func(i, j int) bool { + // Move default backend to end + if isDefaultBackend(routes[i]) { + return false + } + if isDefaultBackend(routes[j]) { + return true + } + + // Move user specified root path match to end + if isAllCatch(routes[i]) { + return false + } + if isAllCatch(routes[j]) { + return true + } + + if routes[i].OriginPathType == routes[j].OriginPathType { + return len(routes[i].OriginPath) > len(routes[j].OriginPath) + } + + if routes[i].OriginPathType == Exact { + return true + } + + if routes[i].OriginPathType != Exact && + routes[j].OriginPathType != Exact { + return routes[i].OriginPathType == Prefix + } + + return false + }) +} + +func constructRouteName(route *WrapperHTTPRoute) string { + var builder strings.Builder + // host-pathType-path + base := route.PathFormat() + builder.WriteString(base) + + var mappings []string + var headerMappings []string + var queryMappings []string + if len(route.HTTPRoute.Match) > 0 { + match := route.HTTPRoute.Match[0] + if len(match.Headers) != 0 { + for k, v := range match.Headers { + var mapping string + switch v.GetMatchType().(type) { + case *networking.StringMatch_Exact: + mapping = CreateConvertedName("exact", k, v.GetExact()) + case *networking.StringMatch_Prefix: + mapping = CreateConvertedName("prefix", k, v.GetPrefix()) + case *networking.StringMatch_Regex: + mapping = CreateConvertedName("regex", k, v.GetRegex()) + } + + headerMappings = append(headerMappings, mapping) + } + + sort.SliceStable(headerMappings, func(i, j int) bool { + return headerMappings[i] < headerMappings[j] + }) + } + + if len(match.QueryParams) != 0 { + for k, v := range match.QueryParams { + var mapping string + switch v.GetMatchType().(type) { + case *networking.StringMatch_Exact: + mapping = strings.Join([]string{"exact", k, v.GetExact()}, ":") + case *networking.StringMatch_Prefix: + mapping = strings.Join([]string{"prefix", k, v.GetPrefix()}, ":") + case *networking.StringMatch_Regex: + mapping = strings.Join([]string{"regex", k, v.GetRegex()}, ":") + } + + queryMappings = append(queryMappings, mapping) + } + + sort.SliceStable(queryMappings, func(i, j int) bool { + return queryMappings[i] < queryMappings[j] + }) + } + } + + mappings = append(mappings, headerMappings...) + mappings = append(mappings, queryMappings...) + + if len(mappings) == 0 { + return CreateConvertedName(base) + } + + return CreateConvertedName(base, CreateConvertedName(mappings...)) +} + +func partMd5(raw string) string { + hash := md5.Sum([]byte(raw)) + encoded := hex.EncodeToString(hash[:]) + return encoded[:4] + encoded[len(encoded)-4:] +} + +func GenerateUniqueRouteName(route *WrapperHTTPRoute) string { + raw := constructRouteName(route) + + // meta-part-clusterId + // meta: ingressNamespace-ingressName + meta := route.Meta() + // host-pathType-path-header-queryParam, md5, then before 4 char and end 4 char + part := partMd5(raw) + routeName := CreateConvertedName(meta, part, route.ClusterId) + + if route.WrapperConfig.AnnotationsConfig.IsCanary() { + return routeName + "-canary" + } + + return routeName +} + +func GenerateUniqueRouteNameWithSuffix(route *WrapperHTTPRoute, suffix string) string { + raw := constructRouteName(route) + + // meta-part-clusterId + // meta: ingressNamespace-ingressName + meta := route.Meta() + // host-pathType-path-header-queryParam, md5, then before 4 char and end 4 char + part := partMd5(raw) + return CreateConvertedName(meta, part, route.ClusterId, suffix) +} + +func SplitServiceFQDN(fqdn string) (string, string, bool) { + parts := strings.Split(fqdn, ".") + if len(parts) > 1 { + return parts[0], parts[1], true + } + return "", "", false +} + +func ConvertBackendService(routeDestination *networking.HTTPRouteDestination) model.BackendService { + parts := strings.Split(routeDestination.Destination.Host, ".") + return model.BackendService{ + Namespace: parts[1], + Name: parts[0], + Port: routeDestination.Destination.Port.Number, + Weight: routeDestination.Weight, + } +} + +func getLoadBalancerIp(svc *v1.Service) []string { + var out []string + + for _, ingress := range svc.Status.LoadBalancer.Ingress { + if ingress.IP != "" { + out = append(out, ingress.IP) + } + + if ingress.Hostname != "" { + hostName := strings.TrimSuffix(ingress.Hostname, SvcHostNameSuffix) + if net.ParseIP(hostName) != nil { + out = append(out, hostName) + } + } + } + + return out +} + +func getSvcIpList(svcList []*v1.Service) []string { + var targetSvcList []*v1.Service + for _, svc := range svcList { + if svc.Spec.Type == v1.ServiceTypeLoadBalancer && + strings.HasPrefix(svc.Name, clusterPrefix) { + targetSvcList = append(targetSvcList, svc) + } + } + + var out []string + for _, svc := range targetSvcList { + out = append(out, getLoadBalancerIp(svc)...) + } + return out +} + +func SortLbIngressList(lbi []v1.LoadBalancerIngress) func(int, int) bool { + return func(i int, j int) bool { + return lbi[i].IP < lbi[j].IP + } +} + +func GetLbStatusList(svcList []*v1.Service) []v1.LoadBalancerIngress { + svcIpList := getSvcIpList(svcList) + lbi := make([]v1.LoadBalancerIngress, 0, len(svcIpList)) + for _, ep := range svcIpList { + lbi = append(lbi, v1.LoadBalancerIngress{IP: ep}) + } + + sort.SliceStable(lbi, SortLbIngressList(lbi)) + return lbi +} diff --git a/ingress/kube/common/tool_test.go b/ingress/kube/common/tool_test.go new file mode 100644 index 000000000..2311d060e --- /dev/null +++ b/ingress/kube/common/tool_test.go @@ -0,0 +1,473 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" + + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pkg/config" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/alibaba/higress/ingress/kube/annotations" +) + +func TestConstructRouteName(t *testing.T) { + testCases := []struct { + input *WrapperHTTPRoute + expect string + }{ + { + input: &WrapperHTTPRoute{ + Host: "test.com", + OriginPathType: Exact, + OriginPath: "/test", + HTTPRoute: &networking.HTTPRoute{}, + }, + expect: "test.com-exact-/test", + }, + { + input: &WrapperHTTPRoute{ + Host: "*.test.com", + OriginPathType: Regex, + OriginPath: "/test/(.*)/?[0-9]", + HTTPRoute: &networking.HTTPRoute{}, + }, + expect: "*.test.com-regex-/test/(.*)/?[0-9]", + }, + { + input: &WrapperHTTPRoute{ + Host: "test.com", + OriginPathType: Exact, + OriginPath: "/test", + HTTPRoute: &networking.HTTPRoute{ + Match: []*networking.HTTPMatchRequest{ + { + Headers: map[string]*networking.StringMatch{ + "b": { + MatchType: &networking.StringMatch_Regex{ + Regex: "a?c.*", + }, + }, + "a": { + MatchType: &networking.StringMatch_Exact{ + Exact: "hello", + }, + }, + }, + }, + }, + }, + }, + expect: "test.com-exact-/test-exact-a-hello-regex-b-a?c.*", + }, + { + input: &WrapperHTTPRoute{ + Host: "test.com", + OriginPathType: Prefix, + OriginPath: "/test", + HTTPRoute: &networking.HTTPRoute{ + Match: []*networking.HTTPMatchRequest{ + { + QueryParams: map[string]*networking.StringMatch{ + "b": { + MatchType: &networking.StringMatch_Regex{ + Regex: "a?c.*", + }, + }, + "a": { + MatchType: &networking.StringMatch_Exact{ + Exact: "hello", + }, + }, + }, + }, + }, + }, + }, + expect: "test.com-prefix-/test-exact:a:hello-regex:b:a?c.*", + }, + { + input: &WrapperHTTPRoute{ + Host: "test.com", + OriginPathType: Prefix, + OriginPath: "/test", + HTTPRoute: &networking.HTTPRoute{ + Match: []*networking.HTTPMatchRequest{ + { + Headers: map[string]*networking.StringMatch{ + "f": { + MatchType: &networking.StringMatch_Regex{ + Regex: "abc?", + }, + }, + "e": { + MatchType: &networking.StringMatch_Exact{ + Exact: "bye", + }, + }, + }, + QueryParams: map[string]*networking.StringMatch{ + "b": { + MatchType: &networking.StringMatch_Regex{ + Regex: "a?c.*", + }, + }, + "a": { + MatchType: &networking.StringMatch_Exact{ + Exact: "hello", + }, + }, + }, + }, + }, + }, + }, + expect: "test.com-prefix-/test-exact-e-bye-regex-f-abc?-exact:a:hello-regex:b:a?c.*", + }, + } + + for _, c := range testCases { + t.Run("", func(t *testing.T) { + out := constructRouteName(c.input) + if out != c.expect { + t.Fatalf("Expect %s, but is %s", c.expect, out) + } + }) + } +} + +func TestGenerateUniqueRouteName(t *testing.T) { + inputWithoutCanary := &WrapperHTTPRoute{ + WrapperConfig: &WrapperConfig{ + Config: &config.Config{ + Meta: config.Meta{ + Name: "foo", + Namespace: "bar", + }, + }, + AnnotationsConfig: &annotations.Ingress{}, + }, + Host: "test.com", + OriginPathType: Prefix, + OriginPath: "/test", + ClusterId: "cluster1", + HTTPRoute: &networking.HTTPRoute{ + Match: []*networking.HTTPMatchRequest{ + { + Headers: map[string]*networking.StringMatch{ + "f": { + MatchType: &networking.StringMatch_Regex{ + Regex: "abc?", + }, + }, + "e": { + MatchType: &networking.StringMatch_Exact{ + Exact: "bye", + }, + }, + }, + QueryParams: map[string]*networking.StringMatch{ + "b": { + MatchType: &networking.StringMatch_Regex{ + Regex: "a?c.*", + }, + }, + "a": { + MatchType: &networking.StringMatch_Exact{ + Exact: "hello", + }, + }, + }, + }, + }, + }, + } + + withoutCanary := GenerateUniqueRouteName(inputWithoutCanary) + t.Log(withoutCanary) + + inputWithCanary := &WrapperHTTPRoute{ + WrapperConfig: &WrapperConfig{ + Config: &config.Config{ + Meta: config.Meta{ + Name: "foo", + Namespace: "bar", + }, + }, + AnnotationsConfig: &annotations.Ingress{ + Canary: &annotations.CanaryConfig{ + Enabled: true, + }, + }, + }, + Host: "test.com", + OriginPathType: Prefix, + OriginPath: "/test", + ClusterId: "cluster1", + HTTPRoute: &networking.HTTPRoute{ + Match: []*networking.HTTPMatchRequest{ + { + Headers: map[string]*networking.StringMatch{ + "f": { + MatchType: &networking.StringMatch_Regex{ + Regex: "abc?", + }, + }, + "e": { + MatchType: &networking.StringMatch_Exact{ + Exact: "bye", + }, + }, + }, + QueryParams: map[string]*networking.StringMatch{ + "b": { + MatchType: &networking.StringMatch_Regex{ + Regex: "a?c.*", + }, + }, + "a": { + MatchType: &networking.StringMatch_Exact{ + Exact: "hello", + }, + }, + }, + }, + }, + }, + } + + withCanary := GenerateUniqueRouteName(inputWithCanary) + t.Log(withCanary) + + if withCanary != withoutCanary+"-canary" { + t.Fatalf("Expect %s, but actual is %s", withCanary, withoutCanary+"-canary") + } +} + +func TestGetLbStatusList(t *testing.T) { + clusterPrefix = "gw-123-" + svcName := clusterPrefix + svcList := []*v1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + }, + Status: v1.ServiceStatus{ + LoadBalancer: v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{ + { + IP: "2.2.2.2", + }, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + }, + Status: v1.ServiceStatus{ + LoadBalancer: v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{ + { + Hostname: "1.1.1.1" + SvcHostNameSuffix, + }, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + }, + Status: v1.ServiceStatus{ + LoadBalancer: v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{ + { + Hostname: "4.4.4.4" + SvcHostNameSuffix, + }, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + }, + Status: v1.ServiceStatus{ + LoadBalancer: v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{ + { + IP: "3.3.3.3", + }, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: svcName, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, + }, + Status: v1.ServiceStatus{ + LoadBalancer: v1.LoadBalancerStatus{ + Ingress: []v1.LoadBalancerIngress{ + { + IP: "5.5.5.5", + }, + }, + }, + }, + }, + } + + lbiList := GetLbStatusList(svcList) + if len(lbiList) != 4 { + t.Fatal("len should be 4") + } + + if lbiList[0].IP != "1.1.1.1" { + t.Fatal("should be 1.1.1.1") + } + + if lbiList[3].IP != "4.4.4.4" { + t.Fatal("should be 4.4.4.4") + } +} + +func TestSortRoutes(t *testing.T) { + input := []*WrapperHTTPRoute{ + { + WrapperConfig: &WrapperConfig{ + Config: &config.Config{ + Meta: config.Meta{ + Name: "foo", + Namespace: "bar", + }, + }, + AnnotationsConfig: &annotations.Ingress{}, + }, + Host: "test.com", + OriginPathType: Prefix, + OriginPath: "/", + ClusterId: "cluster1", + HTTPRoute: &networking.HTTPRoute{ + Name: "test-1", + }, + }, + { + WrapperConfig: &WrapperConfig{ + Config: &config.Config{ + Meta: config.Meta{ + Name: "foo", + Namespace: "bar", + }, + }, + AnnotationsConfig: &annotations.Ingress{}, + }, + Host: "test.com", + OriginPathType: Prefix, + OriginPath: "/a", + ClusterId: "cluster1", + HTTPRoute: &networking.HTTPRoute{ + Name: "test-2", + }, + }, + { + WrapperConfig: &WrapperConfig{ + Config: &config.Config{ + Meta: config.Meta{ + Name: "foo", + Namespace: "bar", + }, + }, + AnnotationsConfig: &annotations.Ingress{}, + }, + HTTPRoute: &networking.HTTPRoute{ + Name: "test-3", + }, + IsDefaultBackend: true, + }, + { + WrapperConfig: &WrapperConfig{ + Config: &config.Config{ + Meta: config.Meta{ + Name: "foo", + Namespace: "bar", + }, + }, + AnnotationsConfig: &annotations.Ingress{}, + }, + Host: "test.com", + OriginPathType: Exact, + OriginPath: "/b", + ClusterId: "cluster1", + HTTPRoute: &networking.HTTPRoute{ + Name: "test-4", + }, + }, + { + WrapperConfig: &WrapperConfig{ + Config: &config.Config{ + Meta: config.Meta{ + Name: "foo", + Namespace: "bar", + }, + }, + AnnotationsConfig: &annotations.Ingress{}, + }, + Host: "test.com", + OriginPathType: Regex, + OriginPath: "/d(.*)", + ClusterId: "cluster1", + HTTPRoute: &networking.HTTPRoute{ + Name: "test-5", + }, + }, + } + + SortHTTPRoutes(input) + if (input[0].HTTPRoute.Name) != "test-4" { + t.Fatal("should be test-4") + } + if (input[1].HTTPRoute.Name) != "test-2" { + t.Fatal("should be test-2") + } + if (input[2].HTTPRoute.Name) != "test-5" { + t.Fatal("should be test-5") + } + if (input[3].HTTPRoute.Name) != "test-1" { + t.Fatal("should be test-1") + } + if (input[4].HTTPRoute.Name) != "test-3" { + t.Fatal("should be test-3") + } +} diff --git a/ingress/kube/ingress/controller.go b/ingress/kube/ingress/controller.go new file mode 100644 index 000000000..76fb015c3 --- /dev/null +++ b/ingress/kube/ingress/controller.go @@ -0,0 +1,1159 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingress + +import ( + "errors" + "fmt" + "path" + "reflect" + "regexp" + "strings" + "sync" + "time" + + "github.com/hashicorp/go-multierror" + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/model" + "istio.io/istio/pilot/pkg/model/credentials" + "istio.io/istio/pilot/pkg/serviceregistry/kube" + "istio.io/istio/pilot/pkg/util/sets" + "istio.io/istio/pkg/config" + "istio.io/istio/pkg/config/constants" + "istio.io/istio/pkg/config/protocol" + "istio.io/istio/pkg/config/schema/gvk" + kubeclient "istio.io/istio/pkg/kube" + "istio.io/istio/pkg/kube/controllers" + ingress "k8s.io/api/networking/v1beta1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/informers/networking/v1beta1" + listerv1 "k8s.io/client-go/listers/core/v1" + networkinglister "k8s.io/client-go/listers/networking/v1beta1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + "github.com/alibaba/higress/ingress/kube/annotations" + "github.com/alibaba/higress/ingress/kube/common" + "github.com/alibaba/higress/ingress/kube/secret" + "github.com/alibaba/higress/ingress/kube/util" + . "github.com/alibaba/higress/ingress/log" +) + +var ( + _ common.IngressController = &controller{} + + // follow specification of ingress-nginx + defaultPathType = ingress.PathTypePrefix +) + +type controller struct { + queue workqueue.RateLimitingInterface + virtualServiceHandlers []model.EventHandler + gatewayHandlers []model.EventHandler + destinationRuleHandlers []model.EventHandler + envoyFilterHandlers []model.EventHandler + + options common.Options + + mutex sync.RWMutex + // key: namespace/name + ingresses map[string]*ingress.Ingress + + ingressInformer cache.SharedInformer + ingressLister networkinglister.IngressLister + serviceInformer cache.SharedInformer + serviceLister listerv1.ServiceLister + // May be nil if ingress class is not supported in the cluster + classes v1beta1.IngressClassInformer + + secretController secret.Controller + + statusSyncer *statusSyncer +} + +// NewController creates a new Kubernetes controller +func NewController(localKubeClient, client kubeclient.Client, options common.Options, secretController secret.Controller) common.IngressController { + q := workqueue.NewRateLimitingQueue(workqueue.DefaultItemBasedRateLimiter()) + + ingressInformer := client.KubeInformer().Networking().V1beta1().Ingresses() + serviceInformer := client.KubeInformer().Core().V1().Services() + + var classes v1beta1.IngressClassInformer + if common.NetworkingIngressAvailable(client) { + classes = client.KubeInformer().Networking().V1beta1().IngressClasses() + _ = classes.Informer() + } else { + IngressLog.Infof("Skipping IngressClass, resource not supported for cluster %s", options.ClusterId) + } + + c := &controller{ + options: options, + queue: q, + ingresses: make(map[string]*ingress.Ingress), + ingressInformer: ingressInformer.Informer(), + ingressLister: ingressInformer.Lister(), + classes: classes, + serviceInformer: serviceInformer.Informer(), + serviceLister: serviceInformer.Lister(), + secretController: secretController, + } + + handler := controllers.LatestVersionHandlerFuncs(controllers.EnqueueForSelf(q)) + c.ingressInformer.AddEventHandler(handler) + + if options.EnableStatus { + c.statusSyncer = newStatusSyncer(localKubeClient, client, c, options.SystemNamespace) + } else { + IngressLog.Infof("Disable status update for cluster %s", options.ClusterId) + } + + return c +} + +func (c *controller) ServiceLister() listerv1.ServiceLister { + return c.serviceLister +} + +func (c *controller) SecretLister() listerv1.SecretLister { + return c.secretController.Lister() +} + +func (c *controller) Run(stop <-chan struct{}) { + if c.statusSyncer != nil { + go c.statusSyncer.run(stop) + } + go c.secretController.Run(stop) + + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + + if !cache.WaitForCacheSync(stop, c.HasSynced) { + IngressLog.Errorf("Failed to sync ingress controller cache for cluster %s", c.options.ClusterId) + return + } + go wait.Until(c.worker, time.Second, stop) + <-stop +} + +func (c *controller) worker() { + for c.processNextWorkItem() { + } +} + +func (c *controller) processNextWorkItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + ingressNamespacedName := key.(types.NamespacedName) + if err := c.onEvent(ingressNamespacedName); err != nil { + IngressLog.Errorf("error processing ingress item (%v) (retrying): %v, cluster: %s", key, err, c.options.ClusterId) + c.queue.AddRateLimited(key) + } else { + c.queue.Forget(key) + } + return true +} + +func (c *controller) onEvent(namespacedName types.NamespacedName) error { + event := model.EventUpdate + ing, err := c.ingressLister.Ingresses(namespacedName.Namespace).Get(namespacedName.Name) + if err != nil { + if kerrors.IsNotFound(err) { + event = model.EventDelete + c.mutex.Lock() + ing = c.ingresses[namespacedName.String()] + delete(c.ingresses, namespacedName.String()) + c.mutex.Unlock() + } else { + return err + } + } + + // ingress deleted, and it is not processed before + if ing == nil { + return nil + } + + // we should check need process only when event is not delete, + // if it is delete event, and previously processed, we need to process too. + if event != model.EventDelete { + shouldProcess, err := c.shouldProcessIngressUpdate(ing) + if err != nil { + return err + } + if !shouldProcess { + IngressLog.Infof("no need process, ingress %s", namespacedName) + return nil + } + } + + drmetadata := config.Meta{ + Name: ing.Name + "-" + "destinationrule", + Namespace: ing.Namespace, + GroupVersionKind: gvk.DestinationRule, + // Set this label so that we do not compare configs and just push. + Labels: map[string]string{constants.AlwaysPushLabel: "true"}, + } + vsmetadata := config.Meta{ + Name: ing.Name + "-" + "virtualservice", + Namespace: ing.Namespace, + GroupVersionKind: gvk.VirtualService, + // Set this label so that we do not compare configs and just push. + Labels: map[string]string{constants.AlwaysPushLabel: "true"}, + } + efmetadata := config.Meta{ + Name: ing.Name + "-" + "envoyfilter", + Namespace: ing.Namespace, + GroupVersionKind: gvk.EnvoyFilter, + // Set this label so that we do not compare configs and just push. + Labels: map[string]string{constants.AlwaysPushLabel: "true"}, + } + gatewaymetadata := config.Meta{ + Name: ing.Name + "-" + "gateway", + Namespace: ing.Namespace, + GroupVersionKind: gvk.Gateway, + // Set this label so that we do not compare configs and just push. + Labels: map[string]string{constants.AlwaysPushLabel: "true"}, + } + + for _, f := range c.destinationRuleHandlers { + f(config.Config{Meta: drmetadata}, config.Config{Meta: drmetadata}, event) + } + + for _, f := range c.virtualServiceHandlers { + f(config.Config{Meta: vsmetadata}, config.Config{Meta: vsmetadata}, event) + } + + for _, f := range c.envoyFilterHandlers { + f(config.Config{Meta: efmetadata}, config.Config{Meta: efmetadata}, event) + } + + for _, f := range c.gatewayHandlers { + f(config.Config{Meta: gatewaymetadata}, config.Config{Meta: gatewaymetadata}, event) + } + + return nil +} + +func (c *controller) RegisterEventHandler(kind config.GroupVersionKind, f model.EventHandler) { + switch kind { + case gvk.VirtualService: + c.virtualServiceHandlers = append(c.virtualServiceHandlers, f) + case gvk.Gateway: + c.gatewayHandlers = append(c.gatewayHandlers, f) + case gvk.DestinationRule: + c.destinationRuleHandlers = append(c.destinationRuleHandlers, f) + case gvk.EnvoyFilter: + c.envoyFilterHandlers = append(c.envoyFilterHandlers, f) + } +} + +func (c *controller) SetWatchErrorHandler(handler func(r *cache.Reflector, err error)) error { + var errs error + if err := c.serviceInformer.SetWatchErrorHandler(handler); err != nil { + errs = multierror.Append(errs, err) + } + if err := c.ingressInformer.SetWatchErrorHandler(handler); err != nil { + errs = multierror.Append(errs, err) + } + if err := c.secretController.Informer().SetWatchErrorHandler(handler); err != nil { + errs = multierror.Append(errs, err) + } + if c.classes != nil { + if err := c.classes.Informer().SetWatchErrorHandler(handler); err != nil { + errs = multierror.Append(errs, err) + } + } + return errs +} + +func (c *controller) HasSynced() bool { + return c.ingressInformer.HasSynced() && c.serviceInformer.HasSynced() && + (c.classes == nil || c.classes.Informer().HasSynced()) && + c.secretController.HasSynced() +} + +func (c *controller) List() []config.Config { + out := make([]config.Config, 0, len(c.ingresses)) + + for _, raw := range c.ingressInformer.GetStore().List() { + ing, ok := raw.(*ingress.Ingress) + if !ok { + continue + } + + if should, err := c.shouldProcessIngress(ing); !should || err != nil { + continue + } + + copiedConfig := ing.DeepCopy() + setDefaultMSEIngressOptionalField(copiedConfig) + + outConfig := config.Config{ + Meta: config.Meta{ + Name: copiedConfig.Name, + Namespace: copiedConfig.Namespace, + Annotations: common.CreateOrUpdateAnnotations(copiedConfig.Annotations, c.options), + Labels: copiedConfig.Labels, + CreationTimestamp: copiedConfig.CreationTimestamp.Time, + }, + Spec: copiedConfig.Spec, + } + + out = append(out, outConfig) + } + + common.RecordIngressNumber(c.options.ClusterId, len(out)) + return out +} + +func extractTLSSecretName(host string, tls []ingress.IngressTLS) string { + if len(tls) == 0 { + return "" + } + + for _, t := range tls { + match := false + for _, h := range t.Hosts { + if h == host { + match = true + } + } + + if match { + return t.SecretName + } + } + + return "" +} + +func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapper *common.WrapperConfig) error { + // Ignore canary config. + if wrapper.AnnotationsConfig.IsCanary() { + return nil + } + + cfg := wrapper.Config + ingressV1Beta, ok := cfg.Spec.(ingress.IngressSpec) + if !ok { + common.IncrementInvalidIngress(c.options.ClusterId, common.Unknown) + return fmt.Errorf("convert type is invalid in cluster %s", c.options.ClusterId) + } + if len(ingressV1Beta.Rules) == 0 && ingressV1Beta.Backend == nil { + common.IncrementInvalidIngress(c.options.ClusterId, common.EmptyRule) + return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId) + } + + for _, rule := range ingressV1Beta.Rules { + cleanHost := common.CleanHost(rule.Host) + // Need create builder for every rule. + domainBuilder := &common.IngressDomainBuilder{ + ClusterId: c.options.ClusterId, + Protocol: common.HTTP, + Host: rule.Host, + Ingress: cfg, + Event: common.Normal, + } + + // Extract the previous gateway and builder + wrapperGateway, exist := convertOptions.Gateways[rule.Host] + preDomainBuilder, _ := convertOptions.IngressDomainCache.Valid[rule.Host] + if !exist { + wrapperGateway = &common.WrapperGateway{ + Gateway: &networking.Gateway{ + Selector: map[string]string{ + c.options.GatewaySelectorKey: c.options.GatewaySelectorValue, + }, + }, + WrapperConfig: wrapper, + ClusterId: c.options.ClusterId, + Host: rule.Host, + } + wrapperGateway.Gateway.Servers = append(wrapperGateway.Gateway.Servers, &networking.Server{ + Port: &networking.Port{ + Number: 80, + Protocol: string(protocol.HTTP), + Name: common.CreateConvertedName("http-80-ingress", c.options.ClusterId, cfg.Namespace, cfg.Name, cleanHost), + }, + Hosts: []string{rule.Host}, + }) + + // Add new gateway, builder + convertOptions.Gateways[rule.Host] = wrapperGateway + convertOptions.IngressDomainCache.Valid[rule.Host] = domainBuilder + } else { + // Fallback to get downstream tls from current ingress. + if wrapperGateway.WrapperConfig.AnnotationsConfig.DownstreamTLS == nil { + wrapperGateway.WrapperConfig.AnnotationsConfig.DownstreamTLS = wrapper.AnnotationsConfig.DownstreamTLS + } + } + + // There are no tls settings, so just skip. + if len(ingressV1Beta.TLS) == 0 { + continue + } + + // Get tls secret matching the rule host + secretName := extractTLSSecretName(rule.Host, ingressV1Beta.TLS) + if secretName == "" { + // There no matching secret, so just skip. + continue + } + + domainBuilder.Protocol = common.HTTPS + domainBuilder.SecretName = path.Join(c.options.ClusterId, cfg.Namespace, secretName) + + // There is a matching secret and the gateway has already a tls secret. + // We should report the duplicated tls secret event. + if wrapperGateway.IsHTTPS() { + domainBuilder.Event = common.DuplicatedTls + domainBuilder.PreIngress = preDomainBuilder.Ingress + convertOptions.IngressDomainCache.Invalid = append(convertOptions.IngressDomainCache.Invalid, + domainBuilder.Build()) + continue + } + + // Append https server + wrapperGateway.Gateway.Servers = append(wrapperGateway.Gateway.Servers, &networking.Server{ + Port: &networking.Port{ + Number: 443, + Protocol: string(protocol.HTTPS), + Name: common.CreateConvertedName("https-443-ingress", c.options.ClusterId, cfg.Namespace, cfg.Name, cleanHost), + }, + Hosts: []string{rule.Host}, + Tls: &networking.ServerTLSSettings{ + Mode: networking.ServerTLSSettings_SIMPLE, + CredentialName: credentials.ToKubernetesIngressResource(c.options.RawClusterId, cfg.Namespace, secretName), + }, + }) + + // Update domain builder + convertOptions.IngressDomainCache.Valid[rule.Host] = domainBuilder + } + + return nil +} + +func (c *controller) ConvertHTTPRoute(convertOptions *common.ConvertOptions, wrapper *common.WrapperConfig) error { + // Canary ingress will be processed in the end. + if wrapper.AnnotationsConfig.IsCanary() { + convertOptions.CanaryIngresses = append(convertOptions.CanaryIngresses, wrapper) + return nil + } + + cfg := wrapper.Config + ingressV1, ok := cfg.Spec.(ingress.IngressSpec) + if !ok { + common.IncrementInvalidIngress(c.options.ClusterId, common.Unknown) + return fmt.Errorf("convert type is invalid in cluster %s", c.options.ClusterId) + } + if len(ingressV1.Rules) == 0 && ingressV1.Backend == nil { + common.IncrementInvalidIngress(c.options.ClusterId, common.EmptyRule) + return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId) + } + + if ingressV1.Backend != nil && ingressV1.Backend.ServiceName != "" { + convertOptions.HasDefaultBackend = true + } + + // In one ingress, we will limit the rule conflict. + // When the host, pathType, path of two rule are same, we think there is a conflict event. + definedRules := sets.NewSet() + + // But in across ingresses case, we will restrict this limit. + // When the host, path of two rule in different ingress are same, we think there is a conflict event. + var tempHostAndPath []string + for _, rule := range ingressV1.Rules { + if rule.HTTP == nil || len(rule.HTTP.Paths) == 0 { + IngressLog.Warnf("invalid ingress rule %s:%s for host %q in cluster %s, no paths defined", cfg.Namespace, cfg.Name, rule.Host, c.options.ClusterId) + continue + } + + wrapperVS, exist := convertOptions.VirtualServices[rule.Host] + if !exist { + wrapperVS = &common.WrapperVirtualService{ + VirtualService: &networking.VirtualService{ + Hosts: []string{rule.Host}, + }, + WrapperConfig: wrapper, + } + convertOptions.VirtualServices[rule.Host] = wrapperVS + } else { + wrapperVS.WrapperConfig.AnnotationsConfig.MergeHostIPAccessControlIfNotExist(wrapper.AnnotationsConfig.IPAccessControl) + } + + // Record the latest app root for per host. + redirect := wrapper.AnnotationsConfig.Redirect + if redirect != nil && redirect.AppRoot != "" { + wrapperVS.AppRoot = redirect.AppRoot + } + + wrapperHttpRoutes := make([]*common.WrapperHTTPRoute, 0, len(rule.HTTP.Paths)) + for _, httpPath := range rule.HTTP.Paths { + wrapperHttpRoute := &common.WrapperHTTPRoute{ + HTTPRoute: &networking.HTTPRoute{}, + WrapperConfig: wrapper, + Host: rule.Host, + ClusterId: c.options.ClusterId, + } + httpMatch := &networking.HTTPMatchRequest{} + + path := httpPath.Path + if wrapper.AnnotationsConfig.NeedRegexMatch() { + wrapperHttpRoute.OriginPathType = common.Regex + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Regex{Regex: httpPath.Path + ".*"}, + } + } else { + switch *httpPath.PathType { + case ingress.PathTypeExact: + wrapperHttpRoute.OriginPathType = common.Exact + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Exact{Exact: httpPath.Path}, + } + case ingress.PathTypePrefix: + wrapperHttpRoute.OriginPathType = common.Prefix + // borrow from implement of official istio code. + if path == "/" { + wrapperVS.ConfiguredDefaultBackend = true + // Optimize common case of / to not needed regex + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Prefix{Prefix: path}, + } + } else { + path = strings.TrimSuffix(path, "/") + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Regex{Regex: regexp.QuoteMeta(path) + common.PrefixMatchRegex}, + } + } + } + } + wrapperHttpRoute.OriginPath = path + wrapperHttpRoute.HTTPRoute.Match = []*networking.HTTPMatchRequest{httpMatch} + wrapperHttpRoute.HTTPRoute.Name = common.GenerateUniqueRouteName(wrapperHttpRoute) + + ingressRouteBuilder := convertOptions.IngressRouteCache.New(wrapperHttpRoute) + + // host and path overlay check across different ingresses. + hostAndPath := wrapperHttpRoute.BasePathFormat() + if preIngress, exist := convertOptions.HostAndPath2Ingress[hostAndPath]; exist { + ingressRouteBuilder.PreIngress = preIngress + ingressRouteBuilder.Event = common.DuplicatedRoute + } + tempHostAndPath = append(tempHostAndPath, hostAndPath) + + // Two duplicated rules in the same ingress. + if ingressRouteBuilder.Event == common.Normal { + pathFormat := wrapperHttpRoute.PathFormat() + if definedRules.Contains(pathFormat) { + ingressRouteBuilder.PreIngress = cfg + ingressRouteBuilder.Event = common.DuplicatedRoute + } + definedRules.Insert(pathFormat) + } + + // backend service check + var event common.Event + wrapperHttpRoute.HTTPRoute.Route, event = c.backendToRouteDestination(&httpPath.Backend, cfg.Namespace, ingressRouteBuilder) + + if ingressRouteBuilder.Event != common.Normal { + event = ingressRouteBuilder.Event + } + + if event != common.Normal { + common.IncrementInvalidIngress(c.options.ClusterId, event) + ingressRouteBuilder.Event = event + } else { + wrapperHttpRoutes = append(wrapperHttpRoutes, wrapperHttpRoute) + } + + convertOptions.IngressRouteCache.Add(ingressRouteBuilder) + } + + for _, item := range tempHostAndPath { + // We only record the first + if _, exist := convertOptions.HostAndPath2Ingress[item]; !exist { + convertOptions.HostAndPath2Ingress[item] = cfg + } + } + + old, f := convertOptions.HTTPRoutes[rule.Host] + if f { + old = append(old, wrapperHttpRoutes...) + convertOptions.HTTPRoutes[rule.Host] = old + } else { + convertOptions.HTTPRoutes[rule.Host] = wrapperHttpRoutes + } + + // Sort, exact -> prefix -> regex + routes := convertOptions.HTTPRoutes[rule.Host] + IngressLog.Debugf("routes of host %s is %v", rule.Host, routes) + common.SortHTTPRoutes(routes) + } + + return nil +} + +func (c *controller) ApplyDefaultBackend(convertOptions *common.ConvertOptions, wrapper *common.WrapperConfig) error { + if wrapper.AnnotationsConfig.IsCanary() { + return nil + } + + cfg := wrapper.Config + ingressV1Beta1, ok := cfg.Spec.(ingress.IngressSpec) + if !ok { + common.IncrementInvalidIngress(c.options.ClusterId, common.Unknown) + return fmt.Errorf("convert type is invalid in cluster %s", c.options.ClusterId) + } + + if ingressV1Beta1.Backend == nil { + return nil + } + + apply := func(host string, op func(vs *common.WrapperVirtualService, defaultRoute *common.WrapperHTTPRoute)) { + wirecardVS, exist := convertOptions.VirtualServices[host] + if !exist || !wirecardVS.ConfiguredDefaultBackend { + if !exist { + wirecardVS = &common.WrapperVirtualService{ + VirtualService: &networking.VirtualService{ + Hosts: []string{host}, + }, + WrapperConfig: wrapper, + } + } + + specDefaultBackend := c.createDefaultRoute(wrapper, ingressV1Beta1.Backend, "*") + if specDefaultBackend != nil { + convertOptions.VirtualServices[host] = wirecardVS + op(wirecardVS, specDefaultBackend) + } + } + } + + // First process * + apply("*", func(_ *common.WrapperVirtualService, defaultRoute *common.WrapperHTTPRoute) { + var hasFound bool + for _, httpRoute := range convertOptions.HTTPRoutes["*"] { + if httpRoute.OriginPathType == common.Prefix && httpRoute.OriginPath == "/" { + hasFound = true + convertOptions.IngressRouteCache.Delete(httpRoute) + + httpRoute.HTTPRoute = defaultRoute.HTTPRoute + httpRoute.WrapperConfig = defaultRoute.WrapperConfig + convertOptions.IngressRouteCache.NewAndAdd(httpRoute) + } + } + if !hasFound { + convertOptions.HTTPRoutes["*"] = append(convertOptions.HTTPRoutes["*"], defaultRoute) + } + }) + + for _, rule := range ingressV1Beta1.Rules { + if rule.Host == "*" { + continue + } + + apply(rule.Host, func(vs *common.WrapperVirtualService, defaultRoute *common.WrapperHTTPRoute) { + convertOptions.HTTPRoutes[rule.Host] = append(convertOptions.HTTPRoutes[rule.Host], defaultRoute) + vs.ConfiguredDefaultBackend = true + + convertOptions.IngressRouteCache.NewAndAdd(defaultRoute) + }) + } + + return nil +} + +func (c *controller) ApplyCanaryIngress(convertOptions *common.ConvertOptions, wrapper *common.WrapperConfig) error { + byHeader, byWeight := wrapper.AnnotationsConfig.CanaryKind() + + cfg := wrapper.Config + ingressV1Beta, ok := cfg.Spec.(ingress.IngressSpec) + if !ok { + common.IncrementInvalidIngress(c.options.ClusterId, common.Unknown) + return fmt.Errorf("convert type is invalid in cluster %s", c.options.ClusterId) + } + if len(ingressV1Beta.Rules) == 0 && ingressV1Beta.Backend == nil { + common.IncrementInvalidIngress(c.options.ClusterId, common.EmptyRule) + return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId) + } + + for _, rule := range ingressV1Beta.Rules { + if rule.HTTP == nil || len(rule.HTTP.Paths) == 0 { + IngressLog.Warnf("invalid ingress rule %s:%s for host %q in cluster %s, no paths defined", cfg.Namespace, cfg.Name, rule.Host, c.options.ClusterId) + continue + } + + routes, exist := convertOptions.HTTPRoutes[rule.Host] + if !exist { + continue + } + + for _, httpPath := range rule.HTTP.Paths { + path := httpPath.Path + + canary := &common.WrapperHTTPRoute{ + HTTPRoute: &networking.HTTPRoute{}, + WrapperConfig: wrapper, + Host: rule.Host, + ClusterId: c.options.ClusterId, + } + httpMatch := &networking.HTTPMatchRequest{} + + if wrapper.AnnotationsConfig.NeedRegexMatch() { + canary.OriginPathType = common.Regex + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Regex{Regex: httpPath.Path + ".*"}, + } + } else { + switch *httpPath.PathType { + case ingress.PathTypeExact: + canary.OriginPathType = common.Exact + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Exact{Exact: httpPath.Path}, + } + case ingress.PathTypePrefix: + canary.OriginPathType = common.Prefix + // borrow from implement of official istio code. + if path == "/" { + // Optimize common case of / to not needed regex + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Prefix{Prefix: path}, + } + } else { + path = strings.TrimSuffix(path, "/") + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Regex{Regex: regexp.QuoteMeta(path) + common.PrefixMatchRegex}, + } + } + } + } + canary.OriginPath = path + canary.HTTPRoute.Match = []*networking.HTTPMatchRequest{httpMatch} + canary.HTTPRoute.Name = common.GenerateUniqueRouteName(canary) + + ingressRouteBuilder := convertOptions.IngressRouteCache.New(canary) + // backend service check + var event common.Event + canary.HTTPRoute.Route, event = c.backendToRouteDestination(&httpPath.Backend, cfg.Namespace, ingressRouteBuilder) + if event != common.Normal { + common.IncrementInvalidIngress(c.options.ClusterId, event) + ingressRouteBuilder.Event = event + convertOptions.IngressRouteCache.Add(ingressRouteBuilder) + continue + } + + canaryConfig := wrapper.AnnotationsConfig.Canary + if byWeight { + canary.HTTPRoute.Route[0].Weight = int32(canaryConfig.Weight) + } + + pos := 0 + var targetRoute *common.WrapperHTTPRoute + for _, route := range routes { + if isCanaryRoute(canary, route) { + targetRoute = route + // Header, Cookie + if byHeader { + IngressLog.Debug("Insert canary route by header") + annotations.ApplyByHeader(canary.HTTPRoute, route.HTTPRoute, canary.WrapperConfig.AnnotationsConfig) + canary.HTTPRoute.Name = common.GenerateUniqueRouteName(canary) + } else { + IngressLog.Debug("Merge canary route by weight") + if route.WeightTotal == 0 { + route.WeightTotal = int32(canaryConfig.WeightTotal) + } + annotations.ApplyByWeight(canary.HTTPRoute, route.HTTPRoute, canary.WrapperConfig.AnnotationsConfig) + } + + break + } + pos += 1 + } + + IngressLog.Debugf("Canary route is %v", canary) + if targetRoute == nil { + continue + } + + if byHeader { + // Inherit policy from normal route + canary.WrapperConfig.AnnotationsConfig.Auth = targetRoute.WrapperConfig.AnnotationsConfig.Auth + + routes = append(routes[:pos+1], routes[pos:]...) + routes[pos] = canary + convertOptions.HTTPRoutes[rule.Host] = routes + + // Recreate route name. + ingressRouteBuilder.RouteName = common.GenerateUniqueRouteName(canary) + convertOptions.IngressRouteCache.Add(ingressRouteBuilder) + } else { + convertOptions.IngressRouteCache.Update(targetRoute) + } + } + } + return nil +} + +func (c *controller) ConvertTrafficPolicy(convertOptions *common.ConvertOptions, wrapper *common.WrapperConfig) error { + if !wrapper.AnnotationsConfig.NeedTrafficPolicy() { + return nil + } + + cfg := wrapper.Config + ingressV1Beta, ok := cfg.Spec.(ingress.IngressSpec) + if !ok { + common.IncrementInvalidIngress(c.options.ClusterId, common.Unknown) + return fmt.Errorf("convert type is invalid in cluster %s", c.options.ClusterId) + } + if len(ingressV1Beta.Rules) == 0 && ingressV1Beta.Backend == nil { + common.IncrementInvalidIngress(c.options.ClusterId, common.EmptyRule) + return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId) + } + + if ingressV1Beta.Backend != nil { + serviceKey, err := c.createServiceKey(ingressV1Beta.Backend, cfg.Namespace) + if err != nil { + IngressLog.Errorf("ignore default service %s within ingress %s/%s", serviceKey.Name, cfg.Namespace, cfg.Name) + } else { + if _, exist := convertOptions.Service2TrafficPolicy[serviceKey]; !exist { + convertOptions.Service2TrafficPolicy[serviceKey] = &common.WrapperTrafficPolicy{ + TrafficPolicy: &networking.TrafficPolicy_PortTrafficPolicy{ + Port: &networking.PortSelector{ + Number: uint32(serviceKey.Port), + }, + }, + WrapperConfig: wrapper, + } + } + } + } + + for _, rule := range ingressV1Beta.Rules { + if rule.HTTP == nil || len(rule.HTTP.Paths) == 0 { + continue + } + + for _, httpPath := range rule.HTTP.Paths { + if httpPath.Backend.ServiceName == "" { + continue + } + + serviceKey, err := c.createServiceKey(&httpPath.Backend, cfg.Namespace) + if err != nil { + IngressLog.Errorf("ignore service %s within ingress %s/%s", serviceKey.Name, cfg.Namespace, cfg.Name) + continue + } + + if _, exist := convertOptions.Service2TrafficPolicy[serviceKey]; exist { + continue + } + + convertOptions.Service2TrafficPolicy[serviceKey] = &common.WrapperTrafficPolicy{ + TrafficPolicy: &networking.TrafficPolicy_PortTrafficPolicy{ + Port: &networking.PortSelector{ + Number: uint32(serviceKey.Port), + }, + }, + WrapperConfig: wrapper, + } + } + } + + return nil +} + +func (c *controller) createDefaultRoute(wrapper *common.WrapperConfig, backend *ingress.IngressBackend, host string) *common.WrapperHTTPRoute { + if backend == nil || backend.ServiceName == "" { + return nil + } + + namespace := wrapper.Config.Namespace + + port := &networking.PortSelector{} + if backend.ServicePort.Type == intstr.Int { + port.Number = uint32(backend.ServicePort.IntVal) + } else { + resolvedPort, err := resolveNamedPort(backend, namespace, c.serviceLister) + if err != nil { + return nil + } + port.Number = uint32(resolvedPort) + } + + routeDestination := []*networking.HTTPRouteDestination{ + { + Destination: &networking.Destination{ + Host: util.CreateServiceFQDN(namespace, backend.ServiceName), + Port: port, + }, + Weight: 100, + }, + } + + route := &common.WrapperHTTPRoute{ + HTTPRoute: &networking.HTTPRoute{ + Route: routeDestination, + }, + WrapperConfig: wrapper, + ClusterId: c.options.ClusterId, + Host: host, + IsDefaultBackend: true, + OriginPathType: common.Prefix, + OriginPath: "/", + } + route.HTTPRoute.Name = common.GenerateUniqueRouteNameWithSuffix(route, "default") + + return route +} + +func (c *controller) createServiceKey(service *ingress.IngressBackend, namespace string) (common.ServiceKey, error) { + serviceKey := common.ServiceKey{} + if service.ServiceName == "" { + return serviceKey, errors.New("service name is empty") + } + + var port int32 + var err error + if service.ServicePort.Type == intstr.Int { + port = service.ServicePort.IntVal + } else { + port, err = resolveNamedPort(service, namespace, c.serviceLister) + if err != nil { + return serviceKey, err + } + } + + return common.ServiceKey{ + Namespace: namespace, + Name: service.ServiceName, + Port: port, + }, nil +} + +func isCanaryRoute(canary, route *common.WrapperHTTPRoute) bool { + return !strings.HasSuffix(route.HTTPRoute.Name, "-canary") && canary.OriginPath == route.OriginPath && + canary.OriginPathType == route.OriginPathType +} + +func (c *controller) backendToRouteDestination(backend *ingress.IngressBackend, namespace string, + builder *common.IngressRouteBuilder) ([]*networking.HTTPRouteDestination, common.Event) { + if backend == nil { + return nil, common.InvalidBackendService + } + + if backend.ServiceName == "" { + return nil, common.InvalidBackendService + } + + builder.PortName = backend.ServicePort.StrVal + + port := &networking.PortSelector{} + if backend.ServicePort.Type == intstr.Int { + port.Number = uint32(backend.ServicePort.IntVal) + } else { + resolvedPort, err := resolveNamedPort(backend, namespace, c.serviceLister) + if err != nil { + return nil, common.PortNameResolveError + } + port.Number = uint32(resolvedPort) + } + + builder.ServiceList = []model.BackendService{ + { + Namespace: namespace, + Name: backend.ServiceName, + Port: port.Number, + Weight: 100, + }, + } + + return []*networking.HTTPRouteDestination{ + { + Destination: &networking.Destination{ + Host: util.CreateServiceFQDN(namespace, backend.ServiceName), + Port: port, + }, + Weight: 100, + }, + }, common.Normal +} + +func resolveNamedPort(backend *ingress.IngressBackend, namespace string, serviceLister listerv1.ServiceLister) (int32, error) { + svc, err := serviceLister.Services(namespace).Get(backend.ServiceName) + if err != nil { + return 0, err + } + for _, port := range svc.Spec.Ports { + if port.Name == backend.ServicePort.StrVal { + return port.Port, nil + } + } + return 0, common.ErrNotFound +} + +func (c *controller) shouldProcessIngressWithClass(ingress *ingress.Ingress, ingressClass *ingress.IngressClass) bool { + if class, exists := ingress.Annotations[kube.IngressClassAnnotation]; exists { + switch c.options.IngressClass { + case "": + return true + case common.DefaultIngressClass: + return class == "" || class == common.DefaultIngressClass + default: + return c.options.IngressClass == class + } + } else if ingressClass != nil { + switch c.options.IngressClass { + case "": + return true + default: + return c.options.IngressClass == ingressClass.Name + } + } else { + ingressClassName := ingress.Spec.IngressClassName + switch c.options.IngressClass { + case "": + return true + case common.DefaultIngressClass: + return ingressClassName == nil || *ingressClassName == "" || + *ingressClassName == common.DefaultIngressClass + default: + return ingressClassName != nil && *ingressClassName == c.options.IngressClass + } + } +} + +func (c *controller) shouldProcessIngress(i *ingress.Ingress) (bool, error) { + var class *ingress.IngressClass + if c.classes != nil && i.Spec.IngressClassName != nil { + classCache, err := c.classes.Lister().Get(*i.Spec.IngressClassName) + if err != nil && !kerrors.IsNotFound(err) { + return false, fmt.Errorf("failed to get ingress class %v from cluster %s: %v", i.Spec.IngressClassName, c.options.ClusterId, err) + } + class = classCache + } + + // first check ingress class + if c.shouldProcessIngressWithClass(i, class) { + // then check namespace + switch c.options.WatchNamespace { + case "": + return true, nil + default: + return c.options.WatchNamespace == i.Namespace, nil + } + } + + return false, nil +} + +// shouldProcessIngressUpdate checks whether we should renotify registered handlers about an update event +func (c *controller) shouldProcessIngressUpdate(ing *ingress.Ingress) (bool, error) { + shouldProcess, err := c.shouldProcessIngress(ing) + if err != nil { + return false, err + } + + namespacedName := ing.Namespace + "/" + ing.Name + if shouldProcess { + // record processed ingress + c.mutex.Lock() + preConfig, exist := c.ingresses[namespacedName] + c.ingresses[namespacedName] = ing + c.mutex.Unlock() + + // We only care about annotations, labels and spec. + if exist { + if !reflect.DeepEqual(preConfig.Annotations, ing.Annotations) { + IngressLog.Debugf("Annotations of ingress %s changed, should process.", namespacedName) + return true, nil + } + if !reflect.DeepEqual(preConfig.Labels, ing.Labels) { + IngressLog.Debugf("Labels of ingress %s changed, should process.", namespacedName) + return true, nil + } + if !reflect.DeepEqual(preConfig.Spec, ing.Spec) { + IngressLog.Debugf("Spec of ingress %s changed, should process.", namespacedName) + return true, nil + } + + return false, nil + } + IngressLog.Debugf("First receive relative ingress %s, should process.", namespacedName) + return true, nil + } + + c.mutex.Lock() + _, preProcessed := c.ingresses[namespacedName] + // previous processed but should not currently, delete it + if preProcessed && !shouldProcess { + delete(c.ingresses, namespacedName) + } + c.mutex.Unlock() + + return preProcessed, nil +} + +// setDefaultMSEIngressOptionalField sets a default value for optional fields when is not defined. +func setDefaultMSEIngressOptionalField(ing *ingress.Ingress) { + for idx, tls := range ing.Spec.TLS { + if len(tls.Hosts) == 0 { + ing.Spec.TLS[idx].Hosts = []string{common.DefaultHost} + } + } + + for idx, rule := range ing.Spec.Rules { + if rule.IngressRuleValue.HTTP == nil { + continue + } + + if rule.Host == "" { + ing.Spec.Rules[idx].Host = common.DefaultHost + } + + for innerIdx := range rule.IngressRuleValue.HTTP.Paths { + p := &rule.IngressRuleValue.HTTP.Paths[innerIdx] + + if p.Path == "" { + p.Path = common.DefaultPath + } + + if p.PathType == nil { + p.PathType = &defaultPathType + // for old k8s version + if !annotations.NeedRegexMatch(ing.Annotations) { + if strings.HasSuffix(p.Path, ".*") { + p.Path = strings.TrimSuffix(p.Path, ".*") + } + + if strings.HasSuffix(p.Path, "/*") { + p.Path = strings.TrimSuffix(p.Path, "/*") + } + } + } + + if *p.PathType == ingress.PathTypeImplementationSpecific { + p.PathType = &defaultPathType + } + } + } +} diff --git a/ingress/kube/ingress/controller_test.go b/ingress/kube/ingress/controller_test.go new file mode 100644 index 000000000..ffca0c1a4 --- /dev/null +++ b/ingress/kube/ingress/controller_test.go @@ -0,0 +1,78 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingress + +import ( + "testing" + + "k8s.io/api/networking/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/alibaba/higress/ingress/kube/common" +) + +func TestShouldProcessIngressUpdate(t *testing.T) { + c := controller{ + options: common.Options{ + IngressClass: "mse", + }, + ingresses: make(map[string]*v1beta1.Ingress), + } + + ingressClass := "mse" + + ingress1 := &v1beta1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-1", + }, + Spec: v1beta1.IngressSpec{ + IngressClassName: &ingressClass, + Rules: []v1beta1.IngressRule{ + { + Host: "test.com", + IngressRuleValue: v1beta1.IngressRuleValue{ + HTTP: &v1beta1.HTTPIngressRuleValue{ + Paths: []v1beta1.HTTPIngressPath{ + { + Path: "/test", + }, + }, + }, + }, + }, + }, + }, + } + + should, _ := c.shouldProcessIngressUpdate(ingress1) + if !should { + t.Fatal("should be true") + } + + ingress2 := *ingress1 + should, _ = c.shouldProcessIngressUpdate(&ingress2) + if should { + t.Fatal("should be false") + } + + ingress3 := *ingress1 + ingress3.Annotations = map[string]string{ + "test": "true", + } + should, _ = c.shouldProcessIngressUpdate(&ingress3) + if !should { + t.Fatal("should be true") + } +} diff --git a/ingress/kube/ingress/status.go b/ingress/kube/ingress/status.go new file mode 100644 index 000000000..a5c4f808d --- /dev/null +++ b/ingress/kube/ingress/status.go @@ -0,0 +1,133 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingress + +import ( + "context" + "reflect" + "sort" + "time" + + kubelib "istio.io/istio/pkg/kube" + coreV1 "k8s.io/api/core/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + listerv1 "k8s.io/client-go/listers/core/v1" + ingresslister "k8s.io/client-go/listers/networking/v1beta1" + "k8s.io/client-go/tools/cache" + + "github.com/alibaba/higress/ingress/kube/common" + . "github.com/alibaba/higress/ingress/log" +) + +// statusSyncer keeps the status IP in each Ingress resource updated +type statusSyncer struct { + client kubernetes.Interface + controller *controller + + watchedNamespace string + + ingressLister ingresslister.IngressLister + ingressClassLister ingresslister.IngressClassLister + // search service in the mse vpc + serviceLister listerv1.ServiceLister +} + +// newStatusSyncer creates a new instance +func newStatusSyncer(localKubeClient, client kubelib.Client, controller *controller, namespace string) *statusSyncer { + return &statusSyncer{ + client: client, + controller: controller, + watchedNamespace: namespace, + ingressLister: client.KubeInformer().Networking().V1beta1().Ingresses().Lister(), + ingressClassLister: client.KubeInformer().Networking().V1beta1().IngressClasses().Lister(), + // search service in the mse vpc + serviceLister: localKubeClient.KubeInformer().Core().V1().Services().Lister(), + } +} + +func (s *statusSyncer) run(stopCh <-chan struct{}) { + cache.WaitForCacheSync(stopCh, s.controller.HasSynced) + + ticker := time.NewTicker(common.DefaultStatusUpdateInterval) + for { + select { + case <-stopCh: + ticker.Stop() + return + case <-ticker.C: + if err := s.runUpdateStatus(); err != nil { + IngressLog.Errorf("update status task fail, err %v", err) + } + } + } +} + +func (s *statusSyncer) runUpdateStatus() error { + svcList, err := s.serviceLister.Services(s.watchedNamespace).List(common.SvcLabelSelector) + if err != nil { + return err + } + + IngressLog.Debugf("found number %d of svc", len(svcList)) + + lbStatusList := common.GetLbStatusList(svcList) + if len(lbStatusList) == 0 { + return nil + } + + return s.updateStatus(lbStatusList) +} + +// updateStatus updates ingress status with the list of IP +func (s *statusSyncer) updateStatus(status []coreV1.LoadBalancerIngress) error { + ingressList, err := s.ingressLister.List(labels.Everything()) + if err != nil { + return err + } + for _, ingress := range ingressList { + shouldTarget, err := s.controller.shouldProcessIngress(ingress) + if err != nil { + IngressLog.Warnf("error determining whether should target ingress %s/%s within cluster %s for status update: %v", + ingress.Namespace, ingress.Name, s.controller.options.ClusterId, err) + return err + } + + if !shouldTarget { + continue + } + + curIPs := ingress.Status.LoadBalancer.Ingress + sort.SliceStable(curIPs, common.SortLbIngressList(curIPs)) + + if reflect.DeepEqual(status, curIPs) { + IngressLog.Debugf("skipping update of Ingress %v/%v within cluster %s (no change)", + ingress.Namespace, ingress.Name, s.controller.options.ClusterId) + continue + } + + ingress.Status.LoadBalancer.Ingress = status + IngressLog.Infof("Update Ingress %v/%v within cluster %s status", + ingress.Namespace, ingress.Name, s.controller.options.ClusterId) + _, err = s.client.NetworkingV1beta1().Ingresses(ingress.Namespace).UpdateStatus(context.TODO(), ingress, metaV1.UpdateOptions{}) + if err != nil { + IngressLog.Warnf("error updating ingress %s/%s within cluster %s status: %v", + ingress.Namespace, ingress.Name, s.controller.options.ClusterId, err) + } + } + + return nil +} diff --git a/ingress/kube/ingressv1/controller.go b/ingress/kube/ingressv1/controller.go new file mode 100644 index 000000000..b43872e5e --- /dev/null +++ b/ingress/kube/ingressv1/controller.go @@ -0,0 +1,1158 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingressv1 + +import ( + "errors" + "fmt" + "path" + "reflect" + "regexp" + "strings" + "sync" + "time" + + "github.com/hashicorp/go-multierror" + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/model" + "istio.io/istio/pilot/pkg/model/credentials" + "istio.io/istio/pilot/pkg/serviceregistry/kube" + "istio.io/istio/pilot/pkg/util/sets" + "istio.io/istio/pkg/config" + "istio.io/istio/pkg/config/constants" + "istio.io/istio/pkg/config/protocol" + "istio.io/istio/pkg/config/schema/gvk" + kubeclient "istio.io/istio/pkg/kube" + "istio.io/istio/pkg/kube/controllers" + ingress "k8s.io/api/networking/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + networkingv1 "k8s.io/client-go/informers/networking/v1" + listerv1 "k8s.io/client-go/listers/core/v1" + networkinglister "k8s.io/client-go/listers/networking/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + "github.com/alibaba/higress/ingress/kube/annotations" + "github.com/alibaba/higress/ingress/kube/common" + "github.com/alibaba/higress/ingress/kube/secret" + "github.com/alibaba/higress/ingress/kube/util" + . "github.com/alibaba/higress/ingress/log" +) + +var ( + _ common.IngressController = &controller{} + + // follow specification of ingress-nginx + defaultPathType = ingress.PathTypePrefix +) + +type controller struct { + queue workqueue.RateLimitingInterface + virtualServiceHandlers []model.EventHandler + gatewayHandlers []model.EventHandler + destinationRuleHandlers []model.EventHandler + envoyFilterHandlers []model.EventHandler + + options common.Options + + mutex sync.RWMutex + // key: namespace/name + ingresses map[string]*ingress.Ingress + + ingressInformer cache.SharedInformer + ingressLister networkinglister.IngressLister + serviceInformer cache.SharedInformer + serviceLister listerv1.ServiceLister + classes networkingv1.IngressClassInformer + + secretController secret.Controller + + statusSyncer *statusSyncer +} + +// NewController creates a new Kubernetes controller +func NewController(localKubeClient, client kubeclient.Client, options common.Options, secretController secret.Controller) common.IngressController { + q := workqueue.NewRateLimitingQueue(workqueue.DefaultItemBasedRateLimiter()) + + ingressInformer := client.KubeInformer().Networking().V1().Ingresses() + serviceInformer := client.KubeInformer().Core().V1().Services() + + classes := client.KubeInformer().Networking().V1().IngressClasses() + classes.Informer() + + c := &controller{ + options: options, + queue: q, + ingresses: make(map[string]*ingress.Ingress), + ingressInformer: ingressInformer.Informer(), + ingressLister: ingressInformer.Lister(), + classes: classes, + serviceInformer: serviceInformer.Informer(), + serviceLister: serviceInformer.Lister(), + secretController: secretController, + } + + handler := controllers.LatestVersionHandlerFuncs(controllers.EnqueueForSelf(q)) + c.ingressInformer.AddEventHandler(handler) + + if options.EnableStatus { + c.statusSyncer = newStatusSyncer(localKubeClient, client, c, options.SystemNamespace) + } else { + IngressLog.Infof("Disable status update for cluster %s", options.ClusterId) + } + + return c +} + +func (c *controller) ServiceLister() listerv1.ServiceLister { + return c.serviceLister +} + +func (c *controller) SecretLister() listerv1.SecretLister { + return c.secretController.Lister() +} + +func (c *controller) Run(stop <-chan struct{}) { + if c.statusSyncer != nil { + go c.statusSyncer.run(stop) + } + go c.secretController.Run(stop) + + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + + if !cache.WaitForCacheSync(stop, c.HasSynced) { + IngressLog.Errorf("Failed to sync ingress controller cache for cluster %s", c.options.ClusterId) + return + } + go wait.Until(c.worker, time.Second, stop) + <-stop +} + +func (c *controller) worker() { + for c.processNextWorkItem() { + } +} + +func (c *controller) processNextWorkItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + ingressNamespacedName := key.(types.NamespacedName) + IngressLog.Debugf("ingress %s push to queue", ingressNamespacedName) + if err := c.onEvent(ingressNamespacedName); err != nil { + IngressLog.Errorf("error processing ingress item (%v) (retrying): %v, cluster: %s", key, err, c.options.ClusterId) + c.queue.AddRateLimited(key) + } else { + c.queue.Forget(key) + } + return true +} + +func (c *controller) onEvent(namespacedName types.NamespacedName) error { + event := model.EventUpdate + ing, err := c.ingressLister.Ingresses(namespacedName.Namespace).Get(namespacedName.Name) + if err != nil { + if kerrors.IsNotFound(err) { + event = model.EventDelete + c.mutex.Lock() + ing = c.ingresses[namespacedName.String()] + delete(c.ingresses, namespacedName.String()) + c.mutex.Unlock() + } else { + return err + } + } + + // ingress deleted, and it is not processed before + if ing == nil { + return nil + } + + IngressLog.Debugf("ingress: %s, event: %s", namespacedName, event) + + // we should check need process only when event is not delete, + // if it is delete event, and previously processed, we need to process too. + if event != model.EventDelete { + shouldProcess, err := c.shouldProcessIngressUpdate(ing) + if err != nil { + return err + } + if !shouldProcess { + IngressLog.Infof("no need process, ingress %s", namespacedName) + return nil + } + } + + drmetadata := config.Meta{ + Name: ing.Name + "-" + "destinationrule", + Namespace: ing.Namespace, + GroupVersionKind: gvk.DestinationRule, + // Set this label so that we do not compare configs and just push. + Labels: map[string]string{constants.AlwaysPushLabel: "true"}, + } + vsmetadata := config.Meta{ + Name: ing.Name + "-" + "virtualservice", + Namespace: ing.Namespace, + GroupVersionKind: gvk.VirtualService, + // Set this label so that we do not compare configs and just push. + Labels: map[string]string{constants.AlwaysPushLabel: "true"}, + } + efmetadata := config.Meta{ + Name: ing.Name + "-" + "envoyfilter", + Namespace: ing.Namespace, + GroupVersionKind: gvk.EnvoyFilter, + // Set this label so that we do not compare configs and just push. + Labels: map[string]string{constants.AlwaysPushLabel: "true"}, + } + gatewaymetadata := config.Meta{ + Name: ing.Name + "-" + "gateway", + Namespace: ing.Namespace, + GroupVersionKind: gvk.Gateway, + // Set this label so that we do not compare configs and just push. + Labels: map[string]string{constants.AlwaysPushLabel: "true"}, + } + + for _, f := range c.destinationRuleHandlers { + f(config.Config{Meta: drmetadata}, config.Config{Meta: drmetadata}, event) + } + + for _, f := range c.virtualServiceHandlers { + f(config.Config{Meta: vsmetadata}, config.Config{Meta: vsmetadata}, event) + } + + for _, f := range c.envoyFilterHandlers { + f(config.Config{Meta: efmetadata}, config.Config{Meta: efmetadata}, event) + } + + for _, f := range c.gatewayHandlers { + f(config.Config{Meta: gatewaymetadata}, config.Config{Meta: gatewaymetadata}, event) + } + + return nil +} + +func (c *controller) RegisterEventHandler(kind config.GroupVersionKind, f model.EventHandler) { + switch kind { + case gvk.VirtualService: + c.virtualServiceHandlers = append(c.virtualServiceHandlers, f) + case gvk.Gateway: + c.gatewayHandlers = append(c.gatewayHandlers, f) + case gvk.DestinationRule: + c.destinationRuleHandlers = append(c.destinationRuleHandlers, f) + case gvk.EnvoyFilter: + c.envoyFilterHandlers = append(c.envoyFilterHandlers, f) + } +} + +func (c *controller) SetWatchErrorHandler(handler func(r *cache.Reflector, err error)) error { + var errs error + if err := c.serviceInformer.SetWatchErrorHandler(handler); err != nil { + errs = multierror.Append(errs, err) + } + if err := c.ingressInformer.SetWatchErrorHandler(handler); err != nil { + errs = multierror.Append(errs, err) + } + if err := c.secretController.Informer().SetWatchErrorHandler(handler); err != nil { + errs = multierror.Append(errs, err) + } + if err := c.classes.Informer().SetWatchErrorHandler(handler); err != nil { + errs = multierror.Append(errs, err) + } + return errs +} + +func (c *controller) HasSynced() bool { + return c.ingressInformer.HasSynced() && c.serviceInformer.HasSynced() && + c.classes.Informer().HasSynced() && + c.secretController.HasSynced() +} + +func (c *controller) List() []config.Config { + out := make([]config.Config, 0, len(c.ingresses)) + + for _, raw := range c.ingressInformer.GetStore().List() { + ing, ok := raw.(*ingress.Ingress) + if !ok { + continue + } + + if should, err := c.shouldProcessIngress(ing); !should || err != nil { + continue + } + + copiedConfig := ing.DeepCopy() + setDefaultMSEIngressOptionalField(copiedConfig) + + outConfig := config.Config{ + Meta: config.Meta{ + Name: copiedConfig.Name, + Namespace: copiedConfig.Namespace, + Annotations: common.CreateOrUpdateAnnotations(copiedConfig.Annotations, c.options), + Labels: copiedConfig.Labels, + CreationTimestamp: copiedConfig.CreationTimestamp.Time, + }, + Spec: copiedConfig.Spec, + } + + out = append(out, outConfig) + } + + common.RecordIngressNumber(c.options.ClusterId, len(out)) + return out +} + +func extractTLSSecretName(host string, tls []ingress.IngressTLS) string { + if len(tls) == 0 { + return "" + } + + for _, t := range tls { + match := false + for _, h := range t.Hosts { + if h == host { + match = true + } + } + + if match { + return t.SecretName + } + } + + return "" +} + +func (c *controller) ConvertGateway(convertOptions *common.ConvertOptions, wrapper *common.WrapperConfig) error { + // Ignore canary config. + if wrapper.AnnotationsConfig.IsCanary() { + return nil + } + + cfg := wrapper.Config + ingressV1, ok := cfg.Spec.(ingress.IngressSpec) + if !ok { + common.IncrementInvalidIngress(c.options.ClusterId, common.Unknown) + return fmt.Errorf("convert type is invalid in cluster %s", c.options.ClusterId) + } + if len(ingressV1.Rules) == 0 && ingressV1.DefaultBackend == nil { + common.IncrementInvalidIngress(c.options.ClusterId, common.EmptyRule) + return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId) + } + + for _, rule := range ingressV1.Rules { + cleanHost := common.CleanHost(rule.Host) + // Need create builder for every rule. + domainBuilder := &common.IngressDomainBuilder{ + ClusterId: c.options.ClusterId, + Protocol: common.HTTP, + Host: rule.Host, + Ingress: cfg, + Event: common.Normal, + } + + // Extract the previous gateway and builder + wrapperGateway, exist := convertOptions.Gateways[rule.Host] + preDomainBuilder, _ := convertOptions.IngressDomainCache.Valid[rule.Host] + if !exist { + wrapperGateway = &common.WrapperGateway{ + Gateway: &networking.Gateway{ + Selector: map[string]string{ + c.options.GatewaySelectorKey: c.options.GatewaySelectorValue, + }, + }, + WrapperConfig: wrapper, + ClusterId: c.options.ClusterId, + Host: rule.Host, + } + wrapperGateway.Gateway.Servers = append(wrapperGateway.Gateway.Servers, &networking.Server{ + Port: &networking.Port{ + Number: 80, + Protocol: string(protocol.HTTP), + Name: common.CreateConvertedName("http-80-ingress", c.options.ClusterId, cfg.Namespace, cfg.Name, cleanHost), + }, + Hosts: []string{rule.Host}, + }) + + // Add new gateway, builder + convertOptions.Gateways[rule.Host] = wrapperGateway + convertOptions.IngressDomainCache.Valid[rule.Host] = domainBuilder + } else { + // Fallback to get downstream tls from current ingress. + if wrapperGateway.WrapperConfig.AnnotationsConfig.DownstreamTLS == nil { + wrapperGateway.WrapperConfig.AnnotationsConfig.DownstreamTLS = wrapper.AnnotationsConfig.DownstreamTLS + } + } + + // There are no tls settings, so just skip. + if len(ingressV1.TLS) == 0 { + continue + } + + // Get tls secret matching the rule host + secretName := extractTLSSecretName(rule.Host, ingressV1.TLS) + if secretName == "" { + // There no matching secret, so just skip. + continue + } + + domainBuilder.Protocol = common.HTTPS + domainBuilder.SecretName = path.Join(c.options.ClusterId, cfg.Namespace, secretName) + + // There is a matching secret and the gateway has already a tls secret. + // We should report the duplicated tls secret event. + if wrapperGateway.IsHTTPS() { + domainBuilder.Event = common.DuplicatedTls + domainBuilder.PreIngress = preDomainBuilder.Ingress + convertOptions.IngressDomainCache.Invalid = append(convertOptions.IngressDomainCache.Invalid, + domainBuilder.Build()) + continue + } + + // Append https server + wrapperGateway.Gateway.Servers = append(wrapperGateway.Gateway.Servers, &networking.Server{ + Port: &networking.Port{ + Number: 443, + Protocol: string(protocol.HTTPS), + Name: common.CreateConvertedName("https-443-ingress", c.options.ClusterId, cfg.Namespace, cfg.Name, cleanHost), + }, + Hosts: []string{rule.Host}, + Tls: &networking.ServerTLSSettings{ + Mode: networking.ServerTLSSettings_SIMPLE, + CredentialName: credentials.ToKubernetesIngressResource(c.options.RawClusterId, cfg.Namespace, secretName), + }, + }) + + // Update domain builder + convertOptions.IngressDomainCache.Valid[rule.Host] = domainBuilder + } + + return nil +} + +func (c *controller) ConvertHTTPRoute(convertOptions *common.ConvertOptions, wrapper *common.WrapperConfig) error { + // Canary ingress will be processed in the end. + if wrapper.AnnotationsConfig.IsCanary() { + convertOptions.CanaryIngresses = append(convertOptions.CanaryIngresses, wrapper) + return nil + } + + cfg := wrapper.Config + ingressV1, ok := cfg.Spec.(ingress.IngressSpec) + if !ok { + common.IncrementInvalidIngress(c.options.ClusterId, common.Unknown) + return fmt.Errorf("convert type is invalid in cluster %s", c.options.ClusterId) + } + if len(ingressV1.Rules) == 0 && ingressV1.DefaultBackend == nil { + common.IncrementInvalidIngress(c.options.ClusterId, common.EmptyRule) + return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId) + } + + if ingressV1.DefaultBackend != nil && ingressV1.DefaultBackend.Service != nil && + ingressV1.DefaultBackend.Service.Name != "" { + convertOptions.HasDefaultBackend = true + } + + // In one ingress, we will limit the rule conflict. + // When the host, pathType, path of two rule are same, we think there is a conflict event. + definedRules := sets.NewSet() + + // But in across ingresses case, we will restrict this limit. + // When the host, path of two rule in different ingress are same, we think there is a conflict event. + var tempHostAndPath []string + for _, rule := range ingressV1.Rules { + if rule.HTTP == nil || len(rule.HTTP.Paths) == 0 { + IngressLog.Warnf("invalid ingress rule %s:%s for host %q in cluster %s, no paths defined", cfg.Namespace, cfg.Name, rule.Host, c.options.ClusterId) + continue + } + + wrapperVS, exist := convertOptions.VirtualServices[rule.Host] + if !exist { + wrapperVS = &common.WrapperVirtualService{ + VirtualService: &networking.VirtualService{ + Hosts: []string{rule.Host}, + }, + WrapperConfig: wrapper, + } + convertOptions.VirtualServices[rule.Host] = wrapperVS + } else { + wrapperVS.WrapperConfig.AnnotationsConfig.MergeHostIPAccessControlIfNotExist(wrapper.AnnotationsConfig.IPAccessControl) + } + + // Record the latest app root for per host. + redirect := wrapper.AnnotationsConfig.Redirect + if redirect != nil && redirect.AppRoot != "" { + wrapperVS.AppRoot = redirect.AppRoot + } + + wrapperHttpRoutes := make([]*common.WrapperHTTPRoute, 0, len(rule.HTTP.Paths)) + for _, httpPath := range rule.HTTP.Paths { + wrapperHttpRoute := &common.WrapperHTTPRoute{ + HTTPRoute: &networking.HTTPRoute{}, + WrapperConfig: wrapper, + Host: rule.Host, + ClusterId: c.options.ClusterId, + } + httpMatch := &networking.HTTPMatchRequest{} + + path := httpPath.Path + if wrapper.AnnotationsConfig.NeedRegexMatch() { + wrapperHttpRoute.OriginPathType = common.Regex + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Regex{Regex: httpPath.Path + ".*"}, + } + } else { + switch *httpPath.PathType { + case ingress.PathTypeExact: + wrapperHttpRoute.OriginPathType = common.Exact + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Exact{Exact: httpPath.Path}, + } + case ingress.PathTypePrefix: + wrapperHttpRoute.OriginPathType = common.Prefix + // borrow from implement of official istio code. + if path == "/" { + wrapperVS.ConfiguredDefaultBackend = true + // Optimize common case of / to not needed regex + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Prefix{Prefix: path}, + } + } else { + path = strings.TrimSuffix(path, "/") + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Regex{Regex: regexp.QuoteMeta(path) + common.PrefixMatchRegex}, + } + } + } + } + wrapperHttpRoute.OriginPath = path + wrapperHttpRoute.HTTPRoute.Match = []*networking.HTTPMatchRequest{httpMatch} + wrapperHttpRoute.HTTPRoute.Name = common.GenerateUniqueRouteName(wrapperHttpRoute) + + ingressRouteBuilder := convertOptions.IngressRouteCache.New(wrapperHttpRoute) + + // host and path overlay check across different ingresses. + hostAndPath := wrapperHttpRoute.BasePathFormat() + if preIngress, exist := convertOptions.HostAndPath2Ingress[hostAndPath]; exist { + ingressRouteBuilder.PreIngress = preIngress + ingressRouteBuilder.Event = common.DuplicatedRoute + } + tempHostAndPath = append(tempHostAndPath, hostAndPath) + + // Two duplicated rules in the same ingress. + if ingressRouteBuilder.Event == common.Normal { + pathFormat := wrapperHttpRoute.PathFormat() + if definedRules.Contains(pathFormat) { + ingressRouteBuilder.PreIngress = cfg + ingressRouteBuilder.Event = common.DuplicatedRoute + } + definedRules.Insert(pathFormat) + } + + // backend service check + var event common.Event + wrapperHttpRoute.HTTPRoute.Route, event = c.backendToRouteDestination(&httpPath.Backend, cfg.Namespace, ingressRouteBuilder) + + if ingressRouteBuilder.Event != common.Normal { + event = ingressRouteBuilder.Event + } + + if event != common.Normal { + common.IncrementInvalidIngress(c.options.ClusterId, event) + ingressRouteBuilder.Event = event + } else { + wrapperHttpRoutes = append(wrapperHttpRoutes, wrapperHttpRoute) + } + + convertOptions.IngressRouteCache.Add(ingressRouteBuilder) + } + + for _, item := range tempHostAndPath { + // We only record the first + if _, exist := convertOptions.HostAndPath2Ingress[item]; !exist { + convertOptions.HostAndPath2Ingress[item] = cfg + } + } + + old, f := convertOptions.HTTPRoutes[rule.Host] + if f { + old = append(old, wrapperHttpRoutes...) + convertOptions.HTTPRoutes[rule.Host] = old + } else { + convertOptions.HTTPRoutes[rule.Host] = wrapperHttpRoutes + } + + // Sort, exact -> prefix -> regex + routes := convertOptions.HTTPRoutes[rule.Host] + IngressLog.Debugf("routes of host %s is %v", rule.Host, routes) + common.SortHTTPRoutes(routes) + } + + return nil +} + +func (c *controller) ApplyDefaultBackend(convertOptions *common.ConvertOptions, wrapper *common.WrapperConfig) error { + if wrapper.AnnotationsConfig.IsCanary() { + return nil + } + + cfg := wrapper.Config + ingressV1, ok := cfg.Spec.(ingress.IngressSpec) + if !ok { + common.IncrementInvalidIngress(c.options.ClusterId, common.Unknown) + return fmt.Errorf("convert type is invalid in cluster %s", c.options.ClusterId) + } + + if ingressV1.DefaultBackend == nil { + return nil + } + + apply := func(host string, op func(vs *common.WrapperVirtualService, defaultRoute *common.WrapperHTTPRoute)) { + wirecardVS, exist := convertOptions.VirtualServices[host] + if !exist || !wirecardVS.ConfiguredDefaultBackend { + if !exist { + wirecardVS = &common.WrapperVirtualService{ + VirtualService: &networking.VirtualService{ + Hosts: []string{host}, + }, + WrapperConfig: wrapper, + } + convertOptions.VirtualServices[host] = wirecardVS + } + + specDefaultBackend := c.createDefaultRoute(wrapper, ingressV1.DefaultBackend, host) + if specDefaultBackend != nil { + convertOptions.VirtualServices[host] = wirecardVS + op(wirecardVS, specDefaultBackend) + } + } + } + + // First process * + apply("*", func(_ *common.WrapperVirtualService, defaultRoute *common.WrapperHTTPRoute) { + var hasFound bool + for _, httpRoute := range convertOptions.HTTPRoutes["*"] { + if httpRoute.OriginPathType == common.Prefix && httpRoute.OriginPath == "/" { + hasFound = true + convertOptions.IngressRouteCache.Delete(httpRoute) + + httpRoute.HTTPRoute = defaultRoute.HTTPRoute + httpRoute.WrapperConfig = defaultRoute.WrapperConfig + convertOptions.IngressRouteCache.NewAndAdd(httpRoute) + } + } + if !hasFound { + convertOptions.HTTPRoutes["*"] = append(convertOptions.HTTPRoutes["*"], defaultRoute) + } + }) + + for _, rule := range ingressV1.Rules { + if rule.Host == "*" { + continue + } + + apply(rule.Host, func(vs *common.WrapperVirtualService, defaultRoute *common.WrapperHTTPRoute) { + convertOptions.HTTPRoutes[rule.Host] = append(convertOptions.HTTPRoutes[rule.Host], defaultRoute) + vs.ConfiguredDefaultBackend = true + + convertOptions.IngressRouteCache.NewAndAdd(defaultRoute) + }) + } + + return nil +} + +func (c *controller) ApplyCanaryIngress(convertOptions *common.ConvertOptions, wrapper *common.WrapperConfig) error { + byHeader, byWeight := wrapper.AnnotationsConfig.CanaryKind() + + cfg := wrapper.Config + ingressV1, ok := cfg.Spec.(ingress.IngressSpec) + if !ok { + common.IncrementInvalidIngress(c.options.ClusterId, common.Unknown) + return fmt.Errorf("convert type is invalid in cluster %s", c.options.ClusterId) + } + if len(ingressV1.Rules) == 0 && ingressV1.DefaultBackend == nil { + common.IncrementInvalidIngress(c.options.ClusterId, common.EmptyRule) + return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId) + } + + for _, rule := range ingressV1.Rules { + if rule.HTTP == nil || len(rule.HTTP.Paths) == 0 { + IngressLog.Warnf("invalid ingress rule %s:%s for host %q in cluster %s, no paths defined", cfg.Namespace, cfg.Name, rule.Host, c.options.ClusterId) + continue + } + + routes, exist := convertOptions.HTTPRoutes[rule.Host] + if !exist { + continue + } + + for _, httpPath := range rule.HTTP.Paths { + path := httpPath.Path + + canary := &common.WrapperHTTPRoute{ + HTTPRoute: &networking.HTTPRoute{}, + WrapperConfig: wrapper, + Host: rule.Host, + ClusterId: c.options.ClusterId, + } + httpMatch := &networking.HTTPMatchRequest{} + + if wrapper.AnnotationsConfig.NeedRegexMatch() { + canary.OriginPathType = common.Regex + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Regex{Regex: httpPath.Path + ".*"}, + } + } else { + switch *httpPath.PathType { + case ingress.PathTypeExact: + canary.OriginPathType = common.Exact + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Exact{Exact: httpPath.Path}, + } + case ingress.PathTypePrefix: + canary.OriginPathType = common.Prefix + // borrow from implement of official istio code. + if path == "/" { + // Optimize common case of / to not needed regex + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Prefix{Prefix: path}, + } + } else { + path = strings.TrimSuffix(path, "/") + httpMatch.Uri = &networking.StringMatch{ + MatchType: &networking.StringMatch_Regex{Regex: regexp.QuoteMeta(path) + common.PrefixMatchRegex}, + } + } + } + } + canary.OriginPath = path + canary.HTTPRoute.Match = []*networking.HTTPMatchRequest{httpMatch} + canary.HTTPRoute.Name = common.GenerateUniqueRouteName(canary) + + ingressRouteBuilder := convertOptions.IngressRouteCache.New(canary) + // backend service check + var event common.Event + canary.HTTPRoute.Route, event = c.backendToRouteDestination(&httpPath.Backend, cfg.Namespace, ingressRouteBuilder) + if event != common.Normal { + common.IncrementInvalidIngress(c.options.ClusterId, event) + ingressRouteBuilder.Event = event + convertOptions.IngressRouteCache.Add(ingressRouteBuilder) + continue + } + + canaryConfig := wrapper.AnnotationsConfig.Canary + if byWeight { + canary.HTTPRoute.Route[0].Weight = int32(canaryConfig.Weight) + } + + pos := 0 + var targetRoute *common.WrapperHTTPRoute + for _, route := range routes { + if isCanaryRoute(canary, route) { + targetRoute = route + // Header, Cookie + if byHeader { + IngressLog.Debug("Insert canary route by header") + annotations.ApplyByHeader(canary.HTTPRoute, route.HTTPRoute, canary.WrapperConfig.AnnotationsConfig) + canary.HTTPRoute.Name = common.GenerateUniqueRouteName(canary) + } else { + IngressLog.Debug("Merge canary route by weight") + if route.WeightTotal == 0 { + route.WeightTotal = int32(canaryConfig.WeightTotal) + } + annotations.ApplyByWeight(canary.HTTPRoute, route.HTTPRoute, canary.WrapperConfig.AnnotationsConfig) + } + + break + } + pos += 1 + } + + IngressLog.Debugf("Canary route is %v", canary) + if targetRoute == nil { + continue + } + + if byHeader { + // Inherit policy from normal route + canary.WrapperConfig.AnnotationsConfig.Auth = targetRoute.WrapperConfig.AnnotationsConfig.Auth + + routes = append(routes[:pos+1], routes[pos:]...) + routes[pos] = canary + convertOptions.HTTPRoutes[rule.Host] = routes + + // Recreate route name. + ingressRouteBuilder.RouteName = common.GenerateUniqueRouteName(canary) + convertOptions.IngressRouteCache.Add(ingressRouteBuilder) + } else { + convertOptions.IngressRouteCache.Update(targetRoute) + } + } + } + return nil +} + +func (c *controller) ConvertTrafficPolicy(convertOptions *common.ConvertOptions, wrapper *common.WrapperConfig) error { + if !wrapper.AnnotationsConfig.NeedTrafficPolicy() { + return nil + } + + cfg := wrapper.Config + ingressV1, ok := cfg.Spec.(ingress.IngressSpec) + if !ok { + common.IncrementInvalidIngress(c.options.ClusterId, common.Unknown) + return fmt.Errorf("convert type is invalid in cluster %s", c.options.ClusterId) + } + if len(ingressV1.Rules) == 0 && ingressV1.DefaultBackend == nil { + common.IncrementInvalidIngress(c.options.ClusterId, common.EmptyRule) + return fmt.Errorf("invalid ingress rule %s:%s in cluster %s, either `defaultBackend` or `rules` must be specified", cfg.Namespace, cfg.Name, c.options.ClusterId) + } + + if ingressV1.DefaultBackend != nil { + serviceKey, err := c.createServiceKey(ingressV1.DefaultBackend.Service, cfg.Namespace) + if err != nil { + IngressLog.Errorf("ignore default service %s within ingress %s/%s", serviceKey.Name, cfg.Namespace, cfg.Name) + } else { + if _, exist := convertOptions.Service2TrafficPolicy[serviceKey]; !exist { + convertOptions.Service2TrafficPolicy[serviceKey] = &common.WrapperTrafficPolicy{ + TrafficPolicy: &networking.TrafficPolicy_PortTrafficPolicy{ + Port: &networking.PortSelector{ + Number: uint32(serviceKey.Port), + }, + }, + WrapperConfig: wrapper, + } + } + } + } + + for _, rule := range ingressV1.Rules { + if rule.HTTP == nil || len(rule.HTTP.Paths) == 0 { + continue + } + + for _, httpPath := range rule.HTTP.Paths { + if httpPath.Backend.Service == nil { + continue + } + + serviceKey, err := c.createServiceKey(httpPath.Backend.Service, cfg.Namespace) + if err != nil { + IngressLog.Errorf("ignore service %s within ingress %s/%s", serviceKey.Name, cfg.Namespace, cfg.Name) + continue + } + + if _, exist := convertOptions.Service2TrafficPolicy[serviceKey]; exist { + continue + } + + convertOptions.Service2TrafficPolicy[serviceKey] = &common.WrapperTrafficPolicy{ + TrafficPolicy: &networking.TrafficPolicy_PortTrafficPolicy{ + Port: &networking.PortSelector{ + Number: uint32(serviceKey.Port), + }, + }, + WrapperConfig: wrapper, + } + } + } + + return nil +} + +func (c *controller) createDefaultRoute(wrapper *common.WrapperConfig, backend *ingress.IngressBackend, host string) *common.WrapperHTTPRoute { + if backend == nil || backend.Service == nil || backend.Service.Name == "" { + return nil + } + + service := backend.Service + namespace := wrapper.Config.Namespace + + port := &networking.PortSelector{} + if service.Port.Number > 0 { + port.Number = uint32(service.Port.Number) + } else { + resolvedPort, err := resolveNamedPort(service, namespace, c.serviceLister) + if err != nil { + return nil + } + port.Number = uint32(resolvedPort) + } + + routeDestination := []*networking.HTTPRouteDestination{ + { + Destination: &networking.Destination{ + Host: util.CreateServiceFQDN(namespace, service.Name), + Port: port, + }, + Weight: 100, + }, + } + + route := &common.WrapperHTTPRoute{ + HTTPRoute: &networking.HTTPRoute{ + Route: routeDestination, + }, + WrapperConfig: wrapper, + ClusterId: c.options.ClusterId, + Host: host, + IsDefaultBackend: true, + OriginPathType: common.Prefix, + OriginPath: "/", + } + route.HTTPRoute.Name = common.GenerateUniqueRouteNameWithSuffix(route, "default") + + return route +} + +func (c *controller) createServiceKey(service *ingress.IngressServiceBackend, namespace string) (common.ServiceKey, error) { + serviceKey := common.ServiceKey{} + if service == nil || service.Name == "" { + return serviceKey, errors.New("service name is empty") + } + + var port int32 + var err error + if service.Port.Number > 0 { + port = service.Port.Number + } else { + port, err = resolveNamedPort(service, namespace, c.serviceLister) + if err != nil { + return serviceKey, err + } + } + + return common.ServiceKey{ + Namespace: namespace, + Name: service.Name, + Port: port, + }, nil +} + +func isCanaryRoute(canary, route *common.WrapperHTTPRoute) bool { + return !strings.HasSuffix(route.HTTPRoute.Name, "-canary") && canary.OriginPath == route.OriginPath && + canary.OriginPathType == route.OriginPathType +} + +func (c *controller) backendToRouteDestination(backend *ingress.IngressBackend, namespace string, + builder *common.IngressRouteBuilder) ([]*networking.HTTPRouteDestination, common.Event) { + if backend == nil || backend.Service == nil { + return nil, common.InvalidBackendService + } + + service := backend.Service + if service.Name == "" { + return nil, common.InvalidBackendService + } + + builder.PortName = service.Port.Name + + port := &networking.PortSelector{} + if service.Port.Number > 0 { + port.Number = uint32(service.Port.Number) + } else { + resolvedPort, err := resolveNamedPort(service, namespace, c.serviceLister) + if err != nil { + return nil, common.PortNameResolveError + } + port.Number = uint32(resolvedPort) + } + + builder.ServiceList = []model.BackendService{ + { + Namespace: namespace, + Name: service.Name, + Port: port.Number, + Weight: 100, + }, + } + + return []*networking.HTTPRouteDestination{ + { + Destination: &networking.Destination{ + Host: util.CreateServiceFQDN(namespace, service.Name), + Port: port, + }, + Weight: 100, + }, + }, common.Normal +} + +func resolveNamedPort(service *ingress.IngressServiceBackend, namespace string, serviceLister listerv1.ServiceLister) (int32, error) { + svc, err := serviceLister.Services(namespace).Get(service.Name) + if err != nil { + return 0, err + } + for _, port := range svc.Spec.Ports { + if port.Name == service.Port.Name { + return port.Port, nil + } + } + return 0, common.ErrNotFound +} + +func (c *controller) shouldProcessIngressWithClass(ingress *ingress.Ingress, ingressClass *ingress.IngressClass) bool { + if class, exists := ingress.Annotations[kube.IngressClassAnnotation]; exists { + switch c.options.IngressClass { + case "": + return true + case common.DefaultIngressClass: + return class == "" || class == common.DefaultIngressClass + default: + return c.options.IngressClass == class + } + } else if ingressClass != nil { + switch c.options.IngressClass { + case "": + return true + default: + return c.options.IngressClass == ingressClass.Name + } + } else { + ingressClassName := ingress.Spec.IngressClassName + switch c.options.IngressClass { + case "": + return true + case common.DefaultIngressClass: + return ingressClassName == nil || *ingressClassName == "" || + *ingressClassName == common.DefaultIngressClass + default: + return ingressClassName != nil && *ingressClassName == c.options.IngressClass + } + } +} + +func (c *controller) shouldProcessIngress(i *ingress.Ingress) (bool, error) { + var class *ingress.IngressClass + if c.classes != nil && i.Spec.IngressClassName != nil { + classCache, err := c.classes.Lister().Get(*i.Spec.IngressClassName) + if err != nil && !kerrors.IsNotFound(err) { + return false, fmt.Errorf("failed to get ingress class %v from cluster %s: %v", i.Spec.IngressClassName, c.options.ClusterId, err) + } + class = classCache + } + + // first check ingress class + if c.shouldProcessIngressWithClass(i, class) { + // then check namespace + switch c.options.WatchNamespace { + case "": + return true, nil + default: + return c.options.WatchNamespace == i.Namespace, nil + } + } + + return false, nil +} + +// shouldProcessIngressUpdate checks whether we should renotify registered handlers about an update event +func (c *controller) shouldProcessIngressUpdate(ing *ingress.Ingress) (bool, error) { + shouldProcess, err := c.shouldProcessIngress(ing) + if err != nil { + return false, err + } + + namespacedName := ing.Namespace + "/" + ing.Name + if shouldProcess { + // record processed ingress + c.mutex.Lock() + preConfig, exist := c.ingresses[namespacedName] + c.ingresses[namespacedName] = ing + c.mutex.Unlock() + + // We only care about annotations, labels and spec. + if exist { + if !reflect.DeepEqual(preConfig.Annotations, ing.Annotations) { + IngressLog.Debugf("Annotations of ingress %s changed, should process.", namespacedName) + return true, nil + } + if !reflect.DeepEqual(preConfig.Labels, ing.Labels) { + IngressLog.Debugf("Labels of ingress %s changed, should process.", namespacedName) + return true, nil + } + if !reflect.DeepEqual(preConfig.Spec, ing.Spec) { + IngressLog.Debugf("Spec of ingress %s changed, should process.", namespacedName) + return true, nil + } + + return false, nil + } + + IngressLog.Debugf("First receive relative ingress %s, should process.", namespacedName) + return true, nil + } + + c.mutex.Lock() + _, preProcessed := c.ingresses[namespacedName] + // previous processed but should not currently, delete it + if preProcessed && !shouldProcess { + delete(c.ingresses, namespacedName) + } + c.mutex.Unlock() + + return preProcessed, nil +} + +// setDefaultMSEIngressOptionalField sets a default value for optional fields when is not defined. +func setDefaultMSEIngressOptionalField(ing *ingress.Ingress) { + for idx, tls := range ing.Spec.TLS { + if len(tls.Hosts) == 0 { + ing.Spec.TLS[idx].Hosts = []string{common.DefaultHost} + } + } + + for idx, rule := range ing.Spec.Rules { + if rule.IngressRuleValue.HTTP == nil { + continue + } + + if rule.Host == "" { + ing.Spec.Rules[idx].Host = common.DefaultHost + } + + for innerIdx := range rule.IngressRuleValue.HTTP.Paths { + p := &rule.IngressRuleValue.HTTP.Paths[innerIdx] + + if p.Path == "" { + p.Path = common.DefaultPath + } + + if p.PathType == nil { + p.PathType = &defaultPathType + // for old k8s version + if !annotations.NeedRegexMatch(ing.Annotations) { + if strings.HasSuffix(p.Path, ".*") { + p.Path = strings.TrimSuffix(p.Path, ".*") + } + + if strings.HasSuffix(p.Path, "/*") { + p.Path = strings.TrimSuffix(p.Path, "/*") + } + } + } + + if *p.PathType == ingress.PathTypeImplementationSpecific { + p.PathType = &defaultPathType + } + } + } +} diff --git a/ingress/kube/ingressv1/controller_test.go b/ingress/kube/ingressv1/controller_test.go new file mode 100644 index 000000000..12395382e --- /dev/null +++ b/ingress/kube/ingressv1/controller_test.go @@ -0,0 +1,78 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingressv1 + +import ( + "testing" + + v1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/alibaba/higress/ingress/kube/common" +) + +func TestShouldProcessIngressUpdate(t *testing.T) { + c := controller{ + options: common.Options{ + IngressClass: "mse", + }, + ingresses: make(map[string]*v1.Ingress), + } + + ingressClass := "mse" + + ingress1 := &v1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-1", + }, + Spec: v1.IngressSpec{ + IngressClassName: &ingressClass, + Rules: []v1.IngressRule{ + { + Host: "test.com", + IngressRuleValue: v1.IngressRuleValue{ + HTTP: &v1.HTTPIngressRuleValue{ + Paths: []v1.HTTPIngressPath{ + { + Path: "/test", + }, + }, + }, + }, + }, + }, + }, + } + + should, _ := c.shouldProcessIngressUpdate(ingress1) + if !should { + t.Fatal("should be true") + } + + ingress2 := *ingress1 + should, _ = c.shouldProcessIngressUpdate(&ingress2) + if should { + t.Fatal("should be false") + } + + ingress3 := *ingress1 + ingress3.Annotations = map[string]string{ + "test": "true", + } + should, _ = c.shouldProcessIngressUpdate(&ingress3) + if !should { + t.Fatal("should be true") + } +} diff --git a/ingress/kube/ingressv1/status.go b/ingress/kube/ingressv1/status.go new file mode 100644 index 000000000..7eb2c7f97 --- /dev/null +++ b/ingress/kube/ingressv1/status.go @@ -0,0 +1,133 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingressv1 + +import ( + "context" + "reflect" + "sort" + "time" + + kubelib "istio.io/istio/pkg/kube" + coreV1 "k8s.io/api/core/v1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + listerv1 "k8s.io/client-go/listers/core/v1" + ingresslister "k8s.io/client-go/listers/networking/v1" + "k8s.io/client-go/tools/cache" + + "github.com/alibaba/higress/ingress/kube/common" + . "github.com/alibaba/higress/ingress/log" +) + +// statusSyncer keeps the status IP in each Ingress resource updated +type statusSyncer struct { + client kubernetes.Interface + controller *controller + + watchedNamespace string + + ingressLister ingresslister.IngressLister + ingressClassLister ingresslister.IngressClassLister + // search service in the mse vpc + serviceLister listerv1.ServiceLister +} + +// newStatusSyncer creates a new instance +func newStatusSyncer(localKubeClient, client kubelib.Client, controller *controller, namespace string) *statusSyncer { + return &statusSyncer{ + client: client, + controller: controller, + watchedNamespace: namespace, + ingressLister: client.KubeInformer().Networking().V1().Ingresses().Lister(), + ingressClassLister: client.KubeInformer().Networking().V1().IngressClasses().Lister(), + // search service in the mse vpc + serviceLister: localKubeClient.KubeInformer().Core().V1().Services().Lister(), + } +} + +func (s *statusSyncer) run(stopCh <-chan struct{}) { + cache.WaitForCacheSync(stopCh, s.controller.HasSynced) + + ticker := time.NewTicker(common.DefaultStatusUpdateInterval) + for { + select { + case <-stopCh: + ticker.Stop() + return + case <-ticker.C: + if err := s.runUpdateStatus(); err != nil { + IngressLog.Errorf("update status task fail, err %v", err) + } + } + } +} + +func (s *statusSyncer) runUpdateStatus() error { + svcList, err := s.serviceLister.Services(s.watchedNamespace).List(common.SvcLabelSelector) + if err != nil { + return err + } + + IngressLog.Debugf("found number %d of svc", len(svcList)) + + lbStatusList := common.GetLbStatusList(svcList) + if len(lbStatusList) == 0 { + return nil + } + + return s.updateStatus(lbStatusList) +} + +// updateStatus updates ingress status with the list of IP +func (s *statusSyncer) updateStatus(status []coreV1.LoadBalancerIngress) error { + ingressList, err := s.ingressLister.List(labels.Everything()) + if err != nil { + return err + } + for _, ingress := range ingressList { + shouldTarget, err := s.controller.shouldProcessIngress(ingress) + if err != nil { + IngressLog.Warnf("error determining whether should target ingress %s/%s within cluster %s for status update: %v", + ingress.Namespace, ingress.Name, s.controller.options.ClusterId, err) + return err + } + + if !shouldTarget { + continue + } + + curIPs := ingress.Status.LoadBalancer.Ingress + sort.SliceStable(curIPs, common.SortLbIngressList(curIPs)) + + if reflect.DeepEqual(status, curIPs) { + IngressLog.Debugf("skipping update of Ingress %v/%v within cluster %s (no change)", + ingress.Namespace, ingress.Name, s.controller.options.ClusterId) + continue + } + + ingress.Status.LoadBalancer.Ingress = status + IngressLog.Infof("Update Ingress %v/%v within cluster %s status", + ingress.Namespace, ingress.Name, s.controller.options.ClusterId) + _, err = s.client.NetworkingV1().Ingresses(ingress.Namespace).UpdateStatus(context.TODO(), ingress, metaV1.UpdateOptions{}) + if err != nil { + IngressLog.Warnf("error updating ingress %s/%s within cluster %s status: %v", + ingress.Namespace, ingress.Name, s.controller.options.ClusterId, err) + } + } + + return nil +} diff --git a/ingress/kube/secret/kube/controller.go b/ingress/kube/secret/kube/controller.go new file mode 100644 index 000000000..0168fabf8 --- /dev/null +++ b/ingress/kube/secret/kube/controller.go @@ -0,0 +1,148 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kube + +import ( + "time" + + "istio.io/istio/pilot/pkg/model" + kubeclient "istio.io/istio/pkg/kube" + "istio.io/istio/pkg/kube/controllers" + v1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + informersv1 "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + listersv1 "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + "github.com/alibaba/higress/ingress/kube/common" + "github.com/alibaba/higress/ingress/kube/secret" + "github.com/alibaba/higress/ingress/kube/util" + . "github.com/alibaba/higress/ingress/log" +) + +var _ secret.Controller = &controller{} + +type controller struct { + queue workqueue.RateLimitingInterface + informer cache.SharedIndexInformer + lister listersv1.SecretLister + handler func(util.ClusterNamespacedName) + clusterId string +} + +// NewController is copied from NewCredentialsController. +func NewController(client kubeclient.Client, options common.Options) secret.Controller { + q := workqueue.NewRateLimitingQueue(workqueue.DefaultItemBasedRateLimiter()) + + informer := client.KubeInformer().InformerFor(&v1.Secret{}, func(k kubernetes.Interface, resync time.Duration) cache.SharedIndexInformer { + return informersv1.NewFilteredSecretInformer( + k, metav1.NamespaceAll, resync, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, + func(options *metav1.ListOptions) { + options.FieldSelector = fields.AndSelectors( + fields.OneTermNotEqualSelector("type", "helm.sh/release.v1"), + fields.OneTermNotEqualSelector("type", string(v1.SecretTypeServiceAccountToken)), + ).String() + }, + ) + }) + + handler := controllers.LatestVersionHandlerFuncs(controllers.EnqueueForSelf(q)) + informer.AddEventHandler(handler) + + return &controller{ + queue: q, + informer: informer, + lister: listersv1.NewSecretLister(informer.GetIndexer()), + clusterId: options.ClusterId, + } +} + +func (c *controller) Lister() listersv1.SecretLister { + return c.lister +} + +func (c *controller) Informer() cache.SharedIndexInformer { + return c.informer +} + +func (c *controller) AddEventHandler(f func(util.ClusterNamespacedName)) { + c.handler = f +} + +func (c *controller) Run(stop <-chan struct{}) { + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + + if !cache.WaitForCacheSync(stop, c.HasSynced) { + IngressLog.Errorf("Failed to sync secret controller cache") + return + } + go wait.Until(c.worker, time.Second, stop) + <-stop +} + +func (c *controller) worker() { + for c.processNextWorkItem() { + } +} + +func (c *controller) processNextWorkItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + ingressNamespacedName := key.(types.NamespacedName) + IngressLog.Debugf("secret %s push to queue", ingressNamespacedName) + if err := c.onEvent(ingressNamespacedName); err != nil { + IngressLog.Errorf("error processing secret item (%v) (retrying): %v", key, err) + c.queue.AddRateLimited(key) + } else { + c.queue.Forget(key) + } + return true +} + +func (c *controller) onEvent(namespacedName types.NamespacedName) error { + _, err := c.lister.Secrets(namespacedName.Namespace).Get(namespacedName.Name) + if err != nil { + if kerrors.IsNotFound(err) { + return nil + } else { + return err + } + } + + // We only care about add or update event. + c.handler(util.ClusterNamespacedName{ + NamespacedName: model.NamespacedName{ + Namespace: namespacedName.Namespace, + Name: namespacedName.Name, + }, + ClusterId: c.clusterId, + }) + return nil +} + +func (c *controller) HasSynced() bool { + return c.informer.HasSynced() +} diff --git a/ingress/kube/secret/model.go b/ingress/kube/secret/model.go new file mode 100644 index 000000000..2b3a42648 --- /dev/null +++ b/ingress/kube/secret/model.go @@ -0,0 +1,34 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package secret + +import ( + listerv1 "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + + "github.com/alibaba/higress/ingress/kube/util" +) + +type Controller interface { + AddEventHandler(func(util.ClusterNamespacedName)) + + Run(stop <-chan struct{}) + + HasSynced() bool + + Lister() listerv1.SecretLister + + Informer() cache.SharedIndexInformer +} diff --git a/ingress/kube/util/util.go b/ingress/kube/util/util.go new file mode 100644 index 000000000..6285ea678 --- /dev/null +++ b/ingress/kube/util/util.go @@ -0,0 +1,84 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "bytes" + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "path" + "strings" + + "github.com/gogo/protobuf/types" + "github.com/golang/protobuf/jsonpb" + "github.com/golang/protobuf/proto" + "istio.io/istio/pilot/pkg/model" +) + +const DefaultDomainSuffix = "cluster.local" + +type ClusterNamespacedName struct { + model.NamespacedName + ClusterId string +} + +func (c ClusterNamespacedName) String() string { + return c.ClusterId + "/" + c.NamespacedName.String() +} + +func SplitNamespacedName(name string) model.NamespacedName { + nsName := strings.Split(name, "/") + if len(nsName) == 2 { + return model.NamespacedName{ + Namespace: nsName[0], + Name: nsName[1], + } + } + + return model.NamespacedName{ + Name: nsName[0], + } +} + +// CreateDestinationRuleName create the same format of DR name with ops. +func CreateDestinationRuleName(istioCluster, namespace, name string) string { + format := path.Join(istioCluster, namespace, name) + hash := md5.Sum([]byte(format)) + return hex.EncodeToString(hash[:]) +} + +func MessageToGoGoStruct(msg proto.Message) (*types.Struct, error) { + if msg == nil { + return nil, errors.New("nil message") + } + + buf := &bytes.Buffer{} + if err := (&jsonpb.Marshaler{OrigName: true}).Marshal(buf, msg); err != nil { + return nil, err + } + + pbs := &types.Struct{} + if err := jsonpb.Unmarshal(buf, pbs); err != nil { + return nil, err + } + + return pbs, nil +} + +func CreateServiceFQDN(namespace, name string) string { + return fmt.Sprintf("%s.%s.svc.%s", name, namespace, DefaultDomainSuffix) +} diff --git a/ingress/kube/util/util_test.go b/ingress/kube/util/util_test.go new file mode 100644 index 000000000..0d56c0182 --- /dev/null +++ b/ingress/kube/util/util_test.go @@ -0,0 +1,108 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "testing" + + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + wasm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/wasm/v3" + v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/wasm/v3" + any "google.golang.org/protobuf/types/known/anypb" + "istio.io/istio/pilot/pkg/model" +) + +func TestSplitNamespacedName(t *testing.T) { + testCases := []struct { + input string + expect model.NamespacedName + }{ + { + input: "", + }, + { + input: "a/", + expect: model.NamespacedName{ + Namespace: "a", + }, + }, + { + input: "a/b", + expect: model.NamespacedName{ + Namespace: "a", + Name: "b", + }, + }, + { + input: "/b", + expect: model.NamespacedName{ + Name: "b", + }, + }, + { + input: "b", + expect: model.NamespacedName{ + Name: "b", + }, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + result := SplitNamespacedName(testCase.input) + if result != testCase.expect { + t.Fatalf("expect is %v, but actual is %v", testCase.expect, result) + } + }) + } +} + +func TestCreateDestinationRuleName(t *testing.T) { + istioCluster := "gw-123-istio" + namespace := "default" + serviceName := "go-httpbin-v1" + t.Log(CreateDestinationRuleName(istioCluster, namespace, serviceName)) +} + +func TestMessageToGoGoStruct(t *testing.T) { + bytes := []byte("test") + wasm := &wasm.Wasm{ + Config: &v3.PluginConfig{ + Name: "basic-auth", + FailOpen: true, + Vm: &v3.PluginConfig_VmConfig{ + VmConfig: &v3.VmConfig{ + Runtime: "envoy.wasm.runtime.null", + Code: &corev3.AsyncDataSource{ + Specifier: &corev3.AsyncDataSource_Local{ + Local: &corev3.DataSource{ + Specifier: &corev3.DataSource_InlineString{ + InlineString: "envoy.wasm.basic_auth", + }, + }, + }, + }, + }, + }, + Configuration: &any.Any{ + TypeUrl: "type.googleapis.com/google.protobuf.StringValue", + Value: bytes, + }, + }, + } + + gogoStruct, _ := MessageToGoGoStruct(wasm) + t.Log(gogoStruct) +} diff --git a/ingress/log/log.go b/ingress/log/log.go new file mode 100644 index 000000000..898c540d8 --- /dev/null +++ b/ingress/log/log.go @@ -0,0 +1,19 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package log + +import "istio.io/pkg/log" + +var IngressLog = log.RegisterScope("ingress", "Higress Ingress process.", 0) diff --git a/ingress/mcp/generator.go b/ingress/mcp/generator.go new file mode 100644 index 000000000..a191b3c4b --- /dev/null +++ b/ingress/mcp/generator.go @@ -0,0 +1,223 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mcp + +import ( + "path" + + "github.com/gogo/protobuf/types" + "github.com/golang/protobuf/ptypes" + "github.com/golang/protobuf/ptypes/any" + extensions "istio.io/api/extensions/v1alpha1" + mcp "istio.io/api/mcp/v1alpha1" + networking "istio.io/api/networking/v1alpha3" + "istio.io/istio/pilot/pkg/model" + "istio.io/istio/pilot/pkg/xds" +) + +type VirtualServiceGenerator struct { + Server *xds.DiscoveryServer +} + +func (c VirtualServiceGenerator) Generate(proxy *model.Proxy, push *model.PushContext, w *model.WatchedResource, + updates *model.PushRequest) ([]*any.Any, model.XdsLogDetails, error) { + resources := make([]*any.Any, 0) + configs := push.AllVirtualServices + for _, config := range configs { + body, err := types.MarshalAny(config.Spec.(*networking.VirtualService)) + if err != nil { + return nil, model.DefaultXdsLogDetails, err + } + createTime, err := types.TimestampProto(config.CreationTimestamp) + if err != nil { + return nil, model.DefaultXdsLogDetails, err + } + resource := &mcp.Resource{ + Body: body, + Metadata: &mcp.Metadata{ + Name: path.Join(config.Namespace, config.Name), + CreateTime: createTime, + }, + } + mcpAny, err := ptypes.MarshalAny(resource) + if err != nil { + return nil, model.DefaultXdsLogDetails, err + } + resources = append(resources, mcpAny) + } + return resources, model.DefaultXdsLogDetails, nil +} + +func (c VirtualServiceGenerator) GenerateDeltas(proxy *model.Proxy, push *model.PushContext, updates *model.PushRequest, + w *model.WatchedResource) ([]*any.Any, []string, model.XdsLogDetails, bool, error) { + // TODO: delta implement + return nil, nil, model.DefaultXdsLogDetails, false, nil +} + +type DestinationRuleGenerator struct { + Server *xds.DiscoveryServer +} + +func (c DestinationRuleGenerator) Generate(proxy *model.Proxy, push *model.PushContext, w *model.WatchedResource, + updates *model.PushRequest) ([]*any.Any, model.XdsLogDetails, error) { + resources := make([]*any.Any, 0) + configs := push.AllDestinationRules + for _, config := range configs { + body, err := types.MarshalAny(config.Spec.(*networking.DestinationRule)) + if err != nil { + return nil, model.DefaultXdsLogDetails, err + } + createTime, err := types.TimestampProto(config.CreationTimestamp) + if err != nil { + return nil, model.DefaultXdsLogDetails, err + } + resource := &mcp.Resource{ + Body: body, + Metadata: &mcp.Metadata{ + Name: path.Join(config.Namespace, config.Name), + CreateTime: createTime, + }, + } + mcpAny, err := ptypes.MarshalAny(resource) + if err != nil { + return nil, model.DefaultXdsLogDetails, err + } + resources = append(resources, mcpAny) + } + return resources, model.DefaultXdsLogDetails, nil +} + +func (c DestinationRuleGenerator) GenerateDeltas(proxy *model.Proxy, push *model.PushContext, updates *model.PushRequest, + w *model.WatchedResource) ([]*any.Any, []string, model.XdsLogDetails, bool, error) { + // TODO: delta implement + return nil, nil, model.DefaultXdsLogDetails, false, nil +} + +type EnvoyFilterGenerator struct { + Server *xds.DiscoveryServer +} + +func (c EnvoyFilterGenerator) Generate(proxy *model.Proxy, push *model.PushContext, w *model.WatchedResource, + updates *model.PushRequest) ([]*any.Any, model.XdsLogDetails, error) { + resources := make([]*any.Any, 0) + configs := push.AllEnvoyFilters + for _, config := range configs { + body, err := types.MarshalAny(config.Spec.(*networking.EnvoyFilter)) + if err != nil { + return nil, model.DefaultXdsLogDetails, err + } + createTime, err := types.TimestampProto(config.CreationTimestamp) + if err != nil { + return nil, model.DefaultXdsLogDetails, err + } + resource := &mcp.Resource{ + Body: body, + Metadata: &mcp.Metadata{ + Name: path.Join(config.Namespace, config.Name), + CreateTime: createTime, + }, + } + mcpAny, err := ptypes.MarshalAny(resource) + if err != nil { + return nil, model.DefaultXdsLogDetails, err + } + resources = append(resources, mcpAny) + } + return resources, model.DefaultXdsLogDetails, nil +} + +func (c EnvoyFilterGenerator) GenerateDeltas(proxy *model.Proxy, push *model.PushContext, updates *model.PushRequest, + w *model.WatchedResource) ([]*any.Any, []string, model.XdsLogDetails, bool, error) { + // TODO: delta implement + return nil, nil, model.DefaultXdsLogDetails, false, nil +} + +type GatewayGenerator struct { + Server *xds.DiscoveryServer +} + +func (c GatewayGenerator) Generate(proxy *model.Proxy, push *model.PushContext, w *model.WatchedResource, + updates *model.PushRequest) ([]*any.Any, model.XdsLogDetails, error) { + resources := make([]*any.Any, 0) + configs := push.AllGateways + for _, config := range configs { + body, err := types.MarshalAny(config.Spec.(*networking.Gateway)) + if err != nil { + return nil, model.DefaultXdsLogDetails, err + } + createTime, err := types.TimestampProto(config.CreationTimestamp) + if err != nil { + return nil, model.DefaultXdsLogDetails, err + } + resource := &mcp.Resource{ + Body: body, + Metadata: &mcp.Metadata{ + Name: path.Join(config.Namespace, config.Name), + CreateTime: createTime, + }, + } + mcpAny, err := ptypes.MarshalAny(resource) + if err != nil { + return nil, model.DefaultXdsLogDetails, err + } + resources = append(resources, mcpAny) + } + return resources, model.DefaultXdsLogDetails, nil +} + +func (c GatewayGenerator) GenerateDeltas(proxy *model.Proxy, push *model.PushContext, updates *model.PushRequest, + w *model.WatchedResource) ([]*any.Any, []string, model.XdsLogDetails, bool, error) { + // TODO: delta implement + return nil, nil, model.DefaultXdsLogDetails, false, nil +} + +type WasmpluginGenerator struct { + Server *xds.DiscoveryServer +} + +func (c WasmpluginGenerator) Generate(proxy *model.Proxy, push *model.PushContext, w *model.WatchedResource, + updates *model.PushRequest) ([]*any.Any, model.XdsLogDetails, error) { + resources := make([]*any.Any, 0) + configs := push.AllWasmplugins + for _, config := range configs { + body, err := types.MarshalAny(config.Spec.(*extensions.WasmPlugin)) + if err != nil { + return nil, model.DefaultXdsLogDetails, err + } + createTime, err := types.TimestampProto(config.CreationTimestamp) + if err != nil { + return nil, model.DefaultXdsLogDetails, err + } + resource := &mcp.Resource{ + Body: body, + Metadata: &mcp.Metadata{ + Name: path.Join(config.Namespace, config.Name), + CreateTime: createTime, + }, + } + mcpAny, err := ptypes.MarshalAny(resource) + if err != nil { + return nil, model.DefaultXdsLogDetails, err + } + resources = append(resources, mcpAny) + } + return resources, model.DefaultXdsLogDetails, nil +} + +func (c WasmpluginGenerator) GenerateDeltas(proxy *model.Proxy, push *model.PushContext, updates *model.PushRequest, + w *model.WatchedResource) ([]*any.Any, []string, model.XdsLogDetails, bool, error) { + // TODO: delta implement + return nil, nil, model.DefaultXdsLogDetails, false, nil +}