From a5edad1a84d76dd2e23dde47853a0ddf3ce6770f Mon Sep 17 00:00:00 2001 From: Xunzhuo Date: Tue, 4 Apr 2023 19:38:44 +0800 Subject: [PATCH] feat: init support hgctl (#273) Signed-off-by: bitliu --- Makefile.core.mk | 33 ++- cmd/hgctl/main.go | 29 +++ cmd/higress/main.go | 114 +---------- go.mod | 12 +- pkg/cmd/hgctl/kubernetes/client.go | 172 ++++++++++++++++ pkg/cmd/hgctl/kubernetes/port-forwarder.go | 155 ++++++++++++++ pkg/cmd/hgctl/root.go | 32 +++ pkg/cmd/hgctl/version.go | 193 ++++++++++++++++++ pkg/cmd/options/global.go | 29 +++ pkg/cmd/root.go | 32 +++ pkg/cmd/server.go | 120 +++++++++++ .../main_test.go => pkg/cmd/server_test.go | 14 +- pkg/cmd/version.go | 38 ++++ pkg/cmd/version/version.go | 62 ++++++ 14 files changed, 906 insertions(+), 129 deletions(-) create mode 100644 cmd/hgctl/main.go create mode 100644 pkg/cmd/hgctl/kubernetes/client.go create mode 100644 pkg/cmd/hgctl/kubernetes/port-forwarder.go create mode 100644 pkg/cmd/hgctl/root.go create mode 100644 pkg/cmd/hgctl/version.go create mode 100644 pkg/cmd/options/global.go create mode 100644 pkg/cmd/root.go create mode 100644 pkg/cmd/server.go rename cmd/higress/main_test.go => pkg/cmd/server_test.go (96%) create mode 100644 pkg/cmd/version.go create mode 100644 pkg/cmd/version/version.go diff --git a/Makefile.core.mk b/Makefile.core.mk index 3d31497cd..54fa570ea 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -6,13 +6,20 @@ export HUB ?= higress-registry.cn-hangzhou.cr.aliyuncs.com/higress export CHARTS ?= higress-registry.cn-hangzhou.cr.aliyuncs.com/charts +VERSION_PACKAGE := github.com/alibaba/higress/pkg/cmd/version + +GIT_COMMIT:=$(shell git rev-parse HEAD) + +GO_LDFLAGS += -X $(VERSION_PACKAGE).higressVersion=$(shell cat VERSION) \ + -X $(VERSION_PACKAGE).gitCommitID=$(GIT_COMMIT) + GO ?= go export GOPROXY ?= https://proxy.golang.com.cn,direct GOARCH_LOCAL := $(TARGET_ARCH) GOOS_LOCAL := $(TARGET_OS) -RELEASE_LDFLAGS='-extldflags -static -s -w' +RELEASE_LDFLAGS='$(GO_LDFLAGS) -extldflags -static -s -w' export OUT:=$(TARGET_OUT) export OUT_LINUX:=$(TARGET_OUT_LINUX) @@ -32,7 +39,9 @@ endif HIGRESS_DOCKER_BUILD_TOP:=${OUT_LINUX}/docker_build -BINARIES:=./cmd/higress +HIGRESS_BINARIES:=./cmd/higress + +HGCTL_BINARIES:=./cmd/hgctl $(OUT): @mkdir -p $@ @@ -52,11 +61,19 @@ go.test.coverage: prebuild .PHONY: build build: prebuild $(OUT) - GOPROXY=$(GOPROXY) GOOS=$(GOOS_LOCAL) GOARCH=$(GOARCH_LOCAL) LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh $(OUT)/ $(BINARIES) + GOPROXY=$(GOPROXY) GOOS=$(GOOS_LOCAL) GOARCH=$(GOARCH_LOCAL) LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh $(OUT)/ $(HIGRESS_BINARIES) .PHONY: build-linux build-linux: prebuild $(OUT) - GOPROXY=$(GOPROXY) GOOS=linux GOARCH=$(GOARCH_LOCAL) LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh $(OUT_LINUX)/ $(BINARIES) + GOPROXY=$(GOPROXY) GOOS=linux GOARCH=$(GOARCH_LOCAL) LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh $(OUT_LINUX)/ $(HIGRESS_BINARIES) + +.PHONY: build-hgctl +build-hgctl: $(OUT) + GOPROXY=$(GOPROXY) GOOS=$(GOOS_LOCAL) GOARCH=$(GOARCH_LOCAL) LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh $(OUT)/ $(HGCTL_BINARIES) + +.PHONY: build-linux-hgctl +build-linux-hgctl: $(OUT) + GOPROXY=$(GOPROXY) GOOS=linux GOARCH=$(GOARCH_LOCAL) LDFLAGS=$(RELEASE_LDFLAGS) tools/hack/gobuild.sh $(OUT_LINUX)/ $(HGCTL_BINARIES) # Create targets for OUT_LINUX/binary # There are two use cases here: @@ -73,14 +90,14 @@ $(OUT_LINUX)/$(shell basename $(1)): $(OUT_LINUX) endif endef -$(foreach bin,$(BINARIES),$(eval $(call build-linux,$(bin),""))) +$(foreach bin,$(HIGRESS_BINARIES),$(eval $(call build-linux,$(bin),""))) # Create helper targets for each binary, like "pilot-discovery" # As an optimization, these still build everything -$(foreach bin,$(BINARIES),$(shell basename $(bin))): build +$(foreach bin,$(HIGRESS_BINARIES),$(shell basename $(bin))): build ifneq ($(OUT_LINUX),$(LOCAL_OUT)) # if we are on linux already, then this rule is handled by build-linux above, which handles BUILD_ALL variable -$(foreach bin,$(BINARIES),${LOCAL_OUT}/$(shell basename $(bin))): build +$(foreach bin,$(HIGRESS_BINARIES),${LOCAL_OUT}/$(shell basename $(bin))): build endif .PHONY: push @@ -185,7 +202,7 @@ delete-cluster: $(tools/kind) ## Delete kind cluster. # kube-load-image loads a local built docker image into kube cluster. .PHONY: kube-load-image -kube-load-image: $(tools/kind) ## Install the EG image to a kind cluster using the provided $IMAGE and $TAG. +kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster using the provided $IMAGE and $TAG. tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG) # run-ingress-e2e-test starts to run ingress e2e tests. diff --git a/cmd/hgctl/main.go b/cmd/hgctl/main.go new file mode 100644 index 000000000..eb3e3c17f --- /dev/null +++ b/cmd/hgctl/main.go @@ -0,0 +1,29 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/alibaba/higress/pkg/cmd/hgctl" +) + +func main() { + if err := hgctl.GetRootCommand().Execute(); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cmd/higress/main.go b/cmd/higress/main.go index cc0357cb7..3c78b8991 100644 --- a/cmd/higress/main.go +++ b/cmd/higress/main.go @@ -17,119 +17,13 @@ package main import ( "fmt" "os" - "time" - "github.com/spf13/cobra" - "istio.io/istio/pilot/pkg/features" - "istio.io/istio/pkg/cmd" - "istio.io/istio/pkg/config/constants" - "istio.io/istio/pkg/keepalive" - "istio.io/pkg/log" - "istio.io/pkg/version" - - "github.com/alibaba/higress/pkg/bootstrap" - innerconstants "github.com/alibaba/higress/pkg/config/constants" + "github.com/alibaba/higress/pkg/cmd" ) -var ( - serverArgs *bootstrap.ServerArgs - loggingOptions = log.DefaultOptions() - - serverProvider = func(args *bootstrap.ServerArgs) (bootstrap.ServerInterface, error) { - return bootstrap.NewServer(serverArgs) - } - - waitForMonitorSignal = func(stop chan struct{}) { - cmd.WaitSignal(stop) - } - - rootCmd = &cobra.Command{ - Use: "higress", - } - - serveCmd = &cobra.Command{ - Use: "serve", - Aliases: []string{"serve"}, - Short: "Starts the higress ingress controller", - Example: "higress serve", - PreRunE: func(c *cobra.Command, args []string) error { - return log.Configure(loggingOptions) - }, - RunE: func(c *cobra.Command, args []string) error { - cmd.PrintFlags(c.Flags()) - log.Infof("Version %s", version.Info.String()) - - stop := make(chan struct{}) - - server, err := serverProvider(serverArgs) - if err != nil { - return fmt.Errorf("fail to create higress server: %v", err) - } - - if err := server.Start(stop); err != nil { - return fmt.Errorf("fail to start higress server: %v", err) - } - - waitForMonitorSignal(stop) - - server.WaitUntilCompletion() - return nil - }, - } -) - -func init() { - serverArgs = &bootstrap.ServerArgs{ - Debug: true, - NativeIstio: true, - HttpAddress: ":8888", - GrpcAddress: ":15051", - GrpcKeepAliveOptions: keepalive.DefaultOption(), - XdsOptions: bootstrap.XdsOptions{ - DebounceAfter: features.DebounceAfter, - DebounceMax: features.DebounceMax, - EnableEDSDebounce: features.EnableEDSDebounce, - }, - } - - serveCmd.PersistentFlags().StringVar(&serverArgs.GatewaySelectorKey, "gatewaySelectorKey", "higress", "gateway resource selector label key") - serveCmd.PersistentFlags().StringVar(&serverArgs.GatewaySelectorValue, "gatewaySelectorValue", "higress-gateway", "gateway resource selector label value") - serveCmd.PersistentFlags().BoolVar(&serverArgs.EnableStatus, "enableStatus", true, "enable the ingress status syncer which use to update the ip in ingress's status") - serveCmd.PersistentFlags().StringVar(&serverArgs.IngressClass, "ingressClass", innerconstants.DefaultIngressClass, "if not empty, only watch the ingresses have the specified class, otherwise watch all ingresses") - serveCmd.PersistentFlags().StringVar(&serverArgs.WatchNamespace, "watchNamespace", "", "if not empty, only wath the ingresses in the specified namespace, otherwise watch in all namespacees") - serveCmd.PersistentFlags().BoolVar(&serverArgs.Debug, "debug", serverArgs.Debug, "if true, enables more debug http api") - serveCmd.PersistentFlags().StringVar(&serverArgs.HttpAddress, "httpAddress", serverArgs.HttpAddress, "the http address") - serveCmd.PersistentFlags().StringVar(&serverArgs.GrpcAddress, "grpcAddress", serverArgs.GrpcAddress, "the grpc address") - serveCmd.PersistentFlags().BoolVar(&serverArgs.KeepStaleWhenEmpty, "keepStaleWhenEmpty", false, "keep the stale service entry when there are no endpoints in the service") - serveCmd.PersistentFlags().StringVar(&serverArgs.RegistryOptions.ClusterRegistriesNamespace, "clusterRegistriesNamespace", - serverArgs.RegistryOptions.ClusterRegistriesNamespace, "Namespace for ConfigMap which stores clusters configs") - serveCmd.PersistentFlags().StringVar(&serverArgs.RegistryOptions.KubeConfig, "kubeconfig", "", - "Use a Kubernetes configuration file instead of in-cluster configuration") - // RegistryOptions Controller options - serveCmd.PersistentFlags().DurationVar(&serverArgs.RegistryOptions.KubeOptions.ResyncPeriod, "resync", 60*time.Second, - "Controller resync interval") - serveCmd.PersistentFlags().StringVar(&serverArgs.RegistryOptions.KubeOptions.DomainSuffix, "domain", constants.DefaultKubernetesDomain, - "DNS domain suffix") - serveCmd.PersistentFlags().StringVar((*string)(&serverArgs.RegistryOptions.KubeOptions.ClusterID), "clusterID", "Kubernetes", - "The ID of the cluster that this instance resides") - serveCmd.PersistentFlags().StringToStringVar(&serverArgs.RegistryOptions.KubeOptions.ClusterAliases, "clusterAliases", map[string]string{}, - "Alias names for clusters") - serveCmd.PersistentFlags().Float32Var(&serverArgs.RegistryOptions.KubeOptions.KubernetesAPIQPS, "kubernetesApiQPS", 80.0, - "Maximum QPS when communicating with the kubernetes API") - - serveCmd.PersistentFlags().IntVar(&serverArgs.RegistryOptions.KubeOptions.KubernetesAPIBurst, "kubernetesApiBurst", 160, - "Maximum burst for throttle when communicating with the kubernetes API") - - loggingOptions.AttachCobraFlags(serveCmd) - serverArgs.GrpcKeepAliveOptions.AttachCobraFlags(serveCmd) - - rootCmd.AddCommand(serveCmd) -} - func main() { - log.EnableKlogWithCobra() - if err := rootCmd.Execute(); err != nil { - log.Error(err) - os.Exit(-1) + if err := cmd.GetRootCommand().Execute(); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) } } diff --git a/go.mod b/go.mod index 1c62c7c9b..9aa6ada9e 100644 --- a/go.mod +++ b/go.mod @@ -27,11 +27,14 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/nacos-group/nacos-sdk-go v1.0.8 github.com/nacos-group/nacos-sdk-go/v2 v2.1.2 + github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.2.1 + github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.1 go.uber.org/atomic v1.9.0 google.golang.org/grpc v1.48.0 google.golang.org/protobuf v1.28.0 + gopkg.in/yaml.v2 v2.4.0 istio.io/api v0.0.0-20211122181927-8da52c66ff23 istio.io/client-go v1.12.0-rc.1.0.20211118171212-b744b6f111e4 istio.io/gogo-genproto v0.0.0-20211115195057-0e34bdd2be67 @@ -39,8 +42,11 @@ require ( istio.io/pkg v0.0.0-20211115195056-e379f31ee62a k8s.io/api v0.22.2 k8s.io/apimachinery v0.22.2 + k8s.io/cli-runtime v0.22.2 k8s.io/client-go v0.22.2 + k8s.io/kubectl v0.22.2 sigs.k8s.io/controller-runtime v0.10.2 + sigs.k8s.io/yaml v1.3.0 ) require ( @@ -142,7 +148,6 @@ require ( github.com/opencontainers/runc v1.0.2 // indirect github.com/openshift/api v0.0.0-20200713203337-b2494ecb17dd // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.12.2 // indirect github.com/prometheus/client_model v0.2.0 // indirect @@ -154,7 +159,6 @@ require ( github.com/sirupsen/logrus v1.8.1 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.3.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect github.com/toolkits/concurrent v0.0.0-20150624120057-a4371d70e3e3 // indirect github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect github.com/yl2chen/cidranger v1.0.2 // indirect @@ -181,21 +185,17 @@ require ( gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.22.2 // indirect - k8s.io/cli-runtime v0.22.2 // indirect k8s.io/component-base v0.22.2 // indirect k8s.io/klog/v2 v2.10.0 // indirect k8s.io/kube-openapi v0.0.0-20211020163157-7327e2aaee2b // indirect - k8s.io/kubectl v0.22.2 // indirect k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect sigs.k8s.io/gateway-api v0.4.0 // indirect sigs.k8s.io/kustomize/api v0.8.11 // indirect sigs.k8s.io/kustomize/kyaml v0.11.0 // indirect sigs.k8s.io/mcs-api v0.1.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect ) replace github.com/dubbogo/gost => github.com/johnlanni/gost v1.11.23-0.20220713132522-0967a24036c6 diff --git a/pkg/cmd/hgctl/kubernetes/client.go b/pkg/cmd/hgctl/kubernetes/client.go new file mode 100644 index 000000000..2093190e6 --- /dev/null +++ b/pkg/cmd/hgctl/kubernetes/client.go @@ -0,0 +1,172 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + "bytes" + "context" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + kubescheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" +) + +type CLIClient interface { + // RESTConfig returns the Kubernetes rest.Config used to configure the clients. + RESTConfig() *rest.Config + + // Pod returns the pod for the given namespaced name. + Pod(namespacedName types.NamespacedName) (*corev1.Pod, error) + + // PodsForSelector finds pods matching selector. + PodsForSelector(namespace string, labelSelectors ...string) (*corev1.PodList, error) + + // PodExec takes a command and the pod data to run the command in the specified pod. + PodExec(namespacedName types.NamespacedName, container string, command string) (stdout string, stderr string, err error) +} + +var _ CLIClient = &client{} + +type client struct { + config *rest.Config + restClient *rest.RESTClient + kube kubernetes.Interface +} + +func NewCLIClient(clientConfig clientcmd.ClientConfig) (CLIClient, error) { + return newClientInternal(clientConfig) +} + +func newClientInternal(clientConfig clientcmd.ClientConfig) (*client, error) { + var ( + c client + err error + ) + + c.config, err = clientConfig.ClientConfig() + if err != nil { + return nil, err + } + setRestDefaults(c.config) + + c.restClient, err = rest.RESTClientFor(c.config) + if err != nil { + return nil, err + } + + c.kube, err = kubernetes.NewForConfig(c.config) + if err != nil { + return nil, err + } + + return &c, err +} + +func setRestDefaults(config *rest.Config) *rest.Config { + if config.GroupVersion == nil || config.GroupVersion.Empty() { + config.GroupVersion = &corev1.SchemeGroupVersion + } + if len(config.APIPath) == 0 { + if len(config.GroupVersion.Group) == 0 { + config.APIPath = "/api" + } else { + config.APIPath = "/apis" + } + } + if len(config.ContentType) == 0 { + config.ContentType = runtime.ContentTypeJSON + } + if config.NegotiatedSerializer == nil { + // This codec factory ensures the resources are not converted. Therefore, resources + // will not be round-tripped through internal versions. Defaulting does not happen + // on the client. + config.NegotiatedSerializer = serializer.NewCodecFactory(kubescheme.Scheme).WithoutConversion() + } + + return config +} + +func (c *client) RESTConfig() *rest.Config { + if c.config == nil { + return nil + } + cpy := *c.config + return &cpy +} + +func (c *client) PodsForSelector(namespace string, podSelectors ...string) (*corev1.PodList, error) { + return c.kube.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: strings.Join(podSelectors, ","), + }) +} + +func (c *client) Pod(namespacedName types.NamespacedName) (*corev1.Pod, error) { + return c.kube.CoreV1().Pods(namespacedName.Namespace).Get(context.TODO(), namespacedName.Name, metav1.GetOptions{}) +} + +func (c *client) PodExec(namespacedName types.NamespacedName, container string, command string) (stdout string, stderr string, err error) { + defer func() { + if err != nil { + if len(stderr) > 0 { + err = fmt.Errorf("error exec into %s/%s container %s: %v\n%s", + namespacedName.Namespace, namespacedName.Name, container, err, stderr) + } else { + err = fmt.Errorf("error exec into %s/%s container %s: %v", + namespacedName.Namespace, namespacedName.Name, container, err) + } + } + }() + + req := c.restClient.Post(). + Resource("pods"). + Namespace(namespacedName.Namespace). + Name(namespacedName.Name). + SubResource("exec"). + Param("container", container). + VersionedParams(&corev1.PodExecOptions{ + Container: container, + Command: strings.Fields(command), + Stdin: false, + Stdout: true, + Stderr: true, + TTY: false, + }, kubescheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(c.config, "POST", req.URL()) + if err != nil { + return "", "", err + } + + var stdoutBuf, stderrBuf bytes.Buffer + err = exec.Stream(remotecommand.StreamOptions{ + Stdin: nil, + Stdout: &stdoutBuf, + Stderr: &stderrBuf, + Tty: false, + }) + + stdout = stdoutBuf.String() + stderr = stderrBuf.String() + return +} diff --git a/pkg/cmd/hgctl/kubernetes/port-forwarder.go b/pkg/cmd/hgctl/kubernetes/port-forwarder.go new file mode 100644 index 000000000..5d4093565 --- /dev/null +++ b/pkg/cmd/hgctl/kubernetes/port-forwarder.go @@ -0,0 +1,155 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + "fmt" + "io" + "net" + "net/http" + "os" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/transport/spdy" +) + +const ( + DefaultLocalAddress = "localhost" +) + +func LocalAvailablePort() (int, error) { + l, err := net.Listen("tcp", fmt.Sprintf("%s:0", DefaultLocalAddress)) + if err != nil { + return 0, err + } + + return l.Addr().(*net.TCPAddr).Port, l.Close() +} + +type PortForwarder interface { + Start() error + + Stop() + + // Address returns the address of the local forwarded address. + Address() string +} + +var _ PortForwarder = &localForwarder{} + +type localForwarder struct { + types.NamespacedName + CLIClient + + localPort int + podPort int + + stopCh chan struct{} +} + +func NewLocalPortForwarder(client CLIClient, namespacedName types.NamespacedName, localPort, podPort int) (PortForwarder, error) { + f := &localForwarder{ + stopCh: make(chan struct{}), + CLIClient: client, + NamespacedName: namespacedName, + localPort: localPort, + podPort: podPort, + } + if f.localPort == 0 { + // get a random port + p, err := LocalAvailablePort() + if err != nil { + return nil, errors.Wrapf(err, "failed to get a local available port") + } + f.localPort = p + } + + return f, nil +} + +func (f *localForwarder) Start() error { + errCh := make(chan error, 1) + readyCh := make(chan struct{}, 1) + go func() { + for { + select { + case <-f.stopCh: + return + default: + } + + fw, err := f.buildKubernetesPortForwarder(readyCh) + if err != nil { + errCh <- err + return + } + + if err := fw.ForwardPorts(); err != nil { + errCh <- err + return + } + + readyCh = nil + } + + }() + + select { + case err := <-errCh: + return errors.Wrap(err, "failed to start port forwarder") + case <-readyCh: + return nil + } +} + +func (f *localForwarder) buildKubernetesPortForwarder(readyCh chan struct{}) (*portforward.PortForwarder, error) { + restClient, err := rest.RESTClientFor(f.RESTConfig()) + if err != nil { + return nil, err + } + + req := restClient.Post().Resource("pods").Namespace(f.Namespace).Name(f.Name).SubResource("portforward") + serverURL := req.URL() + + roundTripper, upgrader, err := spdy.RoundTripperFor(f.RESTConfig()) + if err != nil { + return nil, fmt.Errorf("failure creating roundtripper: %v", err) + } + + dialer := spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, serverURL) + fw, err := portforward.NewOnAddresses(dialer, + []string{DefaultLocalAddress}, + []string{fmt.Sprintf("%d:%d", f.localPort, f.podPort)}, + f.stopCh, + readyCh, + io.Discard, + os.Stderr) + if err != nil { + return nil, fmt.Errorf("failed establishing portforward: %v", err) + } + + return fw, nil +} + +func (f *localForwarder) Stop() { + close(f.stopCh) +} + +func (f *localForwarder) Address() string { + return fmt.Sprintf("%s:%d", DefaultLocalAddress, f.localPort) +} diff --git a/pkg/cmd/hgctl/root.go b/pkg/cmd/hgctl/root.go new file mode 100644 index 000000000..ddfeed100 --- /dev/null +++ b/pkg/cmd/hgctl/root.go @@ -0,0 +1,32 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hgctl + +import "github.com/spf13/cobra" + +// GetRootCommand returns the root cobra command to be executed +// by hgctl main. +func GetRootCommand() *cobra.Command { + rootCmd := &cobra.Command{ + Use: "hgctl", + Long: "A command line utility for operating Higress", + SilenceUsage: true, + DisableAutoGenTag: true, + } + + rootCmd.AddCommand(newVersionCommand()) + + return rootCmd +} diff --git a/pkg/cmd/hgctl/version.go b/pkg/cmd/hgctl/version.go new file mode 100644 index 000000000..75b78467d --- /dev/null +++ b/pkg/cmd/hgctl/version.go @@ -0,0 +1,193 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hgctl + +import ( + "encoding/json" + "fmt" + "io" + "sort" + "strings" + + "github.com/alibaba/higress/pkg/cmd/hgctl/kubernetes" + "github.com/alibaba/higress/pkg/cmd/options" + "github.com/alibaba/higress/pkg/cmd/version" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "gopkg.in/yaml.v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + cmdutil "k8s.io/kubectl/pkg/cmd/util" +) + +const ( + yamlOutput = "yaml" + jsonOutput = "json" + higressCoreContainerName = "higress-core" + higressGatewayContainerName = "higress-gateway" +) + +func newVersionCommand() *cobra.Command { + var ( + output string + client bool + ) + + versionCommand := &cobra.Command{ + Use: "version", + Aliases: []string{"versions", "v"}, + Short: "Show version", + Example: ` # Show versions of both client and server. + hgctl version + + # Show versions of both client and server in JSON format. + hgctl version --output=json + + # Show version of client without server. + hgctl version --client + `, + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(versions(cmd.OutOrStdout(), output, client)) + }, + } + + flags := versionCommand.Flags() + options.AddKubeConfigFlags(flags) + + versionCommand.PersistentFlags().StringVarP(&output, "output", "o", yamlOutput, "One of 'yaml' or 'json'") + + versionCommand.PersistentFlags().BoolVarP(&client, "client", "r", false, "If true, only log client version.") + + return versionCommand +} + +type VersionInfo struct { + ClientVersion string `json:"client" yaml:"client"` + ServerVersions []*ServerVersion `json:"server,omitempty" yaml:"server"` +} + +type ServerVersion struct { + types.NamespacedName `yaml:"namespacedName"` + version.Info `yaml:"versionInfo"` +} + +func Get() VersionInfo { + return VersionInfo{ + ClientVersion: version.Get().HigressVersion, + ServerVersions: make([]*ServerVersion, 0), + } +} + +func retrieveVersion(w io.Writer, v *VersionInfo, containerName string, cmd string, labelSelector string, c kubernetes.CLIClient, f versionFunc) error { + pods, err := c.PodsForSelector(metav1.NamespaceAll, labelSelector) + if err != nil { + return errors.Wrap(err, "list Higress pods failed") + } + + for _, pod := range pods.Items { + if pod.Status.Phase != v1.PodRunning { + + fmt.Fprintf(w, "WARN: pod %s/%s is not running, skipping it.", pod.Namespace, pod.Name) + continue + } + + nn := types.NamespacedName{ + Namespace: pod.Namespace, + Name: pod.Name, + } + stdout, _, err := c.PodExec(nn, containerName, cmd) + if err != nil { + return fmt.Errorf("pod exec on %s/%s failed: %w", nn.Namespace, nn.Name, err) + } + + info, err := f(stdout) + if err != nil { + return err + } + + v.ServerVersions = append(v.ServerVersions, &ServerVersion{ + NamespacedName: nn, + Info: *info, + }) + } + + return nil +} + +type versionFunc func(string) (*version.Info, error) + +func versions(w io.Writer, output string, client bool) error { + v := Get() + + if client { + fmt.Fprintf(w, "clientVersion: %s", v.ClientVersion) + return nil + } + + c, err := kubernetes.NewCLIClient(options.DefaultConfigFlags.ToRawKubeConfigLoader()) + if err != nil { + return fmt.Errorf("failed to build kubernetes client: %w", err) + } + + if err := retrieveVersion(w, &v, higressCoreContainerName, "higress version -ojson", "app=higress-controller", c, func(s string) (*version.Info, error) { + info := &version.Info{} + if err := json.Unmarshal([]byte(s), info); err != nil { + return nil, fmt.Errorf("unmarshall pod exec result failed: %w", err) + } + info.Type = "higress-controller" + return info, nil + }); err != nil { + return err + } + + if err := retrieveVersion(w, &v, higressGatewayContainerName, "envoy --version", "app=higress-gateway", c, func(s string) (*version.Info, error) { + if len(strings.Split(s, ":")) != 2 { + return nil, nil + } + proxyVersion := strings.TrimSpace(strings.Split(s, ":")[1]) + return &version.Info{ + GatewayVersion: proxyVersion, + Type: "higress-gateway", + }, nil + }); err != nil { + return err + } + + sort.Slice(v.ServerVersions, func(i, j int) bool { + if v.ServerVersions[i].Namespace == v.ServerVersions[j].Namespace { + return v.ServerVersions[i].Name < v.ServerVersions[j].Name + } + + return v.ServerVersions[i].Namespace < v.ServerVersions[j].Namespace + }) + + var out []byte + switch output { + case yamlOutput: + out, err = yaml.Marshal(v) + case jsonOutput: + out, err = json.MarshalIndent(v, "", " ") + default: + out, err = json.MarshalIndent(v, "", " ") + } + + if err != nil { + return err + } + fmt.Fprintln(w, string(out)) + + return nil +} diff --git a/pkg/cmd/options/global.go b/pkg/cmd/options/global.go new file mode 100644 index 000000000..056c30523 --- /dev/null +++ b/pkg/cmd/options/global.go @@ -0,0 +1,29 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +import ( + "github.com/spf13/pflag" + "k8s.io/cli-runtime/pkg/genericclioptions" +) + +var DefaultConfigFlags = genericclioptions.NewConfigFlags(true) + +func AddKubeConfigFlags(flags *pflag.FlagSet) { + flags.StringVar(DefaultConfigFlags.KubeConfig, "kubeconfig", *DefaultConfigFlags.KubeConfig, + "Path to the kubeconfig file to use for CLI requests.") + flags.StringVar(DefaultConfigFlags.Context, "context", *DefaultConfigFlags.Context, + "The name of the kubeconfig context to use.") +} diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go new file mode 100644 index 000000000..f4faa5256 --- /dev/null +++ b/pkg/cmd/root.go @@ -0,0 +1,32 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import "github.com/spf13/cobra" + +// GetRootCommand returns the root cobra command to be executed +// by main. +func GetRootCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "higress", + Short: "Higress", + Long: "Next-generation Cloud Native Gateway", + } + + cmd.AddCommand(getServerCommand()) + cmd.AddCommand(getVersionCommand()) + + return cmd +} diff --git a/pkg/cmd/server.go b/pkg/cmd/server.go new file mode 100644 index 000000000..948db6da8 --- /dev/null +++ b/pkg/cmd/server.go @@ -0,0 +1,120 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "time" + + "github.com/alibaba/higress/pkg/bootstrap" + innerconstants "github.com/alibaba/higress/pkg/config/constants" + "github.com/spf13/cobra" + "istio.io/istio/pilot/pkg/features" + "istio.io/istio/pkg/cmd" + "istio.io/istio/pkg/config/constants" + "istio.io/istio/pkg/keepalive" + "istio.io/pkg/log" +) + +var ( + serverArgs *bootstrap.ServerArgs + loggingOptions = log.DefaultOptions() + + serverProvider = func(args *bootstrap.ServerArgs) (bootstrap.ServerInterface, error) { + return bootstrap.NewServer(serverArgs) + } + + waitForMonitorSignal = func(stop chan struct{}) { + cmd.WaitSignal(stop) + } +) + +// getServerCommand returns the server cobra command to be executed. +func getServerCommand() *cobra.Command { + serveCmd := &cobra.Command{ + Use: "serve", + Aliases: []string{"serve"}, + Short: "Starts the higress ingress controller", + Example: "higress serve", + PreRunE: func(c *cobra.Command, args []string) error { + return log.Configure(loggingOptions) + }, + RunE: func(c *cobra.Command, args []string) error { + cmd.PrintFlags(c.Flags()) + + stop := make(chan struct{}) + + server, err := serverProvider(serverArgs) + if err != nil { + return fmt.Errorf("fail to create higress server: %v", err) + } + + if err := server.Start(stop); err != nil { + return fmt.Errorf("fail to start higress server: %v", err) + } + + waitForMonitorSignal(stop) + + server.WaitUntilCompletion() + return nil + }, + } + + serverArgs = &bootstrap.ServerArgs{ + Debug: true, + NativeIstio: true, + HttpAddress: ":8888", + GrpcAddress: ":15051", + GrpcKeepAliveOptions: keepalive.DefaultOption(), + XdsOptions: bootstrap.XdsOptions{ + DebounceAfter: features.DebounceAfter, + DebounceMax: features.DebounceMax, + EnableEDSDebounce: features.EnableEDSDebounce, + }, + } + + serveCmd.PersistentFlags().StringVar(&serverArgs.GatewaySelectorKey, "gatewaySelectorKey", "higress", "gateway resource selector label key") + serveCmd.PersistentFlags().StringVar(&serverArgs.GatewaySelectorValue, "gatewaySelectorValue", "higress-gateway", "gateway resource selector label value") + serveCmd.PersistentFlags().BoolVar(&serverArgs.EnableStatus, "enableStatus", true, "enable the ingress status syncer which use to update the ip in ingress's status") + serveCmd.PersistentFlags().StringVar(&serverArgs.IngressClass, "ingressClass", innerconstants.DefaultIngressClass, "if not empty, only watch the ingresses have the specified class, otherwise watch all ingresses") + serveCmd.PersistentFlags().StringVar(&serverArgs.WatchNamespace, "watchNamespace", "", "if not empty, only wath the ingresses in the specified namespace, otherwise watch in all namespacees") + serveCmd.PersistentFlags().BoolVar(&serverArgs.Debug, "debug", serverArgs.Debug, "if true, enables more debug http api") + serveCmd.PersistentFlags().StringVar(&serverArgs.HttpAddress, "httpAddress", serverArgs.HttpAddress, "the http address") + serveCmd.PersistentFlags().StringVar(&serverArgs.GrpcAddress, "grpcAddress", serverArgs.GrpcAddress, "the grpc address") + serveCmd.PersistentFlags().BoolVar(&serverArgs.KeepStaleWhenEmpty, "keepStaleWhenEmpty", false, "keep the stale service entry when there are no endpoints in the service") + serveCmd.PersistentFlags().StringVar(&serverArgs.RegistryOptions.ClusterRegistriesNamespace, "clusterRegistriesNamespace", + serverArgs.RegistryOptions.ClusterRegistriesNamespace, "Namespace for ConfigMap which stores clusters configs") + serveCmd.PersistentFlags().StringVar(&serverArgs.RegistryOptions.KubeConfig, "kubeconfig", "", + "Use a Kubernetes configuration file instead of in-cluster configuration") + // RegistryOptions Controller options + serveCmd.PersistentFlags().DurationVar(&serverArgs.RegistryOptions.KubeOptions.ResyncPeriod, "resync", 60*time.Second, + "Controller resync interval") + serveCmd.PersistentFlags().StringVar(&serverArgs.RegistryOptions.KubeOptions.DomainSuffix, "domain", constants.DefaultKubernetesDomain, + "DNS domain suffix") + serveCmd.PersistentFlags().StringVar((*string)(&serverArgs.RegistryOptions.KubeOptions.ClusterID), "clusterID", "Kubernetes", + "The ID of the cluster that this instance resides") + serveCmd.PersistentFlags().StringToStringVar(&serverArgs.RegistryOptions.KubeOptions.ClusterAliases, "clusterAliases", map[string]string{}, + "Alias names for clusters") + serveCmd.PersistentFlags().Float32Var(&serverArgs.RegistryOptions.KubeOptions.KubernetesAPIQPS, "kubernetesApiQPS", 80.0, + "Maximum QPS when communicating with the kubernetes API") + + serveCmd.PersistentFlags().IntVar(&serverArgs.RegistryOptions.KubeOptions.KubernetesAPIBurst, "kubernetesApiBurst", 160, + "Maximum burst for throttle when communicating with the kubernetes API") + + loggingOptions.AttachCobraFlags(serveCmd) + serverArgs.GrpcKeepAliveOptions.AttachCobraFlags(serveCmd) + + return serveCmd +} diff --git a/cmd/higress/main_test.go b/pkg/cmd/server_test.go similarity index 96% rename from cmd/higress/main_test.go rename to pkg/cmd/server_test.go index 52f54423d..785a911d7 100644 --- a/cmd/higress/main_test.go +++ b/pkg/cmd/server_test.go @@ -12,18 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package cmd import ( - "github.com/alibaba/higress/pkg/bootstrap" - "github.com/spf13/cobra" - "github.com/stretchr/testify/assert" "os" "testing" "time" + + "github.com/alibaba/higress/pkg/bootstrap" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" ) func TestServe(t *testing.T) { + serveCmd := getServerCommand() runEBackup := serveCmd.RunE argsBackup := os.Args serverProviderBackup := serverProvider @@ -53,7 +55,9 @@ func TestServe(t *testing.T) { time.Sleep(delay) close(stop) } - main() + + serveCmd.Execute() + end := time.Now() cost := end.Sub(start) diff --git a/pkg/cmd/version.go b/pkg/cmd/version.go new file mode 100644 index 000000000..94e4121fe --- /dev/null +++ b/pkg/cmd/version.go @@ -0,0 +1,38 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/alibaba/higress/pkg/cmd/version" + "github.com/spf13/cobra" +) + +// getVersionCommand returns the version cobra command to be executed. +func getVersionCommand() *cobra.Command { + var output string + + cmd := &cobra.Command{ + Use: "version", + Aliases: []string{"versions", "v"}, + Short: "Show versions", + RunE: func(cmd *cobra.Command, args []string) error { + return version.Print(cmd.OutOrStdout(), output) + }, + } + + cmd.PersistentFlags().StringVarP(&output, "output", "o", "", "One of 'yaml' or 'json'") + + return cmd +} diff --git a/pkg/cmd/version/version.go b/pkg/cmd/version/version.go new file mode 100644 index 000000000..1736efbc6 --- /dev/null +++ b/pkg/cmd/version/version.go @@ -0,0 +1,62 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package version + +import ( + "encoding/json" + "fmt" + "io" + + "sigs.k8s.io/yaml" +) + +type Info struct { + Type string `json:"type,omitempty" yaml:"type,omitempty"` + HigressVersion string `json:"higressVersion,omitempty" yaml:"higressVersion,omitempty"` + GitCommitID string `json:"gitCommitID,omitempty" yaml:"gitCommitID,omitempty"` + GatewayVersion string `json:"gatewayVersion,omitempty" yaml:"gatewayVersion,omitempty"` +} + +func Get() Info { + return Info{ + HigressVersion: higressVersion, + GitCommitID: gitCommitID, + } +} + +var ( + higressVersion string + gitCommitID string +) + +// Print shows the versions of the Envoy Gateway. +func Print(w io.Writer, format string) error { + v := Get() + switch format { + case "json": + if marshalled, err := json.MarshalIndent(v, "", " "); err == nil { + _, _ = fmt.Fprintln(w, string(marshalled)) + } + case "yaml": + if marshalled, err := yaml.Marshal(v); err == nil { + _, _ = fmt.Fprintln(w, string(marshalled)) + } + default: + _, _ = fmt.Fprintf(w, "HIGRESS_VERSION: %s\n", v.HigressVersion) + _, _ = fmt.Fprintf(w, "GIT_COMMIT_ID: %s\n", v.GitCommitID) + } + + return nil +}