feat: init support hgctl (#273)

Signed-off-by: bitliu <bitliu@tencent.com>

pkg/cmd/hgctl/kubernetes/client.go

@@ -0,0 +1,172 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	kubescheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/remotecommand"
+)
+
+type CLIClient interface {
+	// RESTConfig returns the Kubernetes rest.Config used to configure the clients.
+	RESTConfig() *rest.Config
+
+	// Pod returns the pod for the given namespaced name.
+	Pod(namespacedName types.NamespacedName) (*corev1.Pod, error)
+
+	// PodsForSelector finds pods matching selector.
+	PodsForSelector(namespace string, labelSelectors ...string) (*corev1.PodList, error)
+
+	// PodExec takes a command and the pod data to run the command in the specified pod.
+	PodExec(namespacedName types.NamespacedName, container string, command string) (stdout string, stderr string, err error)
+}
+
+var _ CLIClient = &client{}
+
+type client struct {
+	config     *rest.Config
+	restClient *rest.RESTClient
+	kube       kubernetes.Interface
+}
+
+func NewCLIClient(clientConfig clientcmd.ClientConfig) (CLIClient, error) {
+	return newClientInternal(clientConfig)
+}
+
+func newClientInternal(clientConfig clientcmd.ClientConfig) (*client, error) {
+	var (
+		c   client
+		err error
+	)
+
+	c.config, err = clientConfig.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+	setRestDefaults(c.config)
+
+	c.restClient, err = rest.RESTClientFor(c.config)
+	if err != nil {
+		return nil, err
+	}
+
+	c.kube, err = kubernetes.NewForConfig(c.config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &c, err
+}
+
+func setRestDefaults(config *rest.Config) *rest.Config {
+	if config.GroupVersion == nil || config.GroupVersion.Empty() {
+		config.GroupVersion = &corev1.SchemeGroupVersion
+	}
+	if len(config.APIPath) == 0 {
+		if len(config.GroupVersion.Group) == 0 {
+			config.APIPath = "/api"
+		} else {
+			config.APIPath = "/apis"
+		}
+	}
+	if len(config.ContentType) == 0 {
+		config.ContentType = runtime.ContentTypeJSON
+	}
+	if config.NegotiatedSerializer == nil {
+		// This codec factory ensures the resources are not converted. Therefore, resources
+		// will not be round-tripped through internal versions. Defaulting does not happen
+		// on the client.
+		config.NegotiatedSerializer = serializer.NewCodecFactory(kubescheme.Scheme).WithoutConversion()
+	}
+
+	return config
+}
+
+func (c *client) RESTConfig() *rest.Config {
+	if c.config == nil {
+		return nil
+	}
+	cpy := *c.config
+	return &cpy
+}
+
+func (c *client) PodsForSelector(namespace string, podSelectors ...string) (*corev1.PodList, error) {
+	return c.kube.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{
+		LabelSelector: strings.Join(podSelectors, ","),
+	})
+}
+
+func (c *client) Pod(namespacedName types.NamespacedName) (*corev1.Pod, error) {
+	return c.kube.CoreV1().Pods(namespacedName.Namespace).Get(context.TODO(), namespacedName.Name, metav1.GetOptions{})
+}
+
+func (c *client) PodExec(namespacedName types.NamespacedName, container string, command string) (stdout string, stderr string, err error) {
+	defer func() {
+		if err != nil {
+			if len(stderr) > 0 {
+				err = fmt.Errorf("error exec into %s/%s container %s: %v\n%s",
+					namespacedName.Namespace, namespacedName.Name, container, err, stderr)
+			} else {
+				err = fmt.Errorf("error exec into %s/%s container %s: %v",
+					namespacedName.Namespace, namespacedName.Name, container, err)
+			}
+		}
+	}()
+
+	req := c.restClient.Post().
+		Resource("pods").
+		Namespace(namespacedName.Namespace).
+		Name(namespacedName.Name).
+		SubResource("exec").
+		Param("container", container).
+		VersionedParams(&corev1.PodExecOptions{
+			Container: container,
+			Command:   strings.Fields(command),
+			Stdin:     false,
+			Stdout:    true,
+			Stderr:    true,
+			TTY:       false,
+		}, kubescheme.ParameterCodec)
+
+	exec, err := remotecommand.NewSPDYExecutor(c.config, "POST", req.URL())
+	if err != nil {
+		return "", "", err
+	}
+
+	var stdoutBuf, stderrBuf bytes.Buffer
+	err = exec.Stream(remotecommand.StreamOptions{
+		Stdin:  nil,
+		Stdout: &stdoutBuf,
+		Stderr: &stderrBuf,
+		Tty:    false,
+	})
+
+	stdout = stdoutBuf.String()
+	stderr = stderrBuf.String()
+	return
+}

pkg/cmd/hgctl/kubernetes/port-forwarder.go

@@ -0,0 +1,155 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/portforward"
+	"k8s.io/client-go/transport/spdy"
+)
+
+const (
+	DefaultLocalAddress = "localhost"
+)
+
+func LocalAvailablePort() (int, error) {
+	l, err := net.Listen("tcp", fmt.Sprintf("%s:0", DefaultLocalAddress))
+	if err != nil {
+		return 0, err
+	}
+
+	return l.Addr().(*net.TCPAddr).Port, l.Close()
+}
+
+type PortForwarder interface {
+	Start() error
+
+	Stop()
+
+	// Address returns the address of the local forwarded address.
+	Address() string
+}
+
+var _ PortForwarder = &localForwarder{}
+
+type localForwarder struct {
+	types.NamespacedName
+	CLIClient
+
+	localPort int
+	podPort   int
+
+	stopCh chan struct{}
+}
+
+func NewLocalPortForwarder(client CLIClient, namespacedName types.NamespacedName, localPort, podPort int) (PortForwarder, error) {
+	f := &localForwarder{
+		stopCh:         make(chan struct{}),
+		CLIClient:      client,
+		NamespacedName: namespacedName,
+		localPort:      localPort,
+		podPort:        podPort,
+	}
+	if f.localPort == 0 {
+		// get a random port
+		p, err := LocalAvailablePort()
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get a local available port")
+		}
+		f.localPort = p
+	}
+
+	return f, nil
+}
+
+func (f *localForwarder) Start() error {
+	errCh := make(chan error, 1)
+	readyCh := make(chan struct{}, 1)
+	go func() {
+		for {
+			select {
+			case <-f.stopCh:
+				return
+			default:
+			}
+
+			fw, err := f.buildKubernetesPortForwarder(readyCh)
+			if err != nil {
+				errCh <- err
+				return
+			}
+
+			if err := fw.ForwardPorts(); err != nil {
+				errCh <- err
+				return
+			}
+
+			readyCh = nil
+		}
+
+	}()
+
+	select {
+	case err := <-errCh:
+		return errors.Wrap(err, "failed to start port forwarder")
+	case <-readyCh:
+		return nil
+	}
+}
+
+func (f *localForwarder) buildKubernetesPortForwarder(readyCh chan struct{}) (*portforward.PortForwarder, error) {
+	restClient, err := rest.RESTClientFor(f.RESTConfig())
+	if err != nil {
+		return nil, err
+	}
+
+	req := restClient.Post().Resource("pods").Namespace(f.Namespace).Name(f.Name).SubResource("portforward")
+	serverURL := req.URL()
+
+	roundTripper, upgrader, err := spdy.RoundTripperFor(f.RESTConfig())
+	if err != nil {
+		return nil, fmt.Errorf("failure creating roundtripper: %v", err)
+	}
+
+	dialer := spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, serverURL)
+	fw, err := portforward.NewOnAddresses(dialer,
+		[]string{DefaultLocalAddress},
+		[]string{fmt.Sprintf("%d:%d", f.localPort, f.podPort)},
+		f.stopCh,
+		readyCh,
+		io.Discard,
+		os.Stderr)
+	if err != nil {
+		return nil, fmt.Errorf("failed establishing portforward: %v", err)
+	}
+
+	return fw, nil
+}
+
+func (f *localForwarder) Stop() {
+	close(f.stopCh)
+}
+
+func (f *localForwarder) Address() string {
+	return fmt.Sprintf("%s:%d", DefaultLocalAddress, f.localPort)
+}

pkg/cmd/hgctl/root.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hgctl
+
+import "github.com/spf13/cobra"
+
+// GetRootCommand returns the root cobra command to be executed
+// by hgctl main.
+func GetRootCommand() *cobra.Command {
+	rootCmd := &cobra.Command{
+		Use:               "hgctl",
+		Long:              "A command line utility for operating Higress",
+		SilenceUsage:      true,
+		DisableAutoGenTag: true,
+	}
+
+	rootCmd.AddCommand(newVersionCommand())
+
+	return rootCmd
+}

pkg/cmd/hgctl/version.go

@@ -0,0 +1,193 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hgctl
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+
+	"github.com/alibaba/higress/pkg/cmd/hgctl/kubernetes"
+	"github.com/alibaba/higress/pkg/cmd/options"
+	"github.com/alibaba/higress/pkg/cmd/version"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+)
+
+const (
+	yamlOutput                  = "yaml"
+	jsonOutput                  = "json"
+	higressCoreContainerName    = "higress-core"
+	higressGatewayContainerName = "higress-gateway"
+)
+
+func newVersionCommand() *cobra.Command {
+	var (
+		output string
+		client bool
+	)
+
+	versionCommand := &cobra.Command{
+		Use:     "version",
+		Aliases: []string{"versions", "v"},
+		Short:   "Show version",
+		Example: `  # Show versions of both client and server.
+  hgctl version
+
+  # Show versions of both client and server in JSON format.
+  hgctl version --output=json
+
+  # Show version of client without server.
+  hgctl version --client
+	  `,
+		Run: func(cmd *cobra.Command, args []string) {
+			cmdutil.CheckErr(versions(cmd.OutOrStdout(), output, client))
+		},
+	}
+
+	flags := versionCommand.Flags()
+	options.AddKubeConfigFlags(flags)
+
+	versionCommand.PersistentFlags().StringVarP(&output, "output", "o", yamlOutput, "One of 'yaml' or 'json'")
+
+	versionCommand.PersistentFlags().BoolVarP(&client, "client", "r", false, "If true, only log client version.")
+
+	return versionCommand
+}
+
+type VersionInfo struct {
+	ClientVersion  string           `json:"client" yaml:"client"`
+	ServerVersions []*ServerVersion `json:"server,omitempty" yaml:"server"`
+}
+
+type ServerVersion struct {
+	types.NamespacedName `yaml:"namespacedName"`
+	version.Info         `yaml:"versionInfo"`
+}
+
+func Get() VersionInfo {
+	return VersionInfo{
+		ClientVersion:  version.Get().HigressVersion,
+		ServerVersions: make([]*ServerVersion, 0),
+	}
+}
+
+func retrieveVersion(w io.Writer, v *VersionInfo, containerName string, cmd string, labelSelector string, c kubernetes.CLIClient, f versionFunc) error {
+	pods, err := c.PodsForSelector(metav1.NamespaceAll, labelSelector)
+	if err != nil {
+		return errors.Wrap(err, "list Higress pods failed")
+	}
+
+	for _, pod := range pods.Items {
+		if pod.Status.Phase != v1.PodRunning {
+
+			fmt.Fprintf(w, "WARN: pod %s/%s is not running, skipping it.", pod.Namespace, pod.Name)
+			continue
+		}
+
+		nn := types.NamespacedName{
+			Namespace: pod.Namespace,
+			Name:      pod.Name,
+		}
+		stdout, _, err := c.PodExec(nn, containerName, cmd)
+		if err != nil {
+			return fmt.Errorf("pod exec on %s/%s failed: %w", nn.Namespace, nn.Name, err)
+		}
+
+		info, err := f(stdout)
+		if err != nil {
+			return err
+		}
+
+		v.ServerVersions = append(v.ServerVersions, &ServerVersion{
+			NamespacedName: nn,
+			Info:           *info,
+		})
+	}
+
+	return nil
+}
+
+type versionFunc func(string) (*version.Info, error)
+
+func versions(w io.Writer, output string, client bool) error {
+	v := Get()
+
+	if client {
+		fmt.Fprintf(w, "clientVersion: %s", v.ClientVersion)
+		return nil
+	}
+
+	c, err := kubernetes.NewCLIClient(options.DefaultConfigFlags.ToRawKubeConfigLoader())
+	if err != nil {
+		return fmt.Errorf("failed to build kubernetes client: %w", err)
+	}
+
+	if err := retrieveVersion(w, &v, higressCoreContainerName, "higress version -ojson", "app=higress-controller", c, func(s string) (*version.Info, error) {
+		info := &version.Info{}
+		if err := json.Unmarshal([]byte(s), info); err != nil {
+			return nil, fmt.Errorf("unmarshall pod exec result failed: %w", err)
+		}
+		info.Type = "higress-controller"
+		return info, nil
+	}); err != nil {
+		return err
+	}
+
+	if err := retrieveVersion(w, &v, higressGatewayContainerName, "envoy --version", "app=higress-gateway", c, func(s string) (*version.Info, error) {
+		if len(strings.Split(s, ":")) != 2 {
+			return nil, nil
+		}
+		proxyVersion := strings.TrimSpace(strings.Split(s, ":")[1])
+		return &version.Info{
+			GatewayVersion: proxyVersion,
+			Type:           "higress-gateway",
+		}, nil
+	}); err != nil {
+		return err
+	}
+
+	sort.Slice(v.ServerVersions, func(i, j int) bool {
+		if v.ServerVersions[i].Namespace == v.ServerVersions[j].Namespace {
+			return v.ServerVersions[i].Name < v.ServerVersions[j].Name
+		}
+
+		return v.ServerVersions[i].Namespace < v.ServerVersions[j].Namespace
+	})
+
+	var out []byte
+	switch output {
+	case yamlOutput:
+		out, err = yaml.Marshal(v)
+	case jsonOutput:
+		out, err = json.MarshalIndent(v, "", "  ")
+	default:
+		out, err = json.MarshalIndent(v, "", "  ")
+	}
+
+	if err != nil {
+		return err
+	}
+	fmt.Fprintln(w, string(out))
+
+	return nil
+}

pkg/cmd/options/global.go

@@ -0,0 +1,29 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package options
+
+import (
+	"github.com/spf13/pflag"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+)
+
+var DefaultConfigFlags = genericclioptions.NewConfigFlags(true)
+
+func AddKubeConfigFlags(flags *pflag.FlagSet) {
+	flags.StringVar(DefaultConfigFlags.KubeConfig, "kubeconfig", *DefaultConfigFlags.KubeConfig,
+		"Path to the kubeconfig file to use for CLI requests.")
+	flags.StringVar(DefaultConfigFlags.Context, "context", *DefaultConfigFlags.Context,
+		"The name of the kubeconfig context to use.")
+}

pkg/cmd/root.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import "github.com/spf13/cobra"
+
+// GetRootCommand returns the root cobra command to be executed
+// by main.
+func GetRootCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "higress",
+		Short: "Higress",
+		Long:  "Next-generation Cloud Native Gateway",
+	}
+
+	cmd.AddCommand(getServerCommand())
+	cmd.AddCommand(getVersionCommand())
+
+	return cmd
+}

pkg/cmd/server.go

@@ -0,0 +1,120 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/alibaba/higress/pkg/bootstrap"
+	innerconstants "github.com/alibaba/higress/pkg/config/constants"
+	"github.com/spf13/cobra"
+	"istio.io/istio/pilot/pkg/features"
+	"istio.io/istio/pkg/cmd"
+	"istio.io/istio/pkg/config/constants"
+	"istio.io/istio/pkg/keepalive"
+	"istio.io/pkg/log"
+)
+
+var (
+	serverArgs     *bootstrap.ServerArgs
+	loggingOptions = log.DefaultOptions()
+
+	serverProvider = func(args *bootstrap.ServerArgs) (bootstrap.ServerInterface, error) {
+		return bootstrap.NewServer(serverArgs)
+	}
+
+	waitForMonitorSignal = func(stop chan struct{}) {
+		cmd.WaitSignal(stop)
+	}
+)
+
+// getServerCommand returns the server cobra command to be executed.
+func getServerCommand() *cobra.Command {
+	serveCmd := &cobra.Command{
+		Use:     "serve",
+		Aliases: []string{"serve"},
+		Short:   "Starts the higress ingress controller",
+		Example: "higress serve",
+		PreRunE: func(c *cobra.Command, args []string) error {
+			return log.Configure(loggingOptions)
+		},
+		RunE: func(c *cobra.Command, args []string) error {
+			cmd.PrintFlags(c.Flags())
+
+			stop := make(chan struct{})
+
+			server, err := serverProvider(serverArgs)
+			if err != nil {
+				return fmt.Errorf("fail to create higress server: %v", err)
+			}
+
+			if err := server.Start(stop); err != nil {
+				return fmt.Errorf("fail to start higress server: %v", err)
+			}
+
+			waitForMonitorSignal(stop)
+
+			server.WaitUntilCompletion()
+			return nil
+		},
+	}
+
+	serverArgs = &bootstrap.ServerArgs{
+		Debug:                true,
+		NativeIstio:          true,
+		HttpAddress:          ":8888",
+		GrpcAddress:          ":15051",
+		GrpcKeepAliveOptions: keepalive.DefaultOption(),
+		XdsOptions: bootstrap.XdsOptions{
+			DebounceAfter:     features.DebounceAfter,
+			DebounceMax:       features.DebounceMax,
+			EnableEDSDebounce: features.EnableEDSDebounce,
+		},
+	}
+
+	serveCmd.PersistentFlags().StringVar(&serverArgs.GatewaySelectorKey, "gatewaySelectorKey", "higress", "gateway resource selector label key")
+	serveCmd.PersistentFlags().StringVar(&serverArgs.GatewaySelectorValue, "gatewaySelectorValue", "higress-gateway", "gateway resource selector label value")
+	serveCmd.PersistentFlags().BoolVar(&serverArgs.EnableStatus, "enableStatus", true, "enable the ingress status syncer which use to update the ip in ingress's status")
+	serveCmd.PersistentFlags().StringVar(&serverArgs.IngressClass, "ingressClass", innerconstants.DefaultIngressClass, "if not empty, only watch the ingresses have the specified class, otherwise watch all ingresses")
+	serveCmd.PersistentFlags().StringVar(&serverArgs.WatchNamespace, "watchNamespace", "", "if not empty, only wath the ingresses in the specified namespace, otherwise watch in all namespacees")
+	serveCmd.PersistentFlags().BoolVar(&serverArgs.Debug, "debug", serverArgs.Debug, "if true, enables more debug http api")
+	serveCmd.PersistentFlags().StringVar(&serverArgs.HttpAddress, "httpAddress", serverArgs.HttpAddress, "the http address")
+	serveCmd.PersistentFlags().StringVar(&serverArgs.GrpcAddress, "grpcAddress", serverArgs.GrpcAddress, "the grpc address")
+	serveCmd.PersistentFlags().BoolVar(&serverArgs.KeepStaleWhenEmpty, "keepStaleWhenEmpty", false, "keep the stale service entry when there are no endpoints in the service")
+	serveCmd.PersistentFlags().StringVar(&serverArgs.RegistryOptions.ClusterRegistriesNamespace, "clusterRegistriesNamespace",
+		serverArgs.RegistryOptions.ClusterRegistriesNamespace, "Namespace for ConfigMap which stores clusters configs")
+	serveCmd.PersistentFlags().StringVar(&serverArgs.RegistryOptions.KubeConfig, "kubeconfig", "",
+		"Use a Kubernetes configuration file instead of in-cluster configuration")
+	// RegistryOptions Controller options
+	serveCmd.PersistentFlags().DurationVar(&serverArgs.RegistryOptions.KubeOptions.ResyncPeriod, "resync", 60*time.Second,
+		"Controller resync interval")
+	serveCmd.PersistentFlags().StringVar(&serverArgs.RegistryOptions.KubeOptions.DomainSuffix, "domain", constants.DefaultKubernetesDomain,
+		"DNS domain suffix")
+	serveCmd.PersistentFlags().StringVar((*string)(&serverArgs.RegistryOptions.KubeOptions.ClusterID), "clusterID", "Kubernetes",
+		"The ID of the cluster that this instance resides")
+	serveCmd.PersistentFlags().StringToStringVar(&serverArgs.RegistryOptions.KubeOptions.ClusterAliases, "clusterAliases", map[string]string{},
+		"Alias names for clusters")
+	serveCmd.PersistentFlags().Float32Var(&serverArgs.RegistryOptions.KubeOptions.KubernetesAPIQPS, "kubernetesApiQPS", 80.0,
+		"Maximum QPS when communicating with the kubernetes API")
+
+	serveCmd.PersistentFlags().IntVar(&serverArgs.RegistryOptions.KubeOptions.KubernetesAPIBurst, "kubernetesApiBurst", 160,
+		"Maximum burst for throttle when communicating with the kubernetes API")
+
+	loggingOptions.AttachCobraFlags(serveCmd)
+	serverArgs.GrpcKeepAliveOptions.AttachCobraFlags(serveCmd)
+
+	return serveCmd
+}

pkg/cmd/server_test.go

@@ -0,0 +1,82 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/alibaba/higress/pkg/bootstrap"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestServe(t *testing.T) {
+	serveCmd := getServerCommand()
+	runEBackup := serveCmd.RunE
+	argsBackup := os.Args
+	serverProviderBackup := serverProvider
+	executed := false
+
+	serverProvider = func(args *bootstrap.ServerArgs) (bootstrap.ServerInterface, error) {
+		return &delayedServer{Args: args, Delay: time.Second * 5}, nil
+	}
+
+	serveCmd.RunE = func(cmd *cobra.Command, args []string) error {
+		executed = true
+		return runEBackup(cmd, args)
+	}
+	defer func() {
+		serverProvider = serverProviderBackup
+		os.Args = argsBackup
+		serveCmd.RunE = runEBackup
+	}()
+
+	a := assert.New(t)
+
+	delay := time.Second * 5
+
+	start := time.Now()
+	os.Args = []string{"/app/higress", "serve"}
+	waitForMonitorSignal = func(stop chan struct{}) {
+		time.Sleep(delay)
+		close(stop)
+	}
+
+	serveCmd.Execute()
+
+	end := time.Now()
+
+	cost := end.Sub(start)
+	a.GreaterOrEqual(cost, delay)
+
+	a.True(executed)
+}
+
+type delayedServer struct {
+	Args  *bootstrap.ServerArgs
+	Delay time.Duration
+	stop  <-chan struct{}
+}
+
+func (d *delayedServer) Start(stop <-chan struct{}) error {
+	d.stop = stop
+	return nil
+}
+
+func (d *delayedServer) WaitUntilCompletion() {
+	<-d.stop
+}

pkg/cmd/version.go

@@ -0,0 +1,38 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"github.com/alibaba/higress/pkg/cmd/version"
+	"github.com/spf13/cobra"
+)
+
+// getVersionCommand returns the version cobra command to be executed.
+func getVersionCommand() *cobra.Command {
+	var output string
+
+	cmd := &cobra.Command{
+		Use:     "version",
+		Aliases: []string{"versions", "v"},
+		Short:   "Show versions",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return version.Print(cmd.OutOrStdout(), output)
+		},
+	}
+
+	cmd.PersistentFlags().StringVarP(&output, "output", "o", "", "One of 'yaml' or 'json'")
+
+	return cmd
+}

pkg/cmd/version/version.go

@@ -0,0 +1,62 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package version
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"sigs.k8s.io/yaml"
+)
+
+type Info struct {
+	Type           string `json:"type,omitempty" yaml:"type,omitempty"`
+	HigressVersion string `json:"higressVersion,omitempty" yaml:"higressVersion,omitempty"`
+	GitCommitID    string `json:"gitCommitID,omitempty" yaml:"gitCommitID,omitempty"`
+	GatewayVersion string `json:"gatewayVersion,omitempty" yaml:"gatewayVersion,omitempty"`
+}
+
+func Get() Info {
+	return Info{
+		HigressVersion: higressVersion,
+		GitCommitID:    gitCommitID,
+	}
+}
+
+var (
+	higressVersion string
+	gitCommitID    string
+)
+
+// Print shows the versions of the Envoy Gateway.
+func Print(w io.Writer, format string) error {
+	v := Get()
+	switch format {
+	case "json":
+		if marshalled, err := json.MarshalIndent(v, "", "  "); err == nil {
+			_, _ = fmt.Fprintln(w, string(marshalled))
+		}
+	case "yaml":
+		if marshalled, err := yaml.Marshal(v); err == nil {
+			_, _ = fmt.Fprintln(w, string(marshalled))
+		}
+	default:
+		_, _ = fmt.Fprintf(w, "HIGRESS_VERSION: %s\n", v.HigressVersion)
+		_, _ = fmt.Fprintf(w, "GIT_COMMIT_ID: %s\n", v.GitCommitID)
+	}
+
+	return nil
+}