feat: init support hgctl (#273)

Signed-off-by: bitliu <bitliu@tencent.com>

pkg/cmd/hgctl/kubernetes/client.go

@@ -0,0 +1,172 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	kubescheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/remotecommand"
+)
+
+type CLIClient interface {
+	// RESTConfig returns the Kubernetes rest.Config used to configure the clients.
+	RESTConfig() *rest.Config
+
+	// Pod returns the pod for the given namespaced name.
+	Pod(namespacedName types.NamespacedName) (*corev1.Pod, error)
+
+	// PodsForSelector finds pods matching selector.
+	PodsForSelector(namespace string, labelSelectors ...string) (*corev1.PodList, error)
+
+	// PodExec takes a command and the pod data to run the command in the specified pod.
+	PodExec(namespacedName types.NamespacedName, container string, command string) (stdout string, stderr string, err error)
+}
+
+var _ CLIClient = &client{}
+
+type client struct {
+	config     *rest.Config
+	restClient *rest.RESTClient
+	kube       kubernetes.Interface
+}
+
+func NewCLIClient(clientConfig clientcmd.ClientConfig) (CLIClient, error) {
+	return newClientInternal(clientConfig)
+}
+
+func newClientInternal(clientConfig clientcmd.ClientConfig) (*client, error) {
+	var (
+		c   client
+		err error
+	)
+
+	c.config, err = clientConfig.ClientConfig()
+	if err != nil {
+		return nil, err
+	}
+	setRestDefaults(c.config)
+
+	c.restClient, err = rest.RESTClientFor(c.config)
+	if err != nil {
+		return nil, err
+	}
+
+	c.kube, err = kubernetes.NewForConfig(c.config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &c, err
+}
+
+func setRestDefaults(config *rest.Config) *rest.Config {
+	if config.GroupVersion == nil || config.GroupVersion.Empty() {
+		config.GroupVersion = &corev1.SchemeGroupVersion
+	}
+	if len(config.APIPath) == 0 {
+		if len(config.GroupVersion.Group) == 0 {
+			config.APIPath = "/api"
+		} else {
+			config.APIPath = "/apis"
+		}
+	}
+	if len(config.ContentType) == 0 {
+		config.ContentType = runtime.ContentTypeJSON
+	}
+	if config.NegotiatedSerializer == nil {
+		// This codec factory ensures the resources are not converted. Therefore, resources
+		// will not be round-tripped through internal versions. Defaulting does not happen
+		// on the client.
+		config.NegotiatedSerializer = serializer.NewCodecFactory(kubescheme.Scheme).WithoutConversion()
+	}
+
+	return config
+}
+
+func (c *client) RESTConfig() *rest.Config {
+	if c.config == nil {
+		return nil
+	}
+	cpy := *c.config
+	return &cpy
+}
+
+func (c *client) PodsForSelector(namespace string, podSelectors ...string) (*corev1.PodList, error) {
+	return c.kube.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{
+		LabelSelector: strings.Join(podSelectors, ","),
+	})
+}
+
+func (c *client) Pod(namespacedName types.NamespacedName) (*corev1.Pod, error) {
+	return c.kube.CoreV1().Pods(namespacedName.Namespace).Get(context.TODO(), namespacedName.Name, metav1.GetOptions{})
+}
+
+func (c *client) PodExec(namespacedName types.NamespacedName, container string, command string) (stdout string, stderr string, err error) {
+	defer func() {
+		if err != nil {
+			if len(stderr) > 0 {
+				err = fmt.Errorf("error exec into %s/%s container %s: %v\n%s",
+					namespacedName.Namespace, namespacedName.Name, container, err, stderr)
+			} else {
+				err = fmt.Errorf("error exec into %s/%s container %s: %v",
+					namespacedName.Namespace, namespacedName.Name, container, err)
+			}
+		}
+	}()
+
+	req := c.restClient.Post().
+		Resource("pods").
+		Namespace(namespacedName.Namespace).
+		Name(namespacedName.Name).
+		SubResource("exec").
+		Param("container", container).
+		VersionedParams(&corev1.PodExecOptions{
+			Container: container,
+			Command:   strings.Fields(command),
+			Stdin:     false,
+			Stdout:    true,
+			Stderr:    true,
+			TTY:       false,
+		}, kubescheme.ParameterCodec)
+
+	exec, err := remotecommand.NewSPDYExecutor(c.config, "POST", req.URL())
+	if err != nil {
+		return "", "", err
+	}
+
+	var stdoutBuf, stderrBuf bytes.Buffer
+	err = exec.Stream(remotecommand.StreamOptions{
+		Stdin:  nil,
+		Stdout: &stdoutBuf,
+		Stderr: &stderrBuf,
+		Tty:    false,
+	})
+
+	stdout = stdoutBuf.String()
+	stderr = stderrBuf.String()
+	return
+}

pkg/cmd/hgctl/kubernetes/port-forwarder.go

@@ -0,0 +1,155 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/portforward"
+	"k8s.io/client-go/transport/spdy"
+)
+
+const (
+	DefaultLocalAddress = "localhost"
+)
+
+func LocalAvailablePort() (int, error) {
+	l, err := net.Listen("tcp", fmt.Sprintf("%s:0", DefaultLocalAddress))
+	if err != nil {
+		return 0, err
+	}
+
+	return l.Addr().(*net.TCPAddr).Port, l.Close()
+}
+
+type PortForwarder interface {
+	Start() error
+
+	Stop()
+
+	// Address returns the address of the local forwarded address.
+	Address() string
+}
+
+var _ PortForwarder = &localForwarder{}
+
+type localForwarder struct {
+	types.NamespacedName
+	CLIClient
+
+	localPort int
+	podPort   int
+
+	stopCh chan struct{}
+}
+
+func NewLocalPortForwarder(client CLIClient, namespacedName types.NamespacedName, localPort, podPort int) (PortForwarder, error) {
+	f := &localForwarder{
+		stopCh:         make(chan struct{}),
+		CLIClient:      client,
+		NamespacedName: namespacedName,
+		localPort:      localPort,
+		podPort:        podPort,
+	}
+	if f.localPort == 0 {
+		// get a random port
+		p, err := LocalAvailablePort()
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get a local available port")
+		}
+		f.localPort = p
+	}
+
+	return f, nil
+}
+
+func (f *localForwarder) Start() error {
+	errCh := make(chan error, 1)
+	readyCh := make(chan struct{}, 1)
+	go func() {
+		for {
+			select {
+			case <-f.stopCh:
+				return
+			default:
+			}
+
+			fw, err := f.buildKubernetesPortForwarder(readyCh)
+			if err != nil {
+				errCh <- err
+				return
+			}
+
+			if err := fw.ForwardPorts(); err != nil {
+				errCh <- err
+				return
+			}
+
+			readyCh = nil
+		}
+
+	}()
+
+	select {
+	case err := <-errCh:
+		return errors.Wrap(err, "failed to start port forwarder")
+	case <-readyCh:
+		return nil
+	}
+}
+
+func (f *localForwarder) buildKubernetesPortForwarder(readyCh chan struct{}) (*portforward.PortForwarder, error) {
+	restClient, err := rest.RESTClientFor(f.RESTConfig())
+	if err != nil {
+		return nil, err
+	}
+
+	req := restClient.Post().Resource("pods").Namespace(f.Namespace).Name(f.Name).SubResource("portforward")
+	serverURL := req.URL()
+
+	roundTripper, upgrader, err := spdy.RoundTripperFor(f.RESTConfig())
+	if err != nil {
+		return nil, fmt.Errorf("failure creating roundtripper: %v", err)
+	}
+
+	dialer := spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, serverURL)
+	fw, err := portforward.NewOnAddresses(dialer,
+		[]string{DefaultLocalAddress},
+		[]string{fmt.Sprintf("%d:%d", f.localPort, f.podPort)},
+		f.stopCh,
+		readyCh,
+		io.Discard,
+		os.Stderr)
+	if err != nil {
+		return nil, fmt.Errorf("failed establishing portforward: %v", err)
+	}
+
+	return fw, nil
+}
+
+func (f *localForwarder) Stop() {
+	close(f.stopCh)
+}
+
+func (f *localForwarder) Address() string {
+	return fmt.Sprintf("%s:%d", DefaultLocalAddress, f.localPort)
+}

pkg/cmd/hgctl/root.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hgctl
+
+import "github.com/spf13/cobra"
+
+// GetRootCommand returns the root cobra command to be executed
+// by hgctl main.
+func GetRootCommand() *cobra.Command {
+	rootCmd := &cobra.Command{
+		Use:               "hgctl",
+		Long:              "A command line utility for operating Higress",
+		SilenceUsage:      true,
+		DisableAutoGenTag: true,
+	}
+
+	rootCmd.AddCommand(newVersionCommand())
+
+	return rootCmd
+}

pkg/cmd/hgctl/version.go

@@ -0,0 +1,193 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hgctl
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+
+	"github.com/alibaba/higress/pkg/cmd/hgctl/kubernetes"
+	"github.com/alibaba/higress/pkg/cmd/options"
+	"github.com/alibaba/higress/pkg/cmd/version"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+)
+
+const (
+	yamlOutput                  = "yaml"
+	jsonOutput                  = "json"
+	higressCoreContainerName    = "higress-core"
+	higressGatewayContainerName = "higress-gateway"
+)
+
+func newVersionCommand() *cobra.Command {
+	var (
+		output string
+		client bool
+	)
+
+	versionCommand := &cobra.Command{
+		Use:     "version",
+		Aliases: []string{"versions", "v"},
+		Short:   "Show version",
+		Example: `  # Show versions of both client and server.
+  hgctl version
+
+  # Show versions of both client and server in JSON format.
+  hgctl version --output=json
+
+  # Show version of client without server.
+  hgctl version --client
+	  `,
+		Run: func(cmd *cobra.Command, args []string) {
+			cmdutil.CheckErr(versions(cmd.OutOrStdout(), output, client))
+		},
+	}
+
+	flags := versionCommand.Flags()
+	options.AddKubeConfigFlags(flags)
+
+	versionCommand.PersistentFlags().StringVarP(&output, "output", "o", yamlOutput, "One of 'yaml' or 'json'")
+
+	versionCommand.PersistentFlags().BoolVarP(&client, "client", "r", false, "If true, only log client version.")
+
+	return versionCommand
+}
+
+type VersionInfo struct {
+	ClientVersion  string           `json:"client" yaml:"client"`
+	ServerVersions []*ServerVersion `json:"server,omitempty" yaml:"server"`
+}
+
+type ServerVersion struct {
+	types.NamespacedName `yaml:"namespacedName"`
+	version.Info         `yaml:"versionInfo"`
+}
+
+func Get() VersionInfo {
+	return VersionInfo{
+		ClientVersion:  version.Get().HigressVersion,
+		ServerVersions: make([]*ServerVersion, 0),
+	}
+}
+
+func retrieveVersion(w io.Writer, v *VersionInfo, containerName string, cmd string, labelSelector string, c kubernetes.CLIClient, f versionFunc) error {
+	pods, err := c.PodsForSelector(metav1.NamespaceAll, labelSelector)
+	if err != nil {
+		return errors.Wrap(err, "list Higress pods failed")
+	}
+
+	for _, pod := range pods.Items {
+		if pod.Status.Phase != v1.PodRunning {
+
+			fmt.Fprintf(w, "WARN: pod %s/%s is not running, skipping it.", pod.Namespace, pod.Name)
+			continue
+		}
+
+		nn := types.NamespacedName{
+			Namespace: pod.Namespace,
+			Name:      pod.Name,
+		}
+		stdout, _, err := c.PodExec(nn, containerName, cmd)
+		if err != nil {
+			return fmt.Errorf("pod exec on %s/%s failed: %w", nn.Namespace, nn.Name, err)
+		}
+
+		info, err := f(stdout)
+		if err != nil {
+			return err
+		}
+
+		v.ServerVersions = append(v.ServerVersions, &ServerVersion{
+			NamespacedName: nn,
+			Info:           *info,
+		})
+	}
+
+	return nil
+}
+
+type versionFunc func(string) (*version.Info, error)
+
+func versions(w io.Writer, output string, client bool) error {
+	v := Get()
+
+	if client {
+		fmt.Fprintf(w, "clientVersion: %s", v.ClientVersion)
+		return nil
+	}
+
+	c, err := kubernetes.NewCLIClient(options.DefaultConfigFlags.ToRawKubeConfigLoader())
+	if err != nil {
+		return fmt.Errorf("failed to build kubernetes client: %w", err)
+	}
+
+	if err := retrieveVersion(w, &v, higressCoreContainerName, "higress version -ojson", "app=higress-controller", c, func(s string) (*version.Info, error) {
+		info := &version.Info{}
+		if err := json.Unmarshal([]byte(s), info); err != nil {
+			return nil, fmt.Errorf("unmarshall pod exec result failed: %w", err)
+		}
+		info.Type = "higress-controller"
+		return info, nil
+	}); err != nil {
+		return err
+	}
+
+	if err := retrieveVersion(w, &v, higressGatewayContainerName, "envoy --version", "app=higress-gateway", c, func(s string) (*version.Info, error) {
+		if len(strings.Split(s, ":")) != 2 {
+			return nil, nil
+		}
+		proxyVersion := strings.TrimSpace(strings.Split(s, ":")[1])
+		return &version.Info{
+			GatewayVersion: proxyVersion,
+			Type:           "higress-gateway",
+		}, nil
+	}); err != nil {
+		return err
+	}
+
+	sort.Slice(v.ServerVersions, func(i, j int) bool {
+		if v.ServerVersions[i].Namespace == v.ServerVersions[j].Namespace {
+			return v.ServerVersions[i].Name < v.ServerVersions[j].Name
+		}
+
+		return v.ServerVersions[i].Namespace < v.ServerVersions[j].Namespace
+	})
+
+	var out []byte
+	switch output {
+	case yamlOutput:
+		out, err = yaml.Marshal(v)
+	case jsonOutput:
+		out, err = json.MarshalIndent(v, "", "  ")
+	default:
+		out, err = json.MarshalIndent(v, "", "  ")
+	}
+
+	if err != nil {
+		return err
+	}
+	fmt.Fprintln(w, string(out))
+
+	return nil
+}