fix: fix hgctl in high kubenetes version problem by auto detecting k8s version and adding helm lookup function (#629)

pkg/cmd/hgctl/helm/render.go

@@ -38,6 +38,7 @@ import (
 	"helm.sh/helm/v3/pkg/engine"
 	"helm.sh/helm/v3/pkg/getter"
 	"helm.sh/helm/v3/pkg/repo"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/yaml"
 )
 
@@ -134,6 +135,12 @@ type RendererOptions struct {
 	// fields for RemoteRenderer
 	Version string
 	RepoURL string
+
+	// Capabilities
+	Capabilities *chartutil.Capabilities
+
+	// rest config
+	restConfig *rest.Config
 }
 
 type RendererOption func(*RendererOptions)
@@ -174,6 +181,18 @@ func WithRepoURL(repo string) RendererOption {
 	}
 }
 
+func WithCapabilities(capabilities *chartutil.Capabilities) RendererOption {
+	return func(opts *RendererOptions) {
+		opts.Capabilities = capabilities
+	}
+}
+
+func WithRestConfig(config *rest.Config) RendererOption {
+	return func(opts *RendererOptions) {
+		opts.restConfig = config
+	}
+}
+
 // LocalFileRenderer load yaml files from local file system
 type LocalFileRenderer struct {
 	Opts     *RendererOptions
@@ -418,8 +437,11 @@ func renderManifest(valsYaml string, cht *chart.Chart, builtIn bool, opts *Rende
 		Name:      opts.Name,
 		Namespace: opts.Namespace,
 	}
-	// TODO need to specify k8s version
+	var caps *chartutil.Capabilities
-	caps := chartutil.DefaultCapabilities
+	caps = opts.Capabilities
+	if caps == nil {
+		caps = chartutil.DefaultCapabilities
+	}
 	// maybe we need a configuration to change this caps
 	resVals, err := chartutil.ToRenderValues(cht, valsMap, RelOpts, caps)
 	if err != nil {
@@ -428,7 +450,7 @@ func renderManifest(valsYaml string, cht *chart.Chart, builtIn bool, opts *Rende
 	if builtIn {
 		resVals["Values"].(chartutil.Values)["enabled"] = true
 	}
-	filesMap, err := engine.Render(cht, resVals)
+	filesMap, err := engine.RenderWithClient(cht, resVals, opts.restConfig)
 	if err != nil {
 		return "", fmt.Errorf("Render chart failed err: %s", err)
 	}

pkg/cmd/hgctl/installer/component.go

@@ -17,6 +17,7 @@ package installer
 import (
 	"github.com/alibaba/higress/pkg/cmd/hgctl/helm"
 	"github.com/alibaba/higress/pkg/cmd/hgctl/util"
+	"helm.sh/helm/v3/pkg/chartutil"
 	"sigs.k8s.io/yaml"
 )
 
@@ -49,6 +50,8 @@ type ComponentOptions struct {
 	ChartName string
 	Version   string
 	Quiet     bool
+	// Capabilities
+	Capabilities *chartutil.Capabilities
 }
 
 type ComponentOption func(*ComponentOptions)
@@ -83,6 +86,12 @@ func WithComponentVersion(version string) ComponentOption {
 	}
 }
 
+func WithComponentCapabilities(capabilities *chartutil.Capabilities) ComponentOption {
+	return func(opts *ComponentOptions) {
+		opts.Capabilities = capabilities
+	}
+}
+
 func WithQuiet() ComponentOption {
 	return func(opts *ComponentOptions) {
 		opts.Quiet = true

pkg/cmd/hgctl/installer/gateway_api.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 
 	"github.com/alibaba/higress/pkg/cmd/hgctl/helm"
+	"github.com/alibaba/higress/pkg/cmd/hgctl/kubernetes"
 	"github.com/alibaba/higress/pkg/cmd/hgctl/manifests"
 )
 
@@ -34,9 +35,10 @@ type GatewayAPIComponent struct {
 	opts     *ComponentOptions
 	renderer helm.Renderer
 	writer   io.Writer
+	kubeCli  kubernetes.CLIClient
 }
 
-func NewGatewayAPIComponent(profile *helm.Profile, writer io.Writer, opts ...ComponentOption) (Component, error) {
+func NewGatewayAPIComponent(kubeCli kubernetes.CLIClient, profile *helm.Profile, writer io.Writer, opts ...ComponentOption) (Component, error) {
 	newOpts := &ComponentOptions{}
 	for _, opt := range opts {
 		opt(newOpts)
@@ -55,6 +57,8 @@ func NewGatewayAPIComponent(profile *helm.Profile, writer io.Writer, opts ...Com
 		helm.WithVersion(newOpts.Version),
 		helm.WithFS(manifests.BuiltinOrDir("")),
 		helm.WithDir(chartDir),
+		helm.WithCapabilities(newOpts.Capabilities),
+		helm.WithRestConfig(kubeCli.RESTConfig()),
 	)
 	if err != nil {
 		return nil, err
@@ -65,6 +69,7 @@ func NewGatewayAPIComponent(profile *helm.Profile, writer io.Writer, opts ...Com
 		renderer: renderer,
 		opts:     newOpts,
 		writer:   writer,
+		kubeCli:  kubeCli,
 	}
 	return gatewayAPIComponent, nil
 }

pkg/cmd/hgctl/installer/higress.go

@@ -17,8 +17,10 @@ package installer
 import (
 	"errors"
 	"fmt"
-	"github.com/alibaba/higress/pkg/cmd/hgctl/helm"
 	"io"
+
+	"github.com/alibaba/higress/pkg/cmd/hgctl/helm"
+	"github.com/alibaba/higress/pkg/cmd/hgctl/kubernetes"
 )
 
 const (
@@ -31,6 +33,7 @@ type HigressComponent struct {
 	opts     *ComponentOptions
 	renderer helm.Renderer
 	writer   io.Writer
+	kubeCli  kubernetes.CLIClient
 }
 
 func (h *HigressComponent) ComponentName() ComponentName {
@@ -89,7 +92,7 @@ func (h *HigressComponent) RenderManifest() (string, error) {
 	return manifest, nil
 }
 
-func NewHigressComponent(profile *helm.Profile, writer io.Writer, opts ...ComponentOption) (Component, error) {
+func NewHigressComponent(kubeCli kubernetes.CLIClient, profile *helm.Profile, writer io.Writer, opts ...ComponentOption) (Component, error) {
 	newOpts := &ComponentOptions{}
 	for _, opt := range opts {
 		opt(newOpts)
@@ -105,6 +108,8 @@ func NewHigressComponent(profile *helm.Profile, writer io.Writer, opts ...Compon
 		helm.WithNamespace(newOpts.Namespace),
 		helm.WithRepoURL(newOpts.RepoURL),
 		helm.WithVersion(newOpts.Version),
+		helm.WithCapabilities(newOpts.Capabilities),
+		helm.WithRestConfig(kubeCli.RESTConfig()),
 	)
 	if err != nil {
 		return nil, err
@@ -115,6 +120,7 @@ func NewHigressComponent(profile *helm.Profile, writer io.Writer, opts ...Compon
 		renderer: renderer,
 		opts:     newOpts,
 		writer:   writer,
+		kubeCli:  kubeCli,
 	}
 	return higressComponent, nil
 }

pkg/cmd/hgctl/installer/installer_k8s.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 
 	"github.com/alibaba/higress/pkg/cmd/hgctl/helm"
 	"github.com/alibaba/higress/pkg/cmd/hgctl/helm/object"
@@ -202,6 +203,19 @@ func (o *K8sInstaller) DeleteManifests(manifestMap map[ComponentName]string) err
 	return nil
 }
 
+// WriteManifests write component manifests to local files
+func (o *K8sInstaller) WriteManifests(manifestMap map[ComponentName]string) error {
+	if o.kubeCli == nil {
+		return errors.New("no injected k8s cli into K8sInstaller")
+	}
+	rootPath, _ := os.Getwd()
+	for name, manifest := range manifestMap {
+		fileName := filepath.Join(rootPath, string(name)+".yaml")
+		util.WriteFileString(fileName, manifest, 0o644)
+	}
+	return nil
+}
+
 // deleteManifest delete manifest to certain namespace
 func (o *K8sInstaller) deleteManifest(manifest string, ns string) error {
 	objs, err := object.ParseK8sObjectsFromYAMLManifest(manifest)
@@ -239,6 +253,14 @@ func NewK8sInstaller(profile *helm.Profile, cli kubernetes.CLIClient, writer io.
 	if profile == nil {
 		return nil, errors.New("install profile is empty")
 	}
+	// initialize server info
+	serverInfo, _ := NewServerInfo(cli)
+	fmt.Fprintf(writer, "\n⌛️ Detecting kubernetes version ... ")
+	capabilities, err := serverInfo.GetCapabilities()
+	if err != nil {
+		return nil, err
+	}
+	fmt.Fprintf(writer, "%s\n", capabilities.KubeVersion.Version)
 	// initialize components
 	components := make(map[ComponentName]Component)
 	opts := []ComponentOption{
@@ -247,11 +269,12 @@ func NewK8sInstaller(profile *helm.Profile, cli kubernetes.CLIClient, writer io.
 		WithComponentVersion(profile.Charts.Higress.Version),
 		WithComponentRepoURL(profile.Charts.Higress.Url),
 		WithComponentChartName(profile.Charts.Higress.Name),
+		WithComponentCapabilities(capabilities),
 	}
 	if quiet {
 		opts = append(opts, WithQuiet())
 	}
-	higressComponent, err := NewHigressComponent(profile, writer, opts...)
+	higressComponent, err := NewHigressComponent(cli, profile, writer, opts...)
 	if err != nil {
 		return nil, fmt.Errorf("NewHigressComponent failed, err: %s", err)
 	}
@@ -267,12 +290,13 @@ func NewK8sInstaller(profile *helm.Profile, cli kubernetes.CLIClient, writer io.
 			WithComponentVersion("1.18.2"),
 			WithComponentRepoURL("embed://istiobase"),
 			WithComponentChartName("istio"),
+			WithComponentCapabilities(capabilities),
 		}
 		if quiet {
 			opts = append(opts, WithQuiet())
 		}
 
-		istioCRDComponent, err := NewIstioCRDComponent(profile, writer, opts...)
+		istioCRDComponent, err := NewIstioCRDComponent(cli, profile, writer, opts...)
 		if err != nil {
 			return nil, fmt.Errorf("NewIstioCRDComponent failed, err: %s", err)
 		}
@@ -285,12 +309,13 @@ func NewK8sInstaller(profile *helm.Profile, cli kubernetes.CLIClient, writer io.
 			WithComponentVersion("1.0.0"),
 			WithComponentRepoURL("embed://gatewayapi"),
 			WithComponentChartName("gatewayAPI"),
+			WithComponentCapabilities(capabilities),
 		}
 		if quiet {
 			opts = append(opts, WithQuiet())
 		}
 
-		gatewayAPIComponent, err := NewGatewayAPIComponent(profile, writer, opts...)
+		gatewayAPIComponent, err := NewGatewayAPIComponent(cli, profile, writer, opts...)
 		if err != nil {
 			return nil, fmt.Errorf("NewGatewayAPIComponent failed, err: %s", err)
 		}

pkg/cmd/hgctl/installer/istio.go

@@ -20,6 +20,7 @@ import (
 	"strings"
 
 	"github.com/alibaba/higress/pkg/cmd/hgctl/helm"
+	"github.com/alibaba/higress/pkg/cmd/hgctl/kubernetes"
 	"github.com/alibaba/higress/pkg/cmd/hgctl/manifests"
 )
 
@@ -33,9 +34,10 @@ type IstioCRDComponent struct {
 	opts     *ComponentOptions
 	renderer helm.Renderer
 	writer   io.Writer
+	kubeCli  kubernetes.CLIClient
 }
 
-func NewIstioCRDComponent(profile *helm.Profile, writer io.Writer, opts ...ComponentOption) (Component, error) {
+func NewIstioCRDComponent(kubeCli kubernetes.CLIClient, profile *helm.Profile, writer io.Writer, opts ...ComponentOption) (Component, error) {
 	newOpts := &ComponentOptions{}
 	for _, opt := range opts {
 		opt(newOpts)
@@ -54,6 +56,8 @@ func NewIstioCRDComponent(profile *helm.Profile, writer io.Writer, opts ...Compo
 			helm.WithVersion(newOpts.Version),
 			helm.WithFS(manifests.BuiltinOrDir("")),
 			helm.WithDir(chartDir),
+			helm.WithCapabilities(newOpts.Capabilities),
+			helm.WithRestConfig(kubeCli.RESTConfig()),
 		)
 		if err != nil {
 			return nil, err
@@ -64,6 +68,8 @@ func NewIstioCRDComponent(profile *helm.Profile, writer io.Writer, opts ...Compo
 			helm.WithNamespace(newOpts.Namespace),
 			helm.WithRepoURL(newOpts.RepoURL),
 			helm.WithVersion(newOpts.Version),
+			helm.WithCapabilities(newOpts.Capabilities),
+			helm.WithRestConfig(kubeCli.RESTConfig()),
 		)
 		if err != nil {
 			return nil, err
@@ -75,6 +81,7 @@ func NewIstioCRDComponent(profile *helm.Profile, writer io.Writer, opts ...Compo
 		renderer: renderer,
 		opts:     newOpts,
 		writer:   writer,
+		kubeCli:  kubeCli,
 	}
 	return istioComponent, nil
 }

pkg/cmd/hgctl/installer/server_info.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package installer
+
+import (
+	"github.com/alibaba/higress/pkg/cmd/hgctl/kubernetes"
+	"github.com/pkg/errors"
+	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/chartutil"
+	"k8s.io/client-go/discovery"
+)
+
+type ServerInfo struct {
+	kubeCli kubernetes.CLIClient
+}
+
+func (c *ServerInfo) GetCapabilities() (*chartutil.Capabilities, error) {
+	// force a discovery cache invalidation to always fetch the latest server version/capabilities.
+	dc := c.kubeCli.KubernetesInterface().Discovery()
+
+	kubeVersion, err := dc.ServerVersion()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get server version from Kubernetes")
+	}
+	// Issue #6361:
+	// Client-Go emits an error when an API service is registered but unimplemented.
+	// We trap that error here and print a warning. But since the discovery client continues
+	// building the API object, it is correctly populated with all valid APIs.
+	// See https://github.com/kubernetes/kubernetes/issues/72051#issuecomment-521157642
+	apiVersions, err := action.GetVersionSet(dc)
+	if err != nil {
+		if discovery.IsGroupDiscoveryFailedError(err) {
+		} else {
+			return nil, errors.Wrap(err, "could not get apiVersions from Kubernetes")
+		}
+	}
+	capabilities := &chartutil.Capabilities{
+		APIVersions: apiVersions,
+		KubeVersion: chartutil.KubeVersion{
+			Version: kubeVersion.GitVersion,
+			Major:   kubeVersion.Major,
+			Minor:   kubeVersion.Minor,
+		},
+		HelmVersion: chartutil.DefaultCapabilities.HelmVersion,
+	}
+	return capabilities, nil
+}
+
+func NewServerInfo(kubCli kubernetes.CLIClient) (*ServerInfo, error) {
+	serverInfo := &ServerInfo{
+		kubeCli: kubCli,
+	}
+	return serverInfo, nil
+}

pkg/cmd/hgctl/kubernetes/client.go

@@ -57,6 +57,9 @@ type CLIClient interface {
 
 	// CreateNamespace create namespace
 	CreateNamespace(namespace string) error
+
+	// KubernetesInterface get kubernetes interface
+	KubernetesInterface() kubernetes.Interface
 }
 
 var _ CLIClient = &client{}
@@ -246,3 +249,8 @@ func (c *client) CreateNamespace(namespace string) error {
 
 	return nil
 }
+
+// KubernetesInterface get kubernetes interface
+func (c *client) KubernetesInterface() kubernetes.Interface {
+	return c.kube
+}

pkg/cmd/hgctl/uninstall.go

@@ -26,13 +26,13 @@ import (
 )
 
 type uninstallArgs struct {
-	// purgeIstioCRD delete  all of Istio resources.
+	// purgeResources delete  all of installed resources.
-	purgeIstioCRD bool
+	purgeResources bool
 }
 
 func addUninstallFlags(cmd *cobra.Command, args *uninstallArgs) {
-	cmd.PersistentFlags().BoolVarP(&args.purgeIstioCRD, "purge-istio-crd", "", false,
+	cmd.PersistentFlags().BoolVarP(&args.purgeResources, "purge-resources", "", false,
-		"Delete  all of Istio resources")
+		"Delete  all of IstioAPI,GatewayAPI resources")
 }
 
 // newUninstallCmd command uninstalls Istio from a cluster
@@ -42,11 +42,11 @@ func newUninstallCmd() *cobra.Command {
 		Use:   "uninstall",
 		Short: "Uninstall higress from a cluster",
 		Long:  "The uninstall command uninstalls higress from a cluster or local environment",
-		Example: `  # Uninstall higress 
+		Example: `# Uninstall higress 
   hgctl uninstal
   
-  # Uninstall higress and istio CRD from a cluster
+  # Uninstall higress, istioAPI and GatewayAPI from a cluster
-  hgctl uninstall --purge-istio-crd
+  hgctl uninstall --purge-resources
 `,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return uninstall(cmd.OutOrStdout(), uiArgs)
@@ -82,7 +82,12 @@ func uninstall(writer io.Writer, uiArgs *uninstallArgs) error {
 	}
 
 	if profile.Global.Install == helm.InstallK8s || profile.Global.Install == helm.InstallLocalK8s {
-		profile.Global.EnableIstioAPI = uiArgs.purgeIstioCRD
+		if profile.Global.EnableIstioAPI {
+			profile.Global.EnableIstioAPI = uiArgs.purgeResources
+		}
+		if profile.Global.EnableGatewayAPI {
+			profile.Global.EnableGatewayAPI = uiArgs.purgeResources
+		}
 	}
 
 	err = uninstallManifests(profile, writer, uiArgs)