diff --git a/helm/core/templates/_helpers.tpl b/helm/core/templates/_helpers.tpl index ec992a390..f10732342 100644 --- a/helm/core/templates/_helpers.tpl +++ b/helm/core/templates/_helpers.tpl @@ -101,3 +101,15 @@ higress: {{ include "controller.name" . }} true {{- end }} {{- end }} + +{{- define "gateway.podMonitor.gvk" -}} +{{- if eq .Values.gateway.metrics.provider "monitoring.coreos.com" -}} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +{{- else if eq .Values.gateway.metrics.provider "operator.victoriametrics.com" -}} +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +{{- else -}} +{{- fail "unexpected gateway.metrics.provider" -}} +{{- end -}} +{{- end -}} diff --git a/helm/core/templates/_pod.tpl b/helm/core/templates/_pod.tpl new file mode 100644 index 000000000..657a4f29d --- /dev/null +++ b/helm/core/templates/_pod.tpl @@ -0,0 +1,314 @@ + +{{/* +Rendering the pod template of gateway component. +*/}} +{{- define "gateway.podTemplate" -}} +{{- $o11y := .Values.global.o11y -}} +template: + metadata: + annotations: + {{- if .Values.global.enableHigressIstio }} + "enableHigressIstio": "true" + {{- end }} + {{- if .Values.gateway.podAnnotations }} + {{- toYaml .Values.gateway.podAnnotations | nindent 6 }} + {{- end }} + labels: + sidecar.istio.io/inject: "false" + {{- with .Values.gateway.revision }} + istio.io/rev: {{ . }} + {{- end }} + {{- include "gateway.selectorLabels" . | nindent 6 }} + spec: + {{- with .Values.gateway.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 6 }} + {{- end }} + serviceAccountName: {{ include "gateway.serviceAccountName" . }} + {{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" + {{- end }} + securityContext: + {{- if .Values.gateway.securityContext }} + {{- toYaml .Values.gateway.securityContext | nindent 6 }} + {{- else if and .Values.gateway.unprivilegedPortSupported (and (not .Values.gateway.hostNetwork) (semverCompare ">=1.22-0" .Capabilities.KubeVersion.GitVersion)) }} + # Safe since 1.22: https://github.com/kubernetes/kubernetes/pull/103326 + sysctls: + - name: net.ipv4.ip_unprivileged_port_start + value: "0" + {{- end }} + containers: + - name: higress-gateway + image: "{{ .Values.gateway.hub | default .Values.global.hub }}/{{ .Values.gateway.image | default "gateway" }}:{{ .Values.gateway.tag | default .Chart.AppVersion }}" + args: + - proxy + - router + - --domain + - $(POD_NAMESPACE).svc.cluster.local + - --proxyLogLevel=warning + - --proxyComponentLogLevel=misc:error + - --log_output_level=all:info + - --serviceCluster=higress-gateway + securityContext: + {{- if .Values.gateway.containerSecurityContext }} + {{- toYaml .Values.gateway.containerSecurityContext | nindent 10 }} + {{- else if and .Values.gateway.unprivilegedPortSupported (and (not .Values.gateway.hostNetwork) (semverCompare ">=1.22-0" .Capabilities.KubeVersion.GitVersion)) }} + # Safe since 1.22: https://github.com/kubernetes/kubernetes/pull/103326 + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false + privileged: false + # When enabling lite metrics, the configuration template files need to be replaced. + {{- if not .Values.global.liteMetrics }} + readOnlyRootFilesystem: true + {{- end }} + runAsUser: 1337 + runAsGroup: 1337 + runAsNonRoot: true + {{- else }} + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: 0 + runAsGroup: 1337 + runAsNonRoot: false + allowPrivilegeEscalation: true + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: INSTANCE_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: HOST_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: PROXY_XDS_VIA_AGENT + value: "true" + - name: ENABLE_INGRESS_GATEWAY_SDS + value: "false" + - name: JWT_POLICY + value: {{ include "controller.jwtPolicy" . }} + - name: ISTIO_META_HTTP10 + value: "1" + - name: ISTIO_META_CLUSTER_ID + value: "{{ $.Values.clusterName | default `Kubernetes` }}" + - name: INSTANCE_NAME + value: "higress-gateway" + {{- if .Values.global.liteMetrics }} + - name: LITE_METRICS + value: "on" + {{- end }} + {{- if include "skywalking.enabled" . }} + - name: ISTIO_BOOTSTRAP_OVERRIDE + value: /etc/istio/custom-bootstrap/custom_bootstrap.json + {{- end }} + {{- with .Values.gateway.networkGateway }} + - name: ISTIO_META_REQUESTED_NETWORK_VIEW + value: "{{.}}" + {{- end }} + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: {{ $val | quote }} + {{- end }} + ports: + - containerPort: 15020 + protocol: TCP + name: istio-prom + - containerPort: 15090 + protocol: TCP + name: http-envoy-prom + {{- if or .Values.global.local .Values.global.kind }} + - containerPort: {{ .Values.gateway.httpPort }} + hostPort: {{ .Values.gateway.httpPort }} + name: http + protocol: TCP + - containerPort: {{ .Values.gateway.httpsPort }} + hostPort: {{ .Values.gateway.httpsPort }} + name: https + protocol: TCP + {{- end }} + readinessProbe: + failureThreshold: {{ .Values.gateway.readinessFailureThreshold }} + httpGet: + path: /healthz/ready + port: 15021 + scheme: HTTP + initialDelaySeconds: {{ .Values.gateway.readinessInitialDelaySeconds }} + periodSeconds: {{ .Values.gateway.readinessPeriodSeconds }} + successThreshold: {{ .Values.gateway.readinessSuccessThreshold }} + timeoutSeconds: {{ .Values.gateway.readinessTimeoutSeconds }} + {{- if not (or .Values.global.local .Values.global.kind) }} + resources: + {{- toYaml .Values.gateway.resources | nindent 10 }} + {{- end }} + volumeMounts: + {{- if eq (include "controller.jwtPolicy" .) "third-party-jwt" }} + - name: istio-token + mountPath: /var/run/secrets/tokens + readOnly: true + {{- end }} + - name: config + mountPath: /etc/istio/config + - name: istio-ca-root-cert + mountPath: /var/run/secrets/istio + - name: istio-data + mountPath: /var/lib/istio/data + - name: podinfo + mountPath: /etc/istio/pod + - name: proxy-socket + mountPath: /etc/istio/proxy + {{- if include "skywalking.enabled" . }} + - mountPath: /etc/istio/custom-bootstrap + name: custom-bootstrap-volume + {{- end }} + {{- if .Values.global.volumeWasmPlugins }} + - mountPath: /opt/plugins + name: local-wasmplugins-volume + {{- end }} + {{- if $o11y.enabled }} + - mountPath: /var/log/proxy + name: log + {{- end }} + {{- if $o11y.enabled }} + {{- $config := $o11y.promtail }} + - name: promtail + image: {{ $config.image.repository }}:{{ $config.image.tag }} + imagePullPolicy: IfNotPresent + args: + - -config.file=/etc/promtail/promtail.yaml + env: + - name: 'HOSTNAME' + valueFrom: + fieldRef: + fieldPath: 'spec.nodeName' + ports: + - containerPort: {{ $config.port }} + name: http-metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: {{ $config.port }} + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + volumeMounts: + - name: promtail-config + mountPath: "/etc/promtail" + - name: log + mountPath: /var/log/proxy + - name: tmp + mountPath: /tmp + {{- end }} + {{- if .Values.gateway.hostNetwork }} + hostNetwork: {{ .Values.gateway.hostNetwork }} + dnsPolicy: ClusterFirstWithHostNet + {{- end }} + {{- with .Values.gateway.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.gateway.affinity }} + affinity: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.gateway.tolerations }} + tolerations: + {{- toYaml . | nindent 6 }} + {{- end }} + volumes: + {{- if eq (include "controller.jwtPolicy" .) "third-party-jwt" }} + - name: istio-token + projected: + sources: + - serviceAccountToken: + audience: istio-ca + expirationSeconds: 43200 + path: istio-token + {{- end }} + - name: istio-ca-root-cert + configMap: + {{- if .Values.global.enableHigressIstio }} + name: istio-ca-root-cert + {{- else }} + name: higress-ca-root-cert + {{- end }} + - name: config + configMap: + name: higress-config + {{- if include "skywalking.enabled" . }} + - configMap: + defaultMode: 420 + name: higress-custom-bootstrap + name: custom-bootstrap-volume + {{- end }} + - name: istio-data + emptyDir: {} + - name: proxy-socket + emptyDir: {} + {{- if $o11y.enabled }} + - name: log + emptyDir: {} + - name: tmp + emptyDir: {} + - name: promtail-config + configMap: + name: higress-promtail + {{- end }} + - name: podinfo + downwardAPI: + defaultMode: 420 + items: + - fieldRef: + apiVersion: v1 + fieldPath: metadata.labels + path: labels + - fieldRef: + apiVersion: v1 + fieldPath: metadata.annotations + path: annotations + - path: cpu-request + resourceFieldRef: + containerName: higress-gateway + divisor: 1m + resource: requests.cpu + - path: cpu-limit + resourceFieldRef: + containerName: higress-gateway + divisor: 1m + resource: limits.cpu + {{- if .Values.global.volumeWasmPlugins }} + - name: local-wasmplugins-volume + hostPath: + path: /opt/plugins + type: Directory + {{- end }} +{{- end -}} diff --git a/helm/core/templates/daemonset.yaml b/helm/core/templates/daemonset.yaml index 1e5951266..d1acd4a2a 100644 --- a/helm/core/templates/daemonset.yaml +++ b/helm/core/templates/daemonset.yaml @@ -6,10 +6,12 @@ {{- if $kernelVersion }} {{- $kernelVersion = regexFind "^(\\d+\\.\\d+\\.\\d+)" $kernelVersion }} {{- if and $kernelVersion (semverCompare "<4.11.0" $kernelVersion) }} - {{- $unprivilegedPortSupported = false }} + {{- $unprivilegedPortSupported = false }} {{- end }} {{- end }} {{- end -}} +{{- $_ := set .Values.gateway "unprivilegedPortSupported" $unprivilegedPortSupported -}} + apiVersion: apps/v1 kind: DaemonSet metadata: @@ -23,310 +25,5 @@ spec: selector: matchLabels: {{- include "gateway.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - {{- if .Values.global.enableHigressIstio }} - "enableHigressIstio": "true" - {{- end }} - {{- if .Values.gateway.podAnnotations }} - {{- toYaml .Values.gateway.podAnnotations | nindent 8 }} - {{- end }} - labels: - sidecar.istio.io/inject: "false" - {{- with .Values.gateway.revision }} - istio.io/rev: {{ . }} - {{- end }} - {{- include "gateway.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.gateway.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "gateway.serviceAccountName" . }} - {{- if .Values.global.priorityClassName }} - priorityClassName: "{{ .Values.global.priorityClassName }}" - {{- end }} - securityContext: - {{- if .Values.gateway.securityContext }} - {{- toYaml .Values.gateway.securityContext | nindent 8 }} - {{- else if and $unprivilegedPortSupported (and (not .Values.gateway.hostNetwork) (semverCompare ">=1.22-0" .Capabilities.KubeVersion.GitVersion)) }} - # Safe since 1.22: https://github.com/kubernetes/kubernetes/pull/103326 - sysctls: - - name: net.ipv4.ip_unprivileged_port_start - value: "0" - {{- end }} - containers: - {{- if $o11y.enabled }} - {{- $config := $o11y.promtail }} - - name: promtail - image: {{ $config.image.repository }}:{{ $config.image.tag }} - imagePullPolicy: IfNotPresent - args: - - -config.file=/etc/promtail/promtail.yaml - env: - - name: 'HOSTNAME' - valueFrom: - fieldRef: - fieldPath: 'spec.nodeName' - ports: - - containerPort: {{ $config.port }} - name: http-metrics - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /ready - port: {{ $config.port }} - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - volumeMounts: - - name: promtail-config - mountPath: "/etc/promtail" - - name: log - mountPath: /var/log/proxy - - name: tmp - mountPath: /tmp - {{- end }} - - name: higress-gateway - image: "{{ .Values.gateway.hub | default .Values.global.hub }}/{{ .Values.gateway.image | default "gateway" }}:{{ .Values.gateway.tag | default .Chart.AppVersion }}" - args: - - proxy - - router - - --domain - - $(POD_NAMESPACE).svc.cluster.local - - --proxyLogLevel=warning - - --proxyComponentLogLevel=misc:error - - --log_output_level=all:info - - --serviceCluster=higress-gateway - securityContext: - {{- if .Values.gateway.containerSecurityContext }} - {{- toYaml .Values.gateway.containerSecurityContext | nindent 12 }} - {{- else if and $unprivilegedPortSupported (and (not .Values.gateway.hostNetwork) (semverCompare ">=1.22-0" .Capabilities.KubeVersion.GitVersion)) }} - # Safe since 1.22: https://github.com/kubernetes/kubernetes/pull/103326 - capabilities: - drop: - - ALL - allowPrivilegeEscalation: false - privileged: false - # When enabling lite metrics, the configuration template files need to be replaced. - {{- if not .Values.global.liteMetrics }} - readOnlyRootFilesystem: true - {{- end }} - runAsUser: 1337 - runAsGroup: 1337 - runAsNonRoot: true - {{- else }} - capabilities: - drop: - - ALL - add: - - NET_BIND_SERVICE - runAsUser: 0 - runAsGroup: 1337 - runAsNonRoot: false - allowPrivilegeEscalation: true - {{- end }} - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: INSTANCE_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: HOST_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - - name: SERVICE_ACCOUNT - valueFrom: - fieldRef: - fieldPath: spec.serviceAccountName - - name: PILOT_XDS_SEND_TIMEOUT - value: 60s - - name: PROXY_XDS_VIA_AGENT - value: "true" - - name: ENABLE_INGRESS_GATEWAY_SDS - value: "false" - - name: JWT_POLICY - value: {{ include "controller.jwtPolicy" . }} - - name: ISTIO_META_HTTP10 - value: "1" - - name: ISTIO_META_CLUSTER_ID - value: "{{ $.Values.clusterName | default `Kubernetes` }}" - - name: INSTANCE_NAME - value: "higress-gateway" - {{- if .Values.global.liteMetrics }} - - name: LITE_METRICS - value: "on" - {{- end }} - {{- if include "skywalking.enabled" . }} - - name: ISTIO_BOOTSTRAP_OVERRIDE - value: /etc/istio/custom-bootstrap/custom_bootstrap.json - {{- end }} - {{- with .Values.gateway.networkGateway }} - - name: ISTIO_META_REQUESTED_NETWORK_VIEW - value: "{{.}}" - {{- end }} - {{- range $key, $val := .Values.env }} - - name: {{ $key }} - value: {{ $val | quote }} - {{- end }} - ports: - - containerPort: 15090 - protocol: TCP - name: http-envoy-prom - {{- if or .Values.global.local .Values.global.kind }} - - containerPort: {{ .Values.gateway.httpPort }} - hostPort: {{ .Values.gateway.httpPort }} - name: http - protocol: TCP - - containerPort: {{ .Values.gateway.httpsPort }} - hostPort: {{ .Values.gateway.httpsPort }} - name: https - protocol: TCP - {{- end }} - readinessProbe: - failureThreshold: {{ .Values.gateway.readinessFailureThreshold }} - httpGet: - path: /healthz/ready - port: 15021 - scheme: HTTP - initialDelaySeconds: {{ .Values.gateway.readinessInitialDelaySeconds }} - periodSeconds: {{ .Values.gateway.readinessPeriodSeconds }} - successThreshold: {{ .Values.gateway.readinessSuccessThreshold }} - timeoutSeconds: {{ .Values.gateway.readinessTimeoutSeconds }} - {{- if not (or .Values.global.local .Values.global.kind) }} - resources: - {{- toYaml .Values.gateway.resources | nindent 12 }} - {{- end }} - volumeMounts: - {{- if eq (include "controller.jwtPolicy" .) "third-party-jwt" }} - - name: istio-token - mountPath: /var/run/secrets/tokens - readOnly: true - {{- end }} - - name: config - mountPath: /etc/istio/config - - name: istio-ca-root-cert - mountPath: /var/run/secrets/istio - - name: istio-data - mountPath: /var/lib/istio/data - - name: podinfo - mountPath: /etc/istio/pod - - name: proxy-socket - mountPath: /etc/istio/proxy - {{- if include "skywalking.enabled" . }} - - mountPath: /etc/istio/custom-bootstrap - name: custom-bootstrap-volume - {{- end }} - {{- if .Values.global.volumeWasmPlugins }} - - mountPath: /opt/plugins - name: local-wasmplugins-volume - {{- end }} - {{- if $o11y.enabled }} - - mountPath: /var/log/proxy - name: log - {{- end }} - {{- if .Values.gateway.hostNetwork }} - hostNetwork: {{ .Values.gateway.hostNetwork }} - dnsPolicy: ClusterFirstWithHostNet - {{- end }} - {{- with .Values.gateway.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.gateway.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.gateway.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - {{- if eq (include "controller.jwtPolicy" .) "third-party-jwt" }} - - name: istio-token - projected: - sources: - - serviceAccountToken: - audience: istio-ca - expirationSeconds: 43200 - path: istio-token - {{- end }} - - name: istio-ca-root-cert - configMap: - {{- if .Values.global.enableHigressIstio }} - name: istio-ca-root-cert - {{- else }} - name: higress-ca-root-cert - {{- end }} - - name: config - configMap: - name: higress-config - {{- if include "skywalking.enabled" . }} - - configMap: - defaultMode: 420 - name: higress-custom-bootstrap - name: custom-bootstrap-volume - {{- end }} - - name: istio-data - emptyDir: {} - - name: proxy-socket - emptyDir: {} - {{- if $o11y.enabled }} - - name: log - emptyDir: {} - - name: tmp - emptyDir: {} - - name: promtail-config - configMap: - name: higress-promtail - {{- end }} - - name: podinfo - downwardAPI: - defaultMode: 420 - items: - - fieldRef: - apiVersion: v1 - fieldPath: metadata.labels - path: labels - - fieldRef: - apiVersion: v1 - fieldPath: metadata.annotations - path: annotations - - path: cpu-request - resourceFieldRef: - containerName: higress-gateway - divisor: 1m - resource: requests.cpu - - path: cpu-limit - resourceFieldRef: - containerName: higress-gateway - divisor: 1m - resource: limits.cpu - {{- if .Values.global.volumeWasmPlugins }} - - name: local-wasmplugins-volume - hostPath: - path: /opt/plugins - type: Directory - {{- end }} + {{- include "gateway.podTemplate" $ | nindent 2 -}} {{- end }} diff --git a/helm/core/templates/deployment.yaml b/helm/core/templates/deployment.yaml index 655341a50..f3d231130 100644 --- a/helm/core/templates/deployment.yaml +++ b/helm/core/templates/deployment.yaml @@ -1,15 +1,16 @@ {{- if eq .Values.gateway.kind "Deployment" -}} -{{- $o11y := .Values.global.o11y }} {{- $unprivilegedPortSupported := true }} {{- range $index, $node := (lookup "v1" "Node" "default" "").items }} {{- $kernelVersion := $node.status.nodeInfo.kernelVersion }} {{- if $kernelVersion }} {{- $kernelVersion = regexFind "^(\\d+\\.\\d+\\.\\d+)" $kernelVersion }} {{- if and $kernelVersion (semverCompare "<4.11.0" $kernelVersion) }} - {{- $unprivilegedPortSupported = false }} + {{- $unprivilegedPortSupported = false }} {{- end }} {{- end }} {{- end -}} +{{- $_ := set .Values.gateway "unprivilegedPortSupported" $unprivilegedPortSupported -}} + apiVersion: apps/v1 kind: Deployment metadata: @@ -38,311 +39,7 @@ spec: {{- else }} maxUnavailable: {{ .Values.gateway.rollingMaxUnavailable }} {{- end }} - template: - metadata: - annotations: - {{- if .Values.global.enableHigressIstio }} - "enableHigressIstio": "true" - {{- end }} - {{- if .Values.gateway.podAnnotations }} - {{- toYaml .Values.gateway.podAnnotations | nindent 8 }} - {{- end }} - labels: - sidecar.istio.io/inject: "false" - {{- with .Values.gateway.revision }} - istio.io/rev: {{ . }} - {{- end }} - {{- include "gateway.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.gateway.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "gateway.serviceAccountName" . }} - {{- if .Values.global.priorityClassName }} - priorityClassName: "{{ .Values.global.priorityClassName }}" - {{- end }} - securityContext: - {{- if .Values.gateway.securityContext }} - {{- toYaml .Values.gateway.securityContext | nindent 8 }} - {{- else if and $unprivilegedPortSupported (and (not .Values.gateway.hostNetwork) (semverCompare ">=1.22-0" .Capabilities.KubeVersion.GitVersion)) }} - # Safe since 1.22: https://github.com/kubernetes/kubernetes/pull/103326 - sysctls: - - name: net.ipv4.ip_unprivileged_port_start - value: "0" - {{- end }} - containers: - {{- if $o11y.enabled }} - {{- $config := $o11y.promtail }} - - name: promtail - image: {{ $config.image.repository }}:{{ $config.image.tag }} - imagePullPolicy: IfNotPresent - args: - - -config.file=/etc/promtail/promtail.yaml - env: - - name: 'HOSTNAME' - valueFrom: - fieldRef: - fieldPath: 'spec.nodeName' - ports: - - containerPort: {{ $config.port }} - name: http-metrics - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /ready - port: {{ $config.port }} - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - volumeMounts: - - name: promtail-config - mountPath: "/etc/promtail" - - name: log - mountPath: /var/log/proxy - - name: tmp - mountPath: /tmp - {{- end }} - - name: higress-gateway - image: "{{ .Values.gateway.hub | default .Values.global.hub }}/{{ .Values.gateway.image | default "gateway" }}:{{ .Values.gateway.tag | default .Chart.AppVersion }}" - args: - - proxy - - router - - --domain - - $(POD_NAMESPACE).svc.cluster.local - - --proxyLogLevel=warning - - --proxyComponentLogLevel=misc:error - - --log_output_level=all:info - - --serviceCluster=higress-gateway - securityContext: - {{- if .Values.gateway.containerSecurityContext }} - {{- toYaml .Values.gateway.containerSecurityContext | nindent 12 }} - {{- else if and $unprivilegedPortSupported (and (not .Values.gateway.hostNetwork) (semverCompare ">=1.22-0" .Capabilities.KubeVersion.GitVersion)) }} - # Safe since 1.22: https://github.com/kubernetes/kubernetes/pull/103326 - capabilities: - drop: - - ALL - allowPrivilegeEscalation: false - privileged: false - # When enabling lite metrics, the configuration template files need to be replaced. - {{- if not .Values.global.liteMetrics }} - readOnlyRootFilesystem: true - {{- end }} - runAsUser: 1337 - runAsGroup: 1337 - runAsNonRoot: true - {{- else }} - capabilities: - drop: - - ALL - add: - - NET_BIND_SERVICE - runAsUser: 0 - runAsGroup: 1337 - runAsNonRoot: false - allowPrivilegeEscalation: true - {{- end }} - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: INSTANCE_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: HOST_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - - name: SERVICE_ACCOUNT - valueFrom: - fieldRef: - fieldPath: spec.serviceAccountName - - name: PROXY_XDS_VIA_AGENT - value: "true" - - name: ENABLE_INGRESS_GATEWAY_SDS - value: "false" - - name: JWT_POLICY - value: {{ include "controller.jwtPolicy" . }} - - name: ISTIO_META_HTTP10 - value: "1" - - name: ISTIO_META_CLUSTER_ID - value: "{{ $.Values.clusterName | default `Kubernetes` }}" - - name: INSTANCE_NAME - value: "higress-gateway" - {{- if .Values.global.liteMetrics }} - - name: LITE_METRICS - value: "on" - {{- end }} - {{- if include "skywalking.enabled" . }} - - name: ISTIO_BOOTSTRAP_OVERRIDE - value: /etc/istio/custom-bootstrap/custom_bootstrap.json - {{- end }} - {{- with .Values.gateway.networkGateway }} - - name: ISTIO_META_REQUESTED_NETWORK_VIEW - value: "{{.}}" - {{- end }} - {{- range $key, $val := .Values.env }} - - name: {{ $key }} - value: {{ $val | quote }} - {{- end }} - ports: - - containerPort: 15020 - protocol: TCP - name: istio-prom - - containerPort: 15090 - protocol: TCP - name: http-envoy-prom - {{- if or .Values.global.local .Values.global.kind }} - - containerPort: {{ .Values.gateway.httpPort }} - hostPort: {{ .Values.gateway.httpPort }} - name: http - protocol: TCP - - containerPort: {{ .Values.gateway.httpsPort }} - hostPort: {{ .Values.gateway.httpsPort }} - name: https - protocol: TCP - {{- end }} - readinessProbe: - failureThreshold: {{ .Values.gateway.readinessFailureThreshold }} - httpGet: - path: /healthz/ready - port: 15021 - scheme: HTTP - initialDelaySeconds: {{ .Values.gateway.readinessInitialDelaySeconds }} - periodSeconds: {{ .Values.gateway.readinessPeriodSeconds }} - successThreshold: {{ .Values.gateway.readinessSuccessThreshold }} - timeoutSeconds: {{ .Values.gateway.readinessTimeoutSeconds }} - {{- if not (or .Values.global.local .Values.global.kind) }} - resources: - {{- toYaml .Values.gateway.resources | nindent 12 }} - {{- end }} - volumeMounts: - {{- if eq (include "controller.jwtPolicy" .) "third-party-jwt" }} - - name: istio-token - mountPath: /var/run/secrets/tokens - readOnly: true - {{- end }} - - name: config - mountPath: /etc/istio/config - - name: istio-ca-root-cert - mountPath: /var/run/secrets/istio - - name: istio-data - mountPath: /var/lib/istio/data - - name: podinfo - mountPath: /etc/istio/pod - - name: proxy-socket - mountPath: /etc/istio/proxy - {{- if include "skywalking.enabled" . }} - - mountPath: /etc/istio/custom-bootstrap - name: custom-bootstrap-volume - {{- end }} - {{- if .Values.global.volumeWasmPlugins }} - - mountPath: /opt/plugins - name: local-wasmplugins-volume - {{- end }} - {{- if $o11y.enabled }} - - mountPath: /var/log/proxy - name: log - {{- end }} - {{- if .Values.gateway.hostNetwork }} - hostNetwork: {{ .Values.gateway.hostNetwork }} - dnsPolicy: ClusterFirstWithHostNet - {{- end }} - {{- with .Values.gateway.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.gateway.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.gateway.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - {{- if eq (include "controller.jwtPolicy" .) "third-party-jwt" }} - - name: istio-token - projected: - sources: - - serviceAccountToken: - audience: istio-ca - expirationSeconds: 43200 - path: istio-token - {{- end }} - - name: istio-ca-root-cert - configMap: - {{- if .Values.global.enableHigressIstio }} - name: istio-ca-root-cert - {{- else }} - name: higress-ca-root-cert - {{- end }} - - name: config - configMap: - name: higress-config - {{- if include "skywalking.enabled" . }} - - configMap: - defaultMode: 420 - name: higress-custom-bootstrap - name: custom-bootstrap-volume - {{- end }} - - name: istio-data - emptyDir: {} - - name: proxy-socket - emptyDir: {} - {{- if $o11y.enabled }} - - name: log - emptyDir: {} - - name: tmp - emptyDir: {} - - name: promtail-config - configMap: - name: higress-promtail - {{- end }} - - name: podinfo - downwardAPI: - defaultMode: 420 - items: - - fieldRef: - apiVersion: v1 - fieldPath: metadata.labels - path: labels - - fieldRef: - apiVersion: v1 - fieldPath: metadata.annotations - path: annotations - - path: cpu-request - resourceFieldRef: - containerName: higress-gateway - divisor: 1m - resource: requests.cpu - - path: cpu-limit - resourceFieldRef: - containerName: higress-gateway - divisor: 1m - resource: limits.cpu - {{- if .Values.global.volumeWasmPlugins }} - - name: local-wasmplugins-volume - hostPath: - path: /opt/plugins - type: Directory - {{- end }} + + {{- include "gateway.podTemplate" $ | nindent 2 -}} + {{- end }} diff --git a/helm/core/templates/podmonitor.yaml b/helm/core/templates/podmonitor.yaml new file mode 100644 index 000000000..8b81f7fff --- /dev/null +++ b/helm/core/templates/podmonitor.yaml @@ -0,0 +1,45 @@ +{{- if .Values.gateway.metrics.enabled }} +{{- include "gateway.podMonitor.gvk" . }} +metadata: + name: {{ printf "%s-metrics" (include "gateway.name" .) | trunc 63 | trimSuffix "-" }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "gateway.labels" . | nindent 4}} + annotations: + {{- .Values.gateway.annotations | toYaml | nindent 4 }} +spec: + jobLabel: "app.kubernetes.io/name" + selector: + matchLabels: + {{- include "gateway.selectorLabels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + podMetricsEndpoints: + - port: istio-prom + path: /stats/prometheus + {{- if .Values.gateway.metrics.interval }} + interval: {{ .Values.gateway.metrics.interval }} + {{- end }} + {{- if .Values.gateway.metrics.scrapeTimeout }} + scrapeTimeout: {{ .Values.gateway.metrics.scrapeTimeout }} + {{- end }} + {{- if .Values.gateway.metrics.honorLabels }} + honorLabels: {{ .Values.gateway.metrics.honorLabels }} + {{- end }} + {{- if .Values.gateway.metrics.metricRelabelings }} + metricRelabelings: {{ toYaml .Values.gateway.metrics.metricRelabelings | nindent 8 }} + {{- end }} + {{- if .Values.gateway.metrics.relabelings }} + relabelings: {{ toYaml .Values.gateway.metrics.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.gateway.metrics.metricRelabelConfigs }} + metricRelabelings: {{ toYaml .Values.gateway.metrics.metricRelabelConfigs | nindent 8 }} + {{- end }} + {{- if .Values.gateway.metrics.relabelConfigs }} + relabelings: {{ toYaml .Values.gateway.metrics.relabelConfigs | nindent 8 }} + {{- end }} + {{- if $.Values.gateway.metrics.rawSpec }} + {{- $.Values.gateway.metrics.rawSpec | toYaml | nindent 6 }} + {{- end }} +{{- end }} diff --git a/helm/core/values.yaml b/helm/core/values.yaml index 215e3cf2b..ce0889920 100644 --- a/helm/core/values.yaml +++ b/helm/core/values.yaml @@ -136,7 +136,6 @@ global: excludeInboundPorts: "" includeInboundPorts: "*" - # istio egress capture allowlist # https://istio.io/docs/tasks/traffic-management/egress.html#calling-external-services-directly # example: includeIPRanges: "172.30.0.0/16,172.20.0.0/16" @@ -322,8 +321,8 @@ global: # Host:Port for submitting traces to the Datadog agent. address: "$(HOST_IP):8126" lightstep: - address: "" # example: lightstep-satellite:443 - accessToken: "" # example: abcdefg1234567 + address: "" # example: lightstep-satellite:443 + accessToken: "" # example: abcdefg1234567 stackdriver: # enables trace output to stdout. debug: false @@ -449,25 +448,25 @@ gateway: prometheus.io/scrape: "true" prometheus.io/path: "/stats/prometheus" sidecar.istio.io/inject: "false" - + # Define the security context for the pod. # If unset, this will be automatically set to the minimum privileges required to bind to port 80 and 443. # On Kubernetes 1.22+, this only requires the `net.ipv4.ip_unprivileged_port_start` sysctl. securityContext: ~ containerSecurityContext: ~ - + service: # Type of service. Set to "None" to disable the service entirely type: LoadBalancer ports: - - name: http2 - port: 80 - protocol: TCP - targetPort: 80 - - name: https - port: 443 - protocol: TCP - targetPort: 443 + - name: http2 + port: 80 + protocol: TCP + targetPort: 80 + - name: https + port: 443 + protocol: TCP + targetPort: 443 annotations: {} loadBalancerIP: "" loadBalancerClass: "" @@ -476,7 +475,7 @@ gateway: rollingMaxSurge: 100% rollingMaxUnavailable: 25% - + resources: requests: cpu: 2000m @@ -484,22 +483,39 @@ gateway: limits: cpu: 2000m memory: 2048Mi - + autoscaling: enabled: false minReplicas: 1 maxReplicas: 5 targetCPUUtilizationPercentage: 80 - + nodeSelector: {} - + tolerations: [] - + affinity: {} - + # If specified, the gateway will act as a network gateway for the given network. networkGateway: "" - + + metrics: + # If true, create PodMonitor or VMPodScrape for gateway + enabled: false + # provider group name for CustomResourceDefinition, can be monitoring.coreos.com or operator.victoriametrics.com + provider: monitoring.coreos.com + interval: "" + scrapeTimeout: "" + honorLabels: false + # for monitoring.coreos.com/v1.PodMonitor + metricRelabelings: [] + relabelings: [] + # for operator.victoriametrics.com/v1beta1.VMPodScrape + metricRelabelConfigs: [] + relabelConfigs: [] + # some more raw podMetricsEndpoints spec + rawSpec: {} + controller: name: "higress-controller" replicas: 1 @@ -510,22 +526,20 @@ controller: env: {} labels: {} - - probe: { - httpGet: { - path: /ready, - port: 8888, - }, - initialDelaySeconds: 1, - periodSeconds: 3, - timeoutSeconds: 5 - } - + + probe: + { + httpGet: { path: /ready, port: 8888 }, + initialDelaySeconds: 1, + periodSeconds: 3, + timeoutSeconds: 5, + } + imagePullSecrets: [] rbac: create: true - + serviceAccount: # Specifies whether a service account should be created create: true @@ -534,37 +548,30 @@ controller: # The name of the service account to use. # If not set and create is true, a name is generated using the fullname template name: "" - + podAnnotations: {} - - podSecurityContext: {} + + podSecurityContext: + {} # fsGroup: 2000 - - ports: [ - { - "name": "http", - "protocol": "TCP", - "port": 8888, - "targetPort": 8888, - }, - { - "name": "http-solver", - "protocol": "TCP", - "port": 8889, - "targetPort": 8889, - }, - { - "name": "grpc", - "protocol": "TCP", - "port": 15051, - "targetPort": 15051, - } - ] - + + ports: + [ + { "name": "http", "protocol": "TCP", "port": 8888, "targetPort": 8888 }, + { + "name": "http-solver", + "protocol": "TCP", + "port": 8889, + "targetPort": 8889, + }, + { "name": "grpc", "protocol": "TCP", "port": 15051, "targetPort": 15051 }, + ] + service: type: ClusterIP - - securityContext: {} + + securityContext: + {} # capabilities: # drop: # - ALL @@ -579,11 +586,11 @@ controller: limits: cpu: 1000m memory: 2048Mi - + nodeSelector: {} - + tolerations: [] - + affinity: {} autoscaling: @@ -594,7 +601,7 @@ controller: automaticHttps: enabled: true email: "" - + ## Discovery Settings pilot: autoscaleEnabled: false @@ -656,7 +663,6 @@ pilot: # Additional labels to apply to the deployment. deploymentLabels: {} - ## Mesh config settings # Install the mesh config map, generated from values.yaml. @@ -666,16 +672,15 @@ pilot: # Additional labels to apply on the pod level for monitoring and logging configuration. podLabels: {} - # Tracing config settings tracing: enable: false sampling: 100 timeout: 500 skywalking: - # access_token: "" - service: "" - port: 11800 + # access_token: "" + service: "" + port: 11800 # zipkin: - # service: "" - # port: 9411 + # service: "" + # port: 9411