Add proxy start script (#972)

Signed-off-by: zty98751 <zty98751@alibaba-inc.com> Co-authored-by: Kent Dong <ch3cho@qq.com>
2026-06-09 04:37:31 +08:00 · 2024-05-19 16:48:55 +08:00
parent dfa1be3b47
commit 0ec6719751
6 changed files with 892 additions and 1 deletions
--- a/helm/core/templates/configmap.yaml
+++ b/helm/core/templates/configmap.yaml
@@ -3,7 +3,11 @@
    # Refer to https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md#21-trust-domain
    trustDomain: "cluster.local"
    accessLogEncoding: TEXT
+    {{- if .Values.global.o11y.enabled }}
+    accessLogFile: "/var/log/proxy/access.log"
+    {{- else }}
    accessLogFile: "/dev/stdout"
+    {{- end }}
    ingressControllerMode: "OFF"
    accessLogFormat: '{"authority":"%REQ(X-ENVOY-ORIGINAL-HOST?:AUTHORITY)%","bytes_received":"%BYTES_RECEIVED%","bytes_sent":"%BYTES_SENT%","downstream_local_address":"%DOWNSTREAM_LOCAL_ADDRESS%","downstream_remote_address":"%DOWNSTREAM_REMOTE_ADDRESS%","duration":"%DURATION%","istio_policy_status":"%DYNAMIC_METADATA(istio.mixer:status)%","method":"%REQ(:METHOD)%","path":"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%","protocol":"%PROTOCOL%","request_id":"%REQ(X-REQUEST-ID)%","requested_server_name":"%REQUESTED_SERVER_NAME%","response_code":"%RESPONSE_CODE%","response_flags":"%RESPONSE_FLAGS%","route_name":"%ROUTE_NAME%","start_time":"%START_TIME%","trace_id":"%REQ(X-B3-TRACEID)%","upstream_cluster":"%UPSTREAM_CLUSTER%","upstream_host":"%UPSTREAM_HOST%","upstream_local_address":"%UPSTREAM_LOCAL_ADDRESS%","upstream_service_time":"%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%","upstream_transport_failure_reason":"%UPSTREAM_TRANSPORT_FAILURE_REASON%","user_agent":"%REQ(USER-AGENT)%","x_forwarded_for":"%REQ(X-FORWARDED-FOR)%"}

--- a/helm/core/templates/controller-deployment.yaml
+++ b/helm/core/templates/controller-deployment.yaml
@@ -70,6 +70,8 @@ spec:
            periodSeconds: 3
            timeoutSeconds: 5
          env:
+          - name: PILOT_ENABLE_HEADLESS_SERVICE_POD_LISTENERS
+            value: "false"
          - name: HIGRESS_SYSTEM_NS
            value: "{{ .Release.Namespace }}"
          - name: DEFAULT_UPSTREAM_CONCURRENCY_THRESHOLD
--- a/helm/core/templates/deployment.yaml
+++ b/helm/core/templates/deployment.yaml
@@ -1,3 +1,4 @@
+{{- $o11y := .Values.global.o11y  }}
 {{- $unprivilegedPortSupported := true }}
 {{- range $index, $node := (lookup "v1" "Node" "default" "").items }}
    {{- $kernelVersion := $node.status.nodeInfo.kernelVersion }}
@@ -67,6 +68,40 @@ spec:
          value: "0"
      {{- end }}
      containers:
+        {{- if $o11y.enabled }}
+          {{- $config := $o11y.promtail }}
+        - name: promtail
+          image: {{ $config.image.repository }}:{{ $config.image.tag }}
+          imagePullPolicy: IfNotPresent
+          args:
+            - -config.file=/etc/promtail/promtail.yaml
+          env:
+            - name: 'HOSTNAME'
+              valueFrom:
+                fieldRef:
+                  fieldPath: 'spec.nodeName'
+          ports:
+            - containerPort: {{ $config.port }}
+              name: http-metrics
+              protocol: TCP
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /ready
+              port: {{ $config.port }}
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          volumeMounts:
+            - name: promtail-config
+              mountPath: "/etc/promtail"
+            - name: log
+              mountPath: /var/log/proxy
+            - name: tmp
+              mountPath: /tmp
+        {{- end }}
        - name: higress-gateway
          image: "{{ .Values.gateway.hub | default .Values.global.hub }}/{{ .Values.gateway.image | default "gateway" }}:{{ .Values.gateway.tag | default .Chart.AppVersion }}"
          args:
@@ -88,7 +123,10 @@ spec:
              - ALL
            allowPrivilegeEscalation: false
            privileged: false
+          # When enabling lite metrics, the configuration template files need to be replaced.
+          {{- if not .Values.global.liteMetrics }}
            readOnlyRootFilesystem: true
+          {{- end }}
            runAsUser: 1337
            runAsGroup: 1337
            runAsNonRoot: true
@@ -102,7 +140,6 @@ spec:
            runAsGroup: 1337
            runAsNonRoot: false
            allowPrivilegeEscalation: true
-            readOnlyRootFilesystem: true
          {{- end }}
          env:
          - name: NODE_NAME
@@ -148,6 +185,10 @@ spec:
            value: "{{ $.Values.clusterName | default `Kubernetes` }}"
          - name: INSTANCE_NAME
            value: "higress-gateway"
+          {{- if .Values.global.liteMetrics }}
+          - name: LITE_METRICS
+            value: "on"
+          {{- end }}
          {{- if include "skywalking.enabled" . }}
          - name: ISTIO_BOOTSTRAP_OVERRIDE
            value: /etc/istio/custom-bootstrap/custom_bootstrap.json
@@ -212,6 +253,10 @@ spec:
          - mountPath: /opt/plugins
            name: local-wasmplugins-volume
          {{- end }}
+          {{- if $o11y.enabled }}
+          - mountPath: /var/log/proxy
+            name: log
+          {{- end }}
      {{- if .Values.gateway.hostNetwork }}
      hostNetwork: {{ .Values.gateway.hostNetwork }}
      dnsPolicy: ClusterFirstWithHostNet
@@ -258,6 +303,15 @@ spec:
        emptyDir: {}
      - name: proxy-socket
        emptyDir: {}
+      {{- if $o11y.enabled }}
+      - name: log
+        emptyDir: {}
+      - name: tmp
+        emptyDir: {}
+      - name: promtail-config
+        configMap:
+          name: higress-promtail
+      {{- end }}
      - name: podinfo
        downwardAPI:
          defaultMode: 420
--- a/helm/core/templates/promtail.yaml
+++ b/helm/core/templates/promtail.yaml
@@ -0,0 +1,64 @@
+{{- $o11y := .Values.global.o11y  }}
+{{- if $o11y.enabled }}
+{{- $config := $o11y.promtail }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: higress-promtail
+  namespace: {{ .Release.Namespace }}
+data:
+  promtail.yaml: |
+    server:
+      log_level: info
+      http_listen_port: {{ $config.port }}
+
+    clients:
+    - url: http://higress-console-loki.{{ .Release.Namespace }}:3100/loki/api/v1/push
+
+    positions:
+      filename: /tmp/promtail-positions.yaml
+    target_config:
+      sync_period: 10s
+    scrape_configs:
+    - job_name: access-logs
+      static_configs:
+      - targets:
+        - localhost
+        labels:
+          __path__: /var/log/proxy/access.log
+      pipeline_stages:
+      - json:
+          expressions:
+            authority:
+            method:
+            path:
+            protocol:
+            request_id:
+            response_code:
+            response_flags:
+            route_name:
+            trace_id:
+            upstream_cluster:
+            upstream_host:
+            upstream_transport_failure_reason:
+            user_agent:
+            x_forwarded_for:
+      - labels:
+          authority:
+          method:
+          path:
+          protocol:
+          request_id:
+          response_code:
+          response_flags:
+          route_name:
+          trace_id:
+          upstream_cluster:
+          upstream_host:
+          upstream_transport_failure_reason:
+          user_agent:
+          x_forwarded_for:
+      - timestamp:
+          source: timestamp
+          format: RFC3339Nano
+{{- end }}
--- a/helm/core/values.yaml
+++ b/helm/core/values.yaml
@@ -1,5 +1,6 @@
 revision: ""
 global:
+  liteMetrics: false
  xdsMaxRecvMsgSize: "104857600"
  defaultUpstreamConcurrencyThreshold: 10000
  enableSRDS: true
@@ -337,6 +338,20 @@ global:
  # Use the Mesh Control Protocol (MCP) for configuring Istiod. Requires an MCP source.
  useMCP: false

+  # Observability (o11y) configurations
+  o11y:
+    enabled: false
+    promtail:
+      image:
+        repository: grafana/promtail
+        tag: 2.9.4
+      port: 3101
+      resources:
+        limits:
+          cpu: 500m
+          memory: 2Gi
+      securityContext: {}
+
  # The name of the CA for workload certificates.
  # For example, when caName=GkeWorkloadCertificate, GKE workload certificates
  # will be used as the certificates for workloads.