Optimize the effectiveness speed of xds and add AI-related metric tags (#1047)

envoy/1.20/patches/envoy/20240610-optimize-xds.patch

@@ -0,0 +1,259 @@
+diff --git a/source/common/router/BUILD b/source/common/router/BUILD
+index 5c58501..4db76cd 100644
+--- a/source/common/router/BUILD
++++ b/source/common/router/BUILD
+@@ -212,6 +212,7 @@ envoy_cc_library(
+         "//envoy/router:rds_interface",
+         "//envoy/router:scopes_interface",
+         "//envoy/thread_local:thread_local_interface",
++        "//source/common/protobuf:utility_lib",
+         "@envoy_api//envoy/config/route/v3:pkg_cc_proto",
+         "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto",
+     ],
+diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc
+index ff7b4c8..5ac4523 100644
+--- a/source/common/router/config_impl.cc
++++ b/source/common/router/config_impl.cc
+@@ -550,19 +550,11 @@ RouteEntryImplBase::RouteEntryImplBase(const VirtualHostImpl& vhost,
+               "not be stripped: {}",
+               path_redirect_);
+   }
+-  ENVOY_LOG(info, "route stats is {}, name is {}", route.stat_prefix(), route.name());
+   if (!route.stat_prefix().empty()) {
+     route_stats_context_ = std::make_unique<RouteStatsContext>(
+         factory_context.scope(), factory_context.routerContext().routeStatNames(), vhost.statName(),
+         route.stat_prefix());
+-  } else if (!route.name().empty()) {
+-    // Added by Ingress
+-    // use route_name as default stat_prefix
+-    route_stats_context_ = std::make_unique<RouteStatsContext>(
+-        factory_context.scope(), factory_context.routerContext().routeStatNames(), vhost.statName(),
+-        route.name());
+   }
+-  // End Added
+ }
+ 
+ bool RouteEntryImplBase::evaluateRuntimeMatch(const uint64_t random_value) const {
+@@ -1415,9 +1407,7 @@ VirtualHostImpl::VirtualHostImpl(
+       retry_shadow_buffer_limit_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(
+           virtual_host, per_request_buffer_limit_bytes, std::numeric_limits<uint32_t>::max())),
+       include_attempt_count_in_request_(virtual_host.include_request_attempt_count()),
+-      include_attempt_count_in_response_(virtual_host.include_attempt_count_in_response()),
+-      virtual_cluster_catch_all_(*vcluster_scope_,
+-                                 factory_context.routerContext().virtualClusterStatNames()) {
++      include_attempt_count_in_response_(virtual_host.include_attempt_count_in_response()) {
+   switch (virtual_host.require_tls()) {
+   case envoy::config::route::v3::VirtualHost::NONE:
+     ssl_requirements_ = SslRequirements::None;
+@@ -1478,10 +1468,14 @@ VirtualHostImpl::VirtualHostImpl(
+     }
+   }
+ 
+-  for (const auto& virtual_cluster : virtual_host.virtual_clusters()) {
+-    virtual_clusters_.push_back(
+-        VirtualClusterEntry(virtual_cluster, *vcluster_scope_,
+-                            factory_context.routerContext().virtualClusterStatNames()));
++  if (!virtual_host.virtual_clusters().empty()) {
++    virtual_cluster_catch_all_ = std::make_unique<CatchAllVirtualCluster>(
++        *vcluster_scope_, factory_context.routerContext().virtualClusterStatNames());
++    for (const auto& virtual_cluster : virtual_host.virtual_clusters()) {
++      virtual_clusters_.push_back(
++          VirtualClusterEntry(virtual_cluster, *vcluster_scope_,
++                              factory_context.routerContext().virtualClusterStatNames()));
++    }
+   }
+ 
+   if (virtual_host.has_cors()) {
+@@ -1774,7 +1768,7 @@ VirtualHostImpl::virtualClusterFromEntries(const Http::HeaderMap& headers) const
+   }
+ 
+   if (!virtual_clusters_.empty()) {
+-    return &virtual_cluster_catch_all_;
++    return virtual_cluster_catch_all_.get();
+   }
+ 
+   return nullptr;
+diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h
+index cf0ddf3..d83eb94 100644
+--- a/source/common/router/config_impl.h
++++ b/source/common/router/config_impl.h
+@@ -352,10 +352,10 @@ private:
+   const bool include_attempt_count_in_response_;
+   absl::optional<envoy::config::route::v3::RetryPolicy> retry_policy_;
+   absl::optional<envoy::config::route::v3::HedgePolicy> hedge_policy_;
+-  const CatchAllVirtualCluster virtual_cluster_catch_all_;
+ #if defined(ALIMESH)
+   std::vector<std::string> allow_server_names_;
+ #endif
++  std::unique_ptr<const CatchAllVirtualCluster> virtual_cluster_catch_all_;
+ };
+ 
+ using VirtualHostSharedPtr = std::shared_ptr<VirtualHostImpl>;
+diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc
+index 594d571..6482615 100644
+--- a/source/common/router/scoped_config_impl.cc
++++ b/source/common/router/scoped_config_impl.cc
+@@ -7,6 +7,8 @@
+ #include "source/common/http/header_utility.h"
+ #endif
+ 
++#include "source/common/protobuf/utility.h"
++
+ namespace Envoy {
+ namespace Router {
+ 
+@@ -239,7 +241,8 @@ HeaderValueExtractorImpl::computeFragment(const Http::HeaderMap& headers) const
+ 
+ ScopedRouteInfo::ScopedRouteInfo(envoy::config::route::v3::ScopedRouteConfiguration&& config_proto,
+                                  ConfigConstSharedPtr&& route_config)
+-    : config_proto_(std::move(config_proto)), route_config_(std::move(route_config)) {
++    : config_proto_(std::move(config_proto)), route_config_(std::move(route_config)),
++      config_hash_(MessageUtil::hash(config_proto)) {
+   // TODO(stevenzzzz): Maybe worth a KeyBuilder abstraction when there are more than one type of
+   // Fragment.
+   for (const auto& fragment : config_proto_.key().fragments()) {
+diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h
+index 9f6a1b2..28e2ee5 100644
+--- a/source/common/router/scoped_config_impl.h
++++ b/source/common/router/scoped_config_impl.h
+@@ -154,11 +154,13 @@ public:
+     return config_proto_;
+   }
+   const std::string& scopeName() const { return config_proto_.name(); }
++  uint64_t configHash() const { return config_hash_; }
+ 
+ private:
+   envoy::config::route::v3::ScopedRouteConfiguration config_proto_;
+   ScopeKey scope_key_;
+   ConfigConstSharedPtr route_config_;
++  const uint64_t config_hash_;
+ };
+ using ScopedRouteInfoConstSharedPtr = std::shared_ptr<const ScopedRouteInfo>;
+ // Ordered map for consistent config dumping.
+diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc
+index 133e91e..9b2096e 100644
+--- a/source/common/router/scoped_rds.cc
++++ b/source/common/router/scoped_rds.cc
+@@ -245,6 +245,11 @@ bool ScopedRdsConfigSubscription::addOrUpdateScopes(
+         dynamic_cast<const envoy::config::route::v3::ScopedRouteConfiguration&>(
+             resource.get().resource());
+     const std::string scope_name = scoped_route_config.name();
++    if (const auto& scope_info_iter = scoped_route_map_.find(scope_name);
++        scope_info_iter != scoped_route_map_.end() &&
++        scope_info_iter->second->configHash() == MessageUtil::hash(scoped_route_config)) {
++      continue;
++    }
+     rds.set_route_config_name(scoped_route_config.route_configuration_name());
+     std::unique_ptr<RdsRouteConfigProviderHelper> rds_config_provider_helper;
+     std::shared_ptr<ScopedRouteInfo> scoped_route_info = nullptr;
+@@ -398,6 +403,7 @@ void ScopedRdsConfigSubscription::onRdsConfigUpdate(const std::string& scope_nam
+   auto new_scoped_route_info = std::make_shared<ScopedRouteInfo>(
+       envoy::config::route::v3::ScopedRouteConfiguration(iter->second->configProto()),
+       std::move(new_rds_config));
++  scoped_route_map_[new_scoped_route_info->scopeName()] = new_scoped_route_info;
+   applyConfigUpdate([new_scoped_route_info](ConfigProvider::ConfigConstSharedPtr config)
+                         -> ConfigProvider::ConfigConstSharedPtr {
+     auto* thread_local_scoped_config =
+diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h
+index d21d812..a510c1f 100644
+--- a/source/common/router/scoped_rds.h
++++ b/source/common/router/scoped_rds.h
+@@ -104,7 +104,7 @@ struct ScopedRdsStats {
+ // A scoped RDS subscription to be used with the dynamic scoped RDS ConfigProvider.
+ class ScopedRdsConfigSubscription
+     : public Envoy::Config::DeltaConfigSubscriptionInstance,
+-      Envoy::Config::SubscriptionBase<envoy::config::route::v3::ScopedRouteConfiguration> {
++      public Envoy::Config::SubscriptionBase<envoy::config::route::v3::ScopedRouteConfiguration> {
+ public:
+   using ScopedRouteConfigurationMap =
+       std::map<std::string, envoy::config::route::v3::ScopedRouteConfiguration>;
+diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc
+index f63f258..69a2f4b 100644
+--- a/test/common/router/scoped_config_impl_test.cc
++++ b/test/common/router/scoped_config_impl_test.cc
+@@ -452,6 +452,24 @@ TEST_F(ScopedRouteInfoTest, Creation) {
+   EXPECT_EQ(info_->scopeKey(), makeKey({"foo", "bar"}));
+ }
+ 
++// Tests that config hash changes if ScopedRouteConfiguration of the ScopedRouteInfo changes.
++TEST_F(ScopedRouteInfoTest, Hash) {
++  const envoy::config::route::v3::ScopedRouteConfiguration config_copy = scoped_route_config_;
++  info_ = std::make_unique<ScopedRouteInfo>(scoped_route_config_, route_config_);
++  EXPECT_EQ(info_->routeConfig().get(), route_config_.get());
++  EXPECT_TRUE(TestUtility::protoEqual(info_->configProto(), config_copy));
++  EXPECT_EQ(info_->scopeName(), "foo_scope");
++  EXPECT_EQ(info_->scopeKey(), makeKey({"foo", "bar"}));
++
++  const auto info2 = std::make_unique<ScopedRouteInfo>(scoped_route_config_, route_config_);
++  ASSERT_EQ(info2->configHash(), info_->configHash());
++
++  // Mutate the config and hash should be different now.
++  scoped_route_config_.set_on_demand(true);
++  const auto info3 = std::make_unique<ScopedRouteInfo>(scoped_route_config_, route_config_);
++  ASSERT_NE(info3->configHash(), info_->configHash());
++}
++
+ class ScopedConfigImplTest : public testing::Test {
+ public:
+   void SetUp() override {
+diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc
+index 09b96a6..b4776c9 100644
+--- a/test/common/router/scoped_rds_test.cc
++++ b/test/common/router/scoped_rds_test.cc
+@@ -13,6 +13,7 @@
+ #include "envoy/stats/scope.h"
+ 
+ #include "source/common/config/api_version.h"
++#include "source/common/config/config_provider_impl.h"
+ #include "source/common/config/grpc_mux_impl.h"
+ #include "source/common/protobuf/message_validator_impl.h"
+ #include "source/common/router/scoped_rds.h"
+@@ -365,6 +366,48 @@ key:
+                             "Didn't find a registered implementation for name: 'filter.unknown'");
+ }
+ 
++// Test that scopes with same config as existing scopes will be skipped in a config push.
++TEST_F(ScopedRdsTest, UnchangedScopesAreSkipped) {
++  setup();
++  init_watcher_.expectReady();
++  const std::string config_yaml = R"EOF(
++name: foo_scope
++route_configuration_name: foo_routes
++key:
++  fragments:
++    - string_key: x-foo-key
++)EOF";
++  const auto resource = parseScopedRouteConfigurationFromYaml(config_yaml);
++  const std::string config_yaml2 = R"EOF(
++name: foo_scope2
++route_configuration_name: foo_routes
++key:
++  fragments:
++    - string_key: x-bar-key
++)EOF";
++  const auto resource_2 = parseScopedRouteConfigurationFromYaml(config_yaml2);
++
++  // Delta API.
++  const auto decoded_resources = TestUtility::decodeResources({resource, resource_2});
++  context_init_manager_.initialize(init_watcher_);
++  EXPECT_NO_THROW(srds_subscription_->onConfigUpdate(decoded_resources.refvec_, {}, "v1"));
++  EXPECT_EQ(1UL,
++            server_factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload")
++                .value());
++  EXPECT_EQ(2UL, all_scopes_.value());
++  pushRdsConfig({"foo_routes"}, "111");
++  Envoy::Router::ScopedRdsConfigSubscription* srds_delta_subscription =
++      static_cast<Envoy::Router::ScopedRdsConfigSubscription*>(srds_subscription_);
++  ASSERT_NE(srds_delta_subscription, nullptr);
++  ASSERT_EQ("v1", srds_delta_subscription->configInfo()->last_config_version_);
++  // Push again the same set of config with different version number, the config will be skipped.
++  EXPECT_NO_THROW(srds_subscription_->onConfigUpdate(decoded_resources.refvec_, {}, "123"));
++  ASSERT_EQ("v1", srds_delta_subscription->configInfo()->last_config_version_);
++  EXPECT_EQ(2UL,
++            server_factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload")
++                .value());
++}
++
+ // Test ignoring the optional unknown factory in the per-virtualhost typed config.
+ TEST_F(ScopedRdsTest, OptionalUnknownFactoryForPerVirtualHostTypedConfig) {
+   OptionalHttpFilters optional_http_filters;

istio/1.12/patches/istio/20240619-ai-stats.patch

@@ -0,0 +1,53 @@
+diff -Naur istio/tools/packaging/common/envoy_bootstrap.json istio-new/tools/packaging/common/envoy_bootstrap.json
+--- istio/tools/packaging/common/envoy_bootstrap.json	2024-06-19 13:39:49.179159469 +0800
++++ istio-new/tools/packaging/common/envoy_bootstrap.json	2024-06-19 13:39:28.299159059 +0800
+@@ -37,6 +37,18 @@
+     "use_all_default_tags": false,
+     "stats_tags": [
+       {
++          "tag_name": "ai_route",
++          "regex": "^wasmcustom\\.route\\.((.*?)\\.)upstream"
++      },
++      {
++          "tag_name": "ai_cluster",
++          "regex": "^wasmcustom\\..*?\\.upstream\\.((.*?)\\.)model"
++      },
++      {
++          "tag_name": "ai_model",
++          "regex": "^wasmcustom\\..*?\\.model\\.((.*?)\\.)(input_token|output_token)"
++      },
++      {
+         "tag_name": "cluster_name",
+         "regex": "^cluster\\.((.*?)\\.)(http1\\.|http2\\.|health_check\\.|zone\\.|external\\.|circuit_breakers\\.|[^\\.]+$)"
+       },
+diff -Naur istio/tools/packaging/common/envoy_bootstrap_lite.json istio-new/tools/packaging/common/envoy_bootstrap_lite.json
+--- istio/tools/packaging/common/envoy_bootstrap_lite.json	2024-06-19 13:39:49.175159469 +0800
++++ istio-new/tools/packaging/common/envoy_bootstrap_lite.json	2024-06-19 13:38:52.283158352 +0800
+@@ -37,6 +37,18 @@
+     "use_all_default_tags": false,
+     "stats_tags": [
+       {
++          "tag_name": "ai_route",
++          "regex": "^wasmcustom\\.route\\.((.*?)\\.)upstream"
++      },
++      {
++          "tag_name": "ai_cluster",
++          "regex": "^wasmcustom\\..*?\\.upstream\\.((.*?)\\.)model"
++      },
++      {
++          "tag_name": "ai_model",
++          "regex": "^wasmcustom\\..*?\\.model\\.((.*?)\\.)(input_token|output_token)"
++      },
++      {
+         "tag_name": "response_code_class",
+         "regex": "_rq(_(\\dxx))$"
+       },
+@@ -60,7 +72,7 @@
+             "prefix": "vhost"
+           },
+           {
+-            "safe_regex": {"regex": "^http.*rds.*", "google_re2":{}}
++            "safe_regex": {"regex": "^http.*\\.rds\\..*", "google_re2":{}}
+           }
+         ]
+       }