lazyload using envoy as global sidecar to support h1/h2 #375
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Patrick0308
force-pushed
the
feat-lazyload-envoy
branch
from
21100fc
to
8af8bf6
Compare
MouceL
reviewed
Jun 25, 2023
| @@ -52,6 +52,18 @@ spec: | |||
| type: ClusterIP | |||
| {{- end }} | |||
| --- | |||
| apiVersion: networking.istio.io/v1beta1 | |||
There was a problem hiding this comment.
又验证了下，下发这个dr后
看起来不需要再修改global-sidecar 为http2-80
grpc请求兜底到global-sidecar
[2023-06-25T07:56:54.322Z] "POST /proto.CustomerService/ListPerson HTTP/2" 200 - via_upstream - "-" 5 249 15 1 "127.0.0.6" "grpc-python/1.19.0 grpc-c/7.0.0 (linux; chttp2; gold)" "b5bc288c-d478-4b38-aa4c-237efe9ee18c" "backend:50051" "10.96.192.135:50051" PassthroughCluster 10.244.0.29:60078 10.96.192.135:50051 127.0.0.6:0 - -
[2023-06-25T07:56:54.321Z] "POST /proto.CustomerService/ListPerson HTTP/2" 200 - via_upstream - "-" 5 249 17 2 "-" "grpc-python/1.19.0 grpc-c/7.0.0 (linux; chttp2; gold)" "42179a91-e024-4c6e-bdb6-e0aec7f1033a" "backend:50051" "10.244.0.29:80" inbound|80|| 127.0.0.6:60535 10.244.0.29:80 10.244.0.28:36956 outbound_.80_._.global-sidecar.mesh-operator.svc.cluster.local default
MouceL
reviewed
Jun 25, 2023
|
hello. 目前我也在尝试用envoy作为global-sidecar的poc，目前遇到流量到global-sidecar后无法分发到正确des的问题。 可以看到global-sidecar envoy的boostrap配置如下： 2023-06-29T07:57:24.661009Z info Envoy command: [-c etc/istio/proxy/envoy-rev0.json --restart-epoch 0 --drain-time-s 45 --parent-shutdown-time-s 60 --service-cluster global-sidecar.istio-mesh-operator --service-node router~10.215.15.24~istio-global-sidecar-ingressgateway-de6f2906-6fbb49dd45-ddk64.istio-mesh-operator~istio-mesh-operator.svc.cluster.local --local-address-ip-version v4 --bootstrap-version 3 --log-format %Y-%m-%dT%T.%fZ %l envoy %n %v -l warning --component-log-level misc:error --config-yaml {
"static_resources": {
"listeners": [
{
"address": {
"socket_address": {
"address": "0.0.0.0",
"port_value": 20000
}
},
"filter_chains": [
{
"filters": [
{
"name": "envoy.filters.network.http_connection_manager",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"codec_type": "AUTO",
"stat_prefix": "health",
"use_remote_address": true,
"route_config": {
"name": "health",
"virtual_hosts": [
{
"name": "all",
"domains": [
"*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"direct_response": {
"status": 200,
"body": {
"inline_string": "success"
}
}
}
]
}
]
},
"http_filters": [
{
"name": "envoy.filters.http.router",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
}
}
]
}
}
]
}
]
},
{
"address": {
"socket_address": {
"address": "0.0.0.0",
"port_value": 80
}
},
"filter_chains": [
{
"filters": [
{
"name": "envoy.filters.network.http_connection_manager",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"codec_type": "AUTO",
"stat_prefix": "egress_http",
"common_http_protocol_options": {
"idle_timeout": "840s"
},
"use_remote_address": true,
"route_config": {
"name": "proxy",
"request_headers_to_remove": [
"Slime-Orig-Dest"
],
"virtual_hosts": [
{
"name": "all",
"domains": [
"*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "original_dst_cluster"
}
}
]
}
]
},
"http_filters": [
{
"name": "envoy.filters.http.lua",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua",
"inlineCode": "function insert_namespace(host, namespace) if string.find(host, '%.') then return host end if not namespace or namespace == '' then return host end local idx = string.find(host, ':') if idx then local prefix = string.sub(host, 1, idx-1) local suffix = string.sub(host, idx) return prefix .. \".\" .. namespace .. suffix end return host .. '.' .. namespace end\nfunction envoy_on_request(request_handle) local sourceNs = request_handle:headers():get(\"Slime-Source-Ns\") local reqHost = request_handle:headers():get(\":authority\") reqHost = insert_namespace(reqHost, sourceNs) request_handle:headers():replace(\":authority\", reqHost) end"
}
},
{
"name": "envoy.filters.http.router",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
}
}
]
}
}
]
}
]
}
],
"clusters": [
{
"cleanupInterval": "8640000s",
"connectTimeout": "10s",
"name": "original_dst_cluster",
"type": "ORIGINAL_DST",
"lbPolicy": "CLUSTER_PROVIDED",
"circuitBreakers": {
"thresholds": [
{
"maxConnections": 4294967295,
"maxPendingRequests": 4294967295,
"maxRequests": 4294967295,
"maxRetries": 4294967295
}
]
},
"protocolSelection": "USE_DOWNSTREAM_PROTOCOL",
"filters": [
{
"name": "istio.metadata_exchange",
"typedConfig": {
"@type": "type.googleapis.com/udpa.type.v1.TypedStruct",
"typeUrl": "type.googleapis.com/envoy.tcp.metadataexchange.config.MetadataExchange",
"value": {
"protocol": "istio-peer-exchange"
}
}
}
]
},
{
"name": "lazyload-accesslog-source",
"type": "STRICT_DNS",
"connect_timeout": "5s",
"http2_protocol_options": {},
"dns_lookup_family": "V4_ONLY",
"load_assignment": {
"cluster_name": "lazyload-accesslog-source",
"endpoints": [{
"lb_endpoints": [{
"endpoint": {
"address": {
"socket_address": {
"address": "lazyload.istio-mesh-operator",
"port_value": 8082
}
}
}
}]
}]
},
"respect_dns_ttl": true
}
]
}
}
--concurrency 2]
2023-06-29T15:57:24.706297Z warning envoy runtime Unable to use runtime singleton for feature envoy.http.headermap.lazy_map_min_size
2023-06-29T15:57:24.706374Z warning envoy runtime Unable to use runtime singleton for feature envoy.http.headermap.lazy_map_min_size
2023-06-29T15:57:24.706928Z warning envoy runtime Unable to use runtime singleton for feature envoy.http.headermap.lazy_map_min_size
2023-06-29T15:57:24.706972Z warning envoy runtime Unable to use runtime singleton for feature envoy.http.headermap.lazy_map_min_size
2023-06-29T07:57:24.771976Z info xdsproxy connected to upstream XDS server: istiod.istio-system.svc:15012
2023-06-29T07:57:25.285948Z warn Envoy proxy is NOT ready: config not received from Pilot (is Pilot running?): cds updates: 1 successful, 0 rejected; lds updates: 0 successful, 0 rejected
2023-06-29T07:57:27.186564Z info Initialization took 6.650089338s
2023-06-29T07:57:27.186611Z info Envoy proxy is ready用一个简单的bookinfo实例触发流量，查看global-sidecar中envoy日志，这一条是从productpage到details 再查看productpage处envoy日志： {
"x_forwarded_for": null,
"start_time": "2023-06-29T08:12:52.929+0000",
"method": "GET",
"requested_server_name": null,
"istio_policy_status": null,
"duration": 1,
"protocol": "HTTP/1.1",
"route_name": null,
"upstream_transport_failure_reason": null,
"response_flags": "-",
"path": "/reviews/0",
"user_agent": "curl/7.64.0",
"bytes_sent": 19,
"request_id": "b28fb898-fce1-4349-bdbf-96e33945a51d",
"upstream_local_address": "10.215.52.22:33616",
"upstream_host": "10.215.15.24:80",
"downstream_remote_address": "10.215.52.22:44750",
"downstream_local_address": "10.212.168.96:9080",
"response_code_details": "via_upstream",
"upstream_service_time": "0",
"response_code": 503,
"tracing_id": null,
"upstream_cluster": "outbound|80||istio-global-sidecar-ingressgateway-a1b89d20.istio-mesh-operator.svc.cluster.local",
"authority": "reviews:9080",
"bytes_received": 0
}
{
"upstream_service_time": "0",
"user_agent": "curl/7.64.0",
"request_id": "b28fb898-fce1-4349-bdbf-96e33945a51d",
"response_code": 503,
"duration": 1,
"method": "GET",
"downstream_remote_address": "10.215.52.22:44752",
"tracing_id": null,
"x_forwarded_for": null,
"response_flags": "-",
"downstream_local_address": "10.212.168.96:9080",
"istio_policy_status": null,
"bytes_sent": 19,
"protocol": "HTTP/1.1",
"response_code_details": "via_upstream",
"route_name": null,
"start_time": "2023-06-29T08:12:52.953+0000",
"upstream_local_address": "10.215.52.22:33616",
"path": "/reviews/0",
"upstream_cluster": "outbound|80||istio-global-sidecar-ingressgateway-a1b89d20.istio-mesh-operator.svc.cluster.local",
"upstream_transport_failure_reason": null,
"requested_server_name": null,
"bytes_received": 0,
"authority": "reviews:9080",
"upstream_host": "10.215.15.24:80"
}
{
"istio_policy_status": null,
"start_time": "2023-06-29T08:12:52.894+0000",
"user_agent": "curl/7.64.0",
"protocol": "HTTP/1.1",
"response_code": 200,
"downstream_remote_address": "10.248.41.74:0",
"upstream_service_time": "61",
"bytes_received": 0,
"upstream_local_address": "127.0.0.1:52520",
"route_name": "default",
"request_id": "b28fb898-fce1-4349-bdbf-96e33945a51d",
"authority": "10.215.52.9:804",
"path": "/productpage",
"upstream_host": "127.0.0.1:9080",
"duration": 62,
"tracing_id": null,
"response_flags": "-",
"requested_server_name": null,
"x_forwarded_for": "10.248.41.74",
"upstream_transport_failure_reason": null,
"bytes_sent": 3769,
"upstream_cluster": "inbound|9080||",
"method": "GET",
"downstream_local_address": "10.215.52.22:9080",
"response_code_details": "via_upstream"
}这个报错可能是什么原因呢？另外这里可以直接用passthrough替换cluster 'original_dst_cluster'吗？ |
lua添加上去就是为了解决这个短域名问题 |
|
@YonkaFang 我们可以先合入 一些chart的调整以及使用的调整，我来跟进 |
MouceL
approved these changes
Jul 3, 2023
|
用 envoy 替换的 gs 的方案，有计划支持 mtls 流量吗
感谢回复，这个问题已经解决了。另外想请问下，用 envoy 作为 gs 的方案有计划支持 mtls 流量吗？ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
lazyload using envoy as global sidecar to support h1/h2. See details on #373.
Close #373
TODO: