Fix cloud proxy entrypoint by avoiding modifying a RO directory (Configmap mount directory) (#2027)

Summary: Fix cloud proxy entrypoint by avoiding modifying a RO directory
(Configmap mount directory)

This bug was introduced between 0a44b36
and c3e0fba on #2018 when the
individual file mounts were changed to a directory mount. Deploying the
cloud proxy from main results in the following error:

```
$ kubectl -n plc logs cloud-proxy-5df85487bf-hrglr
Defaulted container "cloud-proxy-server" out of: cloud-proxy-server, envoy
/scripts/entrypoint.sh: line 20: can't create /usr/local/openresty/nginx/conf/nginx.conf: Read-only file system
```

When I originally tested the final change, I must have only looked at
the resulting directory and missed that the pod was crashing. This issue
was detected during the 0.1.8 cloud prerelease testing.

Relevant Issues: #2017 #2013

Type of change: /kind bugfix

Test Plan: Verified that the cloud proxy image starts up successfully

Signed-off-by: Dom Del Nano <ddelnano@gmail.com>

Author: ddelnano

k8s/cloud/base/proxy_deployment.yaml

@@ -79,7 +79,7 @@ spec:
         - name: certs
           mountPath: /certs
         - name: nginx-config
-          mountPath: /usr/local/openresty/nginx/conf
+          mountPath: /usr/local/openresty/nginx/conf.d
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:

k8s/cloud/base/proxy_nginx_config.yaml

@@ -85,7 +85,7 @@ data:
     etag on;
     expires 60m;
     add_header Cache-Control "public";
-    include /usr/local/openresty/nginx/conf/headers_common.conf;
+    include /usr/local/openresty/nginx/conf.d/headers_common.conf;
 
   pixie_api.conf: |-
     location /api/ {
@@ -232,7 +232,7 @@ data:
             ssl_certificate /certs/tls.crt;
             ssl_certificate_key /certs/tls.key;
 
-            include /usr/local/openresty/nginx/conf/pixie_health_check.conf;
+            include /usr/local/openresty/nginx/conf.d/pixie_health_check.conf;
 
             if ($http_x_forwarded_proto = "http") {
                 return 404;
@@ -250,10 +250,10 @@ data:
             listen 56000 ssl http2;
             server_name @PL_DOMAIN_NAME@ *.cluster.local;
 
-            include /usr/local/openresty/nginx/conf/pixie_compression.conf;
-            include /usr/local/openresty/nginx/conf/pixie_vars.conf;
-            include /usr/local/openresty/nginx/conf/pixie_server_defaults.conf;
-            include /usr/local/openresty/nginx/conf/pixie_api.conf;
+            include /usr/local/openresty/nginx/conf.d/pixie_compression.conf;
+            include /usr/local/openresty/nginx/conf.d/pixie_vars.conf;
+            include /usr/local/openresty/nginx/conf.d/pixie_server_defaults.conf;
+            include /usr/local/openresty/nginx/conf.d/pixie_api.conf;
 
             if ($http_x_forwarded_proto = "http") {
                 return 307 https://$host$request_uri;
@@ -276,13 +276,13 @@ data:
             server_name work.@PL_DOMAIN_NAME@;
 
             error_page 404 = @error404;
-            include /usr/local/openresty/nginx/conf/pixie_compression.conf;
-            include /usr/local/openresty/nginx/conf/pixie_vars.conf;
-            include /usr/local/openresty/nginx/conf/pixie_server_defaults.conf;
-            include /usr/local/openresty/nginx/conf/pixie_health_check.conf;
-            include /usr/local/openresty/nginx/conf/pixie_api.conf;
-            include /usr/local/openresty/nginx/conf/headers_common.conf;
-            include /usr/local/openresty/nginx/conf/private/*.conf;
+            include /usr/local/openresty/nginx/conf.d/pixie_compression.conf;
+            include /usr/local/openresty/nginx/conf.d/pixie_vars.conf;
+            include /usr/local/openresty/nginx/conf.d/pixie_server_defaults.conf;
+            include /usr/local/openresty/nginx/conf.d/pixie_health_check.conf;
+            include /usr/local/openresty/nginx/conf.d/pixie_api.conf;
+            include /usr/local/openresty/nginx/conf.d/headers_common.conf;
+            include /usr/local/openresty/nginx/conf.d/private/*.conf;
 
             # Disable caching by default.
             add_header Cache-Control "no-store";
@@ -342,14 +342,14 @@ data:
             location ~ ^/static(/.*)$ {
                 gzip_static off;
                 root   /assets;
-                include /usr/local/openresty/nginx/conf/pixie_cache.conf;
-                include /usr/local/openresty/nginx/conf/pixie_filter.conf;
+                include /usr/local/openresty/nginx/conf.d/pixie_cache.conf;
+                include /usr/local/openresty/nginx/conf.d/pixie_filter.conf;
                 try_files $1 "/index.html";
             }
 
             location /auth-complete {
                 root   /assets;
-                include /usr/local/openresty/nginx/conf/pixie_filter.conf;
+                include /usr/local/openresty/nginx/conf.d/pixie_filter.conf;
                 try_files $uri "/index.html";
             }
 
@@ -368,8 +368,8 @@ data:
             listen 56000 ssl http2;
             server_name segment.@PL_DOMAIN_NAME@;
 
-            include /usr/local/openresty/nginx/conf/pixie_compression.conf;
-            include /usr/local/openresty/nginx/conf/pixie_vars.conf;
+            include /usr/local/openresty/nginx/conf.d/pixie_compression.conf;
+            include /usr/local/openresty/nginx/conf.d/pixie_vars.conf;
 
             set $segment_cdn "cdn.segment.com";
 

src/cloud/proxy/entrypoint.sh

@@ -17,7 +17,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 if [ -n "$PL_DOMAIN_NAME" ]; then
-    sed -e "s/[@]PL_DOMAIN_NAME[@]/$PL_DOMAIN_NAME/" /usr/local/openresty/nginx/conf/nginx.conf.tmpl > /usr/local/openresty/nginx/conf/nginx.conf
+    sed -e "s/[@]PL_DOMAIN_NAME[@]/$PL_DOMAIN_NAME/" /usr/local/openresty/nginx/conf.d/nginx.conf.tmpl > /usr/local/openresty/nginx/conf/nginx.conf
 else
    echo "PL_DOMAIN_NAME undefined, exiting"
    exit 1