fix(ai-proxy): set llm variables default value to 0

apisix/cli/ngx_tpl.lua

@@ -809,10 +809,10 @@ http {
             set $llm_content_risk_level         '';
             set $request_type               'traditional_http';
 
-            set $llm_time_to_first_token        '';
+            set $llm_time_to_first_token        '0';
             set $llm_model                      '';
-            set $llm_prompt_tokens              '';
-            set $llm_completion_tokens          '';
+            set $llm_prompt_tokens              '0';
+            set $llm_completion_tokens          '0';
 
 
             access_by_lua_block {

apisix/plugins/ai-drivers/openai-base.lua

@@ -97,7 +97,7 @@ local function read_response(ctx, res)
                 return
             end
 
-            if ctx.var.llm_time_to_first_token == "" then
+            if ctx.var.llm_time_to_first_token == "0" then
                 ctx.var.llm_time_to_first_token = math.floor(
                                                 (ngx_now() - ctx.llm_request_start_time) * 1000)
             end

t/APISIX.pm

@@ -860,10 +860,10 @@ _EOC_
             set \$llm_content_risk_level         '';
             set \$request_type               'traditional_http';
 
-            set \$llm_time_to_first_token        '';
+            set \$llm_time_to_first_token        '0';
             set \$llm_model                      '';
-            set \$llm_prompt_tokens              '';
-            set \$llm_completion_tokens          '';
+            set \$llm_prompt_tokens              '0';
+            set \$llm_completion_tokens          '0';
 
             access_log $apisix_home/t/servroot/logs/access.log main;
 

t/cli/test_access_log.sh

@@ -138,6 +138,32 @@ make stop
 
 echo "passed: access log with JSON format"
 
+# access log with unset llm_token JSON format
+
+echo '
+nginx_config:
+  http:
+    access_log_format: |-
+      {"@timestamp": "$time_iso8601", "client_ip": "$remote_addr", "status": "$status", "llm_prompt_tokens": $llm_prompt_tokens}
+    access_log_format_escape: json
+' > conf/config.yaml
+
+make init
+make run
+sleep 0.1
+curl http://127.0.0.1:9080/hello2
+sleep 4
+tail -n 1 logs/access.log > output.log
+
+if [ `grep -c '"llm_prompt_tokens": 0' output.log` -eq '0' ]; then
+    echo "failed: invalid JSON log in access log"
+    exit 1
+fi
+
+make stop
+
+echo "passed: access log with JSON format and llm_prompt_tokens"
+
 # check uninitialized variable in access log when access admin
 git checkout conf/config.yaml
 

t/plugin/ai-proxy3.t

@@ -224,3 +224,108 @@ POST /anything
 --- response_body eval
 qr/.*assistant.*/
 --- no_error_log
+
+
+
+=== TEST 5: create a ai-proxy-multi route with delay streaming ai endpoint(every event delay 200ms)
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                    "uri": "/anything",
+                    "plugins": {
+                        "ai-proxy-multi": {
+                            "instances": [
+                                {
+                                    "name": "self-hosted",
+                                    "provider": "openai-compatible",
+                                    "weight": 1,
+                                    "auth": {
+                                        "header": {
+                                            "Authorization": "Bearer token"
+                                        }
+                                    },
+                                    "options": {
+                                        "model": "gpt-3.5-turbo",
+                                        "stream": true
+                                    },
+                                    "override": {
+                                        "endpoint": "http://localhost:7737/v1/chat/completions?delay=true"
+                                    }
+                                }
+                            ],
+                            "ssl_verify": false
+                        }
+                    }
+                 }]]
+            )
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 6: assert access log contains right llm variable
+--- config
+    location /t {
+        content_by_lua_block {
+            local http = require("resty.http")
+            local httpc = http.new()
+            local core = require("apisix.core")
+            local ok, err = httpc:connect({
+                scheme = "http",
+                host = "localhost",
+                port = ngx.var.server_port,
+            })
+            if not ok then
+                ngx.status = 500
+                ngx.say(err)
+                return
+            end
+            local params = {
+                method = "POST",
+                headers = {
+                    ["Content-Type"] = "application/json",
+                },
+                path = "/anything",
+                body = [[{
+                    "messages": [
+                        { "role": "system", "content": "some content" }
+                    ],
+                    "model": "gpt-4"
+                }]],
+            }
+            local res, err = httpc:request(params)
+            if not res then
+                ngx.status = 500
+                ngx.say(err)
+                return
+            end
+            local final_res = {}
+            local inspect = require("inspect")
+            while true do
+                local chunk, err = res.body_reader() -- will read chunk by chunk
+                if err then
+                    core.log.error("failed to read response chunk: ", err)
+                    break
+                end
+                if not chunk then
+                    break
+                end
+                core.table.insert_tail(final_res, chunk)
+            end
+            ngx.print(#final_res .. final_res[6])
+        }
+    }
+--- response_body_like eval
+qr/6data: \[DONE\]\n\n/
+--- access_log eval
+qr/.*gpt-3.5-turbo 2\d\d 15 20.*/