Merge pull request #12 from VictoriaMetrics/issue-11-rows-limit

Implemented server-side configuration for default max rows limit (#11)

Author: Amper

README.md

@@ -112,7 +112,8 @@ Example (`config.json`):
     "errors": "* | level:ERROR",
     "traces": "* | span_id:*"
   },
-  "viewsDir": "./data/views"
+  "viewsDir": "./data/views",
+  "limit": 1000
 }
 ```
 
@@ -123,6 +124,7 @@ Example (`config.json`):
 | `bearerToken` | string            | Optional bearer token injected into VictoriaLogs requests when `endpoint` is set.                                       | empty             |
 | `tables`      | map[string]string | Mapping from SQL table name to LogsQL filter or pipeline fragment. Keys are case-insensitive.                           | `{ "logs": "*" }` |
 | `viewsDir`    | string            | Directory that stores `.logsql` files for views. Required for `CREATE VIEW`, `DROP VIEW`, and `SHOW VIEWS`.             | `./data/views`    |
+| `limit`       | int               | Maximum number of rows returned by any query.                                                                           | 1000              |
 
 Please note that VictoriaLogs is called via the backend, so if you are using sql-to-logsql in Docker, localhost refers to the localhost of the container, not your computer.
 
@@ -195,12 +197,15 @@ Successful response:
 Errors emit `HTTP 4xx/5xx` with `{ "error": "..." }`. 
 Parser, translator, VictoriaLogs client, and view-store errors map to informative messages (`400`, `409`, `423`, `502`, etc.).
 
-### `GET /api/v1/endpoint`
+### `GET /api/v1/config`
 
-Returns the compile-time endpoint configured on the server (used by the UI to decide whether the endpoint fields should be read-only):
+Returns the endpoint and max rows limit configured on the server (used by the UI to decide whether the endpoint fields should be read-only):
 
 ```json
-{ "endpoint": "https://victoria-logs.example.com" }
+{ 
+  "endpoint": "https://victoria-logs.example.com", 
+  "limit": 1000 
+}
 ```
 
 ### `GET /healthz`

cmd/sql-to-logsql/api/server.go

@@ -30,6 +30,7 @@ type Config struct {
 	BearerToken string            `json:"bearerToken"`
 	Tables      map[string]string `json:"tables"`
 	ViewsDir    string            `json:"viewsDir"`
+	Limit       uint32            `json:"limit"`
 }
 
 type Server struct {
@@ -62,20 +63,23 @@ func NewServer(cfg Config) (*Server, error) {
 	srv := &Server{
 		mux: http.NewServeMux(),
 		sp:  sp,
-		api: vlogs.NewVLogsAPI(vlogs.EndpointConfig{
-			Endpoint:    serverCfg.Endpoint,
-			BearerToken: serverCfg.BearerToken,
-		}),
+		api: vlogs.NewVLogsAPI(
+			vlogs.EndpointConfig{
+				Endpoint:    serverCfg.Endpoint,
+				BearerToken: serverCfg.BearerToken,
+			},
+			serverCfg.Limit,
+		),
 	}
 	srv.mux.HandleFunc("/healthz", withSecurityHeaders(srv.handleHealth))
 	srv.mux.HandleFunc("/api/v1/sql-to-logsql", withSecurityHeaders(srv.handleQuery))
-	srv.mux.HandleFunc("/api/v1/endpoint", withSecurityHeaders(func(w http.ResponseWriter, r *http.Request) {
+	srv.mux.HandleFunc("/api/v1/config", withSecurityHeaders(func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodGet {
 			w.Header().Set("Allow", http.MethodGet)
 			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
 			return
 		}
-		writeJSON(w, http.StatusOK, map[string]string{"endpoint": serverCfg.Endpoint})
+		writeJSON(w, http.StatusOK, map[string]any{"endpoint": serverCfg.Endpoint, "limit": serverCfg.Limit})
 	}))
 	srv.mux.HandleFunc("/", withSecurityHeaders(srv.handleStatic))
 	return srv, nil

cmd/sql-to-logsql/main.go

@@ -36,6 +36,9 @@ func main() {
 	if len(cfg.Tables) == 0 {
 		cfg.Tables = map[string]string{"logs": "*"}
 	}
+	if (cfg.Limit) <= 0 {
+		cfg.Limit = 1000
+	}
 	srv, err := api.NewServer(cfg)
 	if err != nil {
 		log.Fatalf("failed to configure server: %v", err)

cmd/sql-to-logsql/web/ui/src/components/sql-editor/SQLEditor.tsx

@@ -13,21 +13,24 @@ import {Select, SelectContent, SelectItem, SelectTrigger} from "@/components/ui/
 import {SelectValue} from "@radix-ui/react-select";
 import {DEFAULT_EXAMPLE_ID, EXAMPLES} from "@/components/sql-editor/examples.ts";
 import {COMPLETIONS} from "@/components/sql-editor/complections.ts";
-import {CircleXIcon, CircleCheckBigIcon, PlayIcon} from "lucide-react"
+import {CircleXIcon, CircleCheckBigIcon, PlayIcon, ListFilterIcon} from "lucide-react"
 import {Spinner} from "@/components/ui/spinner.tsx";
+import {Badge} from "@/components/ui/badge.tsx";
 
 export interface SqlEditorProps {
   readonly onRun?: (sql: string) => void;
   readonly isLoading?: boolean;
   readonly error?: string;
   readonly success?: string;
+  readonly limit?: number
 }
 
 export function SQLEditor({
   onRun,
   isLoading,
   error,
   success,
+  limit,
 }: SqlEditorProps) {
   const [value, setValue] = useState<string>(DEFAULT_EXAMPLE_ID);
   const [sql, setSql] = useState("");
@@ -141,6 +144,14 @@ export function SQLEditor({
             <span className={"text-green-700"}>{success}</span>
           </CardFooter>
       )}
+      {!error && !success && limit && limit > 0 && (
+        <CardFooter className={"flex gap-1"}>
+          <ListFilterIcon />
+          <span>
+            Any query will be limited to <Badge variant={"secondary"} className={"font-semibold"}>{limit}</Badge> rows.
+          </span>
+        </CardFooter>
+      )}
     </Card>
   );
 }

cmd/sql-to-logsql/web/ui/src/pages/main/Main.tsx

@@ -27,16 +27,18 @@ export function Main() {
   const [loading, setLoading] = useState<boolean>(false);
   const [error, setError] = useState<string>("");
   const [success, setSuccess] = useState<string>("");
+  const [limit, setLimit] = useState<number>(0);
 
   useEffect(() => {
     setLoading(true);
-    fetch(`/api/v1/endpoint`).then(resp => resp.json()).then(data => {
+    fetch(`/api/v1/config`).then(resp => resp.json()).then(data => {
       if (data.endpoint) {
         setEndpointUrl(data.endpoint);
         setBearerToken("secret");
         setEndpointReadOnly(true);
         setEndpointEnabled(false);
       }
+      setLimit(data.limit || 0);
       setLoading(false);
     })
   }, [])
@@ -103,6 +105,7 @@ export function Main() {
             isLoading={loading}
             error={error}
             success={success}
+            limit={limit}
           />
         </div>
         <Docs />

lib/vlogs/api.go

@@ -20,12 +20,14 @@ type EndpointConfig struct {
 
 type API struct {
 	ec     EndpointConfig
+	limit  uint32
 	client *http.Client
 }
 
-func NewVLogsAPI(ec EndpointConfig) *API {
+func NewVLogsAPI(ec EndpointConfig, limit uint32) *API {
 	return &API{
-		ec: ec,
+		ec:    ec,
+		limit: limit,
 		client: &http.Client{
 			Timeout: 60 * time.Second,
 		},
@@ -89,6 +91,7 @@ func (a *API) Query(ctx context.Context, logsQL string, recEC EndpointConfig) ([
 	reqURL = reqURL.JoinPath("/select/logsql/query")
 	form := url.Values{}
 	form.Set("query", logsQL)
+	form.Set("limit", fmt.Sprintf("%d", a.limit))
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, reqURL.String(), strings.NewReader(form.Encode()))
 	if err != nil {
 		return nil, &APIError{