collection: adding allow-unsafe-internals flag

README.md

@@ -32,6 +32,8 @@ The Prometheus collectors will add additional labels to track the cluster name a
 
 > Note: All queries executed to retrieve metrics are run on a **read-only connection** with follower reads enabled by default (`SET default_transaction_use_follower_reads = 'true'`). Follower reads allow queries to be served by replicas that are not necessarily the leaseholder, using slightly historical data (typically up to a few seconds old). This approach minimizes the risk of creating contention or adding load on the primary leaseholder replicas, ensuring that metric collection does not interfere with the performance of foreground workloads.
 
+> Note: In CockroachDB v26 and later, the `--allow-unsafe-internals` flag is required to query `crdb_internal` tables on read-only connections. Use this flag with both the `start` and `collection test` commands when collections need to access internal tables.
+
 ## Database Security
 
 It is recommended to use separate users for managing the configuration and to run the sidecar.
@@ -435,6 +437,7 @@ Examples:
 ./visus start --bindAddr "127.0.0.1:15432" 
 
 Flags:
+      --allow-unsafe-internals   set allow_unsafe_internals = true for read-only database connections
       --bind-addr string     A network address and port to bind to (default "127.0.0.1:8888")
       --bind-cert string     Path to the  TLS certificate for the server
       --bind-key string      Path to the  TLS key for the server

internal/cmd/collection/admin.go

@@ -172,6 +172,7 @@ func putCmd(env *env.Env) *cobra.Command {
 func testCmd(env *env.Env) *cobra.Command {
 	var interval time.Duration
 	var count int
+	var allowUnsafeInternals bool
 	c := &cobra.Command{
 		Use:     "test",
 		Example: `./visus collection test  collection_name  --url "postgresql://root@localhost:26257/defaultdb?sslmode=disable" `,
@@ -182,7 +183,7 @@ func testCmd(env *env.Env) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			conn, err := env.ProvideReadOnlyConnection(ctx, databaseURL)
+			conn, err := env.ProvideReadOnlyConnection(ctx, databaseURL, allowUnsafeInternals)
 			if err != nil {
 				return err
 			}
@@ -223,6 +224,7 @@ func testCmd(env *env.Env) *cobra.Command {
 		},
 	}
 	f := c.Flags()
+	f.BoolVar(&allowUnsafeInternals, "allow-unsafe-internals", false, "set allow_unsafe_internals = true for read-only database connections")
 	f.DurationVar(&interval, "interval", 10*time.Second, "interval of collection")
 	f.IntVar(&count, "count", 1, "number of times to run the collection. Specify 0 for continuos collection")
 	return c

internal/cmd/collection/admin_test.go

@@ -245,6 +245,21 @@ func TestCommands(t *testing.T) {
 			initialStore:  []*store.Collection{collection1},
 			name:          "test collection store error",
 		},
+		{
+			args:               []string{"test", "--interval", "1s", "--allow-unsafe-internals", "--url", "fake://", collection1.Name},
+			expectedOut:        "HELP collection_01_queries total queries per database",
+			expectedStoreNames: []string{collection1.Name},
+			initialStore:       []*store.Collection{collection1},
+			name:               "test collection with allow-unsafe-internals flag",
+			mock:               mockResults(t),
+		},
+		{
+			args:          []string{"test", "--interval", "1s", "--invalid", "--url", "fake://", collection1.Name},
+			expectedError: "unknown flag: --invalid",
+			initialStore:  []*store.Collection{collection1},
+			name:          "test collection with invalid flag",
+			mock:          mockResults(t),
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {

internal/cmd/env/env.go

@@ -77,13 +77,13 @@ func (e *Env) InjectStoreError(err error) {
 
 // ProvideReadOnlyConnection returns the database connection to use for collectors.
 func (e *Env) ProvideReadOnlyConnection(
-	ctx context.Context, databaseURL string,
+	ctx context.Context, databaseURL string, allowUnsafeInternals bool,
 ) (database.Connection, error) {
 	if e.testing {
 		return e.roPool, nil
 	}
 	var err error
-	e.roPool, err = database.ReadOnly(ctx, databaseURL)
+	e.roPool, err = database.ReadOnly(ctx, databaseURL, allowUnsafeInternals)
 	if err != nil {
 		return nil, err
 	}

internal/cmd/server/server.go

@@ -58,7 +58,7 @@ func Command() *cobra.Command {
 				return err
 			}
 			store := store.New(conn)
-			roConn, err := database.ReadOnly(ctx, cfg.URL)
+			roConn, err := database.ReadOnly(ctx, cfg.URL, cfg.AllowUnsafeInternals)
 			if err != nil {
 				return err
 			}
@@ -140,6 +140,7 @@ func Command() *cobra.Command {
 		},
 	}
 	f := c.Flags()
+	f.BoolVar(&cfg.AllowUnsafeInternals, "allow-unsafe-internals", false, "set allow_unsafe_internals = true for read-only database connections")
 	f.StringVar(&cfg.BindAddr, "bind-addr", "127.0.0.1:8888", "A network address and port to bind to")
 	f.StringVar(&cfg.BindCert, "bind-cert", "",
 		"Path to the  TLS certificate for the server")

internal/database/factory.go

@@ -26,40 +26,54 @@ import (
 // New creates a new connection to the database.
 // It waits until a connection can be established, or that the context has been cancelled.
 func New(ctx context.Context, URL string) (*Pool, error) {
-	return new(ctx, URL, false)
+	return new(ctx, URL, false /* readOnly */, false /* allowUnsafeInternals */)
 }
 
-// ReadOnly creates a new connection to the database with follower reads
+// ReadOnly creates a new connection to the database with follower reads.
 // It waits until a connection can be established, or that the context has been cancelled.
-func ReadOnly(ctx context.Context, URL string) (*Pool, error) {
-	return new(ctx, URL, true)
+// If allowUnsafeInternals is true, sets allow_unsafe_internals = true on the connection.
+func ReadOnly(ctx context.Context, URL string, allowUnsafeInternals bool) (*Pool, error) {
+	return new(ctx, URL, true /* readOnly */, allowUnsafeInternals)
 }
 
-func new(ctx context.Context, URL string, ro bool) (*Pool, error) {
-	pool, err := pgxPool(ctx, URL, ro)
+func new(ctx context.Context, URL string, ro bool, allowUnsafeInternals bool) (*Pool, error) {
+	pool, err := pgxPool(ctx, URL, ro, allowUnsafeInternals)
 	if err != nil {
 		return nil, err
 	}
 	res := &Pool{
-		URL:      URL,
-		readOnly: ro,
+		URL:                  URL,
+		readOnly:             ro,
+		allowUnsafeInternals: allowUnsafeInternals,
 	}
 	res.mu.pool = pool
 	return res, nil
 }
 
-func pgxPool(ctx context.Context, URL string, ro bool) (*pgxpool.Pool, error) {
+func pgxPool(
+	ctx context.Context, URL string, ro bool, allowUnsafeInternals bool,
+) (*pgxpool.Pool, error) {
 	var pool *pgxpool.Pool
 	sleepTime := int64(5)
 	poolConfig, err := pgxpool.ParseConfig(URL)
 	if err != nil {
 		log.Fatal(err)
 	}
-	if ro {
+	if ro || allowUnsafeInternals {
 		poolConfig.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error {
-			log.Debug("setting up a read only session")
-			_, err := conn.Exec(ctx, "set session default_transaction_use_follower_reads = true;")
-			return err
+			if ro {
+				log.Debug("setting up a read only session")
+				if _, err := conn.Exec(ctx, "set session default_transaction_use_follower_reads = true;"); err != nil {
+					return err
+				}
+			}
+			if allowUnsafeInternals {
+				log.Debug("setting allow_unsafe_internals = true")
+				if _, err := conn.Exec(ctx, "set allow_unsafe_internals = true;"); err != nil {
+					return err
+				}
+			}
+			return nil
 		}
 	}
 	for {

internal/database/pool.go

@@ -26,9 +26,10 @@ import (
 
 // Pool provides a set of connections to the target database.
 type Pool struct {
-	readOnly bool
-	URL      string
-	mu       struct {
+	readOnly             bool
+	allowUnsafeInternals bool
+	URL                  string
+	mu                   struct {
 		sync.RWMutex
 		pool *pgxpool.Pool
 	}
@@ -70,7 +71,7 @@ func (p *Pool) Refresh(ctx context.Context) error {
 	var err error
 	// We don't lock the pool until we get the new one.
 	// Existing clients may still use the current pool.
-	if pool, err = pgxPool(ctx, p.URL, p.readOnly); err != nil {
+	if pool, err = pgxPool(ctx, p.URL, p.readOnly, p.allowUnsafeInternals); err != nil {
 		return err
 	}
 	p.mu.Lock()

internal/server/config.go

@@ -19,16 +19,17 @@ import "time"
 // Config encapsulates the command-line parameters that can be supplied
 // to configure the behavior of Visus.
 type Config struct {
-	BindAddr          string        // Address to bind the server to.
-	BindCert, BindKey string        // Paths to Certificate and Key.
-	CaCert            string        // Path to the Root CA.
-	Endpoint          string        // Endpoint for metrics
-	Inotify           bool          // Enable inotify for scans.
-	Insecure          bool          // Sanity check to ensure that the operator really means it.
-	ProcMetrics       bool          // Enable collections of process metrics.
-	Prometheus        string        // URL for the node prometheus endpoint
-	Refresh           time.Duration // how often to refresh the configuration.
-	RewriteHistograms bool          // Enable histogram rewriting
-	URL               string        // URL to connect to the database
-	VisusMetrics      bool          // Enable collection of Visus metrics.
+	AllowUnsafeInternals bool          // Enable allow_unsafe_internals for database connections.
+	BindAddr             string        // Address to bind the server to.
+	BindCert, BindKey    string        // Paths to Certificate and Key.
+	CaCert               string        // Path to the Root CA.
+	Endpoint             string        // Endpoint for metrics
+	Inotify              bool          // Enable inotify for scans.
+	Insecure             bool          // Sanity check to ensure that the operator really means it.
+	ProcMetrics          bool          // Enable collections of process metrics.
+	Prometheus           string        // URL for the node prometheus endpoint
+	Refresh              time.Duration // how often to refresh the configuration.
+	RewriteHistograms    bool          // Enable histogram rewriting
+	URL                  string        // URL to connect to the database
+	VisusMetrics         bool          // Enable collection of Visus metrics.
 }

internal/store/sql/mainNode.sql

@@ -16,24 +16,24 @@
 
 WITH touch as (
 UPSERT INTO
-  _visus.node 
-VALUES 
-  (crdb_internal.node_id(), current_timestamp())
+  _visus.node
+VALUES
+  ((SELECT node_id::INT FROM [SHOW node_id]), current_timestamp())
 RETURNING _visus.node.*
 ), nodes as (
 SELECT
   *
 FROM
-  _visus.node 
-WHERE 
+  _visus.node
+WHERE
   updated >= $1
 UNION
 SELECT
   *
 FROM
   touch
 )
-SELECT 
-  max(id) = crdb_internal.node_id()
-FROM 
+SELECT
+  max(id) = (SELECT node_id::INT FROM [SHOW node_id])
+FROM
   nodes;