Surface internal optimization modes (all rows vs first rows) at the SQL and configuration levels

builds/install/misc/firebird.conf

@@ -469,6 +469,21 @@
 #InlineSortThreshold = 1000
 
 
+# ----------------------------
+# Defines whether queries should be optimized to retrieve the first records
+# as soon as possible rather than returning the whole dataset as soon as possible.
+# By default retrieval of all rows is implied by the optimizer.
+#
+# Can be overridden at the session level using the SET OPTIMIZE statement
+# or at the SQL statement level by using the OPTIMIZE FOR clause.
+#
+# Per-database configurable.
+#
+# Type: boolean
+#
+#OptimizeForFirstRows = false
+
+
 # ============================
 # Plugin settings
 # ============================

src/common/classes/Nullable.h

@@ -70,11 +70,21 @@ template <typename T> class BaseNullable
 		return (!specified && !o.specified) || (specified == o.specified && value == o.value);
 	}
 
+	bool operator !=(const BaseNullable<T>& o) const
+	{
+		return !(*this == o);
+	}
+
 	bool operator ==(const T& o) const
 	{
 		return specified && value == o;
 	}
 
+	bool operator !=(const T& o) const
+	{
+		return !(*this == o);
+	}
+
 	void operator =(const T& v)
 	{
 		this->value = v;

src/common/config/config.h

@@ -192,6 +192,7 @@ enum ConfigKey
 	KEY_MAX_STATEMENT_CACHE_SIZE,
 	KEY_PARALLEL_WORKERS,
 	KEY_MAX_PARALLEL_WORKERS,
+	KEY_OPTIMIZE_FOR_FIRST_ROWS,
 	MAX_CONFIG_KEY		// keep it last
 };
 
@@ -310,7 +311,8 @@ constexpr ConfigEntry entries[MAX_CONFIG_KEY] =
 	{TYPE_STRING,	"TempTableDirectory",		false,	""},
 	{TYPE_INTEGER,	"MaxStatementCacheSize",	false,	2 * 1048576},	// bytes
 	{TYPE_INTEGER,	"ParallelWorkers",			true,	1},
-	{TYPE_INTEGER,	"MaxParallelWorkers",		true,	1}
+	{TYPE_INTEGER,	"MaxParallelWorkers",		true,	1},
+	{TYPE_BOOLEAN,	"OptimizeForFirstRows",		false,	false}
 };
 
 
@@ -638,6 +640,8 @@ class Config : public RefCounted, public GlobalStorage
 	CONFIG_GET_GLOBAL_INT(getParallelWorkers, KEY_PARALLEL_WORKERS);
 
 	CONFIG_GET_GLOBAL_INT(getMaxParallelWorkers, KEY_MAX_PARALLEL_WORKERS);
+
+	CONFIG_GET_PER_DB_BOOL(getOptimizeForFirstRows, KEY_OPTIMIZE_FOR_FIRST_ROWS);
 };
 
 // Implementation of interface to access master configuration file

src/common/keywords.cpp

@@ -339,6 +339,7 @@ static const TOK tokens[] =
 	{TOK_ON, "ON", false},
 	{TOK_ONLY, "ONLY", false},
 	{TOK_OPEN, "OPEN", false},
+	{TOK_OPTIMIZE, "OPTIMIZE", true},
 	{TOK_OPTION, "OPTION", true},
 	{TOK_OR, "OR", false},
 	{TOK_ORDER, "ORDER", false},

src/dsql/BoolNodes.cpp

@@ -1124,7 +1124,7 @@ BoolExprNode* ComparativeBoolNode::createRseNode(DsqlCompilerScratch* dsqlScratc
 	const DsqlContextStack::iterator baseDT(dsqlScratch->derivedContext);
 	const DsqlContextStack::iterator baseUnion(dsqlScratch->unionContext);
 
-	RseNode* rse = PASS1_rse(dsqlScratch, select_expr, false, false);
+	RseNode* rse = PASS1_rse(dsqlScratch, select_expr);
 	rse->flags |= RseNode::FLAG_DSQL_COMPARATIVE;
 
 	// Create a conjunct to be injected.
@@ -1709,7 +1709,7 @@ DmlNode* RseBoolNode::parse(thread_db* tdbb, MemoryPool& pool, CompilerScratch*
 	node->rse->flags |= RseNode::FLAG_SUB_QUERY;
 
 	if (blrOp == blr_any || blrOp == blr_exists) // maybe for blr_unique as well?
-		node->rse->flags |= RseNode::FLAG_OPT_FIRST_ROWS;
+		node->rse->firstRows = true;
 
 	if (csb->csb_currentForNode && csb->csb_currentForNode->parBlrBeginCnt <= 1)
 		node->ownSavepoint = false;
@@ -1744,7 +1744,7 @@ BoolExprNode* RseBoolNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
 	const DsqlContextStack::iterator base(*dsqlScratch->context);
 
 	RseBoolNode* node = FB_NEW_POOL(dsqlScratch->getPool()) RseBoolNode(dsqlScratch->getPool(), blrOp,
-		PASS1_rse(dsqlScratch, nodeAs<SelectExprNode>(dsqlRse), false, false));
+		PASS1_rse(dsqlScratch, nodeAs<SelectExprNode>(dsqlRse)));
 
 	// Finish off by cleaning up contexts
 	dsqlScratch->context->clear(base);

src/dsql/DdlNodes.epp

@@ -8683,7 +8683,7 @@ void CreateAlterViewNode::execute(thread_db* tdbb, DsqlCompilerScratch* dsqlScra
 
 	dsqlScratch->resetContextStack();
 	++dsqlScratch->contextNumber;
-	RseNode* rse = PASS1_rse(dsqlScratch, selectExpr, false, false);
+	RseNode* rse = PASS1_rse(dsqlScratch, selectExpr);
 
 	dsqlScratch->getBlrData().clear();
 	dsqlScratch->appendUChar(dsqlScratch->isVersion4() ? blr_version4 : blr_version5);

src/dsql/ExprNodes.cpp

@@ -11208,7 +11208,7 @@ ValueExprNode* SubQueryNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
 
 	const DsqlContextStack::iterator base(*dsqlScratch->context);
 
-	RseNode* rse = PASS1_rse(dsqlScratch, nodeAs<SelectExprNode>(dsqlRse), false, false);
+	RseNode* rse = PASS1_rse(dsqlScratch, nodeAs<SelectExprNode>(dsqlRse));
 
 	SubQueryNode* node = FB_NEW_POOL(dsqlScratch->getPool()) SubQueryNode(dsqlScratch->getPool(), blrOp, rse,
 		rse->dsqlSelectList->items[0], NullNode::instance());

src/dsql/StmtNodes.cpp

@@ -63,6 +63,7 @@
 #include "../dsql/gen_proto.h"
 #include "../dsql/make_proto.h"
 #include "../dsql/pass1_proto.h"
+#include "../dsql/DsqlStatementCache.h"
 
 using namespace Firebird;
 using namespace Jrd;
@@ -1238,7 +1239,7 @@ DeclareCursorNode* DeclareCursorNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
 	dt->querySpec = dsqlSelect->dsqlExpr;
 	dt->alias = dsqlName.c_str();
 
-	rse = PASS1_derived_table(dsqlScratch, dt, NULL, dsqlSelect->dsqlWithLock, dsqlSelect->dsqlSkipLocked);
+	rse = PASS1_derived_table(dsqlScratch, dt, NULL, dsqlSelect);
 
 	// Assign number and store in the dsqlScratch stack.
 	cursorNumber = dsqlScratch->cursorNumber++;
@@ -4931,7 +4932,7 @@ ForNode* ForNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
 		dt->querySpec = dsqlSelect->dsqlExpr;
 		dt->alias = dsqlCursor->dsqlName.c_str();
 
-		node->rse = PASS1_derived_table(dsqlScratch, dt, NULL, dsqlSelect->dsqlWithLock, dsqlSelect->dsqlSkipLocked);
+		node->rse = PASS1_derived_table(dsqlScratch, dt, NULL, dsqlSelect);
 
 		dsqlCursor->rse = node->rse;
 		dsqlCursor->cursorNumber = dsqlScratch->cursorNumber++;
@@ -7528,7 +7529,7 @@ StmtNode* StoreNode::internalDsqlPass(DsqlCompilerScratch* dsqlScratch,
 		if (dsqlRse && dsqlScratch->isPsql() && dsqlReturning)
 			selExpr->dsqlFlags |= RecordSourceNode::DFLAG_SINGLETON;
 
-		RseNode* rse = PASS1_rse(dsqlScratch, selExpr, false, false);
+		RseNode* rse = PASS1_rse(dsqlScratch, selExpr);
 		node->dsqlRse = rse;
 		values = rse->dsqlSelectList;
 		needSavePoint = false;
@@ -8206,9 +8207,10 @@ SelectNode* SelectNode::dsqlPass(DsqlCompilerScratch* dsqlScratch)
 {
 	SelectNode* node = FB_NEW_POOL(dsqlScratch->getPool()) SelectNode(dsqlScratch->getPool());
 	node->dsqlForUpdate = dsqlForUpdate;
+	node->dsqlOptimizeForFirstRows = dsqlOptimizeForFirstRows;
 
 	const DsqlContextStack::iterator base(*dsqlScratch->context);
-	node->dsqlRse = PASS1_rse(dsqlScratch, dsqlExpr, dsqlWithLock, dsqlSkipLocked);
+	node->dsqlRse = PASS1_rse(dsqlScratch, dsqlExpr, this);
 	dsqlScratch->context->clear(base);
 
 	if (dsqlForUpdate)
@@ -9170,6 +9172,25 @@ void SetSessionNode::execute(thread_db* tdbb, DsqlRequest* request, jrd_tra** /*
 //--------------------
 
 
+void SetOptimizeNode::execute(thread_db* tdbb, DsqlRequest* /*request*/, jrd_tra** /*traHandle*/) const
+{
+	const auto attachment = tdbb->getAttachment();
+
+	if (attachment->att_opt_first_rows != optimizeMode)
+	{
+		attachment->att_opt_first_rows = optimizeMode;
+
+		// Clear the local compiled statements cache to allow queries
+		// to be re-optimized accordingly to the new rules
+
+		attachment->att_dsql_instance->dbb_statement_cache->purge(tdbb, false);
+	}
+}
+
+
+//--------------------
+
+
 void SetTimeZoneNode::execute(thread_db* tdbb, DsqlRequest* request, jrd_tra** /*traHandle*/) const
 {
 	Attachment* const attachment = tdbb->getAttachment();

src/dsql/StmtNodes.h

@@ -1344,6 +1344,7 @@ class SelectNode final : public TypedNode<StmtNode, StmtNode::TYPE_SELECT>
 	bool dsqlForUpdate = false;
 	bool dsqlWithLock = false;
 	bool dsqlSkipLocked = false;
+	TriState dsqlOptimizeForFirstRows;
 };
 
 
@@ -1840,6 +1841,37 @@ class SetBindNode : public SessionManagementNode
 };
 
 
+class SetOptimizeNode : public SessionManagementNode
+{
+public:
+	explicit SetOptimizeNode(MemoryPool& pool)
+		: SessionManagementNode(pool)
+	{
+	}
+
+	SetOptimizeNode(MemoryPool& pool, bool mode)
+		: SessionManagementNode(pool),
+		  optimizeMode(mode)
+	{
+	}
+
+public:
+	virtual Firebird::string internalPrint(NodePrinter& printer) const
+	{
+		SessionManagementNode::internalPrint(printer);
+
+		NODE_PRINT(printer, optimizeMode);
+
+		return "SetOptimizeNode";
+	}
+
+	virtual void execute(thread_db* tdbb, DsqlRequest* request, jrd_tra** traHandle) const;
+
+public:
+	TriState optimizeMode;
+};
+
+
 class SetTimeZoneNode : public SessionManagementNode
 {
 public:

src/dsql/gen.cpp

@@ -605,6 +605,12 @@ void GEN_rse(DsqlCompilerScratch* dsqlScratch, RseNode* rse)
 		gen_plan(dsqlScratch, rse->rse_plan);
 	}
 
+	if (rse->firstRows.isAssigned())
+	{
+		dsqlScratch->appendUChar(blr_optimize);
+		dsqlScratch->appendUChar(rse->firstRows.value);
+	}
+
 	dsqlScratch->appendUChar(blr_end);
 }
 

src/dsql/parse-conflicts.txt

@@ -1 +1 @@
-68 shift/reduce conflicts, 19 reduce/reduce conflicts.
+69 shift/reduce conflicts, 21 reduce/reduce conflicts.

src/dsql/parse.y

@@ -689,6 +689,7 @@ using namespace Firebird;
 // tokens added for Firebird 5.0
 
 %token <metaNamePtr> LOCKED
+%token <metaNamePtr> OPTIMIZE
 %token <metaNamePtr> QUARTER
 %token <metaNamePtr> TARGET
 %token <metaNamePtr> TIMEZONE_NAME
@@ -918,6 +919,7 @@ mng_statement
 	| session_reset								{ $$ = $1; }
 	| set_time_zone								{ $$ = $1; }
 	| set_bind									{ $$ = $1; }
+	| set_optimize								{ $$ = $1; }
 	;
 
 
@@ -5481,6 +5483,14 @@ decfloat_trap($setDecFloatTrapsNode)
 		{ $setDecFloatTrapsNode->trap($1); }
 	;
 
+%type <mngNode> set_optimize
+set_optimize
+	: SET OPTIMIZE optimize_mode
+		{ $$ = newNode<SetOptimizeNode>($3); }
+	| SET OPTIMIZE TO DEFAULT
+		{ $$ = newNode<SetOptimizeNode>(); }
+	;
+
 %type <setSessionNode> session_statement
 session_statement
 	: SET SESSION IDLE TIMEOUT long_integer timepart_sesion_idle_tout
@@ -5764,13 +5774,14 @@ ddl_desc
 
 %type <selectNode> select
 select
-	: select_expr for_update_clause lock_clause
+	: select_expr for_update_clause lock_clause optimize_clause
 		{
 			SelectNode* node = newNode<SelectNode>();
 			node->dsqlExpr = $1;
 			node->dsqlForUpdate = $2;
 			node->dsqlWithLock = $3.first;
 			node->dsqlSkipLocked = $3.second;
+			node->dsqlOptimizeForFirstRows = $4;
 			$$ = node;
 		}
 	;
@@ -5799,6 +5810,22 @@ skip_locked_clause_opt
 	| SKIP LOCKED			{ $$ = true; }
 	;
 
+%type <nullableBoolVal>	optimize_clause
+optimize_clause
+	: OPTIMIZE optimize_mode
+		{ $$ = Nullable<bool>::val($2); }
+	| // nothing
+		{ $$ = Nullable<bool>::empty(); }
+	;
+
+%type <boolVal> optimize_mode
+optimize_mode
+	: FOR FIRST ROWS
+		{ $$ = true; }
+	| FOR ALL ROWS
+		{ $$ = false; }
+	;
+
 
 // SELECT expression
 
@@ -9186,6 +9213,7 @@ non_reserved_word
 	| BLOB_APPEND
 	// added in FB 5.0
 	| LOCKED
+	| OPTIMIZE
 	| QUARTER
 	| TARGET
 	| TIMEZONE_NAME

src/dsql/pass1.cpp

@@ -578,15 +578,23 @@ dsql_ctx* PASS1_make_context(DsqlCompilerScratch* dsqlScratch, RecordSourceNode*
 
 // Compile a record selection expression, bumping up the statement scope level everytime an rse is
 // seen. The scope level controls parsing of aliases.
-RseNode* PASS1_rse(DsqlCompilerScratch* dsqlScratch, SelectExprNode* input, bool updateLock, bool skipLocked)
+RseNode* PASS1_rse(DsqlCompilerScratch* dsqlScratch,
+				   SelectExprNode* input,
+				   const SelectNode* select)
 {
 	DEV_BLKCHK(dsqlScratch, dsql_type_req);
 	DEV_BLKCHK(input, dsql_type_nod);
 
+	const bool updateLock = select ? select->dsqlWithLock : false;
+	const bool skipLocked = select ? select->dsqlSkipLocked : false;
+
 	dsqlScratch->scopeLevel++;
 	RseNode* node = pass1_rse(dsqlScratch, input, NULL, NULL, updateLock, skipLocked, 0);
 	dsqlScratch->scopeLevel--;
 
+	if (select)
+		node->firstRows = select->dsqlOptimizeForFirstRows;
+
 	return node;
 }
 
@@ -975,7 +983,7 @@ void PASS1_expand_contexts(DsqlContextStack& contexts, dsql_ctx* context)
 
 // Process derived table which is part of a from clause.
 RseNode* PASS1_derived_table(DsqlCompilerScratch* dsqlScratch, SelectExprNode* input,
-	const char* cte_alias, bool updateLock, bool skipLocked)
+	const char* cte_alias, const SelectNode* select)
 {
 	DEV_BLKCHK(dsqlScratch, dsql_type_req);
 
@@ -1076,7 +1084,7 @@ RseNode* PASS1_derived_table(DsqlCompilerScratch* dsqlScratch, SelectExprNode* i
 			rse = pass1_union(dsqlScratch, unionExpr, NULL, NULL, false, false, 0);
 		}
 		else
-			rse = PASS1_rse(dsqlScratch, input, updateLock, skipLocked);
+			rse = PASS1_rse(dsqlScratch, input, select);
 
 		// Finish off by cleaning up contexts and put them into derivedContext
 		// so create view (ddl) can deal with it.
@@ -1237,7 +1245,7 @@ RseNode* PASS1_derived_table(DsqlCompilerScratch* dsqlScratch, SelectExprNode* i
 			dsqlScratch->currCteAlias ? *dsqlScratch->currCteAlias : NULL;
 		dsqlScratch->resetCTEAlias(alias);
 
-		rse = PASS1_rse(dsqlScratch, input, updateLock, skipLocked);
+		rse = PASS1_rse(dsqlScratch, input, select);
 
 		if (saveCteAlias)
 			dsqlScratch->resetCTEAlias(*saveCteAlias);