YQ-2634: S3 runtime file listing (#925)

* Base feature implementation

* Added interconnect retries for FileQueue

* Added best effort FileQueue pass away

* Global ids for PB events, review fixes

* Added missing peer dirs

* Move FileQueue creation to FillSourceSettings

* Added FileQueueBatchSize pragrma, batch also limited by requested amount

* Round-robin stage in FileQueue, new s3 settings

* FileQueue processes one batch at a time, removing amount from batch request

* Runtime listing fixes

* Clean up

* Added parametrization in partitioning integration tests

* Poison fix

* Fixed partitioning columns, fixed retries, added extra tests

* Unit test imports fixes

* FileQueue consumers count correction

* Review fixes

Author: MetaGigachad

ydb/core/base/events.h

@@ -173,6 +173,7 @@ struct TKikimrEvents : TEvents {
         ES_GRAPH,
         ES_REPLICATION_SERVICE,
         ES_CHANGE_EXCHANGE,
+        ES_S3_FILE_QUEUE,
     };
 };
 

ydb/core/kqp/host/kqp_host.cpp

@@ -1475,6 +1475,7 @@ class TKqpHost : public IKqpHost {
         state->CredentialsFactory = FederatedQuerySetup->CredentialsFactory;
         state->Configuration->WriteThroughDqIntegration = true;
         state->Configuration->AllowAtomicUploadCommit = queryType == EKikimrQueryType::Script;
+        state->MaxTasksPerStage = SessionCtx->ConfigPtr()->MaxTasksPerStage.Get();
 
         state->Configuration->Init(FederatedQuerySetup->S3GatewayConfig, TypesCtx);
 

ydb/library/yql/dq/actors/compute/retry_queue.h

@@ -120,6 +120,11 @@ class TRetryEventsQueue {
         }
         return false;
     }
+    
+    bool HasPendingEvents() {
+        RemoveConfirmedEvents(MyConfirmedSeqNo);
+        return !Events.empty();
+    }
 
     void OnNewRecipientId(const NActors::TActorId& recipientId, bool unsubscribe = true);
     void HandleNodeConnected(ui32 nodeId);

ydb/library/yql/providers/s3/actors/ya.make

@@ -23,6 +23,7 @@ PEERDIR(
     library/cpp/string_utils/base64
     library/cpp/string_utils/quote
     library/cpp/xml/document
+    ydb/core/base
     ydb/core/fq/libs/events
     ydb/library/yql/dq/actors/compute
     ydb/library/yql/minikql/computation
@@ -36,6 +37,8 @@ PEERDIR(
     ydb/library/yql/providers/s3/credentials
     ydb/library/yql/providers/s3/object_listers
     ydb/library/yql/providers/s3/proto
+    ydb/library/yql/providers/s3/range_helpers
+    ydb/library/yql/public/issue
     ydb/library/yql/public/types
     ydb/library/yql/udfs/common/clickhouse/client
 )