feat: Add import scheduler and manager (#29367)

This PR introduces novel managerial roles for importv2:
1. ImportMeta: To manage all the import tasks;
2. ImportScheduler: To process tasks and modify their states;
3. ImportChecker: To ascertain the completion of all tasks and instigate
relevant operations.

issue: #28521

---------

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>

configs/milvus.yaml

@@ -434,7 +434,6 @@ dataCoord:
   import:
     filesPerPreImportTask: 2 # The maximum number of files allowed per pre-import task.
     taskRetention: 10800 # The retention period in seconds for tasks in the Completed or Failed state.
-    inactiveTimeout: 1800 # The timeout duration in seconds for a task in the "InProgress" state if it remains inactive (with no progress updates).
 
   enableGarbageCollection: true
   gc:

internal/datacoord/allocator.go

@@ -18,6 +18,7 @@ package datacoord
 
 import (
 	"context"
+	"time"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
 	"github.com/milvus-io/milvus/internal/proto/rootcoordpb"
@@ -30,6 +31,7 @@ import (
 type allocator interface {
 	allocTimestamp(context.Context) (Timestamp, error)
 	allocID(context.Context) (UniqueID, error)
+	allocN(n int64) (UniqueID, UniqueID, error)
 }
 
 // make sure rootCoordAllocator implements allocator interface
@@ -79,3 +81,25 @@ func (alloc *rootCoordAllocator) allocID(ctx context.Context) (UniqueID, error)
 
 	return resp.ID, nil
 }
+
+// allocID allocates an `UniqueID` from RootCoord, invoking AllocID grpc
+func (alloc *rootCoordAllocator) allocN(n int64) (UniqueID, UniqueID, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if n <= 0 {
+		n = 1
+	}
+	resp, err := alloc.AllocID(ctx, &rootcoordpb.AllocIDRequest{
+		Base: commonpbutil.NewMsgBase(
+			commonpbutil.WithMsgType(commonpb.MsgType_RequestID),
+			commonpbutil.WithSourceID(paramtable.GetNodeID()),
+		),
+		Count: uint32(n),
+	})
+
+	if err = VerifyResponse(resp, err); err != nil {
+		return 0, 0, err
+	}
+	start, count := resp.GetID(), resp.GetCount()
+	return start, start + int64(count), nil
+}

internal/datacoord/broker/coordinator_broker.go

@@ -32,6 +32,7 @@ import (
 	"github.com/milvus-io/milvus/pkg/util/paramtable"
 )
 
+//go:generate mockery --name=Broker --structname=MockBroker --output=./  --filename=mock_coordinator_broker.go --with-expecter --inpackage
 type Broker interface {
 	DescribeCollectionInternal(ctx context.Context, collectionID int64) (*milvuspb.DescribeCollectionResponse, error)
 	ShowPartitionsInternal(ctx context.Context, collectionID int64) ([]int64, error)