Merge branch 'main' into deprecate-old-schema-ops

adapters/handlers/graphql/local/local_component_test.go

@@ -23,7 +23,7 @@ import (
 	"github.com/weaviate/weaviate/entities/models"
 	"github.com/weaviate/weaviate/entities/schema"
 	"github.com/weaviate/weaviate/usecases/config"
-	"github.com/weaviate/weaviate/usecases/modules"
+	usecaseModules "github.com/weaviate/weaviate/usecases/modules"
 )
 
 // These tests are component tests for the local package including all its
@@ -207,7 +207,7 @@ type testCases []testCase
 func (tests testCases) AssertNoError(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			modules := modules.NewProvider()
+			modules := usecaseModules.NewProvider()
 			localSchema, err := Build(&test.localSchema, nil, config.Config{}, modules)
 			require.Nil(t, err, test.name)
 
@@ -240,7 +240,7 @@ func (tests testCases) AssertNoError(t *testing.T) {
 func (tests testCases) AssertErrorLogs(t *testing.T, expectedMsg string) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			modules := modules.NewProvider()
+			modules := usecaseModules.NewProvider()
 			logger, logsHook := logrus.NewNullLogger()
 			localSchema, err := Build(&test.localSchema, logger, config.Config{}, modules)
 			require.Nil(t, err, test.name)

adapters/handlers/grpc/v1/parse_search_request.go

@@ -368,12 +368,6 @@ func extractTargetVectors(req *pb.SearchRequest, class *models.Class) (*[]string
 	var targetVectors *[]string
 	if hs := req.HybridSearch; hs != nil {
 		targetVectors = &hs.TargetVectors
-		if hs.NearText != nil {
-			targetVectors = &hs.NearText.TargetVectors
-		}
-		if hs.NearVector != nil {
-			targetVectors = &hs.NearVector.TargetVectors
-		}
 	}
 	if na := req.NearAudio; na != nil {
 		targetVectors = &na.TargetVectors

adapters/handlers/grpc/v1/parse_search_request_test.go

@@ -204,11 +204,11 @@ func TestGRPCRequest(t *testing.T) {
 					Alpha: 1.0,
 					Query: "nearvecquery",
 					NearVector: &pb.NearVector{
-						VectorBytes:   byteops.Float32ToByteVector([]float32{1, 2, 3}),
-						TargetVectors: []string{"custom"},
-						Certainty:     &one,
-						Distance:      &one,
+						VectorBytes: byteops.Float32ToByteVector([]float32{1, 2, 3}),
+						Certainty:   &one,
+						Distance:    &one,
 					},
+					TargetVectors: []string{"custom"},
 				},
 			},
 			out: dto.GetParams{
@@ -221,12 +221,12 @@ func TestGRPCRequest(t *testing.T) {
 					Query:           "nearvecquery",
 					FusionAlgorithm: 1,
 					NearVectorParams: &searchparams.NearVector{
-						Vector:        []float32{1, 2, 3},
-						TargetVectors: []string{"custom"},
-						Certainty:     1.0,
-						Distance:      1.0,
-						WithDistance:  true,
+						Vector:       []float32{1, 2, 3},
+						Certainty:    1.0,
+						Distance:     1.0,
+						WithDistance: true,
 					},
+					TargetVectors: []string{"custom"},
 				},
 			},
 			error: false,

adapters/handlers/grpc/v1/tenants.go

@@ -27,7 +27,7 @@ func (s *Service) tenantsGet(ctx context.Context, principal *models.Principal, r
 	var err error
 	var tenants []*models.Tenant
 	if req.Params == nil {
-		tenants, err = s.schemaManager.GetConsistentTenants(ctx, principal, req.Collection, req.IsConsistent, []string{})
+		tenants, err = s.schemaManager.GetConsistentTenants(ctx, principal, req.Collection, true, []string{})
 		if err != nil {
 			return nil, err
 		}
@@ -38,7 +38,7 @@ func (s *Service) tenantsGet(ctx context.Context, principal *models.Principal, r
 			if len(requestedNames) == 0 {
 				return nil, fmt.Errorf("must specify at least one tenant name")
 			}
-			tenants, err = s.schemaManager.GetConsistentTenants(ctx, principal, req.Collection, req.IsConsistent, requestedNames)
+			tenants, err = s.schemaManager.GetConsistentTenants(ctx, principal, req.Collection, true, requestedNames)
 			if err != nil {
 				return nil, err
 			}

adapters/handlers/rest/clusterapi/indices.go

@@ -1105,7 +1105,7 @@ func (i *indices) postShard() http.Handler {
 func (i *indices) putShardReinit() http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		args := i.regexpShardReinit.FindStringSubmatch(r.URL.Path)
-		fmt.Println(args)
+
 		if len(args) != 3 {
 			http.Error(w, "invalid URI", http.StatusBadRequest)
 			return

adapters/handlers/rest/configure_api.go

@@ -193,6 +193,8 @@ func MakeAppState(ctx context.Context, options *swag.CommandLineOptionsGroup) *s
 		MemtablesMaxSizeMB:        appState.ServerConfig.Config.Persistence.MemtablesMaxSizeMB,
 		MemtablesMinActiveSeconds: appState.ServerConfig.Config.Persistence.MemtablesMinActiveDurationSeconds,
 		MemtablesMaxActiveSeconds: appState.ServerConfig.Config.Persistence.MemtablesMaxActiveDurationSeconds,
+		MaxSegmentSize:            appState.ServerConfig.Config.Persistence.LSMMaxSegmentSize,
+		HNSWMaxLogSize:            appState.ServerConfig.Config.Persistence.HNSWMaxLogSize,
 		RootPath:                  appState.ServerConfig.Config.Persistence.DataPath,
 		QueryLimit:                appState.ServerConfig.Config.QueryDefaults.Limit,
 		QueryMaximumResults:       appState.ServerConfig.Config.QueryMaximumResults,
@@ -251,8 +253,6 @@ func MakeAppState(ctx context.Context, options *swag.CommandLineOptionsGroup) *s
 		remoteIndexClient, appState.Logger, appState.ServerConfig.Config.Persistence.DataPath)
 	appState.Scaler = scaler
 
-	/// TODO-RAFT START
-	//
 	server2port, err := parseNode2Port(appState)
 	if len(server2port) == 0 || err != nil {
 		appState.Logger.
@@ -268,31 +268,31 @@ func MakeAppState(ctx context.Context, options *swag.CommandLineOptionsGroup) *s
 	dataPath := appState.ServerConfig.Config.Persistence.DataPath
 
 	rConfig := rStore.Config{
-		WorkDir:               filepath.Join(dataPath, "raft"),
-		NodeID:                nodeName,
-		Host:                  addrs[0],
-		RaftPort:              appState.ServerConfig.Config.Raft.Port,
-		RPCPort:               appState.ServerConfig.Config.Raft.InternalRPCPort,
-		RaftRPCMessageMaxSize: appState.ServerConfig.Config.Raft.RPCMessageMaxSize,
-		ServerName2PortMap:    server2port,
-		BootstrapTimeout:      appState.ServerConfig.Config.Raft.BootstrapTimeout,
-		BootstrapExpect:       appState.ServerConfig.Config.Raft.BootstrapExpect,
-		HeartbeatTimeout:      appState.ServerConfig.Config.Raft.HeartbeatTimeout,
-		RecoveryTimeout:       appState.ServerConfig.Config.Raft.RecoveryTimeout,
-		ElectionTimeout:       appState.ServerConfig.Config.Raft.ElectionTimeout,
-		SnapshotInterval:      appState.ServerConfig.Config.Raft.SnapshotInterval,
-		SnapshotThreshold:     appState.ServerConfig.Config.Raft.SnapshotThreshold,
-		UpdateWaitTimeout:     time.Second * 10, // TODO-RAFT read from the flag
-		MetadataOnlyVoters:    appState.ServerConfig.Config.Raft.MetadataOnlyVoters,
-		DB:                    nil,
-		Parser:                schema.NewParser(appState.Cluster, vectorIndex.ParseAndValidateConfig, migrator),
-		AddrResolver:          appState.Cluster,
-		Logger:                appState.Logger,
-		LogLevel:              logLevel(),
-		LogJSONFormat:         !logTextFormat(),
-		IsLocalHost:           appState.ServerConfig.Config.Cluster.Localhost,
-		LoadLegacySchema:      schemaRepo.LoadLegacySchema,
-		SaveLegacySchema:      schemaRepo.SaveLegacySchema,
+		WorkDir:                filepath.Join(dataPath, config.DefaultRaftDir),
+		NodeID:                 nodeName,
+		Host:                   addrs[0],
+		RaftPort:               appState.ServerConfig.Config.Raft.Port,
+		RPCPort:                appState.ServerConfig.Config.Raft.InternalRPCPort,
+		RaftRPCMessageMaxSize:  appState.ServerConfig.Config.Raft.RPCMessageMaxSize,
+		ServerName2PortMap:     server2port,
+		BootstrapTimeout:       appState.ServerConfig.Config.Raft.BootstrapTimeout,
+		BootstrapExpect:        appState.ServerConfig.Config.Raft.BootstrapExpect,
+		HeartbeatTimeout:       appState.ServerConfig.Config.Raft.HeartbeatTimeout,
+		RecoveryTimeout:        appState.ServerConfig.Config.Raft.RecoveryTimeout,
+		ElectionTimeout:        appState.ServerConfig.Config.Raft.ElectionTimeout,
+		SnapshotInterval:       appState.ServerConfig.Config.Raft.SnapshotInterval,
+		SnapshotThreshold:      appState.ServerConfig.Config.Raft.SnapshotThreshold,
+		ConsistencyWaitTimeout: appState.ServerConfig.Config.Raft.ConsistencyWaitTimeout,
+		MetadataOnlyVoters:     appState.ServerConfig.Config.Raft.MetadataOnlyVoters,
+		DB:                     nil,
+		Parser:                 schema.NewParser(appState.Cluster, vectorIndex.ParseAndValidateConfig, migrator),
+		AddrResolver:           appState.Cluster,
+		Logger:                 appState.Logger,
+		LogLevel:               logLevel(),
+		LogJSONFormat:          !logTextFormat(),
+		IsLocalHost:            appState.ServerConfig.Config.Cluster.Localhost,
+		LoadLegacySchema:       schemaRepo.LoadLegacySchema,
+		SaveLegacySchema:       schemaRepo.SaveLegacySchema,
 	}
 	for _, name := range appState.ServerConfig.Config.Raft.Join[:rConfig.BootstrapExpect] {
 		if strings.Contains(name, rConfig.NodeID) {
@@ -322,7 +322,7 @@ func MakeAppState(ctx context.Context, options *swag.CommandLineOptionsGroup) *s
 	}
 
 	appState.SchemaManager = schemaManager
-	appState.RemoteIndexIncoming = sharding.NewRemoteIndexIncoming(repo, appState.ClusterService.SchemaReader())
+	appState.RemoteIndexIncoming = sharding.NewRemoteIndexIncoming(repo, appState.ClusterService.SchemaReader(), appState.Modules)
 	appState.RemoteNodeIncoming = sharding.NewRemoteNodeIncoming(repo)
 	appState.RemoteReplicaIncoming = replica.NewRemoteReplicaIncoming(repo, appState.ClusterService.SchemaReader())
 

adapters/repos/db/aggregations_fixtures_for_test.go

@@ -36,6 +36,25 @@ var productClass = &models.Class{
 	},
 }
 
+func boolRef(b bool) *bool {
+	return &b
+}
+
+var notIndexedClass = &models.Class{
+	Class:               "NotIndexedClass",
+	VectorIndexConfig:   enthnsw.NewDefaultUserConfig(),
+	InvertedIndexConfig: invertedConfig(),
+	Properties: []*models.Property{
+		{
+			Name:            "name",
+			DataType:        schema.DataTypeText.PropString(),
+			Tokenization:    models.PropertyTokenizationWhitespace,
+			IndexFilterable: boolRef(false),
+			IndexInverted:   boolRef(false),
+		},
+	},
+}
+
 var companyClass = &models.Class{
 	Class:               "AggregationsTestCompany",
 	VectorIndexConfig:   enthnsw.NewDefaultUserConfig(),

adapters/repos/db/aggregations_integration_test.go

@@ -129,6 +129,7 @@ func prepareCompanyTestSchemaAndData(repo *DB,
 			Objects: &models.Schema{
 				Classes: []*models.Class{
 					productClass,
+					notIndexedClass,
 					companyClass,
 					arrayTypesClass,
 					customerClass,
@@ -147,6 +148,8 @@ func prepareCompanyTestSchemaAndData(repo *DB,
 				migrator.AddClass(context.Background(), arrayTypesClass, schemaGetter.shardState))
 			require.Nil(t,
 				migrator.AddClass(context.Background(), customerClass, schemaGetter.shardState))
+			require.Nil(t,
+				migrator.AddClass(context.Background(), notIndexedClass, schemaGetter.shardState))
 		})
 
 		schemaGetter.schema = schema
@@ -165,6 +168,20 @@ func prepareCompanyTestSchemaAndData(repo *DB,
 			}
 		})
 
+		t.Run("import products into notIndexed class", func(t *testing.T) {
+			for i, schema := range products {
+				t.Run(fmt.Sprintf("importing product %d", i), func(t *testing.T) {
+					fixture := models.Object{
+						Class:      notIndexedClass.Class,
+						ID:         productsIds[i],
+						Properties: schema,
+					}
+					require.Nil(t,
+						repo.PutObject(context.Background(), &fixture, []float32{0.1, 0.2, 0.01, 0.2}, nil, nil, 0))
+				})
+			}
+		})
+
 		t.Run("import companies", func(t *testing.T) {
 			for j := 0; j < importFactor; j++ {
 				for i, schema := range companies {

adapters/repos/db/aggregator/filtered.go

@@ -124,6 +124,9 @@ func (fa *filteredAggregator) bm25Objects(ctx context.Context, kw *searchparams.
 		return nil, nil, fmt.Errorf("bm25 objects: could not find class %s in schema", fa.params.ClassName)
 	}
 	cfg := inverted.ConfigFromModel(class.InvertedIndexConfig)
+
+	kw.ChooseSearchableProperties(class)
+
 	objs, scores, err := inverted.NewBM25Searcher(cfg.BM25, fa.store, fa.getSchema.ReadOnlyClass,
 		propertyspecific.Indices{}, fa.classSearcher,
 		fa.GetPropertyLengthTracker(), fa.logger, fa.shardVersion,

adapters/repos/db/aggregator/hybrid.go

@@ -48,6 +48,8 @@ func (a *Aggregator) bm25Objects(ctx context.Context, kw *searchparams.KeywordRa
 	}
 	cfg := inverted.ConfigFromModel(class.InvertedIndexConfig)
 
+	kw.ChooseSearchableProperties(class)
+
 	objs, dists, err := inverted.NewBM25Searcher(cfg.BM25, a.store, a.getSchema.ReadOnlyClass,
 		propertyspecific.Indices{}, a.classSearcher,
 		a.GetPropertyLengthTracker(), a.logger, a.shardVersion,

adapters/repos/db/bm25f_test.go

@@ -114,6 +114,14 @@ func SetupClass(t require.TestingT, repo *DB, schemaGetter *fakeSchemaGetter, lo
 				IndexFilterable: &vFalse,
 				IndexSearchable: &vTrue,
 			},
+			// Test that bm25f handles this property being unsearchable
+			{
+				Name:            "notSearchable",
+				DataType:        schema.DataTypeTextArray.PropString(),
+				Tokenization:    models.PropertyTokenizationWhitespace,
+				IndexFilterable: &vFalse,
+				IndexSearchable: &vFalse,
+			},
 		},
 	}
 

adapters/repos/db/clusterintegrationtest/cluster_integration_test.go

@@ -35,6 +35,7 @@ import (
 	"github.com/weaviate/weaviate/entities/schema"
 	"github.com/weaviate/weaviate/entities/search"
 	"github.com/weaviate/weaviate/entities/searchparams"
+	"github.com/weaviate/weaviate/usecases/modules"
 	"github.com/weaviate/weaviate/usecases/objects"
 )
 
@@ -350,7 +351,7 @@ func testDistributed(t *testing.T, dirName string, rnd *rand.Rand, batch bool) {
 		}
 
 		node := nodes[rnd.Intn(len(nodes))]
-		res, err := node.repo.Aggregate(context.Background(), params, nil)
+		res, err := node.repo.Aggregate(context.Background(), params, modules.NewProvider())
 		require.Nil(t, err)
 
 		expectedResult := &aggregation.Result{

adapters/repos/db/clusterintegrationtest/fakes_for_test.go

@@ -36,6 +36,7 @@ import (
 	modstgfs "github.com/weaviate/weaviate/modules/backup-filesystem"
 	ubak "github.com/weaviate/weaviate/usecases/backup"
 	"github.com/weaviate/weaviate/usecases/memwatch"
+	"github.com/weaviate/weaviate/usecases/modules"
 	"github.com/weaviate/weaviate/usecases/sharding"
 )
 
@@ -99,7 +100,7 @@ func (n *node) init(dirName string, shardStateRaw []byte,
 
 	n.migrator = db.NewMigrator(n.repo, logger)
 
-	indices := clusterapi.NewIndices(sharding.NewRemoteIndexIncoming(n.repo, n.schemaManager),
+	indices := clusterapi.NewIndices(sharding.NewRemoteIndexIncoming(n.repo, n.schemaManager, modules.NewProvider()),
 		n.repo, clusterapi.NewNoopAuthHandler())
 	mux := http.NewServeMux()
 	mux.Handle("/indices/", indices.Indices())