Working on filters

all/src/test/scala/spec/SimpleSpec.scala

@@ -82,6 +82,15 @@ class SimpleSpec extends AsyncWordSpec with AsyncIOSpec with Matchers {
         doc(Person.age) should be(19)
       }
     }
+    "search by age for positive result" in {
+      db.people.query.filter(Person.age is 19).search().compile.toList.map { results =>
+        results.length should be(1)
+        val doc = results.head
+        doc.id should be(id2)
+        doc(Person.name) should be("Jane Doe")
+        doc(Person.age) should be(19)
+      }
+    }
     "delete John" in {
       db.people.delete(id1)
     }

core/shared/src/main/scala/lightdb/ObjectMapping.scala

@@ -1,5 +1,6 @@
 package lightdb
 
+import fabric.rw.RW
 import lightdb.data.DataManager
 import lightdb.field.Field
 
@@ -17,8 +18,9 @@ trait ObjectMapping[D <: Document[D]] { om =>
   object field {
     def get[F](name: String): Option[FD[F]] = _fields.find(_.name == name).map(_.asInstanceOf[Field[D, F]])
 
-    def apply[F](name: String, getter: D => F): FD[F] = {
-      replace(Field[D, F](name, getter, Nil, om))
+    def apply[F](name: String, getter: D => F)
+                (implicit rw: RW[F]): FD[F] = {
+      replace(Field[D, F](name, getter, om))
     }
 
     def replace[F](field: Field[D, F]): FD[F] = om.synchronized {

core/shared/src/main/scala/lightdb/field/Field.scala

@@ -1,12 +1,10 @@
 package lightdb.field
 
+import fabric.rw.RW
 import lightdb.{Document, ObjectMapping}
 
 case class Field[D <: Document[D], F](name: String,
                                       getter: D => F,
-                                      features: List[FieldFeature],
-                                      mapping: ObjectMapping[D]) {
-  def withFeature(feature: FieldFeature): Field[D, F] = {
-    mapping.field.replace(copy(features = features ::: List(feature)))
-  }
-}
+                                      mapping: ObjectMapping[D],
+                                      stored: Boolean = true)
+                                     (implicit val rw: RW[F])

core/shared/src/main/scala/lightdb/query/Filter.scala

@@ -7,7 +7,37 @@ sealed trait Filter[D <: Document[D]] {
   def matches(document: D): Boolean
 }
 
+sealed trait Condition
+
+object Condition {
+  case object Filter extends Condition
+  case object Must extends Condition
+  case object MustNot extends Condition
+  case object Should extends Condition
+}
+
 object Filter {
+  case class GroupedFilter[D <: Document[D], F](minimumNumberShouldMatch: Int,
+                                             filters: List[(Filter[D], Condition)]) extends Filter[D] {
+    override def matches(document: D): Boolean = {
+      var should = 0
+      var fail = 0
+      filters.foreach {
+        case (filter, condition) =>
+          val b = filter.matches(document)
+          condition match {
+            case Condition.Must | Condition.Filter if !b => fail +=1
+            case Condition.MustNot if b => fail += 1
+            case Condition.Should if b => should += 1
+            case _ => // Ignore others
+          }
+      }
+      fail == 0 && should >= minimumNumberShouldMatch
+    }
+
+    override def toString: String = s"grouped(minimumNumberShouldMatch: $minimumNumberShouldMatch, conditionalTerms: ${filters.map(ct => s"${ct._1} -> ${ct._2}").mkString(", ")})"
+  }
+
   case class Equals[D <: Document[D], F](field: Field[D, F], value: F) extends Filter[D] {
     override def matches(document: D): Boolean = field.getter(document) == value
   }

core/shared/src/main/scala/lightdb/query/Query.scala

@@ -6,16 +6,17 @@ import lightdb.collection.Collection
 import lightdb.index.SearchResult
 
 case class Query[D <: Document[D]](collection: Collection[D],
-                                   filters: List[Filter[D]] = Nil,
+                                   filter: Option[Filter[D]] = None,
                                    sort: List[Sort] = Nil,
                                    scoreDocs: Boolean = false,
                                    offset: Int = 0,
                                    limit: Int = 100) {
-  def filter(filters: Filter[D]*): Query[D] = copy(filters = this.filters ::: filters.toList)
+  def filter(filter: Filter[D]): Query[D] = copy(filter = Some(filter))
+  def clearFilter: Query[D] = copy(filter = None)
   def sort(sort: Sort*): Query[D] = copy(sort = this.sort ::: sort.toList)
   def limit(limit: Int): Query[D] = copy(limit = limit)
   def scoreDocs(b: Boolean = true): Query[D] = copy(scoreDocs = b)
   def search(): fs2.Stream[IO, SearchResult[D]] = collection.indexer.search(this)
 
-  def matches(document: D): Boolean = filters.forall(_.matches(document))
+  def matches(document: D): Boolean = filter.forall(_.matches(document))
 }

lucene/src/main/scala/lightdb/index/lucene/LuceneIndexer.scala

@@ -1,17 +1,18 @@
 package lightdb.index.lucene
 
 import cats.effect.IO
+import fabric.define.DefType
 import lightdb.collection.Collection
 import lightdb.index.{Indexer, SearchResult}
-import lightdb.query.{Filter, Query}
+import lightdb.query.{Condition, Filter, Query}
 import lightdb.{Document, Id}
 import org.apache.lucene.analysis.Analyzer
 import org.apache.lucene.analysis.standard.StandardAnalyzer
 import org.apache.lucene.index.{IndexWriter, IndexWriterConfig, Term}
 import org.apache.lucene.queryparser.classic.QueryParser
-import org.apache.lucene.search._
+import org.apache.lucene.search.{BooleanClause, BooleanQuery, IndexSearcher, MatchAllDocsQuery, PhraseQuery, ScoreDoc, SearcherFactory, SearcherManager, Sort, SortField, TermQuery, Query => LuceneQuery}
 import org.apache.lucene.store.{ByteBuffersDirectory, FSDirectory}
-import org.apache.lucene.document.{Field, IntField, TextField, Document => LuceneDocument}
+import org.apache.lucene.document.{Field, IntField, IntRange, TextField, Document => LuceneDocument}
 
 import java.nio.file.Path
 import scala.collection.immutable.ArraySeq
@@ -44,10 +45,12 @@ case class LuceneIndexer[D <: Document[D]](collection: Collection[D],
   override def put(value: D): IO[D] = IO {
     val document = new LuceneDocument
     collection.mapping.fields.foreach { field =>
+      def textFieldType = if (field.stored) TextField.TYPE_STORED else TextField.TYPE_NOT_STORED
+      def fieldStore = if (field.stored) Field.Store.YES else Field.Store.NO
       field.getter(value) match {
-        case id: Id[_] => document.add(new Field(field.name, id.value, TextField.TYPE_STORED))
-        case s: String => document.add(new Field(field.name, s, TextField.TYPE_STORED))
-        case i: Int => document.add(new IntField(field.name, i, Field.Store.YES))
+        case id: Id[_] => document.add(new Field(field.name, id.value, textFieldType))
+        case s: String => document.add(new Field(field.name, s, textFieldType))
+        case i: Int => document.add(new IntField(field.name, i, fieldStore))
         case value => throw new RuntimeException(s"Unsupported value: $value (${value.getClass})")
       }
     }
@@ -63,11 +66,11 @@ case class LuceneIndexer[D <: Document[D]](collection: Collection[D],
     doAutoCommit()
   }
 
-  protected def doAutoCommit(): Unit = if (autoCommit) {
+  private def doAutoCommit(): Unit = if (autoCommit) {
     commitBlocking()
   }
 
-  protected def commitBlocking(): Unit = {
+  private def commitBlocking(): Unit = {
     indexWriter.flush()
     indexWriter.commit()
     i.synchronized {
@@ -85,31 +88,55 @@ case class LuceneIndexer[D <: Document[D]](collection: Collection[D],
     indexSearcher.count(new MatchAllDocsQuery)
   }
 
-  override def search(query: Query[D]): fs2.Stream[IO, SearchResult[D]] = {
-    val filters = query.filters.map {
-      case Filter.Equals(field, value) => s"${field.name}:$value"
-      case f => throw new UnsupportedOperationException(s"Unsupported filter: $f")
+  private def condition2Lucene(condition: Condition): BooleanClause.Occur = condition match {
+    case Condition.Filter => BooleanClause.Occur.FILTER
+    case Condition.Must => BooleanClause.Occur.MUST
+    case Condition.MustNot => BooleanClause.Occur.MUST_NOT
+    case Condition.Should => BooleanClause.Occur.SHOULD
+  }
+
+  private def filter2Query(filter: Filter[D]): LuceneQuery = filter match {
+    case Filter.GroupedFilter(minimumNumberShouldMatch, filters) =>
+      val b = new BooleanQuery.Builder
+      b.setMinimumNumberShouldMatch(minimumNumberShouldMatch)
+      if (filters.forall(_._2 == Condition.MustNot)) {           // Work-around for all negative groups, something must match
+        b.add(new MatchAllDocsQuery, BooleanClause.Occur.MUST)
+      }
+      filters.foreach {
+        case (filter, condition) => b.add(filter2Query(filter), condition2Lucene(condition))
+      }
+      b.build()
+    case Filter.Equals(field, value) => value match {
+      case s: String =>
+        val b = new PhraseQuery.Builder
+        s.toLowerCase.split(' ').foreach { word =>
+          b.add(new Term(field.name, word))
+        }
+        b.setSlop(0)
+        b.build()
+      case i: Int => IntField.newExactQuery(field.name, i)
+      case _ => throw new UnsupportedOperationException(s"Unsupported: $value (${field.name})")
     }
-    // TODO: Support filtering better
-    val q = if (filters.isEmpty) {
-      new MatchAllDocsQuery
+  }
+
+  override def search(query: Query[D]): fs2.Stream[IO, SearchResult[D]] = {
+    val q = query.filter.map(filter2Query).getOrElse(new MatchAllDocsQuery)
+    val sortFields = if (query.sort.isEmpty) {
+      List(SortField.FIELD_SCORE)
     } else {
-      val filterString = filters match {
-        case f :: Nil => f
-        case list => list.mkString("(", " AND ", ")")
+      query.sort.map {
+        case lightdb.query.Sort.BestMatch => SortField.FIELD_SCORE
+        case lightdb.query.Sort.IndexOrder => SortField.FIELD_DOC
+        case lightdb.query.Sort.ByField(field, reverse) =>
+          val sortType = field.rw.definition match {
+            case DefType.Str => SortField.Type.STRING
+            case d => throw new RuntimeException(s"Unsupported DefType: $d")
+          }
+          new SortField(field.name, sortType, reverse)
       }
-      parser.parse(filterString)
-    }
-    // TODO Sort
-    query.sort.map {
-      case lightdb.query.Sort.BestMatch => SortField.FIELD_SCORE
-      case lightdb.query.Sort.IndexOrder => SortField.FIELD_DOC
-      case lightdb.query.Sort.ByField(field, reverse) =>
-        field.
-        new SortField(field.name, sortType, reverse)
     }
     // TODO: Offset
-    val topDocs = indexSearcher.search(q, query.limit, sort, query.scoreDocs)
+    val topDocs = indexSearcher.search(q, query.limit, new Sort(sortFields: _*), query.scoreDocs)
     val hits = topDocs.scoreDocs
     val total = topDocs.totalHits.value
     val storedFields = indexSearcher.storedFields()