diff --git a/lucene/src/main/java/com/orientechnologies/lucene/builder/OLuceneQueryBuilder.java b/lucene/src/main/java/com/orientechnologies/lucene/builder/OLuceneQueryBuilder.java index 904c4b812af..13686479071 100644 --- a/lucene/src/main/java/com/orientechnologies/lucene/builder/OLuceneQueryBuilder.java +++ b/lucene/src/main/java/com/orientechnologies/lucene/builder/OLuceneQueryBuilder.java @@ -19,15 +19,16 @@ package com.orientechnologies.lucene.builder; import com.orientechnologies.common.log.OLogManager; +import com.orientechnologies.lucene.parser.OLuceneMultiFieldQueryParser; import com.orientechnologies.orient.core.index.OCompositeKey; import com.orientechnologies.orient.core.index.OIndexDefinition; +import com.orientechnologies.orient.core.metadata.schema.OType; import com.orientechnologies.orient.core.record.impl.ODocument; import com.orientechnologies.orient.core.sql.parser.ParseException; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; -import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.Query; +import java.util.HashMap; import java.util.Map; /** @@ -82,11 +83,7 @@ public Query query(OIndexDefinition index, Object key, Analyzer analyzer) throws } protected Query getQueryParser(OIndexDefinition index, String key, Analyzer analyzer) throws ParseException { - QueryParser queryParser; - if ((key).startsWith("(")) { - queryParser = new QueryParser("", analyzer); - } else { String[] fields; if (index.isAutomatic()) { fields = index.getFields().toArray(new String[index.getFields().size()]); @@ -99,9 +96,17 @@ protected Query getQueryParser(OIndexDefinition index, String key, Analyzer anal } } - queryParser = new MultiFieldQueryParser(fields, analyzer); + Map types = new HashMap(); + for (int i = 0; i < fields.length; i++) { + String field = fields[i]; + types.put(field, index.getTypes()[i]); + } +// for (Map.Entry typeEntry : types.entrySet()) { +// System.out.println("typeEntry = " + typeEntry); +// } + final OLuceneMultiFieldQueryParser queryParser = new OLuceneMultiFieldQueryParser(types, fields, analyzer); queryParser.setAllowLeadingWildcard(allowLeadingWildcard); queryParser.setLowercaseExpandedTerms(lowercaseExpandedTerms); diff --git a/lucene/src/main/java/com/orientechnologies/lucene/operator/OLuceneTextOperator.java b/lucene/src/main/java/com/orientechnologies/lucene/operator/OLuceneTextOperator.java index d7df515719d..6796e9fb22a 100644 --- a/lucene/src/main/java/com/orientechnologies/lucene/operator/OLuceneTextOperator.java +++ b/lucene/src/main/java/com/orientechnologies/lucene/operator/OLuceneTextOperator.java @@ -38,10 +38,8 @@ import com.orientechnologies.orient.core.sql.operator.OIndexReuseType; import com.orientechnologies.orient.core.sql.operator.OQueryTargetOperator; import com.orientechnologies.orient.core.sql.parser.ParseException; -import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.memory.MemoryIndex; -import org.apache.lucene.search.Query; import java.util.ArrayList; import java.util.Arrays; @@ -52,6 +50,8 @@ public class OLuceneTextOperator extends OQueryTargetOperator { + public static final String MEMORY_INDEX = "_memoryIndex"; + public OLuceneTextOperator() { this("LUCENE", 5, false); } @@ -60,8 +60,17 @@ public OLuceneTextOperator(String iKeyword, int iPrecedence, boolean iLogical) { super(iKeyword, iPrecedence, iLogical); } - protected static ODatabaseDocumentInternal getDatabase() { - return ODatabaseRecordThreadLocal.INSTANCE.get(); + @Override + public OIndexReuseType getIndexReuseType(Object iLeft, Object iRight) { + return OIndexReuseType.INDEX_OPERATOR; + } + + @Override + public OIndexSearchResult getOIndexSearchResult(OClass iSchemaClass, OSQLFilterCondition iCondition, + List iIndexSearchResults, OCommandContext context) { + + //FIXME questo non trova l'indice se l'ordine e' errato + return OLuceneOperatorUtil.buildOIndexSearchResult(iSchemaClass, iCondition, iIndexSearchResults, context); } @Override @@ -75,11 +84,6 @@ public OIndexCursor executeIndexQuery(OCommandContext iContext, OIndex index, return cursor; } - @Override - public OIndexReuseType getIndexReuseType(Object iLeft, Object iRight) { - return OIndexReuseType.INDEX_OPERATOR; - } - @Override public ORID getBeginRidRange(Object iLeft, Object iRight) { return null; @@ -91,11 +95,8 @@ public ORID getEndRidRange(Object iLeft, Object iRight) { } @Override - public OIndexSearchResult getOIndexSearchResult(OClass iSchemaClass, OSQLFilterCondition iCondition, - List iIndexSearchResults, OCommandContext context) { - - //FIXME questo non trova l'indice se l'ordine e' errato - return OLuceneOperatorUtil.buildOIndexSearchResult(iSchemaClass, iCondition, iIndexSearchResults, context); + public boolean canBeMerged() { + return false; } @Override @@ -114,25 +115,24 @@ public Object evaluateRecord(OIdentifiable iRecord, ODocument iCurrentResult, OS throw new OCommandExecutionException("Cannot evaluate lucene condition without index configuration."); } - MemoryIndex memoryIndex = (MemoryIndex) iContext.getVariable("_memoryIndex"); + MemoryIndex memoryIndex = (MemoryIndex) iContext.getVariable(MEMORY_INDEX); if (memoryIndex == null) { memoryIndex = new MemoryIndex(); - iContext.setVariable("_memoryIndex", memoryIndex); + iContext.setVariable(MEMORY_INDEX, memoryIndex); } memoryIndex.reset(); - final Document doc = index.buildDocument(iLeft); - for (IndexableField field : doc.getFields()) { - memoryIndex.addField(field.name(), field.stringValue(), index.indexAnalyzer()); - } - - Query query = null; try { - query = index.buildQuery(iRight); + for (IndexableField field : index.buildDocument(iLeft).getFields()) { + memoryIndex.addField(field.name(), field.tokenStream(index.indexAnalyzer(), null)); + } + return memoryIndex.search(index.buildQuery(iRight)) > 0.0f; + } catch (ParseException e) { OLogManager.instance().error(this, "error occurred while building query", e); + } - return memoryIndex.search(query) > 0.0f; + return null; } protected OLuceneFullTextIndex involvedIndex(OIdentifiable iRecord, ODocument iCurrentResult, OSQLFilterCondition iCondition, @@ -166,6 +166,10 @@ protected OLuceneFullTextIndex involvedIndex(OIdentifiable iRecord, ODocument iC return idx; } + protected static ODatabaseDocumentInternal getDatabase() { + return ODatabaseRecordThreadLocal.INSTANCE.get(); + } + private boolean isChained(Object left) { if (left instanceof OSQLFilterItemField) { OSQLFilterItemField field = (OSQLFilterItemField) left; @@ -204,9 +208,4 @@ protected Collection fields(OSQLFilterCondition iCondition) { } return Collections.emptyList(); } - - @Override - public boolean canBeMerged() { - return false; - } } diff --git a/lucene/src/main/java/com/orientechnologies/lucene/parser/OLuceneMultiFieldQueryParser.java b/lucene/src/main/java/com/orientechnologies/lucene/parser/OLuceneMultiFieldQueryParser.java new file mode 100644 index 00000000000..2722b743c52 --- /dev/null +++ b/lucene/src/main/java/com/orientechnologies/lucene/parser/OLuceneMultiFieldQueryParser.java @@ -0,0 +1,88 @@ +package com.orientechnologies.lucene.parser; + +import com.orientechnologies.orient.core.metadata.schema.OType; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.DateTools; +import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.search.LegacyNumericRangeQuery; +import org.apache.lucene.search.Query; + +import java.util.Map; + +/** + * Created by frank on 13/12/2016. + */ +public class OLuceneMultiFieldQueryParser extends MultiFieldQueryParser { + private final Map types; + + public OLuceneMultiFieldQueryParser(Map types, String[] fields, + Analyzer analyzer, + Map boosts) { + super(fields, analyzer, boosts); + this.types = types; + } + + public OLuceneMultiFieldQueryParser(Map types, String[] fields, Analyzer analyzer) { + super(fields, analyzer); + this.types = types; + } + + @Override + protected Query getFieldQuery(String field, String queryText, int slop) throws ParseException { + Query query = getQuery(field, queryText, queryText, true, true); + if (query != null) + return query; + + return super.getFieldQuery(field, queryText, slop); + } + + @Override + protected Query getFieldQuery(String field, String queryText, boolean quoted) throws ParseException { + + Query query = getQuery(field, queryText, queryText, true, true); + if (query != null) + return query; + + return super.getFieldQuery(field, queryText, quoted); + } + + @Override + protected Query getRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) + throws ParseException { + + Query query = getQuery(field, part1, part2, startInclusive, endInclusive); + if (query != null) + return query; + + return super.getRangeQuery(field, part1, part2, startInclusive, endInclusive); + } + + private Query getQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) + throws ParseException { + + if (types.containsKey(field)) { + switch (types.get(field)) { + case LONG: + return LegacyNumericRangeQuery.newLongRange(field, Long.parseLong(part1), Long.parseLong(part2), + startInclusive, endInclusive); + case INTEGER: + return LegacyNumericRangeQuery.newIntRange(field, Integer.parseInt(part1), Integer.parseInt(part2), + startInclusive, endInclusive); + case DOUBLE: + return LegacyNumericRangeQuery.newDoubleRange(field, Double.parseDouble(part1), Double.parseDouble(part2), + startInclusive, endInclusive); + case DATE: + case DATETIME: + try { + return LegacyNumericRangeQuery.newLongRange(field, DateTools.stringToTime(part1), DateTools.stringToTime(part2), + startInclusive, endInclusive); + } catch (java.text.ParseException e) { + throw new ParseException(e.getMessage()); + } + } + } + return null; + } + +} diff --git a/lucene/src/test/java/com/orientechnologies/lucene/test/LuceneRangeTest.java b/lucene/src/test/java/com/orientechnologies/lucene/test/LuceneRangeTest.java new file mode 100644 index 00000000000..269a7fb7369 --- /dev/null +++ b/lucene/src/test/java/com/orientechnologies/lucene/test/LuceneRangeTest.java @@ -0,0 +1,125 @@ +package com.orientechnologies.lucene.test; + +import com.orientechnologies.orient.core.metadata.schema.OClass; +import com.orientechnologies.orient.core.metadata.schema.OSchema; +import com.orientechnologies.orient.core.metadata.schema.OType; +import com.orientechnologies.orient.core.record.impl.ODocument; +import com.orientechnologies.orient.core.sql.OCommandSQL; +import org.apache.lucene.document.DateTools; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.assertj.core.api.Assertions.*; + +/** + * Created by frank on 13/12/2016. + */ +public class LuceneRangeTest extends BaseLuceneTest { + + @Before + public void setUp() throws Exception { + OSchema schema = db.getMetadata().getSchema(); + + OClass cls = schema.createClass("Person"); + cls.createProperty("name", OType.STRING); + cls.createProperty("surname", OType.STRING); + cls.createProperty("date", OType.DATETIME); + cls.createProperty("age", OType.INTEGER); + + List names = Arrays.asList("John", "Robert", "Jane", "andrew", "Scott", "luke", "Enriquez", "Luis", "Gabriel", "Sara"); + for (int i = 0; i < 10; i++) { + db.save(new ODocument("Person") + .field("name", names.get(i)) + .field("surname", "Reese") + //from today back one day a time + .field("date", System.currentTimeMillis() - (i * 3600 * 24 * 1000)) + .field("age", i)); + } + + } + + @Test + public void shouldUseRangeQueryOnSingleIntegerField() throws Exception { + + db.command(new OCommandSQL("create index Person.age on Person(age) FULLTEXT ENGINE LUCENE")).execute(); + + assertThat(db.getMetadata().getIndexManager().getIndex("Person.age").getSize()).isEqualTo(10); + + //range + Collection results = db.command( + new OCommandSQL("SELECT FROM Person WHERE age LUCENE 'age:[5 TO 6]'")) + .execute(); + + assertThat(results).hasSize(2); + + //single value + results = db.command( + new OCommandSQL("SELECT FROM Person WHERE age LUCENE 'age:5'")) + .execute(); + + assertThat(results).hasSize(1); + } + + @Test + public void shouldUseRangeQueryOnSingleDateField() throws Exception { + + db.commit(); + db.command(new OCommandSQL("create index Person.date on Person(date) FULLTEXT ENGINE LUCENE")).execute(); + db.commit(); + + assertThat(db.getMetadata().getIndexManager().getIndex("Person.date").getSize()).isEqualTo(10); + + String today = DateTools.timeToString(System.currentTimeMillis(), DateTools.Resolution.MINUTE); + String fiveDaysAgo = DateTools.timeToString(System.currentTimeMillis() - (5 * 3600 * 24 * 1000), DateTools.Resolution.MINUTE); + + //range + Collection results = db.command( + new OCommandSQL("SELECT FROM Person WHERE date LUCENE 'date:[" + fiveDaysAgo + " TO " + today + "]'")) + .execute(); + + assertThat(results).hasSize(5); + + } + + @Test + public void shouldUseRangeQueryMultipleField() throws Exception { + + db.command(new OCommandSQL("create index Person.composite on Person(name,surname,date,age) FULLTEXT ENGINE LUCENE")).execute(); + + assertThat(db.getMetadata().getIndexManager().getIndex("Person.composite").getSize()).isEqualTo(10); + + db.commit(); + + String today = DateTools.timeToString(System.currentTimeMillis(), DateTools.Resolution.MINUTE); + String fiveDaysAgo = DateTools.timeToString(System.currentTimeMillis() - (5 * 3600 * 24 * 1000), DateTools.Resolution.MINUTE); + + //anme and age range + Collection results = db.command( + new OCommandSQL("SELECT * FROM Person WHERE [name,surname,date,age] LUCENE 'name:luke age:[5 TO 6]'")) + .execute(); + + assertThat(results).hasSize(2); + + //date range + results = db.command( + new OCommandSQL("SELECT FROM Person WHERE [name,surname,date,age] LUCENE 'date:[" + fiveDaysAgo + " TO " + today + "]'")) + .execute(); + + assertThat(results).hasSize(5); + + //age and date range with MUST + results = db.command( + new OCommandSQL( + "SELECT FROM Person WHERE [name,surname,date,age] LUCENE '+age:[4 TO 7] +date:[" + fiveDaysAgo + " TO " + today + + "]'")) + .execute(); + + assertThat(results).hasSize(2); + + } + +} diff --git a/lucene/src/test/java/com/orientechnologies/lucene/LuceneReuseTest.java b/lucene/src/test/java/com/orientechnologies/lucene/test/LuceneReuseTest.java similarity index 85% rename from lucene/src/test/java/com/orientechnologies/lucene/LuceneReuseTest.java rename to lucene/src/test/java/com/orientechnologies/lucene/test/LuceneReuseTest.java index 79d40be7a3b..447036acaa3 100644 --- a/lucene/src/test/java/com/orientechnologies/lucene/LuceneReuseTest.java +++ b/lucene/src/test/java/com/orientechnologies/lucene/test/LuceneReuseTest.java @@ -1,4 +1,4 @@ -package com.orientechnologies.lucene; +package com.orientechnologies.lucene.test; import com.orientechnologies.lucene.test.BaseLuceneTest; import com.orientechnologies.orient.core.metadata.schema.OClass; @@ -28,7 +28,7 @@ public void shouldUseTheRightIndex() { cls.createProperty("surname", OType.STRING); cls.createProperty("age", OType.LONG); - db.command(new OCommandSQL("create index Reuse.comp on Reuse (name,surname,date,age) UNIQUE")).execute(); + db.command(new OCommandSQL("create index Reuse.composite on Reuse (name,surname,date,age) UNIQUE")).execute(); db.command(new OCommandSQL("create index Reuse.surname on Reuse (surname) FULLTEXT ENGINE LUCENE")).execute(); for (int i = 0; i < 10; i++) { @@ -56,12 +56,16 @@ public void shouldUseTheRightLuceneIndex() { cls.createProperty("surname", OType.STRING); cls.createProperty("age", OType.LONG); - db.command(new OCommandSQL("create index Reuse.comp on Reuse (name,surname,date,age) UNIQUE")).execute(); - db.command(new OCommandSQL("create index Reuse.n_surname on Reuse (name,surname) FULLTEXT ENGINE LUCENE")).execute(); + db.command(new OCommandSQL("create index Reuse.composite on Reuse (name,surname,date,age) UNIQUE")).execute(); + + //lucene on name and surname + db.command(new OCommandSQL("create index Reuse.name_surname on Reuse (name,surname) FULLTEXT ENGINE LUCENE")).execute(); for (int i = 0; i < 10; i++) { db.save(new ODocument("Reuse").field("name", "John").field("date", new Date()).field("surname", "Reese").field("age", i)); } + + //additional record db.save(new ODocument("Reuse").field("name", "John").field("date", new Date()).field("surname", "Franklin").field("age", 11)); Collection results = db .command(new OCommandSQL("SELECT FROM Reuse WHERE name='John' and [name,surname] LUCENE 'Reese'")).execute(); diff --git a/lucene/src/test/java/com/orientechnologies/lucene/test/LuceneVsLuceneTest.java b/lucene/src/test/java/com/orientechnologies/lucene/test/LuceneVsLuceneTest.java new file mode 100644 index 00000000000..ed2bd8c831f --- /dev/null +++ b/lucene/src/test/java/com/orientechnologies/lucene/test/LuceneVsLuceneTest.java @@ -0,0 +1,143 @@ +/* + * + * * Copyright 2014 Orient Technologies. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.orientechnologies.lucene.test; + +import com.orientechnologies.common.io.OFileUtils; +import com.orientechnologies.lucene.analyzer.OLucenePerFieldAnalyzerWrapper; +import com.orientechnologies.orient.core.command.script.OCommandScript; +import com.orientechnologies.orient.core.metadata.schema.OSchema; +import com.orientechnologies.orient.core.record.impl.ODocument; +import com.orientechnologies.orient.core.sql.OCommandSQL; +import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.NIOFSDirectory; +import org.assertj.core.api.Assertions; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Created by enricorisa on 08/10/14. + */ +public class LuceneVsLuceneTest extends BaseLuceneTest { + + private IndexWriter indexWriter; + private OLucenePerFieldAnalyzerWrapper analyzer; + + @Before + public void init() { + InputStream stream = ClassLoader.getSystemResourceAsStream("testLuceneIndex.sql"); + + db.command(new OCommandScript("sql", getScriptFromStream(stream))).execute(); + + OSchema schema = db.getMetadata().getSchema(); + + OFileUtils.deleteRecursively(getPath().getAbsoluteFile()); + try { + Directory dir = getDirectory(); + analyzer = new OLucenePerFieldAnalyzerWrapper(new StandardAnalyzer()); + + analyzer.add("title", new StandardAnalyzer()) + .add("Song.title", new StandardAnalyzer()); + + IndexWriterConfig iwc = new IndexWriterConfig(analyzer); + iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + indexWriter = new IndexWriter(dir, iwc); + + } catch (IOException e) { + e.printStackTrace(); + } + db.command(new OCommandSQL("create index Song.title on Song (title) FULLTEXT ENGINE LUCENE")).execute(); + + } + + private File getPath() { + return new File("./target/databases/" + name.getMethodName()); + } + + protected Directory getDirectory() throws IOException { + return NIOFSDirectory.open(getPath().toPath()); + } + + @Test + public void testLuceneVsLucene() throws IOException, ParseException { + + for (ODocument oDocument : db.browseClass("Song")) { + + String title = oDocument.field("title"); + if (title != null) { + Document d = new Document(); + + d.add(new TextField("title", title, Field.Store.YES)); + d.add(new TextField("Song.title", title, Field.Store.YES)); + indexWriter.addDocument(d); + + } + } + + indexWriter.commit(); + indexWriter.close(); + + IndexReader reader = DirectoryReader.open(getDirectory()); + assertThat(reader.numDocs()).isEqualTo(Long.valueOf(db.countClass("Song")).intValue()); + + IndexSearcher searcher = new IndexSearcher(reader); + + Query query = new MultiFieldQueryParser(new String[] { "title" }, analyzer).parse("down the"); + final TopDocs docs = searcher.search(query, Integer.MAX_VALUE); + ScoreDoc[] hits = docs.scoreDocs; + + List oDocs = db + .query(new OSQLSynchQuery("select *,$score from Song where title LUCENE \"down the\"")); + + Assert.assertEquals(oDocs.size(), hits.length); + + int i = 0; + for (ScoreDoc hit : hits) { +// Assert.assertEquals(oDocs.get(i).field("$score"), hit.score); + + assertThat(oDocs.get(i).field("$score")).isEqualTo(hit.score); + i++; + } + reader.close(); + + } + +} diff --git a/lucene/src/test/resources/testLuceneIndex.sql b/lucene/src/test/resources/testLuceneIndex.sql index ad3ec1e9c3d..52f1fd98f08 100644 --- a/lucene/src/test/resources/testLuceneIndex.sql +++ b/lucene/src/test/resources/testLuceneIndex.sql @@ -8,15 +8,14 @@ create property Song.description STRING create class Author extends V create property Author.name STRING +create property Author.score INTEGER begin - - -create vertex Author set name="Bob Dylan" -create vertex Author set name="Grateful Dead" -create vertex Author set name="Lennon McCartney" -create vertex Author set name="Chuck Berry" -create vertex Author set name="Jack Mountain" +create vertex Author set name="Bob Dylan", score =10 +create vertex Author set name="Grateful Dead", score =5 +create vertex Author set name="Lennon McCartney", score =7 +create vertex Author set name="Chuck Berry", score =10 +create vertex Author set name="Jack Mountain", score =4 create vertex Song set title="BELIEVE IT OR NOT", author="Hunter"