Skip to content

Commit 3f004eb

Browse files
jberkeljberkel
authored
Merge pull request #1068 from stephencelis/trigram
Add trigram tokenizer
2 parents d25e5bc + 7f98a7b commit 3f004eb

File tree

12 files changed

+142
-40
lines changed

12 files changed

+142
-40
lines changed

‎Package.swift

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ package.dependencies = [.package(url: "https://github.com/stephencelis/CSQLite.g
4646
package.targets = [
4747
.target(name: "SQLite", exclude: ["Extensions/FTS4.swift", "Extensions/FTS5.swift"]),
4848
.testTarget(name: "SQLiteTests", dependencies: ["SQLite"], path: "Tests/SQLiteTests", exclude: [
49+
"FTSIntegrationTests.swift",
4950
"FTS4Tests.swift",
5051
"FTS5Tests.swift"
5152
])

‎SQLite.xcodeproj/project.pbxproj

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
19A17152E32A9585831E3FE0 /* DateAndTimeFunctions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A17BA55DABB480F9020C8A /* DateAndTimeFunctions.swift */; };
5555
19A1717B10CC941ACB5533D6 /* FTS5.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A1730E4390C775C25677D1 /* FTS5.swift */; };
5656
19A171967CC511C4F6F773C9 /* RowTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A175C1F9CB3BBAB8FCEC7B /* RowTests.swift */; };
57+
19A171BE056457F13BFBC4C3 /* FTSIntegrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A171FDE4D67879B14ACBDF /* FTSIntegrationTests.swift */; };
5758
19A171E6FA242F72A308C594 /* FTS5Tests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A1721B8984686B9963B45D /* FTS5Tests.swift */; };
5859
19A171F12AB8B07F2FD7201A /* Cipher.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A178A39ACA9667A62663CC /* Cipher.swift */; };
5960
19A1720B67ED13E6150C6A3D /* RowTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A175C1F9CB3BBAB8FCEC7B /* RowTests.swift */; };
@@ -80,8 +81,10 @@
8081
19A179B59450FE7C4811AB8A /* Connection+Aggregation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A175A9CB446640AE6F2200 /* Connection+Aggregation.swift */; };
8182
19A179CCF9671E345E5A9811 /* Cipher.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A178A39ACA9667A62663CC /* Cipher.swift */; };
8283
19A179E76EA6207669B60C1B /* Cipher.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A178A39ACA9667A62663CC /* Cipher.swift */; };
84+
19A17C013B00682B70D53DB8 /* FTSIntegrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A171FDE4D67879B14ACBDF /* FTSIntegrationTests.swift */; };
8385
19A17C4B951CB054EE48AB1C /* CipherTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A17399EA9E61235D5D77BF /* CipherTests.swift */; };
8486
19A17C80076860CF7751A056 /* DateAndTimeFunctionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A1729B75C33F9A0B9A89C1 /* DateAndTimeFunctionTests.swift */; };
87+
19A17C9407AC0EE104E5CC85 /* FTSIntegrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A171FDE4D67879B14ACBDF /* FTSIntegrationTests.swift */; };
8588
19A17CB808ACF606726E77A8 /* QueryIntegrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A17BA6B4E282C1315A115C /* QueryIntegrationTests.swift */; };
8689
19A17DC282E36C4F41AA440B /* Errors.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A1710E73A46D5AC721CDA9 /* Errors.swift */; };
8790
19A17DF8D4F13A20F5D2269E /* Result.swift in Sources */ = {isa = PBXBuildFile; fileRef = 19A17E723300E5ED3771DCB5 /* Result.swift */; };
@@ -236,6 +239,7 @@
236239
03A65E631C6BB0F60062603F /* SQLiteTests tvOS.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = "SQLiteTests tvOS.xctest"; sourceTree = BUILT_PRODUCTS_DIR; };
237240
03A65E961C6BB3210062603F /* libsqlite3.tbd */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.text-based-dylib-definition"; name = libsqlite3.tbd; path = Platforms/AppleTVOS.platform/Developer/SDKs/AppleTVOS.sdk/usr/lib/libsqlite3.tbd; sourceTree = DEVELOPER_DIR; };
238241
19A1710E73A46D5AC721CDA9 /* Errors.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Errors.swift; sourceTree = "<group>"; };
242+
19A171FDE4D67879B14ACBDF /* FTSIntegrationTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FTSIntegrationTests.swift; sourceTree = "<group>"; };
239243
19A1721B8984686B9963B45D /* FTS5Tests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FTS5Tests.swift; sourceTree = "<group>"; };
240244
19A1729B75C33F9A0B9A89C1 /* DateAndTimeFunctionTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DateAndTimeFunctionTests.swift; sourceTree = "<group>"; };
241245
19A1730E4390C775C25677D1 /* FTS5.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FTS5.swift; sourceTree = "<group>"; };
@@ -449,6 +453,7 @@
449453
19A1729B75C33F9A0B9A89C1 /* DateAndTimeFunctionTests.swift */,
450454
D4DB368A20C09C9B00D5A58E /* SelectTests.swift */,
451455
19A17BA6B4E282C1315A115C /* QueryIntegrationTests.swift */,
456+
19A171FDE4D67879B14ACBDF /* FTSIntegrationTests.swift */,
452457
);
453458
name = SQLiteTests;
454459
path = Tests/SQLiteTests;
@@ -887,6 +892,7 @@
887892
19A1769C1F3A7542BECF50FF /* DateAndTimeFunctionTests.swift in Sources */,
888893
D4DB368E20C09CFD00D5A58E /* SelectTests.swift in Sources */,
889894
19A17CB808ACF606726E77A8 /* QueryIntegrationTests.swift in Sources */,
895+
19A17C9407AC0EE104E5CC85 /* FTSIntegrationTests.swift in Sources */,
890896
);
891897
runOnlyForDeploymentPostprocessing = 0;
892898
};
@@ -983,6 +989,7 @@
983989
19A17C80076860CF7751A056 /* DateAndTimeFunctionTests.swift in Sources */,
984990
D4DB368C20C09CFB00D5A58E /* SelectTests.swift in Sources */,
985991
19A172A9B536D02D4A98AAAD /* QueryIntegrationTests.swift in Sources */,
992+
19A17C013B00682B70D53DB8 /* FTSIntegrationTests.swift in Sources */,
986993
);
987994
runOnlyForDeploymentPostprocessing = 0;
988995
};
@@ -1047,6 +1054,7 @@
10471054
19A172EB202970561E5C4245 /* DateAndTimeFunctionTests.swift in Sources */,
10481055
D4DB368D20C09CFC00D5A58E /* SelectTests.swift in Sources */,
10491056
19A176406BDE9D9C80CC9FA3 /* QueryIntegrationTests.swift in Sources */,
1057+
19A171BE056457F13BFBC4C3 /* FTSIntegrationTests.swift in Sources */,
10501058
);
10511059
runOnlyForDeploymentPostprocessing = 0;
10521060
};

‎Sources/SQLite/Extensions/FTS4.swift

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,10 +95,10 @@ extension VirtualTable {
9595
public struct Tokenizer {
9696

9797
public static let Simple = Tokenizer("simple")
98-
9998
public static let Porter = Tokenizer("porter")
10099

101-
public static func Unicode61(removeDiacritics: Bool? = nil, tokenchars: Set<Character> = [],
100+
public static func Unicode61(removeDiacritics: Bool? = nil,
101+
tokenchars: Set<Character> = [],
102102
separators: Set<Character> = []) -> Tokenizer {
103103
var arguments = [String]()
104104

@@ -119,6 +119,11 @@ public struct Tokenizer {
119119
return Tokenizer("unicode61", arguments)
120120
}
121121

122+
// https://sqlite.org/fts5.html#the_experimental_trigram_tokenizer
123+
public static func Trigram(caseSensitive: Bool = false) -> Tokenizer {
124+
return Tokenizer("trigram", ["case_sensitive", caseSensitive ? "1" : "0"])
125+
}
126+
122127
public static func Custom(_ name: String) -> Tokenizer {
123128
Tokenizer(Tokenizer.moduleName.quote(), [name.quote()])
124129
}

‎Tests/SQLiteTests/CipherTests.swift

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,22 @@ class CipherTests: XCTestCase {
88
let db1 = try! Connection()
99
let db2 = try! Connection()
1010

11-
override func setUp() {
11+
override func setUpWithError() throws {
1212
// db
1313

14-
try! db1.key("hello")
14+
try db1.key("hello")
1515

16-
try! db1.run("CREATE TABLE foo (bar TEXT)")
17-
try! db1.run("INSERT INTO foo (bar) VALUES ('world')")
16+
try db1.run("CREATE TABLE foo (bar TEXT)")
17+
try db1.run("INSERT INTO foo (bar) VALUES ('world')")
1818

1919
// db2
2020
let key2 = keyData()
21-
try! db2.key(Blob(bytes: key2.bytes, length: key2.length))
21+
try db2.key(Blob(bytes: key2.bytes, length: key2.length))
2222

23-
try! db2.run("CREATE TABLE foo (bar TEXT)")
24-
try! db2.run("INSERT INTO foo (bar) VALUES ('world')")
23+
try db2.run("CREATE TABLE foo (bar TEXT)")
24+
try db2.run("INSERT INTO foo (bar) VALUES ('world')")
2525

26-
super.setUp()
26+
try super.setUpWithError()
2727
}
2828

2929
func test_key() {

‎Tests/SQLiteTests/ConnectionTests.swift

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,9 @@ import SQLite3
1515

1616
class ConnectionTests: SQLiteTestCase {
1717

18-
override func setUp() {
19-
super.setUp()
20-
21-
createUsersTable()
18+
override func setUpWithError() throws {
19+
try super.setUpWithError()
20+
try createUsersTable()
2221
}
2322

2423
func test_init_withInMemory_returnsInMemoryConnection() {

‎Tests/SQLiteTests/CustomAggregationTests.swift

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,12 @@ import SQLite3
1717
#if !os(Linux)
1818

1919
class CustomAggregationTests: SQLiteTestCase {
20-
override func setUp() {
21-
super.setUp()
22-
createUsersTable()
23-
try! insertUser("Alice", age: 30, admin: true)
24-
try! insertUser("Bob", age: 25, admin: true)
25-
try! insertUser("Eve", age: 28, admin: false)
20+
override func setUpWithError() throws {
21+
try super.setUpWithError()
22+
try createUsersTable()
23+
try insertUser("Alice", age: 30, admin: true)
24+
try insertUser("Bob", age: 25, admin: true)
25+
try insertUser("Eve", age: 28, admin: false)
2626
}
2727

2828
func testUnsafeCustomSum() {

‎Tests/SQLiteTests/FTS5Tests.swift

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,16 @@ class FTS5Tests: XCTestCase {
8181
sql(config.tokenizer(.Unicode61(removeDiacritics: true, tokenchars: ["."], separators: ["X"]))))
8282
}
8383

84+
func test_tokenizer_trigram() {
85+
XCTAssertEqual(
86+
"CREATE VIRTUAL TABLE \"virtual_table\" USING fts5(tokenize=trigram case_sensitive 0)",
87+
sql(config.tokenizer(.Trigram())))
88+
89+
XCTAssertEqual(
90+
"CREATE VIRTUAL TABLE \"virtual_table\" USING fts5(tokenize=trigram case_sensitive 1)",
91+
sql(config.tokenizer(.Trigram(caseSensitive: true))))
92+
}
93+
8494
func test_column_size() {
8595
XCTAssertEqual(
8696
"CREATE VIRTUAL TABLE \"virtual_table\" USING fts5(columnsize=1)",

‎Tests/SQLiteTests/FTSIntegrationTests.swift

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import XCTest
2+
#if SQLITE_SWIFT_STANDALONE
3+
import sqlite3
4+
#elseif SQLITE_SWIFT_SQLCIPHER
5+
import SQLCipher
6+
#elseif os(Linux)
7+
import CSQLite
8+
#else
9+
import SQLite3
10+
#endif
11+
@testable import SQLite
12+
13+
class FTSIntegrationTests: SQLiteTestCase {
14+
let email = Expression<String>("email")
15+
let index = VirtualTable("index")
16+
17+
private func createIndex() throws {
18+
try createOrSkip { db in
19+
try db.run(index.create(.FTS5(
20+
FTS5Config()
21+
.column(email)
22+
.tokenizer(.Unicode61()))
23+
))
24+
}
25+
26+
for user in try db.prepare(users) {
27+
try db.run(index.insert(email <- user[email]))
28+
}
29+
}
30+
31+
private func createTrigramIndex() throws {
32+
try createOrSkip { db in
33+
try db.run(index.create(.FTS5(
34+
FTS5Config()
35+
.column(email)
36+
.tokenizer(.Trigram(caseSensitive: false)))
37+
))
38+
}
39+
40+
for user in try db.prepare(users) {
41+
try db.run(index.insert(email <- user[email]))
42+
}
43+
}
44+
45+
override func setUpWithError() throws {
46+
try super.setUpWithError()
47+
try createUsersTable()
48+
try insertUsers("John", "Paul", "George", "Ringo")
49+
}
50+
51+
func testMatch() throws {
52+
try createIndex()
53+
let matches = Array(try db.prepare(index.match("Paul")))
54+
XCTAssertEqual(matches.map { $0[email ]}, ["Paul@example.com"])
55+
}
56+
57+
func testMatchPartial() throws {
58+
try insertUsers("Paula")
59+
try createIndex()
60+
let matches = Array(try db.prepare(index.match("Pa*")))
61+
XCTAssertEqual(matches.map { $0[email ]}, ["Paul@example.com", "Paula@example.com"])
62+
}
63+
64+
func testTrigramIndex() throws {
65+
try createTrigramIndex()
66+
let matches = Array(try db.prepare(index.match("Paul")))
67+
XCTAssertEqual(1, matches.count)
68+
}
69+
70+
private func createOrSkip(_ createIndex: (Connection) throws -> Void) throws {
71+
do {
72+
try createIndex(db)
73+
} catch let error as Result {
74+
try XCTSkipIf(error.description.starts(with: "no such module:") ||
75+
error.description.starts(with: "parse error")
76+
)
77+
throw error
78+
}
79+
}
80+
}

‎Tests/SQLiteTests/QueryIntegrationTests.swift

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,9 @@ class QueryIntegrationTests: SQLiteTestCase {
1616
let email = Expression<String>("email")
1717
let age = Expression<Int>("age")
1818

19-
override func setUp() {
20-
super.setUp()
21-
22-
createUsersTable()
19+
override func setUpWithError() throws {
20+
try super.setUpWithError()
21+
try createUsersTable()
2322
}
2423

2524
// MARK: -
@@ -131,7 +130,7 @@ class QueryIntegrationTests: SQLiteTestCase {
131130
}
132131

133132
func test_upsert() throws {
134-
guard db.satisfiesMinimumVersion(minor: 24) else { return }
133+
try XCTSkipUnless(db.satisfiesMinimumVersion(minor: 24))
135134
let fetchAge = { () throws -> Int? in
136135
try self.db.pluck(self.users.filter(self.email == "alice@example.com")).flatMap { $0[self.age] }
137136
}
@@ -210,7 +209,7 @@ class QueryIntegrationTests: SQLiteTestCase {
210209
}
211210
}
212211

213-
private extension Connection {
212+
extension Connection {
214213
func satisfiesMinimumVersion(minor: Int, patch: Int = 0) -> Bool {
215214
guard let version = try? scalar("SELECT sqlite_version()") as? String else { return false }
216215
let components = version.split(separator: ".", maxSplits: 3).compactMap { Int($0) }

‎Tests/SQLiteTests/SelectTests.swift

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,14 @@ import XCTest
33

44
class SelectTests: SQLiteTestCase {
55

6-
override func setUp() {
7-
super.setUp()
8-
createUsersTable()
9-
createUsersDataTable()
6+
override func setUpWithError() throws {
7+
try super.setUpWithError()
8+
try createUsersTable()
9+
try createUsersDataTable()
1010
}
1111

12-
func createUsersDataTable() {
13-
try! db.execute("""
12+
func createUsersDataTable() throws {
13+
try db.execute("""
1414
CREATE TABLE users_name (
1515
id INTEGER,
1616
user_id INTEGER REFERENCES users(id),

‎Tests/SQLiteTests/StatementTests.swift

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ import XCTest
22
import SQLite
33

44
class StatementTests: SQLiteTestCase {
5-
override func setUp() {
6-
super.setUp()
7-
createUsersTable()
5+
override func setUpWithError() throws {
6+
try super.setUpWithError()
7+
try createUsersTable()
88
}
99

1010
func test_cursor_to_blob() {

‎Tests/SQLiteTests/TestHelpers.swift

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@ class SQLiteTestCase: XCTestCase {
66
var db: Connection!
77
let users = Table("users")
88

9-
override func setUp() {
10-
super.setUp()
11-
db = try! Connection()
9+
override func setUpWithError() throws {
10+
try super.setUpWithError()
11+
db = try Connection()
1212
trace = [String: Int]()
1313

1414
db.trace { SQL in
@@ -17,8 +17,8 @@ class SQLiteTestCase: XCTestCase {
1717
}
1818
}
1919

20-
func createUsersTable() {
21-
try! db.execute("""
20+
func createUsersTable() throws {
21+
try db.execute("""
2222
CREATE TABLE users (
2323
id INTEGER PRIMARY KEY,
2424
email TEXT NOT NULL UNIQUE,

0 commit comments

Comments
 (0)