Revert using UNION instead of UNION ALL.

Syfaro · Apr 23, 2021 · f040cf3 · f040cf3
1 parent 96bc22c
commit f040cf3
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 7 deletions.
diff --git a/fuzzysearch/sqlx-data.json b/fuzzysearch/sqlx-data.json
@@ -173,8 +173,8 @@
       ]
     }
   },
-  "7953d382d35d07effb803c1b80c6efba81e3db90d0abfe3a40cd66b9106ce44d": {
-    "query": "SELECT hash_int hash FROM submission WHERE hash_int IS NOT NULL\n        UNION\n        SELECT hash FROM e621 WHERE hash IS NOT NULL\n        UNION\n        SELECT hash FROM tweet_media WHERE hash IS NOT NULL\n        UNION\n        SELECT hash FROM weasyl WHERE hash IS NOT NULL",
+  "d6a5ed9266d7cf6f08df5e0160c59843a15ef5ef3ccac2e5ccfe2251a50bd0dc": {
+    "query": "SELECT hash_int hash FROM submission WHERE hash_int IS NOT NULL\n        UNION ALL\n        SELECT hash FROM e621 WHERE hash IS NOT NULL\n        UNION ALL\n        SELECT hash FROM tweet_media WHERE hash IS NOT NULL\n        UNION ALL\n        SELECT hash FROM weasyl WHERE hash IS NOT NULL",
     "describe": {
       "columns": [
         {

diff --git a/fuzzysearch/src/main.rs b/fuzzysearch/src/main.rs
@@ -172,11 +172,11 @@ async fn create_tree(conn: &Pool) -> bk_tree::BKTree<Node, Hamming> {
 
     let mut rows = sqlx::query!(
         "SELECT hash_int hash FROM submission WHERE hash_int IS NOT NULL
-        UNION
+        UNION ALL
         SELECT hash FROM e621 WHERE hash IS NOT NULL
-        UNION
+        UNION ALL
         SELECT hash FROM tweet_media WHERE hash IS NOT NULL
-        UNION
+        UNION ALL
         SELECT hash FROM weasyl WHERE hash IS NOT NULL"
     )
     .fetch(conn);
@@ -185,9 +185,12 @@ async fn create_tree(conn: &Pool) -> bk_tree::BKTree<Node, Hamming> {
 
     while let Some(row) = rows.try_next().await.expect("Unable to get row") {
         if let Some(hash) = row.hash {
-            tree.add(Node::new(hash));
-            count += 1;
+            let node = Node::new(hash);
+            if tree.find_exact(&node).is_none() {
+                tree.add(node);
+            }
 
+            count += 1;
             if count % 250_000 == 0 {
                 tracing::debug!(count, "Made progress in loading tree rows");
             }