feat(streets): remove all single-char synonyms except directionals, s…

…ynonyms which seem to only be present for the purpose of spelling correction
pelias · Jul 13, 2020 · 213a66c · 213a66c
1 parent 8206fe0
commit 213a66c
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 28 deletions.
diff --git a/synonyms/custom_name.txt b/synonyms/custom_name.txt
@@ -87,9 +87,9 @@ colline,coli
 collines,colis
 enceinte,en
 fleuve,fl
-grand,gd,gr,g
+grand,gd,gr
 mont,mt,mnt
-petite,p,pt
+petite,pt
 porche,pch
 rivière,riviere,riv
 village,vge
@@ -108,7 +108,6 @@ kleines,kl
 kogel,kg
 niedere,nd
 rhein,rh
-see,s
 spitze,sp
 vordere,vd,vord
 wiese,ws

diff --git a/synonyms/linter.js b/synonyms/linter.js
@@ -41,6 +41,7 @@ function linter(synonyms) {
       letterCasing(line, logprefix, tokens);
       tokensSanityCheck(line, logprefix, tokens);
       // multiWordCheck(line, logprefix, tokens);
+      // tokenLengthCheck(line, logprefix, tokens);
     })
   })
 }
@@ -65,10 +66,18 @@ function tokensSanityCheck(line, logprefix, tokens) {
   }
 }
 
-function multiWordCheck(line, tokens) {
+function multiWordCheck(line, logprefix, tokens) {
   _.each(tokens, token => {
     if (/\s/.test(token)){
-      logger.warn(`multi word synonyms may cause issues with phrase queries:`, token);
+      logger.warn(`${logprefix} multi word synonyms may cause issues with phrase queries:`, token);
+    }
+  });
+}
+
+function tokenLengthCheck(line, logprefix, tokens) {
+  _.each(tokens, token => {
+    if (token.length <= 1) {
+      logger.warn(`${logprefix} short token:`, token);
     }
   });
 }

diff --git a/synonyms/street_synonyms_en.txt b/synonyms/street_synonyms_en.txt
@@ -6,7 +6,7 @@ alleyway, alwy, allyway, allwy
 amble, ambl
 anchorage, ancg
 annex, anx
-apartments, apts, appartments
+apartments, apts
 approach, app, apch, appr
 arcade, arc
 arterial, artl
@@ -50,7 +50,7 @@ cape, cpe, cp
 canyon, cyn, cnyn
 caravan, cvan, cvn, c van
 causeway, csway, cswy, causewy, caus, cause, cway
-center, centre, cetr, cntr, ctr, c, cen
+center, centre, cetr, cntr, ctr, cen
 centers, ctrs
 centreway, cnwy
 chase, ch, chas
@@ -106,7 +106,7 @@ divide, div
 diversion, divers
 down, dn
 downs, dns, dwns
-drive, dr, drv, dv, dve, d
+drive, dr, drv, dv, dve
 driveway, drwy, dvwy, dwy, dway, drvwy
 drove, drov
 easement, esmt
@@ -164,7 +164,7 @@ haven, hvn, havn
 head, hd
 heads, hds
 heath, hth, heth
-heights, hghts, hgts, hieghts, ht, hts, hgths
+heights, hghts, hgts, ht, hts, hgths
 highlands, hghlds, hlds, hglds
 highroad, hrd, high road, hird, hi rd
 highway, hgwy, hw, hway, hwy, hi, hwye, hywy
@@ -175,10 +175,10 @@ impasse, imp
 inlet, inlt
 interchange, intg, intchg, inter change
 intersection, intn, inter section, intsctn
-interstate, inter state, i, ih, i h
-island, is, id, i, isl, isld
+interstate, inter state, ih
+island, is, id, isl, isld
 islands, iss, ids, islds
-junction, jct, jnc, jnct, jctn, jtn, junct, j
+junction, jct, jnc, jnct, jctn, jtn, junct
 junctions, jcts
 key, ky
 keys, kys
@@ -187,7 +187,7 @@ knolls, knls
 ladder, ladr
 lagoon, lagn, lgn, lagon
 landing, ldg, lndg, landng
-lane, l, ln, la
+lane, ln, la
 laneway, lnwy
 light, lgt, lt
 limits, lmts
@@ -280,8 +280,8 @@ right of way, rowy, rightofway, rofw, row, r o w, r of w
 rise, ri
 riverway, rvwy
 riviera, rvra
-road, rd, ro, r, roa, raod
-roads, raods, rds
+road, rd, ro, roa
+roads, rds
 roadside, rdsd, road side
 roadway, rdwy, rdw, rdy
 rocks, rks

diff --git a/test/fixtures/expected.json b/test/fixtures/expected.json
@@ -273,9 +273,9 @@
             "collines,colis",
             "enceinte,en",
             "fleuve,fl",
-            "grand,gd,gr,g",
+            "grand,gd,gr",
             "mont,mt,mnt",
-            "petite,p,pt",
+            "petite,pt",
             "porche,pch",
             "rivière,riviere,riv",
             "village,vge",
@@ -292,7 +292,6 @@
             "kogel,kg",
             "niedere,nd",
             "rhein,rh",
-            "see,s",
             "spitze,sp",
             "vordere,vd,vord",
             "wiese,ws",
@@ -398,7 +397,7 @@
             "amble,ambl",
             "anchorage,ancg",
             "annex,anx",
-            "apartments,apts,appartments",
+            "apartments,apts",
             "approach,app,apch,appr",
             "arcade,arc",
             "arterial,artl",
@@ -442,7 +441,7 @@
             "canyon,cyn,cnyn",
             "caravan,cvan,cvn,c van",
             "causeway,csway,cswy,causewy,caus,cause,cway",
-            "center,centre,cetr,cntr,ctr,c,cen",
+            "center,centre,cetr,cntr,ctr,cen",
             "centers,ctrs",
             "centreway,cnwy",
             "chase,ch,chas",
@@ -498,7 +497,7 @@
             "diversion,divers",
             "down,dn",
             "downs,dns,dwns",
-            "drive,dr,drv,dv,dve,d",
+            "drive,dr,drv,dv,dve",
             "driveway,drwy,dvwy,dwy,dway,drvwy",
             "drove,drov",
             "easement,esmt",
@@ -556,7 +555,7 @@
             "head,hd",
             "heads,hds",
             "heath,hth,heth",
-            "heights,hghts,hgts,hieghts,ht,hts,hgths",
+            "heights,hghts,hgts,ht,hts,hgths",
             "highlands,hghlds,hlds,hglds",
             "highroad,hrd,high road,hird,hi rd",
             "highway,hgwy,hw,hway,hwy,hi,hwye,hywy",
@@ -567,10 +566,10 @@
             "inlet,inlt",
             "interchange,intg,intchg,inter change",
             "intersection,intn,inter section,intsctn",
-            "interstate,inter state,i,ih,i h",
-            "island,is,id,i,isl,isld",
+            "interstate,inter state,ih",
+            "island,is,id,isl,isld",
             "islands,iss,ids,islds",
-            "junction,jct,jnc,jnct,jctn,jtn,junct,j",
+            "junction,jct,jnc,jnct,jctn,jtn,junct",
             "junctions,jcts",
             "key,ky",
             "keys,kys",
@@ -579,7 +578,7 @@
             "ladder,ladr",
             "lagoon,lagn,lgn,lagon",
             "landing,ldg,lndg,landng",
-            "lane,l,ln,la",
+            "lane,ln,la",
             "laneway,lnwy",
             "light,lgt,lt",
             "limits,lmts",
@@ -672,8 +671,8 @@
             "rise,ri",
             "riverway,rvwy",
             "riviera,rvra",
-            "road,rd,ro,r,roa,raod",
-            "roads,raods,rds",
+            "road,rd,ro,roa",
+            "roads,rds",
             "roadside,rdsd,road side",
             "roadway,rdwy,rdw,rdy",
             "rocks,rks",