initial commit

Author: felixguendling

.gitignore

@@ -0,0 +1 @@
+/build*

CMakeLists.txt

@@ -0,0 +1,10 @@
+cmake_minimum_required(VERSION 2.6)
+project(guess)
+
+add_library(guess src/guesser.cc)
+set_target_properties(guess PROPERTIES COMPILE_FLAGS "-std=c++11")
+
+include_directories(include)
+add_executable(guess-example EXCLUDE_FROM_ALL example/main.cc)
+target_link_libraries(guess-example guess)
+set_target_properties(guess-example PROPERTIES COMPILE_FLAGS "-std=c++11")

example/main.cc

@@ -0,0 +1,50 @@
+#include <iostream>
+#include <fstream>
+#include <chrono>
+
+#include "guess/guesser.h"
+
+using namespace guess;
+
+struct new_line_tokenizer : std::ctype<char> {
+  new_line_tokenizer() : std::ctype<char>(get_table()) {}
+  static mask const* get_table() {
+    static mask rc[table_size];
+    rc['\n'] = std::ctype_base::space;
+    return &rc[0];
+  }
+};
+
+int main() {
+  std::vector<std::string> stations;
+
+  try {
+    std::fstream in("stations.txt");
+    in.exceptions(std::ifstream::failbit);
+    in.imbue(std::locale(std::cin.getloc(), new new_line_tokenizer));
+    std::copy(std::istream_iterator<std::string>(in),
+              std::istream_iterator<std::string>(),
+              std::back_inserter(stations));
+  } catch (std::exception const& e) {
+    std::cout << "unable to read file stations.txt\n";
+    return 1;
+  }
+
+  guesser g(stations);
+
+  std::string input;
+  while (std::cout << "$ " && std::getline(std::cin, input)) {
+    using std::chrono::system_clock;
+    using std::chrono::duration_cast;
+    using std::chrono::milliseconds;
+    auto start = system_clock::now();
+    auto candidates = g.guess(input);
+    auto duration = duration_cast<milliseconds>(system_clock::now() - start);
+    std::cout << "\n\t" << duration.count() << "ms\n\n";
+
+    for (auto const& guess : candidates) {
+      std::cout << "  " << stations[guess] << "\n";
+    }
+    std::cout << "\n";
+  }
+}

include/guess/guesser.h

@@ -0,0 +1,35 @@
+#ifndef GUESS_GUESSER_H_
+#define GUESS_GUESSER_H_
+
+#include <string>
+#include <vector>
+
+namespace guess {
+
+struct guesser {
+  guesser(std::vector<std::string> const& candidates);
+
+  std::vector<int> guess(std::string in, int count = 10) const;
+
+private:
+  struct match {
+    match() = default;
+    explicit match(int index) : index(index), cos_sim(0) {}
+    bool operator<(match const& o) const { return cos_sim > o.cos_sim; }
+    int index;
+    double cos_sim;
+  };
+
+  std::vector<match> match_trigrams(std::string& in) const;
+
+  void score_exact_word_matches(std::string& in,
+                                std::vector<match>& matches) const;
+
+  static void normalize(std::string& s);
+
+  std::vector<std::string> candidates_;
+};
+
+}  // namespace guess
+
+#endif  // GUESS_GUESSER_H_

include/guess/string_util.h

@@ -0,0 +1,46 @@
+#ifndef GUESS_STRING_UTIL_H_
+#define GUESS_STRING_UTIL_H_
+
+#include <string>
+
+namespace guess {
+
+template <typename Function>
+inline void for_each_token(char* s, int length, Function f) {
+  int base = 0;
+  int i = 0;
+  while (i < length) {
+    if (s[i] == ' ') {
+      char tmp = s[i];
+      s[i] = '\0';
+      bool exit = f(s + base);
+      s[i] = tmp;
+
+      if (exit) {
+        return;
+      }
+
+      base = i + 1;
+    }
+    ++i;
+  }
+  f(s + base);
+}
+
+template <typename Function>
+inline void for_each_token(std::string const& in, Function f) {
+  for_each_token(const_cast<char*>(in.c_str()), in.size(), f);
+}
+
+void replace_all(std::string& s,
+                 std::string const& from,
+                 std::string const& to) {
+  std::string::size_type pos;
+  while ((pos = s.find(from)) != std::string::npos) {
+    s.replace(pos, from.size(), to);
+  }
+}
+
+}  // namespace guess
+
+#endif  // GUESS_STRING_UTIL_H_

src/guesser.cc

@@ -0,0 +1,127 @@
+#include "guess/guesser.h"
+
+#include <cmath>
+#include <cinttypes>
+#include <fstream>
+#include <string>
+#include <vector>
+
+#include "guess/string_util.h"
+
+namespace guess {
+
+guesser::guesser(std::vector<std::string> const& candidates)
+    : candidates_(candidates) {
+  for (auto& candidate : candidates_) {
+    normalize(candidate);
+  }
+}
+
+std::vector<int> guesser::guess(std::string in, int count) const {
+  auto matches = match_trigrams(in);
+  matches.resize(std::min(100ul, matches.size()));
+
+  score_exact_word_matches(in, matches);
+  matches.resize(count);
+
+  std::vector<int> ret(matches.size());
+  for (int i = 0; i < matches.size(); ++i) {
+    ret[i] = matches[i].index;
+  }
+
+  return ret;
+}
+
+std::vector<guesser::match> guesser::match_trigrams(std::string& in) const {
+  std::vector<match> matches;
+  matches.reserve(candidates_.size());
+  for (int i = 0; i < candidates_.size(); ++i) {
+    matches.emplace_back(i);
+  }
+
+  normalize(in);
+
+  char const* input = in.c_str();
+  double sqrt_len_vec_input = std::sqrt(in.size() - 2);
+
+  char trigram_input[4] = {0};
+  char trigram_candidate[4] = {0};
+
+  for (int i = 0; i < candidates_.size(); ++i) {
+    int match_count = 0;
+    const int len_vec_candidate = candidates_[i].length() - 2;
+
+    char const* substr_input = input;
+    while (substr_input[2] != '\0') {
+      trigram_input[0] = substr_input[0];
+      trigram_input[1] = substr_input[1];
+      trigram_input[2] = substr_input[2];
+      ++substr_input;
+
+      char const* substr_candidate = candidates_[i].c_str();
+      while (substr_candidate[2] != '\0') {
+        trigram_candidate[0] = substr_candidate[0];
+        trigram_candidate[1] = substr_candidate[1];
+        trigram_candidate[2] = substr_candidate[2];
+        ++substr_candidate;
+
+        if (*(uint32_t*) trigram_input == *(uint32_t*) trigram_candidate) {
+          ++match_count;
+          break;
+        }
+      }
+    }
+
+    double denominator = sqrt_len_vec_input * std::sqrt(len_vec_candidate);
+    matches[i].cos_sim = match_count / denominator;
+  }
+  std::sort(std::begin(matches), std::end(matches));
+
+  return matches;
+}
+
+void guesser::score_exact_word_matches(std::string& in,
+                                       std::vector<match>& matches) const {
+  for (int i = 0; i < matches.size(); ++i) {
+    auto& candidate = candidates_[matches[i].index];
+    for_each_token(in, [&](char* input_token) {
+      for_each_token(candidate, [&](char* candidate_token) {
+        if (strcmp(candidate_token, input_token) == 0) {
+          matches[i].cos_sim *= 1.33;
+          return true;
+        }
+        return false;
+      });
+      return false;
+    });
+  }
+  std::sort(std::begin(matches), std::end(matches));
+}
+
+void guesser::normalize(std::string& s) {
+  replace_all(s, "Ä", "a" );
+  replace_all(s, "ä", "a" );
+  replace_all(s, "Ö", "o" );
+  replace_all(s, "ö", "o" );
+  replace_all(s, "Ü", "u" );
+  replace_all(s, "ü", "u" );
+  replace_all(s, "ß", "ss");
+  replace_all(s, "-", " " );
+  replace_all(s, "/", " " );
+  replace_all(s, ".", " " );
+  replace_all(s, ",", " " );
+  replace_all(s, "(", " " );
+  replace_all(s, ")", " " );
+
+  for (int i = 0; i  < s.length(); ++i) {
+    if (!isalnum(s[i])) {
+      s[i] = ' ';
+    }
+  }
+
+  replace_all(s, "  ", " ");
+
+  std::transform(s.begin(), s.end(), s.begin(), ::tolower);
+}
+
+}  // namespace guess