Skip to content

Commit 2bf9bf0

Browse files
hendrikmuhshendrikmuhs
committed
add more compilers and mergers
1 parent 4101e4e commit 2bf9bf0

File tree

4 files changed

+88
-10
lines changed

4 files changed

+88
-10
lines changed

‎python-pybind/src/compiler/py_dictionary_compilers.cpp‎

Lines changed: 67 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,21 @@
1717

1818
#include <pybind11/functional.h>
1919
#include <pybind11/pybind11.h>
20+
#include <pybind11/stl.h>
21+
22+
#include <string>
23+
#include <vector>
2024

2125
#include "keyvi/dictionary/dictionary_types.h"
2226

2327
namespace py = pybind11;
2428
namespace kd = keyvi::dictionary;
2529

26-
void init_keyvi_dictionary_compilers(const py::module_ &m) {
30+
void init_keyvi_dictionary_compilers(const py::module_ &module) {
2731
#define CREATE_COMPILER(compiler, name) \
28-
py::class_<compiler>(m, name) \
32+
py::class_<compiler>(module, name) \
2933
.def(py::init<>()) \
34+
.def(py::init<const keyvi::util::parameters_t &>()) \
3035
.def("__enter__", [](compiler &c) { return &c; }) \
3136
.def("__exit__", [](compiler &c, void *exc_type, void *exc_value, void *traceback) { c.Compile(); }) \
3237
.def("__setitem__", &compiler::Add) \
@@ -47,9 +52,67 @@ void init_keyvi_dictionary_compilers(const py::module_ &m) {
4752
}, \
4853
py::arg("progress_callback") = static_cast<std::function<void(const size_t a, const size_t b)> *>(nullptr)) \
4954
.def("set_manifest", &compiler::SetManifest) \
50-
.def("write_to_file", &compiler::WriteToFile);
51-
55+
.def("write_to_file", &compiler::WriteToFile, py::call_guard<py::gil_scoped_release>());
56+
#define CREATE_SK_COMPILER(compiler, name) \
57+
py::class_<compiler>(module, name) \
58+
.def(py::init<const std::vector<std::string> &>()) \
59+
.def(py::init<const std::vector<std::string> &, const keyvi::util::parameters_t &>()) \
60+
.def("__enter__", [](compiler &c) { return &c; }) \
61+
.def("__exit__", [](compiler &c, void *exc_type, void *exc_value, void *traceback) { c.Compile(); }) \
62+
.def("__setitem__", &compiler::Add) \
63+
.def("add", &compiler::Add) \
64+
.def( \
65+
"compile", \
66+
[](compiler &c, std::function<void(const size_t a, const size_t b)> progress_callback) { \
67+
pybind11::gil_scoped_release release_gil; \
68+
if (progress_callback == nullptr) { \
69+
c.Compile(); \
70+
return; \
71+
} \
72+
auto progress_compiler_callback = [](size_t a, size_t b, void *user_data) { \
73+
auto py_callback = *reinterpret_cast<std::function<void(const size_t, const size_t)> *>(user_data); \
74+
pybind11::gil_scoped_acquire acquire_gil; \
75+
py_callback(a, b); \
76+
}; \
77+
void *user_data = reinterpret_cast<void *>(&progress_callback); \
78+
c.Compile(progress_compiler_callback, user_data); \
79+
}, \
80+
py::arg("progress_callback") = static_cast<std::function<void(const size_t a, const size_t b)> *>(nullptr)) \
81+
.def("set_manifest", &compiler::SetManifest) \
82+
.def("write_to_file", &compiler::WriteToFile, py::call_guard<py::gil_scoped_release>());
83+
#define CREATE_MERGER(merger, name) \
84+
py::class_<merger>(module, name) \
85+
.def(py::init<>()) \
86+
.def(py::init<const keyvi::util::parameters_t &>()) \
87+
.def("__enter__", [](merger &m) { return &m; }) \
88+
.def("__exit__", [](merger &m, void *exc_type, void *exc_value, void *traceback) { m.Merge(); }) \
89+
.def("add", &merger::Add) \
90+
.def("merge", \
91+
[](merger &m) { \
92+
pybind11::gil_scoped_release release_gil; \
93+
m.Merge(); \
94+
}) \
95+
.def("merge", \
96+
[](merger &m, const std::string &filename) { \
97+
pybind11::gil_scoped_release release_gil; \
98+
m.Merge(filename); \
99+
}) \
100+
.def("set_manifest", &merger::SetManifest) \
101+
.def("write_to_file", &merger::WriteToFile, py::call_guard<py::gil_scoped_release>());
52102
CREATE_COMPILER(kd::CompletionDictionaryCompiler, "CompletionDictionaryCompiler");
103+
CREATE_COMPILER(kd::FloatVectorDictionaryCompiler, "FloatVectorDictionaryCompiler");
104+
CREATE_COMPILER(kd::IntDictionaryCompiler, "IntDictionaryCompiler");
105+
CREATE_COMPILER(kd::JsonDictionaryCompiler, "JsonDictionaryCompiler");
106+
CREATE_COMPILER(kd::KeyOnlyDictionaryCompiler, "KeyOnlyDictionaryCompiler");
107+
CREATE_COMPILER(kd::StringDictionaryCompiler, "StringDictionaryCompiler");
108+
CREATE_SK_COMPILER(kd::SecondaryKeyCompletionDictionaryCompiler, "SecondaryKeyCompletionDictionaryCompiler");
109+
CREATE_SK_COMPILER(kd::SecondaryKeyFloatVectorDictionaryCompiler, "SecondaryKeyFloatVectorDictionaryCompiler");
110+
CREATE_SK_COMPILER(kd::SecondaryKeyIntDictionaryCompiler, "SecondaryKeyIntDictionaryCompiler");
111+
CREATE_SK_COMPILER(kd::SecondaryKeyJsonDictionaryCompiler, "SecondaryKeyJsonDictionaryCompiler");
112+
CREATE_SK_COMPILER(kd::SecondaryKeyKeyOnlyDictionaryCompiler, "SecondaryKeyKeyOnlyDictionaryCompiler");
113+
CREATE_SK_COMPILER(kd::SecondaryKeyStringDictionaryCompiler, "SecondaryKeyStringDictionaryCompiler");
114+
CREATE_MERGER(kd::CompletionDictionaryMerger, "CompletionDictionaryMerger");
115+
CREATE_MERGER(kd::IntDictionaryMerger, "IntDictionaryMerger");
53116

54117
#undef CREATE_COMPILER
55118
}

‎python-pybind/src/dictionary/py_dictionary.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ void init_keyvi_dictionary(const py::module_ &m) {
4444
// 'search_tokenized', 'statistics', 'values'
4545
py::class_<kd::Dictionary>(m, "Dictionary")
4646
.def(py::init<const std::string &>())
47+
.def(py::init<const std::string &, kd::loading_strategy_types>())
4748
.def(
4849
"complete_fuzzy_multiword",
4950
[](const kd::Dictionary &d, const std::string &query, const int32_t max_edit_distance,

‎python-pybind/src/py_keyvi.cpp‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,13 @@
1717

1818
#include <pybind11/pybind11.h>
1919

20+
#include "keyvi/dictionary/fsa/internal/memory_map_flags.h"
21+
2022
#define STRINGIFY(x) #x
2123
#define MACRO_STRINGIFY(x) STRINGIFY(x)
2224

2325
namespace py = pybind11;
26+
namespace kd = keyvi::dictionary;
2427

2528
void init_keyvi_dictionary(const py::module_ &);
2629
void init_keyvi_dictionary_compilers(const py::module_ &);
@@ -38,6 +41,17 @@ PYBIND11_MODULE(keyvi_scikit_core, m) {
3841
3942
)pbdoc";
4043

44+
py::enum_<kd::loading_strategy_types>(m, "loading_strategy_types")
45+
.value("default_os", kd::loading_strategy_types::default_os)
46+
.value("lazy", kd::loading_strategy_types::lazy)
47+
.value("populate", kd::loading_strategy_types::populate)
48+
.value("populate_key_part", kd::loading_strategy_types::populate_key_part)
49+
.value("populate_lazy", kd::loading_strategy_types::populate_lazy)
50+
.value("lazy_no_readahead", kd::loading_strategy_types::lazy_no_readahead)
51+
.value("lazy_no_readahead_value_part", kd::loading_strategy_types::lazy_no_readahead_value_part)
52+
.value("populate_key_part_no_readahead_value_part",
53+
kd::loading_strategy_types::populate_key_part_no_readahead_value_part);
54+
4155
init_keyvi_match(m);
4256
py::module keyvi_dictionary = m.def_submodule("dictionary", "keyvi_scikit_core.dictionary");
4357
init_keyvi_dictionary(keyvi_dictionary);

‎python-pybind/tests/match_object_test.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@
66
import warnings
77

88

9-
#from keyvi.compiler import (
10-
# JsonDictionaryCompiler,
11-
# CompletionDictionaryCompiler,
12-
# KeyOnlyDictionaryCompiler,
13-
# StringDictionaryCompiler,
14-
#)
9+
from keyvi_scikit_core.compiler import (
10+
JsonDictionaryCompiler,
11+
CompletionDictionaryCompiler,
12+
KeyOnlyDictionaryCompiler,
13+
StringDictionaryCompiler,
14+
)
1515

1616

1717
""" def test_serialization():

0 commit comments

Comments
 (0)