diff --git a/README.md b/README.md index 1276ed11a0..6a837e5d39 100644 --- a/README.md +++ b/README.md @@ -49,15 +49,18 @@ cmake --build build -j$(nproc) #-j flag for paralleled build process ``` Additional CMake flags can be used to build tests, format code or analyze memory leaks, check [our build docs](docs/build/cmake-flags.md) for more info. -## Build knowledge base: +## Build knowledge base This repo provides *build_kb.py* script to build and prepare knowledge base (KB). ### Usage -`python3 scripts/build_kb.py -o ` -Additionally you can define repo file name (`-f `) or logfile location (`-l `) +`python3 scripts/build_kb.py -i -o ` +Additionally you can define repo file name (`-f `), logfile location (`-l `) or config file path +(`-c `). To get more information, use `python3 scripts/build_kb.py -h` +**Note: flags have higher priority than config file.** + *Example:* ```sh # This command will parse repo.path in current directory @@ -82,4 +85,69 @@ sc-machine provides two network protocols to interact with: 1. **sc-server**: use `./sctipts/run_sc_sever.sh` script to run sc-server 2. **sctp server**: use `./sctipts/run_sctp.sh` script to run sctp server +## Config + +To customize *sc-machine* usage you can create your own config file. + +Allowed options: +1. [Network] + - `Port` - port for redis connection +2. [Repo] + - `Source` - directory containing repo file + - `Log` - directory where error log file will be stored + - `Path` - path to compiled knowledge base folder + - `SavePeriod` - time before KB save +3. [Extensions] + - `Directory` - directory with sc-machine extensions +4. [Stat] + - `UpdatePeriod` - time before KB update + - `Path` - path to folder with sctp statistic +5. [memory] + - `max_loaded_segments` - number of maximum loaded segments from kb +6. [filememory] + - `engine` - engine used for reading KB (only redis supported!) +7. [kpm] + - `threads` - maximum number of threads +8. [redis] + - `host` - host of redis +9. [python] + - `modules_path` - path to `sc-kpm/sc-python/services` +10. [debug] + - `is_debug` - debug mode enable (True|False) + +*Config file example*: +```ini +[Network] +Port = 55770 +[Repo] +Source = ./ +Logfile = ./ +Path = ../kb.bin +SavePeriod = 3600 +[Extensions] +Directory = ../bin/extensions +[Stat] +UpdatePeriod = 1800 +Path = /tmp/sctp_stat +##### sc-memory +[memory] +max_loaded_segments = 1000 + +[filememory] +engine = redis + +[kpm] +max_threads = 32 + +[redis] +host = 127.0.0.1 + +[python] +modules_path = /sc-machine/sc-kpm/sc-python/services + +[debug] +is_debug = True + +``` + *This repository continues the development of [this sc-machine](https://github.com/ostis-dev/sc-machine) from version 0.6.0.* diff --git a/config/config.ini.in b/config/config.ini.in index 7f055e3bbd..84cc1c6a3a 100644 --- a/config/config.ini.in +++ b/config/config.ini.in @@ -16,6 +16,3 @@ modules_path = ${SC_MACHINE_ROOT}/sc-kpm/sc-python/services [debug] is_debug = True - -[web] -path = ${SC_MACHINE_ROOT}/web diff --git a/config/sc-web.ini b/config/sc-machine.conf similarity index 86% rename from config/sc-web.ini rename to config/sc-machine.conf index 434d1958b6..7d9c07a20b 100644 --- a/config/sc-web.ini +++ b/config/sc-machine.conf @@ -1,6 +1,8 @@ [Network] Port = 55770 [Repo] +Source = ./ +Logfile = ./ Path = ../kb.bin SavePeriod = 3600 [Extensions] @@ -8,4 +10,3 @@ Directory = ../bin/extensions [Stat] UpdatePeriod = 1800 Path = /tmp/sctp_stat - diff --git a/scripts/build_kb.py b/scripts/build_kb.py index 65634cadf1..ec0f4c1645 100644 --- a/scripts/build_kb.py +++ b/scripts/build_kb.py @@ -2,6 +2,8 @@ from os.path import join, abspath, relpath, commonprefix, isdir, exists import os import shutil +import re +import sys paths = set() @@ -46,6 +48,31 @@ def copy_kb(output_path: str): print(e) +def parse_config(path: str): + config_dict = {'src': '', 'path': '', 'log': '', 'filename': ''} + with open(path, mode='r') as config: + reading_state = False + for line in config.readlines(): + line = line.replace('\n', '') + if line == '[Repo]': + reading_state = True + elif re.search(r'\[.+\]', line): + reading_state = False + if line.startswith(';'): + continue + if line.find("Source = ") != -1 and reading_state: + config_dict.update({'src': line.replace('Source = ', '')}) + if line.find("Path = ") != -1 and reading_state: + line = line.replace("Path = ", "") + line = line.replace('kb.bin', '') + config_dict.update({'path': line}) + if line.find("Log = ") != -1 and reading_state: + config_dict.update({'log': line.replace("Log = ", "")}) + if line.find("Filename = ") != -1 and reading_state: + config_dict.update({'filename': line.replace("Filename = ", "")}) + return config_dict + + def prepare_kb(kb_to_prepare: str, logfile: str): for script in prepare_scripts: os.system('python3 ' + script + ' ' + kb_to_prepare + ' ' + logfile) @@ -54,23 +81,41 @@ def prepare_kb(kb_to_prepare: str, logfile: str): exit(1) -def build_kb(kb_to_build: str): - bin_folder = join(os.getcwd(), "kb.bin") +def build_kb(bin_folder: str, kb_to_build: str): + bin_folder = join(bin_folder, "kb.bin") os.makedirs(bin_folder, exist_ok=True) os.environ['LD_LIBRARY_PATH'] = join(ostis_path, "/bin") #call sc-builder with required parameters - os.system(" ".join([join(ostis_path, "bin/sc-builder"), "-f", "-c", "-i", kb_to_build, "-o", bin_folder, "-s", join(ostis_path, "config/sc-web.ini"), "-e", join(ostis_path, "bin/extensions")])) + os.system(" ".join([join(ostis_path, "bin/sc-builder"), "-f", "-c", "-i", kb_to_build, "-o", bin_folder, "-e", join(ostis_path, "bin/extensions")])) + + +def main(root_repo_path: str, output_path: str, logfile: str, repo_filename: str, config_file_path: str): + conf = parse_config(config_file_path) + if conf['src'] == '': + root_repo_path = abspath(root_repo_path) + elif root_repo_path == os.getcwd(): + root_repo_path = relpath(ostis_path, conf['src']) + + if conf['path'] == '': + output_path = abspath(output_path) + elif output_path == os.getcwd(): + output_path = relpath(ostis_path, conf['path']) + if conf['log'] == '': + logfile = abspath(logfile) + elif logfile == join(os.getcwd(), "prepare.log"): + logfile = relpath(ostis_path, conf['log']) + logfile = join(logfile, 'prepare.log') -def main(root_repo_path: str, output_path: str, logfile: str, repo_filename: str): - root_repo_path = abspath(root_repo_path) - output_path = abspath(output_path) - logfile = abspath(logfile) + if conf['filename'] != '' and repo_filename == 'repo.path': + repo_filename = conf['filename'] kb_to_prepare = join(output_path, "prepared_kb") if isdir(kb_to_prepare): shutil.rmtree(kb_to_prepare) + + search_knowledge_bases(root_repo_path, output_path, repo_filename) if not paths: @@ -79,15 +124,15 @@ def main(root_repo_path: str, output_path: str, logfile: str, repo_filename: str copy_kb(output_path) prepare_kb(kb_to_prepare, logfile) - build_kb(kb_to_prepare) + build_kb(output_path, kb_to_prepare) shutil.rmtree(kb_to_prepare) if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument(dest="repo_folder", type=str, - help="The entrypoint folder, should contain a repo file (repo.path by default)") + parser.add_argument('-i', '--input', dest="repo_folder", type=str, + help="The entrypoint folder, should contain a repo file (repo.path by default)", default=os.getcwd()) parser.add_argument('-o', '--output', dest="output_path", help="Destination path - path where prepared KB and built KB (kb.bin) will be stored. Default: ", default=os.getcwd()) @@ -98,7 +143,11 @@ def main(root_repo_path: str, output_path: str, logfile: str, repo_filename: str parser.add_argument('-f', '--filename', dest="repo_path_name", help="Repo file name - a filename for repo file that will be used in all subsequent KBs. Default: repo.path", default="repo.path") + parser.add_argument('-c', '--config', dest='config_file_path', + help="Config file path - path to config file (Note: config file has lower priority than flags!)") + args = parser.parse_args() + main(args.repo_folder, args.output_path, - args.errors_file_path, args.repo_path_name) + args.errors_file_path, args.repo_path_name, args.config_file_path) diff --git a/scripts/run_sctp.sh b/scripts/run_sctp.sh index d8de131d19..e54af77b1b 100755 --- a/scripts/run_sctp.sh +++ b/scripts/run_sctp.sh @@ -1,4 +1,4 @@ #!/bin/bash export LD_LIBRARY_PATH=../bin -../bin/sctp-server ../config/sc-web.ini +../bin/sctp-server ../config/sc-machine.conf