[python] Fix config parsing

README.md

@@ -49,15 +49,18 @@ cmake --build build -j$(nproc) #-j flag for paralleled build process
 ```
 Additional CMake flags can be used to build tests, format code or analyze memory leaks, check [our build docs](docs/build/cmake-flags.md) for more info.
 
-## Build knowledge base:
+## Build knowledge base
 
 This repo provides *build_kb.py* script to build and prepare knowledge base (KB).
 
 ### Usage
-`python3 scripts/build_kb.py <path/to/KB/folder> -o <path/to/output/dir>`
-Additionally you can define repo file name (`-f <name>`) or logfile location (`-l <path/to/logfile/dir>`)
+`python3 scripts/build_kb.py -i <path/to/KB/folder> -o <path/to/output/dir>`
+Additionally you can define repo file name (`-f <name>`), logfile location (`-l <path/to/logfile/dir>`) or config file path
+(`-c <path/to/config/file>`).
 To get more information, use `python3 scripts/build_kb.py -h`
 
+**Note: flags have higher priority than config file.**
+
 *Example:*
 ```sh
 # This command will parse repo.path in current directory
@@ -82,4 +85,69 @@ sc-machine provides two network protocols to interact with:
 1. **sc-server**: use `./sctipts/run_sc_sever.sh` script to run sc-server
 2. **sctp server**: use `./sctipts/run_sctp.sh` script to run sctp server
 
+## Config
+
+To customize *sc-machine* usage you can create your own config file.
+
+Allowed options:
+1. [Network]
+  - `Port` - port for redis connection
+2. [Repo]
+  - `Source` - directory containing repo file
+  - `Log` - directory where error log file will be stored
+  - `Path` - path to compiled knowledge base folder
+  - `SavePeriod` - time before KB save
+3. [Extensions]
+  - `Directory` - directory with sc-machine extensions
+4. [Stat]
+  - `UpdatePeriod` - time before KB update
+  - `Path` - path to folder with sctp statistic
+5. [memory]
+  - `max_loaded_segments` - number of maximum loaded segments from kb
+6. [filememory]
+  - `engine` - engine used for reading KB (only redis supported!)
+7. [kpm]
+  - `threads` - maximum number of threads
+8. [redis]
+  - `host` - host of redis 
+9. [python]
+  - `modules_path` - path to `sc-kpm/sc-python/services`
+10. [debug]
+  - `is_debug` - debug mode enable (True|False)
+
+*Config file example*:
+```ini
+[Network]
+Port = 55770
+[Repo]
+Source = ./
+Logfile = ./
+Path = ../kb.bin
+SavePeriod = 3600
+[Extensions]
+Directory = ../bin/extensions
+[Stat]
+UpdatePeriod = 1800
+Path = /tmp/sctp_stat
+##### sc-memory
+[memory]
+max_loaded_segments = 1000
+
+[filememory]
+engine = redis
+
+[kpm]
+max_threads = 32
+
+[redis]
+host = 127.0.0.1
+
+[python]
+modules_path = /sc-machine/sc-kpm/sc-python/services
+
+[debug]
+is_debug = True
+
+```
+
 *This repository continues the development of [this sc-machine](https://github.com/ostis-dev/sc-machine) from version 0.6.0.*

config/config.ini.in

@@ -16,6 +16,3 @@ modules_path = ${SC_MACHINE_ROOT}/sc-kpm/sc-python/services
 
 [debug]
 is_debug = True
-
-[web]
-path = ${SC_MACHINE_ROOT}/web

config/sc-web.ini → config/sc-machine.conf

@@ -1,11 +1,12 @@
 [Network]
 Port = 55770
 [Repo]
+Source = ./
+Logfile = ./
 Path = ../kb.bin
 SavePeriod = 3600
 [Extensions]
 Directory = ../bin/extensions
 [Stat]
 UpdatePeriod = 1800
 Path = /tmp/sctp_stat
-

scripts/build_kb.py

@@ -2,6 +2,8 @@
 from os.path import join, abspath, relpath, commonprefix, isdir, exists
 import os
 import shutil
+import re
+import sys
 
 paths = set()
 
@@ -46,6 +48,31 @@ def copy_kb(output_path: str):
         print(e)
 
 
+def parse_config(path: str):
+    config_dict = {'src': '', 'path': '', 'log': '', 'filename': ''}
+    with open(path, mode='r') as config:
+        reading_state = False
+        for line in config.readlines():
+            line = line.replace('\n', '')
+            if line == '[Repo]':
+                reading_state = True
+            elif re.search(r'\[.+\]', line):
+                reading_state = False
+            if line.startswith(';'): 
+                continue
+            if line.find("Source = ") != -1 and reading_state:
+                config_dict.update({'src': line.replace('Source = ', '')})
+            if line.find("Path = ") != -1 and reading_state:
+                line = line.replace("Path = ", "")
+                line = line.replace('kb.bin', '')
+                config_dict.update({'path': line})
+            if line.find("Log = ") != -1 and reading_state:
+                config_dict.update({'log': line.replace("Log = ", "")})
+            if line.find("Filename = ") != -1 and reading_state:
+                config_dict.update({'filename': line.replace("Filename = ", "")})
+    return config_dict
+
+
 def prepare_kb(kb_to_prepare: str, logfile: str):
     for script in prepare_scripts:
         os.system('python3 ' + script + ' ' + kb_to_prepare + ' ' + logfile)
@@ -54,23 +81,41 @@ def prepare_kb(kb_to_prepare: str, logfile: str):
         exit(1)
 
 
-def build_kb(kb_to_build: str):
-    bin_folder = join(os.getcwd(), "kb.bin")
+def build_kb(bin_folder: str, kb_to_build: str):
+    bin_folder = join(bin_folder, "kb.bin")
     os.makedirs(bin_folder, exist_ok=True)
     os.environ['LD_LIBRARY_PATH'] = join(ostis_path, "/bin")
     #call sc-builder with required parameters
-    os.system(" ".join([join(ostis_path, "bin/sc-builder"), "-f", "-c", "-i", kb_to_build, "-o", bin_folder, "-s", join(ostis_path, "config/sc-web.ini"), "-e", join(ostis_path, "bin/extensions")]))
+    os.system(" ".join([join(ostis_path, "bin/sc-builder"), "-f", "-c", "-i", kb_to_build, "-o", bin_folder, "-e", join(ostis_path, "bin/extensions")]))
+
+
+def main(root_repo_path: str, output_path: str, logfile: str, repo_filename: str, config_file_path: str):
+    conf = parse_config(config_file_path)
+    if conf['src'] == '':
+        root_repo_path = abspath(root_repo_path)
+    elif root_repo_path == os.getcwd():
+        root_repo_path = relpath(ostis_path, conf['src'])
+
+    if conf['path'] == '':
+        output_path = abspath(output_path)
+    elif output_path == os.getcwd():
+        output_path = relpath(ostis_path, conf['path'])
 
+    if conf['log'] == '':
+        logfile = abspath(logfile)
+    elif logfile == join(os.getcwd(), "prepare.log"):
+        logfile = relpath(ostis_path, conf['log'])
+        logfile = join(logfile, 'prepare.log')
 
-def main(root_repo_path: str, output_path: str, logfile: str, repo_filename: str):
-    root_repo_path = abspath(root_repo_path)
-    output_path = abspath(output_path)
-    logfile = abspath(logfile)
+    if conf['filename'] != '' and repo_filename == 'repo.path':
+        repo_filename = conf['filename']
 
     kb_to_prepare = join(output_path, "prepared_kb")
     if isdir(kb_to_prepare):
         shutil.rmtree(kb_to_prepare)
 
+
+
     search_knowledge_bases(root_repo_path, output_path, repo_filename)
 
     if not paths:
@@ -79,15 +124,15 @@ def main(root_repo_path: str, output_path: str, logfile: str, repo_filename: str
 
     copy_kb(output_path)
     prepare_kb(kb_to_prepare, logfile)
-    build_kb(kb_to_prepare)
+    build_kb(output_path, kb_to_prepare)
     shutil.rmtree(kb_to_prepare)
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
 
-    parser.add_argument(dest="repo_folder", type=str,
-                        help="The entrypoint folder, should contain a repo file (repo.path by default)")
+    parser.add_argument('-i', '--input', dest="repo_folder", type=str,
+                        help="The entrypoint folder, should contain a repo file (repo.path by default)", default=os.getcwd())
 
     parser.add_argument('-o', '--output', dest="output_path",
                         help="Destination path - path where prepared KB and built KB (kb.bin) will be stored. Default: <cwd>", default=os.getcwd())
@@ -98,7 +143,11 @@ def main(root_repo_path: str, output_path: str, logfile: str, repo_filename: str
     parser.add_argument('-f', '--filename', dest="repo_path_name",
                         help="Repo file name - a filename for repo file that will be used in all subsequent KBs. Default: repo.path", default="repo.path")
 
+    parser.add_argument('-c', '--config', dest='config_file_path',
+                        help="Config file path - path to config file (Note: config file has lower priority than flags!)")
+
     args = parser.parse_args()
 
+
     main(args.repo_folder, args.output_path,
-         args.errors_file_path, args.repo_path_name)
+         args.errors_file_path, args.repo_path_name, args.config_file_path)

scripts/run_sctp.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 export LD_LIBRARY_PATH=../bin
-../bin/sctp-server ../config/sc-web.ini
+../bin/sctp-server ../config/sc-machine.conf