Merge pull request #11 from BaCa2-project/dev

Broker refactored to use:

1. FastAPI
2. Pydantic

Communication secured
Added logging system

.env.template

@@ -0,0 +1,14 @@
+SERVER_HOST='127.0.0.1'
+SERVER_PORT=8180
+SERVER_URL='127.0.0.1'
+
+ACTIVE_WAIT=true
+
+BACA_URL="http://127.0.0.1:8000/broker_api"
+# BACA2_DIR='example/baca2/dir'
+# PACKAGES_DIR='example/packages/dir'
+# SUBMITS_DIR='example/submits/dir'
+
+# BROKER_SETTINGS -----
+BACA_PASSWORD='tmp-baca-password'
+BROKER_PASSWORD='tmp-broker-password'

.gitignore

@@ -2,6 +2,9 @@
 # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 
+# poetry
+poetry.lock
+
 # User-specific stuff
 .idea/
 .idea/**/workspace.xml
@@ -285,3 +288,16 @@ cython_debug/
 
 # for mac users
 .DS_Store
+
+**/.build/
+
+kolejka_src/
+submits/
+
+# Config file for kolejka server
+
+*/kolejka.conf
+kolejka.conf
+
+/.env
+/logs

README.md

@@ -0,0 +1,26 @@
+# BaCa2 Broker
+
+BaCa2 Broker is an HTTP server that listens for incoming requests build using FastAPI framework.
+It is responsible for managing the communication between BaCa2 and Kolejka checking system - 
+it receives tasks from BaCa2, sends them to Kolejka, and then sends the results back to BaCa2.
+
+## Structure
+
+The application is contained in the `app` directory. It consists of the following modules:
+- `main.py` - the main module that starts the server
+- `handlers.py` - contains handlers for incoming requests that are ment to run in the background.
+- `logger.py` - logger logic for the application
+- `broker` - a package that contains the logic for the broker, it consists of:
+  * `datamaster.py` - logic for managing data
+  * `messenger.py` - responsible for sending and receiving messages from/to Kolejka and BaCa2
+  * `builder.py` - parses data for Kolejka
+  * `master.py` - combines all of the above to manage the whole process
+
+In the `judges` directory there are judge configurations for Kolejka system.
+
+## Running
+
+Broker uses `settings.py` for configuration and can be launched using `run.py` script.
+Certain preferences have to be set using environment variables listed in `.end.template` file.
+`kolejka.conf` file has to be present in the root directory of the broker and list login
+credentials for Kolejka system.

app/broker/__init__.py

@@ -0,0 +1,13 @@
+from .master import BrokerMaster
+from .datamaster import DataMaster, SetSubmit, TaskSubmit
+from .messenger import KolejkaMessenger, BacaMessenger, PackageManager
+
+__all__ = [
+    'BrokerMaster',
+    'DataMaster',
+    'SetSubmit',
+    'TaskSubmit',
+    'KolejkaMessenger',
+    'BacaMessenger',
+    'PackageManager'
+]

app/broker/builder.py

@@ -0,0 +1,159 @@
+import os
+from pathlib import Path
+
+from baca2PackageManager import Package, TSet, TestF
+from yaml import dump
+import settings
+
+from .yaml_tags import get_dumper, File
+
+INCLUDE_TAG = '0tag::include'
+
+
+class Builder:
+    TRANSLATE_CMD = {
+        'test_generator': 'generator',
+        'memory_limit': 'memory',
+        'time_limit': 'time',
+    }
+    IGNORED_KEYS = ['name', 'points', 'weight', 'tests']
+
+    def __init__(self, package: Package, enable_shortcut: bool = True) -> None:
+        self.package = package
+        self.build_namespace = settings.BUILD_NAMESPACE
+        self.build_path = None
+        self.enable_shortcut = enable_shortcut
+        self.common_path = None
+        self.source_size = package.get('source_size')
+
+    @property
+    def is_built(self) -> bool:
+        return self.package.check_build(self.build_namespace)
+
+    @staticmethod
+    def to_yaml(data: dict, path: Path) -> None:
+        with open(path, mode='wt', encoding='utf-8') as file:
+            file.write(dump(data, Dumper=get_dumper()))
+
+        # replace import tags
+        with open(path, mode='r', encoding='utf-8') as file:
+            content = file.read()
+
+        content = content.replace(INCLUDE_TAG, '!include ')
+
+        with open(path, mode='wt', encoding='utf-8') as file:
+            file.write(content)
+
+    def _generate_test_yaml(self):
+        test_yaml = {
+            'memory': '512MB',
+            'kolejka': {
+                'image': 'kolejka/satori:judge',
+                'exclusive': False,
+                'requires': ['cpu:xeon e3-1270 v5'],
+                'collect': ['log.zip'],
+                'limits': {
+                    'time': '600s',
+                    'memory': '10G',
+                    'swap': 0,
+                    'cpus': self.package['cpus'],
+                    'network': self.package['network'],
+                    'storage': '5G',
+                    'workspace': '5G',
+                }
+            }
+        }
+        if self.enable_shortcut:
+            test_yaml['kolejka']['satori'] = {
+                'result': {
+                    'execute_time_real': '/io/executor/run/real_time',
+                    'execute_time_cpu': '/io/executor/run/cpu_time',
+                    'execute_memory': '/io/executor/run/memory',
+                    'compile_log': 'str:/builder/**/stdout,/builder/**/stderr',
+                    'tool_log': 'str:/io/generator/**/stderr,/io/verifier/**/stdout,'
+                                '/io/verifier/**/stderr,/io/hinter/**/stderr',
+                    'checker_log': 'str:/io/checker/**/stdout,/io/checker/**/stderr',
+                    'answer': 'str:/io/executor/run/stdout',
+                    'logs': '/logs/logs',
+                    'debug': '/debug/debug',
+                }
+            }
+
+        return test_yaml
+
+    def _create_common(self,
+                       test_yaml: dict,
+                       judge_type: str = 'main'):
+        self.common_path = self.build_path / 'common'
+        self.common_path.mkdir()
+
+        self.to_yaml(test_yaml, self.common_path / 'test.yaml')
+
+        os.symlink(settings.KOLEJKA_SRC_DIR / 'kolejka-judge', self.common_path / 'kolejka-judge')
+        os.symlink(settings.KOLEJKA_SRC_DIR / 'kolejka-client', self.common_path / 'kolejka-client')
+        os.symlink(settings.JUDGES[judge_type], self.common_path / 'judge.py')
+
+    def build(self):
+        self.build_path = self.package.prepare_build(self.build_namespace)
+
+        test_yaml = self._generate_test_yaml()
+        self._create_common(test_yaml)
+
+        for t_set in self.package.sets():
+            set_builder = SetBuilder(self.package, t_set, self.build_path)
+            set_builder.build()
+
+
+class SetBuilder:
+    def __init__(self, package: Package, t_set: TSet, build_path: Path) -> None:
+        self.package = package
+        self.t_set = t_set
+        self.name = t_set['name']
+        self.build_path = build_path / self.name
+
+    def _generate_test_yaml(self):
+        test_yaml = {
+            INCLUDE_TAG: '../common/test.yaml',
+        }
+        for k, v in self.t_set:
+            key = Builder.TRANSLATE_CMD.get(k, k)
+            if key == 'time':
+                v = f'{v * 1000}ms'
+            if key not in Builder.IGNORED_KEYS and v is not None:
+                test_yaml[key] = v
+        return test_yaml
+
+    def _add_test(self, test_yaml: dict, test: TestF, include_test: bool = True):
+        single_test = {}
+        if include_test:
+            single_test[INCLUDE_TAG] = 'test.yaml'
+
+        if test.get('input') is not None:
+            test_filename = test['name'] + '.in'
+            os.symlink(test['input'], self.build_path / test_filename)
+            single_test['input'] = File(test_filename)
+        if test.get('output') is not None:
+            test_filename = test['name'] + '.out'
+            os.symlink(test['output'], self.build_path / test_filename)
+            single_test['hint'] = File(test_filename)
+
+        for k, v in test:
+            key = Builder.TRANSLATE_CMD.get(k, k)
+            if key == 'time':
+                v = f'{v * 1000}ms'
+            if key not in Builder.IGNORED_KEYS[:] + ['input', 'output', 'hint']:
+                single_test[key] = v
+
+        test_yaml[test['name']] = single_test
+
+    def build(self):
+        os.mkdir(self.build_path)
+
+        test_yaml = self._generate_test_yaml()
+
+        tests_yaml = {}
+        for test in self.t_set.tests():
+            self._add_test(tests_yaml, test)
+
+        Builder.to_yaml(test_yaml, self.build_path / 'test.yaml')
+        Builder.to_yaml(tests_yaml, self.build_path / 'tests.yaml')