Merge pull request #6 from TatraDev/venv_logic

refactor structure project

Author: LaoGunz

.github/workflows/test.yaml

@@ -0,0 +1,32 @@
+name: Python package
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Test with pytest
+        run: |
+           pytest --verbose tests/running_piper_test.py::TestVenv

main.py

@@ -1,12 +1,15 @@
-from piper.services import TestMessageAdder, StringValue, TesseractRecognizer, SpacyNER
-from piper.envs import CurrentEnv, DockerEnv, VirtualEnv
-from piper.configurations import get_configuration
-import time
 import asyncio
 import sys
-from piper.utils import tesrct_utils as tu
+import time
 
 from loguru import logger
+
+from piper.configurations import get_configuration
+from piper.envs import CurrentEnv, DockerEnv, VirtualEnv
+# from piper.services import (SpacyNER, StringValue, TesseractRecognizer,
+#                             TestMessageAdder)
+# from piper.utils import tesrct_utils as tu
+
 logger.add("file.log", level="INFO", backtrace=True, diagnose=True, rotation='5 MB')
 
 if __name__ == '__main__':
@@ -24,15 +27,15 @@
     # print(result)
     # adder.rm_container()
 
-    logger.info(f'main here {time.time()}')
-    cfg = get_configuration()
-    loop = asyncio.get_event_loop()
-    with DockerEnv() as env:
-        # object created
-        recognizer = TesseractRecognizer(port=cfg.docker_app_port)
-        
-        result = loop.run_until_complete(recognizer())
-        logger.info(f'result of recognition is {result}')
+    # logger.info(f'main here {time.time()}')
+    # cfg = get_configuration()
+    # loop = asyncio.get_event_loop()
+    # with DockerEnv() as env:
+    #     # object created
+    #     recognizer = TesseractRecognizer(port=cfg.docker_app_port)
+    #
+    #     result = loop.run_until_complete(recognizer())
+    #     logger.info(f'result of recognition is {result}')
 
     with VirtualEnv() as env:
         env.copy_struct_project()

piper/base/executors.py

@@ -1,24 +1,25 @@
-from abc import abstractmethod, ABC
-from distutils.command.config import config
+import inspect
 import os
+import sys
 import time
+from abc import ABC, abstractmethod
+from distutils.command.config import config
 from typing import Dict
-import inspect
 
 import aiohttp
-from loguru import logger
 import docker
-from pydantic import BaseModel #, BytesObject, ListOfStringsObject
+import requests
+from loguru import logger
+from pydantic import BaseModel  # , BytesObject, ListOfStringsObject
 
-from piper.base.docker import PythonImage
 # from piper.base.docker import PythonTesseractImage
-from piper.base.backend.utils import render_fast_api_backend, render_fast_api_tsrct_backend
-from piper.envs import is_docker_env, is_current_env, get_env
+from piper.base.backend.utils import (render_fast_api_backend,
+                                      render_fast_api_tsrct_backend)
+from piper.base.docker import PythonImage
 from piper.configurations import get_configuration
+from piper.envs import get_env, is_current_env, is_docker_env
 from piper.utils import docker_utils as du
 
-import requests
-import sys
 
 class BaseExecutor:
     pass

piper/configurations.py

@@ -1,7 +1,7 @@
 class Configuration:
-    path = "/Users/olegsokolov/PycharmProjects/piper/applications"
-    path = "/home/pavel/repo/piper_new_out/"
-    test_path = "/home/pavel/repo/piper_test_out/"
+    # path = "/Users/olegsokolov/PycharmProjects/piper/applications"
+    path = "./piper_new_out/"
+    test_path = "./piper_test_out/"
     piper_path = "piper"
     default_env = "docker"
     docker_app_port = 8788
@@ -17,7 +17,7 @@ class Configuration:
 
     # docker start time and counter
     docker_wait_on_iter = 0.5
-    docker_n_iters = 20    
+    docker_n_iters = 20
 
     n_gunicorn_workers = 1
 
@@ -58,8 +58,6 @@ class Configuration:
     ts_config_row = r'--oem 1 --psm 11'
     ts_config = {'ts_lang': 'eng', 'ts_config_row': r'--oem 1 --psm 11'}
 
-    
-
     # models and where to find them
     spacy_models = set(
         [
@@ -70,5 +68,6 @@ class Configuration:
     )
     model_path = '/app/models'
 
+
 def get_configuration():
     return Configuration

piper/services/__init__.py

@@ -1,27 +1,30 @@
-from piper.base.executors import FastAPIExecutor, FastAPITesseractExecutor, VirtualEnvExecutor
-from fastapi.responses import JSONResponse
-
-from pydantic import BaseModel
-from loguru import logger
 import json
-import spacy
 import sys
+
+import spacy
+from fastapi.responses import JSONResponse
+from loguru import logger
+from pydantic import BaseModel
+
+from piper.base.executors import FastAPIExecutor, FastAPITesseractExecutor
 from piper.configurations import get_configuration
 from piper.utils import tesrct_utils as tu
 
-
 logger.add("file.log", level="INFO", backtrace=True, diagnose=True, rotation='5 MB')
 
 
 class StringValue(BaseModel):
     value: str
 
+
 class BytesObject(BaseModel):
     value: bytes
 
+
 class ListOfStringsObject(BaseModel):
     value: list
 
+
 class TestMessageAdder(FastAPIExecutor):
 
     def __init__(self, appender="TEST", **kwargs):
@@ -30,18 +33,19 @@ def __init__(self, appender="TEST", **kwargs):
 
     async def run(self, message: StringValue) -> StringValue:
         return StringValue(value=(message.value + self.appender))
-        
+
 
 class TesseractRecognizer(FastAPITesseractExecutor):
     '''
         Tesseract OCR implementation service
     '''
+
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         cfg = get_configuration()
         self.ts_config = cfg.ts_config
 
-    def set_config_(self, config_):        
+    def set_config_(self, config_):
         if 'ts_lang' not in config_.keys():
             logger.error(f'tesseract config keys must contains ts_lang, keys {config_.keys()}')
             logger.error(f'tesseract config did not set')
@@ -59,23 +63,24 @@ async def sconfig(self, conf) -> ListOfStringsObject:
         # conf = '12'
         logger.info(f'request to set config to {conf}')
         self.set_config_(conf)
-        return JSONResponse(content={'text':'OK'})
-    
-    async def recognize(self, file_content : BytesObject, suf: str) -> ListOfStringsObject:
+        return JSONResponse(content={'text': 'OK'})
+
+    async def recognize(self, file_content: BytesObject, suf: str) -> ListOfStringsObject:
         logger.info(f'file_content {type(file_content)}, file suffix is {suf}')
 
         logger.info(f'current tesseract config is {self.ts_config}')
         text_dict = tu.bytes_handler(file_content, suf, self.ts_config)
         logger.info(f'img_bytes_handler return {type(text_dict)} object')
         return JSONResponse(content=text_dict)
 
-    async def ner(self, txt: str): 
+    async def ner(self, txt: str):
         sn = SpacyNER()
         if sn.available_models and len(sn.available_models) > 0:
             dummy_model = sn.available_models[0]
             sn.set_model(dummy_model)
         return JSONResponse(content=sn.extract_named_ents(txt))
 
+
 # class ModelNameNotInList(BaseException):
 #     def __init__(self, msg):
 #         # pass
@@ -86,6 +91,7 @@ class SpacyNER():
     '''
         Spacy NER service
     '''
+
     def __init__(self):
         cfg = get_configuration()
         self.available_models = set()
@@ -102,34 +108,32 @@ def __init__(self):
             logger.error(f'catch exception {e}')
             sys.exit()
 
-
     def set_model(self, cur_model):
         if cur_model not in self.available_models:
             logger.error(f'there is not {cur_model} in available_models set: {self.available_models}')
             self.nlp = None
             raise ValueError(f'there is not {cur_model} in available_models set: {self.available_models}')
 
-        try:        
+        try:
             nlp = spacy.load(cur_model)
             # nlp = spacy.load('en_default')
             logger.info('spacy nlp object created with model {cur_model}')
         except Exception as e:
             logger.error(f'catch exception {e}')
-            if isinstance(e, OSError):                
+            if isinstance(e, OSError):
                 logger.error(f'you must download spacy model {cur_model}')
             nlp = None
             logger.info('spacy nlp object DID NOT create')
-        
-        self.nlp = nlp
 
+        self.nlp = nlp
 
     def extract_named_ents(self, txt: str):
         logger.debug(f'got data type {type(txt)} and data <<{txt}>> for NER')
         if self.nlp:
             res = []
             doc = self.nlp(txt)
             for ent in doc.ents:
-                res.append((ent.text,  ent.label_))
+                res.append((ent.text, ent.label_))
             return JSONResponse(content=res)
         else:
             logger.error(f'nlp object didn`t create. you should use set_model(model_name)')

piper/utils/docker_utils.py

@@ -1,6 +1,7 @@
-import docker
-import time
 import sys
+import time
+
+import docker
 from loguru import logger
 
 from piper.configurations import get_configuration

piper/utils/tesrct_utils.py

@@ -1,9 +1,10 @@
 import cv2
-import pytesseract
 import numpy as np
-from loguru import logger
 import pdf2image
+import pytesseract
 import requests
+from loguru import logger
+
 from piper.configurations import get_configuration
 
 cfg = get_configuration()