Feature/collect process init status (#713)

* Collect process init status

* remove mark

* remove mark

* remove useless code

* update frontend package socket.io-client

frontend/package.json

@@ -11,7 +11,7 @@
     "iview": "^3.4.2",
     "monaco-editor": "^0.17.0",
     "monaco-editor-webpack-plugin": "^1.7.0",
-    "socket.io-client": "^2.4.0",
+    "socket.io-client": "^3.1.3",
     "vue": "^2.7.10",
     "vue-clipboard2": "^0.3.1",
     "vue-draggable-nested-tree": "^2.2.17",

frontend/src/views/inspector/FlowList.vue

@@ -89,36 +89,40 @@
       </template>
 
       <template v-slot:item.request="{ item }">
-        <span class="flow-list-item-url">
-          <span>{{ item.request.scheme }}</span>
-          <span v-if="item.request.scheme">://</span>
+        <v-row class="my-0 ml-0 mr-1">
+          <span class="flow-list-item-url">
+            <span>{{ item.request.scheme }}</span>
+            <span v-if="item.request.scheme">://</span>
 
-          <span class="flow-list-item-url-host">{{ item.request.host}}</span>
-          <span class="flow-list-item-url-path">{{ item.request.path}}</span>
+            <span class="flow-list-item-url-host">{{ item.request.host}}</span>
+            <span class="flow-list-item-url-path">{{ item.request.path}}</span>
 
-          <span class="flow-list-item-url-params" v-if="item.request.params">?</span>
-          <span class="flow-list-item-url-params">{{ item.request.params }}</span>
-        </span>
+            <span class="flow-list-item-url-params" v-if="item.request.params">?</span>
+            <span class="flow-list-item-url-params">{{ item.request.params }}</span>
+          </span>
 
-        <span class="flow-list-item-copy-btn" @click.stop>
-          <v-tooltip bottom>
-            <template v-slot:activator="{ on, attrs }">
-              <span v-bind="attrs" v-on="on">
-                <v-btn icon x-small plain>
-                  <v-icon
-                    x-small
-                    color="accent"
-                    v-clipboard:copy="item.request.url"
-                    v-clipboard:success="onUrlCopy"
-                    v-clipboard:error="onUrlCopyError"
-                  >mdi-content-copy</v-icon>
-                </v-btn>
-              </span>
-            </template>
-            Copy
-          </v-tooltip>
+          <v-spacer/>
 
-        </span>
+          <span class="flow-list-item-copy-btn" @click.stop>
+            <v-tooltip bottom>
+              <template v-slot:activator="{ on, attrs }">
+                <span v-bind="attrs" v-on="on">
+                  <v-btn icon x-small plain>
+                    <v-icon
+                      x-small
+                      color="accent"
+                      v-clipboard:copy="item.request.url"
+                      v-clipboard:success="onUrlCopy"
+                      v-clipboard:error="onUrlCopyError"
+                    >mdi-content-copy</v-icon>
+                  </v-btn>
+                </span>
+              </template>
+              Copy
+            </v-tooltip>
+
+          </span>
+        </v-row>
       </template>
 
       <template v-slot:item.start_time="{ item }">
@@ -389,6 +393,9 @@ export default {
 .flow-table table>tbody>tr>td>span {
   color: #666 !important;
 }
+.flow-table table>tbody>tr>td>div {
+  color: #666 !important;
+}
 .flow-table {
   width: 100%;
 }
@@ -462,6 +469,7 @@ export default {
 .flow-list-item-url {
   display: inline-block;
   word-break: keep-all;
+  max-width: 900px;
   width: calc(100% - 50px);
   white-space: nowrap;
   overflow: hidden;

lyrebird/application.py

@@ -1,4 +1,5 @@
 from flask import jsonify
+from functools import reduce
 
 """
 Lyrebird context
@@ -24,9 +25,6 @@ def make_fail_response(msg, **kwargs):
     return jsonify(fail_resp)
 
 
-# Lyrebird status contains: 'READY' and 'INITING'
-status = 'INITING'
-
 _cm = None
 _src = None
 
@@ -86,3 +84,66 @@ def raw(self):
 def root_dir():
     if _cm:
         return _cm.ROOT
+
+
+# --------- Lyrebird status ---------
+'''
+[INITING] --> /run lyrebird main method/  --> [READY]
+              /start services          / 
+              /extra mock server ready /
+              /mitm proxy server ready /
+'''
+
+# Lyrebird status contains: 'READY' and 'INITING'
+status = 'INITING'
+
+# mitm proxy check point will be added if args set useing mitm server (--no-mitm False)
+status_checkpoints = {
+    'main': False,
+    'extra_mock': False
+}
+
+
+def status_listener(event):
+    '''
+    event example
+    event = {
+        'channel': 'system',
+        'system': {
+            'action': 'init_module',
+            'status': 'READY',
+            'module': 'main'
+        }
+    }
+    '''
+
+    system = event.get('system')
+    if not system:
+        return
+
+    action = system.get('action')
+    if action != 'init_module':
+        return
+
+    module = system.get('module')
+    if module not in status_checkpoints:
+        return
+
+    module_status = system.get('status')
+    if module_status == 'READY':
+        status_checkpoints[module] = True
+
+    is_all_status_checkpoints_ok = reduce(lambda x, y: x and y, status_checkpoints.values())
+    if is_all_status_checkpoints_ok:
+        global status
+        status = 'READY'
+
+
+def process_status_listener():
+    server['event'].subscribe('system', status_listener)
+
+
+def status_ready():
+    server['event'].publish('system', {
+        'system': {'action': 'init_module', 'status': 'READY', 'module': 'main'}
+    })

lyrebird/manager.py

@@ -85,9 +85,6 @@ def main():
     gen_parser = subparser.add_parser('gen')
     gen_parser.add_argument('path', help='Create plugin project')
 
-    install_parser = subparser.add_parser('install')
-    install_parser.add_argument('extension_name')
-
     args = parser.parse_args()
 
     if args.version:
@@ -188,6 +185,7 @@ def run(args: argparse.Namespace):
     else:
         should_start_mitm = not conf_no_mitm
     if should_start_mitm:
+        application.status_checkpoints['mitm_proxy'] = False
         application.server['proxy'] = LyrebirdProxyServer()
 
     application.server['mock'] = LyrebirdMockServer()
@@ -196,6 +194,9 @@ def run(args: argparse.Namespace):
     application.server['plugin'] = PluginManager()
     application.server['checker'] = LyrebirdCheckerServer()
 
+    # handle progress message
+    application.process_status_listener()
+
     application.start_server()
 
     # int statistics reporter
@@ -225,8 +226,8 @@ def run(args: argparse.Namespace):
     if not args.no_browser:
         webbrowser.open(f'http://localhost:{application.config["mock.port"]}')
 
-    # Lyrebird status contains: 'READY' and 'INITING'
-    application.status = 'READY'
+    # main process is ready, publish system event
+    application.status_ready()
 
     # stop event handler
     def signal_handler(signum, frame):

lyrebird/mitm/proxy_server.py

@@ -45,7 +45,7 @@ def wait_for_mitm_start(self, config, logger):
             except Exception:
                 continue
 
-    def start_mitmdump(self, config, logger, mitmdump_path):
+    def start_mitmdump(self, queue, config, logger, mitmdump_path):
         proxy_port = config.get('proxy.port', 4272)
         mock_port = config.get('mock.port', 9090)
         '''
@@ -82,16 +82,31 @@ def start_mitmdump(self, config, logger, mitmdump_path):
         subprocess.Popen(f'{str(mitmdump_path)} {" ".join(mitm_arguments)}', shell=True, env=mitmenv)
         is_mitm_start = self.wait_for_mitm_start(config, logger)
         if is_mitm_start:
+            self.publish_init_status(queue, 'READY')
             logger.log(60, f'HTTP proxy server start on {proxy_port}')
         else:
+            self.publish_init_status(queue, 'ERROR')
             self.show_mitmdump_start_timeout_help(mitmdump_path, logger)
 
+    def publish_init_status(self, queue, status):
+        queue.put({
+            'type': 'event',
+            "channel": "system",
+            "content": {
+                'system': {
+                    'action': 'init_module',
+                    'status': status,
+                    'module': 'mitm_proxy'
+                }
+            }
+        })
+
     def run(self, queue, config, *args, **kwargs):
         # Init logger
         log.init(config)
         logger = log.get_logger()
         mitm_path = kwargs.get('mitm_path')
-        self.start_mitmdump(config, logger, mitm_path)
+        self.start_mitmdump(queue, config, logger, mitm_path)
 
 
 class UnsupportedPlatform(Exception):

lyrebird/mock/context.py

@@ -121,7 +121,7 @@ def save(self):
 Because of iview table has render preformance problem
 We need to limit render time
 """
-EMIT_INTERVAL = 0.4
+EMIT_INTERVAL = 0.25
 last_emit_time = {}
 
 

lyrebird/mock/extra_mock_server/__init__.py

@@ -1,28 +1,12 @@
-
-import multiprocessing
-
-from lyrebird import application
 from lyrebird.log import get_logger
-from .server import serve
-
+from .server import serve, publish_init_status
+from lyrebird.base_server import ProcessServer
 
 logger = get_logger()
 
 
-class ExtraMockServer():
-    def __init__(self) -> None:
-        self._server_process = None
-
-    def start(self):
-        self._server_process = multiprocessing.Process(
-            group=None,
-            daemon=True,
-            target=serve,
-            kwargs={'config': application.config.raw()})
-        self._server_process.start()
+class ExtraMockServer(ProcessServer):
 
-    def stop(self):
-        if self._server_process:
-            self._server_process.terminate()
-            logger.warning(f'MockServer shutdown')
-            self._server_process = None
+    def run(self, queue, config, *args, **kwargs):
+        publish_init_status(queue, 'READY')
+        serve(queue, config, *args, **kwargs)

lyrebird/mock/extra_mock_server/server.py

@@ -175,16 +175,29 @@ def _cancel_tasks(
                 }
             )
 
-
-def serve(config):
+def publish_init_status(queue, status):
+    queue.put({
+        'type': 'event',
+        "channel": "system",
+        "content": {
+            'system': {
+                'action': 'init_module',
+                'status': status,
+                'module': 'extra_mock'
+            }
+        }
+    })
+
+
+def serve(queue, config, *args, **kwargs):
     loop = asyncio.new_event_loop()
     main_task = loop.create_task(_run_app(config))
 
     try:
         asyncio.set_event_loop(loop)
         loop.run_until_complete(main_task)
     except KeyboardInterrupt:
-        pass
+        publish_init_status(queue, 'ERROR')
     finally:
         _cancel_tasks({main_task}, loop)
         _cancel_tasks(asyncio.all_tasks(loop), loop)

requirements.txt

@@ -7,7 +7,7 @@ packaging==19.0
 Pillow==9.1.1
 portpicker==1.3.1
 qrcode==7.3.1
-requests==2.21.0
+requests==2.25.1
 SQLAlchemy==1.3.22
 aiohttp==3.8.1
 netifaces==0.11.0

requirements.txt.lock

@@ -33,7 +33,7 @@ python-engineio==4.3.4
 python-socketio==5.7.1
 pytz==2022.2.1
 qrcode==7.3.1
-requests==2.21.0
+requests==2.25.1
 six==1.16.0
 soupsieve==2.3.2.post1
 SQLAlchemy==1.3.22