bug fix

Author: jerrychen1990

snippets/decorators.py

@@ -167,7 +167,7 @@ def wrapped(data: Iterable, *args, **kwargs):
             # add a thread pool here
             executors = ThreadPoolExecutor(work_num)
 
-            def _func(x):
+            def _func(x):                
                 return func(x, *args, **kwargs)
             rs_iter = executors.map(_func, data)
             total = None if not hasattr(data, '__len__') else len(data)

snippets/logs.py

@@ -13,60 +13,78 @@
 logger = logging.getLogger(__name__)
 
 
-def getlog(env, name):
+def getlog(env:str, name:str):
     exist = name in logging.Logger.manager.loggerDict
     rs_logger = logging.getLogger(name)
     if not exist:
         logger.info(f"create logger with {env=}, {name=}")
-        if env in ["dev", "local"]:
+        if env.lower() in ["dev", "local"]:
             rs_logger.propagate = False
             rs_logger.setLevel(logging.DEBUG)
-            streamHandler = logging.StreamHandler()
-            streamHandler.setFormatter(fmt=logging.Formatter(
+            stream_handler = logging.StreamHandler()
+            stream_handler.setFormatter(fmt=logging.Formatter(
                 "%(asctime)s [%(levelname)s][%(filename)s:%(lineno)d]:%(message)s", datefmt='%Y-%m-%d %H:%M:%S'))
-            rs_logger.addHandler(streamHandler)
+            rs_logger.addHandler(stream_handler)
         else:
             rs_logger.propagate = False
             rs_logger.setLevel(logging.INFO)
-            streamHandler = logging.StreamHandler()
-            streamHandler.setFormatter(fmt=logging.Formatter(
+            stream_handler = logging.StreamHandler()
+            stream_handler.setFormatter(fmt=logging.Formatter(
                 "%(asctime)s [%(levelname)s]%(message)s", datefmt='%Y-%m-%d-%H:%M:%S'))
-            rs_logger.addHandler(streamHandler)
+            rs_logger.addHandler(stream_handler)
 
     return rs_logger
 
+get_log = getlog
+
 
 _FMT_MAP = {
+    "raw": logging.Formatter("%(message)s"),
     "simple": logging.Formatter(
         "%(asctime)s [%(levelname)s]%(message)s", datefmt='%Y-%m-%d-%H:%M:%S'),
     "detail":   logging.Formatter(
         "%(asctime)s [%(levelname)s][%(filename)s:%(lineno)d]:%(message)s", datefmt='%Y-%m-%d %H:%M:%S')
+    
 
 }
 
 
-def getlog_detail(name, level, format_type: str = "simple", do_print=True, do_file=False, propagate=False,
-                  log_dir: str = None, file_type="time_rotate", file_config=dict(when='d', interval=1, backupCount=7)):
+def getlog_detail(name, level, format_type: str = "simple",
+                  do_print=True, print_format_type=None, print_level=None,
+                  do_file=False, file_format_type=None,  file_level = None, file_type="time_rotate", file_config=dict(when='d', interval=1, backupCount=7),
+                  propagate=False, log_dir: str = None,):
     exist = name in logging.Logger.manager.loggerDict
     rs_logger = logging.getLogger(name)
     if exist:
         return rs_logger
-    fmt = _FMT_MAP[format_type]
-
+    
     rs_logger.propagate = propagate
     rs_logger.setLevel(level)
 
     if do_print:
         streamHandler = logging.StreamHandler()
-        streamHandler.setFormatter(fmt=fmt)
+        streamHandler.setFormatter(fmt=_FMT_MAP[print_format_type if print_format_type else format_type])
+        streamHandler.setLevel(print_level if print_level else level)
         rs_logger.addHandler(streamHandler)
+        
     if do_file:
         log_dir = log_dir or os.environ.get("LOG_DIR", "/tmp/logs")
         file_path = os.path.join(log_dir, name + ".log")
         os.makedirs(log_dir, exist_ok=True)
         if file_type == "time_rotate":
             filehandler = TimedRotatingFileHandler(file_path, **file_config)
             filehandler.suffix = "%Y-%m-%d_%H-%M-%S.log"  # 设置历史文件 后缀
-            filehandler.setFormatter(fmt)
+            filehandler.setFormatter(_FMT_MAP[file_format_type if file_format_type else format_type])
+            filehandler.setLevel(file_level if file_level else level)
             rs_logger.addHandler(filehandler)
     return rs_logger
+
+
+def get_file_log(name, log_dir):
+    return getlog_detail(name=name, format_type="simple",
+                         do_print=True, print_format_type="raw", print_level=logging.INFO,
+                         do_file=True, file_format_type="detail", file_level=logging.DEBUG, log_dir=log_dir)
+
+
+
+

snippets/perf.py

@@ -5,7 +5,7 @@
 from snippets import SNIPPETS_ENV
 from snippets.decorators import batch_process
 from snippets.logs import getlog
-from snippets.utils import create_dir_path, get_current_time_str, jdump, read2list, split_surfix
+from snippets.utils import create_dir_path, get_current_time_str, jdump, read2list, split_suffix
 
 
 logger = getlog(SNIPPETS_ENV, __file__)
@@ -34,25 +34,25 @@ def req_http_service_detail(item, url, build_req_func=default_build_req, build_r
 def perf_test(input_path, url, req_func, output_path=None, work_num=1, max_num=None):
     logger.info("perf starts")
     logger.info(f"input_path: {input_path}, url:{url}, work_num:{work_num}")
-    name, surfix = split_surfix(input_path)
+    name, _ = split_suffix(input_path)
     if not output_path:
         output_path = os.path.join(
             name, f"{get_current_time_str()}.pef{work_num}.jsonl")
     create_dir_path(output_path)
 
-    querys = read2list(input_path)
+    queries = read2list(input_path)
     st = time.time()
     if max_num:
-        querys = querys[:max_num]
+        queries = queries[:max_num]
 
     func = batch_process(work_num=work_num, return_list=True)(req_func)
-    rs = func(data=querys, url=url)
+    rs = func(data=queries, url=url)
     # logger.info(rs)
     cost = time.time() - st
     costs = [e['cost'] for e in rs]
     latency = sum(costs)/len(costs)
 
-    stat = dict(test_cost=cost, latency=latency, test_num=len(querys), qps=len(querys)/cost)
+    stat = dict(test_cost=cost, latency=latency, test_num=len(queries), qps=len(queries)/cost)
     # rs.append(stat)
 
     logger.info(f"dump to {output_path}")

snippets/utils.py

@@ -42,7 +42,7 @@ def default(self, obj):
         if isinstance(obj, set):
             return list(obj)
         if isinstance(obj, BaseModel):
-            return obj.dict(exclude_none=True, exclude_defaults=True)
+            return obj.model_dump(exclude_none=True, exclude_defaults=True)
         if isinstance(obj, datetime):
             return obj.strftime("%Y-%M-%d %H:%m:%S")
         if isinstance(obj, np.integer):
@@ -111,7 +111,7 @@ def jload(fp):
     return rs
 
 
-def split_surfix(filename: str) -> Tuple[str, str]:
+def split_suffix(filename: str) -> Tuple[str, str]:
     name, ext = filename.rsplit(".", 1)
     return name, ext
 
@@ -154,11 +154,11 @@ def get_gen(f):
 # table类的文件转化为list of dict
 
 
-def table2json(path):
+def table2json(path, **kwargs):
     if path.endswith("csv"):
-        df = pd.read_csv(path)
+        df = pd.read_csv(path, **kwargs)
     if path.endswith("xlsx"):
-        df = pd.read_excel(path)
+        df = pd.read_excel(path, **kwargs)
     df.replace(np.nan, None, inplace=True)
     cols = df.columns.tolist()
     cols = [e for e in cols if not e.startswith("Unnamed")]
@@ -197,17 +197,17 @@ def load_lines(fp, return_generator=False):
 def read2list(file_path: Union[str, List], **kwargs) -> List[Union[str, dict]]:
 
     def _read2list(file_path, **kwargs):
-        surfix = os.path.splitext(file_path)[-1].lower()
-        if surfix == ".json":
+        suffix = os.path.splitext(file_path)[-1].lower()
+        if suffix == ".json":
             return jload(file_path, **kwargs)
-        if surfix == ".jsonl":
+        if suffix == ".jsonl":
             return jload_lines(file_path, **kwargs)
-        if surfix in [".xlsx", ".csv"]:
-            return table2json(file_path)
-        if surfix in [".txt"]:
+        if suffix in [".xlsx", ".csv"]:
+            return table2json(file_path, **kwargs)
+        if suffix in [".txt"]:
             return load_lines(file_path, **kwargs)
         else:
-            logger.warn(f"unkown surfix:{surfix}, read as txt")
+            logger.warn(f"unknown suffix:{suffix}, read as txt")
             return load_lines(file_path, **kwargs)
 
     if isinstance(file_path, list):