Merge pull request open-lambda#120 from open-lambda/testing2

use pandas tests inside SOCK containers as OL tests

Author: tylerharter

Makefile

@@ -91,7 +91,7 @@ check-fmt:
 	cd bin-functions && cargo fmt --check
 
 lint:
-	pylint scripts --ignore=build --disable=missing-docstring,multiple-imports,global-statement,invalid-name,W0511,W1510
+	pylint scripts --ignore=build --disable=missing-docstring,multiple-imports,global-statement,invalid-name,W0511,W1510,R0801
 	cd wasm-worker && cargo clippy
 	cd bin-functions && cargo clippy
 

scripts/helper/test.py

@@ -100,11 +100,9 @@ def _wrapper(*args, **kwargs):
         mounts0 = mounts()
         worker = None
 
-        try:
-            worker = WORKER_TYPE()
-            print("Worker started")
-        except Exception as err:
-            print(f"Failed to start worker: {err}")
+        worker = WORKER_TYPE()
+        assert worker
+        print("Worker started")
 
         if worker:
             try:

scripts/package_tests.py

@@ -0,0 +1,71 @@
+#! /bin/env python3
+
+'''Many popular PyPI packages (e.g., pandas, scikit-learn, etc) have
+extensive tests.  We want those packages to work on OL, so we run
+these inside the SOCK containers.  Some of these suites take quite
+long (e.g., 20+ minutes), so we set large timeouts.  Very high memory
+limits (>1GB) are also necessary.
+'''
+
+import argparse
+import os
+
+from helper import DockerWorker, SockWorker, prepare_open_lambda, setup_config
+from helper import TestConfContext, assert_eq
+
+from helper.test import set_test_filter, start_tests, check_test_results, set_worker_type, test
+
+from open_lambda import OpenLambda
+
+# These will be set by argparse in main()
+OL_DIR = None
+
+@test
+def pandas_test():
+    open_lambda = OpenLambda()
+    result = open_lambda.run("pandas-tests", None)
+    assert_eq(result, True)
+
+def run_tests():
+    with TestConfContext(mem_pool_mb=6000, limits={"mem_mb": 2000, "max_runtime_default": 3600}):
+        pandas_test()
+
+def main():
+    global OL_DIR
+
+    parser = argparse.ArgumentParser(description='Run tests for OpenLambda')
+    parser.add_argument('--reuse_config', action="store_true")
+    parser.add_argument('--worker_type', type=str, default="sock")
+    parser.add_argument('--test_filter', type=str, default="")
+    parser.add_argument('--registry', type=str, default="test-registry")
+    parser.add_argument('--ol_dir', type=str, default="test-dir")
+
+    args = parser.parse_args()
+
+    set_test_filter([name for name in args.test_filter.split(",") if name != ''])
+    OL_DIR = args.ol_dir
+
+    setup_config(args.ol_dir)
+    prepare_open_lambda(args.ol_dir)
+
+    trace_config = {
+        "cgroups": True,
+        "memory": True,
+        "evictor": True,
+        "package": True,
+    }
+    with TestConfContext(registry=os.path.abspath(args.registry), trace=trace_config):
+        if args.worker_type == 'docker':
+            set_worker_type(DockerWorker)
+        elif args.worker_type == 'sock':
+            set_worker_type(SockWorker)
+        else:
+            raise RuntimeError(f"Invalid worker type {args.worker_type}")
+
+        start_tests()
+        run_tests()
+
+    check_test_results()
+
+if __name__ == '__main__':
+    main()

test-registry/pandas-tests/f.py

@@ -0,0 +1,45 @@
+# ol-install: pandas,pytest,hypothesis,exceptiongroup
+# TODO: why isn't exceptiongroup automatically installed?
+
+import numpy
+import pandas
+import pytest
+import inspect
+import os, sys
+import subprocess
+import time
+
+def f(event):
+    print("redirecting to stdout.txt, stderr.txt")
+    sys.stdout.close()
+    sys.stderr.close()
+    sys.stdout = open("/tmp/stdout.txt", 'w', 1)
+    sys.stderr = open("/tmp/stderr.txt", 'w', 1)
+
+    pkg = os.path.dirname(pandas.__file__)
+    print(pkg)
+
+    # TODO: look into these to determine whether it's a problem
+    tests_that_fail = [
+        "test_oo_optimizable",
+        "test_oo_optimized_datetime_index_unpickle",
+        "test_missing_required_dependency",
+        "test_util_in_top_level",
+        "test_raw_roundtrip",
+        "test_get_handle_with_path",
+        "test_with_missing_lzma",
+        "test_with_missing_lzma_runtime",
+        "test_multi_thread_string_io_read_csv",
+        "test_multi_thread_path_multipart_read_csv",
+        "test_server_and_default_headers",
+        "test_server_and_custom_headers",
+        "test_server_and_all_custom_headers",
+        "TestS3",
+        "s3",
+    ]
+
+    cmd = [pkg, "-o", "cache_dir=/tmp/.my_cache_dir", "-m", "(not slow)"]
+    cmd.extend(["-k", " and ".join([f"(not {name})" for name in tests_that_fail])])
+    print(" ".join(cmd))
+    result = pytest.main(cmd)
+    return result == pytest.ExitCode.OK