Move tests outside ot coffea_casa (no intention to add them as a part…

… of distribution)

tests/ca.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIUHfc6OUT6HK590ABdtsRzglDZsfowDQYJKoZIhvcNAQEL
+BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMDA1MjUxNDQ5MzlaFw0zMDA1
+MjMxNDQ5MzlaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
+HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggIiMA0GCSqGSIb3DQEB
+AQUAA4ICDwAwggIKAoICAQCb5PcZthd/Ssv4JpsxGUww9KsA+HknTQVAWUN85bC4
+BxLS8IKaE302yGBg8MK9w0z+qTOoMFNMqoo+Yqp7sDGbV/4U9qVPuJfbxddLPyQU
+QenGXsyAMLEmVXCmQHXlcZCsIlY0je6c7o2bJw2oJPPktjMRNfHI+lWtzhMVOhWz
+nKRMwikfaV80Vn80Jx7zY2A2kcsoGL16Y/GLp4AAhEGWtn3GFJDCCWpJM4fBeChZ
+6wOxP/KnNsJZ2PeHb+yTXT347ZFRdKHB6DNDmiWk48BrPgwemKdePt+7YN1UmV88
+YCl54BecgGiu3Iav0gqCHv2JIDZtyUtfBA/rPXsUmRbus0ZW9ASCnVEbt9FtHQZ+
+e/BmLL6WDg58lGE4TeI+EaK1FqxbVEB3K5idrForTZvadKzf2licSpRP+wlRruhZ
+k85lT50aY5tvQIPq+FnX2mQwosMln0hIPXvgqXDMNrSCzUZ9aCLL6PWx+wqLHq9p
+dU/2upPM9AEiPh1M9nEXL+FdbQMB6wNbpo/L3yTfQ1pL3Z33KLDC9OfWCGymgarL
+1uxeUbt6XeCB7Exb+1fT73RtjeO2QFTZrCjKfxR3vKIBA0/2qb0DfktAasekyx3U
+TC2ZhT/wNP4LBP/l1Oasn47e6EPs2slZLPHZXfbdElsBGcGPDwh38RjSVq+fzm7b
+bwIDAQABo1MwUTAdBgNVHQ4EFgQU+UmkXa3pmOtN4bN3LEm6Vbc2KVMwHwYDVR0j
+BBgwFoAU+UmkXa3pmOtN4bN3LEm6Vbc2KVMwDwYDVR0TAQH/BAUwAwEB/zANBgkq
+hkiG9w0BAQsFAAOCAgEAMNWu/3ci9SxiZ2YO/x4VBtvuhB2D0eAF71NXdJW+F2WN
+ol+ut2mHyXeqJTqme6nwMkItuCk/M3/rtxJYa+i77PkRP+dv10jfOLNgc0nMTCKx
+ll6AlhX3Mz4kt4/8g1h6oyee/Tau4VK9UTLzlxHiPSX4HrPR1g8jEyA5K9pwfX0p
+RBLAlaStyemH/HkCX0HpG8PSJfEbmLsbJR5cOKtGAgt+mUnPjKGvuWOkohrOzFci
+dMaWCULZX6PEdeJaXNerP7mR/BH3nOHSWja1zqr4t4h4e7hOXREseUrvUB9pHNkv
+T2UrbbBMGElHVlHtxtaxdu/zNzL1DBxBkLjyw+SberFFQAQR5I0yCWOhrwzONzH6
+XPL81igvUao5BQ7quJpy58EPqGwALAkB5QDsvH4uD7Bgwp8IYAtcRWWGL8WnuEt5
+shS4yK4UXFCgUeg580+gZ7t1nH6f55/A7d/g0DdpvBBTZVYRJFpiJCnCTH6LbjI0
+XXe5vDubTxjLSZemP4w+w+Oe18U2tJ6DXb2rUZGoWECnfbYfVmO2hdLcgp2nenXl
+dcRANXfZT9R34wzVx/LUJA9hjOgfFF38CC2RfOZr9bb1IjLD/X4V5SrxsfgnQ4P/
+/RKGyQ5dpeK/lpdgYJuqMgUffy8KDrQdMaBq8+0RdVL3nm5+tPFKcskMI2f/zyc=
+-----END CERTIFICATE-----

tests/key.pem

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCb5PcZthd/Ssv4
+JpsxGUww9KsA+HknTQVAWUN85bC4BxLS8IKaE302yGBg8MK9w0z+qTOoMFNMqoo+
+Yqp7sDGbV/4U9qVPuJfbxddLPyQUQenGXsyAMLEmVXCmQHXlcZCsIlY0je6c7o2b
+Jw2oJPPktjMRNfHI+lWtzhMVOhWznKRMwikfaV80Vn80Jx7zY2A2kcsoGL16Y/GL
+p4AAhEGWtn3GFJDCCWpJM4fBeChZ6wOxP/KnNsJZ2PeHb+yTXT347ZFRdKHB6DND
+miWk48BrPgwemKdePt+7YN1UmV88YCl54BecgGiu3Iav0gqCHv2JIDZtyUtfBA/r
+PXsUmRbus0ZW9ASCnVEbt9FtHQZ+e/BmLL6WDg58lGE4TeI+EaK1FqxbVEB3K5id
+rForTZvadKzf2licSpRP+wlRruhZk85lT50aY5tvQIPq+FnX2mQwosMln0hIPXvg
+qXDMNrSCzUZ9aCLL6PWx+wqLHq9pdU/2upPM9AEiPh1M9nEXL+FdbQMB6wNbpo/L
+3yTfQ1pL3Z33KLDC9OfWCGymgarL1uxeUbt6XeCB7Exb+1fT73RtjeO2QFTZrCjK
+fxR3vKIBA0/2qb0DfktAasekyx3UTC2ZhT/wNP4LBP/l1Oasn47e6EPs2slZLPHZ
+XfbdElsBGcGPDwh38RjSVq+fzm7bbwIDAQABAoICAHFc4ddfk9yr3oEYSdg9Zitf
+cA6noSpUFtKBVtM3D/fypNyhqscyDubMdVFpIqPtpkq1bewLIDfq99Z/1ytUp+4n
+4YsLBJFhUYSubG26f5j/iWkIPLunLNsMXHt4+oKbv7F80qUq5O5Xhr/heUvhez3A
+xIfqa2VTrQRTi4rvDyLqcIuk0VSXQnUDxUJ+hEJG7IsiH9KLkxWyIc8FQc6eXjej
+gviMset3/0M15q1onCcvACNftiukZVYCsZVabXWH423mC7tpDcu897JcIK20NJOH
+rjZ9mY+uNvHCcZB4a0mzP9XxgBn9QqKNmJ+4JI4UzRdvRkU1kMqKYK0Wqy8CWCqf
+kw7FrRb2AO94bK3CMQ+9G0AJCLD4tlh/Xw9DIh4h2iv3G3T9GVmH0fLnF2LDKy2G
+7Z0oC65s3TjeO2a9SRvlgEmf28EujjCil0RZx6Dnni171GVARohdLZaGDquj5V26
+Sc2iTaHDXKcRCyntORJ7hyJt7PsCU7nj4dMLc4q35mW4Finxt0Is5w+IGIL+fKEI
+0ZrTf6eBRNU4rLagkiCmhBydmJaj8jMfl7XVHXEoOT2AR0dk8XfIK/LO9d/5zTJA
+y1Wx9THg5wUK2KoimQXg+qZFw06fEjlbzEl0DlZOdOrk2O1lX+cYrW0pbUH2OKOh
+TGjMN3f45sCfDEvHQg35AoIBAQDPTDnv7ZiKDI5vpl6+uNnVQRwlKQ+QrD+FxrW3
+52b23eVcPP7kcZ4xDeJ0T6ZdGGJ41SRKUn6gsojQ02yReSn02/BZVmL/shEweXhM
+VrHmMbwOUYGwcVmp9jjSTH4VVEl42oMpc8JCrEAKAda09ZuG4+PbFOtGBskWnBNE
+uhJHlgL5CRODLmy+jZvafsTLHIqfDlq18jxyPz0SsaXGcV5Lm2Ob6DPRKvp8578X
+ehwpJcJMvqAmzwflgVj+Qb2K0/ceDvjajC3BMcp63gjCAojTbbHrIrXzS0ClSjUg
+U8sfUaXe0wOAqsKIl+evQtNwiv/2QMmvC1jHI7miMeu7MSJtAoIBAQDAhR7uAzJc
+quFtjfbwCJswzAsQ8W4YOioTLWFW2W/+nS4MfFlxFEuOeJpvYmViD67UFJrCY9bO
+xXyQvTQPN1iVFAjqG3Ur40/+zFGm1uXR0rF2f0oafceKodT2+TyDcoLkVoi2h93k
+PfgCsen+ajwXYNH4utHN9QoykRKESzwWtJOcY2vBWxItFmumFborv/6oJkCMXWC8
+o5wjIJHXZ9q/bxvNNnENnXafbYWyKlwu6f0CXeTd4Dt4zD7Ab861/F0zP4GXlrR/
+ecPhqxaS5U51VfOZp4q5Ey8HbWxVENMA9z+4dp7P+/A2WLLdyLz23tYo7wDmpqQ0
+CUZ2YNeUSWvLAoIBAQCLxczm7uBEbNT5iUcm+ALA3Mb6c2YwWUP3kpVia9+sItAM
+0n/XTpioYMFJRY9aBCAZczWl+1uwRNElZPk2WWkl1cqIokvcNpeKhMzNRENtgClZ
+yjFU5AjeJcwIWFVHUm670zJPF+NrCzOey8CWgWidmjk/tioxLFAYM6J2W7QJmqdk
+fW8vq3TdQyRMPd+5SARb3NTjC3MgYW1vlmK9nCFFf3+5VubhaUY+RBA/5zDnubL6
+Bip8IGoloIJ95ZvE6Mkd9mBrE8uiEU2CbQWgsw7I6JTng58Fbb8n9BJAOt+hvW7H
+AKbC8eB7M1mffcKNhtuxkdurcE4q7/ax21EkBaw9AoIBAHdA/svIxyWH5GQMkG5X
+pmovupsgMmZngTCn56f4wNsjWib50B2vyK3UHzXn6Y040b8llEfduG4U/vhZeyoB
+yqlt46fAonAxOphG0D1c2LeEn9EbQDfwue4yGM1zzfxOrq3qvHz05IpBqKNiueOS
+wu5oVyiP8O53X327R5ETWYFnEhjJrTH0y+mJ/dy/kLcRExntuAY6wXWYk1tfDXg1
+KNd0Z/BSTO12IMjY+vxGKRwWbVdN+jtGbxCA1E438//e94yLRic0f1KHhsL/S9hq
+mpMsTt1bXx8NtxAOxBBdf5cVkS8eq3mCQmYnw4SGmCcEGfz5L8Gwb/6b0D379w4v
+/MkCggEAEwIOqhgwUjscb1L9NTehFa3CaK8eQwkEvB7aQLhTbTUK4bigSd6f20Ju
+3N/zmHhSkEm6Oq9+zrjm8QqgztrB3Zn+nCWu2n7BHFoCexg4Qov3xuKsC3MMh/oa
+/xqgVgsFOgqk93YWv/6IvCHM8F3Ntk+rcUlt0M1VxXC8Ym/vRHy5VN12asI19K1M
+UbZINK0bVAdEdwaB2OSx14yag9uiaXxKryZe7qTjcR2ELxksVo0nwX7h0NHA1S77
+Ubq57jxR5a/wI9VA5iTUZSUkjxfEdPUEAY1eDdp+Ogqcs1Zsa8Er0JUNPxqJzmlJ
+Q2ypwp0CapUz0d43a47/izrjAtXzug==
+-----END PRIVATE KEY-----

tests/test_coffeacasa.py

@@ -0,0 +1,117 @@
+import os
+import sys
+import socket
+import pytest
+from distributed.security import Security
+from distributed import Client
+from coffea_casa import CoffeaCasaCluster
+
+os.environ["HOST_IP"] = socket.gethostbyname(socket.gethostname())
+
+
+def test_header():
+    with CoffeaCasaCluster(cores=1,
+                           memory="100MB",
+                           disk="100MB",
+                           worker_image="coffeateam/coffea-casa:latest"
+                           ) as cluster:
+        job_script = cluster.job_script()
+        print("HTCondor Job script:", job_script)
+        print("Scheduler specs:", cluster.scheduler_spec)
+        kwargs = CoffeaCasaCluster._modify_job_kwargs({})
+        print("CoffeaCasaCluster arguments:", kwargs)
+        assert cluster._dummy_job.job_header_dict["MY.DaskWorkerCores"] == 1
+        assert cluster._dummy_job.job_header_dict["MY.DaskWorkerDisk"] == 100000000
+        assert cluster._dummy_job.job_header_dict["MY.DaskWorkerMemory"] == 100000000
+
+
+def test_job_script():
+    with CoffeaCasaCluster(cores=4,
+                           processes=4,
+                           memory="500MB",
+                           disk="500MB",
+                           worker_image="coffeateam/coffea-casa:latest",
+                           env_extra=['export LANG="en_US.utf8"',
+                                      'export LC_ALL="en_US.utf8"'],
+                           job_extra={"+Extra": "True"},
+                           ) as cluster:
+        job_script = cluster.job_script()
+        print("HTCondor Job script:", job_script)
+        print("Scheduler specs:", cluster.scheduler_spec)
+        assert "RequestCpus = MY.DaskWorkerCores" in job_script
+        assert "RequestDisk = floor(MY.DaskWorkerDisk / 1024)" in job_script
+        assert "RequestMemory = floor(MY.DaskWorkerMemory / 1048576)" in job_script
+        assert "MY.DaskWorkerCores = 4" in job_script
+        assert "MY.DaskWorkerDisk = 500000000" in job_script
+        assert "MY.DaskWorkerMemory = 500000000" in job_script
+        assert 'MY.JobId = "$(ClusterId).$(ProcId)"' in job_script
+        assert "LANG=en_US.utf8" in job_script
+        assert "LC_ALL=en_US.utf8" in job_script
+        assert "export" not in job_script
+        assert "+Extra = True" in job_script
+        assert (
+            "{} -m distributed.cli.dask_worker tcp://".format(sys.executable)
+            in job_script
+        )
+        assert "--memory-limit 125.00MB" in job_script
+        assert "--nthreads 1" in job_script
+        assert "--nprocs 4" in job_script
+
+
+def test_scheduler():
+    with CoffeaCasaCluster(cores=1,
+                           memory="100MB",
+                           disk="100MB",
+                           worker_image="coffeateam/coffea-casa:latest",
+                           scheduler_options={
+                               "dashboard_address": 8786,
+                               "port": 8788}
+                           ) as cluster:
+        job_script = cluster.job_script()
+        print("HTCondor Job script:", job_script)
+        print("Scheduler specs:", cluster.scheduler_spec)
+        kwargs = CoffeaCasaCluster._modify_job_kwargs({})
+        print("CoffeaCasaCluster arguments:", kwargs)
+        #assert cluster.scheduler_spec["port"] == 8788
+        expected = os.environ["HOST_IP"]
+        assert expected in str(cluster.scheduler_spec)
+        #cluster.scale(1)
+        #assert expected in str(cluster.worker_spec)
+
+
+def test_security():
+    dirname = os.path.dirname(__file__)
+    key = os.path.join(dirname, "key.pem")
+    cert = os.path.join(dirname, "ca.pem")
+    security = Security(
+        tls_ca_file=cert,
+        tls_scheduler_key=key,
+        tls_scheduler_cert=cert,
+        tls_worker_key=key,
+        tls_worker_cert=cert,
+        tls_client_key=key,
+        tls_client_cert=cert,
+        require_encryption=True,
+    )
+
+    with CoffeaCasaCluster(cores=1,
+                           memory="100MB",
+                           disk="100MB",
+                           worker_image="coffeateam/coffea-casa:latest",
+                           security=security) as cluster:
+        job_script = cluster.job_script()
+        print("HTCondor JobAd script:", job_script)
+        print("Scheduler specs:", cluster.scheduler_spec)
+        kwargs = CoffeaCasaCluster._modify_job_kwargs({})
+        print("CoffeaCasaCluster arguments:", kwargs)
+        assert cluster.security == 'mioa'
+        assert cluster.security == security
+        assert cluster.scheduler_spec["options"]["security"] == security
+        assert "--tls-key {}".format(key) in job_script
+        assert "--tls-cert {}".format(cert) in job_script
+        assert "--tls-ca-file {}".format(cert) in job_script
+        #cluster.scale(jobs=1)
+        #with Client(cluster, security=security) as client:
+        #    future = client.submit(lambda x: x + 1, 10)
+        #    result = future.result()
+        #    assert result == 11

tests/test_coffeacasa_args.py

@@ -0,0 +1,48 @@
+import pytest
+import os
+from pathlib import Path
+from coffea_casa import CoffeaCasaCluster
+from distributed.security import Security
+
+# Security settings for coffea-casa Dask scheduler
+SECRETS_DIR = Path('/etc/cmsaf-secrets')
+CA_FILE = SECRETS_DIR / "ca.pem"
+CERT_FILE = SECRETS_DIR / "hostcert.pem"
+
+@pytest.fixture
+def default_modified_kwargs():
+    return CoffeaCasaCluster._modify_job_kwargs({})
+
+
+def test_default_dask_container_port(default_modified_kwargs):
+    assert default_modified_kwargs["job_extra"]["dask_container_port"] == 8787
+
+
+def test_can_override_dask_container_port():
+    kwargs = CoffeaCasaCluster._modify_job_kwargs(dict(job_extra={"dask_container_port": 8788}))
+    assert kwargs["job_extra"]["dask_container_port"] == 8788
+
+
+def test_default_protocol(default_modified_kwargs):
+    if CA_FILE.is_file() and CERT_FILE.is_file():
+        assert default_modified_kwargs["scheduler_options"]["protocol"] == 'tls'
+    else:
+        assert default_modified_kwargs["scheduler_options"]["protocol"] == 'tcp'
+
+
+def test_can_override_security():
+    dirname = os.path.dirname(__file__)
+    key = os.path.join(dirname, "key.pem")
+    cert = os.path.join(dirname, "ca.pem")
+    security = Security(
+        tls_ca_file=cert,
+        tls_scheduler_key=key,
+        tls_scheduler_cert=cert,
+        tls_worker_key=key,
+        tls_worker_cert=cert,
+        tls_client_key=key,
+        tls_client_cert=cert,
+        require_encryption=True)
+    kwargs = CoffeaCasaCluster._modify_job_kwargs(dict(security=security))
+    assert kwargs["security"] == security
+    assert kwargs["scheduler_options"]["protocol"] == 'tls'

tests/test_config.py

@@ -0,0 +1,7 @@
+import dask
+import pytest
+import coffea_casa.config
+
+
+def test_dask_config():
+    assert dask.config.get('jobqueue.coffea-casa.worker-image') == "coffeateam/coffea-casa:latest"