Remote Bazefetcher source

requirements-dev.txt

@@ -1,5 +1,6 @@
-pytest==3.10.1
-pytest-repeat
 hypothesis
-scikit-build
+mock-ssh-server
 pybind11
+pytest-repeat
+pytest==3.10.1
+scikit-build

requirements.txt

@@ -1,6 +1,7 @@
 numpy
 pandas>=1.1.0,<1.2.0
-scipy
+paramiko
+pytz
 rainflow==2.2.0
 requests
-pytz
+scipy

tests/source/test_bazefetcher.py

@@ -1,9 +1,13 @@
 #!/usr/bin/env python3
 from camille.source import Bazefetcher, TagNotFoundError
+from camille.source.bazefetcher import RemoteIO
 from camille.source.bazefetcher import _get_files_between_start_and_end
 from datetime import datetime, timedelta
 from math import pi
 from pytz import utc
+from unittest import mock
+import contextlib
+import mockssh
 import numpy as np
 import pandas as pd
 import pytest
@@ -13,6 +17,26 @@
 baze = Bazefetcher('tests/test_data/baze')
 non_standard = Bazefetcher('tests/test_data/non_standard_names')
 
+
+@contextlib.contextmanager
+def mock_remote_ssh():
+    with contextlib.ExitStack() as exit_stack:
+        users = {'ssh-user': 'tests/test_data/ssh/id_rsa'}
+        server = exit_stack.enter_context(mockssh.Server(users))
+        client = exit_stack.enter_context(server.client('ssh-user'))
+
+        connect_mock = exit_stack.enter_context(mock.patch(
+            'camille.source.bazefetcher.RemoteIO._create_connection'
+        ))
+        connect_mock.return_value = client
+
+        #disable disconnect
+        exit_stack.enter_context(mock.patch(
+            'camille.source.bazefetcher.RemoteIO.__exit__'
+        ))
+        yield
+
+
 t12_31_22 = datetime(2029, 12, 31, 22, tzinfo=utc)
 t12_31_23 = datetime(2029, 12, 31, 23, tzinfo=utc)
 t1_1 = datetime(2030, 1, 1, tzinfo=utc)
@@ -103,12 +127,6 @@ def test_read_outside_timeseries_in_file():
     assert empty_time_series.name == 'value'
     assert empty_time_series.index.name == 'time'
 
-def test_no_directories():
-    with pytest.raises(ValueError) as excinfo:
-        Bazefetcher('tests/test_data/baze/perling')
-    assert ('no file in [\'tests/test_data/baze/perling\'] is a directory'
-            in str(excinfo.value))
-
 def test_non_existing_tag():
     with pytest.raises(TagNotFoundError) as excinfo:
         baze('non-existing-tag', t1_1, t1_1_1)
@@ -183,7 +201,7 @@ def test_snap_both():
 
 def test_many_roots():
     baze_and_authored = Bazefetcher(
-        ['tests/test_data/authored', 'tests/test_data/baze'])
+        paths=['tests/test_data/authored', 'tests/test_data/baze'])
 
     sin_b = baze_and_authored('Sin-T60s-SR01hz', t1_2, t1_4)
     assert len(sin_b) == 17280
@@ -192,31 +210,6 @@ def test_many_roots():
     assert len(i04_status_b) == 5
 
 
-def test_many_roots_same_tag():
-    roots = ['tests/test_data/many_roots/dir'+ str(index)
-             for index in [3, 1, 2]]
-    many_roots = Bazefetcher(roots)
-    tag = "root_tag"
-    root = many_roots(tag, t1_1, t1_5)
-    assert len(root) == 7
-
-    root = many_roots(tag, t1_3, t1_3_21)
-    assert len(root) == 1
-
-    root = many_roots(tag, t1_3, t1_3_21, snap='right')
-    assert len(root) == 2
-
-
-def test_many_roots_same_filename():
-    roots = ['tests/test_data/many_roots/dir'+ str(index)
-             for index in [1, 4]]
-    many_roots = Bazefetcher(roots)
-    with pytest.raises(ValueError) as excinfo:
-        many_roots("root_tag", t1_1, t1_5)
-    assert ('files [\'root_tag_2030-01-03T00.00.00+00.00_2030-01-04T00.00.00'
-            '+00.00.json.gz\'] are not unique' in str(excinfo.value))
-
-
 def test_no_time_boundaries():
     sin_b = baze('Sin-T60s-SR01hz')
     assert len(sin_b) == 34560 # 4 days
@@ -227,9 +220,10 @@ def test_no_time_boundaries():
 
 
 def test_no_unnecessary_files_read():
-    #testing private method to cover the case where unnecessary files were readlllll
-    files = _get_files_between_start_and_end(
-        authored.src_dirs, 'installation-04-status', t1_2, t1_3)
+    # Testing private method to cover the case where unnecessary files were read
+    tag = 'installation-04-status'
+    with authored._tag_protocol(tag) as io:
+        files = _get_files_between_start_and_end(io, tag, t1_2, t1_3)
     assert len(files) == 1
 
 
@@ -256,3 +250,34 @@ def test_load_special_regex_file():
 def test_load_bad_format_file():
     df = baze('bad_format')
     assert df.empty
+
+
+def test_remote_io():
+    path = 'ssh-user@127.0.0.1:tests/test_data/remote'
+    with mock_remote_ssh(), RemoteIO(path) as io:
+            assert io.is_dir()
+            assert not (io / 'not a dir').is_dir()
+            assert sorted(io.iterdir()) == [
+                io / 'hello_world',
+                io / 'not a dir',
+            ]
+            assert (io / 'name').name == 'name'
+            with (io / 'hello_world').open() as f:
+                assert f.read() == b'Hello World!\n'
+
+
+def test_remote_root():
+    with mock_remote_ssh():
+        remote_baze = Bazefetcher('127.0.0.1:tests/test_data/baze')
+
+        sin_b = remote_baze('Sin-T60s-SR01hz', t1_2, t1_4)
+        tan_b = remote_baze('Tan-T60s-SR01hz', t1_2, t1_4)
+        assert len(sin_b) == len(tan_b) == 17280 # 2 days
+        pd.testing.assert_series_equal(
+            sin_b, sin(t1_2, t1_4), check_freq=False)
+        pd.testing.assert_series_equal(
+            tan_b, tan(t1_2, t1_4), check_freq=False)
+        assert sin_b.index[0] == t1_2
+        assert sin_b.index[-1] < t1_4
+        assert (t1_4 - sin_b.index[-1]
+                ).to_pytimedelta() < timedelta(seconds=20)

tests/test_data/remote/hello_world

@@ -0,0 +1 @@
+Hello World!

tests/test_data/ssh/id_rsa

@@ -0,0 +1,38 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAYEAuaQikfcdB8MHO/aIUa75+YkLinqnxlBAJ+pS/TI/UL5t3rLnC0ma
+pIsW+eE7apcwpDsq66biDxVElksHfGByK4b1knmLc4bndGVqoZt/ge7WBO6PgKcIQp29jT
+YgkY+c+HuzvIzAHFx4NIJTNuEbX9MuRoASeAewGcM994jRt0pf74P+5bTyyrRcco5mA27g
+mBUfIqWHgGJuIvsXrViHjwuFZKx5fe1gI+tKLCkPHV+0H8Ga/PWIjTC22+asCdTdcGbQUX
+LkdJ5lvoswEEjzpjm/CjeQxlwu1yS7UzMOPjqC3QM1QezAr87aHXqITGw2L2fb6kxObyEO
+qcFOk7VhurWDoGlcWazrwoPz6A+vWBQrlltl0OpoaaFlYtkWWQ4P3IcuaWg/BNkraSmLYS
+SMCIF43g+LA0qY5R78+ZPT5x/Tru3VTAcX7gU4CCuQ95P4q5n4K9+Ep0e5yQEt3VWIfikD
+aKu/v8X7xzsjchHMAHxOVrx/vmUjD7nZleoeYOa1AAAFkJlZpdaZWaXWAAAAB3NzaC1yc2
+EAAAGBALmkIpH3HQfDBzv2iFGu+fmJC4p6p8ZQQCfqUv0yP1C+bd6y5wtJmqSLFvnhO2qX
+MKQ7Kuum4g8VRJZLB3xgciuG9ZJ5i3OG53RlaqGbf4Hu1gTuj4CnCEKdvY02IJGPnPh7s7
+yMwBxceDSCUzbhG1/TLkaAEngHsBnDPfeI0bdKX++D/uW08sq0XHKOZgNu4JgVHyKlh4Bi
+biL7F61Yh48LhWSseX3tYCPrSiwpDx1ftB/Bmvz1iI0wttvmrAnU3XBm0FFy5HSeZb6LMB
+BI86Y5vwo3kMZcLtcku1MzDj46gt0DNUHswK/O2h16iExsNi9n2+pMTm8hDqnBTpO1Ybq1
+g6BpXFms68KD8+gPr1gUK5ZbZdDqaGmhZWLZFlkOD9yHLmloPwTZK2kpi2EkjAiBeN4Piw
+NKmOUe/PmT0+cf067t1UwHF+4FOAgrkPeT+KuZ+CvfhKdHuckBLd1ViH4pA2irv7/F+8c7
+I3IRzAB8Tla8f75lIw+52ZXqHmDmtQAAAAMBAAEAAAGADxp/hYipX/xKFmLUbHDj/Z1cxw
+N7qvLUhWrXHap85EDaB4heq9htMH3wfJwKjX2Hg5UcxNdrBPciAWhRT5k5MLlmOWiSI/AR
+3LJznnpcQWIBH52ZJ0pvrZvP1DFXLQCWMNE/qlN5PdV4Ii2IP6rI4173IN1Co2lctnvgCs
+ei6Fqwu3ftwRqbtko+S0OyoQzgUP9G7Tayc4uOkmIDZyMQxF6Xl71AlTI+5C08d6NPi4BB
+02OZDIOdfH8+IV3/GT2sMtpLRXPPOPxJ8Jn46G0DfLAe6JaqNQdRo6BSlcjXMFVMwKqXhp
+DSFIvt8gZ+3BUT/h0b82bHl1qNtv7x4l0lhJ3KoXnhQhxkzOLj2/Vamv0HMAVY23natyvl
+k9516QndHpWpXcNKMNymHbgNSP3d/9W8CMYqEgbrvY4ID0aZSzRL7eM0Lby2gFfbEJ+mTU
+uzE1gLakk414Ri5lA42ftjHvHEcW9X1TJFhhRcgSvazv+A/b022x9dLFArXHVijCnxAAAA
+wGP/A1J3neSEdMQ6aek7RlugisJm/L7LV3Gjy480hViVUUhZYbzItatIhq7sSLBL6r3QqW
+/fq2oFckj52uafggjMZ6wfG8v9xK+eq+pU5szBZMyyt9chnkBkCgOWBUTFQfsm3kBTfWW7
+RTJmAFBdjChtWgrsyYqW7busAJPpMUzS6sCluJE/YtDvldI4jPigxI6CMuaOgdEn3nJ+f/
+oaafFZducT9pDMJAa3Ln7x67nudtJl4nXB+6U6XgMonV64IQAAAMEA79/s6zFWfAlvmpy3
+/f4FB5SyEWOyY3NlqFwuOAGt6QXpFrLLwV0kUXo1+XC1/VYcBVaf326zIu6gydvJqSOCdG
+7VJgffU5k3ba4ZjmmtEIfJ+ws4VDCo7Mkq4WKRGT5rX92ZU0z6iKlQ2S2F+md2XtxczB/2
+jLIuQTLJVIdRFgtFknqmy/xrp+HU0VRktc2q9MAEjfpSgd9TScSY4XwQcjamgwGXLrjaVM
+rAjHdCksVOY6wY7J4XISrKwS7WUfRzAAAAwQDGHuNpX3tN3+lsJVKBx8R6wCYYoc1PDhtu
+EKezIWO7vFOVLmkQARSGX0a2Km1cJgnhCULugOVTZE905OabG8Sv74cks/APmZ05Yyoa+4
+oKfqQybecoenV/++nbVvBvpFP/UEhLduBrufw/y91nvVM6dkGXTb0rEqqRyPq8WeEGldhU
+L3tGMUNddHXR4o4n8bBH484/YIhXFtsh8YgCwHMiLoDZMRt7P7Ni/BK9QZ6b4yveaLcUT8
+VGEXljMgPOljcAAAAUamVnbUBBQy1DMDJZVzE3MUxWRFYBAgMEBQYH
+-----END OPENSSH PRIVATE KEY-----

tests/test_data/ssh/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC5pCKR9x0Hwwc79ohRrvn5iQuKeqfGUEAn6lL9Mj9Qvm3esucLSZqkixb54TtqlzCkOyrrpuIPFUSWSwd8YHIrhvWSeYtzhud0ZWqhm3+B7tYE7o+ApwhCnb2NNiCRj5z4e7O8jMAcXHg0glM24Rtf0y5GgBJ4B7AZwz33iNG3Sl/vg/7ltPLKtFxyjmYDbuCYFR8ipYeAYm4i+xetWIePC4VkrHl97WAj60osKQ8dX7QfwZr89YiNMLbb5qwJ1N1wZtBRcuR0nmW+izAQSPOmOb8KN5DGXC7XJLtTMw4+OoLdAzVB7MCvztodeohMbDYvZ9vqTE5vIQ6pwU6TtWG6tYOgaVxZrOvCg/PoD69YFCuWW2XQ6mhpoWVi2RZZDg/chy5paD8E2StpKYthJIwIgXjeD4sDSpjlHvz5k9PnH9Ou7dVMBxfuBTgIK5D3k/irmfgr34SnR7nJAS3dVYh+KQNoq7+/xfvHOyNyEcwAfE5WvH++ZSMPudmV6h5g5rU= jegm@AC-C02YW171LVDV