Merge pull request dandi#804 from dandi/public-instance

Make `--dandi-instance` public and add "instances" command

.github/workflows/test.yml

@@ -39,9 +39,6 @@ jobs:
           - os: ubuntu-18.04
             python: 3.7
             mode: dev-deps
-          - os: ubuntu-18.04
-            python: 3.7
-            mode: dandi-devel
         exclude:
           # Temporarily disabled due to h5py/hdf5 dependency issue
           # See <https://github.com/dandi/dandi-cli/pull/315>
@@ -80,10 +77,6 @@ jobs:
       run: |
         pip install git+https://github.com/NeurodataWithoutBorders/pynwb
 
-    - name: Set DANDI_DEVEL=1
-      if: matrix.mode == 'dandi-devel'
-      run: echo DANDI_DEVEL=1 >> "$GITHUB_ENV"
-
     - name: Run all tests
       if: matrix.mode != 'dandi-api' && github.event_name != 'schedule'
       run: |

dandi/cli/base.py

@@ -47,15 +47,15 @@ def dandiset_path_option(**kwargs):
     )
 
 
-def instance_option():
-    return devel_option(
-        "-i",
-        "--dandi-instance",
-        help="For development: DANDI instance to use",
-        type=click.Choice(sorted(known_instances)),
-        default="dandi",
-        show_default=True,
-    )
+def instance_option(**kwargs):
+    params = {
+        "help": "DANDI instance to use",
+        "type": click.Choice(sorted(known_instances)),
+        "default": "dandi",
+        "show_default": True,
+    }
+    params.update(kwargs)
+    return click.option("-i", "--dandi-instance", **params)
 
 
 def devel_debug_option():

dandi/cli/cmd_delete.py

@@ -9,7 +9,7 @@
 @instance_option()
 @devel_debug_option()
 @map_to_click_exceptions
-def delete(paths, skip_missing, dandi_instance="dandi", devel_debug=False):
+def delete(paths, skip_missing, dandi_instance, devel_debug=False):
     """Delete dandisets and assets from the server.
 
     PATH could be a local path or a URL to an asset, directory, or an entire

dandi/cli/cmd_download.py

@@ -76,7 +76,7 @@ def get_metavar(self, param):
 @click.option(
     "--sync", is_flag=True, help="Delete local assets that do not exist on the server"
 )
-@instance_option()
+@instance_option(default=None)
 # Might be a cool feature, not unlike verifying a checksum, we verify that
 # downloaded file passes the validator, and if not -- alert
 # @click.option(
@@ -94,7 +94,7 @@ def get_metavar(self, param):
 @click.argument("url", nargs=-1)
 @map_to_click_exceptions
 def download(
-    url, output_dir, existing, jobs, format, download_types, sync, dandi_instance=None
+    url, output_dir, existing, jobs, format, download_types, sync, dandi_instance
 ):
     """Download a file or entire folder from DANDI"""
     # We need to import the download module rather than the download function

dandi/cli/cmd_instances.py

@@ -0,0 +1,18 @@
+import sys
+
+import click
+import ruamel.yaml
+
+from .base import map_to_click_exceptions
+from ..consts import known_instances
+
+
+@click.command()
+@map_to_click_exceptions
+def instances():
+    """List known Dandi Archive instances that the CLI can interact with"""
+    yaml = ruamel.yaml.YAML(typ="safe")
+    yaml.default_flow_style = False
+    # Convert _asdict() with dict() so that ruamel doesn't tag the results with
+    # "!!omap" on Python 3.7
+    yaml.dump({k: dict(v._asdict()) for k, v in known_instances.items()}, sys.stdout)

dandi/cli/cmd_upload.py

@@ -84,11 +84,11 @@ def upload(
     paths,
     jobs,
     sync,
+    dandi_instance,
     existing="refresh",
     validation="require",
     dandiset_path=None,
     # Development options should come as kwargs
-    dandi_instance="dandi",
     allow_any_path=False,
     upload_dandiset_metadata=False,
     devel_debug=False,

dandi/cli/command.py

@@ -137,21 +137,23 @@ def main(ctx, log_level, pdb=False):
 from .cmd_delete import delete  # noqa: E402
 from .cmd_digest import digest  # noqa: E402
 from .cmd_download import download  # noqa: E402
+from .cmd_instances import instances  # noqa: E402
 from .cmd_ls import ls  # noqa: E402
 from .cmd_organize import organize  # noqa: E402
 from .cmd_shell_completion import shell_completion  # noqa: E402
 from .cmd_upload import upload  # noqa: E402
 from .cmd_validate import validate  # noqa: E402
 
 __all_commands__ = (
+    delete,
+    digest,
+    download,
+    instances,
     ls,
     organize,
+    shell_completion,
     upload,
-    download,
     validate,
-    digest,
-    delete,
-    shell_completion,
 )
 
 for cmd in __all_commands__:

dandi/cli/tests/test_download.py

@@ -3,7 +3,6 @@
 
 import click
 from click.testing import CliRunner
-import pytest
 
 from ..command import download
 from ...consts import dandiset_metadata_file
@@ -82,9 +81,6 @@ def test_download_bad_type(mocker):
     mock_download.assert_not_called()
 
 
-@pytest.mark.skipif(
-    not os.environ.get("DANDI_DEVEL"), reason="DANDI_DEVEL required to run"
-)
 def test_download_gui_instance_in_dandiset(mocker):
     mock_download = mocker.patch("dandi.download.download")
     runner = CliRunner()
@@ -104,9 +100,6 @@ def test_download_gui_instance_in_dandiset(mocker):
     )
 
 
-@pytest.mark.skipif(
-    not os.environ.get("DANDI_DEVEL"), reason="DANDI_DEVEL required to run"
-)
 def test_download_api_instance_in_dandiset(mocker):
     mock_download = mocker.patch("dandi.download.download")
     runner = CliRunner()
@@ -126,9 +119,6 @@ def test_download_api_instance_in_dandiset(mocker):
     )
 
 
-@pytest.mark.skipif(
-    not os.environ.get("DANDI_DEVEL"), reason="DANDI_DEVEL required to run"
-)
 def test_download_url_instance_match(mocker):
     mock_download = mocker.patch("dandi.download.download")
     r = CliRunner().invoke(
@@ -152,9 +142,6 @@ def test_download_url_instance_match(mocker):
     )
 
 
-@pytest.mark.skipif(
-    not os.environ.get("DANDI_DEVEL"), reason="DANDI_DEVEL required to run"
-)
 def test_download_url_instance_conflict(mocker):
     mock_download = mocker.patch("dandi.download.download")
     r = CliRunner().invoke(
@@ -163,7 +150,7 @@ def test_download_url_instance_conflict(mocker):
         standalone_mode=False,
     )
     assert r.exit_code != 0
-    assert isinstance(r.exception, click.UsageError)
+    assert isinstance(r.exception, click.ClickException)
     assert (
         str(r.exception)
         == "http://localhost:8000/api/dandisets/123456/ does not point to 'dandi' instance"

dandi/cli/tests/test_instances.py

@@ -0,0 +1,32 @@
+import os
+
+from click.testing import CliRunner
+
+from ..command import instances
+
+
+def test_cmd_instances(monkeypatch):
+    redirector_base = os.environ.get(
+        "DANDI_REDIRECTOR_BASE", "https://dandiarchive.org"
+    )
+    instancehost = os.environ.get("DANDI_INSTANCEHOST", "localhost")
+    r = CliRunner().invoke(instances, [])
+    assert r.exit_code == 0
+    assert r.output == (
+        "dandi:\n"
+        "  api: https://api.dandiarchive.org/api\n"
+        "  gui: https://gui.dandiarchive.org\n"
+        f"  redirector: {redirector_base}\n"
+        "dandi-api-local-docker-tests:\n"
+        f"  api: http://{instancehost}:8000/api\n"
+        "  gui: null\n"
+        "  redirector: null\n"
+        "dandi-devel:\n"
+        "  api: null\n"
+        "  gui: https://gui-beta-dandiarchive-org.netlify.app\n"
+        "  redirector: null\n"
+        "dandi-staging:\n"
+        "  api: https://api-staging.dandiarchive.org/api\n"
+        "  gui: https://gui-staging.dandiarchive.org\n"
+        "  redirector: null\n"
+    )

docs/source/cmdline/instances.rst

@@ -0,0 +1,31 @@
+:program:`dandi instances`
+==========================
+
+::
+
+    dandi [<global options>] instances
+
+List known Dandi Archive instances that can be passed to the
+``-i``/``--dandi-instance`` option of other subcommands for the CLI to
+interact with.  Output is in YAML.
+
+Example output:
+
+.. code:: yaml
+
+    dandi:
+      api: https://api.dandiarchive.org/api
+      gui: https://gui.dandiarchive.org
+      redirector: https://dandiarchive.org
+    dandi-api-local-docker-tests:
+      api: http://localhost:8000/api
+      gui: null
+      redirector: null
+    dandi-devel:
+      api: null
+      gui: https://gui-beta-dandiarchive-org.netlify.app
+      redirector: null
+    dandi-staging:
+      api: https://api-staging.dandiarchive.org/api
+      gui: https://gui-staging.dandiarchive.org
+      redirector: null