Added integration test

.gitignore

@@ -6,4 +6,6 @@
 /arduino-cli.yaml
 /wiki
 .idea
-coverage_*.txt
+coverage_*.txt
+__pycache__
+venv

test/README.md

@@ -0,0 +1,15 @@
+## Integration tests
+
+This dir contains integration tests, the aim is to test the Command Line Interface and its output
+from a pure user point of view.
+
+### Installation
+
+    cd test
+    virtualenv --python=python3 venv
+    source venv/bin/activate
+    pip install -r requirements.txt
+
+### Running tests
+
+    pytest

test/conftest.py

@@ -0,0 +1,21 @@
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--runslow", action="store_true", default=False, help="run slow tests"
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "slow: mark test as slow to run")
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--runslow"):
+        # --runslow given in cli: do not skip slow tests
+        return
+    skip_slow = pytest.mark.skip(reason="need --runslow option to run")
+    for item in items:
+        if "slow" in item.keywords:
+            item.add_marker(skip_slow)

test/pytest.ini

@@ -0,0 +1,7 @@
+[pytest]
+filterwarnings =
+    error
+    ignore::DeprecationWarning
+    ignore::ResourceWarning
+
+addopts = -s --verbose --tb=short

test/requirements.txt

@@ -0,0 +1,14 @@
+atomicwrites==1.3.0
+attrs==19.1.0
+importlib-metadata==0.18
+invoke==1.2.0
+more-itertools==7.1.0
+packaging==19.0
+pep8==1.7.1
+pluggy==0.12.0
+py==1.8.0
+pyparsing==2.4.0
+pytest==5.0.1
+six==1.12.0
+wcwidth==0.1.7
+zipp==0.5.2

test/test_main.py

@@ -0,0 +1,103 @@
+from invoke import run, Responder
+import os
+import json
+import pytest
+import semver
+from datetime import datetime
+
+this_test_path = os.path.dirname(os.path.realpath(__file__))
+# Calculate absolute path of the CLI
+cli_path = os.path.join(this_test_path, '..', 'arduino-cli')
+
+# Useful reference: 
+# http://docs.pyinvoke.org/en/1.2/api/runners.html#invoke.runners.Result
+
+
+def cli_line(*args):
+    # Accept a list of arguments cli_line('lib list --format json')
+    # Return a full command line string e.g. 'arduino-cli help --format json'
+    cli_full_line = ' '.join([cli_path, ' '.join(str(arg) for arg in args)])
+    return cli_full_line
+
+
+def run_command(*args):
+    result = run(cli_line(*args), echo=False, hide=True)
+    return result
+
+
+def test_command_help():
+    result = run_command('help')
+    assert result.ok
+    assert result.stderr == ''
+    assert 'Usage' in result.stdout
+
+
+def test_command_lib_list():
+    result = run_command('lib list')
+    assert result.stderr == ''
+    result = run_command('lib list', '--format json')
+    assert '{}' == result.stdout
+
+# def test_command_lib_download():
+#     result = run_command('lib download')
+
+
+def test_command_lib_install():
+    libs = ['\"AzureIoTProtocol_MQTT\"', '\"CMMC MQTT Connector\"', '\"WiFiNINA\"']
+    # Should be safe to run install multiple times
+    result_1 = run_command('lib install {}'.format(' '.join(libs)))
+    assert result_1.ok
+    result_2 = run_command('lib install {}'.format(' '.join(libs)))
+    assert result_2.ok
+
+def test_command_lib_update_index():
+    result = run_command('lib update-index')
+    assert 'Updating index: library_index.json downloaded' == result.stdout.splitlines()[-1].strip()
+
+def test_command_lib_remove():
+    libs = ['\"AzureIoTProtocol_MQTT\"', '\"CMMC MQTT Connector\"', '\"WiFiNINA\"']
+    result = run_command('lib uninstall {}'.format(' '.join(libs)))
+    assert result.ok
+
+@pytest.mark.slow
+def test_command_lib_search():
+    result = run_command('lib search')
+    out_lines = result.stdout.splitlines()
+    libs = []
+    # Create an array with just the name of the vars
+    for line in out_lines:
+        if 'Name: ' in line:
+            libs.append(line.split()[1].strip('\"'))
+    number_of_libs = len(libs)
+    # It would be strange to have less than 2000 Arduino Libs published
+    assert number_of_libs > 2000
+    result = run_command('lib search --format json')
+    libs_found_from_json = json.loads(result.stdout)
+    number_of_libs_from_json = len(libs_found_from_json.get('libraries'))
+    assert number_of_libs == number_of_libs_from_json
+
+
+def test_command_board_list():
+    result = run_command('board list --format json')
+    # check is a valid json and contains a list of ports
+    ports = json.loads(result.stdout).get('ports')
+    assert isinstance(ports, list)
+    for port in ports:
+        assert 'protocol' in port
+        assert 'protocol_label' in port
+
+
+def test_command_board_listall():
+    result = run_command('board listall')
+    assert ['Board', 'Name', 'FQBN'] == result.stdout.splitlines()[0].strip().split()
+
+
+def test_command_version():
+    result = run_command('version --format json')
+    parsed_out = json.loads(result.stdout)
+
+    assert parsed_out.get('Application', False) == 'arduino-cli'
+    assert isinstance(semver.parse(parsed_out.get('VersionString', False)), dict)
+    assert isinstance(parsed_out.get('Commit', False), str)
+    assert datetime.strptime(parsed_out.get('BuildDate')[:-2], '%Y-%m-%dT%H:%M:%S.%f')
+