Universal Deploy

.circleci/config.yml

@@ -48,7 +48,7 @@ jobs:
       - setup_remote_docker
       - run:
           name: Install prefect
-          command: pip install -e ".[dev]"
+          command: pip install -e ".[dev,test]"
 
       - run:
           name: Run documentation tests
@@ -101,7 +101,7 @@ jobs:
 
       - run:
           name: Install testing packages
-          command: pip install "pytest==5.0" pytest-cov pytest-env
+          command: pip install -r test-requirements.txt
 
       - run:
           name: Run tests
@@ -133,7 +133,7 @@ jobs:
 
       - run:
           name: Install testing packages
-          command: pip install "pytest>=4.3,<5.0" pytest-cov pytest-env
+          command: pip install -r test-requirements.txt
 
       - run:
           name: Run tests

MANIFEST.in

@@ -7,4 +7,4 @@ include src/prefect/_version.py
 include src/prefect/_siginfo.py
 include src/prefect/_sig29/*.txt
 graft src/prefect/environments
-graft src/prefect/agent
+graft src/prefect/agent

appveyor.yml

@@ -11,8 +11,8 @@ init:
   - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%"
 
 install:
-- "%PYTHON%/Scripts/pip.exe install pytest==5.2.2"
-- "%PYTHON%/Scripts/pip.exe install ."
+  - "%PYTHON%/Scripts/pip.exe install -r test-requirements.txt"
+  - "%PYTHON%/Scripts/pip.exe install .[all_extras]"
 
 test_script:
 - "%PYTHON%/Scripts/pytest . --skip-formatting"

dev-requirements.txt

@@ -2,8 +2,4 @@ black
 graphviz >= 0.8, < 0.14
 jinja2 >= 2.0, < 3.0
 mypy >= 0.600, < 0.800
-pytest >= 5.0, < 5.3.2
-pytest-cov >= 2.6, < 3.0
-pytest-env >= 0.6.0, < 0.7.0
-pytest-xdist >= 1.23, < 2.0
 Pygments >= 2.2, < 3.0

docs/.vuepress/config.js

@@ -41,9 +41,10 @@ module.exports = {
         ga: 'UA-115585378-1'
       }
     ],
-    ['vuepress-plugin-code-copy', true],
+    ["vuepress-plugin-code-copy", true],
+    'vuepress-plugin-element-tabs',
     [
-      'vuepress-plugin-selected-text-popup',
+    'vuepress-plugin-selected-text-popup',
       {
         github: true,
         githubOwner: 'prefecthq',
@@ -71,8 +72,8 @@ module.exports = {
         link: '/core/'
       },
       {
-        text: 'Prefect Cloud',
-        link: '/cloud/the-basics'
+        text: "Prefect Cloud",
+        link: "/cloud/dataflow"
       },
       {
         text: 'API Reference',
@@ -155,15 +156,12 @@ module.exports = {
         {
           title: 'Welcome',
           collapsable: false,
-          children: [
-            'the-basics',
-            'upandrunning',
-            'flow-deploy',
-            'dataflow',
-            'deployment',
-            'other-considerations',
-            'faq'
-          ]
+          children: ["dataflow", "faq"]
+        },
+        {
+          title: "Tutorial",
+          collapsable: false,
+          children: ["tutorial/configure", "tutorial/first", "tutorial/multiple", "tutorial/docker", "tutorial/k8s", "tutorial/next-steps"]
         },
         {
           title: 'Cloud Concepts',
@@ -174,12 +172,13 @@ module.exports = {
           title: 'Cloud Execution',
           collapsable: true,
           children: [
-            'execution/overview',
-            'execution/remote_environment',
-            'execution/dask_k8s_environment',
-            'execution/k8s_job_environment',
-            'execution/fargate_task_environment',
-            'execution/custom_environment'
+            "execution/overview",
+            "execution/storage_options",
+            "execution/remote_environment",
+            "execution/dask_k8s_environment",
+            "execution/k8s_job_environment",
+            "execution/fargate_task_environment",
+            "execution/custom_environment",
           ]
         },
         {
@@ -188,14 +187,15 @@ module.exports = {
           children: [
             'agent/overview',
             'agent/local',
+            'agent/docker',
             'agent/kubernetes',
             'agent/fargate'
           ]
         },
         {
           title: 'Deployment Recipes',
           collapsable: true,
-          children: getChildren('docs/cloud', 'recipes')
+          children: ["recipes/deployment", "recipes/configuring_storage", "recipes/multi_flow_storage", "recipes/k8s_dask", "recipes/k8s_docker_sidecar",]
         }
       ],
       '/core/': [

docs/cloud/agent/docker.md

@@ -0,0 +1,52 @@
+# Docker Agent
+
+The Docker Agent is designed to work in all environments with access to a Docker daemon. Docker agents are most commonly used on personal machines for testing flow run deployments, but the docker agent is by no means only useful for testing. In fact, it creates flow runs that interact with Prefect Cloud in the same way that it would on any other platform. This allows the Docker Agent to be a fully functioning method of executing flows in conjunction with Prefect Cloud.
+
+[[toc]]
+
+### Requirements
+
+::: warning Docker Daemon
+The Docker Agent requires an accessible Docker daemon. So if you are using this on your local machine make sure that you have Docker running. If Docker is not running, or the agent cannot access a daemon, it will notify users on start.
+:::
+
+### Usage
+
+```
+$ prefect agent start docker
+
+ ____            __           _        _                    _
+|  _ \ _ __ ___ / _| ___  ___| |_     / \   __ _  ___ _ __ | |_
+| |_) | '__/ _ \ |_ / _ \/ __| __|   / _ \ / _` |/ _ \ '_ \| __|
+|  __/| | |  __/  _|  __/ (__| |_   / ___ \ (_| |  __/ | | | |_
+|_|   |_|  \___|_|  \___|\___|\__| /_/   \_\__, |\___|_| |_|\__|
+                                           |___/
+
+2019-09-01 12:24:59,261 - agent - INFO - Starting DockerAgent
+2019-09-01 12:24:59,261 - agent - INFO - Agent documentation can be found at https://docs.prefect.io/cloud/
+2019-09-01 12:24:59,482 - agent - INFO - Agent successfully connected to Prefect Cloud
+2019-09-01 12:24:59,482 - agent - INFO - Waiting for flow runs...
+```
+
+The Docker Agent can be started either through the Prefect CLI or by importing the `DockerAgent` class from the core library.
+
+::: tip Tokens
+There are a few ways in which you can specify a `RUNNER` API token:
+
+- command argument `prefect agent start docker -t MY_TOKEN`
+- environment variable `export PREFECT__CLOUD__AGENT__AUTH_TOKEN=MY_TOKEN`
+- token will be used from `prefect.config.cloud.auth_token` if not provided from one of the two previous methods
+
+:::
+
+### Process
+
+On start, the Docker Agent verifies that it can connect to a Docker daemon. The default daemon location is determined by your system. `npipe:////./pipe/docker_engine` for Windows and `unix://var/run/docker.sock` for Unix. A separate Docker daemon location can be provided either through `base_url` when instantiating a `DockerAgent` object or through `--base-url` on the CLI.
+
+The Docker Agent periodically polls for new flow runs to execute. When a flow run is retrieved from Prefect Cloud, the agent confirms that the flow was registered with a Docker storage option and uses the connected Docker daemon to create a container and run the flow.
+
+The agent will block on the process in between finding the flow run and submitting it for execution if it has to pull the flow's Docker image.
+
+::: tip no-pull
+The docker agent has an optional `--no-pull` flag where it will not attempt to pull the flow's Docker storage from a registry if desired. This is useful for cases in which a user may be testing the process completely dockerly without pushing the flow's Docker storage to a registry. Alternatively, if a flow's Docker storage does not have a `registry_url` specified then the Docker Agent will not attempt to pull the image.
+:::

docs/cloud/agent/fargate.md

@@ -70,7 +70,7 @@ You are now ready to run some flows!
 
 ### Process
 
-The Fargate Agent periodically polls for new flow runs to execute. When a flow run is retrieved from Prefect Cloud the agent checks to make sure that the flow was deployed with a Docker storage option. If so, the agent then creates a Task using the `storage` attribute of that flow, and runs `prefect execute cloud-flow`.
+The Fargate Agent periodically polls for new flow runs to execute. When a flow run is retrieved from Prefect Cloud the agent checks to make sure that the flow was registered with a Docker storage option. If so, the agent then creates a Task using the `storage` attribute of that flow, and runs `prefect execute cloud-flow`.
 
 If it is the first run of a particular flow then a Task Definition will be registered. Each new run of that flow will run using that same Task Definition and it will override some of the environment variables in order to specify which flow run is occurring.
 

docs/cloud/agent/kubernetes.md

@@ -58,24 +58,30 @@ The Prefect CLI provides commands for installing agents on their respective plat
 
 ```
 $ prefect agent install --help
-Usage: prefect agent install [OPTIONS] [NAME]
+Usage: prefect agent install [OPTIONS] NAME
 
   Install an agent. Outputs configuration text which can be used to install
   on various platforms. The Prefect image version will default to your local
   `prefect.__version__`
 
   Arguments:
-      name                        TEXT    The name of an agent to start (e.g. `kubernetes`)
-                                          Defaults to `kubernetes`
+      name                        TEXT    The name of an agent to install (e.g. `kubernetes`, `local`)
 
   Options:
       --token, -t                 TEXT    A Prefect Cloud API token
+      --label, -l                 TEXT    Labels the agent will use to query for flow runs
+                                          Multiple values supported e.g. `-l label1 -l label2`
+
+  Kubernetes Agent Options:
       --api, -a                   TEXT    A Prefect Cloud API URL
       --namespace, -n             TEXT    Agent namespace to launch workloads
       --image-pull-secrets, -i    TEXT    Name of image pull secrets to use for workloads
       --resource-manager                  Enable resource manager on install
-      --label, -l                 TEXT    Labels the agent will use to query for flow runs
-                                          Multiple values supported e.g. `-l label1 -l label2`
+
+  Local Agent Options:
+      --import-path, -p           TEXT    Absolute import paths to provide to the local agent.
+                                          Multiple values supported e.g. `-p /root/my_scripts -p /utilities`
+      --show-flow-logs, -f                Display logging output from flows run by the agent
 
 Options:
   -h, --help  Show this message and exit.