v1: Connection re-use, Streaming, …

CHANGELOG.md

@@ -4,7 +4,7 @@ Presented in reverse chronological order.
 
 ## master
 
-https://github.com/bitcrowd/sshkit.ex/compare/v0.2.0...HEAD
+https://github.com/bitcrowd/sshkit.ex/compare/v0.3.0...HEAD
 
 <!-- Put high-level summary here -->
 
@@ -27,6 +27,16 @@ https://github.com/bitcrowd/sshkit.ex/compare/v0.2.0...HEAD
 
 <!-- Put fixes here -->
 
+## `1.0.0` (??)
+
+* Implement new top-level package API
+  * Explicitly establish, re-use and close connections
+  * Enable streaming data to and from a remote
+  * Make context optional
+* Replace SCP with an alternative file transfer implementation
+
+https://github.com/bitcrowd/sshkit.ex/compare/v0.3.0...v1.0.0
+
 ## `0.3.0` (2019-10-28)
 
 https://github.com/bitcrowd/sshkit.ex/compare/v0.2.0...v0.3.0

README.md

@@ -12,60 +12,68 @@ SSHKit is an Elixir toolkit for performing tasks on one or more servers, built o
 SSHKit is designed to enable server task automation in a structured and repeatable way, e.g. in the context of deployment tools:
 
 ```elixir
-hosts = ["1.eg.io", {"2.eg.io", port: 2222}]
+{:ok, conn} = SSHKit.connect("eg.io", port: 2222)
+{:ok, chan} = SSHKit.run(conn, "apt-get update -y")
+:ok = SSHKit.flush(chan)
+:ok = SSHKit.close(conn)
+```
+
+```elixir
+{:ok, conn} = SSHKit.connect("eg.io", port: 2222)
 
 context =
-  SSHKit.context(hosts)
-  |> SSHKit.path("/var/www/phx")
-  |> SSHKit.user("deploy")
-  |> SSHKit.group("deploy")
-  |> SSHKit.umask("022")
-  |> SSHKit.env(%{"NODE_ENV" => "production"})
-
-[:ok, :ok] = SSHKit.upload(context, ".", recursive: true)
-[{:ok, _, 0}, {:ok, _, 0}] = SSHKit.run(context, "yarn install")
+  SSHKit.Context.new()
+  |> SSHKit.Context.path("/var/www/phx")
+  |> SSHKit.Context.user("deploy")
+  |> SSHKit.Context.group("deploy")
+  |> SSHKit.Context.umask("022")
+  |> SSHKit.Context.env(%{"NODE_ENV" => "production"})
+
+# TODO: Track/report upload progress
+{:ok, _} = SSHKit.upload(conn, ".", recursive: true, context: context)
+
+{:ok, chan} = SSHKit.run(conn, "yarn install", context: context)
+
+status =
+  chan
+  |> SSHKit.stream!()
+  |> Enum.reduce(nil, fn
+    {:stdout, ^chan, data}, status ->
+      IO.write(:stdio, data)
+      status
+
+    {:stderr, ^chan, data}, status ->
+      IO.write(:stderr, data)
+      status
+
+    {:exited, ^chan, status}, _ ->
+      status
+
+    _, status ->
+      status
+  end)
+
+if status != 0 do
+  IO.write(:stderr, "Non-zero exit code #{status}")
+end
+
+:ok = SSHKit.close(conn)
 ```
 
 The [`SSHKit`](https://hexdocs.pm/sshkit/SSHKit.html) module documentation has more guidance and examples for the DSL.
 
-If you need more control, take a look at the [`SSHKit.SSH`](https://hexdocs.pm/sshkit/SSHKit.SSH.html) and [`SSHKit.SCP`](https://hexdocs.pm/sshkit/SSHKit.SCP.html) modules.
-
 ## Installation
 
 Just add `sshkit` to your list of dependencies in `mix.exs`:
 
   ```elixir
   def deps do
-    [{:sshkit, "~> 0.1"}]
+    [{:sshkit, "~> 1.0"}]
   end
   ```
 
 SSHKit should be automatically started unless the `:applications` key is set inside `def application` in your `mix.exs`. In such cases, you need to [remove the `:applications` key in favor of `:extra_applications`](https://elixir-lang.org/blog/2017/01/05/elixir-v1-4-0-released/#application-inference).
 
-## Modules
-
-SSHKit consists of three core modules:
-
-```
-+--------------------+
-| SSHKit             |
-+--------------------+
-|       | SSHKit.SCP |
-|       +------------+
-| SSHKit.SSH         |
-+--------------------+
-```
-
-1. [**`SSHKit.SSH`**](https://hexdocs.pm/sshkit/SSHKit.SSH.html) provides convenience functions for working with SSH connections and for executing commands on remote hosts.
-
-2. [**`SSHKit.SCP`**](https://hexdocs.pm/sshkit/SSHKit.SCP.html) provides convenience functions for transferring files or entire directory trees to or from a remote host via SCP. It is built on top of `SSHKit.SSH`.
-
-3. [**`SSHKit`**](https://hexdocs.pm/sshkit/SSHKit.html) provides the main API for automating tasks on remote hosts in a structured way. It uses both `SSH` and `SCP` to implement its functionality.
-
-Additional modules, e.g. for custom client key handling, are available as separate packages:
-
-* [**`ssh_client_key_api`**](https://hex.pm/packages/ssh_client_key_api): An Elixir implementation for the Erlang `ssh_client_key_api` behavior, to make it easier to specify SSH keys and `known_hosts` files independently of any particular user's home directory.
-
 ## Testing
 
 As usual, to run all tests, use:
@@ -74,7 +82,7 @@ As usual, to run all tests, use:
 mix test
 ```
 
-Apart from unit tests, we also have [functional tests](https://en.wikipedia.org/wiki/Functional_testing). These check SSHKit functionality against real SSH server implementations running inside Docker containers. Therefore, you need to have [Docker](https://www.docker.com/) installed.
+Apart from unit tests, we also have [functional tests](https://en.wikipedia.org/wiki/Functional_testing). These check SSHKit against real SSH server implementations running inside Docker containers. Therefore, you need to have [Docker](https://www.docker.com/) installed.
 
 All functional tests are tagged as such. Hence, if you wish to skip them:
 
@@ -121,9 +129,9 @@ SSHKit source code is released under the MIT License.
 
 Check the [LICENSE][license] file for more information.
 
-  [issues]: https://github.com/bitcrowd/sshkit.ex/issues
-  [pulls]: https://github.com/bitcrowd/sshkit.ex/pulls
-  [docs]: https://hexdocs.pm/sshkit
-  [changelog]: ./CHANGELOG.md
-  [license]: ./LICENSE
-  [writing-docs]: https://hexdocs.pm/elixir/writing-documentation.html
+[issues]: https://github.com/bitcrowd/sshkit.ex/issues
+[pulls]: https://github.com/bitcrowd/sshkit.ex/pulls
+[docs]: https://hexdocs.pm/sshkit
+[changelog]: ./CHANGELOG.md
+[license]: ./LICENSE
+[writing-docs]: https://hexdocs.pm/elixir/writing-documentation.html

config/config.exs

@@ -1 +1,7 @@
 import Config
+
+if Mix.env() == :test do
+  config :sshkit, :ssh, MockErlangSsh
+  config :sshkit, :ssh_connection, MockErlangSshConnection
+  config :sshkit, :ssh_sftp, MockErlangSshSftp
+end

examples/command.exs

@@ -0,0 +1,7 @@
+{:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true)
+
+conn
+|> SSHKit.run!(~S(uname -a && ssh -V))
+|> Enum.each(fn {type, data} -> IO.write("#{type}: #{data}") end)
+
+:ok = SSHKit.close(conn)

examples/context.exs

@@ -0,0 +1,13 @@
+{:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true)
+
+context =
+  SSHKit.Context.new()
+  |> SSHKit.Context.path("/tmp")
+  |> SSHKit.Context.umask("007")
+  |> SSHKit.Context.env(%{"X" => "Y"})
+
+conn
+|> SSHKit.run!(~S(echo $X && pwd && umask && id -un && id -gn), context: context)
+|> Enum.each(fn {type, data} -> IO.write("#{type}: #{data}") end)
+
+:ok = SSHKit.close(conn)

examples/parallel.exs

@@ -0,0 +1,45 @@
+defaults = [user: "deploy", password: "deploy", silently_accept_hosts: true]
+
+hosts =
+  [{"127.0.0.1", port: 2222}, {"127.0.0.1", port: 2223}]
+  |> Enum.map(fn {name, options} -> {name, Keyword.merge(defaults, options)} end)
+
+conns = Enum.map(hosts, fn {name, options} ->
+  {:ok, conn} = SSHKit.connect(name, options)
+  conn
+end)
+
+label = fn conn -> Enum.join([conn.host, conn.port], ":") end
+
+tasks =
+  Enum.map(conns, fn conn ->
+    Task.async(fn ->
+      conn
+      |> SSHKit.exec!("uptime")
+      |> Enum.reduce(nil, fn
+        {:stdout, chan, output}, status ->
+          IO.write("[#{label.(chan.connection)}] (stdout) #{output}")
+          status
+
+        {:stderr, chan, output}, status ->
+          IO.write("[#{label.(chan.connection)}] (stderr) #{output}")
+          status
+
+        {:exit_status, _, status}, _ ->
+          status
+
+        _, status ->
+          status
+      end)
+    end)
+  end)
+
+tasks
+|> Enum.map(&Task.await/1)
+|> Enum.filter(&(&1 != 0))
+|> Enum.zip(conns)
+|> Enum.each(fn {status, conn} ->
+  IO.puts("[#{label.(conn)}] exited with status #{status}")
+end)
+
+:ok = Enum.each(conns, &SSHKit.close/1)

examples/stream.exs

@@ -0,0 +1,28 @@
+{:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true)
+
+stream = SSHKit.exec!(conn, ~S(echo "Who's there?"; read name; echo -n "Hello"; sleep 3; echo " $name."))
+
+:ok = IO.write("> ")
+
+code = Enum.reduce(stream, nil, fn
+  {:stdout, chan, chunk}, status ->
+    :ok = IO.write("#{chunk}")
+
+    if String.ends_with?(chunk, "?\n") do
+      :ok = SSHKit.send(chan, "SSHKit\n")
+      :ok = IO.write("< SSHKit\n")
+      :ok = IO.write("> ")
+    end
+
+    status
+
+  {:exit_status, _, status}, _ ->
+    status
+
+  _, status ->
+    status
+end)
+
+:ok = IO.puts("? #{code}")
+
+:ok = SSHKit.close(conn)

examples/tarpipe.exs

@@ -0,0 +1,107 @@
+{:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true)
+
+ctx =
+  SSHKit.Context.new()
+  |> SSHKit.Context.path("/tmp")
+  # |> SSHKit.Context.user("other")
+  # |> SSHKit.Context.group("other")
+  |> SSHKit.Context.umask("0077")
+
+defmodule TP do
+  def upload!(conn, source, dest, opts \\ []) do
+    ctx = Keyword.get(opts, :context, SSHKit.Context.new())
+
+    Stream.resource(
+      fn ->
+        owner = self()
+
+        tarpipe = spawn(fn ->
+          {:ok, chan} = SSHKit.Channel.open(conn, [])
+          command = SSHKit.Context.build(ctx, "tar -x")
+          :success = SSHKit.Channel.exec(chan, command)
+
+          # TODO: What if command immediately exits or does not exist?
+          # IO.inspect(SSHKit.Channel.recv(chan, 1000))
+
+          {:ok, tar} = :erl_tar.init(chan, :write, fn
+            :position, {^chan, position} ->
+              # IO.inspect(position, label: "position")
+              {:ok, 0}
+
+            :write, {^chan, data} ->
+              # TODO: Send data in chunks based on channel window size?
+              IO.inspect(data, label: "write")
+              # In case of failing upload, check command output:
+              # IO.inspect(SSHKit.Channel.recv(chan, 0))
+              chunk = to_binary(data)
+
+              receive do
+                :cont ->
+                  case SSHKit.Channel.send(chan, chunk) do
+                    :ok -> send(owner, {:write, chan, self(), chunk})
+                    other -> send(owner, {:error, chan, self(), other})
+                  end
+              end
+              :ok
+
+            :close, ^chan ->
+              # IO.puts("close")
+              :ok = SSHKit.Channel.eof(chan)
+              send(owner, {:close, chan, self()})
+              :ok
+          end)
+
+          :ok = :erl_tar.add(tar, to_charlist(source), to_charlist(Path.basename(source)), [])
+          :ok = :erl_tar.close(tar)
+
+          :ok = SSHKit.Channel.close(chan)
+        end)
+
+        tarpipe
+      end,
+      fn tarpipe ->
+        send(tarpipe, :cont)
+
+        receive do
+          {:write, chan, ^tarpipe, data} ->
+            {[{:write, chan, data}], tarpipe}
+
+          {:close, chan, ^tarpipe} ->
+            {:halt, tarpipe}
+
+          {:error, chan, ^tarpipe, error} ->
+            IO.inspect(error, label: "received error")
+            {:halt, tarpipe}
+        end
+
+        # case Tarpipe.proceed(tarpipe) do
+        #   {:write, …} -> {[], tarpipe}
+        #   {:error, …} -> raise
+        # end
+      end,
+      fn tarpipe ->
+        nil # :ok = Tarpipe.close(tarpipe)
+      end
+    )
+  end
+
+  # https://github.com/erlang/otp/blob/OTP-23.2.1/lib/ssh/src/ssh.hrl
+  def to_binary(data) when is_list(data) do
+    :erlang.iolist_to_binary(data)
+  catch
+    _ -> :unicode.characters_to_binary(data)
+  end
+
+  def to_binary(data) when is_binary(data) do
+    data
+  end
+end
+
+stream = TP.upload!(conn, "test/fixtures", "upload", context: ctx)
+
+Enum.each(stream, fn
+  {:write, chan, data} ->
+    IO.puts("Upload, sent #{byte_size(data)} bytes")
+end)
+
+:ok = SSHKit.close(conn)

examples/upload.exs

@@ -0,0 +1,8 @@
+{:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true)
+
+:ok =
+  conn
+  |> SSHKit.upload!("test/fixtures", "/tmp/fixtures")
+  |> Stream.run()
+
+:ok = SSHKit.close(conn)