Merge pull request #42 from invenia/upload_with_retry

Added testsets and an upload function that can do retries

Author: kano31

.travis.yml

@@ -5,6 +5,9 @@ os:
 julia:
   - 0.6
   - nightly
+matrix:
+  allow_failures:
+      - julia: nightly
 notifications:
   email: false
 script:

REQUIRE

@@ -1,3 +1,3 @@
 julia 0.6
 LibCURL 0.2.2
-Compat 0.8.3
+Compat 0.41

appveyor.yml

@@ -3,6 +3,10 @@ environment:
   - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe"
   - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe"
 
+matrix:
+  allow_failures:
+    - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe"
+
 notifications:
   - provider: Email
     on_build_success: false

src/FTPC.jl

@@ -1,6 +1,8 @@
+using Compat
 using LibCURL
 
 import Base: ==
+import Compat.Cvoid
 
 ##############################
 # Type definitions
@@ -113,7 +115,7 @@ end
 # Callbacks
 ##############################
 
-function write_file_cb(buff::Ptr{UInt8}, sz::Csize_t, n::Csize_t, p_wd::Ptr{Void})
+function write_file_cb(buff::Ptr{UInt8}, sz::Csize_t, n::Csize_t, p_wd::Ptr{Cvoid})
     # println("@write_file_cb")
     wd = unsafe_pointer_to_objref(p_wd)
     nbytes = sz * n
@@ -125,9 +127,9 @@ function write_file_cb(buff::Ptr{UInt8}, sz::Csize_t, n::Csize_t, p_wd::Ptr{Void
     nbytes::Csize_t
 end
 
-c_write_file_cb = cfunction(write_file_cb, Csize_t, (Ptr{UInt8}, Csize_t, Csize_t, Ptr{Void}))
+c_write_file_cb = cfunction(write_file_cb, Csize_t, Tuple{Ptr{UInt8}, Csize_t, Csize_t, Ptr{Cvoid}})
 
-function header_command_cb(buff::Ptr{UInt8}, sz::Csize_t, n::Csize_t, p_resp::Ptr{Void})
+function header_command_cb(buff::Ptr{UInt8}, sz::Csize_t, n::Csize_t, p_resp::Ptr{Cvoid})
     # println("@header_cb")
     resp = unsafe_pointer_to_objref(p_resp)
     nbytes = sz * n
@@ -140,25 +142,25 @@ function header_command_cb(buff::Ptr{UInt8}, sz::Csize_t, n::Csize_t, p_resp::Pt
     nbytes::Csize_t
 end
 
-c_header_command_cb = cfunction(header_command_cb, Csize_t, (Ptr{UInt8}, Csize_t, Csize_t, Ptr{Void}))
+c_header_command_cb = cfunction(header_command_cb, Csize_t, Tuple{Ptr{UInt8}, Csize_t, Csize_t, Ptr{Cvoid}})
 
-function curl_read_cb(out::Ptr{Void}, s::Csize_t, n::Csize_t, p_rd::Ptr{Void})
+function curl_read_cb(out::Ptr{Cvoid}, s::Csize_t, n::Csize_t, p_rd::Ptr{Cvoid})
     # println("@curl_read_cb")
     rd = unsafe_pointer_to_objref(p_rd)
     bavail::Csize_t = s * n
     breq::Csize_t = rd.sz - rd.offset
     b2copy = bavail > breq ? breq : bavail
 
     b_read = read(rd.src, UInt8, b2copy)
-    ccall(:memcpy, Ptr{Void}, (Ptr{Void}, Ptr{Void}, UInt), out, b_read, b2copy)
+    ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt), out, b_read, b2copy)
 
     rd.offset += b2copy
 
     r = convert(Csize_t, b2copy)
     r::Csize_t
 end
 
-c_curl_read_cb = cfunction(curl_read_cb, Csize_t, (Ptr{Void}, Csize_t, Csize_t, Ptr{Void}))
+c_curl_read_cb = cfunction(curl_read_cb, Csize_t, Tuple{Ptr{Cvoid}, Csize_t, Csize_t, Ptr{Cvoid}})
 
 
 ##############################

src/FTPClient.jl

@@ -2,7 +2,7 @@ module FTPClient
 
 import Base: convert, show, open, mkdir, ascii, mv
 import Base: readdir, cd, pwd, rm, close, download
-import Compat: readstring, unsafe_string, unsafe_write
+import Compat: readstring, unsafe_string, unsafe_write, @compat
 
 mutable struct FTPClientError <: Exception
     msg::AbstractString

src/FTPObject.jl

@@ -1,3 +1,6 @@
+# FTP code for when the file transfer is complete.
+const complete_transfer_code = 226
+
 """
     FTP(; kwargs...) -> FTP
 
@@ -110,6 +113,94 @@ function upload(ftp::FTP, local_file::IO, remote_name::AbstractString; mode::FTP
 end
 
 
+"""
+    upload(
+        ftp::FTP,
+        local_file_paths::Vector{<:AbstractString},
+        ftp_dir<:AbstractString;
+        retry_callback::Function=(count, options) -> (count < 4, options),
+        retry_wait_seconds::Integer = 5
+    )
+
+Uploads the files specified in local_file_paths to the directory specifed by
+ftp_dir. The files will have the same names.
+
+By default, will try to deliver the files 4 times with a 5 second wait in between
+each failed attempt.
+
+You can specify a function for retry_callback to change behaviour. This function must
+take as parameters the number of attempts that have been made so far, and the current
+ftp connection options as a FTPClient.ConnContext type. It must return a boolean
+that is true if another delivery attempt can be made, and a TPClient.ConnContext
+type that is the connection options to use for all future files to be delivered. This
+allows backup ftp directories to be used for example.
+
+# Arguments
+`ftp::FTP`: The FTP to deliver to. See FTPClient.FTP for details.
+`file_paths::Vector{T}`: The file paths to the files we want to deliver.
+`ftp_dir`: The directory on the ftp server where we want to drop the files.
+`retry_callback::Function=(count, options) -> (count < 4, options)`: Function for retrying
+                                                                     when delivery fails.
+`retry_wait_seconds::Integer = 5`: How many seconds to wait in between retries.
+
+# Returns
+- `Array{Bool,1}`: Returns a vector of booleans with true for each successfully delivered
+                   file and false for any that failed to transfer.
+"""
+@compat function upload(
+    ftp::FTP,
+    local_file_paths::Vector{<:AbstractString},
+    ftp_dir::AbstractString;
+    retry_callback::Function=(count, options) -> (count < 4, options),
+    retry_wait_seconds::Integer = 5
+)
+
+    successful_delivery = Bool[]
+
+    ftp_options = ftp.ctxt
+
+    for single_file in local_file_paths
+        # The location we are going to drop the file in the FTP server
+        server_location = joinpath(ftp_dir, basename(single_file))
+
+        # ftp_put requires an io so open up our file.
+        open(single_file) do single_file_io
+            # Whether or not the current file was successfully delivered to the FTP
+            file_delivery_success = false
+
+            attempts = 1
+            # The loops should break after an appropriate amount of retries.
+            # This way of doing retries makes testing easier.
+            # Defaults to 4 attempts, waiting 5 seconds between each retry.
+            while true
+                try
+                    resp = ftp_put(ftp_options, server_location, single_file_io)
+                    file_delivery_success = resp.code == complete_transfer_code
+                    if file_delivery_success
+                        break
+                    end
+                catch e
+                    warn(e)
+                end
+                sleep(retry_wait_seconds)
+                # It returns ftp_options for testing purposes, where the ftp server
+                # starts not existing then comes into existence during retries.
+                do_retry, ftp_options = retry_callback(attempts, ftp_options)
+                if !do_retry
+                    break
+                end
+                attempts += 1
+            end
+
+            push!(successful_delivery, file_delivery_success)
+
+        end
+    end
+
+    return successful_delivery
+end
+
+
 """
     readdir(ftp::FTP)