Skip to content

Commit

Permalink
Add options to Bucket#create_file
Browse files Browse the repository at this point in the history 
Also, refactor Bucket and Connection to remove duplication.

[closes googleapis#92]
  • Loading branch information
@quartzmo @blowmage
quartzmo authored and blowmage committed Oct 20, 2015
1 parent 1adec26 commit ea3f41d
Show file tree
Hide file tree
Showing 5 changed files with 185 additions and 128 deletions.
77 changes: 38 additions & 39 deletions lib/gcloud/storage/bucket.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,39 @@ def file path, options = {}
# and project team members get access according to their roles.
# * +public+, +public_read+, +publicRead+ - File owner gets OWNER
# access, and allUsers get READER access.
# <code>options[:cache_control]</code>::
# The {Cache-Control}[https://tools.ietf.org/html/rfc7234#section-5.2]
# response header to be returned when the file is downloaded. (+String+)
# <code>options[:content_disposition]</code>::
# The {Content-Disposition}[https://tools.ietf.org/html/rfc6266]
# response header to be returned when the file is downloaded. (+String+)
# <code>options[:content_encoding]</code>::
# The {Content-Encoding
# }[https://tools.ietf.org/html/rfc7231#section-3.1.2.2] response header
# to be returned when the file is downloaded. (+String+)
# <code>options[:content_language]</code>::
# The {Content-Language}[http://tools.ietf.org/html/bcp47] response
# header to be returned when the file is downloaded. (+String+)
# <code>options[:content_type]</code>::
# The {Content-Type}[https://tools.ietf.org/html/rfc2616#section-14.17]
# response header to be returned when the file is downloaded. (+String+)
# <code>options[:chunk_size]</code>::
# The number of bytes per chunk in a resumable upload. Must be divisible
# by 256KB. If it is not divisible by 265KB then it will be lowered to
# the nearest acceptable value. (+Integer+)
# <code>options[:crc32c]</code>::
# The CRC32c checksum of the file data, as described in
# {RFC 4960, Appendix B}[http://tools.ietf.org/html/rfc4960#appendix-B].
# If provided, Cloud Storage will only create the file if the value
# matches the value calculated by the service. See
# {Validation}[https://cloud.google.com/storage/docs/hashes-etags]
# for more information. (+String+)
# <code>options[:md5]</code>::
# The MD5 hash of the file data. If provided, Cloud Storage will only
# create the file if the value matches the value calculated by the
# service. See
# {Validation}[https://cloud.google.com/storage/docs/hashes-etags]
# for more information. (+String+)
#
# === Returns
#
Expand Down Expand Up @@ -560,13 +593,14 @@ def file path, options = {}
def create_file file, path = nil, options = {}
ensure_connection!
ensure_file_exists! file

options[:acl] = File::Acl.predefined_rule_for options[:acl]
resumable = resumable_upload?(file)
resp = @connection.upload_file resumable, name, file, path, options

if resumable_upload? file
upload_resumable file, path, options[:chunk_size], options
if resp.success?
File.from_gapi resp.data, connection
else
upload_multipart file, path, options
fail ApiError.from_response(resp)
end
end
alias_method :upload_file, :create_file
Expand Down Expand Up @@ -733,41 +767,6 @@ def resumable_upload? file #:nodoc:
::File.size?(file).to_i > Upload.resumable_threshold
end

def upload_multipart file, path, options = {}
resp = @connection.insert_file_multipart name, file, path, options

if resp.success?
File.from_gapi resp.data, connection
else
fail ApiError.from_response(resp)
end
end

def upload_resumable file, path, chunk_size, options = {}
chunk_size = verify_chunk_size! chunk_size

resp = @connection.insert_file_resumable name, file,
path, chunk_size, options

if resp.success?
File.from_gapi resp.data, connection
else
fail ApiError.from_response(resp)
end
end

##
# Determines if a chunk_size is valid.
def verify_chunk_size! chunk_size
chunk_size = chunk_size.to_i
chunk_mod = 256 * 1024 # 256KB
if (chunk_size.to_i % chunk_mod) != 0
chunk_size = (chunk_size / chunk_mod) * chunk_mod
end
return if chunk_size.zero?
chunk_size
end

##
# Given nil, empty array, a gapi array of hashes, or any value, returns a
# deeply dup'd and frozen array of simple hashes or values (not gapi
Expand Down
111 changes: 52 additions & 59 deletions lib/gcloud/storage/connection.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,73 +178,25 @@ def list_files bucket_name, options = {}
)
end

# rubocop:disable Metrics/MethodLength
# rubocop:disable Metrics/AbcSize
# Disabled rubocop because the API we need to use
# is verbose. No getting around it.

##
# Stores a new object and metadata.
# Uses a multipart form post.
def insert_file_multipart bucket_name, file, path = nil,
options = {}
local_path = Pathname(file).to_path
upload_path = Pathname(path || local_path).to_path
mime_type = mime_type_for local_path

media = Google::APIClient::UploadIO.new local_path, mime_type

params = { uploadType: "multipart",
bucket: bucket_name,
name: upload_path,
predefinedAcl: options[:acl]
}.delete_if { |_, v| v.nil? }

@client.execute(
api_method: @storage.objects.insert,
media: media,
parameters: params,
body_object: { contentType: mime_type }
)
end

##
# Stores a new object and metadata.
# Uses a resumable upload.
def insert_file_resumable bucket_name, file, path = nil,
chunk_size = nil, options = {}
# Stores a new object and metadata. If resumable is true, a resumable
# upload, otherwise uses a multipart form post.
#
# UploadIO comes from Faraday, which gets it from multipart-post
# The initializer signature is:
# filename_or_io, content_type, filename = nil, opts = {}
def upload_file resumable, bucket_name, file, path = nil, options = {}
local_path = Pathname(file).to_path
options[:content_type] ||= mime_type_for(local_path)
media = file_media local_path, options, resumable
upload_path = Pathname(path || local_path).to_path
# mime_type = options[:mime_type] || mime_type_for local_path
mime_type = mime_type_for local_path

# This comes from Faraday, which gets it from multipart-post
# The signature is:
# filename_or_io, content_type, filename = nil, opts = {}

media = Google::APIClient::UploadIO.new local_path, mime_type
media.chunk_size = chunk_size

params = { uploadType: "resumable",
bucket: bucket_name,
name: upload_path,
predefinedAcl: options[:acl]
}.delete_if { |_, v| v.nil? }

result = @client.execute(
api_method: @storage.objects.insert,
media: media,
parameters: params,
body_object: { contentType: mime_type }
)
result = insert_file resumable, bucket_name, upload_path, media, options
return result unless resumable
upload = result.resumable_upload
result = @client.execute upload while upload.resumable?
result
end

# rubocop:enable Metrics/MethodLength
# rubocop:enable Metrics/AbcSize

##
# Retrieves an object or its metadata.
def get_file bucket_name, file_path, options = {}
Expand Down Expand Up @@ -406,6 +358,35 @@ def storage_class str #:nodoc:
"standard" => "STANDARD" }[str.to_s.downcase]
end

def insert_file resumable, bucket_name, path, media, options
params = { uploadType: (resumable ? "resumable" : "multipart"),
bucket: bucket_name,
name: path,
predefinedAcl: options[:acl]
}.delete_if { |_, v| v.nil? }

@client.execute api_method: @storage.objects.insert,
media: media,
parameters: params,
body_object: insert_file_request(options)
end

def file_media local_path, options, resumable
media = Google::APIClient::UploadIO.new local_path,
options[:content_type]
return media unless resumable && options[:chunk_size]
media.chunk_size = verify_chunk_size!(options.delete(:chunk_size))
media
end

def insert_file_request options = {}
request = {
"md5Hash" => options[:md5],
"crc32c" => options[:crc32c]
}.delete_if { |_, v| v.nil? }
request.merge patch_file_request(options)
end

def patch_file_request options = {}
{
"cacheControl" => options[:cache_control],
Expand All @@ -421,6 +402,18 @@ def incremental_backoff options = {}
yield
end
end

##
# Determines if a chunk_size is valid.
def verify_chunk_size! chunk_size
chunk_size = chunk_size.to_i
chunk_mod = 256 * 1024 # 256KB
if (chunk_size.to_i % chunk_mod) != 0
chunk_size = (chunk_size / chunk_mod) * chunk_mod
end
return if chunk_size.zero?
chunk_size
end
end
end
end
5 changes: 3 additions & 2 deletions lib/gcloud/storage/file.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,9 @@ def md5
end

##
# CRC32c checksum, as described in RFC 4960, Appendix B;
# encoded using base64.
# The CRC32c checksum of the data, as described in
# {RFC 4960, Appendix B}[http://tools.ietf.org/html/rfc4960#appendix-B].
# Encoded using base64 in big-endian byte order.
def crc32c
@gapi["crc32c"]
end
Expand Down
Loading

0 comments on commit ea3f41d

Please sign in to comment.