Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove blobstore info from IaaS metadata endpoints #147

Merged
merged 1 commit into from
Apr 24, 2023
Merged

Remove blobstore info from IaaS metadata endpoints #147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 0 additions & 70 deletions jobs/aws_cpi/spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,76 +91,6 @@ properties:
- 0.pool.ntp.org
- 1.pool.ntp.org

agent.blobstore.credentials_source:
description: Where to get AWS credentials for the aws cpi. This can be set to `static` for to use an `access_key_id` and `secret_access_key` or `env_or_profile` to get the credentials from environment variables or an EC2 instance profile.
agent.blobstore.access_key_id:
description: AWS access_key_id for agent used by s3 blobstore plugin (Required when blobstore.credentials_source is set to `static`)
agent.blobstore.secret_access_key:
description: AWS secret_access_key for agent used by s3 blobstore plugin (Required when blobstore.credentials_source is set to `static`)
agent.blobstore.s3_region:
description: AWS region for agent used by s3 blobstore plugin (Required when blobstore.credentials_source is set to `static`)
agent.blobstore.address:
description: Address for agent to connect to blobstore server used by dav blobstore plugin
agent.blobstore.use_ssl:
description: Whether the s3 blobstore plugin should use SSL to connect to the blobstore server
agent.blobstore.s3_port:
description: Port of agent blobstore server used by s3 blobstore plugin
agent.blobstore.host:
description: Host of agent blobstore server used by s3 blobstore plugin
agent.blobstore.ssl_verify_peer:
description: Whether the agent blobstore plugin should verify its peer when using SSL
agent.blobstore.s3_signature_version:
description: Signature version used to connect to an s3 blobstore
agent.blobstore.server_side_encryption:
description: Server-side encryption algorithm used when storing blobs in S3 (Optional - "AES256"|"aws:kms")
agent.blobstore.sse_kms_key_id:
description: AWS KMS key ID to use for object encryption. All GET and PUT requests for an object protected by AWS KMS will fail if not made via SSL or using SigV4.

blobstore.provider:
description: Provider of the blobstore used by director and agent (dav|local|s3)
default: 'dav'
blobstore.bucket_name:
description: AWS S3 Bucket used by s3 blobstore plugin
blobstore.credentials_source:
description: Where to get AWS credentials for the aws cpi. This can be set to `static` for to use an `access_key_id` and `secret_access_key` or `env_or_profile` to get the credentials from environment variables or an EC2 instance profile.
default: 'static'
blobstore.access_key_id:
description: AWS access_key_id used by s3 blobstore plugin (Required when blobstore.credentials_source is set to `static`)
default: null
blobstore.secret_access_key:
description: AWS secret_access_key used by s3 blobstore plugin (Required when blobstore.credentials_source is set to `static`)
default: null
blobstore.s3_region:
description: AWS region used by s3 blobstore plugin (Required when blobstore.credentials_source is set to `static`)
blobstore.use_ssl:
description: Whether the s3 blobstore plugin should use SSL to connect to the blobstore server
default: true
blobstore.s3_port:
description: Port of blobstore server used by s3 blobstore plugin
default: 443
blobstore.host:
description: Host of blobstore server used by s3 blobstore plugin
blobstore.ssl_verify_peer:
description: Whether the s3 blobstore plugin should verify its peer when using SSL
blobstore.s3_signature_version:
description: Signature version used to connect to an s3 blobstore
blobstore.server_side_encryption:
description: Server-side encryption algorithm used when storing blobs in S3 (Optional - "AES256"|"aws:kms")
blobstore.sse_kms_key_id:
description: AWS KMS key ID to use for object encryption. All GET and PUT requests for an object protected by AWS KMS will fail if not made via SSL or using SigV4.

blobstore.path:
description: local blobstore path
blobstore.address:
description: Address of blobstore server used by dav blobstore plugin
blobstore.port:
description: Port of blobstore server used by dav blobstore plugin
default: 25250
blobstore.agent.user:
description: Username agent uses to connect to blobstore used by dav blobstore plugin (Optional)
blobstore.agent.password:
description: Password agent uses to connect to blobstore used by dav blobstore plugin (Required only when user is provided)

agent.mbus:
description: Agent mbus
nats.user:
Expand Down
47 changes: 0 additions & 47 deletions jobs/aws_cpi/templates/cpi.json.erb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,53 +56,6 @@ params["cloud"]["properties"]["aws"]["region"] = p('aws.region')

agent_params = params["cloud"]["properties"]["agent"]

blobstore_defined = p('blobstore.provider') != 'dav' || !p(['blobstore.agent.user', 'agent.blobstore.address', 'blobstore.address'], nil).nil?
if blobstore_defined
agent_params["blobstore"] = {
"provider" => p('blobstore.provider'),
"options" => {}
}

blobstore = agent_params["blobstore"]

if p('blobstore.provider') == "s3"
blobstore["options"] = {
"bucket_name" => p('blobstore.bucket_name'),
"credentials_source" => p(['agent.blobstore.credentials_source', 'blobstore.credentials_source']),
"access_key_id" => p(['agent.blobstore.access_key_id', 'blobstore.access_key_id'], nil),
"secret_access_key" => p(['agent.blobstore.secret_access_key', 'blobstore.secret_access_key'], nil),
"session_token" => p(['agent.blobstore.session_token', 'blobstore.session_token'], nil),
}

def update_blobstore_options(blobstore, manifest_key, rendered_key=manifest_key)
value = p(["agent.blobstore.#{manifest_key}", "blobstore.#{manifest_key}"], nil)
blobstore["options"][rendered_key] = value unless value.nil?
end

update_blobstore_options(blobstore, 'use_ssl')
update_blobstore_options(blobstore, 's3_port', 'port')
update_blobstore_options(blobstore, 'host')
update_blobstore_options(blobstore, 'ssl_verify_peer')
update_blobstore_options(blobstore, 's3_signature_version', 'signature_version')
update_blobstore_options(blobstore, 's3_region', 'region')
update_blobstore_options(blobstore, 'server_side_encryption')
update_blobstore_options(blobstore, 'sse_kms_key_id')
elsif p('blobstore.provider') == 'local'
blobstore["options"] = {
"blobstore_path" => p('blobstore.path')
}
else
blobstore["options"] = {
"endpoint" => "http://#{p(['agent.blobstore.address', 'blobstore.address'])}:#{p('blobstore.port')}"
}

if_p('blobstore.agent.user') do
blobstore["options"]["user"] = p('blobstore.agent.user')
blobstore["options"]["password"] = p('blobstore.agent.password')
end
end
end

if_p('agent.mbus') do |mbus|
agent_params["mbus"] = mbus
end.else do
Expand Down
231 changes: 0 additions & 231 deletions src/bosh_aws_cpi/spec/unit/bosh_release/jobs/cpi/templates/cpi.json.erb_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,6 @@
'username' => 'admin',
'password' => 'admin'
},
'blobstore' => {
'address' => 'blobstore-address.example.com',
'agent' => {
'user' => 'agent',
'password' => 'agent-password'
}
},
'nats' => {
'address' => 'nats-address.example.com',
'password' => 'nats-password'
Expand Down Expand Up @@ -69,14 +62,6 @@
},
'agent' => {
'ntp'=> %w(0.pool.ntp.org 1.pool.ntp.org),
'blobstore' => {
'provider' => 'dav',
'options' => {
'endpoint' => 'http://blobstore-address.example.com:25250',
'user' => 'agent',
'password' => 'agent-password'
}
},
'mbus'=>'nats://nats:nats-password@nats-address.example.com:4222'
},
'debug'=> {
Expand Down Expand Up @@ -191,222 +176,6 @@
end
end

context 'when using a dav blobstore' do
let(:rendered_blobstore) { subject['cloud']['properties']['agent']['blobstore'] }

it 'renders agent user/password for accessing blobstore' do
expect(rendered_blobstore['options']['user']).to eq('agent')
expect(rendered_blobstore['options']['password']).to eq('agent-password')
end

context 'when enabling signed URLs' do
before do
manifest['properties']['blobstore']['agent'].delete('user')
manifest['properties']['blobstore']['agent'].delete('password')
end

it 'does not render agent user/password for accessing blobstore' do
expect(rendered_blobstore['options']['user']).to be_nil
expect(rendered_blobstore['options']['password']).to be_nil
end
end
end

context 'when using an s3 blobstore' do
let(:rendered_blobstore) { subject['cloud']['properties']['agent']['blobstore'] }

context 'when provided a minimal configuration' do
before do
manifest['properties']['blobstore'].merge!({
'provider' => 's3',
'bucket_name' => 'my_bucket',
'access_key_id' => 'blobstore-access-key-id',
'secret_access_key' => 'blobstore-secret-access-key',
})
end

it 'renders the s3 provider section with the correct defaults' do
expect(rendered_blobstore).to eq(
{
'provider' => 's3',
'options' => {
'bucket_name' => 'my_bucket',
'credentials_source' => 'static',
'access_key_id' => 'blobstore-access-key-id',
'secret_access_key' => 'blobstore-secret-access-key',
'session_token' => nil,
'use_ssl' => true,
'port' => 443,
}
}
)
end
end

context 'when provided a minimal configuration with env_or_profile credentials source' do
before do
manifest['properties']['blobstore'].merge!({
'provider' => 's3',
'bucket_name' => 'my_bucket',
'credentials_source' => 'env_or_profile',
})
end

it 'renders the s3 provider section with the correct defaults' do
expect(rendered_blobstore).to eq(
{
'provider' => 's3',
'options' => {
'bucket_name' => 'my_bucket',
'credentials_source' => 'env_or_profile',
'access_key_id' => nil,
'secret_access_key' => nil,
'session_token' => nil,
'use_ssl' => true,
'port' => 443,
}
}
)
end
end

context 'when provided a maximal configuration' do
before do
manifest['properties']['blobstore'].merge!(
'provider' => 's3',
'bucket_name' => 'my_bucket',
'credentials_source' => 'blobstore-credentials-source',
'access_key_id' => 'blobstore-access-key-id',
'secret_access_key' => 'blobstore-secret-access-key',
'session_token' => 'blobstore-session-token',
's3_region' => 'blobstore-region',
'use_ssl' => false,
's3_port' => 21,
'host' => 'blobstore-host',
'ssl_verify_peer' => true,
's3_signature_version' => '11',
'server_side_encryption' => 'AES256',
'sse_kms_key_id' => 'kms-key'
)
end

it 'renders the s3 provider section correctly' do
expect(rendered_blobstore).to eq(
{
'provider' => 's3',
'options' => {
'bucket_name' => 'my_bucket',
'credentials_source' => 'blobstore-credentials-source',
'access_key_id' => 'blobstore-access-key-id',
'secret_access_key' => 'blobstore-secret-access-key',
'session_token' => nil,
'region' => 'blobstore-region',
'use_ssl' => false,
'host' => 'blobstore-host',
'port' => 21,
'ssl_verify_peer' => true,
'signature_version' => '11',
'server_side_encryption' => 'AES256',
'sse_kms_key_id' => 'kms-key'
}
}
)
end

it 'prefers the agent properties when they are both included' do
manifest['properties']['agent'] = {
'blobstore' => {
'credentials_source' => 'agent-credentials-source',
'access_key_id' => 'agent_access_key_id',
'secret_access_key' => 'agent_secret_access_key',
'session_token' => 'agent_session_token',
's3_region' => 'agent-region',
'use_ssl' => true,
's3_port' => 42,
'host' => 'agent-host',
'ssl_verify_peer' => true,
's3_signature_version' => '99',
'server_side_encryption' => 'from-agent',
'sse_kms_key_id' => 'from-agent'
}
}

manifest['properties']['blobstore'].merge!({
'credentials_source' => 'blobstore-credentials-source',
'access_key_id' => 'blobstore_access_key_id',
'secret_access_key' => 'blobstore_secret_access_key',
'session_token' => 'blobstore_session_token',
's3_region' => 'blobstore-region',
'use_ssl' => false,
's3_port' => 21,
'host' => 'blobstore-host',
'ssl_verify_peer' => false,
's3_signature_version' => '11',
'server_side_encryption' => 'from-root',
'sse_kms_key_id' => 'from-root'
})

expect(rendered_blobstore['options']['access_key_id']).to eq('agent_access_key_id')
expect(rendered_blobstore['options']['secret_access_key']).to eq('agent_secret_access_key')
expect(rendered_blobstore['options']['credentials_source']).to eq('agent-credentials-source')
expect(rendered_blobstore['options']['region']).to eq('agent-region')
expect(rendered_blobstore['options']['use_ssl']).to be true
expect(rendered_blobstore['options']['port']).to eq(42)
expect(rendered_blobstore['options']['host']).to eq('agent-host')
expect(rendered_blobstore['options']['ssl_verify_peer']).to be true
expect(rendered_blobstore['options']['signature_version']).to eq('99')
expect(rendered_blobstore['options']['server_side_encryption']).to eq('from-agent')
expect(rendered_blobstore['options']['sse_kms_key_id']).to eq('from-agent')
end
end

end

context 'when using a local blobstore' do
let(:rendered_blobstore) { subject['cloud']['properties']['agent']['blobstore'] }

context 'when provided a minimal configuration' do
before do
manifest['properties']['blobstore'].merge!({
'provider' => 'local',
'path' => '/fake/path',
})
end

it 'renders the local provider section with the correct defaults' do
expect(rendered_blobstore).to eq(
{
'provider' => 'local',
'options' => {
'blobstore_path' => '/fake/path',
}
}
)
end
end
context 'when provided an incomplete configuration' do
before do
manifest['properties']['blobstore'].merge!({
'provider' => 'local',
})
end

it 'raises an error' do
expect { rendered_blobstore }.to raise_error(/Can't find property 'blobstore.path'/)
end
end
end

context 'when no blobstore is provided' do
before do
manifest['properties'].delete('blobstore')
end

it 'should NOT add any blobstore properties' do
expect(subject['cloud']['properties']['blobstore']).to be_nil
end
end

context 'when registry is NOT provided' do
before do
properties = manifest['properties']
Expand Down