Skip to content

Commit 663ddc0

Browse files
mugamuga
committed
Fix and refactor unit tests
1 parent 269e0af commit 663ddc0

File tree

6 files changed

+342
-174
lines changed

6 files changed

+342
-174
lines changed

‎build.gradle

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,11 @@ subprojects {
3535
provided "org.embulk:embulk-core:0.7.9"
3636
compile "com.amazonaws:aws-java-sdk-s3:1.10.33"
3737
runtime "org.slf4j:jcl-over-slf4j:1.7.12" // aws-sdk uses Apache Commons Logging and Embulk uses slf4j
38+
testCompile "com.amazonaws:aws-java-sdk-sts:1.10.33"
3839
testCompile "junit:junit:4.+"
3940
testCompile "org.mockito:mockito-core:1.+"
40-
testCompile "org.embulk:embulk-core:0.7.0:tests"
41+
testCompile "org.embulk:embulk-standards:0.7.9"
42+
testCompile "org.embulk:embulk-core:0.7.9:tests"
4143
}
4244

4345
gradle.projectsEvaluated {

‎embulk-input-riak_cs/src/test/java/org/embulk/input/riak_cs/TestRiakCsFileInputPlugin.java

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import org.embulk.EmbulkTestRuntime;
44
import org.embulk.config.ConfigSource;
55
import org.embulk.input.s3.AbstractS3FileInputPlugin.PluginTask;
6-
import org.embulk.input.s3.TestS3FileInputPlugin;
76
import org.junit.Before;
87
import org.junit.Rule;
98
import org.junit.Test;
@@ -25,7 +24,12 @@ public void createPlugin()
2524
@Test
2625
public void createS3ClientSuccessfully()
2726
{
28-
ConfigSource config = TestS3FileInputPlugin.config().set("endpoint", "my.endpoint.com");
27+
ConfigSource config = runtime.getExec().newConfigSource()
28+
.set("endpoint", "my.endpoint.com")
29+
.set("bucket", "my_bucket")
30+
.set("path_prefix", "my_path_prefix")
31+
.set("access_key_id", "my_access_key_id")
32+
.set("secret_access_key", "my_secret_access_key");
2933
PluginTask task = config.loadConfig(plugin.getTaskClass());
3034
plugin.newS3Client(task);
3135
}

‎embulk-input-s3/src/test/java/org/embulk/input/s3/TestAwsCredentials.java

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
package org.embulk.input.s3;
2+
3+
import com.amazonaws.auth.BasicAWSCredentials;
4+
import com.amazonaws.auth.BasicSessionCredentials;
5+
import com.amazonaws.auth.policy.Policy;
6+
import com.amazonaws.auth.policy.Resource;
7+
import com.amazonaws.auth.policy.Statement;
8+
import com.amazonaws.auth.policy.actions.S3Actions;
9+
import com.amazonaws.internal.StaticCredentialsProvider;
10+
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
11+
import com.amazonaws.services.securitytoken.model.Credentials;
12+
import com.amazonaws.services.securitytoken.model.GetFederationTokenRequest;
13+
import com.amazonaws.services.securitytoken.model.GetFederationTokenResult;
14+
import org.embulk.EmbulkTestRuntime;
15+
import org.embulk.config.ConfigDiff;
16+
import org.embulk.config.ConfigSource;
17+
import org.embulk.input.s3.TestS3FileInputPlugin.Control;
18+
import org.embulk.spi.FileInputRunner;
19+
import org.embulk.spi.TestPageBuilderReader;
20+
import org.junit.Before;
21+
import org.junit.BeforeClass;
22+
import org.junit.Rule;
23+
import org.junit.Test;
24+
25+
import static org.embulk.input.s3.TestS3FileInputPlugin.assertRecords;
26+
import static org.embulk.input.s3.TestS3FileInputPlugin.parserConfig;
27+
import static org.embulk.input.s3.TestS3FileInputPlugin.schemaConfig;
28+
import static org.junit.Assert.assertEquals;
29+
import static org.junit.Assume.assumeNotNull;
30+
31+
public class TestAwsCredentials
32+
{
33+
private static String EMBULK_S3_TEST_BUCKET;
34+
private static String EMBULK_S3_TEST_ACCESS_KEY_ID;
35+
private static String EMBULK_S3_TEST_SECRET_ACCESS_KEY;
36+
private static final String EMBULK_S3_TEST_PATH_PREFIX = "embulk_input_s3_test";
37+
38+
/*
39+
* This test case requires environment variables:
40+
* EMBULK_S3_TEST_BUCKET
41+
* EMBULK_S3_TEST_ACCESS_KEY_ID
42+
* EMBULK_S3_TEST_SECRET_ACCESS_KEY
43+
* If the variables not set, the test case is skipped.
44+
*/
45+
@BeforeClass
46+
public static void initializeConstantVariables()
47+
{
48+
EMBULK_S3_TEST_BUCKET = System.getenv("EMBULK_S3_TEST_BUCKET");
49+
EMBULK_S3_TEST_ACCESS_KEY_ID = System.getenv("EMBULK_S3_TEST_ACCESS_KEY_ID");
50+
EMBULK_S3_TEST_SECRET_ACCESS_KEY = System.getenv("EMBULK_S3_TEST_SECRET_ACCESS_KEY");
51+
assumeNotNull(EMBULK_S3_TEST_BUCKET, EMBULK_S3_TEST_ACCESS_KEY_ID, EMBULK_S3_TEST_SECRET_ACCESS_KEY);
52+
}
53+
54+
@Rule
55+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
56+
57+
private ConfigSource config;
58+
private FileInputRunner runner;
59+
private TestPageBuilderReader.MockPageOutput output;
60+
61+
@Before
62+
public void createResources()
63+
{
64+
config = runtime.getExec().newConfigSource()
65+
.set("type", "s3")
66+
.set("bucket", EMBULK_S3_TEST_BUCKET)
67+
.set("path_prefix", EMBULK_S3_TEST_PATH_PREFIX)
68+
.set("parser", parserConfig(schemaConfig()));
69+
runner = new FileInputRunner(runtime.getInstance(S3FileInputPlugin.class));
70+
output = new TestPageBuilderReader.MockPageOutput();
71+
}
72+
73+
private void doTest(ConfigSource config)
74+
{
75+
ConfigDiff configDiff = runner.transaction(config, new Control(runner, output));
76+
77+
assertEquals(EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv", configDiff.get(String.class, "last_path"));
78+
assertRecords(config, output);
79+
}
80+
81+
@Test
82+
public void useBasic()
83+
{
84+
ConfigSource config = this.config.deepCopy()
85+
.set("auth_method", "basic")
86+
.set("access_key_id", EMBULK_S3_TEST_ACCESS_KEY_ID)
87+
.set("secret_access_key", EMBULK_S3_TEST_SECRET_ACCESS_KEY);
88+
doTest(config);
89+
}
90+
91+
@Test
92+
public void useEnv()
93+
{
94+
// TODO
95+
}
96+
97+
@Test
98+
public void useInstance()
99+
{
100+
// TODO
101+
}
102+
103+
@Test
104+
public void useProfile()
105+
{
106+
// TODO
107+
}
108+
109+
@Test
110+
public void useProperties()
111+
{
112+
String origAccessKeyId = System.getProperty("aws.accessKeyId");
113+
String origSecretKey = System.getProperty("aws.secretKey");
114+
try {
115+
116+
ConfigSource config = this.config.deepCopy().set("auth_method", "properties");
117+
System.setProperty("aws.accessKeyId", EMBULK_S3_TEST_ACCESS_KEY_ID);
118+
System.setProperty("aws.secretKey", EMBULK_S3_TEST_SECRET_ACCESS_KEY);
119+
doTest(config);
120+
}
121+
finally {
122+
if (origAccessKeyId != null) {
123+
System.setProperty("aws.accessKeyId", origAccessKeyId);
124+
}
125+
if (origSecretKey != null) {
126+
System.setProperty("aws.secretKey", origAccessKeyId);
127+
}
128+
}
129+
}
130+
131+
@Test
132+
public void useAnonymous()
133+
{
134+
// TODO
135+
}
136+
137+
@Test
138+
public void useSession()
139+
{
140+
BasicSessionCredentials sessionCredentials = getSessionCredentials();
141+
ConfigSource config = this.config.deepCopy()
142+
.set("auth_method", "session")
143+
.set("access_key_id", sessionCredentials.getAWSAccessKeyId())
144+
.set("secret_access_key", sessionCredentials.getAWSSecretKey())
145+
.set("session_token", sessionCredentials.getSessionToken());
146+
doTest(config);
147+
}
148+
149+
private static BasicSessionCredentials getSessionCredentials()
150+
{
151+
AWSSecurityTokenServiceClient stsClient = new AWSSecurityTokenServiceClient(
152+
new StaticCredentialsProvider(new BasicAWSCredentials(EMBULK_S3_TEST_ACCESS_KEY_ID, EMBULK_S3_TEST_SECRET_ACCESS_KEY)));
153+
154+
GetFederationTokenRequest getFederationTokenRequest = new GetFederationTokenRequest();
155+
getFederationTokenRequest.setDurationSeconds(7200);
156+
getFederationTokenRequest.setName("dummy");
157+
158+
Policy policy = new Policy().withStatements(new Statement(Statement.Effect.Allow)
159+
.withActions(S3Actions.ListObjects, S3Actions.GetObject)
160+
.withResources(
161+
new Resource("arn:aws:s3:::" + EMBULK_S3_TEST_BUCKET + "/" + EMBULK_S3_TEST_PATH_PREFIX + "/*"),
162+
new Resource("arn:aws:s3:::" + EMBULK_S3_TEST_BUCKET)));
163+
getFederationTokenRequest.setPolicy(policy.toJson());
164+
165+
GetFederationTokenResult federationTokenResult = stsClient.getFederationToken(getFederationTokenRequest);
166+
Credentials sessionCredentials = federationTokenResult.getCredentials();
167+
168+
return new BasicSessionCredentials(
169+
sessionCredentials.getAccessKeyId(),
170+
sessionCredentials.getSecretAccessKey(),
171+
sessionCredentials.getSessionToken());
172+
}
173+
}

0 commit comments

Comments
 (0)