Skip to content

Commit 61923bc

Browse files
jordanw-bqabhishekdas99
authored andcommitted
new: test: #285: Add support for multi-node integration tests (#291)
1 parent bc671d8 commit 61923bc

31 files changed

+914
-37
lines changed

.travis.yml

+5-4
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,10 @@ cache:
1111
- $HOME/.m2
1212
sudo: enabled
1313
before_install:
14-
- docker pull quboleinc/hadoop_mvn_thrift
14+
- /bin/bash rubix_setup_build.sh
1515
install: true
16-
script:
17-
- docker run -it --rm -v "$PWD":/usr/src/rubix -v "$HOME/.m2":/root/.m2 $ci_env -w /usr/src/rubix quboleinc/hadoop_mvn_thrift /bin/bash script.sh
16+
script: chmod a+x docker_build_rubix.sh && /bin/bash rubix_run_build.sh mvn integration-test -rf :rubix-tests
1817
env:
19-
- ci_env=`bash <(curl -s https://codecov.io/env)`
18+
global:
19+
- TRAVIS_ENV=`bash <(curl -s https://codecov.io/env)`
20+
- RUBIX_DOCKER_NETWORK_NAME=network-rubix-build

Dockerfile

+12
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ MAINTAINER Kamesh Vankayala - Qubole Inc.
44
## Install thrift 0.9.3
55
ENV THRIFT_VERSION 0.9.3
66

7+
RUN printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /etc/apt/sources.list
8+
79
RUN buildDeps=" \
810
automake \
911
bison \
@@ -63,4 +65,14 @@ ENV HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/hadoop2/share/hadoop/tools/lib/*
6365
ENV USER=root
6466
ENV PATH $HADOOP_PREFIX/bin/:$PATH
6567

68+
# Install Docker and Docker Compose for integration tests
69+
RUN set -x \
70+
&& curl -fSL "https://download.docker.com/linux/static/stable/x86_64/docker-17.09.0-ce.tgz" -o /tmp/docker-ce.tgz \
71+
&& tar -xvzf /tmp/docker-ce.tgz \
72+
&& cp docker/* /usr/local/bin
73+
74+
RUN set -x \
75+
&& curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose \
76+
&& chmod +x /usr/local/bin/docker-compose
77+
6678
RUN mkdir -p /media/ephemeral0

docker_build_rubix.sh

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash -e
2+
3+
build-rubix-jars-for-tests() {
4+
mvn clean install -DskipTests
5+
6+
HADOOP_LIB_DIR="/usr/lib/hadoop2/share/hadoop/tools/lib/"
7+
RUBIX_JARS=$(ls ${PWD}/rubix-*/target/rubix-*.jar | grep -v tests)
8+
9+
echo "=== Copying RubiX Jars to ${HADOOP_LIB_DIR} ==="
10+
sudo cp ${RUBIX_JARS} ${HADOOP_LIB_DIR}
11+
}
12+
13+
upload-coverage-results() {
14+
echo "=== Running tests with coverage ==="
15+
# "cobertura-integration-test" goal needed for shading JARs
16+
# if run on CI, integration tests will have been run before this
17+
mvn cobertura:cobertura-integration-test -Pno-integration-tests
18+
echo "=== Uploading code coverage results to Codecov ==="
19+
bash <(curl -s https://codecov.io/bash)
20+
}
21+
22+
build-rubix-jars-for-tests
23+
24+
MAVEN_CMD=$@
25+
26+
echo "=== Executing command \"${MAVEN_CMD}\" ==="
27+
${MAVEN_CMD}
28+
if [[ $? -ne 0 ]]; then
29+
exit 1
30+
fi
31+
32+
if [[ -n "${TRAVIS}" ]]; then
33+
upload-coverage-results
34+
fi

log4j_cache.properties

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
log4j.rootLogger=DEBUG, R
2+
3+
log4j.appender.R=org.apache.log4j.RollingFileAppender
4+
log4j.appender.R.File=/Users/jordanw/Development/Projects/Qubole/RubiX/watcher.log
5+
log4j.appender.R.MaxFileSize=100MB
6+
log4j.appender.R.MaxBackupIndex=5
7+
log4j.appender.R.layout=org.apache.log4j.PatternLayout
8+
log4j.appender.R.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss,SSS} %p %t %c{2}: %m%n
9+
10+
log4j.logger.com.qubole.rubix=DEBUG
11+
log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=DEBUG

mvnw

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
3+
MAVEN_CMD="mvn $@"
4+
5+
if [[ "$*" == *-DrunITs* ]]
6+
then
7+
export RUBIX_DOCKER_NETWORK_NAME=network-rubix-build
8+
source ./rubix_setup_build.sh
9+
source ./rubix_run_build.sh "${MAVEN_CMD}"
10+
else
11+
exec ${MAVEN_CMD} -Pno-integration-tests
12+
fi
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
/**
2+
* Copyright (c) 2019. Qubole Inc
3+
* Licensed under the Apache License, Version 2.0 (the License);
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
* Unless required by applicable law or agreed to in writing, software
8+
* distributed under the License is distributed on an AS IS BASIS,
9+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
* See the License for the specific language governing permissions and
11+
* limitations under the License. See accompanying LICENSE file.
12+
*/
13+
package com.qubole.rubix.client.robotframework.container.client;
14+
15+
import com.qubole.rubix.client.robotframework.container.server.RequestServer;
16+
17+
import java.rmi.NotBoundException;
18+
import java.rmi.RemoteException;
19+
import java.rmi.registry.LocateRegistry;
20+
import java.rmi.registry.Registry;
21+
import java.util.HashMap;
22+
import java.util.Map;
23+
24+
public class ContainerRequestClient
25+
{
26+
private static final String REQUEST_SERVER_NAME = "ContainerRequestServer";
27+
private static final int REGISTRY_PORT = 1099;
28+
private static final String FILE_SCHEME = "file:";
29+
30+
public ContainerRequestClient()
31+
{
32+
}
33+
34+
/**
35+
* Get the current cache metrics from the BookKeeper server on a particular node.
36+
*
37+
* @param host The hostname of the container to connect to.
38+
* @return A map of metrics describing cache statistics and interactions for that node.
39+
*/
40+
public Map<String, Double> getCacheMetricsForNode(String host)
41+
{
42+
try {
43+
final RequestServer containerServer = getRequestServer(host);
44+
return containerServer.getCacheMetrics(new GetCacheMetricsRequest());
45+
}
46+
catch (RemoteException | NotBoundException e) {
47+
System.err.println("ContainerRequestClient exception:");
48+
e.printStackTrace();
49+
}
50+
return new HashMap<>();
51+
}
52+
53+
/**
54+
* Read data from a given file into the BookKeeper cache using a client caching file system.
55+
*
56+
* @param host The hostname of the container to connect to.
57+
* @param remotePath The remote path location.
58+
* @param readStart The block to start reading from.
59+
* @param length The amount of data to read.
60+
* @param fileSize The length of the file.
61+
* @param lastModified The time at which the file was last modified.
62+
* @param clusterType The type id of cluster being used.
63+
* @return True if the data was read into the cache correctly, false otherwise.
64+
*/
65+
public boolean cacheDataUsingClientFileSystemForNode(
66+
String host,
67+
String remotePath,
68+
long readStart,
69+
int length,
70+
long fileSize,
71+
long lastModified,
72+
int clusterType)
73+
{
74+
ReadDataRequestParams params = new ReadDataRequestParams(
75+
getPathWithFileScheme(remotePath),
76+
readStart,
77+
length,
78+
fileSize,
79+
lastModified,
80+
clusterType);
81+
82+
try {
83+
RequestServer containerServer = getRequestServer(host);
84+
return containerServer.cacheDataUsingClientFileSystem(new ReadDataWithFileSystemRequest(), params);
85+
}
86+
catch (RemoteException | NotBoundException e) {
87+
System.err.println("ContainerRequestClient exception:");
88+
e.printStackTrace();
89+
}
90+
return false;
91+
}
92+
93+
/**
94+
* Locates a {@link RequestServer} for executing requests on a particular container.
95+
*
96+
* @param host The hostname for the container to connect to.
97+
* @return The {@link RequestServer} used for executing requests.
98+
* @throws RemoteException if the registry could not be located or communicated with.
99+
* @throws NotBoundException if the registry has not been bould.
100+
*/
101+
private static RequestServer getRequestServer(String host) throws RemoteException, NotBoundException
102+
{
103+
Registry registry = LocateRegistry.getRegistry(host, REGISTRY_PORT);
104+
return (RequestServer) registry.lookup(REQUEST_SERVER_NAME);
105+
}
106+
107+
/**
108+
* Add the file scheme to the provided path for proper execution with the BookKeeper server.
109+
*
110+
* @param path The path to update.
111+
* @return The provided path with the file scheme.
112+
*/
113+
private String getPathWithFileScheme(String path)
114+
{
115+
return FILE_SCHEME + path;
116+
}
117+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/**
2+
* Copyright (c) 2019. Qubole Inc
3+
* Licensed under the Apache License, Version 2.0 (the License);
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
* Unless required by applicable law or agreed to in writing, software
8+
* distributed under the License is distributed on an AS IS BASIS,
9+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
* See the License for the specific language governing permissions and
11+
* limitations under the License. See accompanying LICENSE file.
12+
*/
13+
package com.qubole.rubix.client.robotframework.container.client;
14+
15+
import com.qubole.rubix.spi.RetryingBookkeeperClient;
16+
import org.apache.thrift.shaded.TException;
17+
18+
import java.io.Serializable;
19+
import java.util.HashMap;
20+
import java.util.Map;
21+
22+
public class GetCacheMetricsRequest implements Serializable
23+
{
24+
public static final long serialVersionUID = 126L;
25+
26+
public Map<String, Double> execute(RetryingBookkeeperClient client)
27+
{
28+
try {
29+
return client.getCacheMetrics();
30+
}
31+
catch (TException ex) {
32+
ex.printStackTrace();
33+
}
34+
return new HashMap<>();
35+
}
36+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/**
2+
* Copyright (c) 2019. Qubole Inc
3+
* Licensed under the Apache License, Version 2.0 (the License);
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
* Unless required by applicable law or agreed to in writing, software
8+
* distributed under the License is distributed on an AS IS BASIS,
9+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
* See the License for the specific language governing permissions and
11+
* limitations under the License. See accompanying LICENSE file.
12+
*/
13+
package com.qubole.rubix.client.robotframework.container.client;
14+
15+
import java.io.Serializable;
16+
17+
public class ReadDataRequestParams implements Serializable
18+
{
19+
private final String remotePath;
20+
private final long readStart;
21+
private final int length;
22+
private final long fileSize;
23+
private final long lastModified;
24+
private final int clusterType;
25+
26+
public ReadDataRequestParams(String remotePath, long readStart, int length, long fileSize, long lastModified, int clusterType)
27+
{
28+
this.remotePath = remotePath;
29+
this.readStart = readStart;
30+
this.length = length;
31+
this.fileSize = fileSize;
32+
this.lastModified = lastModified;
33+
this.clusterType = clusterType;
34+
}
35+
36+
public String getRemotePath()
37+
{
38+
return remotePath;
39+
}
40+
41+
public long getReadStart()
42+
{
43+
return readStart;
44+
}
45+
46+
public int getLength()
47+
{
48+
return length;
49+
}
50+
51+
public long getFileSize()
52+
{
53+
return fileSize;
54+
}
55+
56+
public long getLastModified()
57+
{
58+
return lastModified;
59+
}
60+
61+
public int getClusterType()
62+
{
63+
return clusterType;
64+
}
65+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/**
2+
* Copyright (c) 2019. Qubole Inc
3+
* Licensed under the Apache License, Version 2.0 (the License);
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
* Unless required by applicable law or agreed to in writing, software
8+
* distributed under the License is distributed on an AS IS BASIS,
9+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
* See the License for the specific language governing permissions and
11+
* limitations under the License. See accompanying LICENSE file.
12+
*/
13+
package com.qubole.rubix.client.robotframework.container.client;
14+
15+
import org.apache.hadoop.fs.FSDataInputStream;
16+
17+
import java.io.IOException;
18+
import java.io.Serializable;
19+
20+
public class ReadDataWithFileSystemRequest implements Serializable
21+
{
22+
public static final long serialVersionUID = 126L;
23+
24+
public boolean execute(FSDataInputStream inputStream, ReadDataRequestParams params)
25+
{
26+
try {
27+
final int readSize = inputStream.read(
28+
new byte[params.getLength()],
29+
(int) params.getReadStart(),
30+
params.getLength());
31+
return readSize == params.getLength();
32+
}
33+
catch (IOException ex) {
34+
ex.printStackTrace();
35+
}
36+
return false;
37+
}
38+
}

0 commit comments

Comments
 (0)