@@ -7,43 +7,6 @@ password=$4
7
7
8
8
cd
9
9
SQOOP_INSTALL_DIR=./sqoop-1.4.6.bin__hadoop-2.0.4-alpha/bin/
10
- # SCRIPT_FILE = s3://supply-emr-qa/scripts/sqoop.challengeSkills.txt\
11
-
12
- # while getopts ":f:" opt; do
13
- # case $opt in
14
- # f)
15
- # SCRIPT_FILE="$OPTARG"
16
- # ;;
17
- # \?)
18
- # echo "Invalid option: -$OPTARG" >&2
19
- # exit 1
20
- # ;;
21
- # :)
22
- # echo "Option -$OPTARG requires an argument." >&2
23
- # exit 1
24
- # ;;
25
- # esac
26
- # done
27
- #
28
- # if [ -z "$SCRIPT_FILE" ]; then
29
- # echo "-f file is required."
30
- # exit 1
31
- # fi
32
- # if [ -f "./scoop.options.txt" ]; then
33
- # rm ./sqoop.options.txt
34
- # fi
35
- #
36
- # echo "Fetching $SCRIPT_FILE... "
37
- # aws s3 cp $SCRIPT_FILE ./sqoop.options.txt
38
- # rc=$?; if [[ $rc != 0 ]]; then exit $rc; fi
39
- #
40
- # echo "$SCRIPT_FILE download done..."
41
- #
42
- #
43
- # if [ ! -f "./sqoop.options.txt" ]; then
44
- # echo "Options file is missing;"
45
- # exit 1
46
- # fi
47
10
48
11
# verify SQOOP_INSTALL DIR is set and is valid
49
12
if [ ! -d " $SQOOP_INSTALL_DIR " ]; then
@@ -52,10 +15,7 @@ if [ ! -d "$SQOOP_INSTALL_DIR" ]; then
52
15
fi
53
16
54
17
# finally execute the command
55
-
56
- echo " RUNNING OPTIONS file"
57
- # $SQOOP_INSTALL_DIR/sqoop --options-file ./sqoop.options.txt
58
-
18
+ echo " RUNNING SQOOP command"
59
19
$SQOOP_INSTALL_DIR /sqoop import --driver $jdbcdriver --connect $connect --username $user --password $password --verbose --query ' SELECT project_result.user_id userId, coder.handle handle, project_result.project_id challengeId, project_technology.name FROM tcs_dw:project_result INNER JOIN tcs_dw:project as project ON project.project_id = project_result.project_id INNER JOIN topcoder_dw:coder as coder ON project_result.user_id = coder.coder_id INNER JOIN tcs_dw:project_technology ON project_technology.project_id = project.project_id WHERE project_result.passed_review_ind = 1 AND $CONDITIONS' --boundary-query ' select min(coder.coder_id), max(coder.coder_id) from topcoder_dw:coder as coder' --target-dir hdfs:///user/supply/skills/input/challenge --split-by project_result.project_id
60
20
# fixme
61
21
# rc=$?; if [[ $rc != 0 ]]; then exit $rc; fi
0 commit comments