Skip to content

Commit 3f62cdf

Browse files
guillimguillim
committed
First commit
0 parents  commit 3f62cdf

File tree

7 files changed

+127
-0
lines changed

7 files changed

+127
-0
lines changed

‎.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
env.txt
2+
.ssh/key
3+
configuration.yml

‎.ssh/keyexample

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
-----BEGIN RSA PRIVATE KEY-----
2+
FDSFSFDSFDSDSSDDS/fsfsdjfljslkjdslfjdlsfjlfjdslfdcdfdsfdsfdsfdsf
3+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
4+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
5+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
6+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
7+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
8+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
9+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
10+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
11+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
12+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
13+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
14+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
15+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
16+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
17+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
18+
djdldjsljdkdlfjdljfljdljsldjdslfjdsljfdsjfdjkdfjskldfldjsfljdfjs
19+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
20+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
21+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
22+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
23+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
24+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
25+
fsfsdjfljslkjdslfjdlsfjlfjdslfdjklfjlfjfjdslfsdjfdslfjdlfjdslfjs
26+
fsfsdjfljslkjdslfjdlsfjlfjdslfd/G+fdsFdsffsfdsfdsfdsfdf=
27+
-----END RSA PRIVATE KEY-----

‎Dockerfile

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
FROM java:8
2+
3+
# help here : https://unix.stackexchange.com/questions/508724/failed-to-fetch-jessie-backports-repository
4+
5+
RUN echo "deb [check-valid-until=no] http://cdn-fastly.deb.debian.org/debian jessie main" > /etc/apt/sources.list.d/jessie.list
6+
RUN echo "deb [check-valid-until=no] http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie-backports.list
7+
RUN sed -i '/deb http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list
8+
RUN apt-get -o Acquire::Check-Valid-Until=false update
9+
10+
RUN apt-get install openssh-client
11+
12+
SHELL ["/bin/bash", "-c"]
13+
14+
RUN curl --create-dirs -o ~/.embulk/bin/embulk -L "https://dl.embulk.org/embulk-latest.jar"
15+
RUN chmod +x ~/.embulk/bin/embulk
16+
RUN echo 'export PATH="$HOME/.embulk/bin:$PATH"' >> ~/.bashrc
17+
RUN source ~/.bashrc
18+
19+
RUN ~/.embulk/bin/embulk gem install embulk-output-postgresql
20+
RUN ~/.embulk/bin/embulk gem install embulk-input-mongodb
21+
22+
RUN mkdir work
23+
COPY * /work/
24+
COPY .ssh/* /work/.ssh/
25+
26+
WORKDIR /work
27+
28+
RUN chmod -R 600 /work/.ssh/*
29+
RUN chmod +x script.sh
30+
31+
ARG SSHKEY=default_SSHKEY
32+
ARG TUNNEL_HOST=default_TUNNEL_HOST
33+
ARG LOCAL_PORT=default_LOCAL_PORT
34+
ARG REMOTE_HOST=default_REMOTE_HOST
35+
ARG REMOTE_PORT=default_REMOTE_PORT
36+
37+
# could be removed if we don't use it in script.sh
38+
ENV SSHKEY=${SSHKEY}
39+
ENV TUNNEL_HOST=${TUNNEL_HOST}
40+
ENV LOCAL_PORT=${LOCAL_PORT}
41+
ENV REMOTE_HOST=${REMOTE_HOST}
42+
ENV REMOTE_PORT=${REMOTE_PORT}
43+
44+
# could be removed i think
45+
EXPOSE 1-65535
46+
47+
# starting in the background ssh tunnel
48+
ENTRYPOINT ssh \
49+
-4 \
50+
-q \
51+
-o StrictHostKeyChecking=no \
52+
-i /work/.ssh/$SSHKEY \
53+
-L *:$LOCAL_PORT:$REMOTE_HOST:$REMOTE_PORT \
54+
-fN \
55+
$TUNNEL_HOST \
56+
&& bash

‎README.md

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Embulk as a Micro-service
2+
3+
This project aims to facilitate deploying embulk as a micro-service through SSH tunneling
4+
5+
### What is does
6+
7+
1. connect to your Mongo database
8+
2. do the job you tell him to do in the configuration.yml file
9+
3. connect to your postgres destination database and write what you want
10+
11+
Every connection is done using SSH tunneling.
12+
13+
### Can it be on hosted on PAAS ?
14+
15+
Yes, you can host it on heroku for instance.
16+
17+
### How can I install it ?
18+
19+
Pre requisite : you need Docker installed on your machine
20+
21+
Then, you have to :
22+
- put your ssh key (private) in the .ssh folder => this key will allow this machine to connect to the remote database
23+
- customize the environment variables in the environment_variables.txt file
24+
- copy configuration_example.yml to configuration.yml and modify it according to your needs
25+
- run `docker build --tag embulk_container .`
26+
- run `docker run --env-file=environment_variables.txt -it embulk_container bash`
27+
- run `./script.sh` inside the docker, in the /work folder

‎configuration_example.yml

Whitespace-only changes.

‎environment_variables.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
SSHKEY=default_env_SSHKEY
2+
TUNNEL_HOST=default_env_TUNNEL_HOST
3+
LOCAL_PORT=default_env_LOCAL_PORT
4+
REMOTE_HOST=default_env_REMOTE_HOST
5+
REMOTE_PORT=default_env_REMOTE_PORT

‎script.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
echo 'begining Embulk script'
2+
3+
# if you need to check what plugins are installed :
4+
# embulk gem list
5+
6+
embulk preview configuration.yml
7+
embulk run configuration.yml
8+
9+
echo 'Finished embulk'

0 commit comments

Comments
 (0)