[add] updated dockerfile. [add] added circleci config

Author: filipecosta90

.circleci/config.yml

@@ -0,0 +1,29 @@
+# Python CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-python/ for more details
+#
+version: 2.1
+jobs:
+  build-multiarch-docker:
+    machine:
+      enabled: true
+    steps:
+      - checkout
+      - run: |
+          echo "$DOCKER_REDISBENCH_PWD" | base64 --decode | docker login --username $DOCKER_REDISBENCH_USER --password-stdin
+      - run:
+          name: Build
+          command: |
+            make -C benchmark/redisgraph docker-release
+          no_output_timeout: 20m
+
+workflows:
+  version: 2
+  commit:
+    jobs:
+      - build-multiarch-docker:
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              only: master

.gitignore

@@ -30,6 +30,7 @@ benchmark/redisgraph/result_redisgraph/**
 # Gradle
 .idea/**/gradle.xml
 .idea/**/libraries
+.idea/**
 
 # Gradle and Maven with auto-import
 # When using Gradle or Maven with auto-import, you should exclude module files,
@@ -207,4 +208,7 @@ venv.bak/
 dmypy.json
 
 # Pyre type checker
-.pyre/
+.pyre/
+
+# Mac
+*.DS_Store

README.md

@@ -8,7 +8,6 @@ This repo contains code for benchmarking Graph databases.
 - each vendor's benchmark is under
  /benchmark/vendor_name/
 - start with README under each folder
-- all test can be reproducible on EC2 or similar enviroment.
 
 Contact: benchmark@tigergraph.com
 

benchmark/redisgraph/Dockerfile

@@ -1,8 +1,21 @@
 FROM python:2.7.10
-COPY . ./
 
-RUN ./scripts/retrieve_seed_and_unique_node_data_from_s3.sh
+COPY scripts ./scripts
+COPY bulk_insert.py ./
+COPY config.py ./
+COPY generate_graph500_inputs.py ./
+COPY generate_twitter_inputs.py ./
+COPY graph_query.py ./
+COPY kn.py ./
+COPY query_runner.py ./
+COPY requirements.txt ./
+COPY docker_entrypoint.sh ./
+
+#RUN ./scripts/retrieve_seed_and_unique_node_data_from_s3.sh
 
 RUN python -m pip install -r requirements.txt
 
-ENTRYPOINT ["./kn.py"]
+
+RUN chmod -R 751 scripts
+RUN chmod 751 docker_entrypoint.sh
+ENTRYPOINT ["./docker_entrypoint.sh"]

benchmark/redisgraph/Makefile

@@ -0,0 +1,46 @@
+# Go parameters
+# DOCKER
+DOCKER_APP_NAME=graph-database-benchmark-redisgraph
+DOCKER_ORG=redisbench
+DOCKER_REPO:=${DOCKER_ORG}/${DOCKER_APP_NAME}
+#DOCKER_TAG:=$(shell git log -1 --pretty=format:"%h")
+DOCKER_TAG=edge
+DOCKER_IMG:="$(DOCKER_REPO):$(DOCKER_TAG)"
+DOCKER_LATEST:="${DOCKER_REPO}:latest"
+
+# DOCKER TASKS
+# Build the container
+docker-build:
+	docker build -t $(DOCKER_APP_NAME):latest -f  Dockerfile .
+
+# Build the container without caching
+docker-build-nc:
+	docker build --no-cache -t $(DOCKER_APP_NAME):latest -f  Dockerfile .
+
+# Make a release by building and publishing the `{version}` ans `latest` tagged containers to ECR
+docker-release: docker-build-nc docker-publish
+
+# Docker publish
+docker-publish: docker-publish-latest docker-publish-version ## Publish the `{version}` ans `latest` tagged containers to ECR
+
+docker-repo-login: ## login to DockerHub with credentials found in env
+	docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD}
+
+docker-publish-latest: docker-tag-latest ## Publish the `latest` tagged container to ECR
+	@echo 'publish latest to $(DOCKER_REPO)'
+	docker push $(DOCKER_LATEST)
+
+docker-publish-version: docker-tag-version ## Publish the `{version}` tagged container to ECR
+	@echo 'publish $(DOCKER_IMG) to $(DOCKER_REPO)'
+	docker push $(DOCKER_IMG)
+
+# Docker tagging
+docker-tag: docker-tag-latest docker-tag-version ## Generate container tags for the `{version}` ans `latest` tags
+
+docker-tag-latest: ## Generate container `{version}` tag
+	@echo 'create tag latest'
+	docker tag $(DOCKER_APP_NAME) $(DOCKER_LATEST)
+
+docker-tag-version: ## Generate container `latest` tag
+	@echo 'create tag $(DOCKER_APP_NAME) $(DOCKER_REPO):$(DOCKER_IMG)'
+	docker tag $(DOCKER_APP_NAME) $(DOCKER_IMG)

benchmark/redisgraph/README.md

@@ -19,7 +19,7 @@ The easiest way to get and install the benchmark code is to use:
 ```bash
 git clone https://github.com/RedisGraph/graph-database-benchmark.git
 cd graph-database-benchmark/benchmark/redisgraph
-sudo pip install -r requirements.txt
+pip install --user -r requirements.txt
 ```
 
 #### Data retrieval 
@@ -32,7 +32,7 @@ Variables:
 1. `EDGE_FILE` (default: `dependent on the use case`)
 1. `NODE_FILE` (default: `dependent on the use case`)
 
-The easiest way to get the datasets required for the benchmark is to use one of the helper scrips:
+The easiest way to get the datasets required for the benchmark is to use one of the helper scripts:
 1. `./scripts/get_graph500_dataset.sh`
 or
 1. `./scripts/get_twitter_dataset.sh`

benchmark/redisgraph/docker_entrypoint.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+echo "--------------------------------------------------"
+echo "graph database benchmark RedisGraph - Docker Image"
+echo "--------------------------------------------------"
+echo "Checking if request binary $1 exists"
+if [ -f ./$1 ]; then
+  ./"$@"
+  echo
+  echo "...done."
+  exit 0
+else
+  echo "$1 binary does not exist."
+  exit 1
+fi

benchmark/redisgraph/graph500_22_seed

@@ -30,7 +30,7 @@ if [ ! -f ${DATA_EDGE_FILE_NAME} ]; then
   curl -O ${EDGE_FILE} | tar zxf -  >  ${DATA_EDGE_FILE_NAME}
 else
   echo "Dataset found locally at ${DATA_EDGE_FILE_NAME}. No need to retrieve again."
-fi./
+fi
 
 echo ""
 echo "---------------------------------------------------------------------------------"

benchmark/redisgraph/scripts/get_twitter_dataset.sh

@@ -5,8 +5,8 @@ DATASET_DIR=${DATASET_DIR:-"graph500_22"}
 EXE_DIR=${EXE_DIR:-$(dirname $0)}
 source ${EXE_DIR}/common.sh
 
-# Prepare input files in RedisGraph bulk import format
-python generate_graph500_inputs.py --inputdir ${DATASET} || exit 1
+## Prepare input files in RedisGraph bulk import format
+#python generate_graph500_inputs.py --inputdir ${DATASET} || exit 1
 
 # Run RedisGraph bulk import script
 python bulk_insert.py graph500_22 -n data/graph500_22_unique_node -r data/graph500_22 \