Mongodb Integration

lib/generators-aux.js

@@ -1,21 +1,22 @@
-
 /**
  * Get the default database key of a given model.
  * @param {string} dataModel data model definition object
  */
 exports.getModelDatabase = function (dataModel) {
-
   // Sanity check: storageType is a required property, but database
   // should be set only for supported storage types.
   const validStorage = {
-    'sql': 'default-sql',
-    'sql-adapter': 'default-sql',
-  }
+    sql: "default-sql",
+    "sql-adapter": "default-sql",
+    cassandra: "default-cassandra",
+    "cassandra-adapter": "default-cassandra",
+    mongodb: "default-mongodb",
+    "mongodb-adapter": "default-mongodb",
+  };
 
   const storageType = dataModel.storageType.toLowerCase();
 
   const defaultDb = validStorage[storageType];
 
-  return dataModel.database || defaultDb || '';
-
-}
+  return dataModel.database || defaultDb || "";
+};

package.json

@@ -11,7 +11,7 @@
     "graphql-server-model-codegen": "index.js"
   },
   "scripts": {
-    "test-unit": "mocha ./test/mocha_unit_test.js",
+    "test-unit": "mocha ./test/mocha_unit.test.js",
     "test-integration": "bash ./test/testenv_cli.sh"
   },
   "author": "Science-DB team",

test/integration_test_misc/Dockerfile.cassandra

@@ -0,0 +1,5 @@
+FROM cassandra:3.11.6
+
+RUN mkdir -p /home/cassandra/.cassandra && chown -R cassandra /home/cassandra
+
+CMD ["cassandra", "-f"]

test/integration_test_misc/animal.csv

@@ -0,0 +1,7 @@
+animal_id,animal_name,category,age,weight,health,birthday,personality
+1,Sally1,Dog,3,5.5,true,2017-10-03T10:15:30Z,energetic;enthusiastic
+2,Sally2,Dog,3,5.5,true,2017-10-03T10:15:30Z,energetic;enthusiastic
+3,Milka1,Cat,2,3.5,true,2017-10-03T10:15:30Z,cute;optimistic
+4,Milka2,Cat,2,3.5,true,2017-10-03T10:15:30Z,cute;optimistic
+5,Lily1,Pig,1,10.5,true,2017-10-03T10:15:30Z,forgetful;funny
+6,Lily2,Pig,1,10.5,true,2017-10-03T10:15:30Z,forgetful;funny

test/integration_test_misc/cassandra-keyspace.cql

@@ -0,0 +1 @@
+CREATE KEYSPACE IF NOT EXISTS sciencedb WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1};

test/integration_test_misc/data_models_storage_config1.json

@@ -6,5 +6,21 @@
     "database": "sciencedb_development",
     "host": "gql_postgres1",
     "dialect": "postgres"
+  },
+  "default-mongodb": {
+    "storageType": "mongodb",
+    "username": "sciencedb",
+    "password": "sciencedb",
+    "database": "sciencedb_development",
+    "host": "gql_mongodb1",
+    "port": "27017"
+  },
+  "default-cassandra": {
+    "storageType": "cassandra",
+    "username": "cassandra",
+    "password": "cassandra",
+    "keyspace": "sciencedb",
+    "host": "cassandra1",
+    "port": "9042"
   }
 }

test/integration_test_misc/docker-compose-test.yml

@@ -11,10 +11,41 @@ services:
     networks:
      - instance1
 
+  gql_mongodb1:
+    image: mongo
+    container_name: mongodb1
+    environment:
+      - MONGO_INITDB_ROOT_USERNAME=sciencedb
+      - MONGO_INITDB_ROOT_PASSWORD=sciencedb
+      - MONGO_INITDB_DATABASE=sciencedb_development
+    volumes:
+      - ./init-mongo.js:/docker-entrypoint-initdb.d/init-mongo.js:ro
+    ports:
+      - 27017:27017
+    networks:
+      - instance1
+
+  gql_cassandra1:
+    container_name: cassandra1
+    build:
+      context: .
+      dockerfile: Dockerfile.cassandra
+    volumes: 
+      - ./docker-entrypoint.sh.cassandra:/docker-entrypoint.sh
+      - ./cassandra-keyspace.cql:/docker-entrypoint-initdb.d/cassandra-keyspace.cql
+    ports: 
+      - 9042:9042
+    environment: 
+      - MAX_HEAP_SIZE=256M
+      - HEAP_NEWSIZE=128M
+    networks: 
+      - instance1
+
   gql_science_db_graphql_server1:
     container_name: server1
     depends_on:
       - gql_postgres1
+      - gql_mongodb1
     build:
       context: .
       dockerfile: Dockerfile.graphql_server

test/integration_test_misc/docker-entrypoint.sh.cassandra

@@ -0,0 +1,105 @@
+#!/bin/bash
+set -e
+
+###############################
+# This file was copied from the Cassandra container and modified according to https://stackoverflow.com/a/42698847.
+# The modification runs all '.sh' and '.cql' files in the 'docker-entrypoint-initdb.d' folder. This mimics the
+# behaviour of e.g. the default postgres container.
+###############################
+
+# first arg is `-f` or `--some-option`
+# or there are no args
+if [ "$#" -eq 0 ] || [ "${1#-}" != "$1" ]; then
+	set -- cassandra -f "$@"
+fi
+
+# allow the container to be started with `--user`
+if [ "$1" = 'cassandra' -a "$(id -u)" = '0' ]; then
+	find "$CASSANDRA_CONF" /var/lib/cassandra /var/log/cassandra \
+		\! -user cassandra -exec chown cassandra '{}' +
+	exec gosu cassandra "$BASH_SOURCE" "$@"
+fi
+
+_ip_address() {
+	# scrape the first non-localhost IP address of the container
+	# in Swarm Mode, we often get two IPs -- the container IP, and the (shared) VIP, and the container IP should always be first
+	ip address | awk '
+		$1 == "inet" && $NF != "lo" {
+			gsub(/\/.+$/, "", $2)
+			print $2
+			exit
+		}
+	'
+}
+
+# "sed -i", but without "mv" (which doesn't work on a bind-mounted file, for example)
+_sed-in-place() {
+	local filename="$1"; shift
+	local tempFile
+	tempFile="$(mktemp)"
+	sed "$@" "$filename" > "$tempFile"
+	cat "$tempFile" > "$filename"
+	rm "$tempFile"
+}
+
+if [ "$1" = 'cassandra' ]; then
+	: ${CASSANDRA_RPC_ADDRESS='0.0.0.0'}
+
+	: ${CASSANDRA_LISTEN_ADDRESS='auto'}
+	if [ "$CASSANDRA_LISTEN_ADDRESS" = 'auto' ]; then
+		CASSANDRA_LISTEN_ADDRESS="$(_ip_address)"
+	fi
+
+	: ${CASSANDRA_BROADCAST_ADDRESS="$CASSANDRA_LISTEN_ADDRESS"}
+
+	if [ "$CASSANDRA_BROADCAST_ADDRESS" = 'auto' ]; then
+		CASSANDRA_BROADCAST_ADDRESS="$(_ip_address)"
+	fi
+	: ${CASSANDRA_BROADCAST_RPC_ADDRESS:=$CASSANDRA_BROADCAST_ADDRESS}
+
+	if [ -n "${CASSANDRA_NAME:+1}" ]; then
+		: ${CASSANDRA_SEEDS:="cassandra"}
+	fi
+	: ${CASSANDRA_SEEDS:="$CASSANDRA_BROADCAST_ADDRESS"}
+
+	_sed-in-place "$CASSANDRA_CONF/cassandra.yaml" \
+		-r 's/(- seeds:).*/\1 "'"$CASSANDRA_SEEDS"'"/'
+
+	for yaml in \
+		broadcast_address \
+		broadcast_rpc_address \
+		cluster_name \
+		endpoint_snitch \
+		listen_address \
+		num_tokens \
+		rpc_address \
+		start_rpc \
+	; do
+		var="CASSANDRA_${yaml^^}"
+		val="${!var}"
+		if [ "$val" ]; then
+			_sed-in-place "$CASSANDRA_CONF/cassandra.yaml" \
+				-r 's/^(# )?('"$yaml"':).*/\2 '"$val"'/'
+		fi
+	done
+
+	for rackdc in dc rack; do
+		var="CASSANDRA_${rackdc^^}"
+		val="${!var}"
+		if [ "$val" ]; then
+			_sed-in-place "$CASSANDRA_CONF/cassandra-rackdc.properties" \
+				-r 's/^('"$rackdc"'=).*/\1 '"$val"'/'
+		fi
+	done
+fi
+
+for f in docker-entrypoint-initdb.d/*; do
+    case "$f" in
+        *.sh)     echo "$0: running $f"; . "$f" ;;
+        *.cql)    echo "$0: running $f" && until cqlsh -f "$f"; do >&2 echo "Cassandra is unavailable - sleeping"; sleep 2; done & ;;
+        *)        echo "$0: ignoring $f" ;;
+    esac
+    echo
+done
+
+exec "$@"

test/integration_test_misc/init-mongo.js

@@ -0,0 +1,10 @@
+db.createUser({
+  user: "sciencedb",
+  pwd: "sciencedb",
+  roles: [
+    {
+      role: "readWrite",
+      db: "sciencedb_development",
+    },
+  ],
+});

test/integration_test_misc/integration_test_models_instance1/animal.json

@@ -0,0 +1,47 @@
+{
+    "model": "animal",
+    "storageType": "mongodb",
+    "attributes": {
+        "animal_id": "String",
+        "category": "String",
+        "animal_name": "String",
+        "age": "Int",
+        "weight": "Float",
+        "health": "Boolean",
+        "birthday": "DateTime",
+        "personality": "[String]",
+        "farm_id": "String",
+        "food_ids": "[String]"
+    },
+    "associations": {
+        "farm": {
+            "type": "to_one",
+            "target": "farm",
+            "targetKey": "farm_id",
+            "keyIn": "animal",
+            "targetStorageType": "mongodb",
+            "label": "farm_name"
+        },
+        "food":{
+            "type": "to_many",
+            "reverseAssociationType": "to_many",
+            "target": "food",
+            "targetKey": "animal_ids",
+            "sourceKey": "food_ids",
+            "keyIn": "animal",
+            "targetStorageType": "mongodb"
+        },
+        "unique_tracker":{
+            "type" : "to_one",
+            "target" : "tracker",
+            "targetKey" : "animal_id",
+            "keyIn": "tracker",
+            "targetStorageType" : "mongodb"
+        }
+    },
+    "internalId": "animal_id",
+    "id": {
+        "name": "animal_id",
+        "type": "String"
+    }
+}

test/integration_test_misc/integration_test_models_instance1/capital.json

@@ -13,6 +13,13 @@
         "targetKey" : "country_id",
         "keyIn": "capital",
         "targetStorageType" : "sql"
+      },
+      "incidents": {
+        "type": "to_many",
+        "target": "Incident",
+        "targetKey": "capital_id",
+        "keyIn": "Incident",
+        "targetStorageType": "cassandra"
       }
     },
     "internalId": "capital_id"

test/integration_test_misc/integration_test_models_instance1/cassandra-city.json

@@ -0,0 +1,28 @@
+{
+  "model" : "city",
+  "storageType" : "cassandra",
+  "attributes" : {
+    "city_id": "String",
+    "name": "String",
+
+    "intArr": "[Int]",
+    "strArr": "[String]",
+    "floatArr": "[Float]",
+    "boolArr": "[Boolean]",
+    "dateTimeArr": "[DateTime]",
+
+    "river_ids": "[String]"
+  },
+  "associations": {
+    "rivers": {
+      "type": "to_many",
+      "target": "river",
+      "targetStorageType": "sql",
+      "sourceKey": "river_ids",
+      "targetKey": "city_ids",
+      "keyIn": "city",
+      "reverseAssociationType":"to_many"
+    }
+  },
+  "internalId": "city_id"
+}

test/integration_test_misc/integration_test_models_instance1/cassandra-incident.json

@@ -0,0 +1,31 @@
+{
+    "model": "Incident",
+    "storageType": "cassandra",
+    "attributes": {
+      "incident_id": "String",
+      "incident_description": "String",
+      "incident_number": "Int",
+      "capital_id": "String"
+    },
+
+    "associations": {
+
+      "instants": {
+        "type": "to_many",
+        "target": "Instant",
+        "targetKey": "incident_assoc_id",
+        "keyIn" : "Instant",
+        "targetStorageType": "cassandra"
+      },
+      "town": {
+        "type": "to_one",
+        "target": "capital",
+        "targetKey": "capital_id",
+        "keyIn" : "Incident",
+        "targetStorageType": "sql"
+      }
+
+    },
+
+    "internalId" : "incident_id"
+  }

test/integration_test_misc/integration_test_models_instance1/cassandra-instant.json

@@ -0,0 +1,27 @@
+{
+    "model": "Instant",
+    "storageType": "cassandra",
+    "attributes": {
+      "instant_id": "String",
+      "incident_assoc_id": "String",
+      "year": "Int",
+      "month": "Int",
+      "day": "Int",
+      "hour": "Int",
+      "minute": "Int"
+    },
+
+    "associations": {
+
+      "incident": {
+        "type": "to_one",
+        "target": "Incident",
+        "targetKey": "incident_assoc_id",
+        "keyIn" : "Instant",
+        "targetStorageType": "cassandra"
+      }
+
+    },
+
+    "internalId" : "instant_id"
+  }