Neo4J: docker-compose and scripts

Author: turiphro

neo4j/docker-compose.yml

@@ -0,0 +1,36 @@
+# Version from: June 2020
+
+version: "3.5"
+services:
+  scripts:
+    build: scripts
+    depends_on:
+      - coordinator
+    environment:
+      - COORDINATOR_NODE=coordinator
+    networks:
+      - cluster
+
+  coordinator: # first node
+    image: neo4j:3.5
+    ports:
+      - 7474:7474  # HTTP
+      - 7687:7687  # Bolt
+    environment:
+      - NEO4J_AUTH=none
+    networks:
+      - cluster
+    volumes:
+      - data:/data  # persistent; remove for ephemeral tasks
+
+
+    # The Open-Source version of Neo4j does *not* support multiple nodes;
+    # hence, we won't support clusters in this setup.
+
+
+networks:
+  cluster:
+    name: cluster
+
+volumes:
+  data: {}

neo4j/scripts/Dockerfile

@@ -0,0 +1,14 @@
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update \
+    && apt-get -y install \
+    httpie ruby ruby-dev \
+    jq \
+    build-essential \
+    net-tools dnsutils # network debugging
+
+RUN gem install faraday
+
+COPY . /scripts
+WORKDIR /scripts

neo4j/scripts/aggregate.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+export DB_HOST=http://$COORDINATOR_NODE:7474
+
+## Example queries
+echo "Available relationship types:"
+http GET "$DB_HOST/db/data/relationship/types"
+echo "Some node:"
+http GET "$DB_HOST/db/data/node/20"
+http GET "$DB_HOST/db/data/node/20/properties"
+http GET "$DB_HOST/db/data/node/20/relationships/all"
+
+## Run aggregations
+echo "Aggregations:"
+CYPHER="db/data/cypher"
+http POST "$DB_HOST/$CYPHER" query="MATCH (n) RETURN DISTINCT labels(n), count(n)"
+http POST "$DB_HOST/$CYPHER" query="MATCH (p:Actor)-[:ACTED_IN]->(m:Movie {name: 'The Matrix'}) RETURN p" | jq ".data[][] | {label: .metadata.labels, name: .data.name}" -c
+
+echo "Limited 'Kevin Bacon' number (here up til 3 degrees from Kevin Bacon = 6 hops)"
+http POST "$DB_HOST/$CYPHER" query='MATCH (Actor {name:"Kevin Bacon"})-[:ACTED_IN*1..6]-(other:Actor) return COUNT(DISTINCT other)'
+
+echo "A*:"
+http POST "$DB_HOST/$CYPHER" query='MATCH (bacon:Actor {name: "Kevin Bacon"}), (neo:Actor {name: "Keanu Reeves"}), path=shortestPath((bacon)-[:ACTED_IN*]-(neo)) RETURN length(path)'  # 2 hops = same movie!
+TOTAL=$(http POST "$DB_HOST/$CYPHER" query='MATCH (p:Actor) RETURN COUNT(DISTINCT p)' | jq ".data[][]")
+echo "Reverse A* up to 2 hops (much slower):"
+http POST "$DB_HOST/$CYPHER" query="MATCH paths=shortestPath((bacon:Actor {name: \"Kevin Bacon\"})-[:ACTED_IN*1..4]-(other:Actor)) WHERE bacon <> other RETURN COUNT(paths) / $TOTAL.0"  # 29% within 2 hops
+echo "Any connection to Kevin Bacon (rest is islands; runtime ~30s):"
+time http POST "$DB_HOST/$CYPHER" query="MATCH paths=shortestPath((bacon:Actor {name: \"Kevin Bacon\"})-[:ACTED_IN*]-(other:Actor)) WHERE bacon <> other RETURN COUNT(paths) / $TOTAL.0"  # 89.847% connected at all (89.838% connected within 6 degrees)

neo4j/scripts/create_users.json

@@ -0,0 +1,14 @@
+{
+    "statements": [
+        {
+            "statement": "CREATE (mr:Person {name: 'Martijn', from: 'Netherlands', occupation: 'software developer'})"
+        },
+        {
+            "statement": "MATCH (mr:Person) WHERE mr.name = 'Martijn' CREATE (on:Person {name: 'Onno', from: 'Netherlands'}), (mr)-[:KNOWS {since: 2018}]->(on), (on)-[:KNOWS]->(mr)"
+        },
+        {
+            "statement": "MATCH (mr:Person {name: 'Martijn'}), (fe:Cat {name: 'Felix'}) CREATE (mr)-[:living_together]->(fe)"
+        }
+    ]
+}
+

neo4j/scripts/importer.rb

@@ -0,0 +1,35 @@
+# Notes by turiphro:
+# - The tsv dataset from the first edition wasn't available anymore;
+#   the neo4j dump is outdated and hard to import from a separate Docker container.
+#   Therefore, I extracted an old version of the dataset from the 1GB+ archive (6GB+ data) at
+#   https://archive.org/download/freebase-data-dump-2010-07-16/freebase-datadump-tsv.tar.bz2
+# - Importer script inspired by importer.rb from 7DB in 7wks 1st edition,
+#   but refactored to send Cypher to the node4j server (since the REST API was removed in 4.0+).
+
+require "faraday"
+
+
+REST_URL = ENV["DB_HOST"]
+REST_CYPHER_ENDPOINT = "db/data/cypher"
+
+puts REST_URL
+conn = Faraday.new(:url => REST_URL) do |builder|
+  builder.adapter :net_http
+end
+
+
+count = 0
+File.open(ARGV[0], :encoding => 'UTF-8').each do |line|
+  _, _, actor, movie = line.split("\t")
+  next if actor.empty? || movie.empty?
+
+  # add actor node, movie node, and relationship
+  cypher_query = 
+    "MERGE (actor:Actor {name:\"#{actor}\"})
+     MERGE (movie:Movie {name:\"#{movie}\"})
+     MERGE (actor)-[:ACTED_IN]->(movie)"
+
+  conn.post(REST_CYPHER_ENDPOINT, "query=#{cypher_query}")
+
+  puts "   #{count} relationships loaded" if (count += 1) % 1000 == 0
+end