Replace submodule for generated test db (#63) (#64)

Finally addresses #35.

Author: fgsch

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "maxmind-db"]
-	path = maxmind-db
-	url = https://github.com/maxmind/MaxMind-DB.git

Makefile.am

@@ -5,6 +5,3 @@ SUBDIRS = src
 DISTCHECK_CONFIGURE_FLAGS = VMOD_DIR='$${libdir}/varnish/vmods'
 
 dist_doc_DATA = LICENSE
-
-EXTRA_DIST = $(top_srcdir)/maxmind-db/LICENSE \
-	$(top_srcdir)/maxmind-db/test-data/*.mmdb

maxmind-db

@@ -5,15 +5,14 @@ server s1 {
 	txresp
 } -start
 
-shell "cp ${vmod_topsrc}/maxmind-db/test-data/*.mmdb ${tmpdir}"
+shell "cp ${vmod_topsrc}/src/tests/libvmod-geoip2.mmdb ${tmpdir}"
 
 varnish v1 -vcl+backend {
 	import geoip2 from "${vmod_topbuild}/src/.libs/libvmod_geoip2.so";
 	import std;
 
 	sub vcl_init {
-		new db1 = geoip2.geoip2(
-		    "${tmpdir}/MaxMind-DB-test-decoder.mmdb");
+		new db1 = geoip2.geoip2("${tmpdir}/libvmod-geoip2.mmdb");
 	}
 
 	sub vcl_deliver {

src/tests/b00000.vtc

@@ -5,19 +5,16 @@ server s1 {
 	txresp
 } -start
 
-shell "cp ${vmod_topsrc}/maxmind-db/test-data/*.mmdb ${tmpdir}"
+shell "cp ${vmod_topsrc}/src/tests/libvmod-geoip2.mmdb ${tmpdir}"
 
 varnish v1 -vcl+backend {
 	import geoip2 from "${vmod_topbuild}/src/.libs/libvmod_geoip2.so";
 	import std;
 
 	sub vcl_init {
-		new db1 = geoip2.geoip2(
-		    "${tmpdir}/GeoIP2-City-Test.mmdb");
-		new db2 = geoip2.geoip2(
-		    "${tmpdir}/GeoIP2-Connection-Type-Test.mmdb");
-		new db3 = geoip2.geoip2(
-		    "${tmpdir}/GeoIP2-ISP-Test.mmdb");
+		new db1 = geoip2.geoip2("${tmpdir}/libvmod-geoip2.mmdb");
+		new db2 = geoip2.geoip2("${tmpdir}/libvmod-geoip2.mmdb");
+		new db3 = geoip2.geoip2("${tmpdir}/libvmod-geoip2.mmdb");
 	}
 
 	sub vcl_deliver {

src/tests/b00001.vtc

@@ -5,7 +5,7 @@ server s1 -repeat 2 {
 	txresp
 } -start
 
-shell "cp ${vmod_topsrc}/maxmind-db/test-data/*.mmdb ${tmpdir}"
+shell "cp ${vmod_topsrc}/src/tests/libvmod-geoip2.mmdb ${tmpdir}"
 
 varnish v1 -arg "-p thread_pools=1" -vcl+backend { } -start
 
@@ -25,7 +25,7 @@ varnish v1 -vcl+backend {
 	import std;
 
 	sub vcl_init {
-		new db = geoip2.geoip2("${tmpdir}/GeoIP2-City-Test.mmdb");
+		new db = geoip2.geoip2("${tmpdir}/libvmod-geoip2.mmdb");
 	}
 
 	sub vcl_deliver {

src/tests/b00002.vtc

@@ -5,14 +5,14 @@ server s1 {
 	txresp
 } -start
 
-shell "cp ${vmod_topsrc}/maxmind-db/test-data/*.mmdb ${tmpdir}"
+shell "cp ${vmod_topsrc}/src/tests/libvmod-geoip2.mmdb ${tmpdir}"
 
 varnish v1 -vcl+backend {
 	import geoip2 from "${vmod_topbuild}/src/.libs/libvmod_geoip2.so";
 	import std;
 
 	sub vcl_init {
-		new db = geoip2.geoip2("${tmpdir}/GeoIP2-City-Test.mmdb");
+		new db = geoip2.geoip2("${tmpdir}/libvmod-geoip2.mmdb");
 		std.log(db.lookup(
 		    "city/names/en",
 		    std.ip("0.0.0.0", "0.0.0.0")));

src/tests/b00004.vtc

@@ -5,15 +5,15 @@ server s1 {
 	txresp
 } -start
 
-shell "cp ${vmod_topsrc}/maxmind-db/test-data/*.mmdb ${tmpdir}"
+shell "cp ${vmod_topsrc}/src/tests/libvmod-geoip2.mmdb ${tmpdir}"
 
 varnish v1 -vcl+backend {
 	import geoip2 from "${vmod_topbuild}/src/.libs/libvmod_geoip2.so";
 	import std;
 	import vtc;
 
 	sub vcl_init {
-		new db = geoip2.geoip2("${tmpdir}/GeoIP2-City-Test.mmdb");
+		new db = geoip2.geoip2("${tmpdir}/libvmod-geoip2.mmdb");
 		std.log(db.lookup("city/names/en", vtc.no_ip()));
 	}
 } -start

src/tests/b00005.vtc

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"encoding/csv"
+	"encoding/hex"
+	"io"
+	"log"
+	"net"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/maxmind/mmdbwriter"
+	"github.com/maxmind/mmdbwriter/inserter"
+	"github.com/maxmind/mmdbwriter/mmdbtype"
+)
+
+func makeRecord(key, mmdbType, value string) mmdbtype.DataType {
+	var mmdbValue mmdbtype.DataType
+	switch mmdbType {
+	case "bool":
+		v, _ := strconv.ParseBool(value)
+		mmdbValue = mmdbtype.Bool(v)
+	case "bytes":
+		v, _ := hex.DecodeString(value)
+		mmdbValue = mmdbtype.Bytes(v)
+	case "double":
+		v, _ := strconv.ParseFloat(value, 64)
+		mmdbValue = mmdbtype.Float64(v)
+	case "float":
+		v, _ := strconv.ParseFloat(value, 32)
+		mmdbValue = mmdbtype.Float32(v)
+	case "int32":
+		v, _ := strconv.ParseInt(value, 10, 32)
+		mmdbValue = mmdbtype.Int32(v)
+	case "uint16":
+		v, _ := strconv.ParseInt(value, 10, 16)
+		mmdbValue = mmdbtype.Uint16(v)
+	case "uint32":
+		v, _ := strconv.ParseInt(value, 10, 32)
+		mmdbValue = mmdbtype.Uint32(v)
+	case "uint64":
+		v, _ := strconv.ParseInt(value, 10, 64)
+		mmdbValue = mmdbtype.Uint64(v)
+	case "string":
+		mmdbValue = mmdbtype.String(value)
+	case "map":
+		keys := strings.Split(key, "/")
+		key = keys[0]
+		mmdbValue = mmdbtype.Map{
+			mmdbtype.String(keys[1]): mmdbtype.Map{
+				mmdbtype.String(keys[2]): mmdbtype.String(value),
+			},
+		}
+	}
+	record := mmdbtype.Map{}
+	record[mmdbtype.String(key)] = mmdbValue
+
+	return record
+}
+
+func main() {
+	writer, err := mmdbwriter.New(
+		mmdbwriter.Options{
+			DatabaseType:            "libvmod-geoip2",
+			Description:             map[string]string{"en": "libvmod-geoip2 test database"},
+			IncludeReservedNetworks: true,
+			Inserter:                inserter.TopLevelMergeWith,
+		},
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	file, err := os.Open("test-data.csv")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	reader := csv.NewReader(file)
+
+	for {
+		row, err := reader.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		_, network, err := net.ParseCIDR(row[0])
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		record := makeRecord(row[1], row[2], row[3])
+		err = writer.Insert(network, record)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	file, err = os.Create("libvmod-geoip2.mmdb")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	_, err = writer.WriteTo(file)
+	if err != nil {
+		log.Fatal(err)
+	}
+}