From 0e7907bd8b80a4759e2291f50f20b3e6e726cb4c Mon Sep 17 00:00:00 2001 From: Alexandre Fiori Date: Thu, 13 Nov 2014 22:29:11 -0500 Subject: [PATCH] freegeoip v3.0 New version of the freegeoip web server. Changes: - Configuration file is gone. All switches are in the command line now. - The updatedb script is gone. Now the server does some background job to download and maintain the database file up to date. - Auto-reload of the database. If you update or overwrite the database file the server reloads it automatically with no service interruption. - Single file front-end. No longer serving any JS or CSS, using it all from the cloud. jQuery is gone and the front-end is based on AngularJS and Bootstrap only. - No more DNS contention. If you're running your own server and need that then use something external, like dnsmasq. - Database date is exposed to clients in the X-Database-Date header. - Preliminary IPv6 support. Closes #21. - Localized country, region and city names. The server uses the Accept-Language header to determine the language of the response. - Official docker support. See the Dockerfile for details. - New package freegeoip for Go. All the above and more tidbits in a Go package that other programs can leverage. This version of the freegeoip web server is backwards compatible with the previous version. The HTTP API is virtually the same. Features of the freegeoip Go package: - Database API that allows any Go program to do IP geolocation lookups. It supports both local and remote databases. - Background goroutine to download and maintain the database up to date. - Background goroutine for monitoring the database file and auto-reloading on demand. Uses fsnotify. - An http.Handler object that any net/http server can use to serve IP geolocation lookups. Supports CSV, XML, JSON and JSONP out of the box. - Extensible encoder interface to support other output formats, or to implement custom responses for the http handler. - Decent test coverage. --- .gitignore | 2 +- .travis.yml | 10 + AUTHORS | 11 + CONTRIBUTORS | 21 + Dockerfile | 26 +- HISTORY.md | 55 ++ LICENSE | 2 +- README.md | 219 ++--- cmd/freegeoip/main.go | 250 +++++ .../freegeoip/public}/crossdomain.xml | 0 {static => cmd/freegeoip/public}/favicon.ico | Bin cmd/freegeoip/public/index.html | 260 +++++ cmd/freegeoip/upstart.conf | 12 + wrk-test.lua => cmd/freegeoip/wrk-test.lua | 0 db.go | 337 +++++++ db/updatedb | 268 ----- db_test.go | 256 +++++ doc.go | 19 + encoder.go | 267 +++++ encoder_test.go | 236 +++++ example_test.go | 118 +++ freegeoip.conf | 44 - freegeoip.go | 919 ++---------------- freegeoip_test.go | 272 +++--- static/css/index.css | 17 - static/css/index.min.css | 1 - static/css/index.min.css.gz | Bin 250 -> 0 bytes static/index.html | 147 --- static/js/index.js | 64 -- static/js/index.min.js | 3 - static/js/index.min.js.gz | Bin 678 -> 0 bytes static/map.html | 68 -- testdata/.placeholder | 0 33 files changed, 2133 insertions(+), 1771 deletions(-) create mode 100644 .travis.yml create mode 100644 AUTHORS create mode 100644 CONTRIBUTORS create mode 100644 HISTORY.md create mode 100644 cmd/freegeoip/main.go rename {static => cmd/freegeoip/public}/crossdomain.xml (100%) rename {static => cmd/freegeoip/public}/favicon.ico (100%) create mode 100644 cmd/freegeoip/public/index.html create mode 100644 cmd/freegeoip/upstart.conf rename wrk-test.lua => cmd/freegeoip/wrk-test.lua (100%) create mode 100644 db.go delete mode 100755 db/updatedb create mode 100644 db_test.go create mode 100644 doc.go create mode 100644 encoder.go create mode 100644 encoder_test.go create mode 100644 example_test.go delete mode 100644 freegeoip.conf delete mode 100644 static/css/index.css delete mode 100644 static/css/index.min.css delete mode 100644 static/css/index.min.css.gz delete mode 100644 static/index.html delete mode 100644 static/js/index.js delete mode 100644 static/js/index.min.js delete mode 100644 static/js/index.min.js.gz delete mode 100644 static/map.html create mode 100644 testdata/.placeholder diff --git a/.gitignore b/.gitignore index 4f5d3e7..2a22d06 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ +db.gz *.pem *.rdb *.swp *.csv *.zip *.sqlite -freegeoip diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..a0b95da --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +language: go +go: + - 1.3 + - release + - tip +script: + - go test -v -cover + +install: + - go get code.google.com/p/go.tools/cmd/cover diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..7d80c4d --- /dev/null +++ b/AUTHORS @@ -0,0 +1,11 @@ +# This is the official list of freegeoip authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS file. +# +# Names should be added to this file as +# Name or Organization +# +# The email address is not required for organizations. +# +# Please keep the list sorted. + +Alexandre Fiori diff --git a/CONTRIBUTORS b/CONTRIBUTORS new file mode 100644 index 0000000..8a2f354 --- /dev/null +++ b/CONTRIBUTORS @@ -0,0 +1,21 @@ +# This is the official list of freegeoip contributors for copyright purposes. +# This file is distinct from the AUTHORS file. +# +# Names should be added to this file as +# Name or Organization +# +# Please keep the list sorted. +# +# Use the following command to generate the list: +# +# git shortlog -se | awk '{print $2 " " $3 " " $4}' +# +# The email address is not required for organizations. + +Alex Goretoy +Gleicon Moraes +Leandro Pereira +Lucas Fontes +Matthias Nehlsen +Melchi +Vladimir Agafonkin diff --git a/Dockerfile b/Dockerfile index 7779022..eb10704 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,10 @@ FROM google/golang -RUN apt-get install -y build-essential libsqlite3-dev pkg-config file supervisord - -WORKDIR /gopath/src/app -ADD . /gopath/src/app/ -RUN go get app -RUN cd /gopath/src/app/ -RUN go build - -#... will download files and process them to create ipdb.sqlite -RUN cd db && ./updatedb -RUN file /gopath/src/app/db/ipdb.sqlite - -RUN /usr/bin/install -o www-data -g www-data -m 0755 -d /var/log/freegeoip - -EXPOSE 8080 - -CMD [] -ENTRYPOINT ["/gopath/bin/app"] +ADD . /gopath/src/github.com/fiorix/freegeoip +WORKDIR /gopath/src/github.com/fiorix/freegeoip/cmd/freegeoip +RUN go get +RUN go install +RUN cp -r public /var/www + +ENTRYPOINT ["/gopath/bin/freegeoip"] +CMD ["-public", "/var/www"] diff --git a/HISTORY.md b/HISTORY.md new file mode 100644 index 0000000..1a8e68d --- /dev/null +++ b/HISTORY.md @@ -0,0 +1,55 @@ +# History of freegeoip.net + +The freegeoip software is the result of a web server research project that +started in 2009, written in Python and hosted on +[Google App Engine](http://appengine.google.com). It was rapidly adopted by +many developers around the world due to its simplistic and straightforward +HTTP API, causing the free account on GAE to exceed its quota every day +after few hours of operation. + +A year later freegeoip 1.0 was released, and the freegeoip.net domain +moved over to its own server infrastructure. The software was rewritten +using the [Cyclone](http://cyclone.io) web framework, backed by +[Twisted](http://twistedmatrix.com) and [PyPy](http://pypy.org) in +production. That's when the first database management tool was created, +a script that would download many pieces of information from the Internet +to create the IP database, an sqlite flat file used by the server. + +This version of the Python server shipped with a much better front-end as +well, but still as a server-side rendered template inherited from the GAE +version. It was only circa 2011 that freegeoip got its first standalone +front-end based on jQuery, and is when Twitter bootstrap was first used. + +Python played an important role in the early life of freegeoip and +allowed the service to grow and evolve fast. It provided a lot of +flexibility in building and maintaining the IP database using multiple +sources of data. This version of the server lasted until 2013, when +it was once again rewritten from scratch, this time in Go. The database +tool, however, remained intact. + +In 2013 the Go version was released as freegeoip 2.0 and this version +had many iterations. The first versions of the server written in Go were +very rustic, practically a verbatim transcription of the Python server. +Took a while until it started looking more like common Go code, and to +have tests. + +Another important change that shipped with v2 was a front-end based on +AngularJS, but still mixed with some jQuery. The Google map in the front +page was made optional to put more focus on the HTTP API. The popularity +of freegeoip has increased considerably over the years of 2013 and 2014, +calling for more. + +Enter freegeoip 3.0, an evolution of the Go server. The foundation of +freegeoip, which is the IP database and HTTP API, now lives in a Go +package that other developers can leverage. The freegeoip web server is +built on this package making its code cleaner, the server faster, +and requires zero maintenance for the IP database. The server downloads +the file from MaxMind and keep it up to date in background. + +This and other changes make it very Docker friendly. + +The front-end has been trimmed down to a single index.html file that loads +CSS and JS from CDNs on the internet. The JS part is based on AngularJS +and handles the search request and response of the public site. The +optional map has become a link to Google Maps following the lat/long +of the query results. diff --git a/LICENSE b/LICENSE index 8daa3aa..27dafae 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2013 Alexandre Fiori. All rights reserved. +Copyright (c) 2009-2013 The freegeoip authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/README.md b/README.md index 5c2945f..ab7857a 100644 --- a/README.md +++ b/README.md @@ -1,188 +1,123 @@ -# freegeoip.net +# freegeoip -freegeoip.net is a public web service for searching -[geolocation](http://en.wikipedia.org/wiki/Geolocation) of IP addresses. -This is the source code of freegeoip.net's web server and script for building -the IP database. +This is the source code of the freegeoip software. It contains both +the web server that empowers freegeoip.net, and a package for the +[Go](http://golang.org) programming language that enables any web server +to support IP geolocation with a simple and clean API. +See http://en.wikipedia.org/wiki/Geolocation for details about geolocation. -## Overview +## Web Server -freegeoip.net is the result of a web server research project that started in -2009 hosted at Google's [App Engine](http://appengine.google.com), -using the Python API. -A year later it moved to its own server infrastructure built on the -[Cyclone](http://cyclone.io) web framework, backed by -[Twisted](http://twistedmatrix.com) and [PyPy](http://pypy.org). +The freegeoip web server is a standalone program that serves an HTTP API +for searching the geolocation of IP addresses. To serve the API, it uses +an IP database that is automatically downloaded and auto-updated from +the internet when the server is running. -The current version is written in Go as the experiments progress with -[go-web](https://github.com/fiorix/go-web) and -[go-redis](https://github.com/fiorix/go-redis). +The API returns data encoded in popular formats such as CSV, XML, JSON +and JSONP. +### Usage -### Install - -List of prerequisites for building and running the server: - -- Go compiler - for `freegeoip.go` -- Git (for downloading Go packages) -- Mercurial (for downloading Go packages) -- libsqlite3-dev, gcc or llvm - for dependency `go-sqlite3` -- Python - for the `updatedb` script -- Redis - (optional) for API usage quotas -- The IP database - -The following instructions are for Debian and Ubuntu servers. - -Make sure Go is installed and both $GOROOT and $GOPATH are set, then run: - - apt-get install build-essential libsqlite3-dev pkg-config - go get github.com/fiorix/freegeoip - cd $GOPATH/src/github.com/fiorix/freegeoip - go build - -On recent OSX you might have to set the CC=clang before `go build` if -the sqlite3 package fails to compile. - -Proceed to building the IP database before starting the server. - - -### Building the IP database - -The IP database is composed of multiple files from multiple sources. It's a -combination of IP subnets, country codes, city names, etc. - -There's a helper script under the `db` directory that automates the process -of building the database, and can be used regularly to update it as well. - -It's a Python script called `updatedb` that creates `ipdb.sqlite`: - - $ cd db - $ ./updatedb - ... will download files and process them to create ipdb.sqlite - $ file ipdb.sqlite - ipdb.sqlite: SQLite 3.x database - -This service includes GeoLite data created by MaxMind, available from -maxmind.com. +Run the server: + ./freegeoip -## Running +Wait for it to download the IP database file for the first time. It does +it in background and writes a message to the console when ready. If you'd +like to use an alternative database source, see the `-db` command line +flag. -The server looks for `freegeoip.conf` in the current directory, but an -alternative config can be specified using the `-c` command line option. +If the server is queried when there is no database available, including +this initial first run, it returns *HTTP 503 (Service Unavailable)*, since +it can't service requests before a proper database is in place. -By default it logs to the stderr, but log file can be specified using -the `-l` command line option. Log files are cycled on SIGHUP. +### Querying -If the server is proxied by Nginx or another HTTP load balancer, edit the -configuration file and set `xheaders="true"` and it'll use X-Real-IP or -X-Forwarded-For HTTP headers (when available) as the client IP. +You can use any HTTP client to test the server. The examples below use +curl and the environment variable $freegeoip, which must be set to the +address of your server, like http://localhost:8080 for example: -Run the server: + export freegeoip=http://localhost:8080 - ./freegeoip [-c freegeoip.conf] [-l freegeoip.log] +Querying the API is very straightforward: you just have to pick a format +of your choice and provide either the IP address or hostname that you'd +like to search for. The syntax is as follows: -Then point the browser to http://localhost:8080. + $freegeoip/{format}/{IP_or_hostname} -If the IP database is unavailable (e.g. file does not exist, bad permissions) -or redis is unreachable (if using redis as the quota backend), all queries -will result in HTTP 503 (Service Unavailable). - -For listening on low ports as non-root user (e.g. www-data) on linux, set -file capabilities at least once before running it: +Examples: - /sbin/setcap 'cap_net_bind_service=+ep' /opt/freegeoip/freegeoip + curl -i $freegeoip/csv/8.8.8.8 -### Running with upstart + curl -i $freegeoip/xml/4.2.2.2 -On Ubuntu, use the following upstart script in `/etc/init/freegeoip.conf` -to start and stop the server: + curl -i $freegeoip/json/github.com - # freegeoip web service - # https://github.com/fiorix/freegeoip +If a domain or hostname is passed in the URL, the server will resolve that +name to its IP address and lookup the IP instead. If the hostname contains +multiple IPs associated to it, the server picks one randomly, which means +it could be either IPv4 or IPv6. - description "freegeoip web service" +If no IP or hostname is provided, then the server queries the IP address +of the HTTP client. - start on runlevel [2345] - stop on runlevel [!2345] +Example: - limit nofile 20000 20000 - setuid www-data - setgid www-data - exec /opt/freegeoip/freegeoip -c /opt/freegeoip/freegeoip.conf -l /var/log/freegeoip/freegeoip.log + curl -i freegeoip.net/json/ (this queries your own IP address) -The log directory must be created with the right permissions before the -daemon can be started. Use the following command for this: +The JSON endpoint also supports JSONP, by adding a `callback` argument +to the request query. - /usr/bin/install -o www-data -g www-data -m 0755 -d /var/log/freegeoip +Example: -Then use `start freegeoip` and `stop freegeoip` to start and stop the server. + curl -i freegeoip.net/json/8.8.8.8?callback=f -Also, use the following configuration file in `/etc/logrotate.d/freegeoip` for -log rotation: +See http://en.wikipedia.org/wiki/JSONP for details on how JSONP works. - /var/log/freegeoip/freegeoip.log - { - rotate 7 - daily - missingok - notifempty - delaycompress - compress - postrotate - reload freegeoip > /dev/null 2>&1 - endscript - } +## freegeoip package for Go -### Running with supervisord +The freegeoip package for the Go programming language provides two APIs: -Use [supervisor](http://supervisord.org) with the following config in -`/etc/supervisor/conf.d/freegeoip.conf`: +- A database API that requires zero maintenance of the IP database; +- A geolocation `http.Handler` that can be used/served by any http server. - [program:freegeoip] - user=www-data - redirect_stderr=true - directory=/opt/freegeoip - command=/opt/freegeoip/freegeoip - stdout_logfile=/var/log/freegeoip/freegeoip.log - stdout_logfile_maxbytes=50MB - stdout_logfile_backups=20 +tl;dr if all you want is code then see the `examples_test.go` file. -Then use `supervisorctl start freegeoip` and `supervisorctl stop freegeiop` -to start and stop the server. +Otherwise check out the godoc reference. +[![GoDoc](https://godoc.org/github.com/fiorix/freegeoip?status.svg)](https://godoc.org/github.com/fiorix/freegeoip) +[![Build Status](https://secure.travis-ci.org/fiorix/freegeoip.png)](http://travis-ci.org/fiorix/freegeoip) -## Usage +### Features -Point the browser to http://localhost:8080 and search for IPs or hostnames. +- Zero maintenance -Use curl from the command line to query the API: +The DB object alone can download an IP database file from the internet and +service lookups to your program right away. It will auto-update the file in +background and always magically work. - $ curl -v http://localhost:8080/{format}/{ip_or_hostname} +- DevOps friendly -It supports csv, json and xml as the output format. JSON supports callbacks -with the `callback` query argument. The client (self) IP is used if -`ip_or_hostname` is omitted in the query. +If you do care about the database and have the commercial version of the +MaxMind database, you can update the database file with your program running +and the DB object will load it in background. You can focus on your stuff. -Examples: +- Extensible - $ curl -v http://localhost:8080/csv/ - $ curl -v http://localhost:8080/xml/ - $ curl -v http://localhost:8080/xml/freegeoip.net - $ curl -v http://localhost:8080/json/github.com?callback=foobar +Besides the database part, the package provides an `http.Handler` object +that you can add to your HTTP server to service IP geolocation lookups with +the same simplistic API of freegeoip.net. There's also an interface for +crafting your own HTTP responses encoded in any format. -If the server is listening on unix sockets, use `nc` to test: +### Install - echo -ne 'GET /json/my-domain.abc HTTP/1.0\r\n\r\n' | nc -U /tmp/freegeoip.sock +Install the package: + go get github.com/fiorix/freegeoip -## Credits +Install the web server: -Thanks to (in no particular order): + go install github.com/fiorix/freegeoip/cmd/freegeoip -- [Gleicon](https://github.com/gleicon) for all the drama. -- Google for the map, Go, and AngularJS. -- Twitter for Bootstrap. -- MaxMind for the current database. -- ipinfodb.com for both the IP and timezones database back in 2010 and 2011. +Test coverage is quite good and may help you find the stuff you need. diff --git a/cmd/freegeoip/main.go b/cmd/freegeoip/main.go new file mode 100644 index 0000000..9009004 --- /dev/null +++ b/cmd/freegeoip/main.go @@ -0,0 +1,250 @@ +// Copyright 2009-2014 The freegeoip authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package main + +import ( + "flag" + "log" + "net" + "net/http" + "net/url" + "runtime" + "strconv" + "strings" + "time" + + "github.com/fiorix/freegeoip" + "github.com/fiorix/go-redis/redis" + "github.com/gorilla/context" +) + +var maxmindFile = "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz" + +func main() { + addr := flag.String("addr", ":8080", "Address in form of ip:port to listen on") + certFile := flag.String("cert", "", "X.509 certificate file") + keyFile := flag.String("key", "", "X.509 key file") + public := flag.String("public", "", "Public directory to serve at the / endpoint") + ipdb := flag.String("db", maxmindFile, "IP database file or URL") + updateIntvl := flag.Duration("update", 24*time.Hour, "Database update check interval") + retryIntvl := flag.Duration("retry", time.Hour, "Max time to wait before retrying update") + useXFF := flag.Bool("use-x-forwarded-for", false, "Use the X-Forwarded-For header when available") + silent := flag.Bool("silent", false, "Do not log requests to stderr") + redisAddr := flag.String("redis", "127.0.0.1:6379", "Redis address in form of ip:port for quota") + quotaMax := flag.Int("quota-max", 0, "Max requests per source IP per interval; Set 0 to turn off") + quotaIntvl := flag.Duration("quota-interval", time.Hour, "Quota expiration interval") + flag.Parse() + + rc, err := redis.Dial(*redisAddr) + if err != nil { + log.Fatal(err) + } + + db, err := openDB(*ipdb, *updateIntvl, *retryIntvl) + if err != nil { + log.Fatal(err) + } + + runtime.GOMAXPROCS(runtime.NumCPU()) + + encoders := map[string]http.Handler{ + "/csv/": freegeoip.NewHandler(db, &freegeoip.CSVEncoder{UseCRLF: true}), + "/xml/": freegeoip.NewHandler(db, &freegeoip.XMLEncoder{Indent: true}), + "/json/": freegeoip.NewHandler(db, &freegeoip.JSONEncoder{}), + } + + if *quotaMax > 0 { + seconds := int((*quotaIntvl).Seconds()) + for path, f := range encoders { + encoders[path] = userQuota(rc, *quotaMax, seconds, f) + } + } + + mux := http.NewServeMux() + for path, handler := range encoders { + mux.Handle(path, handler) + } + + if len(*public) > 0 { + mux.Handle("/", http.FileServer(http.Dir(*public))) + } + + handler := CORS(mux, "GET", "HEAD") + + if !*silent { + log.Println("freegeoip server starting on", *addr) + go logEvents(db) + handler = logHandler(handler) + } + + if *useXFF { + handler = freegeoip.ProxyHandler(handler) + } + + if len(*certFile) > 0 && len(*keyFile) > 0 { + err = http.ListenAndServeTLS(*addr, *certFile, *keyFile, handler) + } else { + err = http.ListenAndServe(*addr, handler) + } + if err != nil { + log.Fatal(err) + } +} + +// openDB opens and returns the IP database. +func openDB(dsn string, updateIntvl, maxRetryIntvl time.Duration) (db *freegeoip.DB, err error) { + _, err = url.Parse(dsn) + if err != nil { + db, err = freegeoip.Open(dsn) + } else { + db, err = freegeoip.OpenURL(dsn, updateIntvl, maxRetryIntvl) + } + return +} + +// CORS is an http handler that checks for allowed request methods (verbs) +// and adds CORS headers to all http responses. +// +// See http://en.wikipedia.org/wiki/Cross-origin_resource_sharing for details. +func CORS(f http.Handler, allow ...string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Method", + strings.Join(allow, ", ")+", OPTIONS") + if r.Method == "OPTIONS" { + w.WriteHeader(200) + return + } + for _, method := range allow { + if r.Method == method { + f.ServeHTTP(w, r) + return + } + } + w.Header().Set("Allow", strings.Join(allow, ", ")+", OPTIONS") + http.Error(w, http.StatusText(http.StatusMethodNotAllowed), + http.StatusMethodNotAllowed) + }) +} + +// userQuota is a handler that provides a rate limiter to the freegeoip API. +// It allows qmax requests per qintvl, in seconds. +// +// If redis is not available it responds with service unavailable. +func userQuota(rc *redis.Client, qmax int, qintvl int, f http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var ip string + if idx := strings.LastIndex(r.RemoteAddr, ":"); idx != -1 { + ip = r.RemoteAddr[:idx] + } else { + ip = r.RemoteAddr + } + sreq, err := rc.Get(ip) + if err != nil { + serviceUnavailable(w, r, err.Error()) + return + } + if len(sreq) == 0 { + err = rc.SetEx(ip, qintvl, "1") + if err != nil { + serviceUnavailable(w, r, err.Error()) + return + } + f.ServeHTTP(w, r) + return + } + nreq, _ := strconv.Atoi(sreq) + if nreq >= qmax { + http.Error(w, "Quota exceeded", http.StatusForbidden) + return + } + _, err = rc.Incr(ip) + if err != nil { + context.Set(r, "log", err.Error()) + } + f.ServeHTTP(w, r) + }) +} + +// serviceUnavailable writes an http error 501 to a client. +func serviceUnavailable(w http.ResponseWriter, r *http.Request, log string) { + context.Set(r, "log", log) + http.Error(w, "Try again later", http.StatusServiceUnavailable) +} + +// logEvents logs database events. +func logEvents(db *freegeoip.DB) { + for { + select { + case file := <-db.NotifyOpen(): + log.Println("database loaded:", file) + case err := <-db.NotifyError(): + log.Println("database error:", err) + case <-db.NotifyClose(): + return + } + } +} + +// logHandler logs http requests. +func logHandler(f http.Handler) http.Handler { + empty := "" + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp := responseWriter{w, http.StatusOK, 0} + start := time.Now() + f.ServeHTTP(&resp, r) + elapsed := time.Since(start) + extra := context.Get(r, "log") + if extra != nil { + defer context.Clear(r) + } else { + extra = empty + } + log.Printf("%q %d %q %q %s %q %db in %s %q", + r.Proto, + resp.status, + r.Method, + r.URL.Path, + remoteIP(r), + r.Header.Get("User-Agent"), + resp.bytes, + elapsed, + extra, + ) + }) +} + +// remoteIP returns the client's address without the port number. +func remoteIP(r *http.Request) string { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return host +} + +// responseWriter is an http.ResponseWriter that records the returned +// status and bytes written to the client. +type responseWriter struct { + http.ResponseWriter + status int + bytes int +} + +// Write implements the http.ResponseWriter interface. +func (f *responseWriter) Write(b []byte) (int, error) { + n, err := f.ResponseWriter.Write(b) + if err != nil { + return 0, err + } + f.bytes += n + return n, nil +} + +// WriteHeader implements the http.ResponseWriter interface. +func (f *responseWriter) WriteHeader(code int) { + f.status = code + f.ResponseWriter.WriteHeader(code) +} diff --git a/static/crossdomain.xml b/cmd/freegeoip/public/crossdomain.xml similarity index 100% rename from static/crossdomain.xml rename to cmd/freegeoip/public/crossdomain.xml diff --git a/static/favicon.ico b/cmd/freegeoip/public/favicon.ico similarity index 100% rename from static/favicon.ico rename to cmd/freegeoip/public/favicon.ico diff --git a/cmd/freegeoip/public/index.html b/cmd/freegeoip/public/index.html new file mode 100644 index 0000000..31d80fd --- /dev/null +++ b/cmd/freegeoip/public/index.html @@ -0,0 +1,260 @@ + + + + + + + + + +freegeoip.net + + + + + + + + + + +
+ +
+
+

About

+

+ freegeoip.net + provides a public HTTP API for software developers + to search the geolocation of IP addresses. It uses + a database of IP addresses that are associated to + cities along with other relevant information like + time zone, latitude and longitude. +

+ +

+ You're allowed up to 10,000 queries per + hour by default. Once this limit is reached, all of + your requests will result in HTTP 403, forbidden, + until your quota is cleared. +

+ +

+ The freegeoip web server is free and open source so + if the public service limit is a problem for you, + download it and run your own instance. +

+ +
+ +

+ Download + GitHub project +

+
+ +
+
+ +
+
+

Searching, please wait...

+

Server returned {{error}}

+ IP database date:
{{lastUpdated}} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IP{{record.ip}}
Country{{record.country_name}}
Region{{record.region_name}}
City{{record.city}}
Zip/Postal code{{record.zip_code}}
Lat/Long + + {{record.latitude}}, {{record.longitude}} + +
Metro code{{record.metro_code||''}}
Time zone + + {{record.time_zone}} + +
+
+ +
+ +
+ +
+
+

Support

+

+ freegeoip.net is + community funded, therefore consider donating if you + use and like it. For any other inquiries or feedback, + contact me + directly. +

+
+ +
+ +
+ +

API

+ +

The HTTP API takes GET requests in the following schema:

+ + 
freegeoip.net/{format}/{IP_or_hostname}
+ +

+ Supported formats are: + csv, + xml, + json + and + jsonp. + If no IP or hostname is provided, then your own IP is looked up. +

+ +
+ +
Examples
+
+ +
+
CSV
+ 
freegeoip.net/csv/8.8.8.8
+
+ +
+
XML
+ 
freegeoip.net/xml/4.2.2.2
+
+ +
+
JSON
+ 
freegeoip.net/json/github.com
+ Make it + JSONP + by adding the callback argument to the request. +
+ +
+ +
+ +
+ + This web site and all documentation is licensed under + Creative Commons 3.0. +
+ This service includes GeoLite2 data created by MaxMind, available + from maxmind.com. +
+ © 2009-2014 freegeoip.net + +
+
+ + + + diff --git a/cmd/freegeoip/upstart.conf b/cmd/freegeoip/upstart.conf new file mode 100644 index 0000000..c85f13e --- /dev/null +++ b/cmd/freegeoip/upstart.conf @@ -0,0 +1,12 @@ +# freegeoip.net web server + +description "freegeoip web server" + +start on runlevel [2345] +stop on runlevel [!2345] + +#pre-start exec /sbin/setcap 'cap_net_bind_service=+ep' /usr/local/bin/freegeoip +#limit nofile 20000 20000 +#setuid www-data +#setgid www-data +exec /usr/local/bin/freegeoip -silent diff --git a/wrk-test.lua b/cmd/freegeoip/wrk-test.lua similarity index 100% rename from wrk-test.lua rename to cmd/freegeoip/wrk-test.lua diff --git a/db.go b/db.go new file mode 100644 index 0000000..5cefb42 --- /dev/null +++ b/db.go @@ -0,0 +1,337 @@ +// Copyright 2009-2014 The freegeoip authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package freegeoip + +import ( + "compress/gzip" + "errors" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "os" + "path/filepath" + "strconv" + "sync" + "time" + + "github.com/howeyc/fsnotify" + "github.com/oschwald/maxminddb-golang" +) + +var ( + // ErrUnavailable may be returned by DB.Lookup when the database + // points to a URL and is not yet available because it's being + // downloaded in background. + ErrUnavailable = errors.New("No database available") + + // Local cached copy of a database downloaded from a URL. + defaultDB = filepath.Join(os.TempDir(), "freegeoip", "db.gz") +) + +// DB is the IP geolocation database. +type DB struct { + file string // Database file name. + reader *maxminddb.Reader // Actual db object. + notifyQuit chan struct{} // Stop auto-update and watch goroutines. + notifyOpen chan string // Notify when a db file is open. + notifyError chan error // Notify when an error occurs. + closed bool // Mark this db as closed. + lastUpdated time.Time // Last time the db was updated. + mu sync.RWMutex // Protects all the above. + + updateInterval time.Duration // Update interval. + maxRetryInterval time.Duration // Max retry interval in case of failure. +} + +// Open creates and initializes a DB from a local file. +// +// The database file is monitored by fsnotify and automatically +// reloads when the file is updated or overwritten. +func Open(dsn string) (db *DB, err error) { + db = &DB{ + file: dsn, + notifyQuit: make(chan struct{}), + notifyOpen: make(chan string, 1), + notifyError: make(chan error, 1), + } + err = db.openFile() + if err != nil { + db.Close() + return nil, err + } + err = db.watchFile() + if err != nil { + db.Close() + return nil, fmt.Errorf("fsnotify failed for %s: %s", dsn, err) + } + return db, nil +} + +// OpenURL creates and initializes a DB from a remote file. +// It automatically downloads and updates the file in background. +func OpenURL(url string, updateInterval, maxRetryInterval time.Duration) (db *DB, err error) { + db = &DB{ + file: defaultDB, + notifyQuit: make(chan struct{}), + notifyOpen: make(chan string, 1), + notifyError: make(chan error, 1), + updateInterval: updateInterval, + maxRetryInterval: maxRetryInterval, + } + db.openFile() // Optional, might fail. + go db.autoUpdate(url) + err = db.watchFile() + if err != nil { + db.Close() + return nil, fmt.Errorf("fsnotify failed for %s: %s", db.file, err) + } + return db, nil +} + +func (db *DB) watchFile() error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return err + } + dbdir := filepath.Dir(db.file) + _, err = os.Stat(dbdir) + if err != nil { + err = os.MkdirAll(dbdir, 0755) + if err != nil { + return err + } + } + go db.watchEvents(watcher) + return watcher.Watch(dbdir) +} + +func (db *DB) watchEvents(watcher *fsnotify.Watcher) { + for { + select { + case ev := <-watcher.Event: + if ev.Name == db.file && (ev.IsCreate() || ev.IsModify()) { + db.openFile() + } + case <-watcher.Error: + case <-db.notifyQuit: + watcher.Close() + return + } + time.Sleep(time.Second) // Suppress high-rate events. + } +} + +func (db *DB) openFile() error { + reader, err := db.newReader(db.file) + if err != nil { + return err + } + stat, err := os.Stat(db.file) + if err != nil { + return err + } + db.setReader(reader, stat.ModTime()) + return nil +} + +func (db *DB) newReader(dbfile string) (*maxminddb.Reader, error) { + f, err := os.Open(dbfile) + if err != nil { + return nil, err + } + defer f.Close() + gzf, err := gzip.NewReader(f) + if err != nil { + return nil, err + } + defer gzf.Close() + b, err := ioutil.ReadAll(gzf) + if err != nil { + return nil, err + } + return maxminddb.FromBytes(b) +} + +func (db *DB) setReader(reader *maxminddb.Reader, modtime time.Time) { + db.mu.Lock() + defer db.mu.Unlock() + if db.closed { + reader.Close() + return + } + if db.reader != nil { + db.reader.Close() + } + db.reader = reader + db.lastUpdated = modtime.UTC() + select { + case db.notifyOpen <- db.file: + default: + } +} + +func (db *DB) autoUpdate(url string) { + var sleep time.Duration + var retrying bool + for { + err := db.runUpdate(url) + if err != nil { + db.sendError(fmt.Errorf("Database update failed: %s", err)) + if !retrying { + retrying = true + sleep = 5 * time.Second + } else { + sleep *= 2 + if sleep > db.maxRetryInterval { + sleep = db.maxRetryInterval + } + } + } else { + retrying = false + sleep = db.updateInterval + } + select { + case <-db.notifyQuit: + return + case <-time.After(sleep): + // Sleep till time for the next update attempt. + } + } +} + +func (db *DB) runUpdate(url string) error { + yes, err := db.needUpdate(url) + if err != nil { + return err + } + if !yes { + return nil + } + tmpfile, err := db.download(url) + if err != nil { + return err + } + return db.renameFile(tmpfile) +} + +func (db *DB) needUpdate(url string) (bool, error) { + stat, err := os.Stat(db.file) + if err != nil { + return true, nil // Local db is missing, must be downloaded. + } + resp, err := http.Head(url) + if err != nil { + return false, err + } + defer resp.Body.Close() + size, err := strconv.Atoi(resp.Header.Get("Content-Length")) + if stat.Size() != int64(size) { + return true, nil + } + return false, nil +} + +func (db *DB) download(url string) (tmpfile string, err error) { + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + tmpfile = filepath.Join(os.TempDir(), + fmt.Sprintf("_freegeoip.%d.db.gz", time.Now().UnixNano())) + f, err := os.Create(tmpfile) + if err != nil { + return "", err + } + defer f.Close() + _, err = io.Copy(f, resp.Body) + if err != nil { + return "", err + } + return tmpfile, nil +} + +func (db *DB) renameFile(name string) error { + os.Rename(db.file, db.file+".bak") // Optional, might fail. + return os.Rename(name, db.file) +} + +// Date returns the UTC date the database file was last modified. +// If no database file has been opened the behaviour of Date is undefined. +func (db *DB) Date() time.Time { + db.mu.Lock() + defer db.mu.Unlock() + return db.lastUpdated +} + +// NotifyClose returns a channel that is closed when the database is closed. +func (db *DB) NotifyClose() <-chan struct{} { + return db.notifyQuit +} + +// NotifyOpen returns a channel that notifies when a new database is +// loaded or reloaded. This can be used to monitor background updates +// when the DB points to a URL. +func (db *DB) NotifyOpen() (filename <-chan string) { + return db.notifyOpen +} + +// NotifyError returns a channel that notifies when an error occurs +// while downloading or reloading a DB that points to a URL. +func (db *DB) NotifyError() (errChan <-chan error) { + return db.notifyError +} + +func (db *DB) sendError(err error) { + db.mu.Lock() + defer db.mu.Unlock() + if db.closed { + return + } + select { + case db.notifyError <- err: + default: + } +} + +// Lookup takes an IP address and a pointer to the result value to decode +// into. The result value pointed to must be a data value that corresponds +// to a record in the database. This may include a struct representation +// of the data, a map capable of holding the data or an empty interface{} +// value. +// +// If result is a pointer to a struct, the struct need not include a field +// for every value that may be in the database. If a field is not present +// in the structure, the decoder will not decode that field, reducing the +// time required to decode the record. +// +// See https://godoc.org/github.com/oschwald/maxminddb-golang#Reader.Lookup +// for details. +func (db *DB) Lookup(addr net.IP, result interface{}) error { + db.mu.RLock() + defer db.mu.RUnlock() + if db.reader != nil { + return db.reader.Lookup(addr, result) + } + return ErrUnavailable +} + +// Close the database. +func (db *DB) Close() { + db.mu.Lock() + defer db.mu.Unlock() + if !db.closed { + db.closed = true + close(db.notifyQuit) + close(db.notifyOpen) + close(db.notifyError) + } + if db.reader != nil { + db.reader.Close() + db.reader = nil + } +} diff --git a/db/updatedb b/db/updatedb deleted file mode 100755 index 4adaeeb..0000000 --- a/db/updatedb +++ /dev/null @@ -1,268 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 -# -# Copyright 2013 Alexandre Fiori -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import csv -import urllib2 -import os -import sqlite3 -import sys -import unicodedata -import zipfile - -dbname = "ipdb.sqlite" - - -def download(url, filename=None): - print "Downloading " + url - - req = urllib2.Request(url, None, {"User-Agent": "Mozilla/5.0"}) - req = urllib2.urlopen(req) - data = req.read() - req.close() - - if filename: - fd = open(filename, "wb") - fd.write(data) - fd.close() - - return data - - -def extract(zipfd, zipname, filename=None): - if filename is None: - filename = zipname - - print "Extracting " + filename - - try: - data = zipfd.read(zipname) - except KeyError: - print "Could not extract %s from zip archive." % zipname - sys.exit(1) - else: - fd = open(filename, "w") - fd.write(data) - fd.close() - - -def import_csv(cursor, csvfile, table, skip_lines=0): - sys.stdout.write("Importing %s: " % csvfile) - sys.stdout.flush() - - fd = open(csvfile) - for n in range(skip_lines): - fd.next() - - q = None - rows = [] - for n, row in enumerate(csv.reader(fd), 1): - if q is None: - question_marks = ",".join(["?"] * len(row)) - q = "insert into %s values (%s)" % (table, question_marks) - - rows.append(map(lambda s: s.decode("latin-1"), row)) - if not n % 100000: - sys.stderr.write(".") - cursor.executemany(q, rows) - rows = [] - - if rows: - cursor.executemany(q, rows) - - fd.close() - print "%d records!" % n - os.unlink(csvfile) - - -class world_regions(dict): - """Imports a csv and only store rows that contains accented characters, - indexing them by their non-accented version:: - - country,region (no accents) -> region with accents - - Expected csv columns: country,region,city - """ - def __init__(self, filename=None, autorm=True): - self.filename = filename - if filename: - fd = open(filename) - for row in csv.reader(fd): - v = map(lambda s: s.decode("utf-8"), row[:2]) - k = self.strip_accents(",".join(v)) - - if k != v: - self[k] = v[1] - fd.close() - if autorm is True: - os.unlink(filename) - - def strip_accents(self, s): - return ''.join((c for c in unicodedata.normalize('NFD', s) - if unicodedata.category(c) != 'Mn')) - - -class world_countries(dict): - def __init__(self, conn): - curs = conn.cursor() - curs.execute("SELECT country_code, country_name from country_blocks") - for (code, name) in curs: - self[code] = name - curs.close() - - -if __name__ == "__main__": - region_url = "http://dev.maxmind.com/static/csv/codes/maxmind/region.csv" - region_csv = os.path.basename(region_url) - if not os.path.exists(region_csv): - download(region_url, region_csv) - - wr_url = "http://blog.freegeoip.net/files/all_cities_in_the_world.csv.zip" - wr_zip = os.path.basename(wr_url) - wr_csv = wr_zip[:-4] - if not os.path.exists(wr_csv): - if not os.path.exists(wr_zip): - download(wr_url, wr_zip) - zipfd = zipfile.ZipFile(wr_zip) - extract(zipfd, wr_csv, wr_csv) - zipfd.close() - os.unlink(wr_zip) - - geolite = "http://geolite.maxmind.com/download/geoip/database/" - - city_url = geolite + "GeoLiteCity_CSV/GeoLiteCity-latest.zip" - city_files = ["GeoLiteCity-Blocks.csv", "GeoLiteCity-Location.csv"] - if not all(map(os.path.exists, city_files)): - print "Checking " + city_url - # Fetch the most recent city database - city_zip = os.path.basename(city_url) - if not os.path.exists(city_zip): - download(city_url, city_zip) - - # Extract city csv files - zipfd = zipfile.ZipFile(city_zip) - for filename in zipfd.namelist(): - n = os.path.basename(filename) - if n in city_files and not os.path.exists(n): - extract(zipfd, filename, n) - zipfd.close() - os.unlink(city_zip) - - # Fetch the country database - country_url = geolite + "GeoIPCountryCSV.zip" - country_zip = os.path.basename(country_url) - if not os.path.exists(country_zip): - download(country_url, country_zip) - - country_csv = "GeoIPCountryWhois.csv" - if not os.path.exists(country_csv): - zipfd = zipfile.ZipFile(country_zip) - extract(zipfd, country_csv) - zipfd.close() - os.unlink(country_zip) - - # Create the IP database - tmpdb = "_" + dbname + ".temp" - if os.path.exists(tmpdb): - os.unlink(tmpdb) - - conn = sqlite3.connect(tmpdb) - curs = conn.cursor() - - curs.execute("""\ - create table country_blocks ( - ip_start_str text, - ip_end_str text, - ip_start text, - ip_end text, - country_code text, - country_name text, - primary key(ip_start))""") - import_csv(curs, country_csv, "country_blocks") - curs.execute("CREATE INDEX cc_idx ON country_blocks(country_code);") - - curs.execute("""\ - create table region_names ( - country_code text, - region_code text, - region_name text, - unique (country_code, region_code))""") - import_csv(curs, region_csv, "region_names") - - curs.execute("""\ - create table city_blocks ( - ip_start int, - ip_end int, - loc_id int, - primary key(ip_start))""") - import_csv(curs, city_files[0], "city_blocks", skip_lines=2) - - curs.execute("""\ - create table city_location ( - loc_id int, - country_code text, - region_code text, - city_name text, - postal_code text, - latitude real, - longitude real, - metro_code text, - area_code text, - primary key(loc_id))""") - import_csv(curs, city_files[1], "city_location", skip_lines=2) - - curs.close() - conn.commit() - - # Fix region names - sys.stdout.write("Updating region names: ") - sys.stdout.flush() - - world_regions = world_regions("all_cities_in_the_world.csv") - world_countries = world_countries(conn) - - count = 0 - regions = conn.cursor() - regions.execute("SELECT rowid, * FROM region_names") - - for region in regions: - region_name = region[-1] # rowid,country_code,region_code,region_name - country_name = world_countries.get(region[1]) - - if country_name: - k = country_name + "," + region_name - if k in world_regions: - new_name = world_regions[k] - if region_name != new_name: - update = conn.cursor() - update.execute("UPDATE region_names SET region_name=? " - "WHERE rowid=?", (new_name, region[0])) - update.close() - count += 1 - - print "%d names updated." % count - - # Fix db consistency. (patch from Alfredo Terrone) - regions.execute("INSERT INTO region_names " - "SELECT DISTINCT country_code,'','' FROM region_names") - regions.close() - conn.commit() - conn.close() - - # Replace any existing db with the new one - if os.path.exists(dbname): - os.unlink(dbname) - os.rename(tmpdb, dbname) diff --git a/db_test.go b/db_test.go new file mode 100644 index 0000000..0cc7a30 --- /dev/null +++ b/db_test.go @@ -0,0 +1,256 @@ +// Copyright 2009-2014 The freegeoip authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package freegeoip + +import ( + "errors" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" +) + +var testFile = "testdata/db.gz" + +func TestDownload(t *testing.T) { + if _, err := os.Stat(testFile); err == nil { + t.Skip("Test database already exists:", testFile) + } + db := &DB{} + dbfile, err := db.download(maxmindFile) + if err != nil { + t.Fatal(err) + } + err = os.Rename(dbfile, testFile) + if err != nil { + t.Fatal(err) + } +} + +func TestNeedUpdateFileMissing(t *testing.T) { + db := &DB{file: "does-not-exist"} + yes, err := db.needUpdate("whatever") + if err != nil { + t.Fatal(err) + } + if !yes { + t.Fatal("Unexpected: db is supposed to need an update") + } +} + +func TestNeedUpdateSameFile(t *testing.T) { + mux := http.NewServeMux() + mux.Handle("/testdata/", http.FileServer(http.Dir("."))) + srv := httptest.NewServer(mux) + defer srv.Close() + db := &DB{file: testFile} + yes, err := db.needUpdate(srv.URL + "/" + testFile) + if err != nil { + t.Fatal(err) + } + if yes { + t.Fatal("Unexpected: db is not supposed to need an update") + } +} + +func TestNeedUpdate(t *testing.T) { + mux := http.NewServeMux() + mux.Handle("/testdata/", http.FileServer(http.Dir("."))) + srv := httptest.NewServer(mux) + defer srv.Close() + file := testFile + ".tmp" + f, err := os.Create(file) + if err != nil { + t.Fatal(err) + } + defer f.Close() + defer os.Remove(file) + db := &DB{file: file} + yes, err := db.needUpdate(srv.URL + "/" + testFile) + if err != nil { + t.Fatal(err) + } + if !yes { + t.Fatal("Unexpected: db is supposed to need an update") + } +} + +func TestOpenFile(t *testing.T) { + db, err := Open(testFile) + if err != nil { + t.Fatal(err) + } + defer db.Close() + select { + case <-db.NotifyOpen(): + case <-db.NotifyClose(): + case <-time.After(time.Second): + t.Fatal("Timed out") + } + db.Date() // Test this? +} + +func TestOpenBadFile(t *testing.T) { + db, err := Open("db_test.go") + if err == nil { + db.Close() + t.Fatal("Unexpected bogus db is open") + } +} + +func TestSendError(t *testing.T) { + db := &DB{notifyError: make(chan error, 1)} + err1 := errors.New("test") + db.sendError(err1) + select { + case err2 := <-db.NotifyError(): + if err2 != err2 { + t.Fatalf("Unexpected error: %#v", err2) + } + default: + t.Fatal("An error is expected but it's not available") + } +} + +func TestSkipSendError(t *testing.T) { + db := &DB{notifyError: make(chan error, 1)} + db.sendError(nil) + db.sendError(nil) + close(db.notifyError) +} + +func TestWatchFile(t *testing.T) { + db, err := Open(testFile) + if err != nil { + t.Fatal(err) + } + defer db.Close() + err = os.Rename(testFile, testFile+".bkp") + if err != nil { + t.Fatal(err) + } + err = os.Rename(testFile+".bkp", testFile) + if err != nil { + t.Fatal(err) + } + select { + case file := <-db.NotifyOpen(): + if file != testFile { + t.Fatal("Unexpected file:", file) + } + case <-time.After(time.Second): + t.Fatal("Timed out") + } +} + +func TestWatchMkdir(t *testing.T) { + mux := http.NewServeMux() + mux.Handle("/testdata/", http.FileServer(http.Dir("."))) + srv := httptest.NewServer(mux) + defer srv.Close() + tmp := defaultDB + defaultDB = filepath.Join(os.TempDir(), "foobar", "db.gz") + defer func() { + defaultDB = tmp + time.Sleep(time.Second) + os.RemoveAll(filepath.Dir(defaultDB)) + }() + db, err := OpenURL(srv.URL+"/"+testFile, time.Hour, time.Minute) + if err != nil { + t.Fatalf("Failed to create %s: %s", filepath.Dir(defaultDB), err) + } + db.Close() +} + +func TestWatchMkdirFail(t *testing.T) { + basedir := filepath.Join(os.TempDir(), "freegeoip-test") + err := os.MkdirAll(basedir, 0444) + if err != nil { + t.Fatal(err) + } + tmp := defaultDB + defaultDB = filepath.Join(basedir, "a", "db.gz") + defer func() { + defaultDB = tmp + time.Sleep(time.Second) + os.Chmod(basedir, 0755) + os.RemoveAll(basedir) + }() + mux := http.NewServeMux() + mux.Handle("/testdata/", http.FileServer(http.Dir("."))) + srv := httptest.NewServer(mux) + defer srv.Close() + db, err := OpenURL(srv.URL+"/"+testFile, time.Hour, time.Minute) + if err == nil { + db.Close() + t.Fatalf("Unexpected creation of dir %s worked", basedir) + } +} + +func TestLookupOnFile(t *testing.T) { + db, err := Open(testFile) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var record testRecord + err = db.Lookup(net.ParseIP("8.8.8.8"), &record) + if err != nil { + t.Fatal(err) + } + if record.Country.ISOCode != "US" { + t.Fatal("Unexpected ISO code:", record.Country.ISOCode) + } +} + +func TestLookupOnURL(t *testing.T) { + mux := http.NewServeMux() + mux.Handle("/testdata/", http.FileServer(http.Dir("."))) + srv := httptest.NewServer(mux) + defer srv.Close() + os.Remove(defaultDB) // In case it exists. + db, err := OpenURL(srv.URL+"/"+testFile, time.Hour, time.Minute) + if err != nil { + t.Fatal(err) + } + defer db.Close() + select { + case file := <-db.NotifyOpen(): + if file != defaultDB { + t.Fatal("Unexpected db file:", file) + } + case err := <-db.NotifyError(): + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("Timed out") + } + var record testRecord + err = db.Lookup(net.ParseIP("8.8.8.8"), &record) + if err != nil { + t.Fatal(err) + } + if record.Country.ISOCode != "US" { + t.Fatal("Unexpected ISO code:", record.Country.ISOCode) + } +} + +func TestLookuUnavailable(t *testing.T) { + db := &DB{} + err := db.Lookup(net.ParseIP("8.8.8.8"), nil) + if err == nil { + t.Fatal("Unexpected lookup worked") + } +} + +type testRecord struct { + Country struct { + ISOCode string `maxminddb:"iso_code"` + } `maxminddb:"country"` +} diff --git a/doc.go b/doc.go new file mode 100644 index 0000000..61663e7 --- /dev/null +++ b/doc.go @@ -0,0 +1,19 @@ +// Copyright 2009-2014 The freegeoip authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Package freegeoip provides an API for searching the geolocation of IP +// addresses. It uses a database that can be either a local file or a +// remote resource from a URL. +// +// Local databases are monitored by fsnotify and reloaded when the file is +// either updated or overwritten. +// +// Remote databases are automatically downloaded and updated in background +// so you can focus on using the API and not managing the database. +// +// Also, the freegeoip package provides http handlers that any Go http +// server (net/http) can use. These handlers can process IP geolocation +// lookup requests and return data in multiple formats like CSV, XML, +// JSON and JSONP. It has also an API for supporting custom formats. +package freegeoip diff --git a/encoder.go b/encoder.go new file mode 100644 index 0000000..bbce3a5 --- /dev/null +++ b/encoder.go @@ -0,0 +1,267 @@ +// Copyright 2009-2014 The freegeoip authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package freegeoip + +import ( + "encoding/csv" + "encoding/json" + "encoding/xml" + "io" + "math" + "net" + "net/http" + "strconv" + "strings" + + // otto is used for testing the JSONP encoder. It's imported here + // to make `go get` download it before `go test` fails. + _ "github.com/robertkrimen/otto" +) + +// A Query object is used to query the IP database. +// +// Currently the only database supported is MaxMind, and the query is a +// data structure with tags that are used by the maxminddb.Lookup function. +type Query interface{} + +// An Encoder that can provide a query specification to be used for +// querying the IP database, and later encode the results of that +// query in a specific format. +type Encoder interface { + // NewQuery returns a query specification that is used to query + // the IP database. It should be a data structure with tags + // associated to its fields describing what fields to query in + // the IP database, such as country and city. + // + // See the maxminddb package documentation for details on + // fields available for the MaxMind database. + NewQuery() Query + + // Encode writes data to the response of an http request + // using the results of a query to the IP database. + // + // It encodes the query object into a specific format such + // as XML or JSON and writes to the response. + // + // The IP passed to the encoder may be the result of a DNS + // lookup, and if there are multiple IPs associated to the + // hostname this will be a random one from the list. + Encode(w http.ResponseWriter, r *http.Request, q Query, ip net.IP) error +} + +// JSONEncoder encodes the results of an IP lookup as JSON. +type JSONEncoder struct { + Indent bool +} + +// NewQuery implements the Encoder interface. +func (f *JSONEncoder) NewQuery() Query { + return &maxmindQuery{} +} + +// Encode implements the Encoder interface. +func (f *JSONEncoder) Encode(w http.ResponseWriter, r *http.Request, q Query, ip net.IP) error { + record := newResponse(q.(*maxmindQuery), ip, requestLang(r)) + callback := r.FormValue("callback") + if len(callback) > 0 { + return f.P(w, r, record, callback) + } + w.Header().Set("Content-Type", "application/json") + if f.Indent { + } + return json.NewEncoder(w).Encode(record) +} + +// P writes JSONP to an http response. +func (f *JSONEncoder) P(w http.ResponseWriter, r *http.Request, record *responseRecord, callback string) error { + w.Header().Set("Content-Type", "application/javascript") + _, err := io.WriteString(w, callback+"(") + if err != nil { + return err + } + err = json.NewEncoder(w).Encode(record) + if err != nil { + return err + } + _, err = io.WriteString(w, ");") + return err +} + +// XMLEncoder encodes the results of an IP lookup as XML. +type XMLEncoder struct { + Indent bool +} + +// NewQuery implements the Encoder interface. +func (f *XMLEncoder) NewQuery() Query { + return &maxmindQuery{} +} + +// Encode implements the Encoder interface. +func (f *XMLEncoder) Encode(w http.ResponseWriter, r *http.Request, q Query, ip net.IP) error { + record := newResponse(q.(*maxmindQuery), ip, requestLang(r)) + w.Header().Set("Content-Type", "application/xml") + _, err := io.WriteString(w, xml.Header) + if err != nil { + return err + } + if f.Indent { + enc := xml.NewEncoder(w) + enc.Indent("", "\t") + err := enc.Encode(record) + if err != nil { + return err + } + _, err = w.Write([]byte("\n")) + return err + } + return xml.NewEncoder(w).Encode(record) +} + +// CSVEncoder encodes the results of an IP lookup as CSV. +type CSVEncoder struct { + UseCRLF bool +} + +// NewQuery implements the Encoder interface. +func (f *CSVEncoder) NewQuery() Query { + return &maxmindQuery{} +} + +// Encode implements the Encoder interface. +func (f *CSVEncoder) Encode(w http.ResponseWriter, r *http.Request, q Query, ip net.IP) error { + record := newResponse(q.(*maxmindQuery), ip, requestLang(r)) + w.Header().Set("Content-Type", "text/csv") + cw := csv.NewWriter(w) + cw.UseCRLF = f.UseCRLF + err := cw.Write([]string{ + record.CountryCode, + record.CountryName, + record.RegionCode, + record.RegionName, + record.City, + record.ZipCode, + record.TimeZone, + strconv.FormatFloat(record.Latitude, 'f', 2, 64), + strconv.FormatFloat(record.Longitude, 'f', 2, 64), + strconv.Itoa(int(record.MetroCode)), + }) + if err != nil { + return err + } + cw.Flush() + return nil +} + +// maxmindQuery is the object used to query the maxmind database. +// +// See the maxminddb package documentation for details. +type maxmindQuery struct { + Country struct { + ISOCode string `maxminddb:"iso_code"` + Names map[string]string `maxminddb:"names"` + } `maxminddb:"country"` + Region []struct { + ISOCode string `maxminddb:"iso_code"` + Names map[string]string `maxminddb:"names"` + } `maxminddb:"subdivisions"` + City struct { + Names map[string]string `maxminddb:"names"` + } `maxminddb:"city"` + Location struct { + Latitude float64 `maxminddb:"latitude"` + Longitude float64 `maxminddb:"longitude"` + MetroCode uint `maxminddb:"metro_code"` + TimeZone string `maxminddb:"time_zone"` + } `maxminddb:"location"` + Postal struct { + Code string `maxminddb:"code"` + } `maxminddb:"postal"` +} + +// responseRecord is the object that gets encoded as the response of an +// IP lookup request. It is encoded to formats such as xml and json. +type responseRecord struct { + XMLName xml.Name `xml:"Response" json:"-"` + IP string `json:"ip"` + CountryCode string `json:"country_code"` + CountryName string `json:"country_name"` + RegionCode string `json:"region_code"` + RegionName string `json:"region_name"` + City string `json:"city"` + ZipCode string `json:"zip_code"` + TimeZone string `json:"time_zone"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + MetroCode uint `json:"metro_code"` +} + +// newResponse translates a maxmindQuery into a responseRecord, setting +// the country, region and city names to their localized name according +// to the given lang. +// +// See the maxminddb documentation for supported languages. +func newResponse(query *maxmindQuery, ip net.IP, lang []string) *responseRecord { + + record := &responseRecord{ + IP: ip.String(), + CountryCode: query.Country.ISOCode, + CountryName: localizedName(query.Country.Names, lang), + City: localizedName(query.City.Names, lang), + ZipCode: query.Postal.Code, + TimeZone: query.Location.TimeZone, + Latitude: roundFloat(query.Location.Latitude, .5, 3), + Longitude: roundFloat(query.Location.Longitude, .5, 3), + MetroCode: query.Location.MetroCode, + } + if len(query.Region) > 0 { + record.RegionCode = query.Region[0].ISOCode + record.RegionName = localizedName(query.Region[0].Names, lang) + } + return record +} + +func requestLang(r *http.Request) (list []string) { + // TODO: Check Accept-Charset, sort languages by qvalue. + l := r.Header.Get("Accept-Language") + if len(l) == 0 { + return nil + } + accpt := strings.Split(l, ",") + if len(accpt) == 0 { + return nil + } + for n, name := range accpt { + accpt[n] = strings.Trim(strings.SplitN(name, ";", 2)[0], " ") + } + return accpt +} + +func localizedName(field map[string]string, accept []string) (name string) { + if accept != nil { + var f string + var ok bool + for _, l := range accept { + f, ok = field[l] + if ok { + return f + } + } + } + return field["en"] +} + +func roundFloat(val float64, roundOn float64, places int) (newVal float64) { + var round float64 + pow := math.Pow(10, float64(places)) + digit := pow * val + _, div := math.Modf(digit) + if div >= roundOn { + round = math.Ceil(digit) + } else { + round = math.Floor(digit) + } + return round / pow +} diff --git a/encoder_test.go b/encoder_test.go new file mode 100644 index 0000000..8b6261f --- /dev/null +++ b/encoder_test.go @@ -0,0 +1,236 @@ +// Copyright 2009-2014 The freegeoip authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package freegeoip + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "encoding/xml" + "io/ioutil" + "net/http" + "testing" + + "github.com/robertkrimen/otto" +) + +func TestCSVEncoder(t *testing.T) { + _, srv, err := runServer("/csv/", &CSVEncoder{}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/csv/8.8.8.8") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + row, err := csv.NewReader(resp.Body).Read() + if err != nil { + t.Fatal(err) + } + if row[0] != "US" { + t.Fatalf("Unexpected country code in record: %#v", row) + } +} + +func TestXMLEncoder(t *testing.T) { + _, srv, err := runServer("/xml/", &XMLEncoder{}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/xml/8.8.8.8") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + var record responseRecord + err = xml.NewDecoder(resp.Body).Decode(&record) + if err != nil { + t.Fatal(err) + } + if record.CountryCode != "US" { + t.Fatalf("Unexpected country code in record: %#v", record.CountryCode) + } +} + +func TestXMLEncoderIndent(t *testing.T) { + // TODO: validate indentation? + _, srv, err := runServer("/xml/", &XMLEncoder{Indent: true}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/xml/8.8.8.8") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + var record responseRecord + err = xml.NewDecoder(resp.Body).Decode(&record) + if err != nil { + t.Fatal(err) + } + if record.CountryCode != "US" { + t.Fatalf("Unexpected country code in record: %#v", record.CountryCode) + } +} + +func TestJSONEncoder(t *testing.T) { + _, srv, err := runServer("/json/", &JSONEncoder{}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/json/8.8.8.8") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + var record responseRecord + err = json.NewDecoder(resp.Body).Decode(&record) + if err != nil { + t.Fatal(err) + } + if record.CountryCode != "US" { + t.Fatalf("Unexpected country code in record: %#v", record.CountryCode) + } +} + +func TestJSONEncoderIndent(t *testing.T) { + // TODO: validate indentation? + _, srv, err := runServer("/json/", &JSONEncoder{Indent: true}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/json/8.8.8.8") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + var record responseRecord + err = json.NewDecoder(resp.Body).Decode(&record) + if err != nil { + t.Fatal(err) + } + if record.CountryCode != "US" { + t.Fatalf("Unexpected country code in record: %#v", record.CountryCode) + } +} + +func TestJSONPEncoder(t *testing.T) { + _, srv, err := runServer("/json/", &JSONEncoder{}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/json/8.8.8.8?callback=f") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + code := bytes.NewBuffer([]byte(` + function f(record) { + set(record.country_code); + }; + `)) + code.Write(b) + vm := otto.New() + var countryCode string + vm.Set("set", func(call otto.FunctionCall) otto.Value { + if len(call.ArgumentList) > 0 { + countryCode = call.Argument(0).String() + } + return otto.UndefinedValue() + }) + _, err = vm.Run(code.Bytes()) + if err != nil { + t.Fatal(err) + } + if countryCode != "US" { + t.Fatalf("Unexpected country code in record: %#v", countryCode) + } +} + +func TestRequestLang(t *testing.T) { + r := http.Request{} + list := requestLang(&r) + if list != nil { + t.Fatal("Unexpected list is not nil") + } + r.Header = map[string][]string{ + "Accept-Language": []string{"en-us,en;q=0.5"}, + } + want := []string{"en-us", "en"} + list = requestLang(&r) + if len(list) != 2 { + t.Fatal("Unexpected list length:", len(list)) + } + for i, lang := range want { + if list[i] != lang { + t.Fatal("Unexpected item in list:", list[i]) + } + } +} + +func TestLocalizedName(t *testing.T) { + names := map[string]string{ + "de": "USA", + "en": "United States", + "es": "Estados Unidos", + "fr": "États-Unis", + "ja": "アメリカ合衆国", + "pt-BR": "Estados Unidos", + "ru": "Сша", + "zh-CN": "美国", + } + mkReq := func(lang string) *http.Request { + return &http.Request{ + Header: map[string][]string{ + "Accept-Language": []string{lang}, + }, + } + } + test := map[string]string{ + "pt-BR,en": "Estados Unidos", + "pt-br": "United States", + "es-ES,ru;q=0.8,q=0.2": "Сша", + "da, en-gb;q=0.8, en;q=0.7": "United States", + "da, fr;q=0.8, en;q=0.7": "États-Unis", + "da, de;q=0.5, zh-CN;q=0.8": "USA", // TODO: Use qvalue. + "da, es": "Estados Unidos", + "es-ES, ja": "アメリカ合衆国", + } + for k, v := range test { + name := localizedName(names, requestLang(mkReq(k))) + if name != v { + t.Fatalf("Unexpected name: want %q, have %q", v, name) + } + } +} diff --git a/example_test.go b/example_test.go new file mode 100644 index 0000000..38f3a66 --- /dev/null +++ b/example_test.go @@ -0,0 +1,118 @@ +// Copyright 2009-2014 The freegeoip authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package freegeoip + +import ( + "encoding/json" + "log" + "net" + "net/http" + "time" +) + +var maxmindFile = "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz" + +func ExampleDatabaseQuery() { + db, err := Open("./testdata.gz") + if err != nil { + log.Fatal(err) + } + var result customQuery + err = db.Lookup(net.ParseIP("8.8.8.8"), &result) + if err != nil { + log.Fatal(err) + } + defer db.Close() + log.Printf("%#v", result) +} + +func ExampleRemoteDatabaseQuery() { + updateInterval := 24 * time.Hour + maxRetryInterval := time.Hour + db, err := OpenURL(maxmindFile, updateInterval, maxRetryInterval) + if err != nil { + log.Fatal(err) + } + defer db.Close() + select { + case <-db.NotifyOpen(): + // Wait for the db to be downloaded. + case err := <-db.NotifyError(): + log.Fatal(err) + } + var result customQuery + err = db.Lookup(net.ParseIP("8.8.8.8"), &result) + if err != nil { + log.Fatal(err) + } + log.Printf("%#v", result) +} + +func ExampleServer() { + db, err := OpenURL(maxmindFile, 24*time.Hour, time.Hour) + if err != nil { + log.Fatal(err) + } + http.Handle("/csv/", NewHandler(db, &CSVEncoder{})) + http.Handle("/xml/", NewHandler(db, &XMLEncoder{})) + http.Handle("/json/", NewHandler(db, &JSONEncoder{})) + http.ListenAndServe(":8080", nil) +} + +func ExampleServerWithCustomEncoder() { + db, err := Open("./testdata/db.gz") + if err != nil { + log.Fatal(err) + } + http.Handle("/custom/json/", NewHandler(db, &customEncoder{})) + http.ListenAndServe(":8080", nil) +} + +// A customEncoder writes a custom JSON object to an http response. +type customEncoder struct{} + +// A customQuery is the query executed in the maxmind database for +// every IP lookup request. +type customQuery struct { + Country struct { + ISOCode string `maxminddb:"iso_code"` + Names map[string]string `maxminddb:"names"` + } `maxminddb:"country"` + Location struct { + Latitude float64 `maxminddb:"latitude"` + Longitude float64 `maxminddb:"longitude"` + TimeZone string `maxminddb:"time_zone"` + } `maxminddb:"location"` +} + +// A customResponse is what gets written to the http response as JSON. +type customResponse struct { + IP string + CountryCode string + CountryName string + Latitude float64 + Longitude float64 + TimeZone string +} + +// NewQuery implements the freegeoip.Encoder interface. +func (f *customEncoder) NewQuery() Query { + return &customQuery{} +} + +// Encode implements the freegeoip.Encoder interface. +func (f *customEncoder) Encode(w http.ResponseWriter, r *http.Request, q Query, ip net.IP) error { + record := q.(*customQuery) + out := &customResponse{ + IP: ip.String(), + CountryCode: record.Country.ISOCode, + CountryName: record.Country.Names["en"], // Set to client lang. + Latitude: record.Location.Latitude, + Longitude: record.Location.Longitude, + TimeZone: record.Location.TimeZone, + } + w.Header().Set("Content-Type", "application/json") + return json.NewEncoder(w).Encode(&out) +} diff --git a/freegeoip.conf b/freegeoip.conf deleted file mode 100644 index 2d40b60..0000000 --- a/freegeoip.conf +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - - - - - ./static - - - - - - - - - - - - - - - - - - - - diff --git a/freegeoip.go b/freegeoip.go index b6cc244..962fb1e 100644 --- a/freegeoip.go +++ b/freegeoip.go @@ -1,876 +1,109 @@ -// Copyright 2009-2014 Alexandre Fiori +// Copyright 2009-2014 The freegeoip authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// Web server of freegeoip.net - -package main +package freegeoip import ( - "database/sql" - "encoding/binary" - "encoding/json" - "encoding/xml" - "expvar" - "flag" "fmt" - "io" - "log" "math/rand" "net" "net/http" - "os" - "os/signal" - "path/filepath" - "runtime" - "runtime/pprof" - "strconv" "strings" - "sync" - "syscall" "time" - - "github.com/fiorix/go-redis/redis" - "github.com/fiorix/go-web/httpxtra" - "github.com/gorilla/context" - - // SQLite driver. - // _ "github.com/mattn/go-sqlite3" - _ "code.google.com/p/gosqlite/sqlite3" ) -var ( - collectStats bool - outputCount = expvar.NewMap("Output") // json, xml or csv - statusCount = expvar.NewMap("Status") // 200, 403, 404, etc - protocolCount = expvar.NewMap("Protocol") // HTTP or HTTPS - - dns *dnsPool -) - -func main() { - flLog := flag.String("l", "", "log to file instead of stderr") - flConfig := flag.String("c", "freegeoip.conf", "set config file") - flProfile := flag.Bool("profile", false, "run cpu and mem profiler") - flag.Parse() - - if *flProfile { - runProfile() - } - - cf, err := loadConfig(*flConfig) - if err != nil { - log.Fatal(err) - } - - collectStats = cf.Debug - - if cf.DNS.Enabled { - t, err := time.ParseDuration(cf.DNS.Timeout) - if err != nil { - log.Fatal("Invalid DNS timeout:", err) - } - dns = new(dnsPool) - dns.init(cf.DNS.MaxConcurrent, t) - } - - if len(*flLog) > 0 { - setLog(*flLog) - } - - runtime.GOMAXPROCS(runtime.NumCPU()) - log.Printf("FreeGeoIP server starting. debug=%t", cf.Debug) - - if cf.Debug && len(cf.DebugSrv) > 0 { - go func() { - // server for expvar's /debug/vars only - log.Printf("Starting DEBUG server on tcp/%s", cf.DebugSrv) - log.Fatal(http.ListenAndServe(cf.DebugSrv, nil)) - }() - } - - mux := http.NewServeMux() - mux.Handle("/", http.FileServer(http.Dir(cf.DocumentRoot))) - - st := time.Now() - db, err := openDB(cf) - if err != nil { - log.Fatal(err) - } - log.Println("IPDB cached in", time.Since(st)) - - lh := lookupHandler(cf, db) - mux.HandleFunc("/csv/", lh) - mux.HandleFunc("/xml/", lh) - mux.HandleFunc("/json/", lh) - - for _, c := range cf.Listen { - go runServer(mux, c) - } - - select {} -} - -func lookupHandler(cf *configFile, db *DB) http.HandlerFunc { - var rl rateLimiter - if cf.Limit.MaxRequests > 0 { - if len(cf.Redis) > 0 { - rl = new(redisQuota) - log.Printf("Using redis to manage quota: %s", cf.Redis) - } else { - rl = new(mapQuota) - log.Printf("Using internal map to manage quota.") - } - rl.init(cf) - } - return func(w http.ResponseWriter, r *http.Request) { - if r.TLS != nil { - w.Header().Set("Strict-Transport-Security", "max-age=31536000") - } - switch r.Method { - case "GET": - w.Header().Set("Access-Control-Allow-Origin", "*") - handleRequest(cf, db, rl, w, r) - case "OPTIONS": - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Content-Type", "text/plain") - w.Header().Set("Access-Control-Allow-Methods", "GET") - w.Header().Set("Access-Control-Allow-Headers", "X-Requested-With") - w.WriteHeader(200) - default: - w.Header().Set("Allow", "GET, OPTIONS") - http.Error(w, http.StatusText(405), 405) - } - } -} - -func handleRequest( - cf *configFile, - db *DB, - rl rateLimiter, - w http.ResponseWriter, - r *http.Request, -) { - // If xheaders is enabled, RemoteAddr might be a copy of - // the X-Real-IP or X-Forwarded-For HTTP headers, which - // can be a comma separated list of IPs. In this case, - // only the first IP in the list is used. - if strings.Index(r.RemoteAddr, ",") > 0 { - r.RemoteAddr = strings.SplitN(r.RemoteAddr, ",", 2)[0] - } - - // Parse remote address. - var ip net.IP - if sIP, _, err := net.SplitHostPort(r.RemoteAddr); err != nil { - ip = net.ParseIP(r.RemoteAddr) // Use X-Real-IP, etc - } else { - ip = net.ParseIP(sIP) - } - +func init() { + rand.Seed(time.Now().UnixNano()) +} + +// A Handler provides http handlers that can process requests and return +// data in multiple formats. +// +// Usage: +// +// handle := NewHandler(db) +// http.Handle("/json/", handle.JSON()) +// +// Note that the url pattern must end with a trailing slash since the +// handler looks for IP addresses or hostnames as parameters, for +// example /json/8.8.8.8 or /json/domain.com. +// +// If no IP or hostname is provided, then the handler will query the +// IP address of the caller. See the ProxyHandler for more. +type Handler struct { + db *DB + enc Encoder +} + +// NewHandler creates and initializes a new Handler. +func NewHandler(db *DB, enc Encoder) *Handler { + return &Handler{db, enc} +} + +// ServeHTTP implements the http.Handler interface. +func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ip := f.queryIP(r) if ip == nil { - // This could be a misconfigured unix socket server. - context.Set(r, "log", "Invalid source IP: "+r.RemoteAddr) - http.Error(w, http.StatusText(500), 500) - return - } - - // Convert remote IP to integer to check quota. - // IPv6 is not supported yet. See issue #21 for details. - nIP, err := ip2int(ip) - if err != nil { - context.Set(r, "log", err.Error()) - http.Error(w, "IPv6 is not supported.", 501) - return - } - - // Check quota. - if rl != nil { - var ok bool - if ok, err = rl.Ok(nIP); err != nil { - context.Set(r, "log", err.Error()) // redis err - http.Error(w, http.StatusText(503), 503) - return - } else if !ok { - // Over quota, soz :( - http.Error(w, http.StatusText(403), 403) - return - } - } - - // Figure out the query: /fmt/{query} or /fmt/{nil} - // In case of {nil} we query the remote IP. - path := strings.SplitN(r.URL.Path, "/", 3) - if len(path) != 3 { - // This handler is for /fmt/ where fmt is json, xml or csv. - log.Fatal("Unexpected URL:", r.URL.Path) - } - - // Process the query, if there's one. - if len(path[2]) > 0 { - // Allow to query by IP or hostname. - if ip = net.ParseIP(path[2]); ip == nil { - if dns == nil || strings.Contains(path[2], " ") { - // DNS lookups not allowed or invalid hostname. - http.Error(w, http.StatusText(404), 404) - return - } else if ip = dns.LookupHost(path[2]); ip == nil { - // DNS lookup failed, assume host not found. - http.Error(w, http.StatusText(404), 404) - return - } - } - - if nIP, err = ip2int(ip); err != nil { // IPv6 fails here. - context.Set(r, "log", err.Error()) - http.Error(w, http.StatusText(404), 404) - return - } - } - - // Query the db. - var record *geoipRecord - if record, err = db.Lookup(ip, nIP); err != nil { http.NotFound(w, r) return } - - // Write response. - switch path[1][0] { - case 'j': - if cb := r.FormValue("callback"); len(cb) > 0 { - w.Header().Set("Content-Type", "text/javascript") - err = record.JSONP(w, cb) - } else { - w.Header().Set("Content-Type", "application/json") - err = record.JSON(w) - } - case 'x': - w.Header().Set("Content-Type", "application/xml") - err = record.XML(w) - case 'c': - w.Header().Set("Content-Type", "application/csv") - err = record.CSV(w) - } - if err != nil { - context.Set(r, "log", err.Error()) - } -} - -func runServer(mux *http.ServeMux, c *serverConfig) { - h := httpxtra.Handler{ - Handler: mux, - XHeaders: c.XHeaders, - } - if c.Log { - h.Logger = httpLogger - } - s := http.Server{ - Addr: c.Addr, - Handler: h, - ReadTimeout: 15 * time.Second, - WriteTimeout: 15 * time.Second, - } - if len(c.KeyFile) > 0 && len(c.CertFile) > 0 { - log.Printf("Starting HTTPS server on tcp/%s "+ - "log=%t xheaders=%t cert=%s key=%s", - c.Addr, - c.Log, - c.XHeaders, - c.CertFile, - c.KeyFile, - ) - log.Fatal(s.ListenAndServeTLS(c.CertFile, c.KeyFile)) - } else { - log.Printf("Starting HTTP server on tcp/%s "+ - "log=%t xheaders=%t", - c.Addr, - c.Log, - c.XHeaders, - ) - log.Fatal(httpxtra.ListenAndServe(s)) - } -} - -type dnsPool struct { - queryChan chan *dnsQuery - queryTimeout time.Duration -} - -type dnsQuery struct { - hostname string - respChan chan net.IP -} - -func (p *dnsPool) init(size int, queryTimeout time.Duration) { - p.queryChan = make(chan *dnsQuery) - p.queryTimeout = queryTimeout - for n := 0; n < size; n++ { - go p.doWork() - } -} - -func (p *dnsPool) doWork() { - for q := range p.queryChan { - q.respChan <- p.resolve(q.hostname) - } -} - -func (p *dnsPool) resolve(hostname string) net.IP { - if a, err := net.LookupHost(hostname); err != nil { - return nil - } else if len(a) == 1 { - return net.ParseIP(a[0]) - } else { - return net.ParseIP(a[rand.Intn(len(a)-1)]) - } -} - -func (p *dnsPool) LookupHost(hostname string) net.IP { - q := &dnsQuery{hostname, make(chan net.IP, 1)} - select { - case p.queryChan <- q: - select { - case ip := <-q.respChan: - close(q.respChan) - return ip - case <-time.After(p.queryTimeout): - return nil - } - default: - return nil - } -} - -type DB struct { - db *sql.DB - stmt *sql.Stmt - - // cache - country map[string]string - region map[regionKey]string - city map[int]locationData -} - -type regionKey struct { - CountryCode, - RegionCode string -} - -type locationData struct { - CountryCode, - RegionCode, - CityName, - ZipCode string - Latitude, - Longitude float32 - MetroCode, - AreaCode string -} - -func openDB(cf *configFile) (*DB, error) { - var ( - db DB - err error - ) - if db.db, err = sql.Open("sqlite3", cf.IPDB.File); err != nil { - return nil, err - } - if _, err = db.db.Exec("PRAGMA cache_size=" + cf.IPDB.CacheSize); err != nil { - return nil, err - } - if db.stmt, err = db.db.Prepare(` - SELECT - loc_id - FROM - city_blocks - WHERE - ip_start <= ? - ORDER BY - ip_start DESC - LIMIT 1 - `); err != nil { - return nil, err - } - if err = db.loadCache(); err != nil { - return nil, err - } - return &db, nil -} - -func (db *DB) loadCache() error { - var ( - wg sync.WaitGroup - errc = make(chan error, 3) - ) - wg.Add(3) - go db.loadCountries(&wg, errc) - go db.loadRegions(&wg, errc) - go db.loadCities(&wg, errc) - wg.Wait() - select { - case err := <-errc: - return err - default: - return nil - } -} - -func (db *DB) loadCountries(wg *sync.WaitGroup, errc chan error) { - defer wg.Done() - db.country = make(map[string]string) - row, err := db.db.Query(` - SELECT - country_code, - country_name - FROM - country_blocks - `) - if err != nil { - errc <- err - return - } - defer row.Close() - var country_code, name string - for row.Next() { - if err = row.Scan( - &country_code, - &name, - ); err != nil { - errc <- err - return - } - db.country[country_code] = name - } -} - -func (db *DB) loadRegions(wg *sync.WaitGroup, errc chan error) { - defer wg.Done() - db.region = make(map[regionKey]string) - row, err := db.db.Query(` - SELECT - country_code, - region_code, - region_name - FROM - region_names - `) + q := f.enc.NewQuery() + err := f.db.Lookup(ip, q) if err != nil { - errc <- err + http.Error(w, "Try again later.", + http.StatusServiceUnavailable) return } - defer row.Close() - var country_code, region_code, name string - for row.Next() { - if err = row.Scan( - &country_code, - ®ion_code, - &name, - ); err != nil { - errc <- err - return - } - db.region[regionKey{country_code, region_code}] = name - } -} - -func (db *DB) loadCities(wg *sync.WaitGroup, errc chan error) { - defer wg.Done() - db.city = make(map[int]locationData) - row, err := db.db.Query("SELECT * FROM city_location") + w.Header().Set("X-Database-Date", f.db.Date().Format(http.TimeFormat)) + err = f.enc.Encode(w, r, q, ip) if err != nil { - errc <- err + f.db.sendError(fmt.Errorf("Failed to encode %#v: %s", q, err)) + http.Error(w, "An unexpected error occurred.", + http.StatusInternalServerError) return } - defer row.Close() - var ( - locId int - loc locationData - ) - for row.Next() { - if err = row.Scan( - &locId, - &loc.CountryCode, - &loc.RegionCode, - &loc.CityName, - &loc.ZipCode, - &loc.Latitude, - &loc.Longitude, - &loc.MetroCode, - &loc.AreaCode, - ); err != nil { - errc <- err - return - } - db.city[locId] = loc - } -} - -func (db *DB) Lookup(ip net.IP, nIP uint32) (*geoipRecord, error) { - for _, net := range reservedIPs { - if net.Contains(ip) { - return &geoipRecord{ - Ip: ip.String(), - CountryCode: "RD", - CountryName: "Reserved", - }, nil - } - } - var locId int - if err := db.stmt.QueryRow(nIP).Scan(&locId); err != nil { - return nil, err - } - return db.newRecord(&ip, locId), nil -} - -func (db *DB) newRecord(ip *net.IP, locId int) *geoipRecord { - city, ok := db.city[locId] - if !ok { - return &geoipRecord{Ip: ip.String()} - } - return &geoipRecord{ - Ip: ip.String(), - CountryCode: city.CountryCode, - CountryName: db.country[city.CountryCode], - RegionCode: city.RegionCode, - RegionName: db.region[regionKey{ - city.CountryCode, - city.RegionCode, - }], - CityName: city.CityName, - ZipCode: city.ZipCode, - Latitude: city.Latitude, - Longitude: city.Longitude, - MetroCode: city.MetroCode, - AreaCode: city.AreaCode, - } } -type geoipRecord struct { - XMLName xml.Name `json:"-" xml:"Response"` - Ip string `json:"ip"` - CountryCode string `json:"country_code"` - CountryName string `json:"country_name"` - RegionCode string `json:"region_code"` - RegionName string `json:"region_name"` - CityName string `json:"city" xml:"City"` - ZipCode string `json:"zipcode"` - Latitude float32 `json:"latitude"` - Longitude float32 `json:"longitude"` - MetroCode string `json:"metro_code"` - AreaCode string `json:"area_code"` -} - -func (r *geoipRecord) JSON(w io.Writer) error { - if collectStats { - outputCount.Add("json", 1) - } - return json.NewEncoder(w).Encode(r) -} - -func (r *geoipRecord) JSONP(w io.Writer, callback string) error { - if collectStats { - outputCount.Add("json", 1) - } - var err error - if _, err = w.Write([]byte(callback)); err != nil { - return err - } - if _, err = w.Write([]byte("(")); err != nil { - return err - } - if err = json.NewEncoder(w).Encode(r); err != nil { - return err - } - if _, err = w.Write([]byte(");")); err != nil { - return err - } - return nil -} - -func (r *geoipRecord) XML(w io.Writer) error { - if collectStats { - outputCount.Add("xml", 1) - } - enc := xml.NewEncoder(w) - enc.Indent("", "\t") - var err error - if _, err = w.Write([]byte(xml.Header)); err != nil { - return err +func (f *Handler) queryIP(r *http.Request) net.IP { + if r.URL.Path[len(r.URL.Path)-1] == '/' { + return f.remoteAddr(r) } - if err = enc.Encode(r); err != nil { - return err + q := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + if ip := net.ParseIP(q); ip != nil { + return ip } - if _, err = w.Write([]byte("\n")); err != nil { - return err - } - return nil -} - -func (r *geoipRecord) CSV(w io.Writer) error { - if collectStats { - outputCount.Add("csv", 1) - } - if _, err := fmt.Fprintf(w, - `"%s","%s","%s","%s","%s","%s","%s","%0.4f","%0.4f","%s","%s"`+"\r\n", - r.Ip, - r.CountryCode, - r.CountryName, - r.RegionCode, - r.RegionName, - r.CityName, - r.ZipCode, - r.Latitude, - r.Longitude, - r.MetroCode, - r.AreaCode, - ); err != nil { - return err - } - return nil -} - -type rateLimiter interface { - init(cf *configFile) // Initializes the limiter - Ok(ipkey uint32) (bool, error) // Returns true if under quota -} - -// mapQuota implements the rateLimiter interface using a map as the backend. -type mapQuota struct { - cf *configFile - mu sync.Mutex - ip map[uint32]int -} - -func (q *mapQuota) init(cf *configFile) { - q.cf = cf - q.ip = make(map[uint32]int) -} - -func (q *mapQuota) Ok(ipkey uint32) (bool, error) { - q.mu.Lock() - defer q.mu.Unlock() - if n, ok := q.ip[ipkey]; ok { - if n < q.cf.Limit.MaxRequests { - q.ip[ipkey]++ - return true, nil - } - return false, nil - } - q.ip[ipkey] = 1 - go func() { - time.Sleep(time.Duration(q.cf.Limit.Expire) * time.Second) - q.mu.Lock() - delete(q.ip, ipkey) - q.mu.Unlock() - }() - return true, nil -} - -// redisQuota implements the rateLimiter interface using Redis as the backend. -type redisQuota struct { - cf *configFile - rc *redis.Client -} - -func (q *redisQuota) init(cf *configFile) { - redis.MaxIdleConnsPerAddr = 5000 - q.cf = cf - q.rc = redis.New(cf.Redis...) - q.rc.Timeout = time.Duration(1500) * time.Millisecond -} - -func (q *redisQuota) Ok(ipkey uint32) (bool, error) { - k := strconv.Itoa(int(ipkey)) // "numeric" key - if ns, err := q.rc.Get(k); err != nil { - return false, fmt.Errorf("redis get: %s", err) - } else if len(ns) == 0 { - if err = q.rc.SetEx(k, q.cf.Limit.Expire, "1"); err != nil { - return false, fmt.Errorf("redis setex: %s", err) - } - } else if n, _ := strconv.Atoi(ns); n < q.cf.Limit.MaxRequests { - if n, err = q.rc.Incr(k); err != nil { - return false, fmt.Errorf("redis incr: %s", err) - } else if n == 1 { - q.rc.Expire(k, q.cf.Limit.Expire) - } - } else { - return false, nil - } - return true, nil -} - -func ip2int(ip net.IP) (uint32, error) { - ipv4 := ip.To4() - if ipv4 == nil { - return 0, fmt.Errorf("IP %s is not IPv4", ip.String()) - } - return binary.BigEndian.Uint32(ipv4), nil -} - -func runProfile() { - f, err := os.Create("freegeoip.cpu.prof") + ip, err := net.LookupIP(q) if err != nil { - log.Fatal(err) + return nil // Not found. } - pprof.StartCPUProfile(f) - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt, os.Kill) - go func() { - <-sig - pprof.StopCPUProfile() - f.Close() - f, err = os.Create("freegeoip.mem.prof") - if err != nil { - log.Fatal(err) - } - pprof.WriteHeapProfile(f) - os.Exit(0) - }() -} - -func setLog(filename string) { - f := openLog(filename) - log.SetOutput(f) - sigc := make(chan os.Signal, 1) - signal.Notify(sigc, syscall.SIGHUP) - go func() { - // Recycle log file on SIGHUP. - var fb *os.File - for { - <-sigc - fb = f - f = openLog(filename) - log.SetOutput(f) - fb.Close() - } - }() -} - -func openLog(filename string) *os.File { - f, err := os.OpenFile( - filename, - os.O_WRONLY|os.O_CREATE|os.O_APPEND, - 0644, - ) - if err != nil { - log.SetOutput(os.Stderr) - log.Fatal(err) - } - return f -} - -func httpLogger(r *http.Request, created time.Time, status, bytes int) { - //fmt.Println(httpxtra.ApacheCommonLog(r, created, status, bytes)) - var ( - s, ip, msg string - err error - ) - if r.TLS == nil { - s = "HTTP" - } else { - s = "HTTPS" - } - if ip, _, err = net.SplitHostPort(r.RemoteAddr); err != nil { - ip = r.RemoteAddr - } - if tmp := context.Get(r, "log"); tmp != nil { - msg = fmt.Sprintf(" (%s)", tmp) - context.Clear(r) - } - log.Printf("%s %d %s %q (%s) :: %d bytes in %s%s", - s, - status, - r.Method, - r.URL.Path, - ip, - bytes, - time.Since(created), - msg, - ) - if collectStats { - protocolCount.Add(s, 1) - statusCount.Add(strconv.Itoa(status), 1) - } -} - -type serverConfig struct { - Log bool `xml:"log,attr"` - XHeaders bool `xml:"xheaders,attr"` - Addr string `xml:"addr,attr"` - CertFile string - KeyFile string -} - -type configFile struct { - XMLName xml.Name `xml:"Server"` - Debug bool `xml:"debug,attr"` - DebugSrv string `xml:"debugsrv,attr"` - DocumentRoot string - - Listen []*serverConfig - - DNS struct { - Enabled bool `xml:",attr"` - Timeout string `xml:",attr"` - MaxConcurrent int `xml:",attr"` - } - - IPDB struct { - File string `xml:",attr"` - CacheSize string `xml:",attr"` - } - - Limit struct { - MaxRequests int `xml:",attr"` - Expire int `xml:",attr"` - } - - Redis []string `xml:"Redis>Addr"` -} - -func loadConfig(filename string) (*configFile, error) { - var cf configFile - if fd, err := os.Open(filename); err != nil { - return nil, err - } else { - if err = xml.NewDecoder(fd).Decode(&cf); err != nil { - return nil, err - } - fd.Close() - } - // Make files' path relative to the config file's directory. - basedir := filepath.Dir(filename) - relativePath(basedir, &cf.IPDB.File) - relativePath(basedir, &cf.DocumentRoot) - for _, l := range cf.Listen { - relativePath(basedir, &l.CertFile) - relativePath(basedir, &l.KeyFile) - } - return &cf, nil -} - -func relativePath(basedir string, filename *string) { - if len(*filename) > 0 && (*filename)[0] != '/' { - *filename = filepath.Join(basedir, *filename) + if len(ip) == 0 { + return nil } + return ip[rand.Intn(len(ip))] } -// http://en.wikipedia.org/wiki/Reserved_IP_addresses -var reservedIPs = []net.IPNet{ - {net.IPv4(0, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)}, - {net.IPv4(10, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)}, - {net.IPv4(100, 64, 0, 0), net.IPv4Mask(255, 192, 0, 0)}, - {net.IPv4(127, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)}, - {net.IPv4(169, 254, 0, 0), net.IPv4Mask(255, 255, 0, 0)}, - {net.IPv4(172, 16, 0, 0), net.IPv4Mask(255, 240, 0, 0)}, - {net.IPv4(192, 0, 0, 0), net.IPv4Mask(255, 255, 255, 248)}, - {net.IPv4(192, 0, 2, 0), net.IPv4Mask(255, 255, 255, 0)}, - {net.IPv4(192, 88, 99, 0), net.IPv4Mask(255, 255, 255, 0)}, - {net.IPv4(192, 168, 0, 0), net.IPv4Mask(255, 255, 0, 0)}, - {net.IPv4(198, 18, 0, 0), net.IPv4Mask(255, 254, 0, 0)}, - {net.IPv4(198, 51, 100, 0), net.IPv4Mask(255, 255, 255, 0)}, - {net.IPv4(203, 0, 113, 0), net.IPv4Mask(255, 255, 255, 0)}, - {net.IPv4(224, 0, 0, 0), net.IPv4Mask(240, 0, 0, 0)}, - {net.IPv4(240, 0, 0, 0), net.IPv4Mask(240, 0, 0, 0)}, - {net.IPv4(255, 255, 255, 255), net.IPv4Mask(255, 255, 255, 255)}, +func (f *Handler) remoteAddr(r *http.Request) net.IP { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return net.ParseIP(r.RemoteAddr) + } + return net.ParseIP(host) +} + +// ProxyHandler is a wrapper for other http handlers that sets the +// client IP address in request.RemoteAddr to the first value of a +// comma separated list of IPs from the X-Forwarded-For request +// header. It resets the original RemoteAddr back after running the +// designated handler f. +func ProxyHandler(f http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + addr := r.Header.Get("X-Forwarded-For") + if len(addr) > 0 { + remoteAddr := r.RemoteAddr + r.RemoteAddr = strings.SplitN(addr, ",", 2)[0] + defer func() { r.RemoteAddr = remoteAddr }() + } + f.ServeHTTP(w, r) + }) } diff --git a/freegeoip_test.go b/freegeoip_test.go index 92f67cb..f748ddf 100644 --- a/freegeoip_test.go +++ b/freegeoip_test.go @@ -1,198 +1,162 @@ -// Copyright 2009-2014 Alexandre Fiori +// Copyright 2009-2014 The freegeoip authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -package main +package freegeoip import ( "bytes" + "encoding/csv" + + "io/ioutil" "net" - "sync" + "net/http" + "net/http/httptest" + "net/url" "testing" - "time" -) - -var ( - testCf *configFile - testDB *DB ) -func TestLoadConfig(t *testing.T) { - var err error - testCf, err = loadConfig("freegeoip.conf") - if err != nil { - t.Fatal(err) +func TestQueryRemoteAddr(t *testing.T) { + want := net.ParseIP("8.8.8.8") + // No query argument, so we query the remote IP. + r := http.Request{ + URL: &url.URL{Path: "/"}, + RemoteAddr: "8.8.8.8:8888", + Header: http.Header{}, } -} - -func TestOpenDB(t *testing.T) { - var err error - testDB, err = openDB(testCf) - if err != nil { - t.Log("Make sure the DB exists: cd db && ./updatedb") - t.Fatal(err) + f := &Handler{} + if ip := f.queryIP(&r); !bytes.Equal(ip, want) { + t.Errorf("Unexpected IP: %s", ip) } } -func TestQueryDB1(t *testing.T) { - record := testQuery(t, "127.0.0.1") - if record.CountryName != "Reserved" { - t.Fatal("Unexpected value:", record.CountryName) +func TestQueryDNS(t *testing.T) { + want4 := net.ParseIP("8.8.8.8") + want6 := net.ParseIP("2001:4860:4860::8888") + r := http.Request{ + URL: &url.URL{Path: "/google-public-dns-a.google.com"}, + RemoteAddr: "127.0.0.1:8080", + Header: make(map[string][]string), } -} - -func TestQueryDB2(t *testing.T) { - record := testQuery(t, "8.8.8.8") - if record.CountryCode != "US" { - t.Fatal("Unexpected value:", record.CountryCode) + f := &Handler{} + ip := f.queryIP(&r) + if ip == nil { + t.Fatal("Failed to resolve", r.URL.Path) } -} - -func TestRecordJSON(t *testing.T) { - record := testQuery(t, "127.0.0.1") - b := bytes.NewBuffer(nil) - record.JSON(b) - if len(b.Bytes()) != 180 { - t.Fatal("Unexpected value:", b.String()) - } -} - -func TestRecordJSONP(t *testing.T) { - record := testQuery(t, "127.0.0.1") - b := bytes.NewBuffer(nil) - record.JSONP(b, "f") - if len(b.Bytes()) != 184 { - t.Fatal("Unexpected value:", b.String()) + if !bytes.Equal(ip, want4) && !bytes.Equal(ip, want6) { + t.Errorf("Unexpected IP: %s", ip) } } -func TestRecordXML(t *testing.T) { - record := testQuery(t, "127.0.0.1") - b := bytes.NewBuffer(nil) - record.XML(b) - if len(b.Bytes()) != 338 { - t.Fatal("Unexpected value:", b.String()) - } -} +// Test the server. -func TestRecordCSV(t *testing.T) { - record := testQuery(t, "127.0.0.1") - b := bytes.NewBuffer(nil) - record.CSV(b) - if len(b.Bytes()) != 65 { - t.Fatal("Unexpected value:", b.String()) +func runServer(pattern string, f Encoder) (*Handler, *httptest.Server, error) { + db, err := Open(testFile) + if err != nil { + return nil, nil, err } -} - -func TestMapQuota(t *testing.T) { - testCf.Limit.MaxRequests = 1 - testCf.Limit.Expire = 1 - rl := new(mapQuota) - rl.init(testCf) - nIP, _ := ip2int(net.ParseIP("127.0.0.1")) - if ok, _ := rl.Ok(nIP); !ok { - t.Fatal("Unexpected value:", ok) - } - if ok, _ := rl.Ok(nIP); ok { - t.Fatal("Unexpected value:", ok) + select { + case <-db.NotifyOpen(): + case err := <-db.NotifyError(): + if err != nil { + return nil, nil, err + } } + mux := http.NewServeMux() + handle := NewHandler(db, f) + mux.Handle(pattern, ProxyHandler(handle)) + return handle, httptest.NewServer(mux), nil } -func TestRedisQuota(t *testing.T) { - if len(testCf.Redis) < 1 { - t.Skip("Redis is not configured") - } - testCf.Limit.MaxRequests = 1 - testCf.Limit.Expire = 1 - rl := new(redisQuota) - rl.init(testCf) - nIP, _ := ip2int(net.ParseIP("127.0.0.1")) - if ok, err := rl.Ok(nIP); err != nil { +func TestLookupUnavailable(t *testing.T) { + handle, srv, err := runServer("/csv/", &CSVEncoder{}) + if err != nil { t.Fatal(err) - } else if !ok { - t.Fatal("Unexpected value:", ok) } - if ok, err := rl.Ok(nIP); err != nil { + defer srv.Close() + handle.db.mu.Lock() + reader := handle.db.reader + handle.db.reader = nil + handle.db.mu.Unlock() + defer func() { + handle.db.mu.Lock() + handle.db.reader = reader + handle.db.mu.Unlock() + }() + resp, err := http.Get(srv.URL + "/csv/8.8.8.8") + if err != nil { t.Fatal(err) - } else if ok { - t.Fatal("Unexpected value:", ok) } -} - -func TestDNSLookup(t *testing.T) { - dp := new(dnsPool) - dp.init(1, 1000*time.Millisecond) - if ip := dp.LookupHost("localhost"); ip == nil { - t.Fatal("Could not resolve host name") - } else if !isLocalhostIP(ip.String()) { - t.Fatal("Unexpected IP: " + ip.String()) - } -} - -func TestConcurrentDNSLookup1(t *testing.T) { - dp := new(dnsPool) - dp.init(1, 1000*time.Millisecond) - go dp.LookupHost("invalid.host.name1") - var wg sync.WaitGroup - wg.Add(1) - go func() { - if ip := dp.LookupHost("localhost"); ip != nil { - t.Error("Unexpected IP: " + ip.String()) - } - wg.Done() - }() - wg.Wait() -} - -func TestConcurrentDNSLookup2(t *testing.T) { - dp := new(dnsPool) - dp.init(2, 1000*time.Millisecond) - go dp.LookupHost("invalid.host.name2") - var wg sync.WaitGroup - wg.Add(1) - go func() { - if ip := dp.LookupHost("localhost"); ip == nil { - t.Error("Could not resolve host name") - } else if !isLocalhostIP(ip.String()) { - t.Error("Unexpected IP: " + ip.String()) + defer resp.Body.Close() + if resp.StatusCode != http.StatusServiceUnavailable { + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) } - wg.Done() - }() - wg.Wait() + t.Fatalf("Unexpected query worked: %s\n%s", resp.Status, b) + } } -func BenchmarkQueryDB(b *testing.B) { - if testDB == nil { - b.Skip("DB is not available") - } - ip := net.ParseIP("8.8.8.8") - nIP, _ := ip2int(ip) - for i := 0; i < b.N; i++ { - _, err := testDB.Lookup(ip, nIP) +func TestLookupNotFound(t *testing.T) { + _, srv, err := runServer("/csv/", &CSVEncoder{}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/csv/fail-me") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + b, err := ioutil.ReadAll(resp.Body) if err != nil { - b.Fatal(err) + t.Fatal(err) } + t.Fatalf("Unexpected query worked: %s\n%s", resp.Status, b) } } -func testQuery(t *testing.T, addr string) *geoipRecord { - if testDB == nil { - t.Skip("DB is not available") +func TestLookupXForwardedFor(t *testing.T) { + _, srv, err := runServer("/csv/", &CSVEncoder{}) + if err != nil { + t.Fatal(err) } - ip := net.ParseIP(addr) - nIP, _ := ip2int(ip) - record, err := testDB.Lookup(ip, nIP) + defer srv.Close() + req, err := http.NewRequest("GET", srv.URL+"/csv/", nil) if err != nil { t.Fatal(err) } - return record + req.Header.Set("X-Forwarded-For", "8.8.8.8") + resp, err := http.DefaultClient.Do(req) + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + row, err := csv.NewReader(resp.Body).Read() + if err != nil { + t.Fatal(err) + } + if row[0] != "US" { + t.Fatalf("Unexpected country code in record: %#v", row) + } } -func isLocalhostIP(ip string) bool { - for _, v := range []string{"::1", "fe80::1", "127.0.0.1"} { - if ip == v { - return true - } +func TestLookupDatabaseDate(t *testing.T) { + _, srv, err := runServer("/csv/", &CSVEncoder{}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/csv/") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + if len(resp.Header.Get("X-Database-Date")) == 0 { + t.Fatal("Header X-Database-Date is missing") } - return false } diff --git a/static/css/index.css b/static/css/index.css deleted file mode 100644 index dc03c03..0000000 --- a/static/css/index.css +++ /dev/null @@ -1,17 +0,0 @@ -body { - padding-top: 10px; - padding-bottom: 40px; -} -.container-narrow { - margin: 0 auto; - max-width: 700px; -} -.container-narrow > hr { - margin: 30px 0; -} -code { - color:#333; -} -div[class="tooltip-inner"] { - max-width: 350px; -} diff --git a/static/css/index.min.css b/static/css/index.min.css deleted file mode 100644 index cbc76df..0000000 --- a/static/css/index.min.css +++ /dev/null @@ -1 +0,0 @@ -body{padding-top:10px;padding-bottom:40px}.container-narrow{margin:0 auto;max-width:700px}.container-narrow>hr{margin:30px 0}.jumbotron{margin:20px 0;text-align:center}.jumbotron h1{font-size:72px;line-height:1}.jumbotron .btn{font-size:21px;padding:14px 24px}.marketing{margin:20px 0}.marketing p+h4{margin-top:28px}code{color:#333}.justify{text-align:justify}div[class="tooltip-inner"]{max-width:350px} \ No newline at end of file diff --git a/static/css/index.min.css.gz b/static/css/index.min.css.gz deleted file mode 100644 index 9194f95fb8363bd18966c9dc9f836244ee92ce17..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 250 zcmVu99f+i~u*}wUB8`hJO14%ZgU+10?0sU$eccUgVkx&jgFI(fugiSQDc|7scVuT+f z4+YnCOcB;VXG!MUXxfn9gp2(7Hl - - - - freegeoip.net: FREE IP Geolocation Web Service - - - - - - - - - - - - -
-
-
-

freegeoip.net

-
-
-
-
- -
-
- -
-
-
-
-
- -
-
-
-

Oooops!

-
Over quota
-

This service is public and free with a hard limit - of 10.000 requests per hour, and you've reached your quota.

-

Check out the Limits section below for more information.

-
-
-

Oooops!

-
No results for
-

Try one of the following:

-
    -
  • Make sure you enter a valid IP address, or
    • -
  • A valid host, or domain name (e.g. google.com)
    • -
-
-
-

Oooops!

-
Service Unavailable
-

The service is unavailable due to maintenance or high load.

-

We apologize for the inconvenience. Please try again later.

-
-
-

Estimated location:

- - - - - - - - - - -
IP
Country
Region
City
Zip code
Latitude and Longitude
Area and Metro codes
- Show map - Hide map -
-
-
-

About

-

freegeoip.net is a public REST API for searching geolocation of IP addresses and host names.

-

It has an internal database with geolocation information, which is queried via the API. There's no sorcery, it's just a database. And although the database is very accurate, don't expect it to be perfect.

-

See the freegeoip blog.

-
-
-
-
-
-

Usage for developers

-

Send HTTP GET requests to: - freegeoip.net/{format}/{ip_or_hostname}

-

The API supports both HTTP and HTTPS.

-

Supported formats are csv, xml or json, - which also support the - callback - query argument for JSONP.

-

The ip_or_hostname part is optional. Your own IP is - searched if one is not provided.

-
-
-

Limits

-

API usage is limited to 10,000 queries per hour. After - reaching this limit, all requests will result in HTTP 403 - (Forbidden) until the roll over.

-

If the usage limit is a problem, please consider running your own - instance of this system. It's open source and freely available - at GitHub.

-

Download the source code.

-
-
-
-

Support

-

freegeoip.net is community funded, - therefore please consider donating if you like and use this service.

- -
- - - - -
- -

For any other inquiries or feedback, - contact me directly.

- -
-
- -

This web site and all documentation is licensed under - Creative Commons 3.0. - This service includes GeoLite data created by MaxMind, available - from maxmind.com.

-
-
- - - diff --git a/static/js/index.js b/static/js/index.js deleted file mode 100644 index 926eb35..0000000 --- a/static/js/index.js +++ /dev/null @@ -1,64 +0,0 @@ -var gmap="//maps.google.com/maps?f=q&source=s_q&hl=en&geocode=&ie=UTF8&iwloc=A&output=embed&"; -function getParameterByName(name) { - var match = RegExp('[?&]' + name + '=([^&]*)').exec(window.location.search); - return match && decodeURIComponent(match[1].replace(/\+/g, ' ')); -}; -angular.module('freegeoip',[]) -.directive('btnSubmit', function(){ - return function(scope,element,attrs){ - scope.$watch(function(){ - return scope.$eval(attrs.btnSubmit); - }, - function(loading){ - if(loading) $(element).button('loading'); - else $(element).button('reset'); - }); - } -}); -freegeoip.$inject = ['$scope','$http']; -function freegeoip($scope,$http){ - $scope.geoip = {} - $scope.search = function(q, showMap) { - $scope.error = null; - $scope.map = showMap; - $scope.searching = true; - $http.get('json/' + (q || "")). - success(function(rs){ - $scope.q = q || rs.ip; - if(!rs.metro_code) rs.metro_code = '-'; - if(!rs.area_code) rs.area_code = '-'; - $scope.geoip = angular.copy(rs); - var qs = ""; - var zoom = 2; - if(rs.country_name&&rs.country_code!='RD'){ - if(rs.region_name){ - if(rs.city){ - zoom=6; - qs=rs.city+','+rs.region_name+','+rs.country_name; - } else { - zoom=4; - qs=rs.region_name+','+rs.country_name; - } - } else { - qs=rs.country_name; - } - } else { - qs = "Africa"; - } - $("#map").attr("src", gmap+"z="+zoom+"&q="+qs); - $scope.searching = false; - }). - error(function(data,st){ - $scope.errorq = q; - $scope.error = st; - $scope.searching = false; - $("#map").attr("src", gmap+"zoom=2&q=africa"); - }); - } - $scope.search(getParameterByName("q"), getParameterByName("map")); -} -$(document).ready(function(){ - $("#map").height($("#map").width()/2); - $("#cb").tooltip(); - $("#cb").click(function(el){el.preventDefault();}); -}); diff --git a/static/js/index.min.js b/static/js/index.min.js deleted file mode 100644 index ce5155c..0000000 --- a/static/js/index.min.js +++ /dev/null @@ -1,3 +0,0 @@ -var gmap="//maps.google.com/maps?f=q&source=s_q&hl=en&geocode=&ie=UTF8&iwloc=A&output=embed&";function getParameterByName(a){return(a=RegExp("[?&]"+a+"=([^&]*)").exec(window.location.search))&&decodeURIComponent(a[1].replace(/\+/g," "))}angular.module("freegeoip",[]).directive("btnSubmit",function(){return function(a,c,d){a.$watch(function(){return a.$eval(d.btnSubmit)},function(a){a?$(c).button("loading"):$(c).button("reset")})}});freegeoip.$inject=["$scope","$http"]; -function freegeoip(a,c){a.geoip={};a.search=function(d,f){a.error=null;a.map=f;a.searching=!0;c.get("json/"+(d||"")).success(function(b){a.q=d||b.ip;b.metro_code||(b.metro_code="-");b.area_code||(b.area_code="-");a.geoip=angular.copy(b);var e="",c=2;b.country_name&&"RD"!=b.country_code?b.region_name?b.city?(c=6,e=b.city+","+b.region_name+","+b.country_name):(c=4,e=b.region_name+","+b.country_name):e=b.country_name:e="Africa";$("#map").attr("src",gmap+"z="+c+"&q="+e);a.searching=!1}).error(function(b, -c){a.errorq=d;a.error=c;a.searching=!1;$("#map").attr("src",gmap+"zoom=2&q=africa")})};a.search(getParameterByName("q"),getParameterByName("map"))}$(document).ready(function(){$("#map").height($("#map").width()/2);$("#cb").tooltip();$("#cb").click(function(a){a.preventDefault()})}); diff --git a/static/js/index.min.js.gz b/static/js/index.min.js.gz deleted file mode 100644 index 0a1e54b232750b31600bfd518d05690fe614b8cb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 678 zcmV;X0$KeZiwFR1yD3uw1BFvjZ__Xke&4U?jw-TAaM^?qT4Ys&fy4_!gFTfpInFiB zNbJ;c+LbN;olCo>Eg*PG?z_)-pT9dVl^bNGsw9j?^r=N=ZC0Srw%o+ynOp?5ZJb6~ z-!6iAA<+aGT5VI5L5}j~>xVZ%zACJimw|13)p&_zf@uKptTEc>)(kTApOsT3dUS8s zpXi4v{@~Cz&M+mfF?)YsF_?^lDTFEn$tK@}>2nTT;5}-#%1vrlf|{tVo~ThdJ?A_K zQtXu6Tz`CL%gP!wo~g;%R5+{(rIC%kg`+Hj0dU?blQo5MqO@sKAcL7hq^sl=M3X5O zY3`8ZE-90E^QB43+(Wdx1Ka5x^oWXdl=259hAZXuoc&3q6fRZ4QqfoB?Y<;=pvFU{ zxkwuC34p??G&dRWizC=!jUIT*+m^?@Y%$Eu9Z8lG7}nZW2oVhD-d8Y=TDWiKd=W*q~avrd|(D2UT-$E@d)IC=Uq)>OqaxT~!hLCDg_M<9nq-Dr(< zy>}w%$}J@26OmVOB4`Diz3pc9_{a`J2`_*XK{-_Wd>?JoJMp_o(xunbBVN)P69pnI zvCjqz@MYiQAe0Isj#RPNEP+&G~P$w=XbLy^w?dd#$Z9iiMtA-NeI(VmdQ>imMv zyxN@k9}(p-LI7OOT&@+wLk7>tXIgjVy<<>24N>vpO(5 zI???I$@q9TP5s-^fA+Mtl;_k^ZS{4F*B@tpdCaf?9{mm9G~n%!rB*j3y-rR(sdRn# zR`x>Xm}hg(_P{Dn{haa9Ip4_D31Qya!qYY%K)T5F4|epdMCEWv{jPAPn!>a03Oqjf M1(xDUfA9nV079fySO5S3 diff --git a/static/map.html b/static/map.html deleted file mode 100644 index 889f886..0000000 --- a/static/map.html +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - FreeGeoIP - - - - - - -
- - diff --git a/testdata/.placeholder b/testdata/.placeholder new file mode 100644 index 0000000..e69de29