diff --git a/.gitignore b/.gitignore index 4f5d3e7..2a22d06 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ +db.gz *.pem *.rdb *.swp *.csv *.zip *.sqlite -freegeoip diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..a0b95da --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +language: go +go: + - 1.3 + - release + - tip +script: + - go test -v -cover + +install: + - go get code.google.com/p/go.tools/cmd/cover diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..7d80c4d --- /dev/null +++ b/AUTHORS @@ -0,0 +1,11 @@ +# This is the official list of freegeoip authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS file. +# +# Names should be added to this file as +# Name or Organization +# +# The email address is not required for organizations. +# +# Please keep the list sorted. + +Alexandre Fiori diff --git a/CONTRIBUTORS b/CONTRIBUTORS new file mode 100644 index 0000000..8a2f354 --- /dev/null +++ b/CONTRIBUTORS @@ -0,0 +1,21 @@ +# This is the official list of freegeoip contributors for copyright purposes. +# This file is distinct from the AUTHORS file. +# +# Names should be added to this file as +# Name or Organization +# +# Please keep the list sorted. +# +# Use the following command to generate the list: +# +# git shortlog -se | awk '{print $2 " " $3 " " $4}' +# +# The email address is not required for organizations. + +Alex Goretoy +Gleicon Moraes +Leandro Pereira +Lucas Fontes +Matthias Nehlsen +Melchi +Vladimir Agafonkin diff --git a/Dockerfile b/Dockerfile index 7779022..eb10704 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,10 @@ FROM google/golang -RUN apt-get install -y build-essential libsqlite3-dev pkg-config file supervisord - -WORKDIR /gopath/src/app -ADD . /gopath/src/app/ -RUN go get app -RUN cd /gopath/src/app/ -RUN go build - -#... will download files and process them to create ipdb.sqlite -RUN cd db && ./updatedb -RUN file /gopath/src/app/db/ipdb.sqlite - -RUN /usr/bin/install -o www-data -g www-data -m 0755 -d /var/log/freegeoip - -EXPOSE 8080 - -CMD [] -ENTRYPOINT ["/gopath/bin/app"] +ADD . /gopath/src/github.com/fiorix/freegeoip +WORKDIR /gopath/src/github.com/fiorix/freegeoip/cmd/freegeoip +RUN go get +RUN go install +RUN cp -r public /var/www + +ENTRYPOINT ["/gopath/bin/freegeoip"] +CMD ["-public", "/var/www"] diff --git a/HISTORY.md b/HISTORY.md new file mode 100644 index 0000000..1a8e68d --- /dev/null +++ b/HISTORY.md @@ -0,0 +1,55 @@ +# History of freegeoip.net + +The freegeoip software is the result of a web server research project that +started in 2009, written in Python and hosted on +[Google App Engine](http://appengine.google.com). It was rapidly adopted by +many developers around the world due to its simplistic and straightforward +HTTP API, causing the free account on GAE to exceed its quota every day +after few hours of operation. + +A year later freegeoip 1.0 was released, and the freegeoip.net domain +moved over to its own server infrastructure. The software was rewritten +using the [Cyclone](http://cyclone.io) web framework, backed by +[Twisted](http://twistedmatrix.com) and [PyPy](http://pypy.org) in +production. That's when the first database management tool was created, +a script that would download many pieces of information from the Internet +to create the IP database, an sqlite flat file used by the server. + +This version of the Python server shipped with a much better front-end as +well, but still as a server-side rendered template inherited from the GAE +version. It was only circa 2011 that freegeoip got its first standalone +front-end based on jQuery, and is when Twitter bootstrap was first used. + +Python played an important role in the early life of freegeoip and +allowed the service to grow and evolve fast. It provided a lot of +flexibility in building and maintaining the IP database using multiple +sources of data. This version of the server lasted until 2013, when +it was once again rewritten from scratch, this time in Go. The database +tool, however, remained intact. + +In 2013 the Go version was released as freegeoip 2.0 and this version +had many iterations. The first versions of the server written in Go were +very rustic, practically a verbatim transcription of the Python server. +Took a while until it started looking more like common Go code, and to +have tests. + +Another important change that shipped with v2 was a front-end based on +AngularJS, but still mixed with some jQuery. The Google map in the front +page was made optional to put more focus on the HTTP API. The popularity +of freegeoip has increased considerably over the years of 2013 and 2014, +calling for more. + +Enter freegeoip 3.0, an evolution of the Go server. The foundation of +freegeoip, which is the IP database and HTTP API, now lives in a Go +package that other developers can leverage. The freegeoip web server is +built on this package making its code cleaner, the server faster, +and requires zero maintenance for the IP database. The server downloads +the file from MaxMind and keep it up to date in background. + +This and other changes make it very Docker friendly. + +The front-end has been trimmed down to a single index.html file that loads +CSS and JS from CDNs on the internet. The JS part is based on AngularJS +and handles the search request and response of the public site. The +optional map has become a link to Google Maps following the lat/long +of the query results. diff --git a/LICENSE b/LICENSE index 8daa3aa..27dafae 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2013 Alexandre Fiori. All rights reserved. +Copyright (c) 2009-2013 The freegeoip authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/README.md b/README.md index 5c2945f..ab7857a 100644 --- a/README.md +++ b/README.md @@ -1,188 +1,123 @@ -# freegeoip.net +# freegeoip -freegeoip.net is a public web service for searching -[geolocation](http://en.wikipedia.org/wiki/Geolocation) of IP addresses. -This is the source code of freegeoip.net's web server and script for building -the IP database. +This is the source code of the freegeoip software. It contains both +the web server that empowers freegeoip.net, and a package for the +[Go](http://golang.org) programming language that enables any web server +to support IP geolocation with a simple and clean API. +See http://en.wikipedia.org/wiki/Geolocation for details about geolocation. -## Overview +## Web Server -freegeoip.net is the result of a web server research project that started in -2009 hosted at Google's [App Engine](http://appengine.google.com), -using the Python API. -A year later it moved to its own server infrastructure built on the -[Cyclone](http://cyclone.io) web framework, backed by -[Twisted](http://twistedmatrix.com) and [PyPy](http://pypy.org). +The freegeoip web server is a standalone program that serves an HTTP API +for searching the geolocation of IP addresses. To serve the API, it uses +an IP database that is automatically downloaded and auto-updated from +the internet when the server is running. -The current version is written in Go as the experiments progress with -[go-web](https://github.com/fiorix/go-web) and -[go-redis](https://github.com/fiorix/go-redis). +The API returns data encoded in popular formats such as CSV, XML, JSON +and JSONP. +### Usage -### Install - -List of prerequisites for building and running the server: - -- Go compiler - for `freegeoip.go` -- Git (for downloading Go packages) -- Mercurial (for downloading Go packages) -- libsqlite3-dev, gcc or llvm - for dependency `go-sqlite3` -- Python - for the `updatedb` script -- Redis - (optional) for API usage quotas -- The IP database - -The following instructions are for Debian and Ubuntu servers. - -Make sure Go is installed and both $GOROOT and $GOPATH are set, then run: - - apt-get install build-essential libsqlite3-dev pkg-config - go get github.com/fiorix/freegeoip - cd $GOPATH/src/github.com/fiorix/freegeoip - go build - -On recent OSX you might have to set the CC=clang before `go build` if -the sqlite3 package fails to compile. - -Proceed to building the IP database before starting the server. - - -### Building the IP database - -The IP database is composed of multiple files from multiple sources. It's a -combination of IP subnets, country codes, city names, etc. - -There's a helper script under the `db` directory that automates the process -of building the database, and can be used regularly to update it as well. - -It's a Python script called `updatedb` that creates `ipdb.sqlite`: - - $ cd db - $ ./updatedb - ... will download files and process them to create ipdb.sqlite - $ file ipdb.sqlite - ipdb.sqlite: SQLite 3.x database - -This service includes GeoLite data created by MaxMind, available from -maxmind.com. +Run the server: + ./freegeoip -## Running +Wait for it to download the IP database file for the first time. It does +it in background and writes a message to the console when ready. If you'd +like to use an alternative database source, see the `-db` command line +flag. -The server looks for `freegeoip.conf` in the current directory, but an -alternative config can be specified using the `-c` command line option. +If the server is queried when there is no database available, including +this initial first run, it returns *HTTP 503 (Service Unavailable)*, since +it can't service requests before a proper database is in place. -By default it logs to the stderr, but log file can be specified using -the `-l` command line option. Log files are cycled on SIGHUP. +### Querying -If the server is proxied by Nginx or another HTTP load balancer, edit the -configuration file and set `xheaders="true"` and it'll use X-Real-IP or -X-Forwarded-For HTTP headers (when available) as the client IP. +You can use any HTTP client to test the server. The examples below use +curl and the environment variable $freegeoip, which must be set to the +address of your server, like http://localhost:8080 for example: -Run the server: + export freegeoip=http://localhost:8080 - ./freegeoip [-c freegeoip.conf] [-l freegeoip.log] +Querying the API is very straightforward: you just have to pick a format +of your choice and provide either the IP address or hostname that you'd +like to search for. The syntax is as follows: -Then point the browser to http://localhost:8080. + $freegeoip/{format}/{IP_or_hostname} -If the IP database is unavailable (e.g. file does not exist, bad permissions) -or redis is unreachable (if using redis as the quota backend), all queries -will result in HTTP 503 (Service Unavailable). - -For listening on low ports as non-root user (e.g. www-data) on linux, set -file capabilities at least once before running it: +Examples: - /sbin/setcap 'cap_net_bind_service=+ep' /opt/freegeoip/freegeoip + curl -i $freegeoip/csv/8.8.8.8 -### Running with upstart + curl -i $freegeoip/xml/4.2.2.2 -On Ubuntu, use the following upstart script in `/etc/init/freegeoip.conf` -to start and stop the server: + curl -i $freegeoip/json/github.com - # freegeoip web service - # https://github.com/fiorix/freegeoip +If a domain or hostname is passed in the URL, the server will resolve that +name to its IP address and lookup the IP instead. If the hostname contains +multiple IPs associated to it, the server picks one randomly, which means +it could be either IPv4 or IPv6. - description "freegeoip web service" +If no IP or hostname is provided, then the server queries the IP address +of the HTTP client. - start on runlevel [2345] - stop on runlevel [!2345] +Example: - limit nofile 20000 20000 - setuid www-data - setgid www-data - exec /opt/freegeoip/freegeoip -c /opt/freegeoip/freegeoip.conf -l /var/log/freegeoip/freegeoip.log + curl -i freegeoip.net/json/ (this queries your own IP address) -The log directory must be created with the right permissions before the -daemon can be started. Use the following command for this: +The JSON endpoint also supports JSONP, by adding a `callback` argument +to the request query. - /usr/bin/install -o www-data -g www-data -m 0755 -d /var/log/freegeoip +Example: -Then use `start freegeoip` and `stop freegeoip` to start and stop the server. + curl -i freegeoip.net/json/8.8.8.8?callback=f -Also, use the following configuration file in `/etc/logrotate.d/freegeoip` for -log rotation: +See http://en.wikipedia.org/wiki/JSONP for details on how JSONP works. - /var/log/freegeoip/freegeoip.log - { - rotate 7 - daily - missingok - notifempty - delaycompress - compress - postrotate - reload freegeoip > /dev/null 2>&1 - endscript - } +## freegeoip package for Go -### Running with supervisord +The freegeoip package for the Go programming language provides two APIs: -Use [supervisor](http://supervisord.org) with the following config in -`/etc/supervisor/conf.d/freegeoip.conf`: +- A database API that requires zero maintenance of the IP database; +- A geolocation `http.Handler` that can be used/served by any http server. - [program:freegeoip] - user=www-data - redirect_stderr=true - directory=/opt/freegeoip - command=/opt/freegeoip/freegeoip - stdout_logfile=/var/log/freegeoip/freegeoip.log - stdout_logfile_maxbytes=50MB - stdout_logfile_backups=20 +tl;dr if all you want is code then see the `examples_test.go` file. -Then use `supervisorctl start freegeoip` and `supervisorctl stop freegeiop` -to start and stop the server. +Otherwise check out the godoc reference. +[![GoDoc](https://godoc.org/github.com/fiorix/freegeoip?status.svg)](https://godoc.org/github.com/fiorix/freegeoip) +[![Build Status](https://secure.travis-ci.org/fiorix/freegeoip.png)](http://travis-ci.org/fiorix/freegeoip) -## Usage +### Features -Point the browser to http://localhost:8080 and search for IPs or hostnames. +- Zero maintenance -Use curl from the command line to query the API: +The DB object alone can download an IP database file from the internet and +service lookups to your program right away. It will auto-update the file in +background and always magically work. - $ curl -v http://localhost:8080/{format}/{ip_or_hostname} +- DevOps friendly -It supports csv, json and xml as the output format. JSON supports callbacks -with the `callback` query argument. The client (self) IP is used if -`ip_or_hostname` is omitted in the query. +If you do care about the database and have the commercial version of the +MaxMind database, you can update the database file with your program running +and the DB object will load it in background. You can focus on your stuff. -Examples: +- Extensible - $ curl -v http://localhost:8080/csv/ - $ curl -v http://localhost:8080/xml/ - $ curl -v http://localhost:8080/xml/freegeoip.net - $ curl -v http://localhost:8080/json/github.com?callback=foobar +Besides the database part, the package provides an `http.Handler` object +that you can add to your HTTP server to service IP geolocation lookups with +the same simplistic API of freegeoip.net. There's also an interface for +crafting your own HTTP responses encoded in any format. -If the server is listening on unix sockets, use `nc` to test: +### Install - echo -ne 'GET /json/my-domain.abc HTTP/1.0\r\n\r\n' | nc -U /tmp/freegeoip.sock +Install the package: + go get github.com/fiorix/freegeoip -## Credits +Install the web server: -Thanks to (in no particular order): + go install github.com/fiorix/freegeoip/cmd/freegeoip -- [Gleicon](https://github.com/gleicon) for all the drama. -- Google for the map, Go, and AngularJS. -- Twitter for Bootstrap. -- MaxMind for the current database. -- ipinfodb.com for both the IP and timezones database back in 2010 and 2011. +Test coverage is quite good and may help you find the stuff you need. diff --git a/cmd/freegeoip/main.go b/cmd/freegeoip/main.go new file mode 100644 index 0000000..9009004 --- /dev/null +++ b/cmd/freegeoip/main.go @@ -0,0 +1,250 @@ +// Copyright 2009-2014 The freegeoip authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package main + +import ( + "flag" + "log" + "net" + "net/http" + "net/url" + "runtime" + "strconv" + "strings" + "time" + + "github.com/fiorix/freegeoip" + "github.com/fiorix/go-redis/redis" + "github.com/gorilla/context" +) + +var maxmindFile = "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz" + +func main() { + addr := flag.String("addr", ":8080", "Address in form of ip:port to listen on") + certFile := flag.String("cert", "", "X.509 certificate file") + keyFile := flag.String("key", "", "X.509 key file") + public := flag.String("public", "", "Public directory to serve at the / endpoint") + ipdb := flag.String("db", maxmindFile, "IP database file or URL") + updateIntvl := flag.Duration("update", 24*time.Hour, "Database update check interval") + retryIntvl := flag.Duration("retry", time.Hour, "Max time to wait before retrying update") + useXFF := flag.Bool("use-x-forwarded-for", false, "Use the X-Forwarded-For header when available") + silent := flag.Bool("silent", false, "Do not log requests to stderr") + redisAddr := flag.String("redis", "127.0.0.1:6379", "Redis address in form of ip:port for quota") + quotaMax := flag.Int("quota-max", 0, "Max requests per source IP per interval; Set 0 to turn off") + quotaIntvl := flag.Duration("quota-interval", time.Hour, "Quota expiration interval") + flag.Parse() + + rc, err := redis.Dial(*redisAddr) + if err != nil { + log.Fatal(err) + } + + db, err := openDB(*ipdb, *updateIntvl, *retryIntvl) + if err != nil { + log.Fatal(err) + } + + runtime.GOMAXPROCS(runtime.NumCPU()) + + encoders := map[string]http.Handler{ + "/csv/": freegeoip.NewHandler(db, &freegeoip.CSVEncoder{UseCRLF: true}), + "/xml/": freegeoip.NewHandler(db, &freegeoip.XMLEncoder{Indent: true}), + "/json/": freegeoip.NewHandler(db, &freegeoip.JSONEncoder{}), + } + + if *quotaMax > 0 { + seconds := int((*quotaIntvl).Seconds()) + for path, f := range encoders { + encoders[path] = userQuota(rc, *quotaMax, seconds, f) + } + } + + mux := http.NewServeMux() + for path, handler := range encoders { + mux.Handle(path, handler) + } + + if len(*public) > 0 { + mux.Handle("/", http.FileServer(http.Dir(*public))) + } + + handler := CORS(mux, "GET", "HEAD") + + if !*silent { + log.Println("freegeoip server starting on", *addr) + go logEvents(db) + handler = logHandler(handler) + } + + if *useXFF { + handler = freegeoip.ProxyHandler(handler) + } + + if len(*certFile) > 0 && len(*keyFile) > 0 { + err = http.ListenAndServeTLS(*addr, *certFile, *keyFile, handler) + } else { + err = http.ListenAndServe(*addr, handler) + } + if err != nil { + log.Fatal(err) + } +} + +// openDB opens and returns the IP database. +func openDB(dsn string, updateIntvl, maxRetryIntvl time.Duration) (db *freegeoip.DB, err error) { + _, err = url.Parse(dsn) + if err != nil { + db, err = freegeoip.Open(dsn) + } else { + db, err = freegeoip.OpenURL(dsn, updateIntvl, maxRetryIntvl) + } + return +} + +// CORS is an http handler that checks for allowed request methods (verbs) +// and adds CORS headers to all http responses. +// +// See http://en.wikipedia.org/wiki/Cross-origin_resource_sharing for details. +func CORS(f http.Handler, allow ...string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Method", + strings.Join(allow, ", ")+", OPTIONS") + if r.Method == "OPTIONS" { + w.WriteHeader(200) + return + } + for _, method := range allow { + if r.Method == method { + f.ServeHTTP(w, r) + return + } + } + w.Header().Set("Allow", strings.Join(allow, ", ")+", OPTIONS") + http.Error(w, http.StatusText(http.StatusMethodNotAllowed), + http.StatusMethodNotAllowed) + }) +} + +// userQuota is a handler that provides a rate limiter to the freegeoip API. +// It allows qmax requests per qintvl, in seconds. +// +// If redis is not available it responds with service unavailable. +func userQuota(rc *redis.Client, qmax int, qintvl int, f http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var ip string + if idx := strings.LastIndex(r.RemoteAddr, ":"); idx != -1 { + ip = r.RemoteAddr[:idx] + } else { + ip = r.RemoteAddr + } + sreq, err := rc.Get(ip) + if err != nil { + serviceUnavailable(w, r, err.Error()) + return + } + if len(sreq) == 0 { + err = rc.SetEx(ip, qintvl, "1") + if err != nil { + serviceUnavailable(w, r, err.Error()) + return + } + f.ServeHTTP(w, r) + return + } + nreq, _ := strconv.Atoi(sreq) + if nreq >= qmax { + http.Error(w, "Quota exceeded", http.StatusForbidden) + return + } + _, err = rc.Incr(ip) + if err != nil { + context.Set(r, "log", err.Error()) + } + f.ServeHTTP(w, r) + }) +} + +// serviceUnavailable writes an http error 501 to a client. +func serviceUnavailable(w http.ResponseWriter, r *http.Request, log string) { + context.Set(r, "log", log) + http.Error(w, "Try again later", http.StatusServiceUnavailable) +} + +// logEvents logs database events. +func logEvents(db *freegeoip.DB) { + for { + select { + case file := <-db.NotifyOpen(): + log.Println("database loaded:", file) + case err := <-db.NotifyError(): + log.Println("database error:", err) + case <-db.NotifyClose(): + return + } + } +} + +// logHandler logs http requests. +func logHandler(f http.Handler) http.Handler { + empty := "" + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp := responseWriter{w, http.StatusOK, 0} + start := time.Now() + f.ServeHTTP(&resp, r) + elapsed := time.Since(start) + extra := context.Get(r, "log") + if extra != nil { + defer context.Clear(r) + } else { + extra = empty + } + log.Printf("%q %d %q %q %s %q %db in %s %q", + r.Proto, + resp.status, + r.Method, + r.URL.Path, + remoteIP(r), + r.Header.Get("User-Agent"), + resp.bytes, + elapsed, + extra, + ) + }) +} + +// remoteIP returns the client's address without the port number. +func remoteIP(r *http.Request) string { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return host +} + +// responseWriter is an http.ResponseWriter that records the returned +// status and bytes written to the client. +type responseWriter struct { + http.ResponseWriter + status int + bytes int +} + +// Write implements the http.ResponseWriter interface. +func (f *responseWriter) Write(b []byte) (int, error) { + n, err := f.ResponseWriter.Write(b) + if err != nil { + return 0, err + } + f.bytes += n + return n, nil +} + +// WriteHeader implements the http.ResponseWriter interface. +func (f *responseWriter) WriteHeader(code int) { + f.status = code + f.ResponseWriter.WriteHeader(code) +} diff --git a/static/crossdomain.xml b/cmd/freegeoip/public/crossdomain.xml similarity index 100% rename from static/crossdomain.xml rename to cmd/freegeoip/public/crossdomain.xml diff --git a/static/favicon.ico b/cmd/freegeoip/public/favicon.ico similarity index 100% rename from static/favicon.ico rename to cmd/freegeoip/public/favicon.ico diff --git a/cmd/freegeoip/public/index.html b/cmd/freegeoip/public/index.html new file mode 100644 index 0000000..31d80fd --- /dev/null +++ b/cmd/freegeoip/public/index.html @@ -0,0 +1,260 @@ + + + + + + + + + +freegeoip.net + + + + + + + + + + +
+ +
+
+

About

+

+ freegeoip.net + provides a public HTTP API for software developers + to search the geolocation of IP addresses. It uses + a database of IP addresses that are associated to + cities along with other relevant information like + time zone, latitude and longitude. +

+ +

+ You're allowed up to 10,000 queries per + hour by default. Once this limit is reached, all of + your requests will result in HTTP 403, forbidden, + until your quota is cleared. +

+ +

+ The freegeoip web server is free and open source so + if the public service limit is a problem for you, + download it and run your own instance. +

+ +
+ +

+ Download + GitHub project +

+
+ +
+
+ +
+
+

Searching, please wait...

+

Server returned {{error}}

+ IP database date:
{{lastUpdated}} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IP{{record.ip}}
Country{{record.country_name}}
Region{{record.region_name}}
City{{record.city}}
Zip/Postal code{{record.zip_code}}
Lat/Long + + {{record.latitude}}, {{record.longitude}} + +
Metro code{{record.metro_code||''}}
Time zone + + {{record.time_zone}} + +
+
+ +
+ +
+ +
+
+

Support

+

+ freegeoip.net is + community funded, therefore consider donating if you + use and like it. For any other inquiries or feedback, + contact me + directly. +

+
+ +
+ +
+ +

API

+ +

The HTTP API takes GET requests in the following schema:

+ + 
freegeoip.net/{format}/{IP_or_hostname}
+ +

+ Supported formats are: + csv, + xml, + json + and + jsonp. + If no IP or hostname is provided, then your own IP is looked up. +

+ +
+ +
Examples
+
+ +
+
CSV
+ 
freegeoip.net/csv/8.8.8.8
+
+ +
+
XML
+ 
freegeoip.net/xml/4.2.2.2
+
+ +
+
JSON
+ 
freegeoip.net/json/github.com
+ Make it + JSONP + by adding the callback argument to the request. +
+ +
+ +
+ +
+ + This web site and all documentation is licensed under + Creative Commons 3.0. +
+ This service includes GeoLite2 data created by MaxMind, available + from maxmind.com. +
+ © 2009-2014 freegeoip.net + +
+
+ + + + diff --git a/cmd/freegeoip/upstart.conf b/cmd/freegeoip/upstart.conf new file mode 100644 index 0000000..c85f13e --- /dev/null +++ b/cmd/freegeoip/upstart.conf @@ -0,0 +1,12 @@ +# freegeoip.net web server + +description "freegeoip web server" + +start on runlevel [2345] +stop on runlevel [!2345] + +#pre-start exec /sbin/setcap 'cap_net_bind_service=+ep' /usr/local/bin/freegeoip +#limit nofile 20000 20000 +#setuid www-data +#setgid www-data +exec /usr/local/bin/freegeoip -silent diff --git a/wrk-test.lua b/cmd/freegeoip/wrk-test.lua similarity index 100% rename from wrk-test.lua rename to cmd/freegeoip/wrk-test.lua diff --git a/db.go b/db.go new file mode 100644 index 0000000..5cefb42 --- /dev/null +++ b/db.go @@ -0,0 +1,337 @@ +// Copyright 2009-2014 The freegeoip authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package freegeoip + +import ( + "compress/gzip" + "errors" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "os" + "path/filepath" + "strconv" + "sync" + "time" + + "github.com/howeyc/fsnotify" + "github.com/oschwald/maxminddb-golang" +) + +var ( + // ErrUnavailable may be returned by DB.Lookup when the database + // points to a URL and is not yet available because it's being + // downloaded in background. + ErrUnavailable = errors.New("No database available") + + // Local cached copy of a database downloaded from a URL. + defaultDB = filepath.Join(os.TempDir(), "freegeoip", "db.gz") +) + +// DB is the IP geolocation database. +type DB struct { + file string // Database file name. + reader *maxminddb.Reader // Actual db object. + notifyQuit chan struct{} // Stop auto-update and watch goroutines. + notifyOpen chan string // Notify when a db file is open. + notifyError chan error // Notify when an error occurs. + closed bool // Mark this db as closed. + lastUpdated time.Time // Last time the db was updated. + mu sync.RWMutex // Protects all the above. + + updateInterval time.Duration // Update interval. + maxRetryInterval time.Duration // Max retry interval in case of failure. +} + +// Open creates and initializes a DB from a local file. +// +// The database file is monitored by fsnotify and automatically +// reloads when the file is updated or overwritten. +func Open(dsn string) (db *DB, err error) { + db = &DB{ + file: dsn, + notifyQuit: make(chan struct{}), + notifyOpen: make(chan string, 1), + notifyError: make(chan error, 1), + } + err = db.openFile() + if err != nil { + db.Close() + return nil, err + } + err = db.watchFile() + if err != nil { + db.Close() + return nil, fmt.Errorf("fsnotify failed for %s: %s", dsn, err) + } + return db, nil +} + +// OpenURL creates and initializes a DB from a remote file. +// It automatically downloads and updates the file in background. +func OpenURL(url string, updateInterval, maxRetryInterval time.Duration) (db *DB, err error) { + db = &DB{ + file: defaultDB, + notifyQuit: make(chan struct{}), + notifyOpen: make(chan string, 1), + notifyError: make(chan error, 1), + updateInterval: updateInterval, + maxRetryInterval: maxRetryInterval, + } + db.openFile() // Optional, might fail. + go db.autoUpdate(url) + err = db.watchFile() + if err != nil { + db.Close() + return nil, fmt.Errorf("fsnotify failed for %s: %s", db.file, err) + } + return db, nil +} + +func (db *DB) watchFile() error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return err + } + dbdir := filepath.Dir(db.file) + _, err = os.Stat(dbdir) + if err != nil { + err = os.MkdirAll(dbdir, 0755) + if err != nil { + return err + } + } + go db.watchEvents(watcher) + return watcher.Watch(dbdir) +} + +func (db *DB) watchEvents(watcher *fsnotify.Watcher) { + for { + select { + case ev := <-watcher.Event: + if ev.Name == db.file && (ev.IsCreate() || ev.IsModify()) { + db.openFile() + } + case <-watcher.Error: + case <-db.notifyQuit: + watcher.Close() + return + } + time.Sleep(time.Second) // Suppress high-rate events. + } +} + +func (db *DB) openFile() error { + reader, err := db.newReader(db.file) + if err != nil { + return err + } + stat, err := os.Stat(db.file) + if err != nil { + return err + } + db.setReader(reader, stat.ModTime()) + return nil +} + +func (db *DB) newReader(dbfile string) (*maxminddb.Reader, error) { + f, err := os.Open(dbfile) + if err != nil { + return nil, err + } + defer f.Close() + gzf, err := gzip.NewReader(f) + if err != nil { + return nil, err + } + defer gzf.Close() + b, err := ioutil.ReadAll(gzf) + if err != nil { + return nil, err + } + return maxminddb.FromBytes(b) +} + +func (db *DB) setReader(reader *maxminddb.Reader, modtime time.Time) { + db.mu.Lock() + defer db.mu.Unlock() + if db.closed { + reader.Close() + return + } + if db.reader != nil { + db.reader.Close() + } + db.reader = reader + db.lastUpdated = modtime.UTC() + select { + case db.notifyOpen <- db.file: + default: + } +} + +func (db *DB) autoUpdate(url string) { + var sleep time.Duration + var retrying bool + for { + err := db.runUpdate(url) + if err != nil { + db.sendError(fmt.Errorf("Database update failed: %s", err)) + if !retrying { + retrying = true + sleep = 5 * time.Second + } else { + sleep *= 2 + if sleep > db.maxRetryInterval { + sleep = db.maxRetryInterval + } + } + } else { + retrying = false + sleep = db.updateInterval + } + select { + case <-db.notifyQuit: + return + case <-time.After(sleep): + // Sleep till time for the next update attempt. + } + } +} + +func (db *DB) runUpdate(url string) error { + yes, err := db.needUpdate(url) + if err != nil { + return err + } + if !yes { + return nil + } + tmpfile, err := db.download(url) + if err != nil { + return err + } + return db.renameFile(tmpfile) +} + +func (db *DB) needUpdate(url string) (bool, error) { + stat, err := os.Stat(db.file) + if err != nil { + return true, nil // Local db is missing, must be downloaded. + } + resp, err := http.Head(url) + if err != nil { + return false, err + } + defer resp.Body.Close() + size, err := strconv.Atoi(resp.Header.Get("Content-Length")) + if stat.Size() != int64(size) { + return true, nil + } + return false, nil +} + +func (db *DB) download(url string) (tmpfile string, err error) { + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + tmpfile = filepath.Join(os.TempDir(), + fmt.Sprintf("_freegeoip.%d.db.gz", time.Now().UnixNano())) + f, err := os.Create(tmpfile) + if err != nil { + return "", err + } + defer f.Close() + _, err = io.Copy(f, resp.Body) + if err != nil { + return "", err + } + return tmpfile, nil +} + +func (db *DB) renameFile(name string) error { + os.Rename(db.file, db.file+".bak") // Optional, might fail. + return os.Rename(name, db.file) +} + +// Date returns the UTC date the database file was last modified. +// If no database file has been opened the behaviour of Date is undefined. +func (db *DB) Date() time.Time { + db.mu.Lock() + defer db.mu.Unlock() + return db.lastUpdated +} + +// NotifyClose returns a channel that is closed when the database is closed. +func (db *DB) NotifyClose() <-chan struct{} { + return db.notifyQuit +} + +// NotifyOpen returns a channel that notifies when a new database is +// loaded or reloaded. This can be used to monitor background updates +// when the DB points to a URL. +func (db *DB) NotifyOpen() (filename <-chan string) { + return db.notifyOpen +} + +// NotifyError returns a channel that notifies when an error occurs +// while downloading or reloading a DB that points to a URL. +func (db *DB) NotifyError() (errChan <-chan error) { + return db.notifyError +} + +func (db *DB) sendError(err error) { + db.mu.Lock() + defer db.mu.Unlock() + if db.closed { + return + } + select { + case db.notifyError <- err: + default: + } +} + +// Lookup takes an IP address and a pointer to the result value to decode +// into. The result value pointed to must be a data value that corresponds +// to a record in the database. This may include a struct representation +// of the data, a map capable of holding the data or an empty interface{} +// value. +// +// If result is a pointer to a struct, the struct need not include a field +// for every value that may be in the database. If a field is not present +// in the structure, the decoder will not decode that field, reducing the +// time required to decode the record. +// +// See https://godoc.org/github.com/oschwald/maxminddb-golang#Reader.Lookup +// for details. +func (db *DB) Lookup(addr net.IP, result interface{}) error { + db.mu.RLock() + defer db.mu.RUnlock() + if db.reader != nil { + return db.reader.Lookup(addr, result) + } + return ErrUnavailable +} + +// Close the database. +func (db *DB) Close() { + db.mu.Lock() + defer db.mu.Unlock() + if !db.closed { + db.closed = true + close(db.notifyQuit) + close(db.notifyOpen) + close(db.notifyError) + } + if db.reader != nil { + db.reader.Close() + db.reader = nil + } +} diff --git a/db/updatedb b/db/updatedb deleted file mode 100755 index 4adaeeb..0000000 --- a/db/updatedb +++ /dev/null @@ -1,268 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 -# -# Copyright 2013 Alexandre Fiori -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import csv -import urllib2 -import os -import sqlite3 -import sys -import unicodedata -import zipfile - -dbname = "ipdb.sqlite" - - -def download(url, filename=None): - print "Downloading " + url - - req = urllib2.Request(url, None, {"User-Agent": "Mozilla/5.0"}) - req = urllib2.urlopen(req) - data = req.read() - req.close() - - if filename: - fd = open(filename, "wb") - fd.write(data) - fd.close() - - return data - - -def extract(zipfd, zipname, filename=None): - if filename is None: - filename = zipname - - print "Extracting " + filename - - try: - data = zipfd.read(zipname) - except KeyError: - print "Could not extract %s from zip archive." % zipname - sys.exit(1) - else: - fd = open(filename, "w") - fd.write(data) - fd.close() - - -def import_csv(cursor, csvfile, table, skip_lines=0): - sys.stdout.write("Importing %s: " % csvfile) - sys.stdout.flush() - - fd = open(csvfile) - for n in range(skip_lines): - fd.next() - - q = None - rows = [] - for n, row in enumerate(csv.reader(fd), 1): - if q is None: - question_marks = ",".join(["?"] * len(row)) - q = "insert into %s values (%s)" % (table, question_marks) - - rows.append(map(lambda s: s.decode("latin-1"), row)) - if not n % 100000: - sys.stderr.write(".") - cursor.executemany(q, rows) - rows = [] - - if rows: - cursor.executemany(q, rows) - - fd.close() - print "%d records!" % n - os.unlink(csvfile) - - -class world_regions(dict): - """Imports a csv and only store rows that contains accented characters, - indexing them by their non-accented version:: - - country,region (no accents) -> region with accents - - Expected csv columns: country,region,city - """ - def __init__(self, filename=None, autorm=True): - self.filename = filename - if filename: - fd = open(filename) - for row in csv.reader(fd): - v = map(lambda s: s.decode("utf-8"), row[:2]) - k = self.strip_accents(",".join(v)) - - if k != v: - self[k] = v[1] - fd.close() - if autorm is True: - os.unlink(filename) - - def strip_accents(self, s): - return ''.join((c for c in unicodedata.normalize('NFD', s) - if unicodedata.category(c) != 'Mn')) - - -class world_countries(dict): - def __init__(self, conn): - curs = conn.cursor() - curs.execute("SELECT country_code, country_name from country_blocks") - for (code, name) in curs: - self[code] = name - curs.close() - - -if __name__ == "__main__": - region_url = "http://dev.maxmind.com/static/csv/codes/maxmind/region.csv" - region_csv = os.path.basename(region_url) - if not os.path.exists(region_csv): - download(region_url, region_csv) - - wr_url = "http://blog.freegeoip.net/files/all_cities_in_the_world.csv.zip" - wr_zip = os.path.basename(wr_url) - wr_csv = wr_zip[:-4] - if not os.path.exists(wr_csv): - if not os.path.exists(wr_zip): - download(wr_url, wr_zip) - zipfd = zipfile.ZipFile(wr_zip) - extract(zipfd, wr_csv, wr_csv) - zipfd.close() - os.unlink(wr_zip) - - geolite = "http://geolite.maxmind.com/download/geoip/database/" - - city_url = geolite + "GeoLiteCity_CSV/GeoLiteCity-latest.zip" - city_files = ["GeoLiteCity-Blocks.csv", "GeoLiteCity-Location.csv"] - if not all(map(os.path.exists, city_files)): - print "Checking " + city_url - # Fetch the most recent city database - city_zip = os.path.basename(city_url) - if not os.path.exists(city_zip): - download(city_url, city_zip) - - # Extract city csv files - zipfd = zipfile.ZipFile(city_zip) - for filename in zipfd.namelist(): - n = os.path.basename(filename) - if n in city_files and not os.path.exists(n): - extract(zipfd, filename, n) - zipfd.close() - os.unlink(city_zip) - - # Fetch the country database - country_url = geolite + "GeoIPCountryCSV.zip" - country_zip = os.path.basename(country_url) - if not os.path.exists(country_zip): - download(country_url, country_zip) - - country_csv = "GeoIPCountryWhois.csv" - if not os.path.exists(country_csv): - zipfd = zipfile.ZipFile(country_zip) - extract(zipfd, country_csv) - zipfd.close() - os.unlink(country_zip) - - # Create the IP database - tmpdb = "_" + dbname + ".temp" - if os.path.exists(tmpdb): - os.unlink(tmpdb) - - conn = sqlite3.connect(tmpdb) - curs = conn.cursor() - - curs.execute("""\ - create table country_blocks ( - ip_start_str text, - ip_end_str text, - ip_start text, - ip_end text, - country_code text, - country_name text, - primary key(ip_start))""") - import_csv(curs, country_csv, "country_blocks") - curs.execute("CREATE INDEX cc_idx ON country_blocks(country_code);") - - curs.execute("""\ - create table region_names ( - country_code text, - region_code text, - region_name text, - unique (country_code, region_code))""") - import_csv(curs, region_csv, "region_names") - - curs.execute("""\ - create table city_blocks ( - ip_start int, - ip_end int, - loc_id int, - primary key(ip_start))""") - import_csv(curs, city_files[0], "city_blocks", skip_lines=2) - - curs.execute("""\ - create table city_location ( - loc_id int, - country_code text, - region_code text, - city_name text, - postal_code text, - latitude real, - longitude real, - metro_code text, - area_code text, - primary key(loc_id))""") - import_csv(curs, city_files[1], "city_location", skip_lines=2) - - curs.close() - conn.commit() - - # Fix region names - sys.stdout.write("Updating region names: ") - sys.stdout.flush() - - world_regions = world_regions("all_cities_in_the_world.csv") - world_countries = world_countries(conn) - - count = 0 - regions = conn.cursor() - regions.execute("SELECT rowid, * FROM region_names") - - for region in regions: - region_name = region[-1] # rowid,country_code,region_code,region_name - country_name = world_countries.get(region[1]) - - if country_name: - k = country_name + "," + region_name - if k in world_regions: - new_name = world_regions[k] - if region_name != new_name: - update = conn.cursor() - update.execute("UPDATE region_names SET region_name=? " - "WHERE rowid=?", (new_name, region[0])) - update.close() - count += 1 - - print "%d names updated." % count - - # Fix db consistency. (patch from Alfredo Terrone) - regions.execute("INSERT INTO region_names " - "SELECT DISTINCT country_code,'','' FROM region_names") - regions.close() - conn.commit() - conn.close() - - # Replace any existing db with the new one - if os.path.exists(dbname): - os.unlink(dbname) - os.rename(tmpdb, dbname) diff --git a/db_test.go b/db_test.go new file mode 100644 index 0000000..0cc7a30 --- /dev/null +++ b/db_test.go @@ -0,0 +1,256 @@ +// Copyright 2009-2014 The freegeoip authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package freegeoip + +import ( + "errors" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" +) + +var testFile = "testdata/db.gz" + +func TestDownload(t *testing.T) { + if _, err := os.Stat(testFile); err == nil { + t.Skip("Test database already exists:", testFile) + } + db := &DB{} + dbfile, err := db.download(maxmindFile) + if err != nil { + t.Fatal(err) + } + err = os.Rename(dbfile, testFile) + if err != nil { + t.Fatal(err) + } +} + +func TestNeedUpdateFileMissing(t *testing.T) { + db := &DB{file: "does-not-exist"} + yes, err := db.needUpdate("whatever") + if err != nil { + t.Fatal(err) + } + if !yes { + t.Fatal("Unexpected: db is supposed to need an update") + } +} + +func TestNeedUpdateSameFile(t *testing.T) { + mux := http.NewServeMux() + mux.Handle("/testdata/", http.FileServer(http.Dir("."))) + srv := httptest.NewServer(mux) + defer srv.Close() + db := &DB{file: testFile} + yes, err := db.needUpdate(srv.URL + "/" + testFile) + if err != nil { + t.Fatal(err) + } + if yes { + t.Fatal("Unexpected: db is not supposed to need an update") + } +} + +func TestNeedUpdate(t *testing.T) { + mux := http.NewServeMux() + mux.Handle("/testdata/", http.FileServer(http.Dir("."))) + srv := httptest.NewServer(mux) + defer srv.Close() + file := testFile + ".tmp" + f, err := os.Create(file) + if err != nil { + t.Fatal(err) + } + defer f.Close() + defer os.Remove(file) + db := &DB{file: file} + yes, err := db.needUpdate(srv.URL + "/" + testFile) + if err != nil { + t.Fatal(err) + } + if !yes { + t.Fatal("Unexpected: db is supposed to need an update") + } +} + +func TestOpenFile(t *testing.T) { + db, err := Open(testFile) + if err != nil { + t.Fatal(err) + } + defer db.Close() + select { + case <-db.NotifyOpen(): + case <-db.NotifyClose(): + case <-time.After(time.Second): + t.Fatal("Timed out") + } + db.Date() // Test this? +} + +func TestOpenBadFile(t *testing.T) { + db, err := Open("db_test.go") + if err == nil { + db.Close() + t.Fatal("Unexpected bogus db is open") + } +} + +func TestSendError(t *testing.T) { + db := &DB{notifyError: make(chan error, 1)} + err1 := errors.New("test") + db.sendError(err1) + select { + case err2 := <-db.NotifyError(): + if err2 != err2 { + t.Fatalf("Unexpected error: %#v", err2) + } + default: + t.Fatal("An error is expected but it's not available") + } +} + +func TestSkipSendError(t *testing.T) { + db := &DB{notifyError: make(chan error, 1)} + db.sendError(nil) + db.sendError(nil) + close(db.notifyError) +} + +func TestWatchFile(t *testing.T) { + db, err := Open(testFile) + if err != nil { + t.Fatal(err) + } + defer db.Close() + err = os.Rename(testFile, testFile+".bkp") + if err != nil { + t.Fatal(err) + } + err = os.Rename(testFile+".bkp", testFile) + if err != nil { + t.Fatal(err) + } + select { + case file := <-db.NotifyOpen(): + if file != testFile { + t.Fatal("Unexpected file:", file) + } + case <-time.After(time.Second): + t.Fatal("Timed out") + } +} + +func TestWatchMkdir(t *testing.T) { + mux := http.NewServeMux() + mux.Handle("/testdata/", http.FileServer(http.Dir("."))) + srv := httptest.NewServer(mux) + defer srv.Close() + tmp := defaultDB + defaultDB = filepath.Join(os.TempDir(), "foobar", "db.gz") + defer func() { + defaultDB = tmp + time.Sleep(time.Second) + os.RemoveAll(filepath.Dir(defaultDB)) + }() + db, err := OpenURL(srv.URL+"/"+testFile, time.Hour, time.Minute) + if err != nil { + t.Fatalf("Failed to create %s: %s", filepath.Dir(defaultDB), err) + } + db.Close() +} + +func TestWatchMkdirFail(t *testing.T) { + basedir := filepath.Join(os.TempDir(), "freegeoip-test") + err := os.MkdirAll(basedir, 0444) + if err != nil { + t.Fatal(err) + } + tmp := defaultDB + defaultDB = filepath.Join(basedir, "a", "db.gz") + defer func() { + defaultDB = tmp + time.Sleep(time.Second) + os.Chmod(basedir, 0755) + os.RemoveAll(basedir) + }() + mux := http.NewServeMux() + mux.Handle("/testdata/", http.FileServer(http.Dir("."))) + srv := httptest.NewServer(mux) + defer srv.Close() + db, err := OpenURL(srv.URL+"/"+testFile, time.Hour, time.Minute) + if err == nil { + db.Close() + t.Fatalf("Unexpected creation of dir %s worked", basedir) + } +} + +func TestLookupOnFile(t *testing.T) { + db, err := Open(testFile) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var record testRecord + err = db.Lookup(net.ParseIP("8.8.8.8"), &record) + if err != nil { + t.Fatal(err) + } + if record.Country.ISOCode != "US" { + t.Fatal("Unexpected ISO code:", record.Country.ISOCode) + } +} + +func TestLookupOnURL(t *testing.T) { + mux := http.NewServeMux() + mux.Handle("/testdata/", http.FileServer(http.Dir("."))) + srv := httptest.NewServer(mux) + defer srv.Close() + os.Remove(defaultDB) // In case it exists. + db, err := OpenURL(srv.URL+"/"+testFile, time.Hour, time.Minute) + if err != nil { + t.Fatal(err) + } + defer db.Close() + select { + case file := <-db.NotifyOpen(): + if file != defaultDB { + t.Fatal("Unexpected db file:", file) + } + case err := <-db.NotifyError(): + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("Timed out") + } + var record testRecord + err = db.Lookup(net.ParseIP("8.8.8.8"), &record) + if err != nil { + t.Fatal(err) + } + if record.Country.ISOCode != "US" { + t.Fatal("Unexpected ISO code:", record.Country.ISOCode) + } +} + +func TestLookuUnavailable(t *testing.T) { + db := &DB{} + err := db.Lookup(net.ParseIP("8.8.8.8"), nil) + if err == nil { + t.Fatal("Unexpected lookup worked") + } +} + +type testRecord struct { + Country struct { + ISOCode string `maxminddb:"iso_code"` + } `maxminddb:"country"` +} diff --git a/doc.go b/doc.go new file mode 100644 index 0000000..61663e7 --- /dev/null +++ b/doc.go @@ -0,0 +1,19 @@ +// Copyright 2009-2014 The freegeoip authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Package freegeoip provides an API for searching the geolocation of IP +// addresses. It uses a database that can be either a local file or a +// remote resource from a URL. +// +// Local databases are monitored by fsnotify and reloaded when the file is +// either updated or overwritten. +// +// Remote databases are automatically downloaded and updated in background +// so you can focus on using the API and not managing the database. +// +// Also, the freegeoip package provides http handlers that any Go http +// server (net/http) can use. These handlers can process IP geolocation +// lookup requests and return data in multiple formats like CSV, XML, +// JSON and JSONP. It has also an API for supporting custom formats. +package freegeoip diff --git a/encoder.go b/encoder.go new file mode 100644 index 0000000..bbce3a5 --- /dev/null +++ b/encoder.go @@ -0,0 +1,267 @@ +// Copyright 2009-2014 The freegeoip authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package freegeoip + +import ( + "encoding/csv" + "encoding/json" + "encoding/xml" + "io" + "math" + "net" + "net/http" + "strconv" + "strings" + + // otto is used for testing the JSONP encoder. It's imported here + // to make `go get` download it before `go test` fails. + _ "github.com/robertkrimen/otto" +) + +// A Query object is used to query the IP database. +// +// Currently the only database supported is MaxMind, and the query is a +// data structure with tags that are used by the maxminddb.Lookup function. +type Query interface{} + +// An Encoder that can provide a query specification to be used for +// querying the IP database, and later encode the results of that +// query in a specific format. +type Encoder interface { + // NewQuery returns a query specification that is used to query + // the IP database. It should be a data structure with tags + // associated to its fields describing what fields to query in + // the IP database, such as country and city. + // + // See the maxminddb package documentation for details on + // fields available for the MaxMind database. + NewQuery() Query + + // Encode writes data to the response of an http request + // using the results of a query to the IP database. + // + // It encodes the query object into a specific format such + // as XML or JSON and writes to the response. + // + // The IP passed to the encoder may be the result of a DNS + // lookup, and if there are multiple IPs associated to the + // hostname this will be a random one from the list. + Encode(w http.ResponseWriter, r *http.Request, q Query, ip net.IP) error +} + +// JSONEncoder encodes the results of an IP lookup as JSON. +type JSONEncoder struct { + Indent bool +} + +// NewQuery implements the Encoder interface. +func (f *JSONEncoder) NewQuery() Query { + return &maxmindQuery{} +} + +// Encode implements the Encoder interface. +func (f *JSONEncoder) Encode(w http.ResponseWriter, r *http.Request, q Query, ip net.IP) error { + record := newResponse(q.(*maxmindQuery), ip, requestLang(r)) + callback := r.FormValue("callback") + if len(callback) > 0 { + return f.P(w, r, record, callback) + } + w.Header().Set("Content-Type", "application/json") + if f.Indent { + } + return json.NewEncoder(w).Encode(record) +} + +// P writes JSONP to an http response. +func (f *JSONEncoder) P(w http.ResponseWriter, r *http.Request, record *responseRecord, callback string) error { + w.Header().Set("Content-Type", "application/javascript") + _, err := io.WriteString(w, callback+"(") + if err != nil { + return err + } + err = json.NewEncoder(w).Encode(record) + if err != nil { + return err + } + _, err = io.WriteString(w, ");") + return err +} + +// XMLEncoder encodes the results of an IP lookup as XML. +type XMLEncoder struct { + Indent bool +} + +// NewQuery implements the Encoder interface. +func (f *XMLEncoder) NewQuery() Query { + return &maxmindQuery{} +} + +// Encode implements the Encoder interface. +func (f *XMLEncoder) Encode(w http.ResponseWriter, r *http.Request, q Query, ip net.IP) error { + record := newResponse(q.(*maxmindQuery), ip, requestLang(r)) + w.Header().Set("Content-Type", "application/xml") + _, err := io.WriteString(w, xml.Header) + if err != nil { + return err + } + if f.Indent { + enc := xml.NewEncoder(w) + enc.Indent("", "\t") + err := enc.Encode(record) + if err != nil { + return err + } + _, err = w.Write([]byte("\n")) + return err + } + return xml.NewEncoder(w).Encode(record) +} + +// CSVEncoder encodes the results of an IP lookup as CSV. +type CSVEncoder struct { + UseCRLF bool +} + +// NewQuery implements the Encoder interface. +func (f *CSVEncoder) NewQuery() Query { + return &maxmindQuery{} +} + +// Encode implements the Encoder interface. +func (f *CSVEncoder) Encode(w http.ResponseWriter, r *http.Request, q Query, ip net.IP) error { + record := newResponse(q.(*maxmindQuery), ip, requestLang(r)) + w.Header().Set("Content-Type", "text/csv") + cw := csv.NewWriter(w) + cw.UseCRLF = f.UseCRLF + err := cw.Write([]string{ + record.CountryCode, + record.CountryName, + record.RegionCode, + record.RegionName, + record.City, + record.ZipCode, + record.TimeZone, + strconv.FormatFloat(record.Latitude, 'f', 2, 64), + strconv.FormatFloat(record.Longitude, 'f', 2, 64), + strconv.Itoa(int(record.MetroCode)), + }) + if err != nil { + return err + } + cw.Flush() + return nil +} + +// maxmindQuery is the object used to query the maxmind database. +// +// See the maxminddb package documentation for details. +type maxmindQuery struct { + Country struct { + ISOCode string `maxminddb:"iso_code"` + Names map[string]string `maxminddb:"names"` + } `maxminddb:"country"` + Region []struct { + ISOCode string `maxminddb:"iso_code"` + Names map[string]string `maxminddb:"names"` + } `maxminddb:"subdivisions"` + City struct { + Names map[string]string `maxminddb:"names"` + } `maxminddb:"city"` + Location struct { + Latitude float64 `maxminddb:"latitude"` + Longitude float64 `maxminddb:"longitude"` + MetroCode uint `maxminddb:"metro_code"` + TimeZone string `maxminddb:"time_zone"` + } `maxminddb:"location"` + Postal struct { + Code string `maxminddb:"code"` + } `maxminddb:"postal"` +} + +// responseRecord is the object that gets encoded as the response of an +// IP lookup request. It is encoded to formats such as xml and json. +type responseRecord struct { + XMLName xml.Name `xml:"Response" json:"-"` + IP string `json:"ip"` + CountryCode string `json:"country_code"` + CountryName string `json:"country_name"` + RegionCode string `json:"region_code"` + RegionName string `json:"region_name"` + City string `json:"city"` + ZipCode string `json:"zip_code"` + TimeZone string `json:"time_zone"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + MetroCode uint `json:"metro_code"` +} + +// newResponse translates a maxmindQuery into a responseRecord, setting +// the country, region and city names to their localized name according +// to the given lang. +// +// See the maxminddb documentation for supported languages. +func newResponse(query *maxmindQuery, ip net.IP, lang []string) *responseRecord { + + record := &responseRecord{ + IP: ip.String(), + CountryCode: query.Country.ISOCode, + CountryName: localizedName(query.Country.Names, lang), + City: localizedName(query.City.Names, lang), + ZipCode: query.Postal.Code, + TimeZone: query.Location.TimeZone, + Latitude: roundFloat(query.Location.Latitude, .5, 3), + Longitude: roundFloat(query.Location.Longitude, .5, 3), + MetroCode: query.Location.MetroCode, + } + if len(query.Region) > 0 { + record.RegionCode = query.Region[0].ISOCode + record.RegionName = localizedName(query.Region[0].Names, lang) + } + return record +} + +func requestLang(r *http.Request) (list []string) { + // TODO: Check Accept-Charset, sort languages by qvalue. + l := r.Header.Get("Accept-Language") + if len(l) == 0 { + return nil + } + accpt := strings.Split(l, ",") + if len(accpt) == 0 { + return nil + } + for n, name := range accpt { + accpt[n] = strings.Trim(strings.SplitN(name, ";", 2)[0], " ") + } + return accpt +} + +func localizedName(field map[string]string, accept []string) (name string) { + if accept != nil { + var f string + var ok bool + for _, l := range accept { + f, ok = field[l] + if ok { + return f + } + } + } + return field["en"] +} + +func roundFloat(val float64, roundOn float64, places int) (newVal float64) { + var round float64 + pow := math.Pow(10, float64(places)) + digit := pow * val + _, div := math.Modf(digit) + if div >= roundOn { + round = math.Ceil(digit) + } else { + round = math.Floor(digit) + } + return round / pow +} diff --git a/encoder_test.go b/encoder_test.go new file mode 100644 index 0000000..8b6261f --- /dev/null +++ b/encoder_test.go @@ -0,0 +1,236 @@ +// Copyright 2009-2014 The freegeoip authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package freegeoip + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "encoding/xml" + "io/ioutil" + "net/http" + "testing" + + "github.com/robertkrimen/otto" +) + +func TestCSVEncoder(t *testing.T) { + _, srv, err := runServer("/csv/", &CSVEncoder{}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/csv/8.8.8.8") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + row, err := csv.NewReader(resp.Body).Read() + if err != nil { + t.Fatal(err) + } + if row[0] != "US" { + t.Fatalf("Unexpected country code in record: %#v", row) + } +} + +func TestXMLEncoder(t *testing.T) { + _, srv, err := runServer("/xml/", &XMLEncoder{}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/xml/8.8.8.8") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + var record responseRecord + err = xml.NewDecoder(resp.Body).Decode(&record) + if err != nil { + t.Fatal(err) + } + if record.CountryCode != "US" { + t.Fatalf("Unexpected country code in record: %#v", record.CountryCode) + } +} + +func TestXMLEncoderIndent(t *testing.T) { + // TODO: validate indentation? + _, srv, err := runServer("/xml/", &XMLEncoder{Indent: true}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/xml/8.8.8.8") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + var record responseRecord + err = xml.NewDecoder(resp.Body).Decode(&record) + if err != nil { + t.Fatal(err) + } + if record.CountryCode != "US" { + t.Fatalf("Unexpected country code in record: %#v", record.CountryCode) + } +} + +func TestJSONEncoder(t *testing.T) { + _, srv, err := runServer("/json/", &JSONEncoder{}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/json/8.8.8.8") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + var record responseRecord + err = json.NewDecoder(resp.Body).Decode(&record) + if err != nil { + t.Fatal(err) + } + if record.CountryCode != "US" { + t.Fatalf("Unexpected country code in record: %#v", record.CountryCode) + } +} + +func TestJSONEncoderIndent(t *testing.T) { + // TODO: validate indentation? + _, srv, err := runServer("/json/", &JSONEncoder{Indent: true}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/json/8.8.8.8") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + var record responseRecord + err = json.NewDecoder(resp.Body).Decode(&record) + if err != nil { + t.Fatal(err) + } + if record.CountryCode != "US" { + t.Fatalf("Unexpected country code in record: %#v", record.CountryCode) + } +} + +func TestJSONPEncoder(t *testing.T) { + _, srv, err := runServer("/json/", &JSONEncoder{}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/json/8.8.8.8?callback=f") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + code := bytes.NewBuffer([]byte(` + function f(record) { + set(record.country_code); + }; + `)) + code.Write(b) + vm := otto.New() + var countryCode string + vm.Set("set", func(call otto.FunctionCall) otto.Value { + if len(call.ArgumentList) > 0 { + countryCode = call.Argument(0).String() + } + return otto.UndefinedValue() + }) + _, err = vm.Run(code.Bytes()) + if err != nil { + t.Fatal(err) + } + if countryCode != "US" { + t.Fatalf("Unexpected country code in record: %#v", countryCode) + } +} + +func TestRequestLang(t *testing.T) { + r := http.Request{} + list := requestLang(&r) + if list != nil { + t.Fatal("Unexpected list is not nil") + } + r.Header = map[string][]string{ + "Accept-Language": []string{"en-us,en;q=0.5"}, + } + want := []string{"en-us", "en"} + list = requestLang(&r) + if len(list) != 2 { + t.Fatal("Unexpected list length:", len(list)) + } + for i, lang := range want { + if list[i] != lang { + t.Fatal("Unexpected item in list:", list[i]) + } + } +} + +func TestLocalizedName(t *testing.T) { + names := map[string]string{ + "de": "USA", + "en": "United States", + "es": "Estados Unidos", + "fr": "États-Unis", + "ja": "アメリカ合衆国", + "pt-BR": "Estados Unidos", + "ru": "Сша", + "zh-CN": "美国", + } + mkReq := func(lang string) *http.Request { + return &http.Request{ + Header: map[string][]string{ + "Accept-Language": []string{lang}, + }, + } + } + test := map[string]string{ + "pt-BR,en": "Estados Unidos", + "pt-br": "United States", + "es-ES,ru;q=0.8,q=0.2": "Сша", + "da, en-gb;q=0.8, en;q=0.7": "United States", + "da, fr;q=0.8, en;q=0.7": "États-Unis", + "da, de;q=0.5, zh-CN;q=0.8": "USA", // TODO: Use qvalue. + "da, es": "Estados Unidos", + "es-ES, ja": "アメリカ合衆国", + } + for k, v := range test { + name := localizedName(names, requestLang(mkReq(k))) + if name != v { + t.Fatalf("Unexpected name: want %q, have %q", v, name) + } + } +} diff --git a/example_test.go b/example_test.go new file mode 100644 index 0000000..38f3a66 --- /dev/null +++ b/example_test.go @@ -0,0 +1,118 @@ +// Copyright 2009-2014 The freegeoip authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package freegeoip + +import ( + "encoding/json" + "log" + "net" + "net/http" + "time" +) + +var maxmindFile = "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz" + +func ExampleDatabaseQuery() { + db, err := Open("./testdata.gz") + if err != nil { + log.Fatal(err) + } + var result customQuery + err = db.Lookup(net.ParseIP("8.8.8.8"), &result) + if err != nil { + log.Fatal(err) + } + defer db.Close() + log.Printf("%#v", result) +} + +func ExampleRemoteDatabaseQuery() { + updateInterval := 24 * time.Hour + maxRetryInterval := time.Hour + db, err := OpenURL(maxmindFile, updateInterval, maxRetryInterval) + if err != nil { + log.Fatal(err) + } + defer db.Close() + select { + case <-db.NotifyOpen(): + // Wait for the db to be downloaded. + case err := <-db.NotifyError(): + log.Fatal(err) + } + var result customQuery + err = db.Lookup(net.ParseIP("8.8.8.8"), &result) + if err != nil { + log.Fatal(err) + } + log.Printf("%#v", result) +} + +func ExampleServer() { + db, err := OpenURL(maxmindFile, 24*time.Hour, time.Hour) + if err != nil { + log.Fatal(err) + } + http.Handle("/csv/", NewHandler(db, &CSVEncoder{})) + http.Handle("/xml/", NewHandler(db, &XMLEncoder{})) + http.Handle("/json/", NewHandler(db, &JSONEncoder{})) + http.ListenAndServe(":8080", nil) +} + +func ExampleServerWithCustomEncoder() { + db, err := Open("./testdata/db.gz") + if err != nil { + log.Fatal(err) + } + http.Handle("/custom/json/", NewHandler(db, &customEncoder{})) + http.ListenAndServe(":8080", nil) +} + +// A customEncoder writes a custom JSON object to an http response. +type customEncoder struct{} + +// A customQuery is the query executed in the maxmind database for +// every IP lookup request. +type customQuery struct { + Country struct { + ISOCode string `maxminddb:"iso_code"` + Names map[string]string `maxminddb:"names"` + } `maxminddb:"country"` + Location struct { + Latitude float64 `maxminddb:"latitude"` + Longitude float64 `maxminddb:"longitude"` + TimeZone string `maxminddb:"time_zone"` + } `maxminddb:"location"` +} + +// A customResponse is what gets written to the http response as JSON. +type customResponse struct { + IP string + CountryCode string + CountryName string + Latitude float64 + Longitude float64 + TimeZone string +} + +// NewQuery implements the freegeoip.Encoder interface. +func (f *customEncoder) NewQuery() Query { + return &customQuery{} +} + +// Encode implements the freegeoip.Encoder interface. +func (f *customEncoder) Encode(w http.ResponseWriter, r *http.Request, q Query, ip net.IP) error { + record := q.(*customQuery) + out := &customResponse{ + IP: ip.String(), + CountryCode: record.Country.ISOCode, + CountryName: record.Country.Names["en"], // Set to client lang. + Latitude: record.Location.Latitude, + Longitude: record.Location.Longitude, + TimeZone: record.Location.TimeZone, + } + w.Header().Set("Content-Type", "application/json") + return json.NewEncoder(w).Encode(&out) +} diff --git a/freegeoip.conf b/freegeoip.conf deleted file mode 100644 index 2d40b60..0000000 --- a/freegeoip.conf +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - - - - - ./static - - - - - - - - - - - - - - - - - - - - diff --git a/freegeoip.go b/freegeoip.go index b6cc244..962fb1e 100644 --- a/freegeoip.go +++ b/freegeoip.go @@ -1,876 +1,109 @@ -// Copyright 2009-2014 Alexandre Fiori +// Copyright 2009-2014 The freegeoip authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// Web server of freegeoip.net - -package main +package freegeoip import ( - "database/sql" - "encoding/binary" - "encoding/json" - "encoding/xml" - "expvar" - "flag" "fmt" - "io" - "log" "math/rand" "net" "net/http" - "os" - "os/signal" - "path/filepath" - "runtime" - "runtime/pprof" - "strconv" "strings" - "sync" - "syscall" "time" - - "github.com/fiorix/go-redis/redis" - "github.com/fiorix/go-web/httpxtra" - "github.com/gorilla/context" - - // SQLite driver. - // _ "github.com/mattn/go-sqlite3" - _ "code.google.com/p/gosqlite/sqlite3" ) -var ( - collectStats bool - outputCount = expvar.NewMap("Output") // json, xml or csv - statusCount = expvar.NewMap("Status") // 200, 403, 404, etc - protocolCount = expvar.NewMap("Protocol") // HTTP or HTTPS - - dns *dnsPool -) - -func main() { - flLog := flag.String("l", "", "log to file instead of stderr") - flConfig := flag.String("c", "freegeoip.conf", "set config file") - flProfile := flag.Bool("profile", false, "run cpu and mem profiler") - flag.Parse() - - if *flProfile { - runProfile() - } - - cf, err := loadConfig(*flConfig) - if err != nil { - log.Fatal(err) - } - - collectStats = cf.Debug - - if cf.DNS.Enabled { - t, err := time.ParseDuration(cf.DNS.Timeout) - if err != nil { - log.Fatal("Invalid DNS timeout:", err) - } - dns = new(dnsPool) - dns.init(cf.DNS.MaxConcurrent, t) - } - - if len(*flLog) > 0 { - setLog(*flLog) - } - - runtime.GOMAXPROCS(runtime.NumCPU()) - log.Printf("FreeGeoIP server starting. debug=%t", cf.Debug) - - if cf.Debug && len(cf.DebugSrv) > 0 { - go func() { - // server for expvar's /debug/vars only - log.Printf("Starting DEBUG server on tcp/%s", cf.DebugSrv) - log.Fatal(http.ListenAndServe(cf.DebugSrv, nil)) - }() - } - - mux := http.NewServeMux() - mux.Handle("/", http.FileServer(http.Dir(cf.DocumentRoot))) - - st := time.Now() - db, err := openDB(cf) - if err != nil { - log.Fatal(err) - } - log.Println("IPDB cached in", time.Since(st)) - - lh := lookupHandler(cf, db) - mux.HandleFunc("/csv/", lh) - mux.HandleFunc("/xml/", lh) - mux.HandleFunc("/json/", lh) - - for _, c := range cf.Listen { - go runServer(mux, c) - } - - select {} -} - -func lookupHandler(cf *configFile, db *DB) http.HandlerFunc { - var rl rateLimiter - if cf.Limit.MaxRequests > 0 { - if len(cf.Redis) > 0 { - rl = new(redisQuota) - log.Printf("Using redis to manage quota: %s", cf.Redis) - } else { - rl = new(mapQuota) - log.Printf("Using internal map to manage quota.") - } - rl.init(cf) - } - return func(w http.ResponseWriter, r *http.Request) { - if r.TLS != nil { - w.Header().Set("Strict-Transport-Security", "max-age=31536000") - } - switch r.Method { - case "GET": - w.Header().Set("Access-Control-Allow-Origin", "*") - handleRequest(cf, db, rl, w, r) - case "OPTIONS": - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Content-Type", "text/plain") - w.Header().Set("Access-Control-Allow-Methods", "GET") - w.Header().Set("Access-Control-Allow-Headers", "X-Requested-With") - w.WriteHeader(200) - default: - w.Header().Set("Allow", "GET, OPTIONS") - http.Error(w, http.StatusText(405), 405) - } - } -} - -func handleRequest( - cf *configFile, - db *DB, - rl rateLimiter, - w http.ResponseWriter, - r *http.Request, -) { - // If xheaders is enabled, RemoteAddr might be a copy of - // the X-Real-IP or X-Forwarded-For HTTP headers, which - // can be a comma separated list of IPs. In this case, - // only the first IP in the list is used. - if strings.Index(r.RemoteAddr, ",") > 0 { - r.RemoteAddr = strings.SplitN(r.RemoteAddr, ",", 2)[0] - } - - // Parse remote address. - var ip net.IP - if sIP, _, err := net.SplitHostPort(r.RemoteAddr); err != nil { - ip = net.ParseIP(r.RemoteAddr) // Use X-Real-IP, etc - } else { - ip = net.ParseIP(sIP) - } - +func init() { + rand.Seed(time.Now().UnixNano()) +} + +// A Handler provides http handlers that can process requests and return +// data in multiple formats. +// +// Usage: +// +// handle := NewHandler(db) +// http.Handle("/json/", handle.JSON()) +// +// Note that the url pattern must end with a trailing slash since the +// handler looks for IP addresses or hostnames as parameters, for +// example /json/8.8.8.8 or /json/domain.com. +// +// If no IP or hostname is provided, then the handler will query the +// IP address of the caller. See the ProxyHandler for more. +type Handler struct { + db *DB + enc Encoder +} + +// NewHandler creates and initializes a new Handler. +func NewHandler(db *DB, enc Encoder) *Handler { + return &Handler{db, enc} +} + +// ServeHTTP implements the http.Handler interface. +func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ip := f.queryIP(r) if ip == nil { - // This could be a misconfigured unix socket server. - context.Set(r, "log", "Invalid source IP: "+r.RemoteAddr) - http.Error(w, http.StatusText(500), 500) - return - } - - // Convert remote IP to integer to check quota. - // IPv6 is not supported yet. See issue #21 for details. - nIP, err := ip2int(ip) - if err != nil { - context.Set(r, "log", err.Error()) - http.Error(w, "IPv6 is not supported.", 501) - return - } - - // Check quota. - if rl != nil { - var ok bool - if ok, err = rl.Ok(nIP); err != nil { - context.Set(r, "log", err.Error()) // redis err - http.Error(w, http.StatusText(503), 503) - return - } else if !ok { - // Over quota, soz :( - http.Error(w, http.StatusText(403), 403) - return - } - } - - // Figure out the query: /fmt/{query} or /fmt/{nil} - // In case of {nil} we query the remote IP. - path := strings.SplitN(r.URL.Path, "/", 3) - if len(path) != 3 { - // This handler is for /fmt/ where fmt is json, xml or csv. - log.Fatal("Unexpected URL:", r.URL.Path) - } - - // Process the query, if there's one. - if len(path[2]) > 0 { - // Allow to query by IP or hostname. - if ip = net.ParseIP(path[2]); ip == nil { - if dns == nil || strings.Contains(path[2], " ") { - // DNS lookups not allowed or invalid hostname. - http.Error(w, http.StatusText(404), 404) - return - } else if ip = dns.LookupHost(path[2]); ip == nil { - // DNS lookup failed, assume host not found. - http.Error(w, http.StatusText(404), 404) - return - } - } - - if nIP, err = ip2int(ip); err != nil { // IPv6 fails here. - context.Set(r, "log", err.Error()) - http.Error(w, http.StatusText(404), 404) - return - } - } - - // Query the db. - var record *geoipRecord - if record, err = db.Lookup(ip, nIP); err != nil { http.NotFound(w, r) return } - - // Write response. - switch path[1][0] { - case 'j': - if cb := r.FormValue("callback"); len(cb) > 0 { - w.Header().Set("Content-Type", "text/javascript") - err = record.JSONP(w, cb) - } else { - w.Header().Set("Content-Type", "application/json") - err = record.JSON(w) - } - case 'x': - w.Header().Set("Content-Type", "application/xml") - err = record.XML(w) - case 'c': - w.Header().Set("Content-Type", "application/csv") - err = record.CSV(w) - } - if err != nil { - context.Set(r, "log", err.Error()) - } -} - -func runServer(mux *http.ServeMux, c *serverConfig) { - h := httpxtra.Handler{ - Handler: mux, - XHeaders: c.XHeaders, - } - if c.Log { - h.Logger = httpLogger - } - s := http.Server{ - Addr: c.Addr, - Handler: h, - ReadTimeout: 15 * time.Second, - WriteTimeout: 15 * time.Second, - } - if len(c.KeyFile) > 0 && len(c.CertFile) > 0 { - log.Printf("Starting HTTPS server on tcp/%s "+ - "log=%t xheaders=%t cert=%s key=%s", - c.Addr, - c.Log, - c.XHeaders, - c.CertFile, - c.KeyFile, - ) - log.Fatal(s.ListenAndServeTLS(c.CertFile, c.KeyFile)) - } else { - log.Printf("Starting HTTP server on tcp/%s "+ - "log=%t xheaders=%t", - c.Addr, - c.Log, - c.XHeaders, - ) - log.Fatal(httpxtra.ListenAndServe(s)) - } -} - -type dnsPool struct { - queryChan chan *dnsQuery - queryTimeout time.Duration -} - -type dnsQuery struct { - hostname string - respChan chan net.IP -} - -func (p *dnsPool) init(size int, queryTimeout time.Duration) { - p.queryChan = make(chan *dnsQuery) - p.queryTimeout = queryTimeout - for n := 0; n < size; n++ { - go p.doWork() - } -} - -func (p *dnsPool) doWork() { - for q := range p.queryChan { - q.respChan <- p.resolve(q.hostname) - } -} - -func (p *dnsPool) resolve(hostname string) net.IP { - if a, err := net.LookupHost(hostname); err != nil { - return nil - } else if len(a) == 1 { - return net.ParseIP(a[0]) - } else { - return net.ParseIP(a[rand.Intn(len(a)-1)]) - } -} - -func (p *dnsPool) LookupHost(hostname string) net.IP { - q := &dnsQuery{hostname, make(chan net.IP, 1)} - select { - case p.queryChan <- q: - select { - case ip := <-q.respChan: - close(q.respChan) - return ip - case <-time.After(p.queryTimeout): - return nil - } - default: - return nil - } -} - -type DB struct { - db *sql.DB - stmt *sql.Stmt - - // cache - country map[string]string - region map[regionKey]string - city map[int]locationData -} - -type regionKey struct { - CountryCode, - RegionCode string -} - -type locationData struct { - CountryCode, - RegionCode, - CityName, - ZipCode string - Latitude, - Longitude float32 - MetroCode, - AreaCode string -} - -func openDB(cf *configFile) (*DB, error) { - var ( - db DB - err error - ) - if db.db, err = sql.Open("sqlite3", cf.IPDB.File); err != nil { - return nil, err - } - if _, err = db.db.Exec("PRAGMA cache_size=" + cf.IPDB.CacheSize); err != nil { - return nil, err - } - if db.stmt, err = db.db.Prepare(` - SELECT - loc_id - FROM - city_blocks - WHERE - ip_start <= ? - ORDER BY - ip_start DESC - LIMIT 1 - `); err != nil { - return nil, err - } - if err = db.loadCache(); err != nil { - return nil, err - } - return &db, nil -} - -func (db *DB) loadCache() error { - var ( - wg sync.WaitGroup - errc = make(chan error, 3) - ) - wg.Add(3) - go db.loadCountries(&wg, errc) - go db.loadRegions(&wg, errc) - go db.loadCities(&wg, errc) - wg.Wait() - select { - case err := <-errc: - return err - default: - return nil - } -} - -func (db *DB) loadCountries(wg *sync.WaitGroup, errc chan error) { - defer wg.Done() - db.country = make(map[string]string) - row, err := db.db.Query(` - SELECT - country_code, - country_name - FROM - country_blocks - `) - if err != nil { - errc <- err - return - } - defer row.Close() - var country_code, name string - for row.Next() { - if err = row.Scan( - &country_code, - &name, - ); err != nil { - errc <- err - return - } - db.country[country_code] = name - } -} - -func (db *DB) loadRegions(wg *sync.WaitGroup, errc chan error) { - defer wg.Done() - db.region = make(map[regionKey]string) - row, err := db.db.Query(` - SELECT - country_code, - region_code, - region_name - FROM - region_names - `) + q := f.enc.NewQuery() + err := f.db.Lookup(ip, q) if err != nil { - errc <- err + http.Error(w, "Try again later.", + http.StatusServiceUnavailable) return } - defer row.Close() - var country_code, region_code, name string - for row.Next() { - if err = row.Scan( - &country_code, - ®ion_code, - &name, - ); err != nil { - errc <- err - return - } - db.region[regionKey{country_code, region_code}] = name - } -} - -func (db *DB) loadCities(wg *sync.WaitGroup, errc chan error) { - defer wg.Done() - db.city = make(map[int]locationData) - row, err := db.db.Query("SELECT * FROM city_location") + w.Header().Set("X-Database-Date", f.db.Date().Format(http.TimeFormat)) + err = f.enc.Encode(w, r, q, ip) if err != nil { - errc <- err + f.db.sendError(fmt.Errorf("Failed to encode %#v: %s", q, err)) + http.Error(w, "An unexpected error occurred.", + http.StatusInternalServerError) return } - defer row.Close() - var ( - locId int - loc locationData - ) - for row.Next() { - if err = row.Scan( - &locId, - &loc.CountryCode, - &loc.RegionCode, - &loc.CityName, - &loc.ZipCode, - &loc.Latitude, - &loc.Longitude, - &loc.MetroCode, - &loc.AreaCode, - ); err != nil { - errc <- err - return - } - db.city[locId] = loc - } -} - -func (db *DB) Lookup(ip net.IP, nIP uint32) (*geoipRecord, error) { - for _, net := range reservedIPs { - if net.Contains(ip) { - return &geoipRecord{ - Ip: ip.String(), - CountryCode: "RD", - CountryName: "Reserved", - }, nil - } - } - var locId int - if err := db.stmt.QueryRow(nIP).Scan(&locId); err != nil { - return nil, err - } - return db.newRecord(&ip, locId), nil -} - -func (db *DB) newRecord(ip *net.IP, locId int) *geoipRecord { - city, ok := db.city[locId] - if !ok { - return &geoipRecord{Ip: ip.String()} - } - return &geoipRecord{ - Ip: ip.String(), - CountryCode: city.CountryCode, - CountryName: db.country[city.CountryCode], - RegionCode: city.RegionCode, - RegionName: db.region[regionKey{ - city.CountryCode, - city.RegionCode, - }], - CityName: city.CityName, - ZipCode: city.ZipCode, - Latitude: city.Latitude, - Longitude: city.Longitude, - MetroCode: city.MetroCode, - AreaCode: city.AreaCode, - } } -type geoipRecord struct { - XMLName xml.Name `json:"-" xml:"Response"` - Ip string `json:"ip"` - CountryCode string `json:"country_code"` - CountryName string `json:"country_name"` - RegionCode string `json:"region_code"` - RegionName string `json:"region_name"` - CityName string `json:"city" xml:"City"` - ZipCode string `json:"zipcode"` - Latitude float32 `json:"latitude"` - Longitude float32 `json:"longitude"` - MetroCode string `json:"metro_code"` - AreaCode string `json:"area_code"` -} - -func (r *geoipRecord) JSON(w io.Writer) error { - if collectStats { - outputCount.Add("json", 1) - } - return json.NewEncoder(w).Encode(r) -} - -func (r *geoipRecord) JSONP(w io.Writer, callback string) error { - if collectStats { - outputCount.Add("json", 1) - } - var err error - if _, err = w.Write([]byte(callback)); err != nil { - return err - } - if _, err = w.Write([]byte("(")); err != nil { - return err - } - if err = json.NewEncoder(w).Encode(r); err != nil { - return err - } - if _, err = w.Write([]byte(");")); err != nil { - return err - } - return nil -} - -func (r *geoipRecord) XML(w io.Writer) error { - if collectStats { - outputCount.Add("xml", 1) - } - enc := xml.NewEncoder(w) - enc.Indent("", "\t") - var err error - if _, err = w.Write([]byte(xml.Header)); err != nil { - return err +func (f *Handler) queryIP(r *http.Request) net.IP { + if r.URL.Path[len(r.URL.Path)-1] == '/' { + return f.remoteAddr(r) } - if err = enc.Encode(r); err != nil { - return err + q := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + if ip := net.ParseIP(q); ip != nil { + return ip } - if _, err = w.Write([]byte("\n")); err != nil { - return err - } - return nil -} - -func (r *geoipRecord) CSV(w io.Writer) error { - if collectStats { - outputCount.Add("csv", 1) - } - if _, err := fmt.Fprintf(w, - `"%s","%s","%s","%s","%s","%s","%s","%0.4f","%0.4f","%s","%s"`+"\r\n", - r.Ip, - r.CountryCode, - r.CountryName, - r.RegionCode, - r.RegionName, - r.CityName, - r.ZipCode, - r.Latitude, - r.Longitude, - r.MetroCode, - r.AreaCode, - ); err != nil { - return err - } - return nil -} - -type rateLimiter interface { - init(cf *configFile) // Initializes the limiter - Ok(ipkey uint32) (bool, error) // Returns true if under quota -} - -// mapQuota implements the rateLimiter interface using a map as the backend. -type mapQuota struct { - cf *configFile - mu sync.Mutex - ip map[uint32]int -} - -func (q *mapQuota) init(cf *configFile) { - q.cf = cf - q.ip = make(map[uint32]int) -} - -func (q *mapQuota) Ok(ipkey uint32) (bool, error) { - q.mu.Lock() - defer q.mu.Unlock() - if n, ok := q.ip[ipkey]; ok { - if n < q.cf.Limit.MaxRequests { - q.ip[ipkey]++ - return true, nil - } - return false, nil - } - q.ip[ipkey] = 1 - go func() { - time.Sleep(time.Duration(q.cf.Limit.Expire) * time.Second) - q.mu.Lock() - delete(q.ip, ipkey) - q.mu.Unlock() - }() - return true, nil -} - -// redisQuota implements the rateLimiter interface using Redis as the backend. -type redisQuota struct { - cf *configFile - rc *redis.Client -} - -func (q *redisQuota) init(cf *configFile) { - redis.MaxIdleConnsPerAddr = 5000 - q.cf = cf - q.rc = redis.New(cf.Redis...) - q.rc.Timeout = time.Duration(1500) * time.Millisecond -} - -func (q *redisQuota) Ok(ipkey uint32) (bool, error) { - k := strconv.Itoa(int(ipkey)) // "numeric" key - if ns, err := q.rc.Get(k); err != nil { - return false, fmt.Errorf("redis get: %s", err) - } else if len(ns) == 0 { - if err = q.rc.SetEx(k, q.cf.Limit.Expire, "1"); err != nil { - return false, fmt.Errorf("redis setex: %s", err) - } - } else if n, _ := strconv.Atoi(ns); n < q.cf.Limit.MaxRequests { - if n, err = q.rc.Incr(k); err != nil { - return false, fmt.Errorf("redis incr: %s", err) - } else if n == 1 { - q.rc.Expire(k, q.cf.Limit.Expire) - } - } else { - return false, nil - } - return true, nil -} - -func ip2int(ip net.IP) (uint32, error) { - ipv4 := ip.To4() - if ipv4 == nil { - return 0, fmt.Errorf("IP %s is not IPv4", ip.String()) - } - return binary.BigEndian.Uint32(ipv4), nil -} - -func runProfile() { - f, err := os.Create("freegeoip.cpu.prof") + ip, err := net.LookupIP(q) if err != nil { - log.Fatal(err) + return nil // Not found. } - pprof.StartCPUProfile(f) - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt, os.Kill) - go func() { - <-sig - pprof.StopCPUProfile() - f.Close() - f, err = os.Create("freegeoip.mem.prof") - if err != nil { - log.Fatal(err) - } - pprof.WriteHeapProfile(f) - os.Exit(0) - }() -} - -func setLog(filename string) { - f := openLog(filename) - log.SetOutput(f) - sigc := make(chan os.Signal, 1) - signal.Notify(sigc, syscall.SIGHUP) - go func() { - // Recycle log file on SIGHUP. - var fb *os.File - for { - <-sigc - fb = f - f = openLog(filename) - log.SetOutput(f) - fb.Close() - } - }() -} - -func openLog(filename string) *os.File { - f, err := os.OpenFile( - filename, - os.O_WRONLY|os.O_CREATE|os.O_APPEND, - 0644, - ) - if err != nil { - log.SetOutput(os.Stderr) - log.Fatal(err) - } - return f -} - -func httpLogger(r *http.Request, created time.Time, status, bytes int) { - //fmt.Println(httpxtra.ApacheCommonLog(r, created, status, bytes)) - var ( - s, ip, msg string - err error - ) - if r.TLS == nil { - s = "HTTP" - } else { - s = "HTTPS" - } - if ip, _, err = net.SplitHostPort(r.RemoteAddr); err != nil { - ip = r.RemoteAddr - } - if tmp := context.Get(r, "log"); tmp != nil { - msg = fmt.Sprintf(" (%s)", tmp) - context.Clear(r) - } - log.Printf("%s %d %s %q (%s) :: %d bytes in %s%s", - s, - status, - r.Method, - r.URL.Path, - ip, - bytes, - time.Since(created), - msg, - ) - if collectStats { - protocolCount.Add(s, 1) - statusCount.Add(strconv.Itoa(status), 1) - } -} - -type serverConfig struct { - Log bool `xml:"log,attr"` - XHeaders bool `xml:"xheaders,attr"` - Addr string `xml:"addr,attr"` - CertFile string - KeyFile string -} - -type configFile struct { - XMLName xml.Name `xml:"Server"` - Debug bool `xml:"debug,attr"` - DebugSrv string `xml:"debugsrv,attr"` - DocumentRoot string - - Listen []*serverConfig - - DNS struct { - Enabled bool `xml:",attr"` - Timeout string `xml:",attr"` - MaxConcurrent int `xml:",attr"` - } - - IPDB struct { - File string `xml:",attr"` - CacheSize string `xml:",attr"` - } - - Limit struct { - MaxRequests int `xml:",attr"` - Expire int `xml:",attr"` - } - - Redis []string `xml:"Redis>Addr"` -} - -func loadConfig(filename string) (*configFile, error) { - var cf configFile - if fd, err := os.Open(filename); err != nil { - return nil, err - } else { - if err = xml.NewDecoder(fd).Decode(&cf); err != nil { - return nil, err - } - fd.Close() - } - // Make files' path relative to the config file's directory. - basedir := filepath.Dir(filename) - relativePath(basedir, &cf.IPDB.File) - relativePath(basedir, &cf.DocumentRoot) - for _, l := range cf.Listen { - relativePath(basedir, &l.CertFile) - relativePath(basedir, &l.KeyFile) - } - return &cf, nil -} - -func relativePath(basedir string, filename *string) { - if len(*filename) > 0 && (*filename)[0] != '/' { - *filename = filepath.Join(basedir, *filename) + if len(ip) == 0 { + return nil } + return ip[rand.Intn(len(ip))] } -// http://en.wikipedia.org/wiki/Reserved_IP_addresses -var reservedIPs = []net.IPNet{ - {net.IPv4(0, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)}, - {net.IPv4(10, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)}, - {net.IPv4(100, 64, 0, 0), net.IPv4Mask(255, 192, 0, 0)}, - {net.IPv4(127, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)}, - {net.IPv4(169, 254, 0, 0), net.IPv4Mask(255, 255, 0, 0)}, - {net.IPv4(172, 16, 0, 0), net.IPv4Mask(255, 240, 0, 0)}, - {net.IPv4(192, 0, 0, 0), net.IPv4Mask(255, 255, 255, 248)}, - {net.IPv4(192, 0, 2, 0), net.IPv4Mask(255, 255, 255, 0)}, - {net.IPv4(192, 88, 99, 0), net.IPv4Mask(255, 255, 255, 0)}, - {net.IPv4(192, 168, 0, 0), net.IPv4Mask(255, 255, 0, 0)}, - {net.IPv4(198, 18, 0, 0), net.IPv4Mask(255, 254, 0, 0)}, - {net.IPv4(198, 51, 100, 0), net.IPv4Mask(255, 255, 255, 0)}, - {net.IPv4(203, 0, 113, 0), net.IPv4Mask(255, 255, 255, 0)}, - {net.IPv4(224, 0, 0, 0), net.IPv4Mask(240, 0, 0, 0)}, - {net.IPv4(240, 0, 0, 0), net.IPv4Mask(240, 0, 0, 0)}, - {net.IPv4(255, 255, 255, 255), net.IPv4Mask(255, 255, 255, 255)}, +func (f *Handler) remoteAddr(r *http.Request) net.IP { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return net.ParseIP(r.RemoteAddr) + } + return net.ParseIP(host) +} + +// ProxyHandler is a wrapper for other http handlers that sets the +// client IP address in request.RemoteAddr to the first value of a +// comma separated list of IPs from the X-Forwarded-For request +// header. It resets the original RemoteAddr back after running the +// designated handler f. +func ProxyHandler(f http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + addr := r.Header.Get("X-Forwarded-For") + if len(addr) > 0 { + remoteAddr := r.RemoteAddr + r.RemoteAddr = strings.SplitN(addr, ",", 2)[0] + defer func() { r.RemoteAddr = remoteAddr }() + } + f.ServeHTTP(w, r) + }) } diff --git a/freegeoip_test.go b/freegeoip_test.go index 92f67cb..f748ddf 100644 --- a/freegeoip_test.go +++ b/freegeoip_test.go @@ -1,198 +1,162 @@ -// Copyright 2009-2014 Alexandre Fiori +// Copyright 2009-2014 The freegeoip authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -package main +package freegeoip import ( "bytes" + "encoding/csv" + + "io/ioutil" "net" - "sync" + "net/http" + "net/http/httptest" + "net/url" "testing" - "time" -) - -var ( - testCf *configFile - testDB *DB ) -func TestLoadConfig(t *testing.T) { - var err error - testCf, err = loadConfig("freegeoip.conf") - if err != nil { - t.Fatal(err) +func TestQueryRemoteAddr(t *testing.T) { + want := net.ParseIP("8.8.8.8") + // No query argument, so we query the remote IP. + r := http.Request{ + URL: &url.URL{Path: "/"}, + RemoteAddr: "8.8.8.8:8888", + Header: http.Header{}, } -} - -func TestOpenDB(t *testing.T) { - var err error - testDB, err = openDB(testCf) - if err != nil { - t.Log("Make sure the DB exists: cd db && ./updatedb") - t.Fatal(err) + f := &Handler{} + if ip := f.queryIP(&r); !bytes.Equal(ip, want) { + t.Errorf("Unexpected IP: %s", ip) } } -func TestQueryDB1(t *testing.T) { - record := testQuery(t, "127.0.0.1") - if record.CountryName != "Reserved" { - t.Fatal("Unexpected value:", record.CountryName) +func TestQueryDNS(t *testing.T) { + want4 := net.ParseIP("8.8.8.8") + want6 := net.ParseIP("2001:4860:4860::8888") + r := http.Request{ + URL: &url.URL{Path: "/google-public-dns-a.google.com"}, + RemoteAddr: "127.0.0.1:8080", + Header: make(map[string][]string), } -} - -func TestQueryDB2(t *testing.T) { - record := testQuery(t, "8.8.8.8") - if record.CountryCode != "US" { - t.Fatal("Unexpected value:", record.CountryCode) + f := &Handler{} + ip := f.queryIP(&r) + if ip == nil { + t.Fatal("Failed to resolve", r.URL.Path) } -} - -func TestRecordJSON(t *testing.T) { - record := testQuery(t, "127.0.0.1") - b := bytes.NewBuffer(nil) - record.JSON(b) - if len(b.Bytes()) != 180 { - t.Fatal("Unexpected value:", b.String()) - } -} - -func TestRecordJSONP(t *testing.T) { - record := testQuery(t, "127.0.0.1") - b := bytes.NewBuffer(nil) - record.JSONP(b, "f") - if len(b.Bytes()) != 184 { - t.Fatal("Unexpected value:", b.String()) + if !bytes.Equal(ip, want4) && !bytes.Equal(ip, want6) { + t.Errorf("Unexpected IP: %s", ip) } } -func TestRecordXML(t *testing.T) { - record := testQuery(t, "127.0.0.1") - b := bytes.NewBuffer(nil) - record.XML(b) - if len(b.Bytes()) != 338 { - t.Fatal("Unexpected value:", b.String()) - } -} +// Test the server. -func TestRecordCSV(t *testing.T) { - record := testQuery(t, "127.0.0.1") - b := bytes.NewBuffer(nil) - record.CSV(b) - if len(b.Bytes()) != 65 { - t.Fatal("Unexpected value:", b.String()) +func runServer(pattern string, f Encoder) (*Handler, *httptest.Server, error) { + db, err := Open(testFile) + if err != nil { + return nil, nil, err } -} - -func TestMapQuota(t *testing.T) { - testCf.Limit.MaxRequests = 1 - testCf.Limit.Expire = 1 - rl := new(mapQuota) - rl.init(testCf) - nIP, _ := ip2int(net.ParseIP("127.0.0.1")) - if ok, _ := rl.Ok(nIP); !ok { - t.Fatal("Unexpected value:", ok) - } - if ok, _ := rl.Ok(nIP); ok { - t.Fatal("Unexpected value:", ok) + select { + case <-db.NotifyOpen(): + case err := <-db.NotifyError(): + if err != nil { + return nil, nil, err + } } + mux := http.NewServeMux() + handle := NewHandler(db, f) + mux.Handle(pattern, ProxyHandler(handle)) + return handle, httptest.NewServer(mux), nil } -func TestRedisQuota(t *testing.T) { - if len(testCf.Redis) < 1 { - t.Skip("Redis is not configured") - } - testCf.Limit.MaxRequests = 1 - testCf.Limit.Expire = 1 - rl := new(redisQuota) - rl.init(testCf) - nIP, _ := ip2int(net.ParseIP("127.0.0.1")) - if ok, err := rl.Ok(nIP); err != nil { +func TestLookupUnavailable(t *testing.T) { + handle, srv, err := runServer("/csv/", &CSVEncoder{}) + if err != nil { t.Fatal(err) - } else if !ok { - t.Fatal("Unexpected value:", ok) } - if ok, err := rl.Ok(nIP); err != nil { + defer srv.Close() + handle.db.mu.Lock() + reader := handle.db.reader + handle.db.reader = nil + handle.db.mu.Unlock() + defer func() { + handle.db.mu.Lock() + handle.db.reader = reader + handle.db.mu.Unlock() + }() + resp, err := http.Get(srv.URL + "/csv/8.8.8.8") + if err != nil { t.Fatal(err) - } else if ok { - t.Fatal("Unexpected value:", ok) } -} - -func TestDNSLookup(t *testing.T) { - dp := new(dnsPool) - dp.init(1, 1000*time.Millisecond) - if ip := dp.LookupHost("localhost"); ip == nil { - t.Fatal("Could not resolve host name") - } else if !isLocalhostIP(ip.String()) { - t.Fatal("Unexpected IP: " + ip.String()) - } -} - -func TestConcurrentDNSLookup1(t *testing.T) { - dp := new(dnsPool) - dp.init(1, 1000*time.Millisecond) - go dp.LookupHost("invalid.host.name1") - var wg sync.WaitGroup - wg.Add(1) - go func() { - if ip := dp.LookupHost("localhost"); ip != nil { - t.Error("Unexpected IP: " + ip.String()) - } - wg.Done() - }() - wg.Wait() -} - -func TestConcurrentDNSLookup2(t *testing.T) { - dp := new(dnsPool) - dp.init(2, 1000*time.Millisecond) - go dp.LookupHost("invalid.host.name2") - var wg sync.WaitGroup - wg.Add(1) - go func() { - if ip := dp.LookupHost("localhost"); ip == nil { - t.Error("Could not resolve host name") - } else if !isLocalhostIP(ip.String()) { - t.Error("Unexpected IP: " + ip.String()) + defer resp.Body.Close() + if resp.StatusCode != http.StatusServiceUnavailable { + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) } - wg.Done() - }() - wg.Wait() + t.Fatalf("Unexpected query worked: %s\n%s", resp.Status, b) + } } -func BenchmarkQueryDB(b *testing.B) { - if testDB == nil { - b.Skip("DB is not available") - } - ip := net.ParseIP("8.8.8.8") - nIP, _ := ip2int(ip) - for i := 0; i < b.N; i++ { - _, err := testDB.Lookup(ip, nIP) +func TestLookupNotFound(t *testing.T) { + _, srv, err := runServer("/csv/", &CSVEncoder{}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/csv/fail-me") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + b, err := ioutil.ReadAll(resp.Body) if err != nil { - b.Fatal(err) + t.Fatal(err) } + t.Fatalf("Unexpected query worked: %s\n%s", resp.Status, b) } } -func testQuery(t *testing.T, addr string) *geoipRecord { - if testDB == nil { - t.Skip("DB is not available") +func TestLookupXForwardedFor(t *testing.T) { + _, srv, err := runServer("/csv/", &CSVEncoder{}) + if err != nil { + t.Fatal(err) } - ip := net.ParseIP(addr) - nIP, _ := ip2int(ip) - record, err := testDB.Lookup(ip, nIP) + defer srv.Close() + req, err := http.NewRequest("GET", srv.URL+"/csv/", nil) if err != nil { t.Fatal(err) } - return record + req.Header.Set("X-Forwarded-For", "8.8.8.8") + resp, err := http.DefaultClient.Do(req) + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + row, err := csv.NewReader(resp.Body).Read() + if err != nil { + t.Fatal(err) + } + if row[0] != "US" { + t.Fatalf("Unexpected country code in record: %#v", row) + } } -func isLocalhostIP(ip string) bool { - for _, v := range []string{"::1", "fe80::1", "127.0.0.1"} { - if ip == v { - return true - } +func TestLookupDatabaseDate(t *testing.T) { + _, srv, err := runServer("/csv/", &CSVEncoder{}) + if err != nil { + t.Fatal(err) + } + defer srv.Close() + resp, err := http.Get(srv.URL + "/csv/") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } + if len(resp.Header.Get("X-Database-Date")) == 0 { + t.Fatal("Header X-Database-Date is missing") } - return false } diff --git a/static/css/index.css b/static/css/index.css deleted file mode 100644 index dc03c03..0000000 --- a/static/css/index.css +++ /dev/null @@ -1,17 +0,0 @@ -body { - padding-top: 10px; - padding-bottom: 40px; -} -.container-narrow { - margin: 0 auto; - max-width: 700px; -} -.container-narrow > hr { - margin: 30px 0; -} -code { - color:#333; -} -div[class="tooltip-inner"] { - max-width: 350px; -} diff --git a/static/css/index.min.css b/static/css/index.min.css deleted file mode 100644 index cbc76df..0000000 --- a/static/css/index.min.css +++ /dev/null @@ -1 +0,0 @@ -body{padding-top:10px;padding-bottom:40px}.container-narrow{margin:0 auto;max-width:700px}.container-narrow>hr{margin:30px 0}.jumbotron{margin:20px 0;text-align:center}.jumbotron h1{font-size:72px;line-height:1}.jumbotron .btn{font-size:21px;padding:14px 24px}.marketing{margin:20px 0}.marketing p+h4{margin-top:28px}code{color:#333}.justify{text-align:justify}div[class="tooltip-inner"]{max-width:350px} \ No newline at end of file diff --git a/static/css/index.min.css.gz b/static/css/index.min.css.gz deleted file mode 100644 index 9194f95..0000000 Binary files a/static/css/index.min.css.gz and /dev/null differ diff --git a/static/index.html b/static/index.html deleted file mode 100644 index 52f6ac8..0000000 --- a/static/index.html +++ /dev/null @@ -1,147 +0,0 @@ - - - - - freegeoip.net: FREE IP Geolocation Web Service - - - - - - - - - - - - -
-
-
-

freegeoip.net

-
-
-
-
- -
-
- -
-
-
-
-
- -
-
-
-

Oooops!

-
Over quota
-

This service is public and free with a hard limit - of 10.000 requests per hour, and you've reached your quota.

-

Check out the Limits section below for more information.

-
-
-

Oooops!

-
No results for
-

Try one of the following:

-
    -
  • Make sure you enter a valid IP address, or
    • -
  • A valid host, or domain name (e.g. google.com)
    • -
-
-
-

Oooops!

-
Service Unavailable
-

The service is unavailable due to maintenance or high load.

-

We apologize for the inconvenience. Please try again later.

-
-
-

Estimated location:

- - - - - - - - - - -
IP
Country
Region
City
Zip code
Latitude and Longitude
Area and Metro codes
- Show map - Hide map -
-
-
-

About

-

freegeoip.net is a public REST API for searching geolocation of IP addresses and host names.

-

It has an internal database with geolocation information, which is queried via the API. There's no sorcery, it's just a database. And although the database is very accurate, don't expect it to be perfect.

-

See the freegeoip blog.

-
-
-
-
-
-

Usage for developers

-

Send HTTP GET requests to: - freegeoip.net/{format}/{ip_or_hostname}

-

The API supports both HTTP and HTTPS.

-

Supported formats are csv, xml or json, - which also support the - callback - query argument for JSONP.

-

The ip_or_hostname part is optional. Your own IP is - searched if one is not provided.

-
-
-

Limits

-

API usage is limited to 10,000 queries per hour. After - reaching this limit, all requests will result in HTTP 403 - (Forbidden) until the roll over.

-

If the usage limit is a problem, please consider running your own - instance of this system. It's open source and freely available - at GitHub.

-

Download the source code.

-
-
-
-

Support

-

freegeoip.net is community funded, - therefore please consider donating if you like and use this service.

- -
- - - - -
- -

For any other inquiries or feedback, - contact me directly.

- -
-
- -

This web site and all documentation is licensed under - Creative Commons 3.0. - This service includes GeoLite data created by MaxMind, available - from maxmind.com.

-
-
- - - diff --git a/static/js/index.js b/static/js/index.js deleted file mode 100644 index 926eb35..0000000 --- a/static/js/index.js +++ /dev/null @@ -1,64 +0,0 @@ -var gmap="//maps.google.com/maps?f=q&source=s_q&hl=en&geocode=&ie=UTF8&iwloc=A&output=embed&"; -function getParameterByName(name) { - var match = RegExp('[?&]' + name + '=([^&]*)').exec(window.location.search); - return match && decodeURIComponent(match[1].replace(/\+/g, ' ')); -}; -angular.module('freegeoip',[]) -.directive('btnSubmit', function(){ - return function(scope,element,attrs){ - scope.$watch(function(){ - return scope.$eval(attrs.btnSubmit); - }, - function(loading){ - if(loading) $(element).button('loading'); - else $(element).button('reset'); - }); - } -}); -freegeoip.$inject = ['$scope','$http']; -function freegeoip($scope,$http){ - $scope.geoip = {} - $scope.search = function(q, showMap) { - $scope.error = null; - $scope.map = showMap; - $scope.searching = true; - $http.get('json/' + (q || "")). - success(function(rs){ - $scope.q = q || rs.ip; - if(!rs.metro_code) rs.metro_code = '-'; - if(!rs.area_code) rs.area_code = '-'; - $scope.geoip = angular.copy(rs); - var qs = ""; - var zoom = 2; - if(rs.country_name&&rs.country_code!='RD'){ - if(rs.region_name){ - if(rs.city){ - zoom=6; - qs=rs.city+','+rs.region_name+','+rs.country_name; - } else { - zoom=4; - qs=rs.region_name+','+rs.country_name; - } - } else { - qs=rs.country_name; - } - } else { - qs = "Africa"; - } - $("#map").attr("src", gmap+"z="+zoom+"&q="+qs); - $scope.searching = false; - }). - error(function(data,st){ - $scope.errorq = q; - $scope.error = st; - $scope.searching = false; - $("#map").attr("src", gmap+"zoom=2&q=africa"); - }); - } - $scope.search(getParameterByName("q"), getParameterByName("map")); -} -$(document).ready(function(){ - $("#map").height($("#map").width()/2); - $("#cb").tooltip(); - $("#cb").click(function(el){el.preventDefault();}); -}); diff --git a/static/js/index.min.js b/static/js/index.min.js deleted file mode 100644 index ce5155c..0000000 --- a/static/js/index.min.js +++ /dev/null @@ -1,3 +0,0 @@ -var gmap="//maps.google.com/maps?f=q&source=s_q&hl=en&geocode=&ie=UTF8&iwloc=A&output=embed&";function getParameterByName(a){return(a=RegExp("[?&]"+a+"=([^&]*)").exec(window.location.search))&&decodeURIComponent(a[1].replace(/\+/g," "))}angular.module("freegeoip",[]).directive("btnSubmit",function(){return function(a,c,d){a.$watch(function(){return a.$eval(d.btnSubmit)},function(a){a?$(c).button("loading"):$(c).button("reset")})}});freegeoip.$inject=["$scope","$http"]; -function freegeoip(a,c){a.geoip={};a.search=function(d,f){a.error=null;a.map=f;a.searching=!0;c.get("json/"+(d||"")).success(function(b){a.q=d||b.ip;b.metro_code||(b.metro_code="-");b.area_code||(b.area_code="-");a.geoip=angular.copy(b);var e="",c=2;b.country_name&&"RD"!=b.country_code?b.region_name?b.city?(c=6,e=b.city+","+b.region_name+","+b.country_name):(c=4,e=b.region_name+","+b.country_name):e=b.country_name:e="Africa";$("#map").attr("src",gmap+"z="+c+"&q="+e);a.searching=!1}).error(function(b, -c){a.errorq=d;a.error=c;a.searching=!1;$("#map").attr("src",gmap+"zoom=2&q=africa")})};a.search(getParameterByName("q"),getParameterByName("map"))}$(document).ready(function(){$("#map").height($("#map").width()/2);$("#cb").tooltip();$("#cb").click(function(a){a.preventDefault()})}); diff --git a/static/js/index.min.js.gz b/static/js/index.min.js.gz deleted file mode 100644 index 0a1e54b..0000000 Binary files a/static/js/index.min.js.gz and /dev/null differ diff --git a/static/map.html b/static/map.html deleted file mode 100644 index 889f886..0000000 --- a/static/map.html +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - FreeGeoIP - - - - - - -
- - diff --git a/testdata/.placeholder b/testdata/.placeholder new file mode 100644 index 0000000..e69de29