Move freegeoip daemon code to its own package

Moving contents from cmd/freegeoip/main.go to apiserver package for
better test coverage.

This change updates the -addr command line flag and its behavior,
and is backwards incomplatible. People using -addr must switch over
to using -http now. In order to enable HTTPS, one must use -https
and the server might listen on both HTTP and HTTPS.

The -pprof flag changed to -internal-server and serves not only pprof
but also metrics for prometheus (http://prometheus.io). These are
under /debug/pprof (https://golang.org/pkg/net/http/pprof/) and
/metrics accordingly.

Bringing back the -read-timeout and -write-timeout command line flags
for server tuning.

Fixed a race condition bug in the redis quota algorithm, at the
exchange of 1 redis incr per request following advice from
pattern #2 from http://redis.io/commands/incr.

Also added rate limit response headers for all HTTP and HTTPS requests,
inspired by GitHub's API:

 X-RateLimit-Limit: number of requests allowed per interval (def. 1h)
 X-RateLimit-Remaining: number of requests remaining, per user
 X-RateLimit-Reset: time in seconds before resetting the limit

Added the -logtostdout command line flag to close #146.

Minor fix to the background database download back off algorithm,
added -api-prefix and -cors-origin command line flags, and tests.

apiserver/cmd.go

@@ -0,0 +1,135 @@
+// Copyright 2009-2015 The freegeoip authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package apiserver
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	// embed pprof server.
+	_ "net/http/pprof"
+
+	"github.com/fiorix/freegeoip"
+	"github.com/fiorix/go-redis/redis"
+	gorilla "github.com/gorilla/handlers"
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/net/http2"
+)
+
+// Version tag.
+var Version = "3.0.7"
+
+var maxmindDB = "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz"
+
+var (
+	flAPIPrefix      = flag.String("api-prefix", "/", "Prefix for API endpoints")
+	flCORSOrigin     = flag.String("cors-origin", "*", "CORS origin API endpoints")
+	flHTTPAddr       = flag.String("http", ":8080", "Address in form of ip:port to listen on for HTTP")
+	flHTTPSAddr      = flag.String("https", "", "Address in form of ip:port to listen on for HTTPS")
+	flCertFile       = flag.String("cert", "cert.pem", "X.509 certificate file")
+	flKeyFile        = flag.String("key", "key.pem", "X.509 key file")
+	flReadTimeout    = flag.Duration("read-timeout", 30*time.Second, "Read timeout for HTTP and HTTPS client conns")
+	flWriteTimeout   = flag.Duration("write-timeout", 15*time.Second, "Write timeout for HTTP and HTTPS client conns")
+	flPublicDir      = flag.String("public", "", "Public directory to serve at the {prefix}/ endpoint")
+	flDB             = flag.String("db", maxmindDB, "IP database file or URL")
+	flUpdateIntvl    = flag.Duration("update", 24*time.Hour, "Database update check interval")
+	flRetryIntvl     = flag.Duration("retry", time.Hour, "Max time to wait before retrying to download database")
+	flUseXFF         = flag.Bool("use-x-forwarded-for", false, "Use the X-Forwarded-For header when available (e.g. when running behind proxies)")
+	flSilent         = flag.Bool("silent", false, "Do not log HTTP or HTTPS requests to stderr")
+	flLogToStdout    = flag.Bool("logtostdout", false, "Log to stdout instead of stderr")
+	flRedisAddr      = flag.String("redis", "127.0.0.1:6379", "Redis address in form of ip:port[,ip:port] for quota")
+	flRedisTimeout   = flag.Duration("redis-timeout", 500*time.Millisecond, "Redis read/write timeout")
+	flQuotaMax       = flag.Int("quota-max", 0, "Max requests per source IP per interval; set 0 to turn off")
+	flQuotaIntvl     = flag.Duration("quota-interval", time.Hour, "Quota expiration interval per source IP querying the API")
+	flVersion        = flag.Bool("version", false, "Show version and exit")
+	flInternalServer = flag.String("internal-server", "", "Address in form of ip:port to listen on for /metrics and /debug/pprof")
+)
+
+// Run is the entrypoint for the freegeoip daemon tool.
+func Run() error {
+	flag.Parse()
+
+	if *flVersion {
+		fmt.Printf("freegeoip v%s\n", Version)
+		return nil
+	}
+
+	if *flLogToStdout {
+		log.SetOutput(os.Stdout)
+	}
+
+	log.SetPrefix("[freegeoip] ")
+
+	addrs := strings.Split(*flRedisAddr, ",")
+	rc, err := redis.Dial(addrs...)
+	if err != nil {
+		return err
+	}
+	rc.Timeout = *flRedisTimeout
+
+	db, err := openDB(*flDB, *flUpdateIntvl, *flRetryIntvl)
+	if err != nil {
+		return err
+	}
+	go watchEvents(db)
+
+	ah := NewHandler(&HandlerConfig{
+		Prefix:    *flAPIPrefix,
+		Origin:    *flCORSOrigin,
+		PublicDir: *flPublicDir,
+		DB:        db,
+		RateLimiter: RateLimiter{
+			Redis:    rc,
+			Max:      *flQuotaMax,
+			Interval: *flQuotaIntvl,
+		},
+	})
+
+	if !*flSilent {
+		ah = gorilla.CombinedLoggingHandler(os.Stderr, ah)
+	}
+
+	if *flUseXFF {
+		ah = freegeoip.ProxyHandler(ah)
+	}
+
+	if len(*flInternalServer) > 0 {
+		http.Handle("/metrics", prometheus.Handler())
+		log.Println("freegeoip internal server starting on", *flInternalServer)
+		go func() { log.Fatal(http.ListenAndServe(*flInternalServer, nil)) }()
+	}
+
+	if *flHTTPAddr != "" {
+		log.Println("freegeoip http server starting on", *flHTTPAddr)
+		srv := &http.Server{
+			Addr:         *flHTTPAddr,
+			Handler:      ah,
+			ReadTimeout:  *flReadTimeout,
+			WriteTimeout: *flWriteTimeout,
+			ConnState:    ConnStateMetrics(httpConnsGauge),
+		}
+		go func() { log.Fatal(srv.ListenAndServe()) }()
+	}
+
+	if *flHTTPSAddr != "" {
+		log.Println("freegeoip https server starting on", *flHTTPSAddr)
+		srv := &http.Server{
+			Addr:         *flHTTPSAddr,
+			Handler:      ah,
+			ReadTimeout:  *flReadTimeout,
+			WriteTimeout: *flWriteTimeout,
+			ConnState:    ConnStateMetrics(httpsConnsGauge),
+		}
+		http2.ConfigureServer(srv, nil)
+		go func() { log.Fatal(srv.ListenAndServeTLS(*flCertFile, *flKeyFile)) }()
+	}
+
+	select {}
+}

apiserver/cmd_test.go

@@ -0,0 +1,26 @@
+// Copyright 2009-2015 The freegeoip authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package apiserver
+
+import (
+	"flag"
+	"testing"
+	"time"
+)
+
+func TestCmd(t *testing.T) {
+	flag.Set("http", ":0")
+	flag.Set("db", "../testdata/db.gz")
+	flag.Set("silent", "true")
+	errc := make(chan error)
+	go func() {
+		errc <- Run()
+	}()
+	select {
+	case err := <-errc:
+		t.Fatal(err)
+	case <-time.After(time.Second):
+	}
+}

apiserver/cors.go

@@ -0,0 +1,44 @@
+// Copyright 2009-2015 The freegeoip authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package apiserver
+
+import (
+	"net/http"
+	"strings"
+)
+
+// cors is an HTTP handler for managing cross-origin resource sharing.
+// Ref: https://en.wikipedia.org/wiki/Cross-origin_resource_sharing.
+func cors(f http.Handler, origin string, methods ...string) http.Handler {
+	ms := strings.Join(methods, ", ") + ", OPTIONS"
+	md := make(map[string]struct{})
+	for _, method := range methods {
+		md[method] = struct{}{}
+	}
+	cf := func(w http.ResponseWriter, r *http.Request) {
+		orig := origin
+		if orig == "*" {
+			if ro := r.Header.Get("Origin"); ro != "" {
+				orig = ro
+			}
+		}
+		w.Header().Set("Access-Control-Allow-Origin", orig)
+		w.Header().Set("Access-Control-Allow-Methods", ms)
+		w.Header().Set("Access-Control-Allow-Credentials", "true")
+		if r.Method == "OPTIONS" {
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+		if _, exists := md[r.Method]; exists {
+			f.ServeHTTP(w, r)
+			return
+		}
+		w.Header().Set("Allow", ms)
+		http.Error(w,
+			http.StatusText(http.StatusMethodNotAllowed),
+			http.StatusMethodNotAllowed)
+	}
+	return http.HandlerFunc(cf)
+}

apiserver/cors_test.go

@@ -0,0 +1,89 @@
+// Copyright 2009-2015 The freegeoip authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package apiserver
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+)
+
+func TestCORS(t *testing.T) {
+	// set up the test server
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		io.WriteString(w, "hello world")
+	}
+	mux := http.NewServeMux()
+	mux.Handle("/", cors(http.HandlerFunc(handler), "*", "GET"))
+	ts := httptest.NewServer(mux)
+	defer ts.Close()
+	// create and issue an OPTIONS request and
+	// validate response status and headers.
+	req, err := http.NewRequest("OPTIONS", ts.URL, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Add("Origin", ts.URL)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("Unexpected response status: %s", resp.Status)
+	}
+	if resp.ContentLength != 0 {
+		t.Fatalf("Unexpected Content-Length. Want 0, have %d",
+			resp.ContentLength)
+	}
+	want := []struct {
+		Name  string
+		Value string
+	}{
+		{"Access-Control-Allow-Origin", ts.URL},
+		{"Access-Control-Allow-Methods", "GET, OPTIONS"},
+		{"Access-Control-Allow-Credentials", "true"},
+	}
+	for _, th := range want {
+		if v := resp.Header.Get(th.Name); v != th.Value {
+			t.Fatalf("Unexpected value for %q. Want %q, have %q",
+				th.Name, th.Value, v)
+		}
+	}
+	// issue a GET request and validate response headers and body
+	resp, err = http.Get(ts.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	want[0].Value = "*" // Origin
+	for _, th := range want {
+		if v := resp.Header.Get(th.Name); v != th.Value {
+			t.Fatalf("Unexpected value for %q. Want %q, have %q",
+				th.Name, th.Value, v)
+		}
+	}
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	wb := []byte("hello world")
+	if !bytes.Equal(b, wb) {
+		t.Fatalf("Unexpected response body. Want %q, have %q", b, wb)
+	}
+	// issue a POST request and validate response status
+	resp, err = http.PostForm(ts.URL, url.Values{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusMethodNotAllowed {
+		t.Fatalf("Unexpected response status: %s", resp.Status)
+	}
+}

apiserver/db.go

@@ -0,0 +1,40 @@
+// Copyright 2009-2015 The freegeoip authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package apiserver
+
+import (
+	"log"
+	"net/url"
+	"time"
+
+	"github.com/fiorix/freegeoip"
+)
+
+// openDB opens and returns the IP database.
+func openDB(dsn string, updateIntvl, maxRetryIntvl time.Duration) (db *freegeoip.DB, err error) {
+	u, err := url.Parse(dsn)
+	if err != nil || len(u.Scheme) == 0 {
+		db, err = freegeoip.Open(dsn)
+	} else {
+		db, err = freegeoip.OpenURL(dsn, updateIntvl, maxRetryIntvl)
+	}
+	return
+}
+
+// watchEvents logs and collect metrics of database events.
+func watchEvents(db *freegeoip.DB) {
+	for {
+		select {
+		case file := <-db.NotifyOpen():
+			log.Println("database loaded:", file)
+			dbEventCounter.WithLabelValues("loaded", file).Inc()
+		case err := <-db.NotifyError():
+			log.Println("database error:", err)
+			dbEventCounter.WithLabelValues("failed", err.Error()).Inc()
+		case <-db.NotifyClose():
+			return
+		}
+	}
+}

apiserver/doc.go

@@ -0,0 +1,7 @@
+// Copyright 2009-2015 The freegeoip authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package apiserver provides the freegeoip web server API, used by
+// the freegeoip daemon tool.
+package apiserver