From 1e28275e0dae7462a13d7026906f8e3a60fc451e Mon Sep 17 00:00:00 2001 From: Julien Vehent Date: Tue, 29 Dec 2015 17:33:35 -0500 Subject: [PATCH] tlsobs-runner: an api client to scan targets regularly and notify on assertions failures --- conf/runner.yaml | 102 ++++++++ tlsobs-runner/assertions.go | 129 ++++++++++ tlsobs-runner/main.go | 229 ++++++++++++++++++ tlsobs-runner/notifications.go | 94 +++++++ .../mozillaEvaluationWorker.go | 20 ++ worker/worker.go | 4 + 6 files changed, 578 insertions(+) create mode 100644 conf/runner.yaml create mode 100644 tlsobs-runner/assertions.go create mode 100644 tlsobs-runner/main.go create mode 100644 tlsobs-runner/notifications.go diff --git a/conf/runner.yaml b/conf/runner.yaml new file mode 100644 index 000000000..f71aab446 --- /dev/null +++ b/conf/runner.yaml @@ -0,0 +1,102 @@ +runs: + - targets: + - accounts.firefox.com + - addon.mozilla.org + - addons.mozilla.org + - api.accounts.firefox.com + - blocklist.addons.mozilla.org + - browserid.org + - builder.addons.mozilla.org + - controller-review.apk.firefox.com + - controller.apk.firefox.com + - diresworb.org + - find.firefox.com + - firefoxos.persona.org + - firefoxusercontent.com + - forum.addons.mozilla.org + - forums.addons.mozilla.org + - gmail.login.persona.org + - hello.firefox.com + - loads.services.mozilla.com + - location.services.mozilla.com + - login.anosrep.org + - login.mozilla.org + - login.persona.org + - marketplace-static.addons.mozilla.net + - marketplace.firefox.com + - mozillausercontent.com + - oauth.accounts.firefox.com + - persona.org + - profile.accounts.firefox.com + - push.hello.firefox.com + - push.services.mozilla.com + - push1.push.hello.firefox.com + - pyrepo.addons.mozilla.org + - readinglist.services.mozilla.com + - receiptcheck.marketplace.firefox.com + - scrypt.accounts.firefox.com + - search.services.mozilla.com + - services.addons.mozilla.org + - shavar.services.mozilla.com + - static.addons.mozilla.net + - static.login.persona.org + - static.marketplace.firefox.com + - tracking.services.mozilla.com + - updates.push.services.mozilla.com + - verifier.accounts.firefox.com + - verifier.login.persona.org + - versioncheck-bg.addons.mozilla.org + - versioncheck.addons.mozilla.org + - www.browserid.org + - www.persona.org + - yahoo.login.persona.org + assertions: + - certificate: + validity: + notAfter: ">30d" + - analysis: + analyzer: mozillaEvaluationWorker + result: '{"level": "intermediate"}' + # daily at midnight + cron: "0 0 * * *" + notifications: + email: + recipients: + - cloudsec@mozilla.com + + - targets: + - download.mozilla.org + - download-installer.cdn.mozilla.net + assertions: + - analysis: + analyzer: mozillaEvaluationWorker + result: '{"level": "old"}' + # daily at midnight + cron: "0 0 * * *" + notifications: + email: + recipients: + - cloudsec@mozilla.com + + #- targets: + # - mana.mozilla.org + # - public.etherpad-mozilla.org + # - irccloud.mozilla.com + # - discourse.mozilla-community.org + # - api.mig.mozilla.org + # assertions: + # - certificate: + # validity: + # notAfter: ">30d" + # - analysis: + # analyzer: mozillaEvaluationWorker + # result: '{"level": "intermediate"}' + # cron: "* * * * *" + # notifications: + # email: + # recipients: + # - infosec@mozilla.com + +smtp: + relay: "gator1:25" + from: "cloudsec@mozilla.com" diff --git a/tlsobs-runner/assertions.go b/tlsobs-runner/assertions.go new file mode 100644 index 000000000..ef100f7e5 --- /dev/null +++ b/tlsobs-runner/assertions.go @@ -0,0 +1,129 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. +*/ + +package main + +import ( + "fmt" + "log" + "strconv" + "time" + + "github.com/mozilla/tls-observatory/certificate" + "github.com/mozilla/tls-observatory/database" + "github.com/mozilla/tls-observatory/worker" + _ "github.com/mozilla/tls-observatory/worker/mozillaEvaluationWorker" +) + +func (r Run) AssertNotBefore(a Assertion, target string, cnb time.Time, notifchan chan Notification) { + if a.Certificate.Validity.NotBefore == "" { + return + } + nbmintime, nbmaxtime, err := parseValidity(a.Certificate.Validity.NotBefore) + if err != nil { + log.Printf("[error] failed to parse validity string %q: %v", + a.Certificate.Validity.NotBefore, err) + return + } + if cnb.Before(nbmintime) || cnb.After(nbmaxtime) { + notifchan <- Notification{ + Target: target, + Body: []byte(fmt.Sprintf(`Assertion certificate.validity.notBefore=%q failed because certificate starts on %q`, + a.Certificate.Validity.NotBefore, cnb.String())), + Conf: r.Notifications, + } + } else { + debugprint("Assertion certificate.validity.notBefore=%q passed because certificate starts on %q", + a.Certificate.Validity.NotBefore, cnb.String()) + } + return +} + +func (r Run) AssertNotAfter(a Assertion, target string, cna time.Time, notifchan chan Notification) { + if a.Certificate.Validity.NotAfter == "" { + return + } + nbmintime, nbmaxtime, err := parseValidity(a.Certificate.Validity.NotAfter) + if err != nil { + log.Printf("[error] failed to parse validity string %q: %v", + a.Certificate.Validity.NotAfter, err) + return + } + if cna.Before(nbmintime) || cna.After(nbmaxtime) { + notifchan <- Notification{ + Target: target, + Body: []byte(fmt.Sprintf(`Assertion certificate.validity.notAfter=%q failed because certificate expires on %q`, + a.Certificate.Validity.NotAfter, cna.String())), + Conf: r.Notifications, + } + } else { + debugprint("Assertion certificate.validity.notAfter=%q passed because certificate expires on %q", + a.Certificate.Validity.NotAfter, cna.String()) + } + + return +} + +func parseValidity(validity string) (mintime, maxtime time.Time, err error) { + var ( + isDays bool = false + n uint64 = 0 + ) + suffix := validity[len(validity)-1] + if suffix == 'd' { + isDays = true + suffix = 'h' + } + n, err = strconv.ParseUint(validity[1:len(validity)-1], 10, 64) + if err != nil { + return + } + if isDays { + n = n * 24 + } + duration := fmt.Sprintf("%d%c", n, suffix) + d, err := time.ParseDuration(duration) + switch validity[0] { + case '>': + mintime = time.Now().Add(d) + maxtime = time.Date(9998, time.January, 11, 11, 11, 11, 11, time.UTC) + case '<': + // modification date is older than date + mintime = time.Date(1111, time.January, 11, 11, 11, 11, 11, time.UTC) + maxtime = time.Now().Add(d) + } + debugprint("Parsed validity time with mintime '%s' and maxtime '%s'\n", + mintime.String(), maxtime.String()) + return +} + +func (r Run) AssertAnalysis(a Assertion, results database.Scan, cert certificate.Certificate, notifchan chan Notification) { + analyzer := a.Analysis.Analyzer + if analyzer == "" { + return + } + for _, ran := range results.AnalysisResults { + if ran.Analyzer != analyzer { + continue + } + if _, ok := worker.AvailableWorkers[analyzer]; !ok { + log.Printf("[error] analyzer %q not found\n", analyzer) + return + } + runner := worker.AvailableWorkers[analyzer].Runner + pass, body, err := runner.(worker.HasAssert).Assert(ran.Result, []byte(a.Analysis.Result)) + if err != nil { + log.Printf("[error] analyzer %q failed with error %v", analyzer, err) + return + } + if !pass { + notifchan <- Notification{ + Target: results.Target, + Body: body, + Conf: r.Notifications, + } + } + } +} diff --git a/tlsobs-runner/main.go b/tlsobs-runner/main.go new file mode 100644 index 000000000..e9709fec1 --- /dev/null +++ b/tlsobs-runner/main.go @@ -0,0 +1,229 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. +*/ + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "log" + "net/http" + "sync" + "time" + + "github.com/gorhill/cronexpr" + "github.com/mozilla/tls-observatory/certificate" + "github.com/mozilla/tls-observatory/database" + + "gopkg.in/yaml.v2" +) + +type Configuration struct { + Runs []Run + Smtp struct { + Relay string + From string + } +} +type Run struct { + Targets []string + Assertions []Assertion + Cron string + Notifications NotificationsConf +} + +type Assertion struct { + Certificate struct { + Validity struct { + NotBefore string + NotAfter string + } + } + Analysis struct { + Analyzer string + Result string `json:"result"` + } +} + +type NotificationsConf struct { + Irc struct { + Channels []string + } + Email struct { + Recipients []string + } +} + +var ( + cfgFile string + observatory string + debug bool + conf Configuration +) + +func main() { + flag.StringVar(&observatory, "observatory", "https://tls-observatory.services.mozilla.com", "URL of the observatory") + flag.StringVar(&cfgFile, "c", "/etc/tls-observatory/runner.yaml", "YAML configuration file") + flag.BoolVar(&debug, "debug", false, "Set debug logging") + flag.Parse() + + // load the local configuration file + fd, err := ioutil.ReadFile(cfgFile) + if err != nil { + log.Fatal(err) + } + err = yaml.Unmarshal(fd, &conf) + if err != nil { + log.Fatalf("error: %v", err) + } + exit := make(chan bool) + for i, run := range conf.Runs { + go run.start(i) + } + <-exit +} + +func (r Run) start(id int) { + for { + cexpr, err := cronexpr.Parse(r.Cron) + if err != nil { + panic(err) + } + // sleep until the next run is scheduled to happen + nrun := cexpr.Next(time.Now()) + waitduration := nrun.Sub(time.Now()) + log.Printf("[info] run %d will start at %v (in %v)", id, nrun, waitduration) + time.Sleep(waitduration) + + notifchan := make(chan Notification) + done := make(chan bool) + go processNotifications(notifchan, done) + var wg sync.WaitGroup + for _, target := range r.Targets { + debugprint("scanning target %s", target) + id, err := r.scan(target) + debugprint("got scan id %s", id) + if err != nil { + log.Printf("[error] failed to launch against %q: %v", target, err) + continue + } + wg.Add(1) + go r.evaluate(id, notifchan, &wg) + } + wg.Wait() + close(notifchan) + <-done + } +} + +type scan struct { + ID string `json:"scan_id"` +} + +func (r Run) scan(target string) (id string, err error) { + defer func() { + if e := recover(); e != nil { + err = fmt.Errorf("scan(target=%q) -> %v", e) + } + }() + resp, err := http.Post(observatory+"/api/v1/scan?target="+target, "application/json", nil) + if err != nil { + panic(err) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + panic(err) + } + var s scan + err = json.Unmarshal(body, &s) + if err != nil { + panic(err) + } + if s.ID == "" { + panic("failed to launch scan on target " + target) + } + id = s.ID + return +} + +func (r Run) evaluate(id string, notifchan chan Notification, wg *sync.WaitGroup) { + defer func() { + if e := recover(); e != nil { + log.Printf("[error] evaluate(id=%q) -> %v", id, e) + } + wg.Done() + }() + var ( + results database.Scan + cert certificate.Certificate + err error + ) + for { + resp, err := http.Get(observatory + "/api/v1/results?id=" + id) + if err != nil { + panic(err) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + panic(err) + } + err = json.Unmarshal(body, &results) + if err != nil { + panic(err) + } + if results.Complperc >= 100 { + debugprint("scan id %s completed", id) + break + } + time.Sleep(1 * time.Second) + } + debugprint("getting certificate id %d", results.Cert_id) + if !results.Has_tls && results.Cert_id < 1 { + log.Printf("[info] target %q is not TLS enabled", results.Target) + return + } + cert, err = getCert(results.Cert_id) + if err != nil { + panic(err) + } + for _, a := range r.Assertions { + r.AssertNotBefore(a, results.Target, cert.Validity.NotBefore, notifchan) + r.AssertNotAfter(a, results.Target, cert.Validity.NotAfter, notifchan) + r.AssertAnalysis(a, results, cert, notifchan) + } + return +} + +func getCert(id int64) (cert certificate.Certificate, err error) { + defer func() { + if e := recover(); e != nil { + err = fmt.Errorf("getCert(id=%q) -> %v", e) + } + }() + resp, err := http.Get(fmt.Sprintf("%s/api/v1/certificate?id=%d", observatory, id)) + if err != nil { + panic(err) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + panic(err) + } + err = json.Unmarshal(body, &cert) + if err != nil { + panic(err) + } + return +} + +func debugprint(format string, a ...interface{}) { + if debug { + log.Printf("[debug] "+format, a...) + } +} diff --git a/tlsobs-runner/notifications.go b/tlsobs-runner/notifications.go new file mode 100644 index 000000000..47022deb4 --- /dev/null +++ b/tlsobs-runner/notifications.go @@ -0,0 +1,94 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. +*/ + +package main + +import ( + "fmt" + "log" + "net/smtp" + "time" +) + +type Notification struct { + Target string `json:"target"` + Body []byte `json:"body"` + Conf NotificationsConf `json:"-"` +} + +func processNotifications(notifchan chan Notification, done chan bool) { + emailntfs := make(map[string][]byte) + ircntfs := make(map[string][]byte) + for n := range notifchan { + log.Printf("[info] received notification for target %s with body: %s", n.Target, n.Body) + for _, rcpt := range n.Conf.Email.Recipients { + var body []byte + if _, ok := emailntfs[rcpt]; ok { + body = emailntfs[rcpt] + } + emailntfs[rcpt] = []byte(fmt.Sprintf("%s\n%s: %s", body, n.Target, n.Body)) + } + for _, rcpt := range n.Conf.Irc.Channels { + var body []byte + if _, ok := ircntfs[rcpt]; ok { + body = ircntfs[rcpt] + } + ircntfs[rcpt] = []byte(fmt.Sprintf("%s\n%s: %s", body, n.Target, n.Body)) + } + } + for rcpt, body := range emailntfs { + sendMail(rcpt, body) + } + done <- true +} + +func sendMail(rcpt string, body []byte) (err error) { + defer func() { + if e := recover(); e != nil { + err = fmt.Errorf("sendMail-> %v", e) + } + }() + // Connect to the remote SMTP server. + c, err := smtp.Dial(conf.Smtp.Relay) + if err != nil { + panic(err) + } + + // Set the sender and recipient first + err = c.Mail(conf.Smtp.From) + if err != nil { + panic(err) + } + err = c.Rcpt(rcpt) + if err != nil { + panic(err) + } + // Send the email body. + wc, err := c.Data() + if err != nil { + panic(err) + } + _, err = fmt.Fprintf(wc, `From: %s +To: %s +Subject: TLS Observatory runner results +Date: %s + +%s +`, conf.Smtp.From, rcpt, time.Now().Format("Mon, 2 Jan 2006 15:04:05 -0700"), body) + if err != nil { + panic(err) + } + err = wc.Close() + if err != nil { + panic(err) + } + + // Send the QUIT command and close the connection. + err = c.Quit() + if err != nil { + panic(err) + } + return +} diff --git a/worker/mozillaEvaluationWorker/mozillaEvaluationWorker.go b/worker/mozillaEvaluationWorker/mozillaEvaluationWorker.go index c3c2e06ee..37d7a46d1 100644 --- a/worker/mozillaEvaluationWorker/mozillaEvaluationWorker.go +++ b/worker/mozillaEvaluationWorker/mozillaEvaluationWorker.go @@ -638,3 +638,23 @@ func (e eval) PrintAnalysis(r []byte) (results []string, err error) { } return } + +func (e eval) Assert(evresults, assertresults []byte) (pass bool, body []byte, err error) { + var evres, assertres EvaluationResults + err = json.Unmarshal(evresults, &evres) + if err != nil { + return + } + err = json.Unmarshal(assertresults, &assertres) + if err != nil { + return + } + if evres.Level != assertres.Level { + body = []byte(fmt.Sprintf(`Assertion mozillaEvaluationWorker.level=%q failed because measured leved is %q`, + assertres.Level, evres.Level)) + pass = false + } else { + pass = true + } + return +} diff --git a/worker/worker.go b/worker/worker.go index 692ecc245..b151124a8 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -56,3 +56,7 @@ type Worker interface { type HasAnalysisPrinter interface { PrintAnalysis([]byte) ([]string, error) } + +type HasAssert interface { + Assert([]byte, []byte) (bool, []byte, error) +}