diff --git a/CHANGELOG.md b/CHANGELOG.md index bb91e88d..459aebcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,42 @@ +## 0.12.0 (August 3, 2017) + +FEATURES: + +- Server: + + * Check versions and path existence in memory to limit DB queries + * Add log-level and log-format options + * Log with fields [GH #19] + * Add a compare API point [GH #20] + * Split types into a types package + * Add tf_versions API point + * Add version API point + * Sanitize raw SQL queries + +- UI: + + * Remove sb-admin theme + * Make charts in overview clickable, linking to search view + * Fix display bugs in state view [GH #18] + * Select first resource of first module on state view load + * Add a compare function to state view [GH #21] + * Use $routeParams instead of parsing $location.url() + * Make state view work without reloading the page + * Order resource attributes in state view + * Display long resource attributes and titles with ellipsis + * Support permalinks and fix form in search view [GH #16] + * Add tf_version filtering to search view + * Allow to clear filters in search view + * Remove unused sorting in tables + +FIXES: + +- Server: + + * Do not import non-ASCII attribute values [GH #17] + * Remove --no-sync from docker-compose.yml + + ## 0.11.0 (August 1, 2017) FEATURES: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..ecee40b5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,16 @@ +# Contributing to Terraboard + +Thank you for considering contributing to Terraboard. It’s people like you that will help make Terraboard a great tool. + +There are various ways in which you can help with this project. + + +## Find and report bugs + +By testing Terraboard, you can help find, identify and report bugs in the application. All bugs can be reported on the [GitHub project +page](https://github.com/camptocamp/terraboard/issues). + + +## Provide Pull Requests + +We welcome contributions in the form of [Pull Requests](https://github.com/camptocamp/terraboard/pulls). diff --git a/Dockerfile b/Dockerfile index d1e81734..5d623aed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,8 @@ RUN go get github.com/aws/aws-sdk-go \ github.com/Sirupsen/logrus \ github.com/hashicorp/terraform \ github.com/jinzhu/gorm github.com/lib/pq \ - github.com/jessevdk/go-flags + github.com/jessevdk/go-flags \ + github.com/pmezard/go-difflib || echo WORKDIR /go/src/github.com/camptocamp/terraboard COPY . . RUN CGO_ENABLED=1 GOOS=linux go build \ diff --git a/README.md b/README.md index 05c48b7f..b51d9c2b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Terraboard -Web Dashboard to inspect Terraform States +:chart_with_upwards_trend: A web dashboard to inspect Terraform States :earth_africa: [![Docker Pulls](https://img.shields.io/docker/pulls/camptocamp/terraboard.svg)](https://hub.docker.com/r/camptocamp/terraboard/) @@ -12,12 +12,25 @@ Web Dashboard to inspect Terraform States ## What is it? Terraboard is a web dashboard to visualize and query -[Terraform](https://terraform.io) states. +[Terraform](https://terraform.io) states. It currently features: + +- an overview page listing the most recently updated state files with their + activity +- a state page with state file details, including versions and resource + attributes +- a search interface to query resources by type, name or attributes +- a diff interface to compare state between versions + +It currently only supports S3 as a remote state backend, and dynamoDB for +retrieving lock informations. + ![Screenshot Overview](screenshots/main.png) ![Screenshot State](screenshots/state.png) +![Screenshot Compare](screenshots/compare.png) + ### Requirements @@ -32,6 +45,10 @@ requires: - `s3:ListBucketVersions` - `s3:GetObjectVersion` * A running PostgreSQL database +* If you want to retrieve lock states + [from a dynamoDB table](https://www.terraform.io/docs/backends/types/s3.html#dynamodb_table), + you need to make sure the provided AWS credentials have `dynamodb:Scan` access to that + table. ## Use with Docker @@ -42,6 +59,7 @@ $ docker run -d -p 8080:8080 \ -e AWS_ACCESS_KEY_ID= \ -e AWS_SECRET_ACCESS_KEY= \ -e AWS_BUCKET= \ + -e AWS_DYNAMODB_TABLE= \ -e DB_PASSWORD="mygreatpasswd" \ --link postgres:db \ camptocamp/terraboard:latest @@ -68,6 +86,28 @@ $ go get github.com/camptocamp/terraboard ## Development +### Architecture + +Terraboard is made of two components: + +#### A server process + +The server is written in go and runs a web server which serves: + + - the API on known access points, taking the data from the PostgreSQL + database + - the index page (from [static/index.html](static/index.html)) on all other + URLs + +The server also has a routine which regularly (every 1 minute) feeds +the PostgreSQL database from the S3 bucket. + +#### A web UI + +The UI is an AngularJS application served from `index.html`. All the UI code +can be found in the [static/](static/) directory. + + ### Testing @@ -78,10 +118,5 @@ $ docker-compose build && docker-compose up -d ### Contributing -Please report bugs on the [GitHub project -page](https://github.com/camptocamp/terraboard/issues). - -We welcome contributions in the form of [Pull -Requests](https://github.com/camptocamp/terraboard/pulls). - +See [CONTRIBUTING.md](CONTRIBUTING.md) diff --git a/api/api.go b/api/api.go index 58ae3c5a..bc1eecc4 100644 --- a/api/api.go +++ b/api/api.go @@ -6,6 +6,7 @@ import ( "io" "net/http" + "github.com/camptocamp/terraboard/compare" "github.com/camptocamp/terraboard/db" "github.com/camptocamp/terraboard/s3" "github.com/camptocamp/terraboard/util" @@ -28,6 +29,7 @@ func ListStates(w http.ResponseWriter, r *http.Request, d *db.Database) { j, err := json.Marshal(states) if err != nil { JSONError(w, "Failed to marshal states", err) + return } io.WriteString(w, string(j)) } @@ -40,6 +42,7 @@ func ListTerraformVersionsWithCount(w http.ResponseWriter, r *http.Request, d *d j, err := json.Marshal(versions) if err != nil { JSONError(w, "Failed to marshal states", err) + return } io.WriteString(w, string(j)) } @@ -57,6 +60,7 @@ func ListStateStats(w http.ResponseWriter, r *http.Request, d *db.Database) { j, err := json.Marshal(response) if err != nil { JSONError(w, "Failed to marshal states", err) + return } io.WriteString(w, string(j)) } @@ -70,6 +74,7 @@ func GetState(w http.ResponseWriter, r *http.Request, d *db.Database) { versionId, err = d.DefaultVersion(st) if err != nil { JSONError(w, "Failed to retrieve default version", err) + return } } state := d.GetState(st, versionId) @@ -77,6 +82,7 @@ func GetState(w http.ResponseWriter, r *http.Request, d *db.Database) { jState, err := json.Marshal(state) if err != nil { JSONError(w, "Failed to marshal state", err) + return } io.WriteString(w, string(jState)) } @@ -89,20 +95,46 @@ func GetStateActivity(w http.ResponseWriter, r *http.Request, d *db.Database) { jActivity, err := json.Marshal(activity) if err != nil { JSONError(w, "Failed to marshal state activity", err) + return } io.WriteString(w, string(jActivity)) } +func StateCompare(w http.ResponseWriter, r *http.Request, d *db.Database) { + w.Header().Set("Access-Control-Allow-Origin", "*") + st := util.TrimBase(r, "api/state/compare/") + query := r.URL.Query() + fromVersion := query.Get("from") + toVersion := query.Get("to") + + from := d.GetState(st, fromVersion) + to := d.GetState(st, toVersion) + compare, err := compare.Compare(from, to) + if err != nil { + JSONError(w, "Failed to compare state versions", err) + return + } + + jCompare, err := json.Marshal(compare) + if err != nil { + JSONError(w, "Failed to marshal state compare", err) + return + } + io.WriteString(w, string(jCompare)) +} + func GetLocks(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") locks, err := s3.GetLocks() if err != nil { JSONError(w, "Failed to get locks", err) + return } j, err := json.Marshal(locks) if err != nil { JSONError(w, "Failed to marshal locks", err) + return } io.WriteString(w, string(j)) } @@ -121,6 +153,7 @@ func SearchAttribute(w http.ResponseWriter, r *http.Request, d *db.Database) { j, err := json.Marshal(response) if err != nil { JSONError(w, "Failed to marshal json", err) + return } io.WriteString(w, string(j)) } @@ -131,6 +164,7 @@ func ListResourceTypes(w http.ResponseWriter, r *http.Request, d *db.Database) { j, err := json.Marshal(result) if err != nil { JSONError(w, "Failed to marshal json", err) + return } io.WriteString(w, string(j)) } @@ -141,6 +175,7 @@ func ListResourceTypesWithCount(w http.ResponseWriter, r *http.Request, d *db.Da j, err := json.Marshal(result) if err != nil { JSONError(w, "Failed to marshal json", err) + return } io.WriteString(w, string(j)) } @@ -151,6 +186,7 @@ func ListResourceNames(w http.ResponseWriter, r *http.Request, d *db.Database) { j, err := json.Marshal(result) if err != nil { JSONError(w, "Failed to marshal json", err) + return } io.WriteString(w, string(j)) } @@ -162,6 +198,18 @@ func ListAttributeKeys(w http.ResponseWriter, r *http.Request, d *db.Database) { j, err := json.Marshal(result) if err != nil { JSONError(w, "Failed to marshal json", err) + return + } + io.WriteString(w, string(j)) +} + +func ListTfVersions(w http.ResponseWriter, r *http.Request, d *db.Database) { + w.Header().Set("Access-Control-Allow-Origin", "*") + result, _ := d.ListTfVersions() + j, err := json.Marshal(result) + if err != nil { + JSONError(w, "Failed to marshal json", err) + return } io.WriteString(w, string(j)) } diff --git a/compare/compare.go b/compare/compare.go new file mode 100644 index 00000000..c50762df --- /dev/null +++ b/compare/compare.go @@ -0,0 +1,191 @@ +package compare + +import ( + "fmt" + "sort" + "strings" + + log "github.com/Sirupsen/logrus" + "github.com/camptocamp/terraboard/types" + "github.com/pmezard/go-difflib/difflib" +) + +// Return all resources of a state +func stateResources(state types.State) (res []string) { + for _, m := range state.Modules { + for _, r := range m.Resources { + res = append(res, fmt.Sprintf("%s.%s.%s", m.Path, r.Type, r.Name)) + } + } + return +} + +// Returns elements only in s1 +func sliceDiff(s1, s2 []string) (diff []string) { + for _, e1 := range s1 { + found := false + for _, e2 := range s2 { + if e1 == e2 { + found = true + break + } + } + + if !found { + diff = append(diff, e1) + } + } + return +} + +// Returns elements in both s1 and s2 +func sliceInter(s1, s2 []string) (inter []string) { + for _, e1 := range s1 { + for _, e2 := range s2 { + if e1 == e2 { + inter = append(inter, e1) + break + } + } + } + return +} + +func getResource(state types.State, key string) (res types.Resource) { + for _, m := range state.Modules { + if strings.HasPrefix(key, m.Path) { + for _, r := range m.Resources { + if key == fmt.Sprintf("%s.%s.%s", m.Path, r.Type, r.Name) { + return r + } + } + } else { + continue + } + } + return +} + +// Return all attributes of a resource +func resourceAttributes(res types.Resource) (attrs []string) { + for _, a := range res.Attributes { + attrs = append(attrs, a.Key) + } + sort.Strings(attrs) + return +} + +func getResourceAttribute(res types.Resource, key string) (val string) { + for _, attr := range res.Attributes { + if attr.Key == key { + return attr.Value + } + } + return +} + +func formatResource(res types.Resource) (out string) { + out = fmt.Sprintf("resource \"%s\" \"%s\" {\n", res.Type, res.Name) + for _, attr := range resourceAttributes(res) { + out += fmt.Sprintf(" %s = \"%s\"\n", attr, getResourceAttribute(res, attr)) + } + out += "}\n" + + return +} + +func stateInfo(state types.State) (info string) { + return fmt.Sprintf("%s (%s)", state.Path, state.Version.LastModified) +} + +// Compare a resource in two states +func compareResource(st1, st2 types.State, key string) (comp types.ResourceDiff) { + res1 := getResource(st1, key) + attrs1 := resourceAttributes(res1) + res2 := getResource(st2, key) + attrs2 := resourceAttributes(res2) + + // Only in old + comp.OnlyInOld = make(map[string]string) + for _, attr := range sliceDiff(attrs1, attrs2) { + comp.OnlyInOld[attr] = getResourceAttribute(res1, attr) + } + + // Only in new + comp.OnlyInNew = make(map[string]string) + for _, attr := range sliceDiff(attrs2, attrs1) { + comp.OnlyInNew[attr] = getResourceAttribute(res2, attr) + } + + // Compute unified diff + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines(formatResource(res1)), + B: difflib.SplitLines(formatResource(res2)), + FromFile: stateInfo(st1), + ToFile: stateInfo(st2), + Context: 3, + Eol: "\n", + } + result, _ := difflib.GetUnifiedDiffString(diff) + comp.UnifiedDiff = result + + return +} + +func Compare(from, to types.State) (comp types.StateCompare, err error) { + if from.Path == "" { + err = fmt.Errorf("from version is unknown") + return + } + fromResources := stateResources(from) + comp.Stats.From = types.StateInfo{ + Path: from.Path, + VersionID: from.Version.VersionID, + ResourceCount: len(fromResources), + TFVersion: from.TFVersion, + Serial: from.Serial, + } + + if to.Path == "" { + err = fmt.Errorf("to version is unknown") + return + } + toResources := stateResources(to) + comp.Stats.To = types.StateInfo{ + Path: to.Path, + VersionID: to.Version.VersionID, + ResourceCount: len(toResources), + TFVersion: to.TFVersion, + Serial: to.Serial, + } + + // OnlyInOld + onlyInOld := sliceDiff(fromResources, toResources) + comp.Differences.OnlyInOld = make(map[string]string) + for _, r := range onlyInOld { + comp.Differences.OnlyInOld[r] = formatResource(getResource(from, r)) + } + + // OnlyInNew + onlyInNew := sliceDiff(toResources, fromResources) + comp.Differences.OnlyInNew = make(map[string]string) + for _, r := range onlyInNew { + comp.Differences.OnlyInNew[r] = formatResource(getResource(to, r)) + } + comp.Differences.InBoth = sliceInter(toResources, fromResources) + comp.Differences.ResourceDiff = make(map[string]types.ResourceDiff) + + for _, r := range comp.Differences.InBoth { + if c := compareResource(to, from, r); c.UnifiedDiff != "" { + comp.Differences.ResourceDiff[r] = c + } + } + + log.WithFields(log.Fields{ + "path": from.Path, + "from": from.Version.VersionID, + "to": to.Version.VersionID, + }).Info("Comparing state versions") + + return +} diff --git a/config/config.go b/config/config.go index df6513a2..b0df6d9a 100644 --- a/config/config.go +++ b/config/config.go @@ -1,9 +1,11 @@ package config import ( + "errors" "fmt" "os" + log "github.com/Sirupsen/logrus" "github.com/jessevdk/go-flags" ) @@ -13,6 +15,11 @@ type Config struct { Port int `short:"p" long:"port" description:"Port to listen on." default:"8080"` + Log struct { + Level string `short:"l" long:"log-level" description:"Set log level ('debug', 'info', 'warn', 'error', 'fatal', 'panic')." env:"TERRABOARD_LOG_LEVEL" default:"info"` + Format string `long:"log-format" description:"Set log format ('plain', 'json')." env:"TERRABOARD_LOG_FORMAT" default:"plain"` + } `group:"Logging Options"` + DB struct { Host string `long:"db-host" env:"DB_HOST" description:"Database host." default:"db"` User string `long:"db-user" env:"DB_USER" description:"Database user." default:"gorm"` @@ -42,3 +49,34 @@ func LoadConfig(version string) *Config { return &c } + +func (c Config) SetupLogging() (err error) { + switch c.Log.Level { + case "debug": + log.SetLevel(log.DebugLevel) + case "info": + log.SetLevel(log.InfoLevel) + case "warn": + log.SetLevel(log.WarnLevel) + case "error": + log.SetLevel(log.ErrorLevel) + case "fatal": + log.SetLevel(log.FatalLevel) + case "panic": + log.SetLevel(log.PanicLevel) + default: + errMsg := fmt.Sprintf("Wrong log level '%v'", c.Log.Level) + return errors.New(errMsg) + } + + switch c.Log.Format { + case "plain": + case "json": + log.SetFormatter(&log.JSONFormatter{}) + default: + errMsg := fmt.Sprintf("Wrong log format '%v'", c.Log.Format) + return errors.New(errMsg) + } + + return +} diff --git a/db/db.go b/db/db.go index b3853480..a5ef02ac 100644 --- a/db/db.go +++ b/db/db.go @@ -1,15 +1,15 @@ package db import ( - "database/sql" + "encoding/base64" "fmt" "net/url" "strconv" "strings" - "time" log "github.com/Sirupsen/logrus" "github.com/aws/aws-sdk-go/service/s3" + "github.com/camptocamp/terraboard/types" "github.com/hashicorp/terraform/terraform" "github.com/jinzhu/gorm" @@ -20,47 +20,9 @@ type Database struct { *gorm.DB } -type Version struct { - ID uint `sql:"AUTO_INCREMENT" gorm:"primary_key" json:"-"` - VersionID string `gorm:"index" json:"version_id"` - LastModified time.Time `json:"last_modified"` -} - -type State struct { - gorm.Model `json:"-"` - Path string `gorm:"index" json:"path"` - Version Version `json:"version"` - VersionID sql.NullInt64 `gorm:"index" json:"-"` - TFVersion string `json:"terraform_version"` - Serial int64 `json:"serial"` - Modules []Module `json:"modules"` -} - -type Module struct { - ID uint `sql:"AUTO_INCREMENT" gorm:"primary_key" json:"-"` - StateID sql.NullInt64 `gorm:"index" json:"-"` - Path string `json:"path"` - Resources []Resource `json:"resources"` -} - -type Resource struct { - ID uint `sql:"AUTO_INCREMENT" gorm:"primary_key" json:"-"` - ModuleID sql.NullInt64 `gorm:"index" json:"-"` - Type string `gorm:"index" json:"type"` - Name string `gorm:"index" json:"name"` - Attributes []Attribute `json:"attributes"` -} - -type Attribute struct { - ID uint `sql:"AUTO_INCREMENT" gorm:"primary_key" json:"-"` - ResourceID sql.NullInt64 `gorm:"index" json:"-"` - Key string `gorm:"index" json:"key"` - Value string `gorm:"index" json:"value"` -} - var pageSize = 20 -func Init(host, user, dbname, password string) *Database { +func Init(host, user, dbname, password, logLevel string) *Database { var err error connString := fmt.Sprintf("host=%s user=%s dbname=%s sslmode=disable password=%s", host, user, dbname, password) db, err := gorm.Open("postgres", connString) @@ -69,16 +31,18 @@ func Init(host, user, dbname, password string) *Database { } log.Infof("Automigrate") - db.AutoMigrate(&Version{}, &State{}, &Module{}, &Resource{}, &Attribute{}) + db.AutoMigrate(&types.Version{}, &types.State{}, &types.Module{}, &types.Resource{}, &types.Attribute{}) - db.LogMode(true) + if logLevel == "debug" { + db.LogMode(true) + } return &Database{db} } -func (db *Database) stateS3toDB(state *terraform.State, path string, versionId string) (st State) { - var version Version - db.First(&version, Version{VersionID: versionId}) - st = State{ +func (db *Database) stateS3toDB(state *terraform.State, path string, versionId string) (st types.State) { + var version types.Version + db.First(&version, types.Version{VersionID: versionId}) + st = types.State{ Path: path, Version: version, TFVersion: state.TFVersion, @@ -86,17 +50,24 @@ func (db *Database) stateS3toDB(state *terraform.State, path string, versionId s } for _, m := range state.Modules { - mod := Module{ + mod := types.Module{ Path: strings.Join(m.Path, "/"), } for n, r := range m.Resources { - res := Resource{ + res := types.Resource{ Type: r.Type, Name: n, } for k, v := range r.Primary.Attributes { - res.Attributes = append(res.Attributes, Attribute{ + if !isASCII(v) { + log.WithFields(log.Fields{ + "key": k, + "value_base64": base64.StdEncoding.EncodeToString([]byte(v)), + }).Info("Attribute has non-ASCII value, skipping") + continue + } + res.Attributes = append(res.Attributes, types.Attribute{ Key: k, Value: v, }) @@ -109,6 +80,15 @@ func (db *Database) stateS3toDB(state *terraform.State, path string, versionId s return } +func isASCII(s string) bool { + for _, c := range s { + if c > 127 { + return false + } + } + return true +} + func (db *Database) InsertState(path string, versionId string, state *terraform.State) error { st := db.stateS3toDB(state, path, versionId) db.Create(&st) @@ -116,30 +96,30 @@ func (db *Database) InsertState(path string, versionId string, state *terraform. } func (db *Database) InsertVersion(version *s3.ObjectVersion) error { - var v Version - db.FirstOrCreate(&v, Version{ + var v types.Version + db.FirstOrCreate(&v, types.Version{ VersionID: *version.VersionId, LastModified: *version.LastModified, }) return nil } -func (db *Database) GetState(path, versionId string) (state State) { +func (db *Database) GetState(path, versionId string) (state types.State) { db.Joins("JOIN versions on states.version_id=versions.id"). Preload("Version").Preload("Modules").Preload("Modules.Resources").Preload("Modules.Resources.Attributes"). Find(&state, "states.path = ? AND versions.version_id = ?", path, versionId) return } -func (db *Database) GetStateActivity(path string) (states []StateStat) { +func (db *Database) GetStateActivity(path string) (states []types.StateStat) { sql := "SELECT t.path, t.serial, t.tf_version, t.version_id, t.last_modified, count(resources.*) as resource_count" + - fmt.Sprintf(" FROM (SELECT states.id, states.path, states.serial, states.tf_version, versions.version_id, versions.last_modified FROM states JOIN versions ON versions.id = states.version_id WHERE states.path = '%s' ORDER BY states.path, versions.last_modified ASC) t", path) + + " FROM (SELECT states.id, states.path, states.serial, states.tf_version, versions.version_id, versions.last_modified FROM states JOIN versions ON versions.id = states.version_id WHERE states.path = ? ORDER BY states.path, versions.last_modified ASC) t" + " JOIN modules ON modules.state_id = t.id" + " JOIN resources ON resources.module_id = modules.id" + " GROUP BY t.path, t.serial, t.tf_version, t.version_id, t.last_modified" + " ORDER BY last_modified ASC" - db.Raw(sql).Find(&states) + db.Raw(sql, path).Find(&states) return } @@ -155,20 +135,10 @@ func (db *Database) KnownVersions() (versions []string) { return } -type SearchResult struct { - Path string `gorm:"column:path" json:"path"` - VersionId string `gorm:"column:version_id" json:"version_id"` - TFVersion string `gorm:"column:tf_version" json:"tf_version"` - Serial int64 `gorm:"column:serial" json:"serial"` - ModulePath string `gorm:"column:module_path" json:"module_path"` - ResourceType string `gorm:"column:type" json:"resource_type"` - ResourceName string `gorm:"column:name" json:"resource_name"` - AttributeKey string `gorm:"column:key" json:"attribute_key"` - AttributeValue string `gorm:"column:value" json:"attribute_value"` -} - -func (db *Database) SearchAttribute(query url.Values) (results []SearchResult, page int, total int) { - log.Infof("Searching for attribute with query=%v", query) +func (db *Database) SearchAttribute(query url.Values) (results []types.SearchResult, page int, total int) { + log.WithFields(log.Fields{ + "query": query, + }).Info("Searching for attribute with query") targetVersion := string(query.Get("versionid")) @@ -185,33 +155,43 @@ func (db *Database) SearchAttribute(query url.Values) (results []SearchResult, p " JOIN attributes ON resources.id = attributes.resource_id" var where []string + var params []interface{} if targetVersion != "" && targetVersion != "*" { // filter by version unless we want all (*) or most recent ("") - where = append(where, fmt.Sprintf("states.version_id = '%s'", targetVersion)) + where = append(where, "states.version_id = ?") + params = append(params, targetVersion) } - if v := query.Get("type"); string(v) != "" { - where = append(where, fmt.Sprintf("resources.type LIKE '%s'", fmt.Sprintf("%%%s%%", string(v)))) + if v := string(query.Get("type")); v != "" { + where = append(where, "resources.type LIKE ?") + params = append(params, fmt.Sprintf("%%%s%%", v)) } - if v := query.Get("name"); string(v) != "" { - where = append(where, fmt.Sprintf("resources.name LIKE '%s'", fmt.Sprintf("%%%s%%", v))) + if v := string(query.Get("name")); v != "" { + where = append(where, "resources.name LIKE ?") + params = append(params, fmt.Sprintf("%%%s%%", v)) } - if v := query.Get("key"); string(v) != "" { - where = append(where, fmt.Sprintf("attributes.key LIKE '%s'", fmt.Sprintf("%%%s%%", v))) + if v := string(query.Get("key")); v != "" { + where = append(where, "attributes.key LIKE ?") + params = append(params, fmt.Sprintf("%%%s%%", v)) } - if v := query.Get("value"); string(v) != "" { - where = append(where, fmt.Sprintf("attributes.value LIKE '%s'", fmt.Sprintf("%%%s%%", v))) + if v := string(query.Get("value")); v != "" { + where = append(where, "attributes.value LIKE ?") + params = append(params, fmt.Sprintf("%%%s%%", v)) + } + + if v := query.Get("tf_version"); string(v) != "" { + where = append(where, fmt.Sprintf("states.tf_version LIKE '%s'", fmt.Sprintf("%%%s%%", v))) } if len(where) > 0 { - sqlQuery += fmt.Sprintf(" WHERE %s", strings.Join(where, " AND ")) + sqlQuery += " WHERE " + strings.Join(where, " AND ") } // Count everything - row := db.Raw("SELECT count(*)" + sqlQuery).Row() + row := db.Raw("SELECT count(*)"+sqlQuery, params...).Row() row.Scan(&total) // Now get results @@ -219,18 +199,36 @@ func (db *Database) SearchAttribute(query url.Values) (results []SearchResult, p sql := "SELECT states.path, states.version_id, states.tf_version, states.serial, modules.path as module_path, resources.type, resources.name, attributes.key, attributes.value" + sqlQuery + " ORDER BY states.path, states.serial, modules.path, resources.type, resources.name, attributes.key" + - fmt.Sprintf(" LIMIT %v", pageSize) + " LIMIT ?" + + params = append(params, pageSize) if v := string(query.Get("page")); v != "" { page, _ = strconv.Atoi(v) // TODO: err o := (page - 1) * pageSize - sql += fmt.Sprintf(" OFFSET %v", o) + sql += " OFFSET ?" + params = append(params, o) } else { page = 1 } - db.Raw(sql).Find(&results) + db.Raw(sql, params...).Find(&results) + + return +} +func (db *Database) ListStatesVersions() (statesVersions map[string][]string) { + rows, _ := db.Table("states"). + Joins("JOIN versions ON versions.id = states.version_id"). + Select("states.path, versions.version_id").Rows() + defer rows.Close() + statesVersions = make(map[string][]string) + for rows.Next() { + var path string + var versionId string + rows.Scan(&path, &versionId) + statesVersions[versionId] = append(statesVersions[versionId], path) + } return } @@ -247,7 +245,10 @@ func (db *Database) ListStates() (states []string) { func (db *Database) ListTerraformVersionsWithCount(query url.Values) (results []map[string]string, err error) { orderBy := string(query.Get("orderBy")) - sql := "SELECT t.tf_version, COUNT(*) FROM (SELECT DISTINCT ON(states.path) states.id, states.path, states.serial, states.tf_version, versions.version_id, versions.last_modified FROM states JOIN versions ON versions.id = states.version_id ORDER BY states.path, versions.last_modified DESC) t GROUP BY t.tf_version ORDER BY " + sql := "SELECT t.tf_version, COUNT(*)" + + " FROM (SELECT DISTINCT ON(states.path) states.id, states.path, states.serial, states.tf_version, versions.version_id, versions.last_modified" + + " FROM states JOIN versions ON versions.id = states.version_id ORDER BY states.path, versions.last_modified DESC) t" + + " GROUP BY t.tf_version ORDER BY " if orderBy == "version" { sql += "string_to_array(t.tf_version, '.')::int[] DESC" @@ -273,16 +274,7 @@ func (db *Database) ListTerraformVersionsWithCount(query url.Values) (results [] return } -type StateStat struct { - Path string `json:"path"` - TFVersion string `json:"terraform_version"` - Serial int64 `json:"serial"` - VersionID string `json:"version_id"` - LastModified time.Time `json:"last_modified"` - ResourceCount int `json:"resource_count"` -} - -func (db *Database) ListStateStats(query url.Values) (states []StateStat, page int, total int) { +func (db *Database) ListStateStats(query url.Values) (states []types.StateStat, page int, total int) { row := db.Table("states").Select("count(DISTINCT path)").Row() row.Scan(&total) @@ -300,9 +292,9 @@ func (db *Database) ListStateStats(query url.Values) (states []StateStat, page i " GROUP BY t.path, t.serial, t.tf_version, t.version_id, t.last_modified" + " ORDER BY last_modified DESC" + " LIMIT 20" + - fmt.Sprintf(" OFFSET %v", offset) + " OFFSET ?" - db.Raw(sql).Find(&states) + db.Raw(sql, offset).Find(&states) return } @@ -323,7 +315,8 @@ func (db *Database) listField(table, field string) (results []string, err error) } func (db *Database) listFieldWithCount(table, field string) (results []map[string]string, err error) { - rows, err := db.Table(table).Select(fmt.Sprintf("%s, COUNT(*)", field)).Group(field).Order("count DESC").Rows() + rows, err := db.Table(table).Select("?, COUNT(*)", field). + Group(field).Order("count DESC").Rows() defer rows.Close() if err != nil { return results, err @@ -347,8 +340,16 @@ func (db *Database) ListResourceTypes() ([]string, error) { } func (db *Database) ListResourceTypesWithCount() (results []map[string]string, err error) { - sql := "SELECT resources.type, COUNT(*) FROM (SELECT DISTINCT ON(states.path) states.id, states.path, states.serial, states.tf_version, versions.version_id, versions.last_modified FROM states JOIN versions ON versions.id = states.version_id ORDER BY states.path, versions.last_modified DESC) t JOIN modules ON modules.state_id = t.id JOIN resources ON resources.module_id = modules.id GROUP BY resources.type ORDER BY count DESC" - + sql := "SELECT resources.type, COUNT(*)" + + " FROM (SELECT DISTINCT ON(states.path) states.id, states.path, states.serial, states.tf_version, versions.version_id, versions.last_modified" + + " FROM states" + + " JOIN versions ON versions.id = states.version_id" + + " ORDER BY states.path, versions.last_modified DESC) t" + + " JOIN modules ON modules.state_id = t.id" + + " JOIN resources ON resources.module_id = modules.id" + + " GROUP BY resources.type" + + " ORDER BY count DESC" + rows, err := db.Raw(sql).Rows() defer rows.Close() if err != nil { @@ -371,9 +372,13 @@ func (db *Database) ListResourceNames() ([]string, error) { return db.listField("resources", "name") } +func (db *Database) ListTfVersions() ([]string, error) { + return db.listField("states", "tf_version") +} + func (db *Database) ListAttributeKeys(resourceType string) (results []string, err error) { query := db.Table("attributes"). - Select(fmt.Sprintf("DISTINCT %s", "key")). + Select("DISTINCT key"). Joins("JOIN resources ON attributes.resource_id = resources.id") if resourceType != "" { @@ -400,9 +405,9 @@ func (db *Database) DefaultVersion(path string) (version string, err error) { " (SELECT states.path, max(states.serial) as mx FROM states GROUP BY states.path) t" + " JOIN states ON t.path = states.path AND t.mx = states.serial" + " JOIN versions on states.version_id=versions.id" + - fmt.Sprintf(" WHERE states.path = '%s'", path) + " WHERE states.path = ?" - row := db.Raw(sqlQuery).Row() + row := db.Raw(sqlQuery, path).Row() row.Scan(&version) return } diff --git a/docker-compose.yml b/docker-compose.yml index 949eafe9..5dd1deed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,6 @@ services: context: . dockerfile: Dockerfile image: camptocamp/terraboard:devel - command: --no-sync ports: - 80:8080 environment: diff --git a/main.go b/main.go index c0569a9c..3e65f7d7 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "io" "io/ioutil" @@ -39,6 +40,19 @@ func handleWithDB(apiF func(w http.ResponseWriter, r *http.Request, d *db.Databa }) } +func isKnownStateVersion(statesVersions map[string][]string, versionId, path string) bool { + if v, ok := statesVersions[versionId]; ok { + for _, s := range v { + if s == path { + return true + } + } + return false + } else { + return false + } +} + // Refresh the DB from S3 // This should be the only direct bridge between S3 and the DB func refreshDB(d *db.Database) { @@ -46,23 +60,40 @@ func refreshDB(d *db.Database) { log.Infof("Refreshing DB from S3") states, err := s3.GetStates() if err != nil { - log.Errorf("Failed to build cache: %s", err) + log.WithFields(log.Fields{ + "error": err, + }).Error("Failed to retrieve states from S3. Retrying in 1 minute.") + time.Sleep(1 * time.Minute) + continue } + statesVersions := d.ListStatesVersions() for _, st := range states { versions, _ := s3.GetVersions(st) for _, v := range versions { - d.InsertVersion(v) + if _, ok := statesVersions[*v.VersionId]; ok { + log.WithFields(log.Fields{ + "version_id": *v.VersionId, + }).Debug("Version is already in the database, skipping") + } else { + d.InsertVersion(v) + } - s := d.GetState(st, *v.VersionId) - if s.Path == st { - log.Infof("State %s/%s is already in the DB, skipping", st, *v.VersionId) + if isKnownStateVersion(statesVersions, *v.VersionId, st) { + log.WithFields(log.Fields{ + "path": st, + "version_id": *v.VersionId, + }).Debug("State is already in the database, skipping") continue } state, _ := s3.GetState(st, *v.VersionId) d.InsertState(st, *v.VersionId, state) if err != nil { - log.Errorf("Failed to insert state %s/%s: %v", st, *v.VersionId, err) + log.WithFields(log.Fields{ + "path": st, + "version_id": *v.VersionId, + "error": err, + }).Error("Failed to insert state in the database") } } } @@ -73,19 +104,39 @@ func refreshDB(d *db.Database) { var version = "undefined" +func getVersion(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + + j, err := json.Marshal(map[string]string{ + "version": version, + "copyright": "Copyright © 2017 Camptocamp", + }) + if err != nil { + api.JSONError(w, "Failed to marshal version", err) + return + } + io.WriteString(w, string(j)) +} + // Main func main() { c := config.LoadConfig(version) log.Infof("Terraboard v%s is starting...", version) + err := c.SetupLogging() + if err != nil { + log.Fatal(err) + } + // Set up S3 s3.Setup(c) // Set up the DB and start S3->DB sync database := db.Init( c.DB.Host, c.DB.User, - c.DB.Name, c.DB.Password) + c.DB.Name, c.DB.Password, + c.Log.Level) if c.DB.NoSync { log.Infof("Not syncing database, as requested.") } else { @@ -101,17 +152,20 @@ func main() { http.Handle(util.AddBase("static/"), http.StripPrefix(util.AddBase("static"), staticFs)) // Handle API points + http.HandleFunc(util.AddBase("api/version"), getVersion) http.HandleFunc(util.AddBase("api/states"), handleWithDB(api.ListStates, database)) http.HandleFunc(util.AddBase("api/states/stats"), handleWithDB(api.ListStateStats, database)) http.HandleFunc(util.AddBase("api/states/tfversion/count"), handleWithDB(api.ListTerraformVersionsWithCount, database)) http.HandleFunc(util.AddBase("api/state/"), handleWithDB(api.GetState, database)) http.HandleFunc(util.AddBase("api/state/activity/"), handleWithDB(api.GetStateActivity, database)) + http.HandleFunc(util.AddBase("api/state/compare/"), handleWithDB(api.StateCompare, database)) http.HandleFunc(util.AddBase("api/locks"), api.GetLocks) http.HandleFunc(util.AddBase("api/search/attribute"), handleWithDB(api.SearchAttribute, database)) http.HandleFunc(util.AddBase("api/resource/types"), handleWithDB(api.ListResourceTypes, database)) http.HandleFunc(util.AddBase("api/resource/types/count"), handleWithDB(api.ListResourceTypesWithCount, database)) http.HandleFunc(util.AddBase("api/resource/names"), handleWithDB(api.ListResourceNames, database)) http.HandleFunc(util.AddBase("api/attribute/keys"), handleWithDB(api.ListAttributeKeys, database)) + http.HandleFunc(util.AddBase("api/tf_versions"), handleWithDB(api.ListTfVersions, database)) // Start server log.Fatal(http.ListenAndServe(fmt.Sprintf(":%v", c.Port), nil)) diff --git a/s3/s3.go b/s3/s3.go index ce434194..92f50bc1 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -113,7 +113,10 @@ func GetVersions(prefix string) (versions []*s3.ObjectVersion, err error) { } func GetState(st, versionId string) (state *terraform.State, err error) { - log.Infof("Retrieving %s/%s from S3", st, versionId) + log.WithFields(log.Fields{ + "path": st, + "version_id": versionId, + }).Info("Retrieving state from S3") input := &s3.GetObjectInput{ Bucket: aws.String(bucket), Key: aws.String(st), @@ -123,7 +126,11 @@ func GetState(st, versionId string) (state *terraform.State, err error) { } result, err := svc.GetObjectWithContext(context.Background(), input) if err != nil { - log.Errorf("Error retrieving %s/%s from S3: %v", st, versionId, err) + log.WithFields(log.Fields{ + "path": st, + "version_id": versionId, + "error": err, + }).Error("Error retrieving state from S3") errObj := make(map[string]string) errObj["error"] = fmt.Sprintf("State file not found: %v", st) errObj["details"] = fmt.Sprintf("%v", err) @@ -134,7 +141,11 @@ func GetState(st, versionId string) (state *terraform.State, err error) { content, err := ioutil.ReadAll(result.Body) if err != nil { - log.Errorf("Error reading %s/%s from S3: %v", st, versionId, err) + log.WithFields(log.Fields{ + "path": st, + "version_id": versionId, + "error": err, + }).Error("Error reading state from S3") errObj := make(map[string]string) errObj["error"] = fmt.Sprintf("Failed to read S3 response: %v", st) errObj["details"] = fmt.Sprintf("%v", err) diff --git a/screenshots/compare.png b/screenshots/compare.png new file mode 100644 index 00000000..c9d15f1d Binary files /dev/null and b/screenshots/compare.png differ diff --git a/screenshots/main.png b/screenshots/main.png index 83b45ed3..f91026e7 100644 Binary files a/screenshots/main.png and b/screenshots/main.png differ diff --git a/static/footer.html b/static/footer.html new file mode 100644 index 00000000..703fbd32 --- /dev/null +++ b/static/footer.html @@ -0,0 +1,11 @@ +
+
+ + +
+
+ diff --git a/static/index.html b/static/index.html index a6ba78a0..71139685 100644 --- a/static/index.html +++ b/static/index.html @@ -9,14 +9,12 @@ - - @@ -28,10 +26,16 @@ + + + + +
+
diff --git a/static/main.html b/static/main.html index 75d4aaf0..da1fd981 100644 --- a/static/main.html +++ b/static/main.html @@ -1,14 +1,34 @@
- + +

Resource types

- + +

Terraform versions

- + +

States locked

@@ -22,34 +42,19 @@

States locked

- - Path - - + Path - - TF Version - - + TF Version - - Serial - - + Serial - - Time - - + Time - - Resources - - + Resources Activity diff --git a/static/navbar.html b/static/navbar.html index 71508020..a86d5e50 100644 --- a/static/navbar.html +++ b/static/navbar.html @@ -1,4 +1,4 @@ -
+
@@ -9,14 +9,21 @@ - - Camptocamp - + Terraboard
+
diff --git a/static/search.html b/static/search.html index 79e240ba..0fe07a30 100644 --- a/static/search.html +++ b/static/search.html @@ -1,63 +1,91 @@ -
-
+
+
+ +
+
- - {{$parent.resType}} - - {{k}} - -
-
+
- - {{$parent.resID}} - - {{k}} - -
-
+
- - {{$parent.attrKey}} - - {{k}} - - -
-
-
-
-
- +
+
- +
+
+
+
+ + {{$parent.tfVersion}} + + {{k}} + + +
+
+ + {{$parent.resType}} + + {{k}} + + +
+
+ + {{$parent.resID}} + + {{k}} + + +
+
+ + {{$parent.attrKey}} + + {{k}} + + +
+
+ +
+ +
+