Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Filebeat][New Input] Http Input #18298

Merged
merged 29 commits into from
May 25, 2020
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
0a7972c
MVP for http input
P1llus May 5, 2020
c985208
adding temp error message on event send failure
P1llus May 5, 2020
5102356
updated comment that was there from old input
P1llus May 5, 2020
7c824c9
modify config
P1llus May 5, 2020
ebb8ed1
Merge branch 'master' into http_input_module
P1llus May 6, 2020
c94b6b9
changing default config
P1llus May 6, 2020
aaafd65
cleaning up the code and refactor checks to its own functions
P1llus May 7, 2020
458dcb5
mage fmt
P1llus May 7, 2020
f51160e
Changed name from httpinput to http_endpoint, moved http server to it…
P1llus May 9, 2020
1ccb3c2
updated code based on PR comments. Changed clientauth after confirmin…
P1llus May 12, 2020
c185b48
change packagename
P1llus May 12, 2020
72a8652
change packagename
P1llus May 12, 2020
7db39a9
updated code to be more idiomatic with comments from noemi
P1llus May 14, 2020
04be474
removing variable declaration to be more idiomatic
P1llus May 14, 2020
aa27ac9
adding basic test, currently not working
P1llus May 15, 2020
4d2cc8b
forgot to add method validation
P1llus May 15, 2020
07c00d6
make ALLL the tests!
P1llus May 15, 2020
a59503e
added changes from PR comments and mage fmt
P1llus May 17, 2020
23a5c52
small change on response header and adding documentation
P1llus May 17, 2020
219eb08
including new input docs
P1llus May 18, 2020
7f6abed
changing to older string formatting to support python version in nose…
P1llus May 18, 2020
f6032a4
wrong doc reference?
P1llus May 18, 2020
e188040
removing response header, updating docs and modify based on PR comments
P1llus May 19, 2020
94d2e27
needed to remove responseheader from defaultconf as well
P1llus May 19, 2020
111b1e3
mage fmt update
P1llus May 19, 2020
fce4e8e
validation for response body added
P1llus May 25, 2020
c9b9a81
Mage fmt update
P1llus May 25, 2020
d17e5d0
updated changelog
P1llus May 25, 2020
cffcade
Merge branch 'master' into http_input_module
P1llus May 25, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
cleaning up the code and refactor checks to its own functions
  • Loading branch information
P1llus committed May 7, 2020
commit aaafd65ed87940ead55e1ddfce592193e28958d6
1 change: 1 addition & 0 deletions x-pack/filebeat/input/httpinput/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,5 +37,6 @@ func defaultConfig() config {
c.ListenAddress = ""
c.ListenPort = "8000"
c.URL = "/"
c.Prefix = "json"
return c
}
176 changes: 119 additions & 57 deletions x-pack/filebeat/input/httpinput/input.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"strings"
"sync"
"time"
"fmt"

"github.com/pkg/errors"

Expand Down Expand Up @@ -47,6 +48,7 @@ type HttpInput struct {
httpMux *http.ServeMux // Current HTTP Handler
httpRequest http.Request // Current Request
httpResponse http.ResponseWriter // Current ResponseWriter
eventObject *map[string]interface{} // Current event object
}

// NewInput creates a new httpjson input
Expand Down Expand Up @@ -151,119 +153,179 @@ func (in *HttpInput) Wait() {
in.Stop()
}

func (in *HttpInput) createServer() error {
// Merge listening address and port
var address strings.Builder
P1llus marked this conversation as resolved.
Show resolved Hide resolved
address.WriteString(in.config.ListenAddress + ":" + in.config.ListenPort)

in.httpMux = http.NewServeMux()
in.httpMux.HandleFunc(in.config.URL, in.apiResponse)

if in.config.UseSSL == true {
P1llus marked this conversation as resolved.
Show resolved Hide resolved
in.httpServer = &http.Server{Addr: address.String(), Handler: in.httpMux}
return in.httpServer.ListenAndServeTLS(in.config.SSLCertificate, in.config.SSLKey)
}
if in.config.UseSSL == false {
in.httpServer = &http.Server{Addr: address.String(), Handler: in.httpMux}
return in.httpServer.ListenAndServe()
}
return errors.New("SSL settings missing")
}

// Create a response to the request
func (in *HttpInput) apiResponse(w http.ResponseWriter, r *http.Request) {
var err string
var status uint

// Storing for validation
in.httpRequest = *r
in.httpResponse = w

// Validates request, writes response directly on error.
objmap := in.validateRequest()
status, err = in.createEvent()

if objmap == nil || len(objmap) == 0 {
in.log.Error("Request could not be processed")
if err != "" || status != 0 {
in.sendResponse(status, err)
return
}

// On success, returns the configured response parameters
in.sendResponse(http.StatusOK, in.config.ResponseBody)
}

func (in *HttpInput) createEvent() (uint, string) {
var err string
var status uint

status, err = in.validateRequest()

// Check if any of the validations failed, and if so, return them
if err != "" || status != 0 {
return status, err
}

// Create the event
ok := in.outlet.OnEvent(beat.Event{
Timestamp: time.Now().UTC(),
Fields: common.MapStr{
"message": "testing",
in.config.Prefix: objmap,
in.config.Prefix: in.eventObject,
},
})

// If event cannot be sent
if !ok {
in.log.Error("Failed to send event")
return http.StatusInternalServerError, in.createErrorMessage("unable to send event")
}

// On success, returns the configured response parameters
w.Write([]byte(in.config.ResponseBody))
w.WriteHeader(in.config.ResponseCode)
return 0, ""
}

func (in *HttpInput) createServer() error {
// Merge listening address and port
var address strings.Builder
address.WriteString(in.config.ListenAddress + ":" + in.config.ListenPort)

in.httpMux = http.NewServeMux()
in.httpMux.HandleFunc(in.config.URL, in.apiResponse)
func (in *HttpInput) validateRequest() (uint, string) {
// Only allow POST requests
var err string
var status uint

if in.config.UseSSL == true {
in.httpServer = &http.Server{Addr: address.String(), Handler: in.httpMux}
return in.httpServer.ListenAndServeTLS(in.config.SSLCertificate, in.config.SSLKey)
// Check auth settings and credentials
if in.config.BasicAuth == true {
P1llus marked this conversation as resolved.
Show resolved Hide resolved
status, err = in.validateAuth()
}
if in.config.UseSSL == false {
in.httpServer = &http.Server{Addr: address.String(), Handler: in.httpMux}
return in.httpServer.ListenAndServe()

if err != "" && status != 0 {
return status, err
}
return errors.New("SSL settings missing")
}

func (in *HttpInput) validateRequest() map[string]interface{} {
// Check auth settings and credentials
if in.config.BasicAuth == true {
if in.config.Username == "" || in.config.Password == "" {
in.log.Fatal("Username and password required when basicauth is enabled")
return nil
}
// Validate headers
status, err = in.validateHeader()

username, password, _ := in.httpRequest.BasicAuth()
if in.config.Username != username || in.config.Password != password {
in.httpResponse.WriteHeader(http.StatusUnauthorized)
in.httpResponse.Write([]byte(`{"message": "Incorrect username or password"}`))
return nil
}
if err != "" && status != 0 {
return status, err
}

// Only allow POST requests
if in.httpRequest.Method != http.MethodPost {
in.httpResponse.WriteHeader(http.StatusMethodNotAllowed)
in.httpResponse.Write([]byte(`{"message": "only post request supported"}`))
return nil
// Validate body
status, err = in.validateBody()

if err != "" && status != 0 {
return status, err
}

return 0, ""

}

func (in *HttpInput) validateHeader() (uint, string) {
// Only allow JSON
if in.httpRequest.Header.Get("Content-Type") != "application/json" {
in.httpResponse.WriteHeader(http.StatusUnsupportedMediaType)
in.httpResponse.Write([]byte(`{"message": "wrong content-type header"}`))
return nil
return http.StatusUnsupportedMediaType, in.createErrorMessage("wrong content-type header, expecting application/json")
}

// Only accept JSON in return
if in.httpRequest.Header.Get("Accept") != "application/json" {
in.httpResponse.WriteHeader(http.StatusNotAcceptable)
in.httpResponse.Write([]byte(`{"message": "wrong accept header"}`))
return nil
return http.StatusNotAcceptable, in.createErrorMessage("wrong accept header, expecting application/json")
}
return 0, ""
}

func (in *HttpInput) validateAuth() (uint, string) {
// Check if username or password is missing
if in.config.Username == "" || in.config.Password == "" {
return http.StatusUnauthorized, in.createErrorMessage("Username and password required when basicauth is enabled")
}

// Check if username and password combination is correct
username, password, _ := in.httpRequest.BasicAuth()
if in.config.Username != username || in.config.Password != password {
return http.StatusUnauthorized, in.createErrorMessage("Incorrect username or password")
}

return 0, ""
}

func (in *HttpInput) validateBody() (uint, string) {
// Checks if body is empty
if in.httpRequest.Body == http.NoBody {
in.httpResponse.WriteHeader(http.StatusNotAcceptable)
in.httpResponse.Write([]byte(`{"message": "empty body"}`))
return nil
return http.StatusNotAcceptable, in.createErrorMessage("body can not be empty")
}


// Write full []byte to string
body, err := ioutil.ReadAll(in.httpRequest.Body)

P1llus marked this conversation as resolved.
Show resolved Hide resolved
// If body cannot be read
P1llus marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
in.httpResponse.WriteHeader(http.StatusInternalServerError)
in.httpResponse.Write([]byte(`{"message": "failure"}`))
return nil
return http.StatusInternalServerError, in.createErrorMessage("unable to read body")
}


// Declare interface for request body
objmap := make(map[string]interface{})

err = json.Unmarshal(body, &objmap)
P1llus marked this conversation as resolved.
Show resolved Hide resolved

// If body can be read, but not converted to JSON
if err != nil {
in.httpResponse.WriteHeader(http.StatusBadRequest)
in.httpResponse.Write([]byte(`{"message": "malformed JSON body"}`))
return nil
return http.StatusBadRequest, in.createErrorMessage("malformed JSON body")
}
return objmap
// Assign the current Unmarshaled object when no errors
in.eventObject = &objmap

return 0, ""
}

func (in *HttpInput) validateMethod() (uint, string) {
// Ensure HTTP method is POST
P1llus marked this conversation as resolved.
Show resolved Hide resolved
if in.httpRequest.Method != http.MethodPost {
return http.StatusMethodNotAllowed, in.createErrorMessage("only POST requests supported")
}

return 0, ""
}

func (in *HttpInput) createErrorMessage(r string ) string {
return fmt.Sprintf(`{"message": "%v"}`, r)
}

func (in *HttpInput) sendResponse(h uint, b string) {
in.httpResponse.WriteHeader(int(h))
in.httpResponse.Write([]byte(b))
}