func (t *TCPMessage) check100Continue() {
if t.expectType != httpExpectNotSet || len(t.packets[0].Data) < 25 {
return
}
if t.methodType != httpMethodWithBody {
return
}
if t.seqMissing || t.headerPacket == -1 {
return
}
last := t.packets[len(t.packets)-1]
// reading last 4 bytes for double CRLF
if !bytes.HasSuffix(last.Data, bEmptyLine) {
return
}
for _, p := range t.packets[:t.headerPacket+1] {
if h := proto.Header(p.Data, bExpectHeader); len(h) > 0 {
if bytes.Equal(bExpect100Value, h) {
t.expectType = httpExpect100Continue
}
return
}
}
t.expectType = httpExpectEmpty
}
// isMultipart returns true if message contains from multiple tcp packets
func (t *TCPMessage) IsMultipart() bool {
if len(t.packets) > 1 {
return true
}
payload := t.packets[0].Data
if len(payload) < 4 {
return false
}
m := payload[:4]
if t.IsIncoming {
// If one GET, OPTIONS, or HEAD request
if bytes.Equal(m, []byte("GET ")) || bytes.Equal(m, []byte("OPTI")) || bytes.Equal(m, []byte("HEAD")) {
return false
} else {
// Sometimes header comes after the body :(
if bytes.Equal(m, []byte("POST")) || bytes.Equal(m, []byte("PUT ")) || bytes.Equal(m, []byte("PATC")) {
if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 {
l, _ := strconv.Atoi(string(length))
// If content-length equal current body length
if l > 0 && l == t.Size() {
return false
}
}
}
}
} else {
if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 {
if length[0] == '0' {
return false
}
l, _ := strconv.Atoi(string(length))
// If content-length equal current body length
if l > 0 && l == t.Size() {
return false
}
}
}
return true
}
func TestRAWInputIPv4(t *testing.T) {
wg := new(sync.WaitGroup)
quit := make(chan int)
listener, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatal(err)
}
origin := &http.Server{
Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
}
go origin.Serve(listener)
defer listener.Close()
originAddr := listener.Addr().String()
var respCounter, reqCounter int64
input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "X-Real-IP")
defer input.Close()
output := NewTestOutput(func(data []byte) {
if data[0] == '1' {
body := payloadBody(data)
if len(proto.Header(body, []byte("X-Real-IP"))) == 0 {
t.Error("Should have X-Real-IP header", string(body))
}
atomic.AddInt64(&reqCounter, 1)
} else {
atomic.AddInt64(&respCounter, 1)
}
if Settings.debug {
log.Println(reqCounter, respCounter)
}
wg.Done()
})
Plugins.Inputs = []io.Reader{input}
Plugins.Outputs = []io.Writer{output}
client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})
go Start(quit)
for i := 0; i < 100; i++ {
// request + response
wg.Add(2)
client.Get("/")
time.Sleep(2 * time.Millisecond)
}
wg.Wait()
close(quit)
}
func (t *TCPMessage) updateBodyType() {
// if there is cache
if t.bodyType != httpBodyNotSet {
return
}
// Headers not received
if t.headerPacket == -1 {
return
}
switch t.methodType {
case httpMethodNotFound:
return
case httpMethodWithoutBody:
t.bodyType = httpBodyEmpty
return
case httpMethodWithBody:
var lengthB, encB []byte
for _, p := range t.packets[:t.headerPacket+1] {
lengthB = proto.Header(p.Data, []byte("Content-Length"))
if len(lengthB) > 0 {
break
}
}
if len(lengthB) > 0 {
t.bodyType = httpBodyContentLength
t.contentLength, _ = strconv.Atoi(string(lengthB))
return
} else {
for _, p := range t.packets[:t.headerPacket+1] {
encB = proto.Header(p.Data, []byte("Transfer-Encoding"))
if len(encB) > 0 {
t.bodyType = httpBodyChunked
return
}
}
}
}
t.bodyType = httpBodyEmpty
}
func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
if c.conn == nil || !c.isAlive() {
Debug("Connecting:", c.baseURL)
c.Connect()
}
timeout := time.Now().Add(5 * time.Second)
c.conn.SetWriteDeadline(timeout)
data = proto.SetHeader(data, []byte("Host"), []byte(c.baseURL.Host))
if c.config.Debug {
Debug("Sending:", string(data))
}
if _, err = c.conn.Write(data); err != nil {
Debug("Write error:", err, c.baseURL)
return
}
c.conn.SetReadDeadline(timeout)
n, err := c.conn.Read(c.respBuf)
if err != nil {
Debug("READ ERRORR!", err, c.conn)
return
}
payload := c.respBuf[:n]
if c.config.Debug {
Debug("Received:", string(payload))
}
if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
status := payload[9:12]
// 3xx requests
if status[0] == '3' {
c.redirectsCount += 1
location, _, _, _ := proto.Header(payload, []byte("Location"))
redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n")
if c.config.Debug {
Debug("Redirecting to: " + string(location))
}
return c.Send(redirectPayload)
}
}
c.redirectsCount = 0
return payload, err
}
func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
if len(m.config.methods) > 0 {
method := proto.Method(payload)
matched := false
for _, m := range m.config.methods {
if bytes.Equal(method, m) {
matched = true
break
}
}
if !matched {
return
}
}
if len(m.config.headers) > 0 {
for _, header := range m.config.headers {
payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value))
}
}
if len(m.config.params) > 0 {
for _, param := range m.config.params {
payload = proto.SetPathParam(payload, param.Name, param.Value)
}
}
if len(m.config.urlRegexp) > 0 {
path := proto.Path(payload)
matched := false
for _, f := range m.config.urlRegexp {
if f.regexp.Match(path) {
matched = true
break
}
}
if !matched {
return
}
}
if len(m.config.urlNegativeRegexp) > 0 {
path := proto.Path(payload)
for _, f := range m.config.urlNegativeRegexp {
if f.regexp.Match(path) {
return
}
}
}
if len(m.config.headerFilters) > 0 {
for _, f := range m.config.headerFilters {
value := proto.Header(payload, f.name)
if len(value) > 0 && !f.regexp.Match(value) {
return
}
}
}
if len(m.config.headerNegativeFilters) > 0 {
for _, f := range m.config.headerNegativeFilters {
value := proto.Header(payload, f.name)
if len(value) > 0 && f.regexp.Match(value) {
return
}
}
}
if len(m.config.headerHashFilters) > 0 {
for _, f := range m.config.headerHashFilters {
value := proto.Header(payload, f.name)
if len(value) > 0 {
hasher := fnv.New32a()
hasher.Write(value)
if (hasher.Sum32() % 100) >= f.percent {
return
}
}
}
}
if len(m.config.paramHashFilters) > 0 {
for _, f := range m.config.paramHashFilters {
value, s, _ := proto.PathParam(payload, f.name)
if s != -1 {
hasher := fnv.New32a()
hasher.Write(value)
if (hasher.Sum32() % 100) >= f.percent {
return
}
}
}
}
if len(m.config.urlRewrite) > 0 {
path := proto.Path(payload)
for _, f := range m.config.urlRewrite {
if f.src.Match(path) {
path = f.src.ReplaceAll(path, f.target)
payload = proto.SetPath(payload, path)
break
}
}
}
return payload
}
func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
// Don't exit on panic
defer func() {
if r := recover(); r != nil {
Debug("[HTTPClient]", r, string(data))
if _, ok := r.(error); !ok {
log.Println("[HTTPClient] Failed to send request: ", string(data))
log.Println("PANIC: pkg:", r, debug.Stack())
}
}
}()
if c.conn == nil || !c.isAlive() {
Debug("[HTTPClient] Connecting:", c.baseURL)
if err = c.Connect(); err != nil {
log.Println("[HTTPClient] Connection error:", err)
response = errorPayload(HTTP_CONNECTION_ERROR)
return
}
}
timeout := time.Now().Add(c.config.Timeout)
c.conn.SetWriteDeadline(timeout)
if !c.config.OriginalHost {
data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
}
if c.config.Debug {
Debug("[HTTPClient] Sending:", string(data))
}
if _, err = c.conn.Write(data); err != nil {
Debug("[HTTPClient] Write error:", err, c.baseURL)
response = errorPayload(HTTP_TIMEOUT)
return
}
c.conn.SetReadDeadline(timeout)
n, err := c.conn.Read(c.respBuf)
// If response large then our buffer, we need to read all response buffer
// Otherwise it will corrupt response of next request
// Parsing response body is non trivial thing, especially with keep-alive
// Simples case is to to close connection if response too large
//
// See https://github.com/buger/gor/issues/184
if n == len(c.respBuf) {
c.Disconnect()
}
if err != nil {
Debug("[HTTPClient] Response read error", err, c.conn)
response = errorPayload(HTTP_TIMEOUT)
return
}
payload := c.respBuf[:n]
if c.config.Debug {
Debug("[HTTPClient] Received:", string(payload))
}
if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
status := payload[9:12]
// 3xx requests
if status[0] == '3' {
c.redirectsCount++
location := proto.Header(payload, []byte("Location"))
redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n")
if c.config.Debug {
Debug("[HTTPClient] Redirecting to: " + string(location))
}
return c.Send(redirectPayload)
}
}
c.redirectsCount = 0
return payload, err
}
func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
// Don't exit on panic
defer func() {
if r := recover(); r != nil {
Debug("[HTTPClient]", r, string(data))
if _, ok := r.(error); !ok {
log.Println("[HTTPClient] Failed to send request: ", string(data))
log.Println("PANIC: pkg:", r, debug.Stack())
}
}
}()
if c.conn == nil || !c.isAlive() {
Debug("[HTTPClient] Connecting:", c.baseURL)
if err = c.Connect(); err != nil {
log.Println("[HTTPClient] Connection error:", err)
response = errorPayload(HTTP_CONNECTION_ERROR)
return
}
}
timeout := time.Now().Add(c.config.Timeout)
c.conn.SetWriteDeadline(timeout)
if !c.config.OriginalHost {
data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
}
if c.auth != "" {
data = proto.SetHeader(data, []byte("Authorization"), []byte(c.auth))
}
if c.config.Debug {
Debug("[HTTPClient] Sending:", string(data))
}
if _, err = c.conn.Write(data); err != nil {
Debug("[HTTPClient] Write error:", err, c.baseURL)
response = errorPayload(HTTP_TIMEOUT)
return
}
var readBytes, n int
var currentChunk []byte
timeout = time.Now().Add(c.config.Timeout)
chunked := false
contentLength := -1
currentContentLength := 0
chunks := 0
for {
c.conn.SetReadDeadline(timeout)
if readBytes < len(c.respBuf) {
n, err = c.conn.Read(c.respBuf[readBytes:])
readBytes += n
chunks++
if err != nil {
if err == io.EOF {
err = nil
}
break
}
// First chunk
if chunked || contentLength != -1 {
currentContentLength += n
} else {
// If headers are finished
if bytes.Contains(c.respBuf[:readBytes], proto.EmptyLine) {
if bytes.Equal(proto.Header(c.respBuf, []byte("Transfer-Encoding")), []byte("chunked")) {
chunked = true
} else {
status, _ := strconv.Atoi(string(proto.Status(c.respBuf)))
if (status >= 100 && status < 200) || status == 204 || status == 304 {
contentLength = 0
} else {
l := proto.Header(c.respBuf, []byte("Content-Length"))
if len(l) > 0 {
contentLength, _ = strconv.Atoi(string(l))
}
}
}
currentContentLength += len(proto.Body(c.respBuf[:readBytes]))
}
}
if chunked {
// Check if chunked message finished
if bytes.HasSuffix(c.respBuf[:readBytes], chunkedSuffix) {
break
}
} else if contentLength != -1 {
if currentContentLength > contentLength {
Debug("[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
c.Disconnect()
break
} else if currentContentLength == contentLength {
break
}
}
} else {
if currentChunk == nil {
currentChunk = make([]byte, readChunkSize)
}
n, err = c.conn.Read(currentChunk)
if err == io.EOF {
break
} else if err != nil {
Debug("[HTTPClient] Read the whole body error:", err, c.baseURL)
break
}
readBytes += int(n)
chunks++
currentContentLength += n
if chunked {
// Check if chunked message finished
if bytes.HasSuffix(currentChunk[:n], chunkedSuffix) {
break
}
} else if contentLength != -1 {
if currentContentLength > contentLength {
Debug("[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
c.Disconnect()
break
} else if currentContentLength == contentLength {
break
}
} else {
Debug("[HTTPClient] disconnected, can't find Content-Length or Chunked")
c.Disconnect()
break
}
}
if readBytes >= maxResponseSize {
Debug("[HTTPClient] Body is more than the max size", maxResponseSize,
c.baseURL)
break
}
// For following chunks expect less timeout
timeout = time.Now().Add(c.config.Timeout / 5)
}
if err != nil {
Debug("[HTTPClient] Response read error", err, c.conn, readBytes)
response = errorPayload(HTTP_TIMEOUT)
return
}
if readBytes > len(c.respBuf) {
readBytes = len(c.respBuf)
}
payload := make([]byte, readBytes)
copy(payload, c.respBuf[:readBytes])
if c.config.Debug {
Debug("[HTTPClient] Received:", string(payload))
}
if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
status := payload[9:12]
// 3xx requests
if status[0] == '3' {
c.redirectsCount++
location := proto.Header(payload, []byte("Location"))
redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n")
if c.config.Debug {
Debug("[HTTPClient] Redirecting to: " + string(location))
}
return c.Send(redirectPayload)
}
}
if bytes.Equal(proto.Status(payload), []byte("400")) {
c.Disconnect()
Debug("[HTTPClient] Closed connection on 400 response")
}
c.redirectsCount = 0
return payload, err
}
func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
if len(resp) == 0 {
// nil http response - skipped elasticsearch export for this request
return
}
t := time.Now()
rtt := p.RttDurationToMs(stop.Sub(start))
req = payloadBody(req)
esResp := ESRequestResponse{
ReqURL: string(proto.Path(req)),
ReqMethod: string(proto.Method(req)),
ReqUserAgent: string(proto.Header(req, []byte("User-Agent"))),
ReqAcceptLanguage: string(proto.Header(req, []byte("Accept-Language"))),
ReqAccept: string(proto.Header(req, []byte("Accept"))),
ReqAcceptEncoding: string(proto.Header(req, []byte("Accept-Encoding"))),
ReqIfModifiedSince: string(proto.Header(req, []byte("If-Modified-Since"))),
ReqConnection: string(proto.Header(req, []byte("Connection"))),
ReqCookies: string(proto.Header(req, []byte("Cookie"))),
RespStatus: string(proto.Status(resp)),
RespStatusCode: string(proto.Status(resp)),
RespProto: string(proto.Method(resp)),
RespContentLength: string(proto.Header(resp, []byte("Content-Length"))),
RespContentType: string(proto.Header(resp, []byte("Content-Type"))),
RespTransferEncoding: string(proto.Header(resp, []byte("Transfer-Encoding"))),
RespContentEncoding: string(proto.Header(resp, []byte("Content-Encoding"))),
RespExpires: string(proto.Header(resp, []byte("Expires"))),
RespCacheControl: string(proto.Header(resp, []byte("Cache-Control"))),
RespVary: string(proto.Header(resp, []byte("Vary"))),
RespSetCookie: string(proto.Header(resp, []byte("Set-Cookie"))),
Rtt: rtt,
Timestamp: t,
}
j, err := json.Marshal(&esResp)
if err != nil {
log.Println(err)
} else {
p.indexor.Index(p.Index, "RequestResponse", "", "", "", &t, j)
}
return
}
func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
// Don't exit on panic
defer func() {
if r := recover(); r != nil {
Debug("[HTTPClient]", r, string(data))
if _, ok := r.(error); !ok {
log.Println("[HTTPClient] Failed to send request: ", string(data))
log.Println("PANIC: pkg:", r, debug.Stack())
}
}
}()
if c.conn == nil || !c.isAlive() {
Debug("[HTTPClient] Connecting:", c.baseURL)
if err = c.Connect(); err != nil {
log.Println("[HTTPClient] Connection error:", err)
return
}
}
timeout := time.Now().Add(5 * time.Second)
c.conn.SetWriteDeadline(timeout)
data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
if c.config.Debug {
Debug("[HTTPClient] Sending:", string(data))
}
if _, err = c.conn.Write(data); err != nil {
Debug("[HTTPClient] Write error:", err, c.baseURL)
return
}
c.conn.SetReadDeadline(timeout)
n, err := c.conn.Read(c.respBuf)
if err != nil {
Debug("[HTTPClient] Response read error", err, c.conn)
return
}
payload := c.respBuf[:n]
if c.config.Debug {
Debug("[HTTPClient] Received:", string(payload))
}
if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
status := payload[9:12]
// 3xx requests
if status[0] == '3' {
c.redirectsCount += 1
location, _, _, _ := proto.Header(payload, []byte("Location"))
redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n")
if c.config.Debug {
Debug("[HTTPClient] Redirecting to: " + string(location))
}
return c.Send(redirectPayload)
}
}
c.redirectsCount = 0
return payload, err
}
func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
if len(m.config.methods) > 0 && !m.config.methods.Contains(proto.Method(payload)) {
return
}
if m.config.urlRegexp.regexp != nil {
host, _, _, _ := proto.Header(payload, []byte("Host"))
fullPath := append(host, proto.Path(payload)...)
if !m.config.urlRegexp.regexp.Match(fullPath) {
return
}
}
if len(m.config.headerFilters) > 0 {
for _, f := range m.config.headerFilters {
value, s, _, _ := proto.Header(payload, f.name)
if s != -1 && !f.regexp.Match(value) {
return
}
}
}
if len(m.config.headerHashFilters) > 0 {
for _, f := range m.config.headerHashFilters {
value, s, _, _ := proto.Header(payload, f.name)
if s == -1 {
return
}
hasher := fnv.New32a()
hasher.Write(value)
if (hasher.Sum32() % 100) >= f.percent {
return
}
}
}
if len(m.config.urlRewrite) > 0 {
path := proto.Path(payload)
for _, f := range m.config.urlRewrite {
if f.src.Match(path) {
path = f.src.ReplaceAll(path, f.target)
payload = proto.SetPath(payload, path)
break
}
}
}
if len(m.config.headers) > 0 {
for _, header := range m.config.headers {
payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value))
}
}
return payload
}