Пример #1
0
func (t *TCPMessage) check100Continue() {
	if t.expectType != httpExpectNotSet || len(t.packets[0].Data) < 25 {
		return
	}

	if t.methodType != httpMethodWithBody {
		return
	}

	if t.seqMissing || t.headerPacket == -1 {
		return
	}

	last := t.packets[len(t.packets)-1]
	// reading last 4 bytes for double CRLF
	if !bytes.HasSuffix(last.Data, bEmptyLine) {
		return
	}

	for _, p := range t.packets[:t.headerPacket+1] {
		if h := proto.Header(p.Data, bExpectHeader); len(h) > 0 {
			if bytes.Equal(bExpect100Value, h) {
				t.expectType = httpExpect100Continue
			}
			return
		}
	}

	t.expectType = httpExpectEmpty
}
Пример #2
0
// isMultipart returns true if message contains from multiple tcp packets
func (t *TCPMessage) IsMultipart() bool {
	if len(t.packets) > 1 {
		return true
	}

	payload := t.packets[0].Data

	if len(payload) < 4 {
		return false
	}

	m := payload[:4]

	if t.IsIncoming {
		// If one GET, OPTIONS, or HEAD request
		if bytes.Equal(m, []byte("GET ")) || bytes.Equal(m, []byte("OPTI")) || bytes.Equal(m, []byte("HEAD")) {
			return false
		} else {
			// Sometimes header comes after the body :(
			if bytes.Equal(m, []byte("POST")) || bytes.Equal(m, []byte("PUT ")) || bytes.Equal(m, []byte("PATC")) {
				if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 {
					l, _ := strconv.Atoi(string(length))

					// If content-length equal current body length
					if l > 0 && l == t.Size() {
						return false
					}
				}
			}
		}
	} else {
		if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 {
			if length[0] == '0' {
				return false
			}

			l, _ := strconv.Atoi(string(length))

			// If content-length equal current body length
			if l > 0 && l == t.Size() {
				return false
			}
		}
	}

	return true
}
Пример #3
0
func TestRAWInputIPv4(t *testing.T) {
	wg := new(sync.WaitGroup)
	quit := make(chan int)

	listener, err := net.Listen("tcp", ":0")
	if err != nil {
		t.Fatal(err)
	}
	origin := &http.Server{
		Handler:      http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
		ReadTimeout:  10 * time.Second,
		WriteTimeout: 10 * time.Second,
	}
	go origin.Serve(listener)
	defer listener.Close()

	originAddr := listener.Addr().String()

	var respCounter, reqCounter int64

	input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "X-Real-IP")
	defer input.Close()

	output := NewTestOutput(func(data []byte) {
		if data[0] == '1' {
			body := payloadBody(data)
			if len(proto.Header(body, []byte("X-Real-IP"))) == 0 {
				t.Error("Should have X-Real-IP header", string(body))
			}
			atomic.AddInt64(&reqCounter, 1)
		} else {
			atomic.AddInt64(&respCounter, 1)
		}

		if Settings.debug {
			log.Println(reqCounter, respCounter)
		}

		wg.Done()
	})

	Plugins.Inputs = []io.Reader{input}
	Plugins.Outputs = []io.Writer{output}

	client := NewHTTPClient("http://"+listener.Addr().String(), &HTTPClientConfig{})

	go Start(quit)

	for i := 0; i < 100; i++ {
		// request + response
		wg.Add(2)
		client.Get("/")
		time.Sleep(2 * time.Millisecond)
	}

	wg.Wait()

	close(quit)
}
Пример #4
0
func (t *TCPMessage) updateBodyType() {
	// if there is cache
	if t.bodyType != httpBodyNotSet {
		return
	}

	// Headers not received
	if t.headerPacket == -1 {
		return
	}

	switch t.methodType {
	case httpMethodNotFound:
		return
	case httpMethodWithoutBody:
		t.bodyType = httpBodyEmpty
		return
	case httpMethodWithBody:
		var lengthB, encB []byte

		for _, p := range t.packets[:t.headerPacket+1] {
			lengthB = proto.Header(p.Data, []byte("Content-Length"))

			if len(lengthB) > 0 {
				break
			}
		}

		if len(lengthB) > 0 {
			t.bodyType = httpBodyContentLength
			t.contentLength, _ = strconv.Atoi(string(lengthB))
			return
		} else {
			for _, p := range t.packets[:t.headerPacket+1] {
				encB = proto.Header(p.Data, []byte("Transfer-Encoding"))

				if len(encB) > 0 {
					t.bodyType = httpBodyChunked
					return
				}
			}
		}
	}

	t.bodyType = httpBodyEmpty
}
Пример #5
0
func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
	if c.conn == nil || !c.isAlive() {
		Debug("Connecting:", c.baseURL)
		c.Connect()
	}

	timeout := time.Now().Add(5 * time.Second)

	c.conn.SetWriteDeadline(timeout)

	data = proto.SetHeader(data, []byte("Host"), []byte(c.baseURL.Host))

	if c.config.Debug {
		Debug("Sending:", string(data))
	}

	if _, err = c.conn.Write(data); err != nil {
		Debug("Write error:", err, c.baseURL)
		return
	}

	c.conn.SetReadDeadline(timeout)
	n, err := c.conn.Read(c.respBuf)

	if err != nil {
		Debug("READ ERRORR!", err, c.conn)
		return
	}

	payload := c.respBuf[:n]

	if c.config.Debug {
		Debug("Received:", string(payload))
	}

	if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
		status := payload[9:12]

		// 3xx requests
		if status[0] == '3' {
			c.redirectsCount += 1

			location, _, _, _ := proto.Header(payload, []byte("Location"))
			redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n")

			if c.config.Debug {
				Debug("Redirecting to: " + string(location))
			}

			return c.Send(redirectPayload)
		}
	}

	c.redirectsCount = 0

	return payload, err
}
Пример #6
0
func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
	if len(m.config.methods) > 0 {
		method := proto.Method(payload)

		matched := false

		for _, m := range m.config.methods {
			if bytes.Equal(method, m) {
				matched = true
				break
			}
		}

		if !matched {
			return
		}
	}

	if len(m.config.headers) > 0 {
		for _, header := range m.config.headers {
			payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value))
		}
	}

	if len(m.config.params) > 0 {
		for _, param := range m.config.params {
			payload = proto.SetPathParam(payload, param.Name, param.Value)
		}
	}

	if len(m.config.urlRegexp) > 0 {
		path := proto.Path(payload)

		matched := false

		for _, f := range m.config.urlRegexp {
			if f.regexp.Match(path) {
				matched = true
				break
			}
		}

		if !matched {
			return
		}
	}

	if len(m.config.urlNegativeRegexp) > 0 {
		path := proto.Path(payload)

		for _, f := range m.config.urlNegativeRegexp {
			if f.regexp.Match(path) {
				return
			}
		}
	}

	if len(m.config.headerFilters) > 0 {
		for _, f := range m.config.headerFilters {
			value := proto.Header(payload, f.name)

			if len(value) > 0 && !f.regexp.Match(value) {
				return
			}
		}
	}

	if len(m.config.headerNegativeFilters) > 0 {
		for _, f := range m.config.headerNegativeFilters {
			value := proto.Header(payload, f.name)

			if len(value) > 0 && f.regexp.Match(value) {
				return
			}
		}
	}

	if len(m.config.headerHashFilters) > 0 {
		for _, f := range m.config.headerHashFilters {
			value := proto.Header(payload, f.name)

			if len(value) > 0 {
				hasher := fnv.New32a()
				hasher.Write(value)

				if (hasher.Sum32() % 100) >= f.percent {
					return
				}
			}
		}
	}

	if len(m.config.paramHashFilters) > 0 {
		for _, f := range m.config.paramHashFilters {
			value, s, _ := proto.PathParam(payload, f.name)

			if s != -1 {
				hasher := fnv.New32a()
				hasher.Write(value)

				if (hasher.Sum32() % 100) >= f.percent {
					return
				}
			}
		}
	}

	if len(m.config.urlRewrite) > 0 {
		path := proto.Path(payload)

		for _, f := range m.config.urlRewrite {
			if f.src.Match(path) {
				path = f.src.ReplaceAll(path, f.target)
				payload = proto.SetPath(payload, path)

				break
			}
		}
	}

	return payload
}
Пример #7
0
func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
	// Don't exit on panic
	defer func() {
		if r := recover(); r != nil {
			Debug("[HTTPClient]", r, string(data))

			if _, ok := r.(error); !ok {
				log.Println("[HTTPClient] Failed to send request: ", string(data))
				log.Println("PANIC: pkg:", r, debug.Stack())
			}
		}
	}()

	if c.conn == nil || !c.isAlive() {
		Debug("[HTTPClient] Connecting:", c.baseURL)
		if err = c.Connect(); err != nil {
			log.Println("[HTTPClient] Connection error:", err)
			response = errorPayload(HTTP_CONNECTION_ERROR)
			return
		}
	}

	timeout := time.Now().Add(c.config.Timeout)

	c.conn.SetWriteDeadline(timeout)

	if !c.config.OriginalHost {
		data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
	}

	if c.config.Debug {
		Debug("[HTTPClient] Sending:", string(data))
	}

	if _, err = c.conn.Write(data); err != nil {
		Debug("[HTTPClient] Write error:", err, c.baseURL)
		response = errorPayload(HTTP_TIMEOUT)
		return
	}

	c.conn.SetReadDeadline(timeout)
	n, err := c.conn.Read(c.respBuf)

	// If response large then our buffer, we need to read all response buffer
	// Otherwise it will corrupt response of next request
	// Parsing response body is non trivial thing, especially with keep-alive
	// Simples case is to to close connection if response too large
	//
	// See https://github.com/buger/gor/issues/184
	if n == len(c.respBuf) {
		c.Disconnect()
	}

	if err != nil {
		Debug("[HTTPClient] Response read error", err, c.conn)
		response = errorPayload(HTTP_TIMEOUT)
		return
	}

	payload := c.respBuf[:n]

	if c.config.Debug {
		Debug("[HTTPClient] Received:", string(payload))
	}

	if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
		status := payload[9:12]

		// 3xx requests
		if status[0] == '3' {
			c.redirectsCount++

			location := proto.Header(payload, []byte("Location"))
			redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n")

			if c.config.Debug {
				Debug("[HTTPClient] Redirecting to: " + string(location))
			}

			return c.Send(redirectPayload)
		}
	}

	c.redirectsCount = 0

	return payload, err
}
Пример #8
0
func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
	// Don't exit on panic
	defer func() {
		if r := recover(); r != nil {
			Debug("[HTTPClient]", r, string(data))

			if _, ok := r.(error); !ok {
				log.Println("[HTTPClient] Failed to send request: ", string(data))
				log.Println("PANIC: pkg:", r, debug.Stack())
			}
		}
	}()

	if c.conn == nil || !c.isAlive() {
		Debug("[HTTPClient] Connecting:", c.baseURL)
		if err = c.Connect(); err != nil {
			log.Println("[HTTPClient] Connection error:", err)
			response = errorPayload(HTTP_CONNECTION_ERROR)
			return
		}
	}

	timeout := time.Now().Add(c.config.Timeout)

	c.conn.SetWriteDeadline(timeout)

	if !c.config.OriginalHost {
		data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
	}

	if c.auth != "" {
		data = proto.SetHeader(data, []byte("Authorization"), []byte(c.auth))
	}

	if c.config.Debug {
		Debug("[HTTPClient] Sending:", string(data))
	}

	if _, err = c.conn.Write(data); err != nil {
		Debug("[HTTPClient] Write error:", err, c.baseURL)
		response = errorPayload(HTTP_TIMEOUT)
		return
	}

	var readBytes, n int
	var currentChunk []byte
	timeout = time.Now().Add(c.config.Timeout)
	chunked := false
	contentLength := -1
	currentContentLength := 0
	chunks := 0

	for {
		c.conn.SetReadDeadline(timeout)

		if readBytes < len(c.respBuf) {
			n, err = c.conn.Read(c.respBuf[readBytes:])
			readBytes += n
			chunks++

			if err != nil {
				if err == io.EOF {
					err = nil
				}
				break
			}

			// First chunk
			if chunked || contentLength != -1 {
				currentContentLength += n
			} else {
				// If headers are finished
				if bytes.Contains(c.respBuf[:readBytes], proto.EmptyLine) {
					if bytes.Equal(proto.Header(c.respBuf, []byte("Transfer-Encoding")), []byte("chunked")) {
						chunked = true
					} else {
						status, _ := strconv.Atoi(string(proto.Status(c.respBuf)))
						if (status >= 100 && status < 200) || status == 204 || status == 304 {
							contentLength = 0
						} else {
							l := proto.Header(c.respBuf, []byte("Content-Length"))
							if len(l) > 0 {
								contentLength, _ = strconv.Atoi(string(l))
							}
						}
					}

					currentContentLength += len(proto.Body(c.respBuf[:readBytes]))
				}
			}

			if chunked {
				// Check if chunked message finished
				if bytes.HasSuffix(c.respBuf[:readBytes], chunkedSuffix) {
					break
				}
			} else if contentLength != -1 {
				if currentContentLength > contentLength {
					Debug("[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
					c.Disconnect()
					break
				} else if currentContentLength == contentLength {
					break
				}
			}
		} else {
			if currentChunk == nil {
				currentChunk = make([]byte, readChunkSize)
			}

			n, err = c.conn.Read(currentChunk)

			if err == io.EOF {
				break
			} else if err != nil {
				Debug("[HTTPClient] Read the whole body error:", err, c.baseURL)
				break
			}

			readBytes += int(n)
			chunks++
			currentContentLength += n

			if chunked {
				// Check if chunked message finished
				if bytes.HasSuffix(currentChunk[:n], chunkedSuffix) {
					break
				}
			} else if contentLength != -1 {
				if currentContentLength > contentLength {
					Debug("[HTTPClient] disconnected, wrong length", currentContentLength, contentLength)
					c.Disconnect()
					break
				} else if currentContentLength == contentLength {
					break
				}
			} else {
				Debug("[HTTPClient] disconnected, can't find Content-Length or Chunked")
				c.Disconnect()
				break
			}
		}

		if readBytes >= maxResponseSize {
			Debug("[HTTPClient] Body is more than the max size", maxResponseSize,
				c.baseURL)
			break
		}

		// For following chunks expect less timeout
		timeout = time.Now().Add(c.config.Timeout / 5)
	}

	if err != nil {
		Debug("[HTTPClient] Response read error", err, c.conn, readBytes)
		response = errorPayload(HTTP_TIMEOUT)
		return
	}

	if readBytes > len(c.respBuf) {
		readBytes = len(c.respBuf)
	}
	payload := make([]byte, readBytes)
	copy(payload, c.respBuf[:readBytes])

	if c.config.Debug {
		Debug("[HTTPClient] Received:", string(payload))
	}

	if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
		status := payload[9:12]

		// 3xx requests
		if status[0] == '3' {
			c.redirectsCount++

			location := proto.Header(payload, []byte("Location"))
			redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n")

			if c.config.Debug {
				Debug("[HTTPClient] Redirecting to: " + string(location))
			}

			return c.Send(redirectPayload)
		}
	}

	if bytes.Equal(proto.Status(payload), []byte("400")) {
		c.Disconnect()
		Debug("[HTTPClient] Closed connection on 400 response")
	}

	c.redirectsCount = 0

	return payload, err
}
Пример #9
0
func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
	if len(resp) == 0 {
		// nil http response - skipped elasticsearch export for this request
		return
	}
	t := time.Now()
	rtt := p.RttDurationToMs(stop.Sub(start))
	req = payloadBody(req)

	esResp := ESRequestResponse{
		ReqURL:               string(proto.Path(req)),
		ReqMethod:            string(proto.Method(req)),
		ReqUserAgent:         string(proto.Header(req, []byte("User-Agent"))),
		ReqAcceptLanguage:    string(proto.Header(req, []byte("Accept-Language"))),
		ReqAccept:            string(proto.Header(req, []byte("Accept"))),
		ReqAcceptEncoding:    string(proto.Header(req, []byte("Accept-Encoding"))),
		ReqIfModifiedSince:   string(proto.Header(req, []byte("If-Modified-Since"))),
		ReqConnection:        string(proto.Header(req, []byte("Connection"))),
		ReqCookies:           string(proto.Header(req, []byte("Cookie"))),
		RespStatus:           string(proto.Status(resp)),
		RespStatusCode:       string(proto.Status(resp)),
		RespProto:            string(proto.Method(resp)),
		RespContentLength:    string(proto.Header(resp, []byte("Content-Length"))),
		RespContentType:      string(proto.Header(resp, []byte("Content-Type"))),
		RespTransferEncoding: string(proto.Header(resp, []byte("Transfer-Encoding"))),
		RespContentEncoding:  string(proto.Header(resp, []byte("Content-Encoding"))),
		RespExpires:          string(proto.Header(resp, []byte("Expires"))),
		RespCacheControl:     string(proto.Header(resp, []byte("Cache-Control"))),
		RespVary:             string(proto.Header(resp, []byte("Vary"))),
		RespSetCookie:        string(proto.Header(resp, []byte("Set-Cookie"))),
		Rtt:                  rtt,
		Timestamp:            t,
	}
	j, err := json.Marshal(&esResp)
	if err != nil {
		log.Println(err)
	} else {
		p.indexor.Index(p.Index, "RequestResponse", "", "", "", &t, j)
	}
	return
}
Пример #10
0
func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
	// Don't exit on panic
	defer func() {
		if r := recover(); r != nil {
			Debug("[HTTPClient]", r, string(data))

			if _, ok := r.(error); !ok {
				log.Println("[HTTPClient] Failed to send request: ", string(data))
				log.Println("PANIC: pkg:", r, debug.Stack())
			}
		}
	}()

	if c.conn == nil || !c.isAlive() {
		Debug("[HTTPClient] Connecting:", c.baseURL)
		if err = c.Connect(); err != nil {
			log.Println("[HTTPClient] Connection error:", err)
			return
		}
	}

	timeout := time.Now().Add(5 * time.Second)

	c.conn.SetWriteDeadline(timeout)

	data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))

	if c.config.Debug {
		Debug("[HTTPClient] Sending:", string(data))
	}

	if _, err = c.conn.Write(data); err != nil {
		Debug("[HTTPClient] Write error:", err, c.baseURL)
		return
	}

	c.conn.SetReadDeadline(timeout)
	n, err := c.conn.Read(c.respBuf)

	if err != nil {
		Debug("[HTTPClient] Response read error", err, c.conn)
		return
	}

	payload := c.respBuf[:n]

	if c.config.Debug {
		Debug("[HTTPClient] Received:", string(payload))
	}

	if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
		status := payload[9:12]

		// 3xx requests
		if status[0] == '3' {
			c.redirectsCount += 1

			location, _, _, _ := proto.Header(payload, []byte("Location"))
			redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n")

			if c.config.Debug {
				Debug("[HTTPClient] Redirecting to: " + string(location))
			}

			return c.Send(redirectPayload)
		}
	}

	c.redirectsCount = 0

	return payload, err
}
Пример #11
0
func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
	if len(m.config.methods) > 0 && !m.config.methods.Contains(proto.Method(payload)) {
		return
	}

	if m.config.urlRegexp.regexp != nil {
		host, _, _, _ := proto.Header(payload, []byte("Host"))
		fullPath := append(host, proto.Path(payload)...)

		if !m.config.urlRegexp.regexp.Match(fullPath) {
			return
		}
	}

	if len(m.config.headerFilters) > 0 {
		for _, f := range m.config.headerFilters {
			value, s, _, _ := proto.Header(payload, f.name)

			if s != -1 && !f.regexp.Match(value) {
				return
			}
		}
	}

	if len(m.config.headerHashFilters) > 0 {
		for _, f := range m.config.headerHashFilters {
			value, s, _, _ := proto.Header(payload, f.name)

			if s == -1 {
				return
			}

			hasher := fnv.New32a()
			hasher.Write(value)

			if (hasher.Sum32() % 100) >= f.percent {
				return
			}
		}
	}

	if len(m.config.urlRewrite) > 0 {
		path := proto.Path(payload)

		for _, f := range m.config.urlRewrite {
			if f.src.Match(path) {
				path = f.src.ReplaceAll(path, f.target)
				payload = proto.SetPath(payload, path)

				break
			}
		}
	}

	if len(m.config.headers) > 0 {
		for _, header := range m.config.headers {
			payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value))
		}
	}

	return payload
}