func TestHTTPModifierURLRewrite(t *testing.T) {
var url, newURL []byte
rewrites := UrlRewriteMap{}
payload := func(url []byte) []byte {
return []byte("POST " + string(url) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
}
err := rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
if err != nil {
t.Error("Should not error on /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
}
modifier := NewHTTPModifier(&HTTPModifierConfig{
urlRewrite: rewrites,
})
url = []byte("/v1/user/joe/ping")
if newURL = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(newURL, url) {
t.Error("Request url should have been rewritten, wasn't", string(newURL))
}
url = []byte("/v1/user/ping")
if newURL = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(newURL, url) {
t.Error("Request url should have been rewritten, wasn't", string(newURL))
}
}
func (o *KafkaOutput) Write(data []byte) (n int, err error) {
headers := make(map[string]string)
proto.ParseHeaders([][]byte{data}, func(header []byte, value []byte) bool {
headers[string(header)] = string(value)
return true
})
req := payloadBody(data)
kafkaMessage := KafkaMessage{
ReqURL: string(proto.Path(req)),
ReqMethod: string(proto.Method(req)),
ReqBody: string(proto.Body(req)),
ReqHeaders: headers,
}
jsonMessage, _ := json.Marshal(&kafkaMessage)
message := sarama.StringEncoder(jsonMessage)
o.producer.Input() <- &sarama.ProducerMessage{
Topic: o.config.topic,
Value: message,
}
return len(message), nil
}
func process(buf []byte) {
// First byte indicate payload type, possible values:
// 1 - Request
// 2 - Response
// 3 - ReplayedResponse
payloadType := buf[0]
headerSize := bytes.IndexByte(buf, '\n') + 1
header := buf[:headerSize-1]
// Header contains space separated values of: request type, request id, and request start time (or round-trip time for responses)
meta := bytes.Split(header, []byte(" "))
// For each request you should receive 3 payloads (request, response, replayed response) with same request id
reqID := string(meta[1])
payload := buf[headerSize:]
Debug("Received payload:", string(buf))
switch payloadType {
case '1': // Request
if bytes.Equal(proto.Path(payload), []byte("/token")) {
originalTokens[reqID] = []byte{}
Debug("Found token request:", reqID)
} else {
token, vs, _ := proto.PathParam(payload, []byte("token"))
if vs != -1 { // If there is GET token param
if alias, ok := tokenAliases[string(token)]; ok {
// Rewrite original token to alias
payload = proto.SetPathParam(payload, []byte("token"), alias)
// Copy modified payload to our buffer
buf = append(buf[:headerSize], payload...)
}
}
}
// Emitting data back
os.Stdout.Write(encode(buf))
case '2': // Original response
if _, ok := originalTokens[reqID]; ok {
// Token is inside response body
secureToken := proto.Body(payload)
originalTokens[reqID] = secureToken
Debug("Remember origial token:", string(secureToken))
}
case '3': // Replayed response
if originalToken, ok := originalTokens[reqID]; ok {
delete(originalTokens, reqID)
secureToken := proto.Body(payload)
tokenAliases[string(originalToken)] = secureToken
Debug("Create alias for new token token, was:", string(originalToken), "now:", string(secureToken))
}
}
}
func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
if len(resp) == 0 {
// nil http response - skipped elasticsearch export for this request
return
}
t := time.Now()
rtt := p.RttDurationToMs(stop.Sub(start))
req = payloadBody(req)
esResp := ESRequestResponse{
ReqURL: string(proto.Path(req)),
ReqMethod: string(proto.Method(req)),
ReqUserAgent: string(proto.Header(req, []byte("User-Agent"))),
ReqAcceptLanguage: string(proto.Header(req, []byte("Accept-Language"))),
ReqAccept: string(proto.Header(req, []byte("Accept"))),
ReqAcceptEncoding: string(proto.Header(req, []byte("Accept-Encoding"))),
ReqIfModifiedSince: string(proto.Header(req, []byte("If-Modified-Since"))),
ReqConnection: string(proto.Header(req, []byte("Connection"))),
ReqCookies: string(proto.Header(req, []byte("Cookie"))),
RespStatus: string(proto.Status(resp)),
RespStatusCode: string(proto.Status(resp)),
RespProto: string(proto.Method(resp)),
RespContentLength: string(proto.Header(resp, []byte("Content-Length"))),
RespContentType: string(proto.Header(resp, []byte("Content-Type"))),
RespTransferEncoding: string(proto.Header(resp, []byte("Transfer-Encoding"))),
RespContentEncoding: string(proto.Header(resp, []byte("Content-Encoding"))),
RespExpires: string(proto.Header(resp, []byte("Expires"))),
RespCacheControl: string(proto.Header(resp, []byte("Cache-Control"))),
RespVary: string(proto.Header(resp, []byte("Vary"))),
RespSetCookie: string(proto.Header(resp, []byte("Set-Cookie"))),
Rtt: rtt,
Timestamp: t,
}
j, err := json.Marshal(&esResp)
if err != nil {
log.Println(err)
} else {
p.indexor.Index(p.Index, "RequestResponse", "", "", "", &t, j)
}
return
}
func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
if len(m.config.methods) > 0 {
method := proto.Method(payload)
matched := false
for _, m := range m.config.methods {
if bytes.Equal(method, m) {
matched = true
break
}
}
if !matched {
return
}
}
if len(m.config.headers) > 0 {
for _, header := range m.config.headers {
payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value))
}
}
if len(m.config.params) > 0 {
for _, param := range m.config.params {
payload = proto.SetPathParam(payload, param.Name, param.Value)
}
}
if len(m.config.urlRegexp) > 0 {
path := proto.Path(payload)
matched := false
for _, f := range m.config.urlRegexp {
if f.regexp.Match(path) {
matched = true
break
}
}
if !matched {
return
}
}
if len(m.config.urlNegativeRegexp) > 0 {
path := proto.Path(payload)
for _, f := range m.config.urlNegativeRegexp {
if f.regexp.Match(path) {
return
}
}
}
if len(m.config.headerFilters) > 0 {
for _, f := range m.config.headerFilters {
value := proto.Header(payload, f.name)
if len(value) > 0 && !f.regexp.Match(value) {
return
}
}
}
if len(m.config.headerNegativeFilters) > 0 {
for _, f := range m.config.headerNegativeFilters {
value := proto.Header(payload, f.name)
if len(value) > 0 && f.regexp.Match(value) {
return
}
}
}
if len(m.config.headerHashFilters) > 0 {
for _, f := range m.config.headerHashFilters {
value := proto.Header(payload, f.name)
if len(value) > 0 {
hasher := fnv.New32a()
hasher.Write(value)
if (hasher.Sum32() % 100) >= f.percent {
return
}
}
}
}
if len(m.config.paramHashFilters) > 0 {
for _, f := range m.config.paramHashFilters {
value, s, _ := proto.PathParam(payload, f.name)
if s != -1 {
hasher := fnv.New32a()
hasher.Write(value)
if (hasher.Sum32() % 100) >= f.percent {
return
}
}
}
}
if len(m.config.urlRewrite) > 0 {
path := proto.Path(payload)
for _, f := range m.config.urlRewrite {
if f.src.Match(path) {
path = f.src.ReplaceAll(path, f.target)
payload = proto.SetPath(payload, path)
break
}
}
}
return payload
}
func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
if len(m.config.methods) > 0 && !m.config.methods.Contains(proto.Method(payload)) {
return
}
if m.config.urlRegexp.regexp != nil {
host, _, _, _ := proto.Header(payload, []byte("Host"))
fullPath := append(host, proto.Path(payload)...)
if !m.config.urlRegexp.regexp.Match(fullPath) {
return
}
}
if len(m.config.headerFilters) > 0 {
for _, f := range m.config.headerFilters {
value, s, _, _ := proto.Header(payload, f.name)
if s != -1 && !f.regexp.Match(value) {
return
}
}
}
if len(m.config.headerHashFilters) > 0 {
for _, f := range m.config.headerHashFilters {
value, s, _, _ := proto.Header(payload, f.name)
if s == -1 {
return
}
hasher := fnv.New32a()
hasher.Write(value)
if (hasher.Sum32() % 100) >= f.percent {
return
}
}
}
if len(m.config.urlRewrite) > 0 {
path := proto.Path(payload)
for _, f := range m.config.urlRewrite {
if f.src.Match(path) {
path = f.src.ReplaceAll(path, f.target)
payload = proto.SetPath(payload, path)
break
}
}
}
if len(m.config.headers) > 0 {
for _, header := range m.config.headers {
payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value))
}
}
return payload
}