func NewForwardInput(logger *logging.Logger, bind string, port Port) (*ForwardInput, error) {
_codec := codec.MsgpackHandle{}
_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
_codec.RawToString = false
addr, err := net.ResolveTCPAddr("tcp", bind)
if err != nil {
logger.Error("%s", err.Error())
return nil, err
}
listener, err := net.ListenTCP("tcp", addr)
if err != nil {
logger.Error("%s", err.Error())
return nil, err
}
return &ForwardInput{
port: port,
logger: logger,
bind: bind,
listener: listener,
codec: &_codec,
clients: make(map[*net.TCPConn]*forwardClient),
clientsMtx: sync.Mutex{},
entries: 0,
wg: sync.WaitGroup{},
acceptChan: make(chan *net.TCPConn),
shutdownChan: make(chan struct{}),
isShuttingDown: uintptr(0),
}, nil
}
func scriptBody(sessionId []byte, graphName, script string, bindings map[string]interface{}) (out []byte, err error) {
var (
mh = new(codec.MsgpackHandle)
enc = codec.NewEncoderBytes(&out, mh)
reqId = uuid.NewV4()
isSessionless = bytes.Equal(sessionId, sessionlessUuid[:])
)
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
meta := map[string]interface{}{
"inSession": !isSessionless,
"isolate": isSessionless,
"graphObjName": "g",
}
if isSessionless {
meta["graphName"] = graphName
}
err = enc.Encode([]interface{}{
sessionId,
reqId[:],
meta,
"groovy",
script,
bindings,
})
return
}
// TBD: Extract more info
func UnpackProxyRequest(raw []byte) (*http.Request, error) {
var (
mh codec.MsgpackHandle
h = &mh
)
var v []interface{}
mh.SliceType = reflect.TypeOf(Headers(nil))
codec.NewDecoderBytes(raw, h).Decode(&v)
r, err := http.NewRequest(string(v[0].([]uint8)), string(v[1].([]uint8)), bytes.NewBuffer(v[4].([]byte)))
if err != nil {
return nil, err
}
r.Header = CocaineHeaderToHttpHeader(v[3].(Headers))
r.Host = r.Header.Get("Host")
if xRealIp := r.Header.Get("X-Real-IP"); xRealIp != "" {
r.RemoteAddr = xRealIp
}
err = decompressBody(r)
if err != nil {
return nil, err
}
return r, nil
}
func TestSpanMsgPack(t *testing.T) {
span := Span{Id: TestId("33f25a1a750a471db5bafa59309d7d6f"),
SpanData: SpanData{
Begin: 1234,
End: 5678,
Description: "getFileDescriptors",
Parents: []SpanId{},
TracerId: "testTracerId",
}}
mh := new(codec.MsgpackHandle)
mh.WriteExt = true
w := bytes.NewBuffer(make([]byte, 0, 2048))
enc := codec.NewEncoder(w, mh)
err := enc.Encode(span)
if err != nil {
t.Fatal("Error encoding span as msgpack: " + err.Error())
}
buf := w.Bytes()
fmt.Printf("span: %s\n", hex.EncodeToString(buf))
mh = new(codec.MsgpackHandle)
mh.WriteExt = true
dec := codec.NewDecoder(bytes.NewReader(buf), mh)
var span2 Span
err = dec.Decode(&span2)
if err != nil {
t.Fatal("Failed to reverse msgpack encoding for " + span.String())
}
ExpectSpansEqual(t, &span, &span2)
}
func DecodeRequest(body []byte, req *Request) (err error) {
mh := codec.MsgpackHandle{RawToString: true}
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
dec := codec.NewDecoderBytes(body, &mh)
err = dec.Decode(req)
return
}
func decodeBody(body []byte) (retVal []interface{}, err error) {
var mh = new(codec.MsgpackHandle)
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
dec := codec.NewDecoderBytes(body, mh)
if e := dec.Decode(&retVal); e != nil {
err = errors.New("rexpro: msgpack decode error")
}
return
}
func (gofetcher *Gofetcher) ParseRequest(method string, requestBody []byte) (request *Request) {
var (
mh codec.MsgpackHandle
h = &mh
timeout int64 = DefaultTimeout
cookies Cookies
headers = make(http.Header)
followRedirects bool = DefaultFollowRedirects
body *bytes.Buffer
)
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
var res []interface{}
codec.NewDecoderBytes(requestBody, h).Decode(&res)
url := string(res[0].([]uint8))
switch {
case method == "GET" || method == "HEAD" || method == "DELETE":
if len(res) > 1 {
timeout = parseTimeout(res[1])
}
if len(res) > 2 {
cookies = parseCookies(res[2].(map[string]interface{}))
}
if len(res) > 3 {
headers = parseHeaders(res[3].(map[string]interface{}))
}
if len(res) > 4 {
followRedirects = res[4].(bool)
}
case method == "POST" || method == "PUT" || method == "PATCH":
if len(res) > 1 {
body = bytes.NewBuffer(res[1].([]byte))
}
if len(res) > 2 {
timeout = parseTimeout(res[2])
}
if len(res) > 3 {
cookies = parseCookies(res[3].(map[string]interface{}))
}
if len(res) > 4 {
headers = parseHeaders(res[4].(map[string]interface{}))
}
if len(res) > 5 {
followRedirects = res[5].(bool)
}
}
request = &Request{Method: method, URL: url, Timeout: timeout,
FollowRedirects: followRedirects,
Cookies: cookies, Headers: headers}
if body != nil {
request.Body = body
}
return request
}
func (cdc *HrpcClientCodec) ReadResponseBody(body interface{}) error {
mh := new(codec.MsgpackHandle)
mh.WriteExt = true
dec := codec.NewDecoder(io.LimitReader(cdc.rwc, int64(cdc.length)), mh)
err := dec.Decode(body)
if err != nil {
return errors.New(fmt.Sprintf("Failed to read response body: %s",
err.Error()))
}
return nil
}
func (cdc *HrpcServerCodec) WriteResponse(resp *rpc.Response, msg interface{}) error {
var err error
buf := EMPTY
if msg != nil {
mh := new(codec.MsgpackHandle)
mh.WriteExt = true
w := bytes.NewBuffer(make([]byte, 0, 128))
enc := codec.NewEncoder(w, mh)
err := enc.Encode(msg)
if err != nil {
return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to marshal "+
"response message: %s", err.Error()))
}
buf = w.Bytes()
}
hdr := common.HrpcResponseHeader{}
hdr.MethodId = common.HrpcMethodNameToId(resp.ServiceMethod)
hdr.Seq = resp.Seq
hdr.ErrLength = uint32(len(resp.Error))
hdr.Length = uint32(len(buf))
writer := bufio.NewWriterSize(cdc.conn, 256)
err = binary.Write(writer, binary.LittleEndian, &hdr)
if err != nil {
return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write response "+
"header: %s", err.Error()))
}
if hdr.ErrLength > 0 {
_, err = io.WriteString(writer, resp.Error)
if err != nil {
return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write error "+
"string: %s", err.Error()))
}
}
if hdr.Length > 0 {
var length int
length, err = writer.Write(buf)
if err != nil {
return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write response "+
"message: %s", err.Error()))
}
if uint32(length) != hdr.Length {
return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write all of "+
"response message: %s", err.Error()))
}
}
err = writer.Flush()
if err != nil {
return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write the response "+
"bytes: %s", err.Error()))
}
return nil
}
func WriteFile(path string, obj interface{}) (err error) {
mh := codec.MsgpackHandle{}
mh.AsSymbols = codec.AsSymbolNone
mh.RawToString = true
buf := &bytes.Buffer{}
encoder := codec.NewEncoder(buf, &mh)
err = encoder.Encode(obj)
if err != nil {
return
}
err = ioutil.WriteFile(path, buf.Bytes(), os.FileMode(0777))
return
}
func (shd *shard) decodeSpan(sid common.SpanId, buf []byte) (*common.Span, error) {
r := bytes.NewBuffer(buf)
mh := new(codec.MsgpackHandle)
mh.WriteExt = true
decoder := codec.NewDecoder(r, mh)
data := common.SpanData{}
err := decoder.Decode(&data)
if err != nil {
return nil, err
}
if data.Parents == nil {
data.Parents = []common.SpanId{}
}
return &common.Span{Id: common.SpanId(sid), SpanData: data}, nil
}
func (shd *ShardLoader) writeShardInfo(info *ShardInfo) error {
mh := new(codec.MsgpackHandle)
mh.WriteExt = true
w := new(bytes.Buffer)
enc := codec.NewEncoder(w, mh)
err := enc.Encode(info)
if err != nil {
return errors.New(fmt.Sprintf("msgpack encoding error: %s",
err.Error()))
}
err = shd.ldb.Put(shd.dld.writeOpts, []byte{SHARD_INFO_KEY}, w.Bytes())
if err != nil {
return errors.New(fmt.Sprintf("leveldb write error: %s",
err.Error()))
}
return nil
}
// TBD: Extract more info
func UnpackProxyRequest(raw []byte) *http.Request {
var (
mh codec.MsgpackHandle
h = &mh
)
var v []interface{}
mh.SliceType = reflect.TypeOf(Headers(nil))
codec.NewDecoderBytes(raw, h).Decode(&v)
r, err := http.NewRequest(string(v[0].([]uint8)), string(v[1].([]uint8)), bytes.NewBuffer(v[4].([]byte)))
if err != nil {
fmt.Println("Error", err)
}
r.Header = CocaineHeaderToHttpHeader(v[3].(Headers))
if xRealIp := r.Header.Get("X-Real-IP"); xRealIp != "" {
r.RemoteAddr = xRealIp
}
return r
}
func (cdc *HrpcServerCodec) ReadRequestBody(body interface{}) error {
if cdc.lg.TraceEnabled() {
cdc.lg.Tracef("Reading HRPC %d-byte request body from %s\n",
cdc.length, cdc.conn.RemoteAddr())
}
mh := new(codec.MsgpackHandle)
mh.WriteExt = true
dec := codec.NewDecoder(io.LimitReader(cdc.conn, int64(cdc.length)), mh)
err := dec.Decode(body)
if err != nil {
return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to read request "+
"body from %s: %s", cdc.conn.RemoteAddr(), err.Error()))
}
if cdc.lg.TraceEnabled() {
cdc.lg.Tracef("Read body from %s: %s\n",
cdc.conn.RemoteAddr(), asJson(&body))
}
return nil
}
func (s *session) sessionBody(kill bool) (out []byte, err error) {
var (
mh = new(codec.MsgpackHandle)
enc = codec.NewEncoderBytes(&out, mh)
reqId = uuid.NewV4()
)
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
err = enc.Encode([]interface{}{
s.sId,
reqId[:],
map[string]interface{}{
"graphName": s.r.graphName,
"graphObjName": "g",
"killSession": kill,
},
s.username,
s.password,
})
return
}
func newForwardInput(factory *ForwardInputFactory, logger *log.Logger, bind string, port ik.Port) (*ForwardInput, error) {
_codec := codec.MsgpackHandle{}
_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
_codec.RawToString = false
listener, err := net.Listen("tcp", bind)
if err != nil {
logger.Fatal(err.Error())
return nil, err
}
return &ForwardInput{
factory: factory,
port: port,
logger: logger,
bind: bind,
listener: listener,
codec: &_codec,
running: false,
clients: make(map[net.Conn]*forwardClient),
}, nil
}
func (cdc *HrpcClientCodec) WriteRequest(req *rpc.Request, msg interface{}) error {
methodId := common.HrpcMethodNameToId(req.ServiceMethod)
if methodId == common.METHOD_ID_NONE {
return errors.New(fmt.Sprintf("HrpcClientCodec: Unknown method name %s",
req.ServiceMethod))
}
mh := new(codec.MsgpackHandle)
mh.WriteExt = true
w := bytes.NewBuffer(make([]byte, 0, 2048))
enc := codec.NewEncoder(w, mh)
err := enc.Encode(msg)
if err != nil {
return errors.New(fmt.Sprintf("HrpcClientCodec: Unable to marshal "+
"message as msgpack: %s", err.Error()))
}
buf := w.Bytes()
if len(buf) > common.MAX_HRPC_BODY_LENGTH {
return errors.New(fmt.Sprintf("HrpcClientCodec: message body is %d "+
"bytes, but the maximum message size is %d bytes.",
len(buf), common.MAX_HRPC_BODY_LENGTH))
}
hdr := common.HrpcRequestHeader{
Magic: common.HRPC_MAGIC,
MethodId: methodId,
Seq: req.Seq,
Length: uint32(len(buf)),
}
err = binary.Write(cdc.rwc, binary.LittleEndian, &hdr)
if err != nil {
return errors.New(fmt.Sprintf("Error writing header bytes: %s",
err.Error()))
}
_, err = cdc.rwc.Write(buf)
if err != nil {
return errors.New(fmt.Sprintf("Error writing body bytes: %s",
err.Error()))
}
return nil
}
func NewForwardOutput(logger *logging.Logger, bind string, retryInterval time.Duration, connectionTimeout time.Duration, writeTimeout time.Duration, flushInterval time.Duration, journalGroupPath string, maxJournalChunkSize int64, metadata string) (*ForwardOutput, error) {
_codec := codec.MsgpackHandle{}
_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
_codec.RawToString = false
_codec.StructToArray = true
journalFactory := NewFileJournalGroupFactory(
logger,
randSource,
time.Now,
".log",
os.FileMode(0600),
maxJournalChunkSize,
)
output := &ForwardOutput{
logger: logger,
codec: &_codec,
bind: bind,
retryInterval: retryInterval,
connectionTimeout: connectionTimeout,
writeTimeout: writeTimeout,
wg: sync.WaitGroup{},
flushInterval: flushInterval,
emitterChan: make(chan FluentRecordSet),
spoolerShutdownChan: make(chan struct{}),
isShuttingDown: 0,
completion: sync.Cond{L: &sync.Mutex{}},
hasShutdownCompleted: false,
metadata: metadata,
}
journalGroup, err := journalFactory.GetJournalGroup(journalGroupPath, output)
if err != nil {
return nil, err
}
output.journalGroup = journalGroup
output.journal = journalGroup.GetJournal("output")
return output, nil
}
func (locator *Locator) unpackchunk(chunk RawMessage) ResolveResult {
defer func() {
if err := recover(); err != nil {
log.Println("defer", err)
}
}()
var v []interface{}
var MH codec.MsgpackHandle
MH.MapType = reflect.TypeOf(map[int]string(nil))
err := codec.NewDecoderBytes(chunk, &MH).Decode(&v)
if err != nil {
log.Println("unpack chunk error", err)
}
if len(v) == 3 {
v_endpoint := v[0].([]interface{})
endpoint := Endpoint{string(v_endpoint[0].([]uint8)), v_endpoint[1].(uint64)}
version := v[1].(int64)
api := v[2].(map[int]string)
return ResolveResult{true, endpoint, version, api}
} else {
panic("Bad format")
}
}
func (shd *ShardLoader) readShardInfo() (*ShardInfo, error) {
buf, err := shd.ldb.Get(shd.dld.readOpts, []byte{SHARD_INFO_KEY})
if err != nil {
return nil, errors.New(fmt.Sprintf("readShardInfo(%s): failed to "+
"read shard info key: %s", shd.path, err.Error()))
}
if len(buf) == 0 {
return nil, errors.New(fmt.Sprintf("readShardInfo(%s): got zero-"+
"length value for shard info key.", shd.path))
}
mh := new(codec.MsgpackHandle)
mh.WriteExt = true
r := bytes.NewBuffer(buf)
decoder := codec.NewDecoder(r, mh)
shardInfo := &ShardInfo{
LayoutVersion: UNKNOWN_LAYOUT_VERSION,
}
err = decoder.Decode(shardInfo)
if err != nil {
return nil, errors.New(fmt.Sprintf("readShardInfo(%s): msgpack "+
"decoding failed for shard info key: %s", shd.path, err.Error()))
}
return shardInfo, nil
}
func doWriteSpans(name string, N int, maxSpansPerRpc uint32, b *testing.B) {
htraceBld := &MiniHTracedBuilder{Name: "doWriteSpans",
Cnf: map[string]string{
conf.HTRACE_LOG_LEVEL: "INFO",
conf.HTRACE_NUM_HRPC_HANDLERS: "20",
},
WrittenSpans: common.NewSemaphore(int64(1 - N)),
}
ht, err := htraceBld.Build()
if err != nil {
panic(err)
}
defer ht.Close()
rnd := rand.New(rand.NewSource(1))
allSpans := make([]*common.Span, N)
for n := 0; n < N; n++ {
allSpans[n] = test.NewRandomSpan(rnd, allSpans[0:n])
}
// Determine how many calls to WriteSpans we should make. Each writeSpans
// message should be small enough so that it doesn't exceed the max RPC
// body length limit. TODO: a production-quality golang client would do
// this internally rather than needing us to do it here in the unit test.
bodyLen := (4 * common.MAX_HRPC_BODY_LENGTH) / 5
reqs := make([][]*common.Span, 0, 4)
curReq := -1
curReqLen := bodyLen
var curReqSpans uint32
mh := new(codec.MsgpackHandle)
mh.WriteExt = true
var mbuf [8192]byte
buf := mbuf[:0]
enc := codec.NewEncoderBytes(&buf, mh)
for n := 0; n < N; n++ {
span := allSpans[n]
if (curReqSpans >= maxSpansPerRpc) ||
(curReqLen >= bodyLen) {
reqs = append(reqs, make([]*common.Span, 0, 16))
curReqLen = 0
curReq++
curReqSpans = 0
}
buf = mbuf[:0]
enc.ResetBytes(&buf)
err := enc.Encode(span)
if err != nil {
panic(fmt.Sprintf("Error encoding span %s: %s\n",
span.String(), err.Error()))
}
bufLen := len(buf)
if bufLen > (bodyLen / 5) {
panic(fmt.Sprintf("Span too long at %d bytes\n", bufLen))
}
curReqLen += bufLen
reqs[curReq] = append(reqs[curReq], span)
curReqSpans++
}
ht.Store.lg.Infof("num spans: %d. num WriteSpansReq calls: %d\n", N, len(reqs))
var hcl *htrace.Client
hcl, err = htrace.NewClient(ht.ClientConf(), nil)
if err != nil {
panic(fmt.Sprintf("failed to create client: %s", err.Error()))
}
defer hcl.Close()
// Reset the timer to avoid including the time required to create new
// random spans in the benchmark total.
if b != nil {
b.ResetTimer()
}
// Write many random spans.
for reqIdx := range reqs {
go func(i int) {
err = hcl.WriteSpans(reqs[i])
if err != nil {
panic(fmt.Sprintf("failed to send WriteSpans request %d: %s",
i, err.Error()))
}
}(reqIdx)
}
// Wait for all the spans to be written.
ht.Store.WrittenSpans.Wait()
}
func DecodeEntries(conn net.Conn) ([]FluentRecordSet, error) {
var mh codec.MsgpackHandle
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
dec := codec.NewDecoder(conn, &mh)
v := []interface{}{nil, nil, nil}
err := dec.Decode(&v)
if err != nil {
return nil, err
}
tag, ok := v[0].([]byte)
if !ok {
return nil, errors.New("Failed to decode tag field")
}
var retval []FluentRecordSet
switch timestamp_or_entries := v[1].(type) {
case int, uint, int64, uint64, int32, uint32, float32, float64:
timestamp := toInt64(timestamp_or_entries)
data, ok := v[2].(map[string]interface{})
if !ok {
return nil, errors.New("Failed to decode data field")
}
coerceInPlace(data)
retval = []FluentRecordSet{
{
Tag: string(tag), // XXX: byte => rune
Records: []FluentRecordType{
&TinyFluentRecord{
Timestamp: int64(timestamp),
Data: data,
},
},
},
}
case []interface{}: // Forward
if !ok {
return nil, errors.New("Unexpected payload format")
}
recordSet, err := decodeRecordSet(tag, timestamp_or_entries)
if err != nil {
return nil, err
}
retval = []FluentRecordSet{recordSet}
case []byte: // PackedForward
reader := bytes.NewReader(timestamp_or_entries)
entries := make([]interface{}, 0)
for {
entry := make([]interface{}, 0)
err := codec.NewDecoder(reader, &mh).Decode(&entry)
if err == io.EOF {
break
} else if err != nil {
return nil, errors.New("Unexpected payload format")
}
entries = append(entries, entry)
}
recordSet, err := decodeRecordSet(tag, entries)
if err != nil {
return nil, err
}
retval = []FluentRecordSet{recordSet}
default:
return nil, errors.New(fmt.Sprintf("Unknown type: %t", timestamp_or_entries))
}
return retval, nil
}
func getHandle() codec.MsgpackHandle {
h := codec.MsgpackHandle{RawToString: true}
h.MapType = reflect.TypeOf(map[string]interface{}(nil))
return h
}
func (self *OutputForward) Init(config map[string]string) error {
_codec := codec.MsgpackHandle{}
_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
_codec.RawToString = false
_codec.StructToArray = true
self.host = "localhost"
self.port = 8888
self.flush_interval = 10
self.sync_interval = 2
self.buffer_path = "/tmp/test"
self.buffer_queue_limit = 64 * 1024 * 1024
self.buffer_chunk_limit = 8 * 1024 * 1024
self.connect_timeout = 10
self.codec = &_codec
value := config["host"]
if len(value) > 0 {
self.host = value
}
value = config["port"]
if len(value) > 0 {
self.port, _ = strconv.Atoi(value)
}
value = config["connect_timeout"]
if len(value) > 0 {
self.connect_timeout, _ = strconv.Atoi(value)
}
value = config["flush_interval"]
if len(value) > 0 {
self.flush_interval, _ = strconv.Atoi(value)
}
value = config["sync_interval"]
if len(value) > 0 {
sync_interval, err := strconv.Atoi(value)
if err != nil {
return err
}
self.sync_interval = sync_interval
}
value = config["buffer_path"]
if len(value) > 0 {
self.buffer_path = value
}
value = config["buffer_queue_limit"]
if len(value) > 0 {
buffer_queue_limit, err := strconv.Atoi(value)
if err != nil {
return err
}
self.buffer_queue_limit = int64(buffer_queue_limit) * 1024 * 1024
}
value = config["buffer_chunk_limit"]
if len(value) > 0 {
buffer_chunk_limit, err := strconv.Atoi(value)
if err != nil {
return err
}
self.buffer_chunk_limit = int64(buffer_chunk_limit) * 1024 * 1024
}
return nil
}
func ParseParams(req *http.Request) *Params {
var p Params
ct := req.Header.Get("Content-Type")
ct = strings.Split(ct, ";")[0]
if ct == "multipart/form-data" {
if err := req.ParseMultipartForm(10000000); err != nil {
log.Println("Request.ParseMultipartForm Error", err)
}
} else {
if err := req.ParseForm(); err != nil {
log.Println("Request.ParseForm Error", err)
}
}
tmap := make(map[string]interface{}, len(req.Form))
for k, v := range req.Form {
if strings.ToLower(v[0]) == "true" {
tmap[k] = true
} else if strings.ToLower(v[0]) == "false" {
tmap[k] = false
} else {
tmap[k] = v[0]
}
}
if req.MultipartForm != nil {
for k, v := range req.MultipartForm.File {
tmap[k] = v[0]
}
}
if ct == "application/json" && req.ContentLength > 0 {
err := json.NewDecoder(req.Body).Decode(&p.Values)
if err != nil {
log.Println("Content-Type is \"application/json\" but no valid json data received", err)
p.Values = tmap
}
for k, v := range tmap {
if _, pres := p.Values[k]; !pres {
p.Values[k] = v
}
}
} else if ct == "application/x-msgpack" {
var mh codec.MsgpackHandle
p.isBinary = true
mh.MapType = reflect.TypeOf(p.Values)
body, _ := ioutil.ReadAll(req.Body)
if len(body) > 0 {
buff := bytes.NewBuffer(body)
first := body[0]
if (first >= 0x80 && first <= 0x8f) || (first == 0xde || first == 0xdf) {
err := codec.NewDecoder(buff, &mh).Decode(&p.Values)
if err != nil && err != io.EOF {
log.Println("Failed decoding msgpack", err)
}
} else {
if p.Values == nil {
p.Values = make(map[string]interface{}, 0)
}
var err error
for err == nil {
vals := make([]interface{}, 0)
err = codec.NewDecoder(buff, &mh).Decode(&vals)
if err != nil && err != io.EOF {
log.Println("Failed decoding msgpack", err)
} else {
for i := len(vals) - 1; i >= 1; i -= 2 {
p.Values[string(vals[i-1].([]byte))] = vals[i]
}
}
}
}
} else {
p.Values = make(map[string]interface{}, 0)
}
for k, v := range tmap {
if _, pres := p.Values[k]; !pres {
p.Values[k] = v
}
}
} else {
p.Values = tmap
}
for k, v := range mux.Vars(req) {
const ID = "id"
if strings.Contains(k, ID) {
id, perr := strconv.ParseUint(v, 10, 64)
if perr != nil {
p.Values[k] = v
} else {
p.Values[k] = id
}
} else {
p.Values[k] = v
}
}
return &p
}
func NewTDOutput(
logger *logging.Logger,
endpoint string,
connectionTimeout time.Duration,
writeTimeout time.Duration,
flushInterval time.Duration,
parallelism int,
journalGroupPath string,
maxJournalChunkSize int64,
apiKey string,
databaseName string,
tableName string,
tempDir string,
useSsl bool,
rootCAs *x509.CertPool,
httpProxy string,
metadata string,
) (*TDOutput, error) {
_codec := codec.MsgpackHandle{}
_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
_codec.RawToString = false
_codec.StructToArray = true
journalFactory := NewFileJournalGroupFactory(
logger,
randSource,
time.Now,
".log",
os.FileMode(0600),
maxJournalChunkSize,
)
router := (td_client.EndpointRouter)(nil)
if endpoint != "" {
router = &td_client.FixedEndpointRouter{endpoint}
}
httpProxy_ := (interface{})(nil)
if httpProxy != "" {
httpProxy_ = httpProxy
}
client, err := td_client.NewTDClient(td_client.Settings{
ApiKey: apiKey,
Router: router,
ConnectionTimeout: connectionTimeout,
// ReadTimeout: readTimeout, // TODO
SendTimeout: writeTimeout,
Ssl: useSsl,
RootCAs: rootCAs,
Proxy: httpProxy_,
})
if err != nil {
return nil, err
}
output := &TDOutput{
logger: logger,
codec: &_codec,
wg: sync.WaitGroup{},
flushInterval: flushInterval,
emitterChan: make(chan FluentRecordSet),
isShuttingDown: 0,
client: client,
databaseName: databaseName,
tableName: tableName,
tempDir: tempDir,
sem: make(chan struct{}, parallelism),
gcChan: make(chan *os.File, 10),
completion: sync.Cond{L: &sync.Mutex{}},
hasShutdownCompleted: false,
metadata: metadata,
}
journalGroup, err := journalFactory.GetJournalGroup(journalGroupPath, output)
if err != nil {
return nil, err
}
output.journalGroup = journalGroup
return output, nil
}
func codecHandle() *codec.MsgpackHandle {
var mh codec.MsgpackHandle
mh.WriteExt = true
return &mh
}
func (cdc *HrpcClientCodec) WriteRequest(rr *rpc.Request, msg interface{}) error {
methodId := common.HrpcMethodNameToId(rr.ServiceMethod)
if methodId == common.METHOD_ID_NONE {
return errors.New(fmt.Sprintf("HrpcClientCodec: Unknown method name %s",
rr.ServiceMethod))
}
mh := new(codec.MsgpackHandle)
mh.WriteExt = true
w := bytes.NewBuffer(make([]byte, 0, 2048))
var err error
enc := codec.NewEncoder(w, mh)
if methodId == common.METHOD_ID_WRITE_SPANS {
spans := msg.([]*common.Span)
req := &common.WriteSpansReq{
NumSpans: len(spans),
}
err = enc.Encode(req)
if err != nil {
return errors.New(fmt.Sprintf("HrpcClientCodec: Unable to marshal "+
"message as msgpack: %s", err.Error()))
}
for spanIdx := range spans {
err = enc.Encode(spans[spanIdx])
if err != nil {
return errors.New(fmt.Sprintf("HrpcClientCodec: Unable to marshal "+
"span %d out of %d as msgpack: %s", spanIdx, len(spans), err.Error()))
}
}
} else {
err = enc.Encode(msg)
if err != nil {
return errors.New(fmt.Sprintf("HrpcClientCodec: Unable to marshal "+
"message as msgpack: %s", err.Error()))
}
}
buf := w.Bytes()
if len(buf) > common.MAX_HRPC_BODY_LENGTH {
return errors.New(fmt.Sprintf("HrpcClientCodec: message body is %d "+
"bytes, but the maximum message size is %d bytes.",
len(buf), common.MAX_HRPC_BODY_LENGTH))
}
hdr := common.HrpcRequestHeader{
Magic: common.HRPC_MAGIC,
MethodId: methodId,
Seq: rr.Seq,
Length: uint32(len(buf)),
}
err = binary.Write(cdc.rwc, binary.LittleEndian, &hdr)
if err != nil {
return errors.New(fmt.Sprintf("Error writing header bytes: %s",
err.Error()))
}
if cdc.testHooks != nil && cdc.testHooks.HandleWriteRequestBody != nil {
cdc.testHooks.HandleWriteRequestBody()
}
_, err = cdc.rwc.Write(buf)
if err != nil {
return errors.New(fmt.Sprintf("Error writing body bytes: %s",
err.Error()))
}
return nil
}