Beispiel #1
0
func NewForwardInput(logger *logging.Logger, bind string, port Port) (*ForwardInput, error) {
	_codec := codec.MsgpackHandle{}
	_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
	_codec.RawToString = false
	addr, err := net.ResolveTCPAddr("tcp", bind)
	if err != nil {
		logger.Error("%s", err.Error())
		return nil, err
	}
	listener, err := net.ListenTCP("tcp", addr)
	if err != nil {
		logger.Error("%s", err.Error())
		return nil, err
	}
	return &ForwardInput{
		port:           port,
		logger:         logger,
		bind:           bind,
		listener:       listener,
		codec:          &_codec,
		clients:        make(map[*net.TCPConn]*forwardClient),
		clientsMtx:     sync.Mutex{},
		entries:        0,
		wg:             sync.WaitGroup{},
		acceptChan:     make(chan *net.TCPConn),
		shutdownChan:   make(chan struct{}),
		isShuttingDown: uintptr(0),
	}, nil
}
Beispiel #2
0
func scriptBody(sessionId []byte, graphName, script string, bindings map[string]interface{}) (out []byte, err error) {
	var (
		mh            = new(codec.MsgpackHandle)
		enc           = codec.NewEncoderBytes(&out, mh)
		reqId         = uuid.NewV4()
		isSessionless = bytes.Equal(sessionId, sessionlessUuid[:])
	)
	mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
	meta := map[string]interface{}{
		"inSession":    !isSessionless,
		"isolate":      isSessionless,
		"graphObjName": "g",
	}
	if isSessionless {
		meta["graphName"] = graphName
	}
	err = enc.Encode([]interface{}{
		sessionId,
		reqId[:],
		meta,
		"groovy",
		script,
		bindings,
	})
	return
}
// TBD: Extract more info
func UnpackProxyRequest(raw []byte) (*http.Request, error) {
	var (
		mh codec.MsgpackHandle
		h  = &mh
	)
	var v []interface{}
	mh.SliceType = reflect.TypeOf(Headers(nil))
	codec.NewDecoderBytes(raw, h).Decode(&v)
	r, err := http.NewRequest(string(v[0].([]uint8)), string(v[1].([]uint8)), bytes.NewBuffer(v[4].([]byte)))
	if err != nil {
		return nil, err
	}
	r.Header = CocaineHeaderToHttpHeader(v[3].(Headers))

	r.Host = r.Header.Get("Host")

	if xRealIp := r.Header.Get("X-Real-IP"); xRealIp != "" {
		r.RemoteAddr = xRealIp
	}

	err = decompressBody(r)
	if err != nil {
		return nil, err
	}

	return r, nil
}
func TestSpanMsgPack(t *testing.T) {
	span := Span{Id: TestId("33f25a1a750a471db5bafa59309d7d6f"),
		SpanData: SpanData{
			Begin:       1234,
			End:         5678,
			Description: "getFileDescriptors",
			Parents:     []SpanId{},
			TracerId:    "testTracerId",
		}}
	mh := new(codec.MsgpackHandle)
	mh.WriteExt = true
	w := bytes.NewBuffer(make([]byte, 0, 2048))
	enc := codec.NewEncoder(w, mh)
	err := enc.Encode(span)
	if err != nil {
		t.Fatal("Error encoding span as msgpack: " + err.Error())
	}
	buf := w.Bytes()
	fmt.Printf("span: %s\n", hex.EncodeToString(buf))
	mh = new(codec.MsgpackHandle)
	mh.WriteExt = true
	dec := codec.NewDecoder(bytes.NewReader(buf), mh)
	var span2 Span
	err = dec.Decode(&span2)
	if err != nil {
		t.Fatal("Failed to reverse msgpack encoding for " + span.String())
	}
	ExpectSpansEqual(t, &span, &span2)
}
Beispiel #5
0
func DecodeRequest(body []byte, req *Request) (err error) {
	mh := codec.MsgpackHandle{RawToString: true}
	mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
	dec := codec.NewDecoderBytes(body, &mh)
	err = dec.Decode(req)
	return
}
Beispiel #6
0
func decodeBody(body []byte) (retVal []interface{}, err error) {
	var mh = new(codec.MsgpackHandle)
	mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
	dec := codec.NewDecoderBytes(body, mh)
	if e := dec.Decode(&retVal); e != nil {
		err = errors.New("rexpro: msgpack decode error")
	}
	return
}
Beispiel #7
0
func (gofetcher *Gofetcher) ParseRequest(method string, requestBody []byte) (request *Request) {
	var (
		mh              codec.MsgpackHandle
		h                     = &mh
		timeout         int64 = DefaultTimeout
		cookies         Cookies
		headers              = make(http.Header)
		followRedirects bool = DefaultFollowRedirects
		body            *bytes.Buffer
	)
	mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
	var res []interface{}
	codec.NewDecoderBytes(requestBody, h).Decode(&res)
	url := string(res[0].([]uint8))
	switch {
	case method == "GET" || method == "HEAD" || method == "DELETE":
		if len(res) > 1 {
			timeout = parseTimeout(res[1])
		}
		if len(res) > 2 {
			cookies = parseCookies(res[2].(map[string]interface{}))
		}
		if len(res) > 3 {
			headers = parseHeaders(res[3].(map[string]interface{}))
		}
		if len(res) > 4 {
			followRedirects = res[4].(bool)
		}
	case method == "POST" || method == "PUT" || method == "PATCH":
		if len(res) > 1 {
			body = bytes.NewBuffer(res[1].([]byte))
		}
		if len(res) > 2 {
			timeout = parseTimeout(res[2])
		}
		if len(res) > 3 {
			cookies = parseCookies(res[3].(map[string]interface{}))
		}
		if len(res) > 4 {
			headers = parseHeaders(res[4].(map[string]interface{}))
		}
		if len(res) > 5 {
			followRedirects = res[5].(bool)
		}
	}

	request = &Request{Method: method, URL: url, Timeout: timeout,
		FollowRedirects: followRedirects,
		Cookies:         cookies, Headers: headers}
	if body != nil {
		request.Body = body
	}
	return request
}
Beispiel #8
0
func (cdc *HrpcClientCodec) ReadResponseBody(body interface{}) error {
	mh := new(codec.MsgpackHandle)
	mh.WriteExt = true
	dec := codec.NewDecoder(io.LimitReader(cdc.rwc, int64(cdc.length)), mh)
	err := dec.Decode(body)
	if err != nil {
		return errors.New(fmt.Sprintf("Failed to read response body: %s",
			err.Error()))
	}
	return nil
}
Beispiel #9
0
func (cdc *HrpcServerCodec) WriteResponse(resp *rpc.Response, msg interface{}) error {
	var err error
	buf := EMPTY
	if msg != nil {
		mh := new(codec.MsgpackHandle)
		mh.WriteExt = true
		w := bytes.NewBuffer(make([]byte, 0, 128))
		enc := codec.NewEncoder(w, mh)
		err := enc.Encode(msg)
		if err != nil {
			return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to marshal "+
				"response message: %s", err.Error()))
		}
		buf = w.Bytes()
	}
	hdr := common.HrpcResponseHeader{}
	hdr.MethodId = common.HrpcMethodNameToId(resp.ServiceMethod)
	hdr.Seq = resp.Seq
	hdr.ErrLength = uint32(len(resp.Error))
	hdr.Length = uint32(len(buf))
	writer := bufio.NewWriterSize(cdc.conn, 256)
	err = binary.Write(writer, binary.LittleEndian, &hdr)
	if err != nil {
		return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write response "+
			"header: %s", err.Error()))
	}
	if hdr.ErrLength > 0 {
		_, err = io.WriteString(writer, resp.Error)
		if err != nil {
			return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write error "+
				"string: %s", err.Error()))
		}
	}
	if hdr.Length > 0 {
		var length int
		length, err = writer.Write(buf)
		if err != nil {
			return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write response "+
				"message: %s", err.Error()))
		}
		if uint32(length) != hdr.Length {
			return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write all of "+
				"response message: %s", err.Error()))
		}
	}
	err = writer.Flush()
	if err != nil {
		return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to write the response "+
			"bytes: %s", err.Error()))
	}
	return nil
}
Beispiel #10
0
func WriteFile(path string, obj interface{}) (err error) {
	mh := codec.MsgpackHandle{}
	mh.AsSymbols = codec.AsSymbolNone
	mh.RawToString = true
	buf := &bytes.Buffer{}
	encoder := codec.NewEncoder(buf, &mh)
	err = encoder.Encode(obj)
	if err != nil {
		return
	}
	err = ioutil.WriteFile(path, buf.Bytes(), os.FileMode(0777))
	return
}
func (shd *shard) decodeSpan(sid common.SpanId, buf []byte) (*common.Span, error) {
	r := bytes.NewBuffer(buf)
	mh := new(codec.MsgpackHandle)
	mh.WriteExt = true
	decoder := codec.NewDecoder(r, mh)
	data := common.SpanData{}
	err := decoder.Decode(&data)
	if err != nil {
		return nil, err
	}
	if data.Parents == nil {
		data.Parents = []common.SpanId{}
	}
	return &common.Span{Id: common.SpanId(sid), SpanData: data}, nil
}
Beispiel #12
0
func (shd *ShardLoader) writeShardInfo(info *ShardInfo) error {
	mh := new(codec.MsgpackHandle)
	mh.WriteExt = true
	w := new(bytes.Buffer)
	enc := codec.NewEncoder(w, mh)
	err := enc.Encode(info)
	if err != nil {
		return errors.New(fmt.Sprintf("msgpack encoding error: %s",
			err.Error()))
	}
	err = shd.ldb.Put(shd.dld.writeOpts, []byte{SHARD_INFO_KEY}, w.Bytes())
	if err != nil {
		return errors.New(fmt.Sprintf("leveldb write error: %s",
			err.Error()))
	}
	return nil
}
Beispiel #13
0
// TBD: Extract more info
func UnpackProxyRequest(raw []byte) *http.Request {
	var (
		mh codec.MsgpackHandle
		h  = &mh
	)
	var v []interface{}
	mh.SliceType = reflect.TypeOf(Headers(nil))
	codec.NewDecoderBytes(raw, h).Decode(&v)
	r, err := http.NewRequest(string(v[0].([]uint8)), string(v[1].([]uint8)), bytes.NewBuffer(v[4].([]byte)))
	if err != nil {
		fmt.Println("Error", err)
	}
	r.Header = CocaineHeaderToHttpHeader(v[3].(Headers))
	if xRealIp := r.Header.Get("X-Real-IP"); xRealIp != "" {
		r.RemoteAddr = xRealIp
	}
	return r
}
Beispiel #14
0
func (cdc *HrpcServerCodec) ReadRequestBody(body interface{}) error {
	if cdc.lg.TraceEnabled() {
		cdc.lg.Tracef("Reading HRPC %d-byte request body from %s\n",
			cdc.length, cdc.conn.RemoteAddr())
	}
	mh := new(codec.MsgpackHandle)
	mh.WriteExt = true
	dec := codec.NewDecoder(io.LimitReader(cdc.conn, int64(cdc.length)), mh)
	err := dec.Decode(body)
	if err != nil {
		return createErrAndWarn(cdc.lg, fmt.Sprintf("Failed to read request "+
			"body from %s: %s", cdc.conn.RemoteAddr(), err.Error()))
	}
	if cdc.lg.TraceEnabled() {
		cdc.lg.Tracef("Read body from %s: %s\n",
			cdc.conn.RemoteAddr(), asJson(&body))
	}
	return nil
}
Beispiel #15
0
func (s *session) sessionBody(kill bool) (out []byte, err error) {
	var (
		mh    = new(codec.MsgpackHandle)
		enc   = codec.NewEncoderBytes(&out, mh)
		reqId = uuid.NewV4()
	)
	mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
	err = enc.Encode([]interface{}{
		s.sId,
		reqId[:],
		map[string]interface{}{
			"graphName":    s.r.graphName,
			"graphObjName": "g",
			"killSession":  kill,
		},
		s.username,
		s.password,
	})
	return
}
Beispiel #16
0
func newForwardInput(factory *ForwardInputFactory, logger *log.Logger, bind string, port ik.Port) (*ForwardInput, error) {
	_codec := codec.MsgpackHandle{}
	_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
	_codec.RawToString = false
	listener, err := net.Listen("tcp", bind)
	if err != nil {
		logger.Fatal(err.Error())
		return nil, err
	}
	return &ForwardInput{
		factory:  factory,
		port:     port,
		logger:   logger,
		bind:     bind,
		listener: listener,
		codec:    &_codec,
		running:  false,
		clients:  make(map[net.Conn]*forwardClient),
	}, nil
}
Beispiel #17
0
func (cdc *HrpcClientCodec) WriteRequest(req *rpc.Request, msg interface{}) error {
	methodId := common.HrpcMethodNameToId(req.ServiceMethod)
	if methodId == common.METHOD_ID_NONE {
		return errors.New(fmt.Sprintf("HrpcClientCodec: Unknown method name %s",
			req.ServiceMethod))
	}
	mh := new(codec.MsgpackHandle)
	mh.WriteExt = true
	w := bytes.NewBuffer(make([]byte, 0, 2048))
	enc := codec.NewEncoder(w, mh)
	err := enc.Encode(msg)
	if err != nil {
		return errors.New(fmt.Sprintf("HrpcClientCodec: Unable to marshal "+
			"message as msgpack: %s", err.Error()))
	}
	buf := w.Bytes()
	if len(buf) > common.MAX_HRPC_BODY_LENGTH {
		return errors.New(fmt.Sprintf("HrpcClientCodec: message body is %d "+
			"bytes, but the maximum message size is %d bytes.",
			len(buf), common.MAX_HRPC_BODY_LENGTH))
	}
	hdr := common.HrpcRequestHeader{
		Magic:    common.HRPC_MAGIC,
		MethodId: methodId,
		Seq:      req.Seq,
		Length:   uint32(len(buf)),
	}
	err = binary.Write(cdc.rwc, binary.LittleEndian, &hdr)
	if err != nil {
		return errors.New(fmt.Sprintf("Error writing header bytes: %s",
			err.Error()))
	}
	_, err = cdc.rwc.Write(buf)
	if err != nil {
		return errors.New(fmt.Sprintf("Error writing body bytes: %s",
			err.Error()))
	}
	return nil
}
Beispiel #18
0
func NewForwardOutput(logger *logging.Logger, bind string, retryInterval time.Duration, connectionTimeout time.Duration, writeTimeout time.Duration, flushInterval time.Duration, journalGroupPath string, maxJournalChunkSize int64, metadata string) (*ForwardOutput, error) {
	_codec := codec.MsgpackHandle{}
	_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
	_codec.RawToString = false
	_codec.StructToArray = true

	journalFactory := NewFileJournalGroupFactory(
		logger,
		randSource,
		time.Now,
		".log",
		os.FileMode(0600),
		maxJournalChunkSize,
	)
	output := &ForwardOutput{
		logger:               logger,
		codec:                &_codec,
		bind:                 bind,
		retryInterval:        retryInterval,
		connectionTimeout:    connectionTimeout,
		writeTimeout:         writeTimeout,
		wg:                   sync.WaitGroup{},
		flushInterval:        flushInterval,
		emitterChan:          make(chan FluentRecordSet),
		spoolerShutdownChan:  make(chan struct{}),
		isShuttingDown:       0,
		completion:           sync.Cond{L: &sync.Mutex{}},
		hasShutdownCompleted: false,
		metadata:             metadata,
	}
	journalGroup, err := journalFactory.GetJournalGroup(journalGroupPath, output)
	if err != nil {
		return nil, err
	}
	output.journalGroup = journalGroup
	output.journal = journalGroup.GetJournal("output")
	return output, nil
}
Beispiel #19
0
func (locator *Locator) unpackchunk(chunk RawMessage) ResolveResult {
	defer func() {
		if err := recover(); err != nil {
			log.Println("defer", err)
		}
	}()
	var v []interface{}
	var MH codec.MsgpackHandle
	MH.MapType = reflect.TypeOf(map[int]string(nil))
	err := codec.NewDecoderBytes(chunk, &MH).Decode(&v)
	if err != nil {
		log.Println("unpack chunk error", err)
	}
	if len(v) == 3 {
		v_endpoint := v[0].([]interface{})
		endpoint := Endpoint{string(v_endpoint[0].([]uint8)), v_endpoint[1].(uint64)}
		version := v[1].(int64)
		api := v[2].(map[int]string)
		return ResolveResult{true, endpoint, version, api}
	} else {
		panic("Bad format")
	}
}
Beispiel #20
0
func (shd *ShardLoader) readShardInfo() (*ShardInfo, error) {
	buf, err := shd.ldb.Get(shd.dld.readOpts, []byte{SHARD_INFO_KEY})
	if err != nil {
		return nil, errors.New(fmt.Sprintf("readShardInfo(%s): failed to "+
			"read shard info key: %s", shd.path, err.Error()))
	}
	if len(buf) == 0 {
		return nil, errors.New(fmt.Sprintf("readShardInfo(%s): got zero-"+
			"length value for shard info key.", shd.path))
	}
	mh := new(codec.MsgpackHandle)
	mh.WriteExt = true
	r := bytes.NewBuffer(buf)
	decoder := codec.NewDecoder(r, mh)
	shardInfo := &ShardInfo{
		LayoutVersion: UNKNOWN_LAYOUT_VERSION,
	}
	err = decoder.Decode(shardInfo)
	if err != nil {
		return nil, errors.New(fmt.Sprintf("readShardInfo(%s): msgpack "+
			"decoding failed for shard info key: %s", shd.path, err.Error()))
	}
	return shardInfo, nil
}
func doWriteSpans(name string, N int, maxSpansPerRpc uint32, b *testing.B) {
	htraceBld := &MiniHTracedBuilder{Name: "doWriteSpans",
		Cnf: map[string]string{
			conf.HTRACE_LOG_LEVEL:         "INFO",
			conf.HTRACE_NUM_HRPC_HANDLERS: "20",
		},
		WrittenSpans: common.NewSemaphore(int64(1 - N)),
	}
	ht, err := htraceBld.Build()
	if err != nil {
		panic(err)
	}
	defer ht.Close()
	rnd := rand.New(rand.NewSource(1))
	allSpans := make([]*common.Span, N)
	for n := 0; n < N; n++ {
		allSpans[n] = test.NewRandomSpan(rnd, allSpans[0:n])
	}
	// Determine how many calls to WriteSpans we should make.  Each writeSpans
	// message should be small enough so that it doesn't exceed the max RPC
	// body length limit.  TODO: a production-quality golang client would do
	// this internally rather than needing us to do it here in the unit test.
	bodyLen := (4 * common.MAX_HRPC_BODY_LENGTH) / 5
	reqs := make([][]*common.Span, 0, 4)
	curReq := -1
	curReqLen := bodyLen
	var curReqSpans uint32
	mh := new(codec.MsgpackHandle)
	mh.WriteExt = true
	var mbuf [8192]byte
	buf := mbuf[:0]
	enc := codec.NewEncoderBytes(&buf, mh)
	for n := 0; n < N; n++ {
		span := allSpans[n]
		if (curReqSpans >= maxSpansPerRpc) ||
			(curReqLen >= bodyLen) {
			reqs = append(reqs, make([]*common.Span, 0, 16))
			curReqLen = 0
			curReq++
			curReqSpans = 0
		}
		buf = mbuf[:0]
		enc.ResetBytes(&buf)
		err := enc.Encode(span)
		if err != nil {
			panic(fmt.Sprintf("Error encoding span %s: %s\n",
				span.String(), err.Error()))
		}
		bufLen := len(buf)
		if bufLen > (bodyLen / 5) {
			panic(fmt.Sprintf("Span too long at %d bytes\n", bufLen))
		}
		curReqLen += bufLen
		reqs[curReq] = append(reqs[curReq], span)
		curReqSpans++
	}
	ht.Store.lg.Infof("num spans: %d.  num WriteSpansReq calls: %d\n", N, len(reqs))
	var hcl *htrace.Client
	hcl, err = htrace.NewClient(ht.ClientConf(), nil)
	if err != nil {
		panic(fmt.Sprintf("failed to create client: %s", err.Error()))
	}
	defer hcl.Close()

	// Reset the timer to avoid including the time required to create new
	// random spans in the benchmark total.
	if b != nil {
		b.ResetTimer()
	}

	// Write many random spans.
	for reqIdx := range reqs {
		go func(i int) {
			err = hcl.WriteSpans(reqs[i])
			if err != nil {
				panic(fmt.Sprintf("failed to send WriteSpans request %d: %s",
					i, err.Error()))
			}
		}(reqIdx)
	}
	// Wait for all the spans to be written.
	ht.Store.WrittenSpans.Wait()
}
Beispiel #22
0
func DecodeEntries(conn net.Conn) ([]FluentRecordSet, error) {
	var mh codec.MsgpackHandle
	mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
	dec := codec.NewDecoder(conn, &mh)
	v := []interface{}{nil, nil, nil}
	err := dec.Decode(&v)
	if err != nil {
		return nil, err
	}
	tag, ok := v[0].([]byte)
	if !ok {
		return nil, errors.New("Failed to decode tag field")
	}

	var retval []FluentRecordSet
	switch timestamp_or_entries := v[1].(type) {
	case int, uint, int64, uint64, int32, uint32, float32, float64:
		timestamp := toInt64(timestamp_or_entries)
		data, ok := v[2].(map[string]interface{})
		if !ok {
			return nil, errors.New("Failed to decode data field")
		}
		coerceInPlace(data)
		retval = []FluentRecordSet{
			{
				Tag: string(tag), // XXX: byte => rune
				Records: []FluentRecordType{
					&TinyFluentRecord{
						Timestamp: int64(timestamp),
						Data:      data,
					},
				},
			},
		}
	case []interface{}: // Forward
		if !ok {
			return nil, errors.New("Unexpected payload format")
		}
		recordSet, err := decodeRecordSet(tag, timestamp_or_entries)
		if err != nil {
			return nil, err
		}
		retval = []FluentRecordSet{recordSet}
	case []byte: // PackedForward
		reader := bytes.NewReader(timestamp_or_entries)
		entries := make([]interface{}, 0)
		for {
			entry := make([]interface{}, 0)
			err := codec.NewDecoder(reader, &mh).Decode(&entry)
			if err == io.EOF {
				break
			} else if err != nil {
				return nil, errors.New("Unexpected payload format")
			}
			entries = append(entries, entry)
		}
		recordSet, err := decodeRecordSet(tag, entries)
		if err != nil {
			return nil, err
		}
		retval = []FluentRecordSet{recordSet}
	default:
		return nil, errors.New(fmt.Sprintf("Unknown type: %t", timestamp_or_entries))
	}
	return retval, nil
}
Beispiel #23
0
func getHandle() codec.MsgpackHandle {
	h := codec.MsgpackHandle{RawToString: true}
	h.MapType = reflect.TypeOf(map[string]interface{}(nil))
	return h
}
Beispiel #24
0
func (self *OutputForward) Init(config map[string]string) error {
	_codec := codec.MsgpackHandle{}
	_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
	_codec.RawToString = false
	_codec.StructToArray = true

	self.host = "localhost"
	self.port = 8888
	self.flush_interval = 10
	self.sync_interval = 2
	self.buffer_path = "/tmp/test"
	self.buffer_queue_limit = 64 * 1024 * 1024
	self.buffer_chunk_limit = 8 * 1024 * 1024
	self.connect_timeout = 10
	self.codec = &_codec

	value := config["host"]
	if len(value) > 0 {
		self.host = value
	}

	value = config["port"]
	if len(value) > 0 {
		self.port, _ = strconv.Atoi(value)
	}

	value = config["connect_timeout"]
	if len(value) > 0 {
		self.connect_timeout, _ = strconv.Atoi(value)
	}

	value = config["flush_interval"]
	if len(value) > 0 {
		self.flush_interval, _ = strconv.Atoi(value)
	}

	value = config["sync_interval"]
	if len(value) > 0 {
		sync_interval, err := strconv.Atoi(value)
		if err != nil {
			return err
		}
		self.sync_interval = sync_interval
	}

	value = config["buffer_path"]
	if len(value) > 0 {
		self.buffer_path = value
	}

	value = config["buffer_queue_limit"]
	if len(value) > 0 {
		buffer_queue_limit, err := strconv.Atoi(value)
		if err != nil {
			return err
		}
		self.buffer_queue_limit = int64(buffer_queue_limit) * 1024 * 1024
	}

	value = config["buffer_chunk_limit"]
	if len(value) > 0 {
		buffer_chunk_limit, err := strconv.Atoi(value)
		if err != nil {
			return err
		}
		self.buffer_chunk_limit = int64(buffer_chunk_limit) * 1024 * 1024
	}
	return nil
}
Beispiel #25
0
func ParseParams(req *http.Request) *Params {
	var p Params
	ct := req.Header.Get("Content-Type")
	ct = strings.Split(ct, ";")[0]
	if ct == "multipart/form-data" {
		if err := req.ParseMultipartForm(10000000); err != nil {
			log.Println("Request.ParseMultipartForm Error", err)
		}
	} else {
		if err := req.ParseForm(); err != nil {
			log.Println("Request.ParseForm Error", err)
		}
	}
	tmap := make(map[string]interface{}, len(req.Form))
	for k, v := range req.Form {
		if strings.ToLower(v[0]) == "true" {
			tmap[k] = true
		} else if strings.ToLower(v[0]) == "false" {
			tmap[k] = false
		} else {
			tmap[k] = v[0]
		}
	}

	if req.MultipartForm != nil {
		for k, v := range req.MultipartForm.File {
			tmap[k] = v[0]
		}
	}

	if ct == "application/json" && req.ContentLength > 0 {
		err := json.NewDecoder(req.Body).Decode(&p.Values)
		if err != nil {
			log.Println("Content-Type is \"application/json\" but no valid json data received", err)
			p.Values = tmap
		}
		for k, v := range tmap {
			if _, pres := p.Values[k]; !pres {
				p.Values[k] = v
			}
		}
	} else if ct == "application/x-msgpack" {
		var mh codec.MsgpackHandle
		p.isBinary = true
		mh.MapType = reflect.TypeOf(p.Values)
		body, _ := ioutil.ReadAll(req.Body)
		if len(body) > 0 {
			buff := bytes.NewBuffer(body)
			first := body[0]
			if (first >= 0x80 && first <= 0x8f) || (first == 0xde || first == 0xdf) {
				err := codec.NewDecoder(buff, &mh).Decode(&p.Values)
				if err != nil && err != io.EOF {
					log.Println("Failed decoding msgpack", err)
				}
			} else {
				if p.Values == nil {
					p.Values = make(map[string]interface{}, 0)
				}
				var err error
				for err == nil {
					vals := make([]interface{}, 0)
					err = codec.NewDecoder(buff, &mh).Decode(&vals)
					if err != nil && err != io.EOF {
						log.Println("Failed decoding msgpack", err)
					} else {
						for i := len(vals) - 1; i >= 1; i -= 2 {
							p.Values[string(vals[i-1].([]byte))] = vals[i]
						}
					}
				}
			}
		} else {
			p.Values = make(map[string]interface{}, 0)
		}
		for k, v := range tmap {
			if _, pres := p.Values[k]; !pres {
				p.Values[k] = v
			}
		}
	} else {
		p.Values = tmap
	}

	for k, v := range mux.Vars(req) {
		const ID = "id"
		if strings.Contains(k, ID) {
			id, perr := strconv.ParseUint(v, 10, 64)
			if perr != nil {
				p.Values[k] = v
			} else {
				p.Values[k] = id
			}
		} else {
			p.Values[k] = v
		}
	}

	return &p
}
Beispiel #26
0
func NewTDOutput(
	logger *logging.Logger,
	endpoint string,
	connectionTimeout time.Duration,
	writeTimeout time.Duration,
	flushInterval time.Duration,
	parallelism int,
	journalGroupPath string,
	maxJournalChunkSize int64,
	apiKey string,
	databaseName string,
	tableName string,
	tempDir string,
	useSsl bool,
	rootCAs *x509.CertPool,
	httpProxy string,
	metadata string,
) (*TDOutput, error) {
	_codec := codec.MsgpackHandle{}
	_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
	_codec.RawToString = false
	_codec.StructToArray = true

	journalFactory := NewFileJournalGroupFactory(
		logger,
		randSource,
		time.Now,
		".log",
		os.FileMode(0600),
		maxJournalChunkSize,
	)
	router := (td_client.EndpointRouter)(nil)
	if endpoint != "" {
		router = &td_client.FixedEndpointRouter{endpoint}
	}
	httpProxy_ := (interface{})(nil)
	if httpProxy != "" {
		httpProxy_ = httpProxy
	}
	client, err := td_client.NewTDClient(td_client.Settings{
		ApiKey:            apiKey,
		Router:            router,
		ConnectionTimeout: connectionTimeout,
		// ReadTimeout: readTimeout, // TODO
		SendTimeout: writeTimeout,
		Ssl:         useSsl,
		RootCAs:     rootCAs,
		Proxy:       httpProxy_,
	})
	if err != nil {
		return nil, err
	}
	output := &TDOutput{
		logger:               logger,
		codec:                &_codec,
		wg:                   sync.WaitGroup{},
		flushInterval:        flushInterval,
		emitterChan:          make(chan FluentRecordSet),
		isShuttingDown:       0,
		client:               client,
		databaseName:         databaseName,
		tableName:            tableName,
		tempDir:              tempDir,
		sem:                  make(chan struct{}, parallelism),
		gcChan:               make(chan *os.File, 10),
		completion:           sync.Cond{L: &sync.Mutex{}},
		hasShutdownCompleted: false,
		metadata:             metadata,
	}
	journalGroup, err := journalFactory.GetJournalGroup(journalGroupPath, output)
	if err != nil {
		return nil, err
	}
	output.journalGroup = journalGroup
	return output, nil
}
Beispiel #27
0
func codecHandle() *codec.MsgpackHandle {
	var mh codec.MsgpackHandle
	mh.WriteExt = true
	return &mh
}
func (cdc *HrpcClientCodec) WriteRequest(rr *rpc.Request, msg interface{}) error {
	methodId := common.HrpcMethodNameToId(rr.ServiceMethod)
	if methodId == common.METHOD_ID_NONE {
		return errors.New(fmt.Sprintf("HrpcClientCodec: Unknown method name %s",
			rr.ServiceMethod))
	}
	mh := new(codec.MsgpackHandle)
	mh.WriteExt = true
	w := bytes.NewBuffer(make([]byte, 0, 2048))

	var err error
	enc := codec.NewEncoder(w, mh)
	if methodId == common.METHOD_ID_WRITE_SPANS {
		spans := msg.([]*common.Span)
		req := &common.WriteSpansReq{
			NumSpans: len(spans),
		}
		err = enc.Encode(req)
		if err != nil {
			return errors.New(fmt.Sprintf("HrpcClientCodec: Unable to marshal "+
				"message as msgpack: %s", err.Error()))
		}
		for spanIdx := range spans {
			err = enc.Encode(spans[spanIdx])
			if err != nil {
				return errors.New(fmt.Sprintf("HrpcClientCodec: Unable to marshal "+
					"span %d out of %d as msgpack: %s", spanIdx, len(spans), err.Error()))
			}
		}
	} else {
		err = enc.Encode(msg)
		if err != nil {
			return errors.New(fmt.Sprintf("HrpcClientCodec: Unable to marshal "+
				"message as msgpack: %s", err.Error()))
		}
	}
	buf := w.Bytes()
	if len(buf) > common.MAX_HRPC_BODY_LENGTH {
		return errors.New(fmt.Sprintf("HrpcClientCodec: message body is %d "+
			"bytes, but the maximum message size is %d bytes.",
			len(buf), common.MAX_HRPC_BODY_LENGTH))
	}
	hdr := common.HrpcRequestHeader{
		Magic:    common.HRPC_MAGIC,
		MethodId: methodId,
		Seq:      rr.Seq,
		Length:   uint32(len(buf)),
	}
	err = binary.Write(cdc.rwc, binary.LittleEndian, &hdr)
	if err != nil {
		return errors.New(fmt.Sprintf("Error writing header bytes: %s",
			err.Error()))
	}
	if cdc.testHooks != nil && cdc.testHooks.HandleWriteRequestBody != nil {
		cdc.testHooks.HandleWriteRequestBody()
	}
	_, err = cdc.rwc.Write(buf)
	if err != nil {
		return errors.New(fmt.Sprintf("Error writing body bytes: %s",
			err.Error()))
	}
	return nil
}