func DecodeRequest(body []byte, req *Request) (err error) {
mh := codec.MsgpackHandle{RawToString: true}
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
dec := codec.NewDecoderBytes(body, &mh)
err = dec.Decode(req)
return
}
func NewForwardInput(logger *logging.Logger, bind string, port Port) (*ForwardInput, error) {
_codec := codec.MsgpackHandle{}
_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
_codec.RawToString = false
addr, err := net.ResolveTCPAddr("tcp", bind)
if err != nil {
logger.Error("%s", err.Error())
return nil, err
}
listener, err := net.ListenTCP("tcp", addr)
if err != nil {
logger.Error("%s", err.Error())
return nil, err
}
return &ForwardInput{
port: port,
logger: logger,
bind: bind,
listener: listener,
codec: &_codec,
clients: make(map[*net.TCPConn]*forwardClient),
clientsMtx: sync.Mutex{},
entries: 0,
wg: sync.WaitGroup{},
acceptChan: make(chan *net.TCPConn),
shutdownChan: make(chan struct{}),
isShuttingDown: uintptr(0),
}, nil
}
func scriptBody(sessionId []byte, graphName, script string, bindings map[string]interface{}) (out []byte, err error) {
var (
mh = new(codec.MsgpackHandle)
enc = codec.NewEncoderBytes(&out, mh)
reqId = uuid.NewV4()
isSessionless = bytes.Equal(sessionId, sessionlessUuid[:])
)
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
meta := map[string]interface{}{
"inSession": !isSessionless,
"isolate": isSessionless,
"graphObjName": "g",
}
if isSessionless {
meta["graphName"] = graphName
}
err = enc.Encode([]interface{}{
sessionId,
reqId[:],
meta,
"groovy",
script,
bindings,
})
return
}
func decodeBody(body []byte) (retVal []interface{}, err error) {
var mh = new(codec.MsgpackHandle)
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
dec := codec.NewDecoderBytes(body, mh)
if e := dec.Decode(&retVal); e != nil {
err = errors.New("rexpro: msgpack decode error")
}
return
}
func (gofetcher *Gofetcher) ParseRequest(method string, requestBody []byte) (request *Request) {
var (
mh codec.MsgpackHandle
h = &mh
timeout int64 = DefaultTimeout
cookies Cookies
headers = make(http.Header)
followRedirects bool = DefaultFollowRedirects
body *bytes.Buffer
)
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
var res []interface{}
codec.NewDecoderBytes(requestBody, h).Decode(&res)
url := string(res[0].([]uint8))
switch {
case method == "GET" || method == "HEAD" || method == "DELETE":
if len(res) > 1 {
timeout = parseTimeout(res[1])
}
if len(res) > 2 {
cookies = parseCookies(res[2].(map[string]interface{}))
}
if len(res) > 3 {
headers = parseHeaders(res[3].(map[string]interface{}))
}
if len(res) > 4 {
followRedirects = res[4].(bool)
}
case method == "POST" || method == "PUT" || method == "PATCH":
if len(res) > 1 {
body = bytes.NewBuffer(res[1].([]byte))
}
if len(res) > 2 {
timeout = parseTimeout(res[2])
}
if len(res) > 3 {
cookies = parseCookies(res[3].(map[string]interface{}))
}
if len(res) > 4 {
headers = parseHeaders(res[4].(map[string]interface{}))
}
if len(res) > 5 {
followRedirects = res[5].(bool)
}
}
request = &Request{Method: method, URL: url, Timeout: timeout,
FollowRedirects: followRedirects,
Cookies: cookies, Headers: headers}
if body != nil {
request.Body = body
}
return request
}
func (s *session) sessionBody(kill bool) (out []byte, err error) {
var (
mh = new(codec.MsgpackHandle)
enc = codec.NewEncoderBytes(&out, mh)
reqId = uuid.NewV4()
)
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
err = enc.Encode([]interface{}{
s.sId,
reqId[:],
map[string]interface{}{
"graphName": s.r.graphName,
"graphObjName": "g",
"killSession": kill,
},
s.username,
s.password,
})
return
}
func newForwardInput(factory *ForwardInputFactory, logger *log.Logger, bind string, port ik.Port) (*ForwardInput, error) {
_codec := codec.MsgpackHandle{}
_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
_codec.RawToString = false
listener, err := net.Listen("tcp", bind)
if err != nil {
logger.Fatal(err.Error())
return nil, err
}
return &ForwardInput{
factory: factory,
port: port,
logger: logger,
bind: bind,
listener: listener,
codec: &_codec,
running: false,
clients: make(map[net.Conn]*forwardClient),
}, nil
}
func NewForwardOutput(logger *logging.Logger, bind string, retryInterval time.Duration, connectionTimeout time.Duration, writeTimeout time.Duration, flushInterval time.Duration, journalGroupPath string, maxJournalChunkSize int64, metadata string) (*ForwardOutput, error) {
_codec := codec.MsgpackHandle{}
_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
_codec.RawToString = false
_codec.StructToArray = true
journalFactory := NewFileJournalGroupFactory(
logger,
randSource,
time.Now,
".log",
os.FileMode(0600),
maxJournalChunkSize,
)
output := &ForwardOutput{
logger: logger,
codec: &_codec,
bind: bind,
retryInterval: retryInterval,
connectionTimeout: connectionTimeout,
writeTimeout: writeTimeout,
wg: sync.WaitGroup{},
flushInterval: flushInterval,
emitterChan: make(chan FluentRecordSet),
spoolerShutdownChan: make(chan struct{}),
isShuttingDown: 0,
completion: sync.Cond{L: &sync.Mutex{}},
hasShutdownCompleted: false,
metadata: metadata,
}
journalGroup, err := journalFactory.GetJournalGroup(journalGroupPath, output)
if err != nil {
return nil, err
}
output.journalGroup = journalGroup
output.journal = journalGroup.GetJournal("output")
return output, nil
}
func (locator *Locator) unpackchunk(chunk RawMessage) ResolveResult {
defer func() {
if err := recover(); err != nil {
log.Println("defer", err)
}
}()
var v []interface{}
var MH codec.MsgpackHandle
MH.MapType = reflect.TypeOf(map[int]string(nil))
err := codec.NewDecoderBytes(chunk, &MH).Decode(&v)
if err != nil {
log.Println("unpack chunk error", err)
}
if len(v) == 3 {
v_endpoint := v[0].([]interface{})
endpoint := Endpoint{string(v_endpoint[0].([]uint8)), v_endpoint[1].(uint64)}
version := v[1].(int64)
api := v[2].(map[int]string)
return ResolveResult{true, endpoint, version, api}
} else {
panic("Bad format")
}
}
func DecodeEntries(conn net.Conn) ([]FluentRecordSet, error) {
var mh codec.MsgpackHandle
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
dec := codec.NewDecoder(conn, &mh)
v := []interface{}{nil, nil, nil}
err := dec.Decode(&v)
if err != nil {
return nil, err
}
tag, ok := v[0].([]byte)
if !ok {
return nil, errors.New("Failed to decode tag field")
}
var retval []FluentRecordSet
switch timestamp_or_entries := v[1].(type) {
case int, uint, int64, uint64, int32, uint32, float32, float64:
timestamp := toInt64(timestamp_or_entries)
data, ok := v[2].(map[string]interface{})
if !ok {
return nil, errors.New("Failed to decode data field")
}
coerceInPlace(data)
retval = []FluentRecordSet{
{
Tag: string(tag), // XXX: byte => rune
Records: []FluentRecordType{
&TinyFluentRecord{
Timestamp: int64(timestamp),
Data: data,
},
},
},
}
case []interface{}: // Forward
if !ok {
return nil, errors.New("Unexpected payload format")
}
recordSet, err := decodeRecordSet(tag, timestamp_or_entries)
if err != nil {
return nil, err
}
retval = []FluentRecordSet{recordSet}
case []byte: // PackedForward
reader := bytes.NewReader(timestamp_or_entries)
entries := make([]interface{}, 0)
for {
entry := make([]interface{}, 0)
err := codec.NewDecoder(reader, &mh).Decode(&entry)
if err == io.EOF {
break
} else if err != nil {
return nil, errors.New("Unexpected payload format")
}
entries = append(entries, entry)
}
recordSet, err := decodeRecordSet(tag, entries)
if err != nil {
return nil, err
}
retval = []FluentRecordSet{recordSet}
default:
return nil, errors.New(fmt.Sprintf("Unknown type: %t", timestamp_or_entries))
}
return retval, nil
}
func getHandle() codec.MsgpackHandle {
h := codec.MsgpackHandle{RawToString: true}
h.MapType = reflect.TypeOf(map[string]interface{}(nil))
return h
}
func ParseParams(req *http.Request) *Params {
var p Params
ct := req.Header.Get("Content-Type")
ct = strings.Split(ct, ";")[0]
if ct == "multipart/form-data" {
if err := req.ParseMultipartForm(10000000); err != nil {
log.Println("Request.ParseMultipartForm Error", err)
}
} else {
if err := req.ParseForm(); err != nil {
log.Println("Request.ParseForm Error", err)
}
}
tmap := make(map[string]interface{}, len(req.Form))
for k, v := range req.Form {
if strings.ToLower(v[0]) == "true" {
tmap[k] = true
} else if strings.ToLower(v[0]) == "false" {
tmap[k] = false
} else {
tmap[k] = v[0]
}
}
if req.MultipartForm != nil {
for k, v := range req.MultipartForm.File {
tmap[k] = v[0]
}
}
if ct == "application/json" && req.ContentLength > 0 {
err := json.NewDecoder(req.Body).Decode(&p.Values)
if err != nil {
log.Println("Content-Type is \"application/json\" but no valid json data received", err)
p.Values = tmap
}
for k, v := range tmap {
if _, pres := p.Values[k]; !pres {
p.Values[k] = v
}
}
} else if ct == "application/x-msgpack" {
var mh codec.MsgpackHandle
p.isBinary = true
mh.MapType = reflect.TypeOf(p.Values)
body, _ := ioutil.ReadAll(req.Body)
if len(body) > 0 {
buff := bytes.NewBuffer(body)
first := body[0]
if (first >= 0x80 && first <= 0x8f) || (first == 0xde || first == 0xdf) {
err := codec.NewDecoder(buff, &mh).Decode(&p.Values)
if err != nil && err != io.EOF {
log.Println("Failed decoding msgpack", err)
}
} else {
if p.Values == nil {
p.Values = make(map[string]interface{}, 0)
}
var err error
for err == nil {
vals := make([]interface{}, 0)
err = codec.NewDecoder(buff, &mh).Decode(&vals)
if err != nil && err != io.EOF {
log.Println("Failed decoding msgpack", err)
} else {
for i := len(vals) - 1; i >= 1; i -= 2 {
p.Values[string(vals[i-1].([]byte))] = vals[i]
}
}
}
}
} else {
p.Values = make(map[string]interface{}, 0)
}
for k, v := range tmap {
if _, pres := p.Values[k]; !pres {
p.Values[k] = v
}
}
} else {
p.Values = tmap
}
for k, v := range mux.Vars(req) {
const ID = "id"
if strings.Contains(k, ID) {
id, perr := strconv.ParseUint(v, 10, 64)
if perr != nil {
p.Values[k] = v
} else {
p.Values[k] = id
}
} else {
p.Values[k] = v
}
}
return &p
}
func NewTDOutput(
logger *logging.Logger,
endpoint string,
connectionTimeout time.Duration,
writeTimeout time.Duration,
flushInterval time.Duration,
parallelism int,
journalGroupPath string,
maxJournalChunkSize int64,
apiKey string,
databaseName string,
tableName string,
tempDir string,
useSsl bool,
rootCAs *x509.CertPool,
httpProxy string,
metadata string,
) (*TDOutput, error) {
_codec := codec.MsgpackHandle{}
_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
_codec.RawToString = false
_codec.StructToArray = true
journalFactory := NewFileJournalGroupFactory(
logger,
randSource,
time.Now,
".log",
os.FileMode(0600),
maxJournalChunkSize,
)
router := (td_client.EndpointRouter)(nil)
if endpoint != "" {
router = &td_client.FixedEndpointRouter{endpoint}
}
httpProxy_ := (interface{})(nil)
if httpProxy != "" {
httpProxy_ = httpProxy
}
client, err := td_client.NewTDClient(td_client.Settings{
ApiKey: apiKey,
Router: router,
ConnectionTimeout: connectionTimeout,
// ReadTimeout: readTimeout, // TODO
SendTimeout: writeTimeout,
Ssl: useSsl,
RootCAs: rootCAs,
Proxy: httpProxy_,
})
if err != nil {
return nil, err
}
output := &TDOutput{
logger: logger,
codec: &_codec,
wg: sync.WaitGroup{},
flushInterval: flushInterval,
emitterChan: make(chan FluentRecordSet),
isShuttingDown: 0,
client: client,
databaseName: databaseName,
tableName: tableName,
tempDir: tempDir,
sem: make(chan struct{}, parallelism),
gcChan: make(chan *os.File, 10),
completion: sync.Cond{L: &sync.Mutex{}},
hasShutdownCompleted: false,
metadata: metadata,
}
journalGroup, err := journalFactory.GetJournalGroup(journalGroupPath, output)
if err != nil {
return nil, err
}
output.journalGroup = journalGroup
return output, nil
}
func (self *OutputForward) Init(config map[string]string) error {
_codec := codec.MsgpackHandle{}
_codec.MapType = reflect.TypeOf(map[string]interface{}(nil))
_codec.RawToString = false
_codec.StructToArray = true
self.host = "localhost"
self.port = 8888
self.flush_interval = 10
self.sync_interval = 2
self.buffer_path = "/tmp/test"
self.buffer_queue_limit = 64 * 1024 * 1024
self.buffer_chunk_limit = 8 * 1024 * 1024
self.connect_timeout = 10
self.codec = &_codec
value := config["host"]
if len(value) > 0 {
self.host = value
}
value = config["port"]
if len(value) > 0 {
self.port, _ = strconv.Atoi(value)
}
value = config["connect_timeout"]
if len(value) > 0 {
self.connect_timeout, _ = strconv.Atoi(value)
}
value = config["flush_interval"]
if len(value) > 0 {
self.flush_interval, _ = strconv.Atoi(value)
}
value = config["sync_interval"]
if len(value) > 0 {
sync_interval, err := strconv.Atoi(value)
if err != nil {
return err
}
self.sync_interval = sync_interval
}
value = config["buffer_path"]
if len(value) > 0 {
self.buffer_path = value
}
value = config["buffer_queue_limit"]
if len(value) > 0 {
buffer_queue_limit, err := strconv.Atoi(value)
if err != nil {
return err
}
self.buffer_queue_limit = int64(buffer_queue_limit) * 1024 * 1024
}
value = config["buffer_chunk_limit"]
if len(value) > 0 {
buffer_chunk_limit, err := strconv.Atoi(value)
if err != nil {
return err
}
self.buffer_chunk_limit = int64(buffer_chunk_limit) * 1024 * 1024
}
return nil
}
