func (t *Transformer) toMsg(incoming interface{}, msg *message.Msg) error {
switch newMsg := incoming.(type) {
case map[string]interface{}: // we're a proper message.Msg, so copy the data over
msg.Op = message.OpTypeFromString(newMsg["op"].(string))
msg.Timestamp = newMsg["ts"].(int64)
msg.Namespace = newMsg["ns"].(string)
switch data := newMsg["data"].(type) {
case otto.Value:
exported, err := data.Export()
if err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return nil
}
d, err := mejson.Unmarshal(exported.(map[string]interface{}))
if err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return nil
}
msg.Data = map[string]interface{}(d)
case map[string]interface{}:
d, err := mejson.Unmarshal(data)
if err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return nil
}
msg.Data = map[string]interface{}(d)
default:
msg.Data = data
}
case bool: // skip this doc if we're a bool and we're false
if !newMsg {
msg.Op = message.Noop
return nil
}
default: // something went wrong
return fmt.Errorf("returned doc was not a map[string]interface{}")
}
return nil
}
/*
* tail the oplog
*/
func (m *Mongodb) tailData() (err error) {
var (
collection = m.mongoSession.DB("local").C("oplog.rs")
result oplogDoc // hold the document
query = bson.M{
"ts": bson.M{"$gte": m.oplogTime},
"ns": m.getNamespace(),
}
iter = collection.Find(query).LogReplay().Sort("$natural").Tail(m.oplogTimeout)
)
for {
for iter.Next(&result) {
if stop := m.pipe.Stopped; stop {
return
}
if result.validOp() {
var doc bson.M
switch result.Op {
case "i":
doc = result.O
case "d":
doc = result.O
case "u":
doc, err = m.getOriginalDoc(result.O2)
if err != nil { // errors aren't fatal here, but we need to send it down the pipe
m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("Mongodb error (%s)", err.Error()), nil)
continue
}
default:
m.pipe.Err <- NewError(ERROR, m.path, "Mongodb error (unknown op type)", nil)
continue
}
msg := message.NewMsg(message.OpTypeFromString(result.Op), doc)
msg.Timestamp = int64(result.Ts) >> 32
m.oplogTime = result.Ts
m.pipe.Send(msg)
}
result = oplogDoc{}
}
// we've exited the mongo read loop, lets figure out why
// check here again if we've been asked to quit
if stop := m.pipe.Stopped; stop {
return
}
if iter.Timeout() {
continue
}
if iter.Err() != nil {
return NewError(CRITICAL, m.path, fmt.Sprintf("Mongodb error (error reading collection %s)", iter.Err()), nil)
}
// query will change,
query = bson.M{
"ts": bson.M{"$gte": m.oplogTime},
"ns": m.getNamespace(),
}
iter = collection.Find(query).LogReplay().Tail(m.oplogTimeout)
}
}
func (t *Transformer) transformOne(msg *message.Msg) (*message.Msg, error) {
var (
doc interface{}
value otto.Value
outDoc otto.Value
result interface{}
err error
)
// short circuit for deletes and commands
if msg.Op == message.Command {
return msg, nil
}
now := time.Now().Nanosecond()
fullDoc := map[string]interface{}{
"data": msg.Data,
"ts": msg.Timestamp,
"op": msg.Op.String(),
}
if msg.IsMap() {
if doc, err = mejson.Marshal(msg.Data); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
fullDoc["data"] = doc
}
if value, err = t.vm.ToValue(fullDoc); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
// now that we have finished casting our map to a bunch of different types,
// lets run our transformer on the document
beforeVM := time.Now().Nanosecond()
if outDoc, err = t.vm.Call(`module.exports`, nil, value); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
if result, err = outDoc.Export(); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
afterVM := time.Now().Nanosecond()
fullDoc, ok := result.(map[string]interface{})
if !ok {
t.pipe.Err <- t.transformerError(ERROR, fmt.Errorf("returned doc was not a map[string]interface{}"), msg)
return msg, fmt.Errorf("returned doc was not a map[string]interface{}")
}
msg.Op = message.OpTypeFromString(fullDoc["op"].(string))
msg.Timestamp = fullDoc["ts"].(int64)
switch data := fullDoc["data"].(type) {
case otto.Value:
exported, err := data.Export()
if err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
d, err := mejson.Unmarshal(exported.(map[string]interface{}))
if err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
msg.Data = map[string]interface{}(d)
case map[string]interface{}:
d, err := mejson.Unmarshal(data)
if err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
msg.Data = map[string]interface{}(d)
default:
msg.Data = data
}
if t.debug {
then := time.Now().Nanosecond()
fmt.Printf("document transformed in %dus. %d to marshal, %d in the vm, %d to unmarshal\n", (then-now)/1000, (beforeVM-now)/1000, (afterVM-beforeVM)/1000, (then-afterVM)/1000)
}
return msg, nil
}
