/*
* dump each message to the file
*/
func (d *File) dumpMessage(msg *message.Msg) (*message.Msg, error) {
var line string
if msg.IsMap() {
ba, err := json.Marshal(msg.Map())
if err != nil {
d.pipe.Err <- NewError(ERROR, d.path, fmt.Sprintf("Can't unmarshal document (%s)", err.Error()), msg.Data)
return msg, nil
}
line = string(ba)
} else {
line = fmt.Sprintf("%v", msg.Data)
}
if strings.HasPrefix(d.uri, "stdout://") {
fmt.Println(line)
} else {
_, err := fmt.Fprintln(d.filehandle, line)
if err != nil {
d.pipe.Err <- NewError(ERROR, d.path, fmt.Sprintf("Error writing to file (%s)", err.Error()), msg.Data)
return msg, nil
}
}
return msg, nil
}
// writeMessage writes one message to the destination mongo, or sends an error down the pipe
// TODO this can be cleaned up. I'm not sure whether this should pipe the error, or whether the
// caller should pipe the error
func (m *Mongodb) writeMessage(msg *message.Msg) (*message.Msg, error) {
collection := m.mongoSession.DB(m.database).C(m.collection)
if !msg.IsMap() {
m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (document must be a bson document, got %T instead)", msg.Data), msg.Data)
return msg, nil
}
doc := msg.Map()
if m.bulk {
m.bulkWriteChannel <- doc
} else if msg.Op == message.Delete {
err := collection.Remove(doc)
if err != nil {
m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error removing (%s)", err.Error()), msg.Data)
}
} else {
err := collection.Insert(doc)
if mgo.IsDup(err) {
err = collection.Update(bson.M{"_id": doc["_id"]}, doc)
}
if err != nil {
m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (%s)", err.Error()), msg.Data)
}
}
return msg, nil
}
func (e *Elasticsearch) runCommand(msg *message.Msg) error {
if !msg.IsMap() {
return nil
}
if _, hasKey := msg.Map()["flush"]; hasKey {
e.indexer.Flush()
}
return nil
}
/*
* dump each message to the file
*/
func (d *Twitter) dumpMessage(msg *message.Msg) (*message.Msg, error) {
var line string
if msg.IsMap() {
ba, err := json.Marshal(msg.Map())
if err != nil {
d.pipe.Err <- NewError(ERROR, d.path, fmt.Sprintf("Can't unmarshal document (%s)", err.Error()), msg.Data)
return msg, nil
}
line = string(ba)
} else {
line = fmt.Sprintf("%v", msg.Data)
}
fmt.Println(line)
return msg, nil
}
// applyOp applies one operation to the database
func (r *Rethinkdb) applyOp(msg *message.Msg) (*message.Msg, error) {
var (
resp gorethink.WriteResponse
err error
)
_, msgTable, err := msg.SplitNamespace()
if err != nil {
r.pipe.Err <- NewError(ERROR, r.path, fmt.Sprintf("rethinkdb error (msg namespace improperly formatted, must be database.table, got %s)", msg.Namespace), msg.Data)
return msg, nil
}
if !msg.IsMap() {
r.pipe.Err <- NewError(ERROR, r.path, "rethinkdb error (document must be a json document)", msg.Data)
return msg, nil
}
doc := msg.Map()
switch msg.Op {
case message.Delete:
id, err := msg.IDString("id")
if err != nil {
r.pipe.Err <- NewError(ERROR, r.path, "rethinkdb error (cannot delete an object with a nil id)", msg.Data)
return msg, nil
}
resp, err = gorethink.Table(msgTable).Get(id).Delete().RunWrite(r.client)
case message.Insert:
resp, err = gorethink.Table(msgTable).Insert(doc).RunWrite(r.client)
case message.Update:
resp, err = gorethink.Table(msgTable).Insert(doc, gorethink.InsertOpts{Conflict: "replace"}).RunWrite(r.client)
}
if err != nil {
r.pipe.Err <- NewError(ERROR, r.path, "rethinkdb error (%s)", err)
return msg, nil
}
err = r.handleResponse(&resp)
if err != nil {
r.pipe.Err <- NewError(ERROR, r.path, "rethinkdb error (%s)", err)
}
return msg, nil
}
// writeMessage writes one message to the destination mongo, or sends an error down the pipe
// TODO this can be cleaned up. I'm not sure whether this should pipe the error, or whether the
// caller should pipe the error
func (m *Mongodb) writeMessage(msg *message.Msg) (*message.Msg, error) {
_, msgColl, err := msg.SplitNamespace()
if err != nil {
m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (msg namespace improperly formatted, must be database.collection, got %s)", msg.Namespace), msg.Data)
return msg, nil
}
collection := m.mongoSession.DB(m.database).C(msgColl)
if !msg.IsMap() {
m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (document must be a bson document, got %T instead)", msg.Data), msg.Data)
return msg, nil
}
doc := &SyncDoc{
Doc: msg.Map(),
Collection: msgColl,
}
if m.bulk {
m.bulkWriteChannel <- doc
} else if msg.Op == message.Delete {
err := collection.Remove(doc.Doc)
if err != nil {
m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error removing (%s)", err.Error()), msg.Data)
}
} else {
err := collection.Insert(doc.Doc)
if mgo.IsDup(err) {
err = collection.Update(bson.M{"_id": doc.Doc["_id"]}, doc.Doc)
}
if err != nil {
m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("mongodb error (%s)", err.Error()), msg.Data)
}
}
return msg, nil
}
func (t *Transformer) transformOne(msg *message.Msg) (*message.Msg, error) {
var (
doc interface{}
value otto.Value
outDoc otto.Value
result interface{}
err error
)
// short circuit for deletes and commands
if msg.Op == message.Delete || msg.Op == message.Command {
return msg, nil
}
now := time.Now().Nanosecond()
if msg.IsMap() {
if doc, err = mejson.Marshal(msg.Data); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
} else {
doc = msg.Data
}
if value, err = t.vm.ToValue(doc); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
// now that we have finished casting our map to a bunch of different types,
// lets run our transformer on the document
beforeVM := time.Now().Nanosecond()
if outDoc, err = t.vm.Call(`module.exports`, nil, value); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
if result, err = outDoc.Export(); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
afterVM := time.Now().Nanosecond()
switch r := result.(type) {
case map[string]interface{}:
doc, err := mejson.Unmarshal(r)
if err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
msg.Data = map[string]interface{}(doc)
default:
msg.Data = r
}
if t.debug {
then := time.Now().Nanosecond()
fmt.Printf("document transformed in %dus. %d to marshal, %d in the vm, %d to unmarshal\n", (then-now)/1000, (beforeVM-now)/1000, (afterVM-beforeVM)/1000, (then-afterVM)/1000)
}
return msg, nil
}
func (t *Transformer) transformOne(msg *message.Msg) (*message.Msg, error) {
var (
doc interface{}
value otto.Value
outDoc otto.Value
result interface{}
err error
)
// short circuit for deletes and commands
if msg.Op == message.Command {
return msg, nil
}
now := time.Now().Nanosecond()
currMsg := map[string]interface{}{
"data": msg.Data,
"ts": msg.Timestamp,
"op": msg.Op.String(),
"ns": msg.Namespace,
}
if msg.IsMap() {
if doc, err = mejson.Marshal(msg.Data); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
currMsg["data"] = doc
}
if value, err = t.vm.ToValue(currMsg); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
// now that we have finished casting our map to a bunch of different types,
// lets run our transformer on the document
beforeVM := time.Now().Nanosecond()
if outDoc, err = t.vm.Call(`module.exports`, nil, value); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
if result, err = outDoc.Export(); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
afterVM := time.Now().Nanosecond()
if err = t.toMsg(result, msg); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, err
}
if t.debug {
then := time.Now().Nanosecond()
fmt.Printf("document transformed in %dus. %d to marshal, %d in the vm, %d to unmarshal\n", (then-now)/1000, (beforeVM-now)/1000, (afterVM-beforeVM)/1000, (then-afterVM)/1000)
}
return msg, nil
}
func (t *Transformer) transformOne(msg *message.Msg) (*message.Msg, error) {
var (
doc interface{}
value otto.Value
outDoc otto.Value
result interface{}
err error
)
// short circuit for deletes and commands
if msg.Op == message.Command {
return msg, nil
}
now := time.Now().Nanosecond()
fullDoc := map[string]interface{}{
"data": msg.Data,
"ts": msg.Timestamp,
"op": msg.Op.String(),
}
if msg.IsMap() {
if doc, err = mejson.Marshal(msg.Data); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
fullDoc["data"] = doc
}
if value, err = t.vm.ToValue(fullDoc); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
// now that we have finished casting our map to a bunch of different types,
// lets run our transformer on the document
beforeVM := time.Now().Nanosecond()
if outDoc, err = t.vm.Call(`module.exports`, nil, value); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
if result, err = outDoc.Export(); err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
afterVM := time.Now().Nanosecond()
fullDoc, ok := result.(map[string]interface{})
if !ok {
t.pipe.Err <- t.transformerError(ERROR, fmt.Errorf("returned doc was not a map[string]interface{}"), msg)
return msg, fmt.Errorf("returned doc was not a map[string]interface{}")
}
msg.Op = message.OpTypeFromString(fullDoc["op"].(string))
msg.Timestamp = fullDoc["ts"].(int64)
switch data := fullDoc["data"].(type) {
case otto.Value:
exported, err := data.Export()
if err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
d, err := mejson.Unmarshal(exported.(map[string]interface{}))
if err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
msg.Data = map[string]interface{}(d)
case map[string]interface{}:
d, err := mejson.Unmarshal(data)
if err != nil {
t.pipe.Err <- t.transformerError(ERROR, err, msg)
return msg, nil
}
msg.Data = map[string]interface{}(d)
default:
msg.Data = data
}
if t.debug {
then := time.Now().Nanosecond()
fmt.Printf("document transformed in %dus. %d to marshal, %d in the vm, %d to unmarshal\n", (then-now)/1000, (beforeVM-now)/1000, (afterVM-beforeVM)/1000, (then-afterVM)/1000)
}
return msg, nil
}