Beispiel #1
0
func (t *Transformer) toMsg(incoming interface{}, msg *message.Msg) error {

	switch newMsg := incoming.(type) {
	case map[string]interface{}: // we're a proper message.Msg, so copy the data over
		msg.Op = message.OpTypeFromString(newMsg["op"].(string))
		msg.Timestamp = newMsg["ts"].(int64)
		msg.Namespace = newMsg["ns"].(string)

		switch data := newMsg["data"].(type) {
		case otto.Value:
			exported, err := data.Export()
			if err != nil {
				t.pipe.Err <- t.transformerError(ERROR, err, msg)
				return nil
			}
			d, err := mejson.Unmarshal(exported.(map[string]interface{}))
			if err != nil {
				t.pipe.Err <- t.transformerError(ERROR, err, msg)
				return nil
			}
			msg.Data = map[string]interface{}(d)
		case map[string]interface{}:
			d, err := mejson.Unmarshal(data)
			if err != nil {
				t.pipe.Err <- t.transformerError(ERROR, err, msg)
				return nil
			}
			msg.Data = map[string]interface{}(d)
		default:
			msg.Data = data
		}
	case bool: // skip this doc if we're a bool and we're false
		if !newMsg {
			msg.Op = message.Noop
			return nil
		}
	default: // something went wrong
		return fmt.Errorf("returned doc was not a map[string]interface{}")
	}

	return nil
}
Beispiel #2
0
/*
 * tail the oplog
 */
func (m *Mongodb) tailData() (err error) {

	var (
		collection = m.mongoSession.DB("local").C("oplog.rs")
		result     oplogDoc // hold the document
		query      = bson.M{
			"ts": bson.M{"$gte": m.oplogTime},
			"ns": m.getNamespace(),
		}

		iter = collection.Find(query).LogReplay().Sort("$natural").Tail(m.oplogTimeout)
	)

	for {
		for iter.Next(&result) {
			if stop := m.pipe.Stopped; stop {
				return
			}
			if result.validOp() {

				var doc bson.M
				switch result.Op {
				case "i":
					doc = result.O
				case "d":
					doc = result.O
				case "u":
					doc, err = m.getOriginalDoc(result.O2)
					if err != nil { // errors aren't fatal here, but we need to send it down the pipe
						m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("Mongodb error (%s)", err.Error()), nil)
						continue
					}
				default:
					m.pipe.Err <- NewError(ERROR, m.path, "Mongodb error (unknown op type)", nil)
					continue
				}

				msg := message.NewMsg(message.OpTypeFromString(result.Op), doc)
				msg.Timestamp = int64(result.Ts) >> 32

				m.oplogTime = result.Ts
				m.pipe.Send(msg)
			}
			result = oplogDoc{}
		}

		// we've exited the mongo read loop, lets figure out why
		// check here again if we've been asked to quit
		if stop := m.pipe.Stopped; stop {
			return
		}
		if iter.Timeout() {
			continue
		}
		if iter.Err() != nil {
			return NewError(CRITICAL, m.path, fmt.Sprintf("Mongodb error (error reading collection %s)", iter.Err()), nil)
		}

		// query will change,
		query = bson.M{
			"ts": bson.M{"$gte": m.oplogTime},
			"ns": m.getNamespace(),
		}
		iter = collection.Find(query).LogReplay().Tail(m.oplogTimeout)
	}
}
func (t *Transformer) transformOne(msg *message.Msg) (*message.Msg, error) {

	var (
		doc    interface{}
		value  otto.Value
		outDoc otto.Value
		result interface{}
		err    error
	)

	// short circuit for deletes and commands
	if msg.Op == message.Command {
		return msg, nil
	}

	now := time.Now().Nanosecond()
	fullDoc := map[string]interface{}{
		"data": msg.Data,
		"ts":   msg.Timestamp,
		"op":   msg.Op.String(),
	}
	if msg.IsMap() {
		if doc, err = mejson.Marshal(msg.Data); err != nil {
			t.pipe.Err <- t.transformerError(ERROR, err, msg)
			return msg, nil
		}
		fullDoc["data"] = doc
	}

	if value, err = t.vm.ToValue(fullDoc); err != nil {
		t.pipe.Err <- t.transformerError(ERROR, err, msg)
		return msg, nil
	}

	// now that we have finished casting our map to a bunch of different types,
	// lets run our transformer on the document
	beforeVM := time.Now().Nanosecond()
	if outDoc, err = t.vm.Call(`module.exports`, nil, value); err != nil {
		t.pipe.Err <- t.transformerError(ERROR, err, msg)
		return msg, nil
	}

	if result, err = outDoc.Export(); err != nil {
		t.pipe.Err <- t.transformerError(ERROR, err, msg)
		return msg, nil
	}

	afterVM := time.Now().Nanosecond()

	fullDoc, ok := result.(map[string]interface{})
	if !ok {
		t.pipe.Err <- t.transformerError(ERROR, fmt.Errorf("returned doc was not a map[string]interface{}"), msg)
		return msg, fmt.Errorf("returned doc was not a map[string]interface{}")
	}

	msg.Op = message.OpTypeFromString(fullDoc["op"].(string))
	msg.Timestamp = fullDoc["ts"].(int64)
	switch data := fullDoc["data"].(type) {
	case otto.Value:
		exported, err := data.Export()
		if err != nil {
			t.pipe.Err <- t.transformerError(ERROR, err, msg)
			return msg, nil
		}
		d, err := mejson.Unmarshal(exported.(map[string]interface{}))
		if err != nil {
			t.pipe.Err <- t.transformerError(ERROR, err, msg)
			return msg, nil
		}
		msg.Data = map[string]interface{}(d)
	case map[string]interface{}:
		d, err := mejson.Unmarshal(data)
		if err != nil {
			t.pipe.Err <- t.transformerError(ERROR, err, msg)
			return msg, nil
		}
		msg.Data = map[string]interface{}(d)
	default:
		msg.Data = data
	}

	if t.debug {
		then := time.Now().Nanosecond()
		fmt.Printf("document transformed in %dus.  %d to marshal, %d in the vm, %d to unmarshal\n", (then-now)/1000, (beforeVM-now)/1000, (afterVM-beforeVM)/1000, (then-afterVM)/1000)
	}

	return msg, nil
}