Esempio n. 1
0
func (r *Rethinkdb) sendAllDocuments(table string) error {
	if r.debug {
		fmt.Printf("sending all documents for %s\n", table)
	}

	cursor, err := gorethink.Table(table).Run(r.client)
	if err != nil {
		return err
	}
	defer cursor.Close()

	var doc map[string]interface{}
	for cursor.Next(&doc) {
		if stop := r.pipe.Stopped; stop {
			return nil
		}

		msg := message.NewMsg(message.Insert, r.prepareDocument(doc), r.computeNamespace(table))
		r.pipe.Send(msg)
	}

	if err := cursor.Err(); err != nil {
		return err
	}

	return nil
}
Esempio n. 2
0
func TestFilestoreUpdates(t *testing.T) {
	fs := NewFilestore("somelongkey", "/tmp/transporter.state")

	data := []struct {
		path string
		in   *message.Msg
		out  *message.Msg
	}{
		{
			"somepath",
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
		},
		{
			"somepath",
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 2}, "db.coll"),
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 2}, "db.coll"),
		},
	}

	for _, d := range data {
		err := fs.Set(d.path, &MsgState{Msg: d.in, Extra: make(map[string]interface{})})
		if err != nil {
			t.Errorf("got error: %s\n", err)
			t.FailNow()
		}
	}

	d := data[len(data)-1]
	fh, err := os.Open("/tmp/transporter.state")
	if err != nil {
		t.Errorf("got error: %s\n", err)
		t.FailNow()
	}
	states := make(map[string]*MsgState)
	dec := gob.NewDecoder(fh)
	err = dec.Decode(&states)
	if err != nil {
		t.Errorf("got error: %s\n", err)
		t.FailNow()
	}
	out := states["somelongkey-somepath"]
	if !reflect.DeepEqual(out.Msg, d.out) {
		t.Errorf("wanted: %s, got: %s", d.out, out.Msg)
	}

}
Esempio n. 3
0
func TestFilestore(t *testing.T) {
	fs := NewFilestore("somelongkey", "/tmp/transporter.state")

	data := []struct {
		path string
		in   *message.Msg
		out  *message.Msg
	}{
		{
			"somepath",
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
		},
		{
			"somepath/morepath",
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
		},
	}

	for _, d := range data {
		err := fs.Set(d.path, &MsgState{Msg: d.in, Extra: make(map[string]interface{})})
		if err != nil {
			t.Errorf("got error: %s\n", err)
			t.FailNow()
		}
	}

	for _, d := range data {
		out, err := fs.Get(d.path)
		if err != nil {
			t.Errorf("got error: %s\n", err)
			t.FailNow()
		}
		if !reflect.DeepEqual(out.Msg, d.out) {
			t.Errorf("wanted: %s, got: %s", d.out, out.Msg)
		}
	}

}
Esempio n. 4
0
func (r *Rethinkdb) sendChanges(table string, ccursor *gorethink.Cursor) error {
	defer ccursor.Close()
	if r.debug {
		fmt.Printf("sending changes for %s\n", table)
	}
	var change rethinkDbChangeNotification
	for ccursor.Next(&change) {
		if stop := r.pipe.Stopped; stop {
			return nil
		}

		if r.debug {
			fmt.Printf("change: %#v\n", change)
		}

		var msg *message.Msg
		if change.Error != "" {
			return errors.New(change.Error)
		} else if change.OldVal != nil && change.NewVal != nil {
			msg = message.NewMsg(message.Update, r.prepareDocument(change.NewVal), r.computeNamespace(table))
		} else if change.NewVal != nil {
			msg = message.NewMsg(message.Insert, r.prepareDocument(change.NewVal), r.computeNamespace(table))
		} else if change.OldVal != nil {
			msg = message.NewMsg(message.Delete, r.prepareDocument(change.OldVal), r.computeNamespace(table))
		}

		if msg != nil {
			r.pipe.Send(msg)
			if r.debug {
				fmt.Printf("msg: %#v\n", msg)
			}
		}
	}

	if err := ccursor.Err(); err != nil {
		return err
	}

	return nil
}
Esempio n. 5
0
// catdata pulls down the original collections
func (m *Mongodb) catData() (err error) {
	collections, _ := m.mongoSession.DB(m.database).CollectionNames()
	for _, collection := range collections {
		if strings.HasPrefix(collection, "system.") {
			continue
		} else if match := m.collectionMatch.MatchString(collection); !match {
			continue
		}

		var (
			query  = bson.M{}
			result bson.M // hold the document
		)

		iter := m.mongoSession.DB(m.database).C(collection).Find(query).Sort("_id").Iter()

		for {
			for iter.Next(&result) {
				if stop := m.pipe.Stopped; stop {
					return
				}

				// set up the message
				msg := message.NewMsg(message.Insert, result, m.computeNamespace(collection))

				m.pipe.Send(msg)
				result = bson.M{}
			}

			// we've exited the mongo read loop, lets figure out why
			// check here again if we've been asked to quit
			if stop := m.pipe.Stopped; stop {
				return
			}

			if iter.Err() != nil && m.restartable {
				fmt.Printf("got err reading collection. reissuing query %v\n", iter.Err())
				time.Sleep(1 * time.Second)
				iter = m.mongoSession.DB(m.database).C(collection).Find(query).Sort("_id").Iter()
				continue
			}
			break
		}
	}
	return
}
Esempio n. 6
0
func BenchmarkTransformOne(b *testing.B) {
	tpipe := pipe.NewPipe(nil, "path")
	transformer := &Transformer{
		pipe: tpipe,
		path: "path",
		fn:   "module.exports=function(doc) { return doc }",
	}
	err := transformer.initEnvironment()
	if err != nil {
		panic(err)
	}

	msg := message.NewMsg(message.Insert, map[string]interface{}{"id": bson.NewObjectId(), "name": "nick"}, "database.collection")
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		transformer.transformOne(msg)
	}
}
Esempio n. 7
0
// read each message from the file
func (d *File) readFile() (err error) {
	filename := strings.Replace(d.uri, "file://", "", 1)
	d.filehandle, err = os.Open(filename)
	if err != nil {
		d.pipe.Err <- NewError(CRITICAL, d.path, fmt.Sprintf("Can't open input file (%s)", err.Error()), nil)
		return err
	}

	decoder := json.NewDecoder(d.filehandle)
	for {
		var doc map[string]interface{}
		if err := decoder.Decode(&doc); err == io.EOF {
			break
		} else if err != nil {
			d.pipe.Err <- NewError(ERROR, d.path, fmt.Sprintf("Can't marshal document (%s)", err.Error()), nil)
			return err
		}
		d.pipe.Send(message.NewMsg(message.Insert, doc, fmt.Sprintf("file.%s", filename)))
	}
	return nil
}
Esempio n. 8
0
/*
 * tail the oplog
 */
func (m *Mongodb) tailData() (err error) {

	var (
		collection = m.mongoSession.DB("local").C("oplog.rs")
		result     oplogDoc // hold the document
		query      = bson.M{
			"ts": bson.M{"$gte": m.oplogTime},
		}

		iter    = collection.Find(query).LogReplay().Sort("$natural").Tail(m.oplogTimeout)
		retries = 0
	)

	for {
		for iter.Next(&result) {
			retries = 0
			if stop := m.pipe.Stopped; stop {
				return
			}
			if result.validOp() {
				_, coll, _ := m.splitNamespace(result.Ns)

				if strings.HasPrefix(coll, "system.") {
					continue
				} else if match := m.collectionMatch.MatchString(coll); !match {
					continue
				}

				var doc bson.M
				switch result.Op {
				case "i":
					doc = result.O
				case "d":
					doc = result.O
				case "u":
					doc, err = m.getOriginalDoc(result.O2, coll)
					if err != nil { // errors aren't fatal here, but we need to send it down the pipe
						m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("Mongodb error (%s)", err.Error()), nil)
						continue
					}
				default:
					m.pipe.Err <- NewError(ERROR, m.path, "Mongodb error (unknown op type)", nil)
					continue
				}

				msg := message.NewMsg(message.OpTypeFromString(result.Op), doc, m.computeNamespace(coll))
				msg.Timestamp = int64(result.Ts) >> 32

				m.oplogTime = result.Ts
				m.pipe.Send(msg)
			}
			result = oplogDoc{}
		}

		// we've exited the mongo read loop, lets figure out why
		// check here again if we've been asked to quit
		if stop := m.pipe.Stopped; stop {
			return
		}
		if iter.Timeout() {
			continue
		}
		if iter.Err() != nil {
			// Do not return, just retry up to 5 times
			retries++
			if retries > 4 {
				return NewError(CRITICAL, m.path, fmt.Sprintf("Mongodb error (error reading collection %s)", iter.Err()), nil)
			}
		}

		// query will change,
		query = bson.M{
			"ts": bson.M{"$gte": m.oplogTime},
		}
		iter = collection.Find(query).LogReplay().Tail(m.oplogTimeout)
	}
}
Esempio n. 9
0
func TestTransformOne(t *testing.T) {
	bsonID1 := bson.NewObjectId()
	bsonID2 := bson.ObjectIdHex("54a4420502a14b9641000001")
	tpipe := pipe.NewPipe(nil, "path")
	go func(p *pipe.Pipe) {
		for range p.Err {
			// noop
		}
	}(tpipe)

	data := []struct {
		fn  string
		in  *message.Msg
		out *message.Msg
		err bool
	}{
		{
			// just pass through
			"module.exports=function(doc) { return doc }",
			message.NewMsg(message.Insert, map[string]interface{}{"id": "id1", "name": "nick"}, "database.collection"),
			message.NewMsg(message.Insert, map[string]interface{}{"id": "id1", "name": "nick"}, "database.collection"),
			false,
		},
		{
			// delete the 'name' property
			"module.exports=function(doc) { doc['data'] = _.omit(doc['data'], ['name']); return doc }",
			message.NewMsg(message.Insert, map[string]interface{}{"id": "id2", "name": "nick"}, "database.collection"),
			message.NewMsg(message.Insert, map[string]interface{}{"id": "id2"}, "database.collection"),
			false,
		},
		{
			// delete's should be processed the same
			"module.exports=function(doc) { doc['data'] =  _.omit(doc['data'], ['name']); return doc }",
			message.NewMsg(message.Delete, map[string]interface{}{"id": "id2", "name": "nick"}, "database.collection"),
			message.NewMsg(message.Delete, map[string]interface{}{"id": "id2"}, "database.collection"),
			false,
		},
		{
			// delete's and commands should pass through, and the transformer fn shouldn't run
			"module.exports=function(doc) { return _.omit(doc['data'], ['name']) }",
			message.NewMsg(message.Command, map[string]interface{}{"id": "id2", "name": "nick"}, "database.collection"),
			message.NewMsg(message.Command, map[string]interface{}{"id": "id2", "name": "nick"}, "database.collection"),
			false,
		},
		{
			// bson should marshal and unmarshal properly
			"module.exports=function(doc) { return doc }",
			message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.collection"),
			message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.collection"),
			false,
		},
		{
			// we should be able to change the bson
			"module.exports=function(doc) { doc['data']['id']['$oid'] = '54a4420502a14b9641000001'; return doc }",
			message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.collection"),
			message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID2, "name": "nick"}, "database.collection"),
			false,
		}, {
			// we should be able to skip a nil message
			"module.exports=function(doc) { return false }",
			message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.collection"),
			message.NewMsg(message.Noop, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.collection"),
			false,
		},
		{
			// this throws an error
			"module.exports=function(doc) { return doc['data']['name'] }",
			message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.collection"),
			message.NewMsg(message.Insert, "nick", "database.collection"),
			true,
		},
		{
			// we should be able to change the namespace
			"module.exports=function(doc) { doc['ns'] = 'database.table'; return doc }",
			message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.collection"),
			message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.table"),
			false,
		},
	}
	for _, v := range data {

		transformer := &Transformer{pipe: tpipe, path: "path", fn: v.fn}
		err := transformer.initEnvironment()
		if err != nil {
			panic(err)
		}
		msg, err := transformer.transformOne(v.in)

		if (err != nil) != v.err {
			t.Errorf("error expected %t but actually got %v", v.err, err)
			continue
		}
		if (!reflect.DeepEqual(msg, v.out) || err != nil) && !v.err {
			t.Errorf("expected:\n(%T) %+v\ngot:\n(%T) %+v with error (%v)\n", v.out, v.out, msg, msg, err)
		}
	}
}