Esempio n. 1
0
func (r *Rethinkdb) sendAllDocuments() error {
	if r.debug {
		fmt.Printf("sending all documents\n")
	}

	cursor, err := gorethink.Table(r.table).Run(r.client)
	if err != nil {
		return err
	}
	defer cursor.Close()

	var doc map[string]interface{}
	for cursor.Next(&doc) {
		if stop := r.pipe.Stopped; stop {
			return nil
		}

		msg := message.NewMsg(message.Insert, r.prepareDocument(doc))
		r.pipe.Send(msg)
	}

	if err := cursor.Err(); err != nil {
		return err
	}

	return nil
}
Esempio n. 2
0
func TestFilestoreUpdates(t *testing.T) {
	fs := NewFilestore("somelongkey", "/tmp/transporter.state")

	data := []struct {
		path string
		in   *message.Msg
		out  *message.Msg
	}{
		{
			"somepath",
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
		},
		{
			"somepath",
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 2}, "db.coll"),
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 2}, "db.coll"),
		},
	}

	for _, d := range data {
		err := fs.Set(d.path, &MsgState{Msg: d.in, Extra: make(map[string]interface{})})
		if err != nil {
			t.Errorf("got error: %s\n", err)
			t.FailNow()
		}
	}

	d := data[len(data)-1]
	fh, err := os.Open("/tmp/transporter.state")
	if err != nil {
		t.Errorf("got error: %s\n", err)
		t.FailNow()
	}
	states := make(map[string]*MsgState)
	dec := gob.NewDecoder(fh)
	err = dec.Decode(&states)
	if err != nil {
		t.Errorf("got error: %s\n", err)
		t.FailNow()
	}
	out := states["somelongkey-somepath"]
	if !reflect.DeepEqual(out.Msg, d.out) {
		t.Errorf("wanted: %s, got: %s", d.out, out.Msg)
	}

}
Esempio n. 3
0
func TestFilestore(t *testing.T) {
	fs := NewFilestore("somelongkey", "/tmp/transporter.state")

	data := []struct {
		path string
		in   *message.Msg
		out  *message.Msg
	}{
		{
			"somepath",
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
		},
		{
			"somepath/morepath",
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
			message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
		},
	}

	for _, d := range data {
		err := fs.Set(d.path, &MsgState{Msg: d.in, Extra: make(map[string]interface{})})
		if err != nil {
			t.Errorf("got error: %s\n", err)
			t.FailNow()
		}
	}

	for _, d := range data {
		out, err := fs.Get(d.path)
		if err != nil {
			t.Errorf("got error: %s\n", err)
			t.FailNow()
		}
		if !reflect.DeepEqual(out.Msg, d.out) {
			t.Errorf("wanted: %s, got: %s", d.out, out.Msg)
		}
	}

}
Esempio n. 4
0
func (r *Rethinkdb) sendChanges(table string, ccursor *gorethink.Cursor) error {
	defer ccursor.Close()
	if r.debug {
		fmt.Printf("sending changes for %s\n", table)
	}
	var change rethinkDbChangeNotification
	for ccursor.Next(&change) {
		if stop := r.pipe.Stopped; stop {
			return nil
		}

		if r.debug {
			fmt.Printf("change: %#v\n", change)
		}

		var msg *message.Msg
		if change.Error != "" {
			return errors.New(change.Error)
		} else if change.OldVal != nil && change.NewVal != nil {
			msg = message.NewMsg(message.Update, r.prepareDocument(change.NewVal), r.computeNamespace(table))
		} else if change.NewVal != nil {
			msg = message.NewMsg(message.Insert, r.prepareDocument(change.NewVal), r.computeNamespace(table))
		} else if change.OldVal != nil {
			msg = message.NewMsg(message.Delete, r.prepareDocument(change.OldVal), r.computeNamespace(table))
		}

		if msg != nil {
			r.pipe.Send(msg)
			if r.debug {
				fmt.Printf("msg: %#v\n", msg)
			}
		}
	}

	if err := ccursor.Err(); err != nil {
		return err
	}

	return nil
}
Esempio n. 5
0
// catdata pulls down the original collections
func (m *Mongodb) catData() (err error) {
	collections, _ := m.mongoSession.DB(m.database).CollectionNames()
	for _, collection := range collections {
		if strings.HasPrefix(collection, "system.") {
			continue
		} else if match := m.collectionMatch.MatchString(collection); !match {
			continue
		}

		var (
			query  = bson.M{}
			result bson.M // hold the document
		)

		iter := m.mongoSession.DB(m.database).C(collection).Find(query).Sort("_id").Iter()

		for {
			for iter.Next(&result) {
				if stop := m.pipe.Stopped; stop {
					return
				}

				// set up the message
				msg := message.NewMsg(message.Insert, result, m.computeNamespace(collection))

				m.pipe.Send(msg)
				result = bson.M{}
			}

			// we've exited the mongo read loop, lets figure out why
			// check here again if we've been asked to quit
			if stop := m.pipe.Stopped; stop {
				return
			}

			if iter.Err() != nil && m.restartable {
				fmt.Printf("got err reading collection. reissuing query %v\n", iter.Err())
				time.Sleep(1 * time.Second)
				iter = m.mongoSession.DB(m.database).C(collection).Find(query).Sort("_id").Iter()
				continue
			}
			break
		}
	}
	return
}
func BenchmarkTransformOne(b *testing.B) {
	tpipe := pipe.NewPipe(nil, "path")
	transformer := &Transformer{
		pipe: tpipe,
		path: "path",
		fn:   "module.exports=function(doc) { return doc }",
	}
	err := transformer.initEnvironment()
	if err != nil {
		panic(err)
	}

	msg := message.NewMsg(message.Insert, map[string]interface{}{"id": bson.NewObjectId(), "name": "nick"})
	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		transformer.transformOne(msg)
	}
}
Esempio n. 7
0
// read each message from the file
func (d *File) readFile() (err error) {
	filename := strings.Replace(d.uri, "file://", "", 1)
	d.filehandle, err = os.Open(filename)
	if err != nil {
		d.pipe.Err <- NewError(CRITICAL, d.path, fmt.Sprintf("Can't open input file (%s)", err.Error()), nil)
		return err
	}

	decoder := json.NewDecoder(d.filehandle)
	for {
		var doc map[string]interface{}
		if err := decoder.Decode(&doc); err == io.EOF {
			break
		} else if err != nil {
			d.pipe.Err <- NewError(ERROR, d.path, fmt.Sprintf("Can't marshal document (%s)", err.Error()), nil)
			return err
		}
		d.pipe.Send(message.NewMsg(message.Insert, doc, fmt.Sprint("file.%s", filename)))
	}
	return nil
}
Esempio n. 8
0
// catdata pulls down the original collection
func (m *Mongodb) catData() (err error) {
	var (
		collection = m.mongoSession.DB(m.database).C(m.collection)
		query      = bson.M{}
		result     bson.M // hold the document
	)

	iter := collection.Find(query).Sort("_id").Iter()

	for {
		for iter.Next(&result) {
			if stop := m.pipe.Stopped; stop {
				return
			}

			// set up the message
			msg := message.NewMsg(message.Insert, result)

			m.pipe.Send(msg)
			result = bson.M{}
		}

		// we've exited the mongo read loop, lets figure out why
		// check here again if we've been asked to quit
		if stop := m.pipe.Stopped; stop {
			return
		}

		if iter.Err() != nil && m.restartable {
			fmt.Printf("got err reading collection. reissuing query %v\n", iter.Err())
			time.Sleep(1 * time.Second)
			iter = collection.Find(query).Sort("_id").Iter()
			continue
		}

		return
	}
}
Esempio n. 9
0
// read each message from twitter
func (d *Twitter) readTwitter() (err error) {

	d.client = twitterstream.NewClient(d.consumerKey, d.consumerSecret, d.accessToken, d.accessSecret)

	for {
		var conn *twitterstream.Connection

		if d.keywords == "" {
			log.Println("Sampling Twitter")
			conn, err = d.client.Sample()
		} else {
			log.Printf("Tracking %s", d.keywords)
			conn, err = d.client.Track(d.keywords)
		}

		if err == nil {

			for {
				if tweet, err := conn.Next(); err == nil {
					doc := structs.Map(tweet)
					msg := message.NewMsg(message.Insert, doc)
					if msg != nil {
						d.pipe.Send(msg)
					} else {
						break
					}
				} else {
					break
				}
			}
		} else {
			log.Println("Sleeping before reconnecting")
			time.Sleep(time.Duration(10 * time.Second))
		}
	}
}
Esempio n. 10
0
func (cloudant *Cloudant) sendChanges() error {
	var err error
	options := couchdb.Options{"include_docs": "true"}
	if cloudant.tail {
		options["feed"] = "continuous"
	}

	//fmt.Printf("--> %+v", options)

	defer func() {
		cloudant.changes.Close()
	}()
	cloudant.changes, err = cloudant.client.DB(cloudant.database).Changes(options)
	if err != nil {
		return err
	}

	for cloudant.changes.Next() {
		if stop := cloudant.pipe.Stopped; stop {
			return nil
		}

		op := message.Update
		if cloudant.changes.Deleted {
			op = message.Delete
		}
		var doc map[string]interface{}
		if err := json.Unmarshal([]byte(cloudant.changes.Doc), &doc); err != nil {
			return err
		}

		cloudant.pipe.Send(message.NewMsg(op, doc, "cloudant.transporter"))
	}

	return nil
}
Esempio n. 11
0
/*
 * tail the oplog
 */
func (m *Mongodb) tailData() (err error) {

	var (
		collection = m.mongoSession.DB("local").C("oplog.rs")
		result     oplogDoc // hold the document
		query      = bson.M{
			"ts": bson.M{"$gte": m.oplogTime},
			"ns": m.getNamespace(),
		}

		iter = collection.Find(query).LogReplay().Sort("$natural").Tail(m.oplogTimeout)
	)

	for {
		for iter.Next(&result) {
			if stop := m.pipe.Stopped; stop {
				return
			}
			if result.validOp() {

				var doc bson.M
				switch result.Op {
				case "i":
					doc = result.O
				case "d":
					doc = result.O
				case "u":
					doc, err = m.getOriginalDoc(result.O2)
					if err != nil { // errors aren't fatal here, but we need to send it down the pipe
						m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("Mongodb error (%s)", err.Error()), nil)
						continue
					}
				default:
					m.pipe.Err <- NewError(ERROR, m.path, "Mongodb error (unknown op type)", nil)
					continue
				}

				msg := message.NewMsg(message.OpTypeFromString(result.Op), doc)
				msg.Timestamp = int64(result.Ts) >> 32

				m.oplogTime = result.Ts
				m.pipe.Send(msg)
			}
			result = oplogDoc{}
		}

		// we've exited the mongo read loop, lets figure out why
		// check here again if we've been asked to quit
		if stop := m.pipe.Stopped; stop {
			return
		}
		if iter.Timeout() {
			continue
		}
		if iter.Err() != nil {
			return NewError(CRITICAL, m.path, fmt.Sprintf("Mongodb error (error reading collection %s)", iter.Err()), nil)
		}

		// query will change,
		query = bson.M{
			"ts": bson.M{"$gte": m.oplogTime},
			"ns": m.getNamespace(),
		}
		iter = collection.Find(query).LogReplay().Tail(m.oplogTimeout)
	}
}
func TestTransformOne(t *testing.T) {
	bsonID1 := bson.NewObjectId()
	bsonID2 := bson.ObjectIdHex("54a4420502a14b9641000001")
	tpipe := pipe.NewPipe(nil, "path")
	go func(p *pipe.Pipe) {
		for range p.Err {
			// noop
		}
	}(tpipe)

	data := []struct {
		fn  string
		in  *message.Msg
		out *message.Msg
		err bool
	}{
		{
			// just pass through
			"module.exports=function(doc) { return doc }",
			message.NewMsg(message.Insert, map[string]interface{}{"id": "id1", "name": "nick"}),
			message.NewMsg(message.Insert, map[string]interface{}{"id": "id1", "name": "nick"}),
			false,
		},
		{
			// delete the 'name' property
			"module.exports=function(doc) { doc['data'] = _.omit(doc['data'], ['name']); return doc }",
			message.NewMsg(message.Insert, map[string]interface{}{"id": "id2", "name": "nick"}),
			message.NewMsg(message.Insert, map[string]interface{}{"id": "id2"}),
			false,
		},
		{
			// delete's should be processed the same
			"module.exports=function(doc) { doc['data'] =  _.omit(doc['data'], ['name']); return doc }",
			message.NewMsg(message.Delete, map[string]interface{}{"id": "id2", "name": "nick"}),
			message.NewMsg(message.Delete, map[string]interface{}{"id": "id2"}),
			false,
		},
		{
			// delete's and commands should pass through, and the transformer fn shouldn't run
			"module.exports=function(doc) { return _.omit(doc['data'], ['name']) }",
			message.NewMsg(message.Command, map[string]interface{}{"id": "id2", "name": "nick"}),
			message.NewMsg(message.Command, map[string]interface{}{"id": "id2", "name": "nick"}),
			false,
		},
		{
			// bson should marshal and unmarshal properly
			"module.exports=function(doc) { return doc }",
			message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}),
			message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}),
			false,
		},
		{
			// we should be able to change the bson
			"module.exports=function(doc) { doc['data']['id']['$oid'] = '54a4420502a14b9641000001'; return doc }",
			message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}),
			message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID2, "name": "nick"}),
			false,
		},
		{
			// this throws an error
			"module.exports=function(doc) { return doc['data']['name'] }",
			message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}),
			message.NewMsg(message.Insert, "nick"),
			true,
		},
	}
	for _, v := range data {

		transformer := &Transformer{pipe: tpipe, path: "path", fn: v.fn}
		err := transformer.initEnvironment()
		if err != nil {
			panic(err)
		}

		msg, err := transformer.transformOne(v.in)
		if (err != nil) != v.err {
			t.Errorf("error expected %t but actually got %v", v.err, err)
			continue
		}
		if (!reflect.DeepEqual(msg.Data, v.out.Data) || err != nil) && !v.err {
			t.Errorf("expected:\n(%T) %+v\ngot:\n(%T) %+v with error (%v)\n", v.out.Data, v.out.Data, msg.Data, msg.Data, err)
		}
	}
}