func (r *Rethinkdb) sendAllDocuments() error {
if r.debug {
fmt.Printf("sending all documents\n")
}
cursor, err := gorethink.Table(r.table).Run(r.client)
if err != nil {
return err
}
defer cursor.Close()
var doc map[string]interface{}
for cursor.Next(&doc) {
if stop := r.pipe.Stopped; stop {
return nil
}
msg := message.NewMsg(message.Insert, r.prepareDocument(doc))
r.pipe.Send(msg)
}
if err := cursor.Err(); err != nil {
return err
}
return nil
}
func TestFilestoreUpdates(t *testing.T) {
fs := NewFilestore("somelongkey", "/tmp/transporter.state")
data := []struct {
path string
in *message.Msg
out *message.Msg
}{
{
"somepath",
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
},
{
"somepath",
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 2}, "db.coll"),
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 2}, "db.coll"),
},
}
for _, d := range data {
err := fs.Set(d.path, &MsgState{Msg: d.in, Extra: make(map[string]interface{})})
if err != nil {
t.Errorf("got error: %s\n", err)
t.FailNow()
}
}
d := data[len(data)-1]
fh, err := os.Open("/tmp/transporter.state")
if err != nil {
t.Errorf("got error: %s\n", err)
t.FailNow()
}
states := make(map[string]*MsgState)
dec := gob.NewDecoder(fh)
err = dec.Decode(&states)
if err != nil {
t.Errorf("got error: %s\n", err)
t.FailNow()
}
out := states["somelongkey-somepath"]
if !reflect.DeepEqual(out.Msg, d.out) {
t.Errorf("wanted: %s, got: %s", d.out, out.Msg)
}
}
func TestFilestore(t *testing.T) {
fs := NewFilestore("somelongkey", "/tmp/transporter.state")
data := []struct {
path string
in *message.Msg
out *message.Msg
}{
{
"somepath",
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
},
{
"somepath/morepath",
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
},
}
for _, d := range data {
err := fs.Set(d.path, &MsgState{Msg: d.in, Extra: make(map[string]interface{})})
if err != nil {
t.Errorf("got error: %s\n", err)
t.FailNow()
}
}
for _, d := range data {
out, err := fs.Get(d.path)
if err != nil {
t.Errorf("got error: %s\n", err)
t.FailNow()
}
if !reflect.DeepEqual(out.Msg, d.out) {
t.Errorf("wanted: %s, got: %s", d.out, out.Msg)
}
}
}
func (r *Rethinkdb) sendChanges(table string, ccursor *gorethink.Cursor) error {
defer ccursor.Close()
if r.debug {
fmt.Printf("sending changes for %s\n", table)
}
var change rethinkDbChangeNotification
for ccursor.Next(&change) {
if stop := r.pipe.Stopped; stop {
return nil
}
if r.debug {
fmt.Printf("change: %#v\n", change)
}
var msg *message.Msg
if change.Error != "" {
return errors.New(change.Error)
} else if change.OldVal != nil && change.NewVal != nil {
msg = message.NewMsg(message.Update, r.prepareDocument(change.NewVal), r.computeNamespace(table))
} else if change.NewVal != nil {
msg = message.NewMsg(message.Insert, r.prepareDocument(change.NewVal), r.computeNamespace(table))
} else if change.OldVal != nil {
msg = message.NewMsg(message.Delete, r.prepareDocument(change.OldVal), r.computeNamespace(table))
}
if msg != nil {
r.pipe.Send(msg)
if r.debug {
fmt.Printf("msg: %#v\n", msg)
}
}
}
if err := ccursor.Err(); err != nil {
return err
}
return nil
}
// catdata pulls down the original collections
func (m *Mongodb) catData() (err error) {
collections, _ := m.mongoSession.DB(m.database).CollectionNames()
for _, collection := range collections {
if strings.HasPrefix(collection, "system.") {
continue
} else if match := m.collectionMatch.MatchString(collection); !match {
continue
}
var (
query = bson.M{}
result bson.M // hold the document
)
iter := m.mongoSession.DB(m.database).C(collection).Find(query).Sort("_id").Iter()
for {
for iter.Next(&result) {
if stop := m.pipe.Stopped; stop {
return
}
// set up the message
msg := message.NewMsg(message.Insert, result, m.computeNamespace(collection))
m.pipe.Send(msg)
result = bson.M{}
}
// we've exited the mongo read loop, lets figure out why
// check here again if we've been asked to quit
if stop := m.pipe.Stopped; stop {
return
}
if iter.Err() != nil && m.restartable {
fmt.Printf("got err reading collection. reissuing query %v\n", iter.Err())
time.Sleep(1 * time.Second)
iter = m.mongoSession.DB(m.database).C(collection).Find(query).Sort("_id").Iter()
continue
}
break
}
}
return
}
func BenchmarkTransformOne(b *testing.B) {
tpipe := pipe.NewPipe(nil, "path")
transformer := &Transformer{
pipe: tpipe,
path: "path",
fn: "module.exports=function(doc) { return doc }",
}
err := transformer.initEnvironment()
if err != nil {
panic(err)
}
msg := message.NewMsg(message.Insert, map[string]interface{}{"id": bson.NewObjectId(), "name": "nick"})
b.ResetTimer()
for i := 0; i < b.N; i++ {
transformer.transformOne(msg)
}
}
// read each message from the file
func (d *File) readFile() (err error) {
filename := strings.Replace(d.uri, "file://", "", 1)
d.filehandle, err = os.Open(filename)
if err != nil {
d.pipe.Err <- NewError(CRITICAL, d.path, fmt.Sprintf("Can't open input file (%s)", err.Error()), nil)
return err
}
decoder := json.NewDecoder(d.filehandle)
for {
var doc map[string]interface{}
if err := decoder.Decode(&doc); err == io.EOF {
break
} else if err != nil {
d.pipe.Err <- NewError(ERROR, d.path, fmt.Sprintf("Can't marshal document (%s)", err.Error()), nil)
return err
}
d.pipe.Send(message.NewMsg(message.Insert, doc, fmt.Sprint("file.%s", filename)))
}
return nil
}
// catdata pulls down the original collection
func (m *Mongodb) catData() (err error) {
var (
collection = m.mongoSession.DB(m.database).C(m.collection)
query = bson.M{}
result bson.M // hold the document
)
iter := collection.Find(query).Sort("_id").Iter()
for {
for iter.Next(&result) {
if stop := m.pipe.Stopped; stop {
return
}
// set up the message
msg := message.NewMsg(message.Insert, result)
m.pipe.Send(msg)
result = bson.M{}
}
// we've exited the mongo read loop, lets figure out why
// check here again if we've been asked to quit
if stop := m.pipe.Stopped; stop {
return
}
if iter.Err() != nil && m.restartable {
fmt.Printf("got err reading collection. reissuing query %v\n", iter.Err())
time.Sleep(1 * time.Second)
iter = collection.Find(query).Sort("_id").Iter()
continue
}
return
}
}
// read each message from twitter
func (d *Twitter) readTwitter() (err error) {
d.client = twitterstream.NewClient(d.consumerKey, d.consumerSecret, d.accessToken, d.accessSecret)
for {
var conn *twitterstream.Connection
if d.keywords == "" {
log.Println("Sampling Twitter")
conn, err = d.client.Sample()
} else {
log.Printf("Tracking %s", d.keywords)
conn, err = d.client.Track(d.keywords)
}
if err == nil {
for {
if tweet, err := conn.Next(); err == nil {
doc := structs.Map(tweet)
msg := message.NewMsg(message.Insert, doc)
if msg != nil {
d.pipe.Send(msg)
} else {
break
}
} else {
break
}
}
} else {
log.Println("Sleeping before reconnecting")
time.Sleep(time.Duration(10 * time.Second))
}
}
}
func (cloudant *Cloudant) sendChanges() error {
var err error
options := couchdb.Options{"include_docs": "true"}
if cloudant.tail {
options["feed"] = "continuous"
}
//fmt.Printf("--> %+v", options)
defer func() {
cloudant.changes.Close()
}()
cloudant.changes, err = cloudant.client.DB(cloudant.database).Changes(options)
if err != nil {
return err
}
for cloudant.changes.Next() {
if stop := cloudant.pipe.Stopped; stop {
return nil
}
op := message.Update
if cloudant.changes.Deleted {
op = message.Delete
}
var doc map[string]interface{}
if err := json.Unmarshal([]byte(cloudant.changes.Doc), &doc); err != nil {
return err
}
cloudant.pipe.Send(message.NewMsg(op, doc, "cloudant.transporter"))
}
return nil
}
/*
* tail the oplog
*/
func (m *Mongodb) tailData() (err error) {
var (
collection = m.mongoSession.DB("local").C("oplog.rs")
result oplogDoc // hold the document
query = bson.M{
"ts": bson.M{"$gte": m.oplogTime},
"ns": m.getNamespace(),
}
iter = collection.Find(query).LogReplay().Sort("$natural").Tail(m.oplogTimeout)
)
for {
for iter.Next(&result) {
if stop := m.pipe.Stopped; stop {
return
}
if result.validOp() {
var doc bson.M
switch result.Op {
case "i":
doc = result.O
case "d":
doc = result.O
case "u":
doc, err = m.getOriginalDoc(result.O2)
if err != nil { // errors aren't fatal here, but we need to send it down the pipe
m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("Mongodb error (%s)", err.Error()), nil)
continue
}
default:
m.pipe.Err <- NewError(ERROR, m.path, "Mongodb error (unknown op type)", nil)
continue
}
msg := message.NewMsg(message.OpTypeFromString(result.Op), doc)
msg.Timestamp = int64(result.Ts) >> 32
m.oplogTime = result.Ts
m.pipe.Send(msg)
}
result = oplogDoc{}
}
// we've exited the mongo read loop, lets figure out why
// check here again if we've been asked to quit
if stop := m.pipe.Stopped; stop {
return
}
if iter.Timeout() {
continue
}
if iter.Err() != nil {
return NewError(CRITICAL, m.path, fmt.Sprintf("Mongodb error (error reading collection %s)", iter.Err()), nil)
}
// query will change,
query = bson.M{
"ts": bson.M{"$gte": m.oplogTime},
"ns": m.getNamespace(),
}
iter = collection.Find(query).LogReplay().Tail(m.oplogTimeout)
}
}
func TestTransformOne(t *testing.T) {
bsonID1 := bson.NewObjectId()
bsonID2 := bson.ObjectIdHex("54a4420502a14b9641000001")
tpipe := pipe.NewPipe(nil, "path")
go func(p *pipe.Pipe) {
for range p.Err {
// noop
}
}(tpipe)
data := []struct {
fn string
in *message.Msg
out *message.Msg
err bool
}{
{
// just pass through
"module.exports=function(doc) { return doc }",
message.NewMsg(message.Insert, map[string]interface{}{"id": "id1", "name": "nick"}),
message.NewMsg(message.Insert, map[string]interface{}{"id": "id1", "name": "nick"}),
false,
},
{
// delete the 'name' property
"module.exports=function(doc) { doc['data'] = _.omit(doc['data'], ['name']); return doc }",
message.NewMsg(message.Insert, map[string]interface{}{"id": "id2", "name": "nick"}),
message.NewMsg(message.Insert, map[string]interface{}{"id": "id2"}),
false,
},
{
// delete's should be processed the same
"module.exports=function(doc) { doc['data'] = _.omit(doc['data'], ['name']); return doc }",
message.NewMsg(message.Delete, map[string]interface{}{"id": "id2", "name": "nick"}),
message.NewMsg(message.Delete, map[string]interface{}{"id": "id2"}),
false,
},
{
// delete's and commands should pass through, and the transformer fn shouldn't run
"module.exports=function(doc) { return _.omit(doc['data'], ['name']) }",
message.NewMsg(message.Command, map[string]interface{}{"id": "id2", "name": "nick"}),
message.NewMsg(message.Command, map[string]interface{}{"id": "id2", "name": "nick"}),
false,
},
{
// bson should marshal and unmarshal properly
"module.exports=function(doc) { return doc }",
message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}),
message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}),
false,
},
{
// we should be able to change the bson
"module.exports=function(doc) { doc['data']['id']['$oid'] = '54a4420502a14b9641000001'; return doc }",
message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}),
message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID2, "name": "nick"}),
false,
},
{
// this throws an error
"module.exports=function(doc) { return doc['data']['name'] }",
message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}),
message.NewMsg(message.Insert, "nick"),
true,
},
}
for _, v := range data {
transformer := &Transformer{pipe: tpipe, path: "path", fn: v.fn}
err := transformer.initEnvironment()
if err != nil {
panic(err)
}
msg, err := transformer.transformOne(v.in)
if (err != nil) != v.err {
t.Errorf("error expected %t but actually got %v", v.err, err)
continue
}
if (!reflect.DeepEqual(msg.Data, v.out.Data) || err != nil) && !v.err {
t.Errorf("expected:\n(%T) %+v\ngot:\n(%T) %+v with error (%v)\n", v.out.Data, v.out.Data, msg.Data, msg.Data, err)
}
}
}