func (r *Rethinkdb) sendAllDocuments(table string) error {
if r.debug {
fmt.Printf("sending all documents for %s\n", table)
}
cursor, err := gorethink.Table(table).Run(r.client)
if err != nil {
return err
}
defer cursor.Close()
var doc map[string]interface{}
for cursor.Next(&doc) {
if stop := r.pipe.Stopped; stop {
return nil
}
msg := message.NewMsg(message.Insert, r.prepareDocument(doc), r.computeNamespace(table))
r.pipe.Send(msg)
}
if err := cursor.Err(); err != nil {
return err
}
return nil
}
func TestFilestoreUpdates(t *testing.T) {
fs := NewFilestore("somelongkey", "/tmp/transporter.state")
data := []struct {
path string
in *message.Msg
out *message.Msg
}{
{
"somepath",
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
},
{
"somepath",
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 2}, "db.coll"),
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 2}, "db.coll"),
},
}
for _, d := range data {
err := fs.Set(d.path, &MsgState{Msg: d.in, Extra: make(map[string]interface{})})
if err != nil {
t.Errorf("got error: %s\n", err)
t.FailNow()
}
}
d := data[len(data)-1]
fh, err := os.Open("/tmp/transporter.state")
if err != nil {
t.Errorf("got error: %s\n", err)
t.FailNow()
}
states := make(map[string]*MsgState)
dec := gob.NewDecoder(fh)
err = dec.Decode(&states)
if err != nil {
t.Errorf("got error: %s\n", err)
t.FailNow()
}
out := states["somelongkey-somepath"]
if !reflect.DeepEqual(out.Msg, d.out) {
t.Errorf("wanted: %s, got: %s", d.out, out.Msg)
}
}
func TestFilestore(t *testing.T) {
fs := NewFilestore("somelongkey", "/tmp/transporter.state")
data := []struct {
path string
in *message.Msg
out *message.Msg
}{
{
"somepath",
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
},
{
"somepath/morepath",
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
message.NewMsg(message.Insert, map[string]interface{}{"id": "nick1", "field1": 1}, "db.coll"),
},
}
for _, d := range data {
err := fs.Set(d.path, &MsgState{Msg: d.in, Extra: make(map[string]interface{})})
if err != nil {
t.Errorf("got error: %s\n", err)
t.FailNow()
}
}
for _, d := range data {
out, err := fs.Get(d.path)
if err != nil {
t.Errorf("got error: %s\n", err)
t.FailNow()
}
if !reflect.DeepEqual(out.Msg, d.out) {
t.Errorf("wanted: %s, got: %s", d.out, out.Msg)
}
}
}
func (r *Rethinkdb) sendChanges(table string, ccursor *gorethink.Cursor) error {
defer ccursor.Close()
if r.debug {
fmt.Printf("sending changes for %s\n", table)
}
var change rethinkDbChangeNotification
for ccursor.Next(&change) {
if stop := r.pipe.Stopped; stop {
return nil
}
if r.debug {
fmt.Printf("change: %#v\n", change)
}
var msg *message.Msg
if change.Error != "" {
return errors.New(change.Error)
} else if change.OldVal != nil && change.NewVal != nil {
msg = message.NewMsg(message.Update, r.prepareDocument(change.NewVal), r.computeNamespace(table))
} else if change.NewVal != nil {
msg = message.NewMsg(message.Insert, r.prepareDocument(change.NewVal), r.computeNamespace(table))
} else if change.OldVal != nil {
msg = message.NewMsg(message.Delete, r.prepareDocument(change.OldVal), r.computeNamespace(table))
}
if msg != nil {
r.pipe.Send(msg)
if r.debug {
fmt.Printf("msg: %#v\n", msg)
}
}
}
if err := ccursor.Err(); err != nil {
return err
}
return nil
}
// catdata pulls down the original collections
func (m *Mongodb) catData() (err error) {
collections, _ := m.mongoSession.DB(m.database).CollectionNames()
for _, collection := range collections {
if strings.HasPrefix(collection, "system.") {
continue
} else if match := m.collectionMatch.MatchString(collection); !match {
continue
}
var (
query = bson.M{}
result bson.M // hold the document
)
iter := m.mongoSession.DB(m.database).C(collection).Find(query).Sort("_id").Iter()
for {
for iter.Next(&result) {
if stop := m.pipe.Stopped; stop {
return
}
// set up the message
msg := message.NewMsg(message.Insert, result, m.computeNamespace(collection))
m.pipe.Send(msg)
result = bson.M{}
}
// we've exited the mongo read loop, lets figure out why
// check here again if we've been asked to quit
if stop := m.pipe.Stopped; stop {
return
}
if iter.Err() != nil && m.restartable {
fmt.Printf("got err reading collection. reissuing query %v\n", iter.Err())
time.Sleep(1 * time.Second)
iter = m.mongoSession.DB(m.database).C(collection).Find(query).Sort("_id").Iter()
continue
}
break
}
}
return
}
func BenchmarkTransformOne(b *testing.B) {
tpipe := pipe.NewPipe(nil, "path")
transformer := &Transformer{
pipe: tpipe,
path: "path",
fn: "module.exports=function(doc) { return doc }",
}
err := transformer.initEnvironment()
if err != nil {
panic(err)
}
msg := message.NewMsg(message.Insert, map[string]interface{}{"id": bson.NewObjectId(), "name": "nick"}, "database.collection")
b.ResetTimer()
for i := 0; i < b.N; i++ {
transformer.transformOne(msg)
}
}
// read each message from the file
func (d *File) readFile() (err error) {
filename := strings.Replace(d.uri, "file://", "", 1)
d.filehandle, err = os.Open(filename)
if err != nil {
d.pipe.Err <- NewError(CRITICAL, d.path, fmt.Sprintf("Can't open input file (%s)", err.Error()), nil)
return err
}
decoder := json.NewDecoder(d.filehandle)
for {
var doc map[string]interface{}
if err := decoder.Decode(&doc); err == io.EOF {
break
} else if err != nil {
d.pipe.Err <- NewError(ERROR, d.path, fmt.Sprintf("Can't marshal document (%s)", err.Error()), nil)
return err
}
d.pipe.Send(message.NewMsg(message.Insert, doc, fmt.Sprintf("file.%s", filename)))
}
return nil
}
/*
* tail the oplog
*/
func (m *Mongodb) tailData() (err error) {
var (
collection = m.mongoSession.DB("local").C("oplog.rs")
result oplogDoc // hold the document
query = bson.M{
"ts": bson.M{"$gte": m.oplogTime},
}
iter = collection.Find(query).LogReplay().Sort("$natural").Tail(m.oplogTimeout)
retries = 0
)
for {
for iter.Next(&result) {
retries = 0
if stop := m.pipe.Stopped; stop {
return
}
if result.validOp() {
_, coll, _ := m.splitNamespace(result.Ns)
if strings.HasPrefix(coll, "system.") {
continue
} else if match := m.collectionMatch.MatchString(coll); !match {
continue
}
var doc bson.M
switch result.Op {
case "i":
doc = result.O
case "d":
doc = result.O
case "u":
doc, err = m.getOriginalDoc(result.O2, coll)
if err != nil { // errors aren't fatal here, but we need to send it down the pipe
m.pipe.Err <- NewError(ERROR, m.path, fmt.Sprintf("Mongodb error (%s)", err.Error()), nil)
continue
}
default:
m.pipe.Err <- NewError(ERROR, m.path, "Mongodb error (unknown op type)", nil)
continue
}
msg := message.NewMsg(message.OpTypeFromString(result.Op), doc, m.computeNamespace(coll))
msg.Timestamp = int64(result.Ts) >> 32
m.oplogTime = result.Ts
m.pipe.Send(msg)
}
result = oplogDoc{}
}
// we've exited the mongo read loop, lets figure out why
// check here again if we've been asked to quit
if stop := m.pipe.Stopped; stop {
return
}
if iter.Timeout() {
continue
}
if iter.Err() != nil {
// Do not return, just retry up to 5 times
retries++
if retries > 4 {
return NewError(CRITICAL, m.path, fmt.Sprintf("Mongodb error (error reading collection %s)", iter.Err()), nil)
}
}
// query will change,
query = bson.M{
"ts": bson.M{"$gte": m.oplogTime},
}
iter = collection.Find(query).LogReplay().Tail(m.oplogTimeout)
}
}
func TestTransformOne(t *testing.T) {
bsonID1 := bson.NewObjectId()
bsonID2 := bson.ObjectIdHex("54a4420502a14b9641000001")
tpipe := pipe.NewPipe(nil, "path")
go func(p *pipe.Pipe) {
for range p.Err {
// noop
}
}(tpipe)
data := []struct {
fn string
in *message.Msg
out *message.Msg
err bool
}{
{
// just pass through
"module.exports=function(doc) { return doc }",
message.NewMsg(message.Insert, map[string]interface{}{"id": "id1", "name": "nick"}, "database.collection"),
message.NewMsg(message.Insert, map[string]interface{}{"id": "id1", "name": "nick"}, "database.collection"),
false,
},
{
// delete the 'name' property
"module.exports=function(doc) { doc['data'] = _.omit(doc['data'], ['name']); return doc }",
message.NewMsg(message.Insert, map[string]interface{}{"id": "id2", "name": "nick"}, "database.collection"),
message.NewMsg(message.Insert, map[string]interface{}{"id": "id2"}, "database.collection"),
false,
},
{
// delete's should be processed the same
"module.exports=function(doc) { doc['data'] = _.omit(doc['data'], ['name']); return doc }",
message.NewMsg(message.Delete, map[string]interface{}{"id": "id2", "name": "nick"}, "database.collection"),
message.NewMsg(message.Delete, map[string]interface{}{"id": "id2"}, "database.collection"),
false,
},
{
// delete's and commands should pass through, and the transformer fn shouldn't run
"module.exports=function(doc) { return _.omit(doc['data'], ['name']) }",
message.NewMsg(message.Command, map[string]interface{}{"id": "id2", "name": "nick"}, "database.collection"),
message.NewMsg(message.Command, map[string]interface{}{"id": "id2", "name": "nick"}, "database.collection"),
false,
},
{
// bson should marshal and unmarshal properly
"module.exports=function(doc) { return doc }",
message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.collection"),
message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.collection"),
false,
},
{
// we should be able to change the bson
"module.exports=function(doc) { doc['data']['id']['$oid'] = '54a4420502a14b9641000001'; return doc }",
message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.collection"),
message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID2, "name": "nick"}, "database.collection"),
false,
}, {
// we should be able to skip a nil message
"module.exports=function(doc) { return false }",
message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.collection"),
message.NewMsg(message.Noop, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.collection"),
false,
},
{
// this throws an error
"module.exports=function(doc) { return doc['data']['name'] }",
message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.collection"),
message.NewMsg(message.Insert, "nick", "database.collection"),
true,
},
{
// we should be able to change the namespace
"module.exports=function(doc) { doc['ns'] = 'database.table'; return doc }",
message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.collection"),
message.NewMsg(message.Insert, map[string]interface{}{"id": bsonID1, "name": "nick"}, "database.table"),
false,
},
}
for _, v := range data {
transformer := &Transformer{pipe: tpipe, path: "path", fn: v.fn}
err := transformer.initEnvironment()
if err != nil {
panic(err)
}
msg, err := transformer.transformOne(v.in)
if (err != nil) != v.err {
t.Errorf("error expected %t but actually got %v", v.err, err)
continue
}
if (!reflect.DeepEqual(msg, v.out) || err != nil) && !v.err {
t.Errorf("expected:\n(%T) %+v\ngot:\n(%T) %+v with error (%v)\n", v.out, v.out, msg, msg, err)
}
}
}