func (s *oplogSuite) TestRestartsOnError(c *gc.C) {
_, session := s.startMongo(c)
oplog := s.makeFakeOplog(c, session)
tailer := mongo.NewOplogTailer(oplog, nil, time.Time{})
defer tailer.Stop()
// First, ensure that the tailer is seeing oplog inserts.
s.insertDoc(c, session, oplog, &mongo.OplogDoc{
Timestamp: 1,
OperationId: 99,
})
doc := s.getNextOplog(c, tailer)
c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
s.emptyCapped(c, oplog)
// Ensure that the tailer still works.
s.insertDoc(c, session, oplog, &mongo.OplogDoc{
Timestamp: 2,
OperationId: 42,
})
doc = s.getNextOplog(c, tailer)
c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(2))
}
func TestTimestamp(t *testing.T) {
data := []struct {
in M
want bson.MongoTimestamp
wantok bool
}{
{
map[string]interface{}{"$timestamp": map[string]interface{}{"t": 1392822881, "i": 1}},
bson.MongoTimestamp(5982128723015499777),
true,
},
{
map[string]interface{}{"$ugh": map[string]interface{}{"t": 1392822881, "i": 1}},
bson.MongoTimestamp(5982128723015499777),
false,
},
}
for _, d := range data {
b, ok := d.in.timestamp()
if ok != d.wantok {
t.Errorf("got %t, want %t, (%v)", ok, d.wantok, d.in)
t.FailNow()
}
if ok && b != d.want {
t.Errorf("wanted: %v (%T), got: %v (%T)", d.want, d.want, b, b)
}
}
}
func TestValidOplogLimitChecking(t *testing.T) {
testutil.VerifyTestType(t, testutil.UnitTestType)
Convey("With a MongoRestore instance with oplogLimit of 5:0", t, func() {
mr := &MongoRestore{
oplogLimit: bson.MongoTimestamp(int64(5) << 32),
}
Convey("an oplog entry with ts=1000:0 should be invalid", func() {
So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(1000)<<32)), ShouldBeFalse)
})
Convey("an oplog entry with ts=5:1 should be invalid", func() {
So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(5)<<32|1)), ShouldBeFalse)
})
Convey("an oplog entry with ts=5:0 should be invalid", func() {
So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(5)<<32)), ShouldBeFalse)
})
Convey("an oplog entry with ts=4:9 should be valid", func() {
So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(4)<<32|9)), ShouldBeTrue)
})
Convey("an oplog entry with ts=4:0 should be valid", func() {
So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(4)<<32)), ShouldBeTrue)
})
Convey("an oplog entry with ts=0:1 should be valid", func() {
So(mr.TimestampBeforeLimit(bson.MongoTimestamp(1)), ShouldBeTrue)
})
})
Convey("With a MongoRestore instance with no oplogLimit", t, func() {
mr := &MongoRestore{}
Convey("an oplog entry with ts=1000:0 should be valid", func() {
So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(1000)<<32)), ShouldBeTrue)
})
Convey("an oplog entry with ts=5:1 should be valid", func() {
So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(5)<<32|1)), ShouldBeTrue)
})
Convey("an oplog entry with ts=5:0 should be valid", func() {
So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(5)<<32)), ShouldBeTrue)
})
})
}
func OplogDeamon() {
url := fmt.Sprintf("mongodb://%s:%s", Conf.MongoIP, Conf.MongoPort)
session, err := mgo.Dial(url)
if err != nil {
panic(err)
}
defer session.Close()
c := session.DB("local").C("oplog.rs")
cp := initCheckpoint()
log.Info("init check point is %d ", cp)
iter := c.Find(bson.M{"ts": bson.M{"$gte": bson.MongoTimestamp(cp)}}).Tail(-1 * time.Second)
defer iter.Close()
count := uint64(0)
for {
log.Info("start oplog manarger !!! ")
for result := new(Oplog); iter.Next(&result); result = new(Oplog) {
count++
checkpoint_threshold(count, result)
if !checkTopicCollection(result.Ns) {
log.Info("the namespace(%s) not in topics: %s", result.Ns, strings.Join(Conf.TopicCollections, "|"))
continue
}
// new event
newEvent(result)
}
if iter.Err() != nil {
iter.Close()
}
if iter.Timeout() {
continue
}
// reload checkp point from file
cp = initCheckpoint()
iter = c.Find(bson.M{"ts": bson.M{"$gte": bson.MongoTimestamp(cp)}}).Tail(-1 * time.Second)
}
}
func (s *oplogSuite) TestNoRepeatsAfterIterRestart(c *gc.C) {
// A bunch of documents with the same timestamp but different ids.
// These will be split across 2 iterators.
docs := make([]*mongo.OplogDoc, 11)
for i := 0; i < 10; i++ {
id := int64(i + 10)
docs[i] = &mongo.OplogDoc{
Timestamp: 1,
OperationId: id,
}
}
// Add one more with a different timestamp.
docs[10] = &mongo.OplogDoc{
Timestamp: 2,
OperationId: 42,
}
session := newFakeSession(
// First block of documents, all time 1
newFakeIterator(nil, docs[:5]...),
// Second block, some time 1, one time 2
newFakeIterator(nil, docs[5:]...),
)
tailer := mongo.NewOplogTailer(session, time.Time{})
defer tailer.Stop()
for id := int64(10); id < 15; id++ {
doc := s.getNextOplog(c, tailer)
c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
c.Assert(doc.OperationId, gc.Equals, id)
}
// Check the query doesn't exclude any in the first request.
session.checkLastArgs(c, mongo.NewMongoTimestamp(time.Time{}), nil)
// The OplogTailer will fall off the end of the iterator and get a new one.
// Ensure that only previously unreported entries are now reported.
for id := int64(15); id < 20; id++ {
doc := s.getNextOplog(c, tailer)
c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
c.Assert(doc.OperationId, gc.Equals, id)
}
// Check we got the next block correctly
session.checkLastArgs(c, bson.MongoTimestamp(1), []int64{10, 11, 12, 13, 14})
doc := s.getNextOplog(c, tailer)
c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(2))
c.Assert(doc.OperationId, gc.Equals, int64(42))
}
func (s *oplogSuite) TestNoRepeatsAfterIterRestart(c *gc.C) {
_, session := s.startMongo(c)
oplog := s.makeFakeOplog(c, session)
tailer := mongo.NewOplogTailer(oplog, nil, time.Time{})
defer tailer.Stop()
// Insert a bunch of oplog entries with the same timestamp (but
// with different ids) and see them reported.
for id := int64(10); id < 15; id++ {
s.insertDoc(c, session, oplog, &mongo.OplogDoc{
Timestamp: 1,
OperationId: id,
})
doc := s.getNextOplog(c, tailer)
c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
c.Assert(doc.OperationId, gc.Equals, id)
}
// Force the OplogTailer's iterator to be recreated.
s.emptyCapped(c, oplog)
// Reinsert the oplog entries that were already there before and a
// few more.
for id := int64(10); id < 20; id++ {
s.insertDoc(c, session, oplog, &mongo.OplogDoc{
Timestamp: 1,
OperationId: id,
})
}
// Insert an entry for a later timestamp.
s.insertDoc(c, session, oplog, &mongo.OplogDoc{
Timestamp: 2,
OperationId: 42,
})
// Ensure that only previously unreported entries are now reported.
for id := int64(15); id < 20; id++ {
doc := s.getNextOplog(c, tailer)
c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
c.Assert(doc.OperationId, gc.Equals, id)
}
doc := s.getNextOplog(c, tailer)
c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(2))
c.Assert(doc.OperationId, gc.Equals, int64(42))
}
// ParseTimestampFlag takes in a string the form of <time_t>:<ordinal>,
// where <time_t> is the seconds since the UNIX epoch, and <ordinal> represents
// a counter of operations in the oplog that occurred in the specified second.
// It parses this timestamp string and returns a bson.MongoTimestamp type.
func ParseTimestampFlag(ts string) (bson.MongoTimestamp, error) {
var seconds, increment int
timestampFields := strings.Split(ts, ":")
if len(timestampFields) > 2 {
return 0, fmt.Errorf("too many : characters")
}
seconds, err := strconv.Atoi(timestampFields[0])
if err != nil {
return 0, fmt.Errorf("error parsing timestamp seconds: %v", err)
}
// parse the increment field if it exists
if len(timestampFields) == 2 {
if len(timestampFields[1]) > 0 {
increment, err = strconv.Atoi(timestampFields[1])
if err != nil {
return 0, fmt.Errorf("error parsing timestamp increment: %v", err)
}
} else {
// handle the case where the user writes "<time_t>:" with no ordinal
increment = 0
}
}
timestamp := (int64(seconds) << 32) | int64(increment)
return bson.MongoTimestamp(timestamp), nil
}
// WriteLogWithOplog writes out a log record to the a (probably fake)
// oplog collection and the logs collection.
func WriteLogWithOplog(
oplog *mgo.Collection,
envUUID string,
entity names.Tag,
t time.Time,
module string,
location string,
level loggo.Level,
msg string,
) error {
doc := &logDoc{
Id: bson.NewObjectId(),
Time: t,
EnvUUID: envUUID,
Entity: entity.String(),
Module: module,
Location: location,
Level: level,
Message: msg,
}
err := oplog.Insert(bson.D{
{"ts", bson.MongoTimestamp(time.Now().Unix() << 32)}, // an approximation which will do
{"h", rand.Int63()}, // again, a suitable fake
{"op", "i"}, // this will always be an insert
{"ns", "logs.logs"},
{"o", doc},
})
if err != nil {
return err
}
session := oplog.Database.Session
logs := session.DB("logs").C("logs")
return logs.Insert(doc)
}
func (s *oplogSuite) TestNewMongoTimestamp(c *gc.C) {
t := time.Date(2015, 6, 24, 12, 47, 0, 0, time.FixedZone("somewhere", 5*3600))
expected := bson.MongoTimestamp(6163845091342417920)
c.Assert(mongo.NewMongoTimestamp(t), gc.Equals, expected)
c.Assert(mongo.NewMongoTimestamp(t.In(time.UTC)), gc.Equals, expected)
}
func TailOps(session *mgo.Session, channel OpChan, errChan chan error, options *Options) error {
s := session.Copy()
defer s.Close()
FillEmptyOptions(session, options)
duration, err := time.ParseDuration(*options.CursorTimeout)
if err != nil {
panic("Invalid value for CursorTimeout")
}
currTimestamp := options.After(s, options)
iter := GetOpLogQuery(s, currTimestamp, options).Tail(duration)
for {
entry := make(OpLogEntry)
for iter.Next(entry) {
op := &Op{"", "", "", nil, bson.MongoTimestamp(0)}
op.ParseLogEntry(entry)
if op.Id != "" {
if options.Filter == nil || options.Filter(op) {
channel <- op
}
}
currTimestamp = op.Timestamp
}
if err = iter.Close(); err != nil {
errChan <- err
return err
}
if iter.Timeout() {
continue
}
iter = GetOpLogQuery(s, currTimestamp, options).Tail(duration)
}
return nil
}
func TestParseLargeBSON(t *testing.T) {
largeArray := make([]interface{}, 5000)
for i := 0; i < 5000; i++ {
largeArray[i] = float64(i)
}
expectedOp := map[string]interface{}{
"ts": bson.MongoTimestamp(6048257058866724865), "h": int64(-6825742652110581687), "v": 2, "op": "i", "ns": "testdb.testdb", "o": map[string]interface{}{
"_id": bson.ObjectIdHex("53efb9c067fd92348e823860"),
"val": largeArray}}
f, err := os.Open("./largetestdata.bson")
if err != nil {
t.Fatal("Error loading file", err)
}
defer f.Close()
foundExpectedOp := false
scanner := New(f)
for scanner.Scan() {
op := map[string]interface{}{}
if err := bson.Unmarshal(scanner.Bytes(), &op); err != nil {
t.Fatal("Error unmarshalling: ", err)
}
if reflect.DeepEqual(op, expectedOp) {
foundExpectedOp = true
}
}
if scanner.Err() != nil {
t.Fatal("Scanner error: ", scanner.Err())
}
if !foundExpectedOp {
t.Fatal("Didn't find the expected operation")
}
}
// NewMongoTimestamp returns a bson.MongoTimestamp repesentation for
// the time.Time given. Note that these timestamps are not the same
// the usual MongoDB time fields. These are an internal format used
// only in a few places such as the replication oplog.
//
// See: http://docs.mongodb.org/manual/reference/bson-types/#timestamps
func NewMongoTimestamp(t time.Time) bson.MongoTimestamp {
unixTime := t.Unix()
if unixTime < 0 {
unixTime = 0
}
return bson.MongoTimestamp(unixTime << 32)
}
// get the cursor for the oplog collection, based on the options
// passed in to mongooplog
func buildTailingCursor(oplog *mgo.Collection,
sourceOptions *SourceOptions) *mgo.Iter {
// how many seconds in the past we need
secondsInPast := time.Duration(sourceOptions.Seconds) * time.Second
// the time threshold for oplog queries
threshold := time.Now().Add(-secondsInPast)
// convert to a unix timestamp (seconds since epoch)
thresholdAsUnix := threshold.Unix()
// shift it appropriately, to prepare it to be converted to an
// oplog timestamp
thresholdShifted := uint64(thresholdAsUnix) << 32
// build the oplog query
oplogQuery := bson.M{
"ts": bson.M{
"$gte": bson.MongoTimestamp(thresholdShifted),
},
}
// TODO: wait time
return oplog.Find(oplogQuery).Iter()
}
/* bsonify a mongo Timestamp */
func (m M) timestamp() (timestamp bson.MongoTimestamp, ok bool) {
if len(m) != 1 {
return
}
if value, contains := m["$timestamp"]; contains {
if ts, ismap := value.(map[string]interface{}); ismap {
t, isok := ts["t"]
if !isok {
return
}
tt, isok := t.(int)
if !isok {
return
}
i, isok := ts["i"]
if !isok {
return
}
ii, isok := i.(int)
if !isok {
return
}
ok = true
var concat int64
concat = int64(uint64(tt)<<32 | uint64(ii))
timestamp = bson.MongoTimestamp(concat)
}
}
return
}
func TestTimestampValue(t *testing.T) {
Convey("When converting JSON with Timestamp values", t, func() {
testTS := bson.MongoTimestamp(123456<<32 | 55)
Convey("works for Timestamp literal", func() {
jsonMap := map[string]interface{}{
"ts": json.Timestamp{123456, 55},
}
err := ConvertJSONDocumentToBSON(jsonMap)
So(err, ShouldBeNil)
So(jsonMap["ts"], ShouldEqual, testTS)
})
Convey(`works for Timestamp document`, func() {
Convey(`{"ts":{"$timestamp":{"t":123456, "i":55}}}`, func() {
jsonMap := map[string]interface{}{
"ts": map[string]interface{}{
"$timestamp": map[string]interface{}{
"t": 123456.0,
"i": 55.0,
},
},
}
bsonMap, err := ConvertJSONValueToBSON(jsonMap)
So(err, ShouldBeNil)
So(bsonMap.(map[string]interface{})["ts"], ShouldEqual, testTS)
})
})
})
}
//mongoTimestamp returns a valid mongoTimestamp
func (m mongoTimestamp) MongoTimestamp() bson.MongoTimestamp {
var b [12]byte
var result uint64
binary.BigEndian.PutUint32(b[:4], uint32(m.Unix()))
binary.Read(bytes.NewReader(b[:]), binary.BigEndian, &result)
return bson.MongoTimestamp(result)
}
// writeLogToOplog writes out a log record to the a (probably fake)
// oplog collection.
func (s *LogTailerSuite) writeLogToOplog(doc interface{}) error {
return s.oplogColl.Insert(bson.D{
{"ts", bson.MongoTimestamp(time.Now().Unix() << 32)}, // an approximation which will do
{"h", rand.Int63()}, // again, a suitable fake
{"op", "i"}, // this will always be an insert
{"ns", "logs.logs"},
{"o", doc},
})
}
func TestInsert(t *testing.T) {
if err := omgo.RunMgFun(mg_instance, mg_collection, func(c *mgo.Collection) error {
return c.Insert(&DongoData{
Id: bson.NewObjectId(),
Name: "yoer",
Date: bson.MongoTimestamp(time.Now().UnixNano()),
})
}); nil != err {
t.Error(err.Error())
}
}
func (s *oplogSuite) TestRestartsOnErrCursor(c *gc.C) {
session := newFakeSession(
// First iterator terminates with an ErrCursor
newFakeIterator(mgo.ErrCursor, &mongo.OplogDoc{Timestamp: 1, OperationId: 99}),
newFakeIterator(nil, &mongo.OplogDoc{Timestamp: 2, OperationId: 42}),
)
tailer := mongo.NewOplogTailer(session, time.Time{})
defer tailer.Stop()
// First, ensure that the tailer is seeing oplog rows and handles
// the ErrCursor that occurs at the end.
doc := s.getNextOplog(c, tailer)
c.Check(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
session.checkLastArgs(c, mongo.NewMongoTimestamp(time.Time{}), nil)
// Ensure that the tailer continues after getting a new iterator.
doc = s.getNextOplog(c, tailer)
c.Check(doc.Timestamp, gc.Equals, bson.MongoTimestamp(2))
session.checkLastArgs(c, bson.MongoTimestamp(1), []int64{99})
}
func TestMarshalTimestamp(t *testing.T) {
data := []struct {
in bson.MongoTimestamp
want []byte
}{
{bson.MongoTimestamp(0), []byte("{\"$timestamp\":{\"i\":0,\"t\":0}}")},
{bson.MongoTimestamp(5982128723015499777), []byte("{\"$timestamp\":{\"i\":1,\"t\":1392822881}}")},
}
for _, d := range data {
mejson := marshalTimestamp(d.in)
b, err := json.Marshal(mejson)
if err != nil {
t.FailNow()
}
if !reflect.DeepEqual(b, d.want) {
t.Errorf("wanted: %s, got: %s", d.want, b)
}
}
}
func (s *oplogSuite) TestDiesOnFatalError(c *gc.C) {
expectedErr := errors.New("oh no, the collection went away!")
session := newFakeSession(
newFakeIterator(expectedErr, &mongo.OplogDoc{Timestamp: 1}),
)
tailer := mongo.NewOplogTailer(session, time.Time{})
defer tailer.Stop()
doc := s.getNextOplog(c, tailer)
c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
s.assertStopped(c, tailer)
c.Assert(tailer.Err(), gc.Equals, expectedErr)
}
func TestTimestampBSONToJSON(t *testing.T) {
testutil.VerifyTestType(t, testutil.UnitTestType)
Convey("Converting a BSON Timestamp to JSON", t, func() {
Convey("should produce a json.Timestamp", func() {
// {t:803434343, i:9} == bson.MongoTimestamp(803434343*2**32 + 9)
_jObj, err := ConvertBSONValueToJSON(bson.MongoTimestamp(uint64(803434343<<32) | uint64(9)))
So(err, ShouldBeNil)
jObj, ok := _jObj.(json.Timestamp)
So(ok, ShouldBeTrue)
So(jObj, ShouldResemble, json.Timestamp{Seconds: 803434343, Increment: 9})
So(jObj, ShouldNotResemble, json.Timestamp{Seconds: 803434343, Increment: 8})
})
})
}
func (s *oplogSuite) TestDiesOnFatalError(c *gc.C) {
_, session := s.startMongo(c)
oplog := s.makeFakeOplog(c, session)
s.insertDoc(c, session, oplog, &mongo.OplogDoc{Timestamp: 1})
tailer := mongo.NewOplogTailer(oplog, nil, time.Time{})
defer tailer.Stop()
doc := s.getNextOplog(c, tailer)
c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
// Induce a fatal error by removing the oplog collection.
err := oplog.DropCollection()
c.Assert(err, jc.ErrorIsNil)
s.assertStopped(c, tailer)
// The actual error varies by MongoDB version so just check that
// there is one.
c.Assert(tailer.Err(), gc.Not(jc.ErrorIsNil))
}
// NewSynchronizer
// - connect
// - get optime
func NewSynchronizer(config Config) *Synchronizer {
p := new(Synchronizer)
p.config = config
if s, err := mgo.DialWithTimeout(p.config.From, time.Second*3); err == nil {
p.srcSession = s
p.srcSession.SetSocketTimeout(0)
p.srcSession.SetSyncTimeout(0)
p.srcSession.SetMode(mgo.Strong, false) // always read from primary
p.srcSession.SetCursorTimeout(0)
log.Printf("connected to %s\n", p.config.From)
} else {
log.Println(err, p.config.From)
return nil
}
if s, err := mgo.DialWithTimeout(p.config.To, time.Second*3); err == nil {
p.dstSession = s
p.dstSession.SetSocketTimeout(0)
p.dstSession.SetSyncTimeout(0)
p.dstSession.SetSafe(&mgo.Safe{W: 1})
p.dstSession.SetMode(mgo.Eventual, false)
log.Printf("connected to %s\n", p.config.To)
} else {
log.Println(err, p.config.To)
return nil
}
if p.config.StartOptime > 0 {
p.optime = bson.MongoTimestamp(int64(p.config.StartOptime) << 32)
} else {
if optime, err := utils.GetOptime(p.srcSession); err == nil {
p.optime = optime
} else {
log.Println(err)
return nil
}
}
log.Printf("optime is %v %v\n", utils.GetTimestampFromOptime(p.optime), utils.GetTimeFromOptime(p.optime))
return p
}
func TestParseBSON(t *testing.T) {
expected := []map[string]interface{}{
map[string]interface{}{"ts": bson.MongoTimestamp(6021954198109683713), "h": int64(920013897904662416), "v": 2, "op": "c", "ns": "testdb.$cmd", "o": map[string]interface{}{"create": "test"}},
map[string]interface{}{"ts": bson.MongoTimestamp(6021954253944258561), "h": int64(-7024883673281943103), "v": 2, "op": "i", "ns": "testdb.test", "o": map[string]interface{}{"_id": bson.ObjectIdHex("5392477d53a5b29c16f834f1"), "message": "insert test", "number": 1}},
map[string]interface{}{"ts": bson.MongoTimestamp(6021954314073800705), "h": int64(8562537077519333892), "v": 2, "op": "i", "ns": "testdb.test", "o": map[string]interface{}{"_id": bson.ObjectIdHex("5392478b53a5b29c16f834f2"), "message": "update test", "number": 2}},
map[string]interface{}{"ts": bson.MongoTimestamp(6021954326958702593), "h": int64(4976203120731500765), "v": 2, "op": "i", "ns": "testdb.test", "o": map[string]interface{}{"_id": bson.ObjectIdHex("5392479553a5b29c16f834f3"), "message": "delete test", "number": 3}},
map[string]interface{}{"ts": bson.MongoTimestamp(6021954408563081217), "h": int64(5650666146636305048), "v": 2, "op": "u", "ns": "testdb.test", "o2": map[string]interface{}{"_id": bson.ObjectIdHex("5392478b53a5b29c16f834f2")}, "o": map[string]interface{}{"_id": bson.ObjectIdHex("5392478b53a5b29c16f834f2"), "message": "update test", "number": 5}},
map[string]interface{}{"ts": bson.MongoTimestamp(6021954451512754177), "h": int64(-4953188477403348903), "v": 2, "op": "d", "ns": "testdb.test", "b": true, "o": map[string]interface{}{"_id": bson.ObjectIdHex("5392479553a5b29c16f834f3")}},
}
f, err := os.Open("./testdata.bson")
if err != nil {
t.Fatal("Got error", err)
}
defer f.Close()
nextOpIndex := 0
scanner := New(f)
for scanner.Scan() {
op := map[string]interface{}{}
if err := bson.Unmarshal(scanner.Bytes(), &op); err != nil {
t.Fatal("Got error in unmarshalling: ", err)
}
if !reflect.DeepEqual(op, expected[nextOpIndex]) {
t.Fatal("Op did not match expected!")
}
nextOpIndex++
}
if scanner.Err() != nil {
t.Fatal("Scanner error", scanner.Err())
}
if nextOpIndex != 6 {
t.Fatal("Did not see all ops!", nextOpIndex)
}
}
func newMongoTimestamp(s, i int) bson.MongoTimestamp {
return bson.MongoTimestamp(int64(s)<<32 + int64(i))
}
func nowAsMongoTimestamp() bson.MongoTimestamp {
return bson.MongoTimestamp(time.Now().Unix() << 32)
}
{bson.M{"_": time.Unix(0, 258e6)}, // Note the NS <=> MS conversion.
"\x09_\x00\x02\x01\x00\x00\x00\x00\x00\x00"},
{bson.M{"_": nil},
"\x0A_\x00"},
{bson.M{"_": bson.RegEx{"ab", "cd"}},
"\x0B_\x00ab\x00cd\x00"},
{bson.M{"_": bson.JavaScript{"code", nil}},
"\x0D_\x00\x05\x00\x00\x00code\x00"},
{bson.M{"_": bson.Symbol("sym")},
"\x0E_\x00\x04\x00\x00\x00sym\x00"},
{bson.M{"_": bson.JavaScript{"code", bson.M{"": nil}}},
"\x0F_\x00\x14\x00\x00\x00\x05\x00\x00\x00code\x00" +
"\x07\x00\x00\x00\x0A\x00\x00"},
{bson.M{"_": 258},
"\x10_\x00\x02\x01\x00\x00"},
{bson.M{"_": bson.MongoTimestamp(258)},
"\x11_\x00\x02\x01\x00\x00\x00\x00\x00\x00"},
{bson.M{"_": int64(258)},
"\x12_\x00\x02\x01\x00\x00\x00\x00\x00\x00"},
{bson.M{"_": int64(258 << 32)},
"\x12_\x00\x00\x00\x00\x00\x02\x01\x00\x00"},
{bson.M{"_": bson.MaxKey},
"\x7F_\x00"},
{bson.M{"_": bson.MinKey},
"\xFF_\x00"},
}
func (s *S) TestMarshalAllItems(c *C) {
for i, item := range allItems {
data, err := bson.Marshal(item.obj)
c.Assert(err, IsNil)
// ConvertJSONValueToBSON walks through a document or an array and
// replaces any extended JSON value with its corresponding BSON type.
func ConvertJSONValueToBSON(x interface{}) (interface{}, error) {
switch v := x.(type) {
case nil:
return nil, nil
case bool:
return v, nil
case map[string]interface{}: // document
for key, jsonValue := range v {
bsonValue, err := ParseJSONValue(jsonValue)
if err != nil {
return nil, err
}
v[key] = bsonValue
}
return v, nil
case bson.D:
for i := range v {
var err error
v[i].Value, err = ParseJSONValue(v[i].Value)
if err != nil {
return nil, err
}
}
return v, nil
case []interface{}: // array
for i, jsonValue := range v {
bsonValue, err := ParseJSONValue(jsonValue)
if err != nil {
return nil, err
}
v[i] = bsonValue
}
return v, nil
case string, float64, int32, int64:
return v, nil // require no conversion
case json.ObjectId: // ObjectId
s := string(v)
if !bson.IsObjectIdHex(s) {
return nil, errors.New("expected ObjectId to contain 24 hexadecimal characters")
}
return bson.ObjectIdHex(s), nil
case json.Decimal128:
return v.Decimal128, nil
case json.Date: // Date
n := int64(v)
return time.Unix(n/1e3, n%1e3*1e6), nil
case json.ISODate: // ISODate
n := string(v)
return util.FormatDate(n)
case json.NumberLong: // NumberLong
return int64(v), nil
case json.NumberInt: // NumberInt
return int32(v), nil
case json.NumberFloat: // NumberFloat
return float64(v), nil
case json.BinData: // BinData
data, err := base64.StdEncoding.DecodeString(v.Base64)
if err != nil {
return nil, err
}
return bson.Binary{v.Type, data}, nil
case json.DBRef: // DBRef
var err error
v.Id, err = ParseJSONValue(v.Id)
if err != nil {
return nil, err
}
return mgo.DBRef{v.Collection, v.Id, v.Database}, nil
case json.DBPointer: // DBPointer, for backwards compatibility
return bson.DBPointer{v.Namespace, v.Id}, nil
case json.RegExp: // RegExp
return bson.RegEx{v.Pattern, v.Options}, nil
case json.Timestamp: // Timestamp
ts := (int64(v.Seconds) << 32) | int64(v.Increment)
return bson.MongoTimestamp(ts), nil
case json.JavaScript: // Javascript
return bson.JavaScript{v.Code, v.Scope}, nil
case json.MinKey: // MinKey
return bson.MinKey, nil
case json.MaxKey: // MaxKey
return bson.MaxKey, nil
case json.Undefined: // undefined
return bson.Undefined, nil
default:
return nil, fmt.Errorf("conversion of JSON value '%v' of type '%T' not supported", v, v)
}
}
func (s *oplogSuite) TestNewMongoTimestampBeforeUnixEpoch(c *gc.C) {
c.Assert(mongo.NewMongoTimestamp(time.Time{}), gc.Equals, bson.MongoTimestamp(0))
}