Example #1
0
func (s *oplogSuite) TestRestartsOnError(c *gc.C) {
	_, session := s.startMongo(c)

	oplog := s.makeFakeOplog(c, session)
	tailer := mongo.NewOplogTailer(oplog, nil, time.Time{})
	defer tailer.Stop()

	// First, ensure that the tailer is seeing oplog inserts.
	s.insertDoc(c, session, oplog, &mongo.OplogDoc{
		Timestamp:   1,
		OperationId: 99,
	})
	doc := s.getNextOplog(c, tailer)
	c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))

	s.emptyCapped(c, oplog)

	// Ensure that the tailer still works.
	s.insertDoc(c, session, oplog, &mongo.OplogDoc{
		Timestamp:   2,
		OperationId: 42,
	})
	doc = s.getNextOplog(c, tailer)
	c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(2))
}
Example #2
0
func TestTimestamp(t *testing.T) {
	data := []struct {
		in     M
		want   bson.MongoTimestamp
		wantok bool
	}{
		{
			map[string]interface{}{"$timestamp": map[string]interface{}{"t": 1392822881, "i": 1}},
			bson.MongoTimestamp(5982128723015499777),
			true,
		},
		{
			map[string]interface{}{"$ugh": map[string]interface{}{"t": 1392822881, "i": 1}},
			bson.MongoTimestamp(5982128723015499777),
			false,
		},
	}

	for _, d := range data {
		b, ok := d.in.timestamp()
		if ok != d.wantok {
			t.Errorf("got %t, want %t, (%v)", ok, d.wantok, d.in)
			t.FailNow()
		}
		if ok && b != d.want {
			t.Errorf("wanted: %v (%T), got: %v (%T)", d.want, d.want, b, b)
		}
	}
}
Example #3
0
func TestValidOplogLimitChecking(t *testing.T) {

	testutil.VerifyTestType(t, testutil.UnitTestType)

	Convey("With a MongoRestore instance with oplogLimit of 5:0", t, func() {
		mr := &MongoRestore{
			oplogLimit: bson.MongoTimestamp(int64(5) << 32),
		}

		Convey("an oplog entry with ts=1000:0 should be invalid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(1000)<<32)), ShouldBeFalse)
		})

		Convey("an oplog entry with ts=5:1 should be invalid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(5)<<32|1)), ShouldBeFalse)
		})

		Convey("an oplog entry with ts=5:0 should be invalid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(5)<<32)), ShouldBeFalse)
		})

		Convey("an oplog entry with ts=4:9 should be valid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(4)<<32|9)), ShouldBeTrue)
		})

		Convey("an oplog entry with ts=4:0 should be valid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(4)<<32)), ShouldBeTrue)
		})

		Convey("an oplog entry with ts=0:1 should be valid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(1)), ShouldBeTrue)
		})
	})

	Convey("With a MongoRestore instance with no oplogLimit", t, func() {
		mr := &MongoRestore{}

		Convey("an oplog entry with ts=1000:0 should be valid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(1000)<<32)), ShouldBeTrue)
		})

		Convey("an oplog entry with ts=5:1 should be valid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(5)<<32|1)), ShouldBeTrue)
		})

		Convey("an oplog entry with ts=5:0 should be valid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(5)<<32)), ShouldBeTrue)
		})
	})

}
Example #4
0
func OplogDeamon() {

	url := fmt.Sprintf("mongodb://%s:%s", Conf.MongoIP, Conf.MongoPort)
	session, err := mgo.Dial(url)
	if err != nil {
		panic(err)
	}
	defer session.Close()

	c := session.DB("local").C("oplog.rs")

	cp := initCheckpoint()

	log.Info("init check point is %d ", cp)

	iter := c.Find(bson.M{"ts": bson.M{"$gte": bson.MongoTimestamp(cp)}}).Tail(-1 * time.Second)
	defer iter.Close()

	count := uint64(0)
	for {

		log.Info("start oplog manarger !!! ")

		for result := new(Oplog); iter.Next(&result); result = new(Oplog) {

			count++
			checkpoint_threshold(count, result)

			if !checkTopicCollection(result.Ns) {
				log.Info("the namespace(%s) not in topics: %s", result.Ns, strings.Join(Conf.TopicCollections, "|"))
				continue
			}

			// new event
			newEvent(result)

		}

		if iter.Err() != nil {
			iter.Close()
		}
		if iter.Timeout() {
			continue
		}

		// reload checkp point from file
		cp = initCheckpoint()
		iter = c.Find(bson.M{"ts": bson.M{"$gte": bson.MongoTimestamp(cp)}}).Tail(-1 * time.Second)
	}

}
Example #5
0
func (s *oplogSuite) TestNoRepeatsAfterIterRestart(c *gc.C) {
	// A bunch of documents with the same timestamp but different ids.
	// These will be split across 2 iterators.
	docs := make([]*mongo.OplogDoc, 11)
	for i := 0; i < 10; i++ {
		id := int64(i + 10)
		docs[i] = &mongo.OplogDoc{
			Timestamp:   1,
			OperationId: id,
		}
	}
	// Add one more with a different timestamp.
	docs[10] = &mongo.OplogDoc{
		Timestamp:   2,
		OperationId: 42,
	}
	session := newFakeSession(
		// First block of documents, all time 1
		newFakeIterator(nil, docs[:5]...),
		// Second block, some time 1, one time 2
		newFakeIterator(nil, docs[5:]...),
	)
	tailer := mongo.NewOplogTailer(session, time.Time{})
	defer tailer.Stop()

	for id := int64(10); id < 15; id++ {
		doc := s.getNextOplog(c, tailer)
		c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
		c.Assert(doc.OperationId, gc.Equals, id)
	}

	// Check the query doesn't exclude any in the first request.
	session.checkLastArgs(c, mongo.NewMongoTimestamp(time.Time{}), nil)

	// The OplogTailer will fall off the end of the iterator and get a new one.

	// Ensure that only previously unreported entries are now reported.
	for id := int64(15); id < 20; id++ {
		doc := s.getNextOplog(c, tailer)
		c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
		c.Assert(doc.OperationId, gc.Equals, id)
	}

	// Check we got the next block correctly
	session.checkLastArgs(c, bson.MongoTimestamp(1), []int64{10, 11, 12, 13, 14})

	doc := s.getNextOplog(c, tailer)
	c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(2))
	c.Assert(doc.OperationId, gc.Equals, int64(42))
}
Example #6
0
func (s *oplogSuite) TestNoRepeatsAfterIterRestart(c *gc.C) {
	_, session := s.startMongo(c)

	oplog := s.makeFakeOplog(c, session)
	tailer := mongo.NewOplogTailer(oplog, nil, time.Time{})
	defer tailer.Stop()

	// Insert a bunch of oplog entries with the same timestamp (but
	// with different ids) and see them reported.
	for id := int64(10); id < 15; id++ {
		s.insertDoc(c, session, oplog, &mongo.OplogDoc{
			Timestamp:   1,
			OperationId: id,
		})

		doc := s.getNextOplog(c, tailer)
		c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
		c.Assert(doc.OperationId, gc.Equals, id)
	}

	// Force the OplogTailer's iterator to be recreated.
	s.emptyCapped(c, oplog)

	// Reinsert the oplog entries that were already there before and a
	// few more.
	for id := int64(10); id < 20; id++ {
		s.insertDoc(c, session, oplog, &mongo.OplogDoc{
			Timestamp:   1,
			OperationId: id,
		})
	}

	// Insert an entry for a later timestamp.
	s.insertDoc(c, session, oplog, &mongo.OplogDoc{
		Timestamp:   2,
		OperationId: 42,
	})

	// Ensure that only previously unreported entries are now reported.
	for id := int64(15); id < 20; id++ {
		doc := s.getNextOplog(c, tailer)
		c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
		c.Assert(doc.OperationId, gc.Equals, id)
	}

	doc := s.getNextOplog(c, tailer)
	c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(2))
	c.Assert(doc.OperationId, gc.Equals, int64(42))
}
Example #7
0
// ParseTimestampFlag takes in a string the form of <time_t>:<ordinal>,
// where <time_t> is the seconds since the UNIX epoch, and <ordinal> represents
// a counter of operations in the oplog that occurred in the specified second.
// It parses this timestamp string and returns a bson.MongoTimestamp type.
func ParseTimestampFlag(ts string) (bson.MongoTimestamp, error) {
	var seconds, increment int
	timestampFields := strings.Split(ts, ":")
	if len(timestampFields) > 2 {
		return 0, fmt.Errorf("too many : characters")
	}

	seconds, err := strconv.Atoi(timestampFields[0])
	if err != nil {
		return 0, fmt.Errorf("error parsing timestamp seconds: %v", err)
	}

	// parse the increment field if it exists
	if len(timestampFields) == 2 {
		if len(timestampFields[1]) > 0 {
			increment, err = strconv.Atoi(timestampFields[1])
			if err != nil {
				return 0, fmt.Errorf("error parsing timestamp increment: %v", err)
			}
		} else {
			// handle the case where the user writes "<time_t>:" with no ordinal
			increment = 0
		}
	}

	timestamp := (int64(seconds) << 32) | int64(increment)
	return bson.MongoTimestamp(timestamp), nil
}
Example #8
0
// WriteLogWithOplog writes out a log record to the a (probably fake)
// oplog collection and the logs collection.
func WriteLogWithOplog(
	oplog *mgo.Collection,
	envUUID string,
	entity names.Tag,
	t time.Time,
	module string,
	location string,
	level loggo.Level,
	msg string,
) error {
	doc := &logDoc{
		Id:       bson.NewObjectId(),
		Time:     t,
		EnvUUID:  envUUID,
		Entity:   entity.String(),
		Module:   module,
		Location: location,
		Level:    level,
		Message:  msg,
	}
	err := oplog.Insert(bson.D{
		{"ts", bson.MongoTimestamp(time.Now().Unix() << 32)}, // an approximation which will do
		{"h", rand.Int63()},                                  // again, a suitable fake
		{"op", "i"},                                          // this will always be an insert
		{"ns", "logs.logs"},
		{"o", doc},
	})
	if err != nil {
		return err
	}

	session := oplog.Database.Session
	logs := session.DB("logs").C("logs")
	return logs.Insert(doc)
}
Example #9
0
func (s *oplogSuite) TestNewMongoTimestamp(c *gc.C) {
	t := time.Date(2015, 6, 24, 12, 47, 0, 0, time.FixedZone("somewhere", 5*3600))

	expected := bson.MongoTimestamp(6163845091342417920)
	c.Assert(mongo.NewMongoTimestamp(t), gc.Equals, expected)
	c.Assert(mongo.NewMongoTimestamp(t.In(time.UTC)), gc.Equals, expected)
}
Example #10
0
func TailOps(session *mgo.Session, channel OpChan, errChan chan error, options *Options) error {
	s := session.Copy()
	defer s.Close()
	FillEmptyOptions(session, options)
	duration, err := time.ParseDuration(*options.CursorTimeout)
	if err != nil {
		panic("Invalid value for CursorTimeout")
	}
	currTimestamp := options.After(s, options)
	iter := GetOpLogQuery(s, currTimestamp, options).Tail(duration)
	for {
		entry := make(OpLogEntry)
		for iter.Next(entry) {
			op := &Op{"", "", "", nil, bson.MongoTimestamp(0)}
			op.ParseLogEntry(entry)
			if op.Id != "" {
				if options.Filter == nil || options.Filter(op) {
					channel <- op
				}
			}
			currTimestamp = op.Timestamp
		}
		if err = iter.Close(); err != nil {
			errChan <- err
			return err
		}
		if iter.Timeout() {
			continue
		}
		iter = GetOpLogQuery(s, currTimestamp, options).Tail(duration)
	}
	return nil
}
Example #11
0
func TestParseLargeBSON(t *testing.T) {
	largeArray := make([]interface{}, 5000)
	for i := 0; i < 5000; i++ {
		largeArray[i] = float64(i)
	}
	expectedOp := map[string]interface{}{
		"ts": bson.MongoTimestamp(6048257058866724865), "h": int64(-6825742652110581687), "v": 2, "op": "i", "ns": "testdb.testdb", "o": map[string]interface{}{
			"_id": bson.ObjectIdHex("53efb9c067fd92348e823860"),
			"val": largeArray}}

	f, err := os.Open("./largetestdata.bson")
	if err != nil {
		t.Fatal("Error loading file", err)
	}
	defer f.Close()
	foundExpectedOp := false
	scanner := New(f)
	for scanner.Scan() {
		op := map[string]interface{}{}
		if err := bson.Unmarshal(scanner.Bytes(), &op); err != nil {
			t.Fatal("Error unmarshalling: ", err)
		}
		if reflect.DeepEqual(op, expectedOp) {
			foundExpectedOp = true
		}
	}
	if scanner.Err() != nil {
		t.Fatal("Scanner error: ", scanner.Err())
	}
	if !foundExpectedOp {
		t.Fatal("Didn't find the expected operation")
	}

}
Example #12
0
// NewMongoTimestamp returns a bson.MongoTimestamp repesentation for
// the time.Time given. Note that these timestamps are not the same
// the usual MongoDB time fields. These are an internal format used
// only in a few places such as the replication oplog.
//
// See: http://docs.mongodb.org/manual/reference/bson-types/#timestamps
func NewMongoTimestamp(t time.Time) bson.MongoTimestamp {
	unixTime := t.Unix()
	if unixTime < 0 {
		unixTime = 0
	}
	return bson.MongoTimestamp(unixTime << 32)
}
Example #13
0
// get the cursor for the oplog collection, based on the options
// passed in to mongooplog
func buildTailingCursor(oplog *mgo.Collection,
	sourceOptions *SourceOptions) *mgo.Iter {

	// how many seconds in the past we need
	secondsInPast := time.Duration(sourceOptions.Seconds) * time.Second
	// the time threshold for oplog queries
	threshold := time.Now().Add(-secondsInPast)
	// convert to a unix timestamp (seconds since epoch)
	thresholdAsUnix := threshold.Unix()

	// shift it appropriately, to prepare it to be converted to an
	// oplog timestamp
	thresholdShifted := uint64(thresholdAsUnix) << 32

	// build the oplog query
	oplogQuery := bson.M{
		"ts": bson.M{
			"$gte": bson.MongoTimestamp(thresholdShifted),
		},
	}

	// TODO: wait time
	return oplog.Find(oplogQuery).Iter()

}
Example #14
0
/* bsonify a mongo Timestamp */
func (m M) timestamp() (timestamp bson.MongoTimestamp, ok bool) {
	if len(m) != 1 {
		return
	}

	if value, contains := m["$timestamp"]; contains {
		if ts, ismap := value.(map[string]interface{}); ismap {
			t, isok := ts["t"]
			if !isok {
				return
			}
			tt, isok := t.(int)
			if !isok {
				return
			}

			i, isok := ts["i"]
			if !isok {
				return
			}
			ii, isok := i.(int)
			if !isok {
				return
			}

			ok = true
			var concat int64
			concat = int64(uint64(tt)<<32 | uint64(ii))
			timestamp = bson.MongoTimestamp(concat)
		}
	}

	return
}
Example #15
0
func TestTimestampValue(t *testing.T) {

	Convey("When converting JSON with Timestamp values", t, func() {
		testTS := bson.MongoTimestamp(123456<<32 | 55)

		Convey("works for Timestamp literal", func() {

			jsonMap := map[string]interface{}{
				"ts": json.Timestamp{123456, 55},
			}

			err := ConvertJSONDocumentToBSON(jsonMap)
			So(err, ShouldBeNil)
			So(jsonMap["ts"], ShouldEqual, testTS)
		})

		Convey(`works for Timestamp document`, func() {
			Convey(`{"ts":{"$timestamp":{"t":123456, "i":55}}}`, func() {
				jsonMap := map[string]interface{}{
					"ts": map[string]interface{}{
						"$timestamp": map[string]interface{}{
							"t": 123456.0,
							"i": 55.0,
						},
					},
				}

				bsonMap, err := ConvertJSONValueToBSON(jsonMap)
				So(err, ShouldBeNil)
				So(bsonMap.(map[string]interface{})["ts"], ShouldEqual, testTS)
			})
		})
	})
}
Example #16
0
//mongoTimestamp returns a valid mongoTimestamp
func (m mongoTimestamp) MongoTimestamp() bson.MongoTimestamp {
	var b [12]byte
	var result uint64
	binary.BigEndian.PutUint32(b[:4], uint32(m.Unix()))
	binary.Read(bytes.NewReader(b[:]), binary.BigEndian, &result)

	return bson.MongoTimestamp(result)
}
Example #17
0
// writeLogToOplog writes out a log record to the a (probably fake)
// oplog collection.
func (s *LogTailerSuite) writeLogToOplog(doc interface{}) error {
	return s.oplogColl.Insert(bson.D{
		{"ts", bson.MongoTimestamp(time.Now().Unix() << 32)}, // an approximation which will do
		{"h", rand.Int63()},                                  // again, a suitable fake
		{"op", "i"},                                          // this will always be an insert
		{"ns", "logs.logs"},
		{"o", doc},
	})
}
Example #18
0
func TestInsert(t *testing.T) {
	if err := omgo.RunMgFun(mg_instance, mg_collection, func(c *mgo.Collection) error {
		return c.Insert(&DongoData{
			Id:   bson.NewObjectId(),
			Name: "yoer",
			Date: bson.MongoTimestamp(time.Now().UnixNano()),
		})
	}); nil != err {
		t.Error(err.Error())
	}
}
Example #19
0
func (s *oplogSuite) TestRestartsOnErrCursor(c *gc.C) {
	session := newFakeSession(
		// First iterator terminates with an ErrCursor
		newFakeIterator(mgo.ErrCursor, &mongo.OplogDoc{Timestamp: 1, OperationId: 99}),
		newFakeIterator(nil, &mongo.OplogDoc{Timestamp: 2, OperationId: 42}),
	)
	tailer := mongo.NewOplogTailer(session, time.Time{})
	defer tailer.Stop()

	// First, ensure that the tailer is seeing oplog rows and handles
	// the ErrCursor that occurs at the end.
	doc := s.getNextOplog(c, tailer)
	c.Check(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
	session.checkLastArgs(c, mongo.NewMongoTimestamp(time.Time{}), nil)

	// Ensure that the tailer continues after getting a new iterator.
	doc = s.getNextOplog(c, tailer)
	c.Check(doc.Timestamp, gc.Equals, bson.MongoTimestamp(2))
	session.checkLastArgs(c, bson.MongoTimestamp(1), []int64{99})
}
Example #20
0
func TestMarshalTimestamp(t *testing.T) {
	data := []struct {
		in   bson.MongoTimestamp
		want []byte
	}{
		{bson.MongoTimestamp(0), []byte("{\"$timestamp\":{\"i\":0,\"t\":0}}")},
		{bson.MongoTimestamp(5982128723015499777), []byte("{\"$timestamp\":{\"i\":1,\"t\":1392822881}}")},
	}

	for _, d := range data {
		mejson := marshalTimestamp(d.in)
		b, err := json.Marshal(mejson)

		if err != nil {
			t.FailNow()
		}
		if !reflect.DeepEqual(b, d.want) {
			t.Errorf("wanted: %s, got: %s", d.want, b)
		}
	}
}
Example #21
0
func (s *oplogSuite) TestDiesOnFatalError(c *gc.C) {
	expectedErr := errors.New("oh no, the collection went away!")
	session := newFakeSession(
		newFakeIterator(expectedErr, &mongo.OplogDoc{Timestamp: 1}),
	)

	tailer := mongo.NewOplogTailer(session, time.Time{})
	defer tailer.Stop()

	doc := s.getNextOplog(c, tailer)
	c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
	s.assertStopped(c, tailer)
	c.Assert(tailer.Err(), gc.Equals, expectedErr)
}
Example #22
0
func TestTimestampBSONToJSON(t *testing.T) {
	testutil.VerifyTestType(t, testutil.UnitTestType)

	Convey("Converting a BSON Timestamp to JSON", t, func() {
		Convey("should produce a json.Timestamp", func() {
			// {t:803434343, i:9} == bson.MongoTimestamp(803434343*2**32 + 9)
			_jObj, err := ConvertBSONValueToJSON(bson.MongoTimestamp(uint64(803434343<<32) | uint64(9)))
			So(err, ShouldBeNil)
			jObj, ok := _jObj.(json.Timestamp)
			So(ok, ShouldBeTrue)

			So(jObj, ShouldResemble, json.Timestamp{Seconds: 803434343, Increment: 9})
			So(jObj, ShouldNotResemble, json.Timestamp{Seconds: 803434343, Increment: 8})
		})
	})
}
Example #23
0
func (s *oplogSuite) TestDiesOnFatalError(c *gc.C) {
	_, session := s.startMongo(c)
	oplog := s.makeFakeOplog(c, session)
	s.insertDoc(c, session, oplog, &mongo.OplogDoc{Timestamp: 1})

	tailer := mongo.NewOplogTailer(oplog, nil, time.Time{})
	defer tailer.Stop()

	doc := s.getNextOplog(c, tailer)
	c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))

	// Induce a fatal error by removing the oplog collection.
	err := oplog.DropCollection()
	c.Assert(err, jc.ErrorIsNil)

	s.assertStopped(c, tailer)
	// The actual error varies by MongoDB version so just check that
	// there is one.
	c.Assert(tailer.Err(), gc.Not(jc.ErrorIsNil))
}
Example #24
0
// NewSynchronizer
//   - connect
//   - get optime
func NewSynchronizer(config Config) *Synchronizer {
	p := new(Synchronizer)
	p.config = config
	if s, err := mgo.DialWithTimeout(p.config.From, time.Second*3); err == nil {
		p.srcSession = s
		p.srcSession.SetSocketTimeout(0)
		p.srcSession.SetSyncTimeout(0)
		p.srcSession.SetMode(mgo.Strong, false) // always read from primary
		p.srcSession.SetCursorTimeout(0)
		log.Printf("connected to %s\n", p.config.From)
	} else {
		log.Println(err, p.config.From)
		return nil
	}
	if s, err := mgo.DialWithTimeout(p.config.To, time.Second*3); err == nil {
		p.dstSession = s
		p.dstSession.SetSocketTimeout(0)
		p.dstSession.SetSyncTimeout(0)
		p.dstSession.SetSafe(&mgo.Safe{W: 1})
		p.dstSession.SetMode(mgo.Eventual, false)
		log.Printf("connected to %s\n", p.config.To)
	} else {
		log.Println(err, p.config.To)
		return nil
	}
	if p.config.StartOptime > 0 {
		p.optime = bson.MongoTimestamp(int64(p.config.StartOptime) << 32)
	} else {
		if optime, err := utils.GetOptime(p.srcSession); err == nil {
			p.optime = optime
		} else {
			log.Println(err)
			return nil
		}
	}
	log.Printf("optime is %v %v\n", utils.GetTimestampFromOptime(p.optime), utils.GetTimeFromOptime(p.optime))
	return p
}
Example #25
0
func TestParseBSON(t *testing.T) {
	expected := []map[string]interface{}{
		map[string]interface{}{"ts": bson.MongoTimestamp(6021954198109683713), "h": int64(920013897904662416), "v": 2, "op": "c", "ns": "testdb.$cmd", "o": map[string]interface{}{"create": "test"}},
		map[string]interface{}{"ts": bson.MongoTimestamp(6021954253944258561), "h": int64(-7024883673281943103), "v": 2, "op": "i", "ns": "testdb.test", "o": map[string]interface{}{"_id": bson.ObjectIdHex("5392477d53a5b29c16f834f1"), "message": "insert test", "number": 1}},
		map[string]interface{}{"ts": bson.MongoTimestamp(6021954314073800705), "h": int64(8562537077519333892), "v": 2, "op": "i", "ns": "testdb.test", "o": map[string]interface{}{"_id": bson.ObjectIdHex("5392478b53a5b29c16f834f2"), "message": "update test", "number": 2}},
		map[string]interface{}{"ts": bson.MongoTimestamp(6021954326958702593), "h": int64(4976203120731500765), "v": 2, "op": "i", "ns": "testdb.test", "o": map[string]interface{}{"_id": bson.ObjectIdHex("5392479553a5b29c16f834f3"), "message": "delete test", "number": 3}},
		map[string]interface{}{"ts": bson.MongoTimestamp(6021954408563081217), "h": int64(5650666146636305048), "v": 2, "op": "u", "ns": "testdb.test", "o2": map[string]interface{}{"_id": bson.ObjectIdHex("5392478b53a5b29c16f834f2")}, "o": map[string]interface{}{"_id": bson.ObjectIdHex("5392478b53a5b29c16f834f2"), "message": "update test", "number": 5}},
		map[string]interface{}{"ts": bson.MongoTimestamp(6021954451512754177), "h": int64(-4953188477403348903), "v": 2, "op": "d", "ns": "testdb.test", "b": true, "o": map[string]interface{}{"_id": bson.ObjectIdHex("5392479553a5b29c16f834f3")}},
	}

	f, err := os.Open("./testdata.bson")
	if err != nil {
		t.Fatal("Got error", err)
	}
	defer f.Close()

	nextOpIndex := 0
	scanner := New(f)
	for scanner.Scan() {
		op := map[string]interface{}{}
		if err := bson.Unmarshal(scanner.Bytes(), &op); err != nil {
			t.Fatal("Got error in unmarshalling: ", err)
		}

		if !reflect.DeepEqual(op, expected[nextOpIndex]) {
			t.Fatal("Op did not match expected!")
		}
		nextOpIndex++
	}
	if scanner.Err() != nil {
		t.Fatal("Scanner error", scanner.Err())
	}

	if nextOpIndex != 6 {
		t.Fatal("Did not see all ops!", nextOpIndex)
	}
}
Example #26
0
func newMongoTimestamp(s, i int) bson.MongoTimestamp {
	return bson.MongoTimestamp(int64(s)<<32 + int64(i))
}
Example #27
0
func nowAsMongoTimestamp() bson.MongoTimestamp {
	return bson.MongoTimestamp(time.Now().Unix() << 32)
}
Example #28
0
	{bson.M{"_": time.Unix(0, 258e6)}, // Note the NS <=> MS conversion.
		"\x09_\x00\x02\x01\x00\x00\x00\x00\x00\x00"},
	{bson.M{"_": nil},
		"\x0A_\x00"},
	{bson.M{"_": bson.RegEx{"ab", "cd"}},
		"\x0B_\x00ab\x00cd\x00"},
	{bson.M{"_": bson.JavaScript{"code", nil}},
		"\x0D_\x00\x05\x00\x00\x00code\x00"},
	{bson.M{"_": bson.Symbol("sym")},
		"\x0E_\x00\x04\x00\x00\x00sym\x00"},
	{bson.M{"_": bson.JavaScript{"code", bson.M{"": nil}}},
		"\x0F_\x00\x14\x00\x00\x00\x05\x00\x00\x00code\x00" +
			"\x07\x00\x00\x00\x0A\x00\x00"},
	{bson.M{"_": 258},
		"\x10_\x00\x02\x01\x00\x00"},
	{bson.M{"_": bson.MongoTimestamp(258)},
		"\x11_\x00\x02\x01\x00\x00\x00\x00\x00\x00"},
	{bson.M{"_": int64(258)},
		"\x12_\x00\x02\x01\x00\x00\x00\x00\x00\x00"},
	{bson.M{"_": int64(258 << 32)},
		"\x12_\x00\x00\x00\x00\x00\x02\x01\x00\x00"},
	{bson.M{"_": bson.MaxKey},
		"\x7F_\x00"},
	{bson.M{"_": bson.MinKey},
		"\xFF_\x00"},
}

func (s *S) TestMarshalAllItems(c *C) {
	for i, item := range allItems {
		data, err := bson.Marshal(item.obj)
		c.Assert(err, IsNil)
Example #29
0
// ConvertJSONValueToBSON walks through a document or an array and
// replaces any extended JSON value with its corresponding BSON type.
func ConvertJSONValueToBSON(x interface{}) (interface{}, error) {
	switch v := x.(type) {
	case nil:
		return nil, nil
	case bool:
		return v, nil
	case map[string]interface{}: // document
		for key, jsonValue := range v {
			bsonValue, err := ParseJSONValue(jsonValue)
			if err != nil {
				return nil, err
			}
			v[key] = bsonValue
		}
		return v, nil
	case bson.D:
		for i := range v {
			var err error
			v[i].Value, err = ParseJSONValue(v[i].Value)
			if err != nil {
				return nil, err
			}
		}
		return v, nil

	case []interface{}: // array
		for i, jsonValue := range v {
			bsonValue, err := ParseJSONValue(jsonValue)
			if err != nil {
				return nil, err
			}
			v[i] = bsonValue
		}
		return v, nil

	case string, float64, int32, int64:
		return v, nil // require no conversion

	case json.ObjectId: // ObjectId
		s := string(v)
		if !bson.IsObjectIdHex(s) {
			return nil, errors.New("expected ObjectId to contain 24 hexadecimal characters")
		}
		return bson.ObjectIdHex(s), nil

	case json.Decimal128:
		return v.Decimal128, nil

	case json.Date: // Date
		n := int64(v)
		return time.Unix(n/1e3, n%1e3*1e6), nil

	case json.ISODate: // ISODate
		n := string(v)
		return util.FormatDate(n)

	case json.NumberLong: // NumberLong
		return int64(v), nil

	case json.NumberInt: // NumberInt
		return int32(v), nil

	case json.NumberFloat: // NumberFloat
		return float64(v), nil
	case json.BinData: // BinData
		data, err := base64.StdEncoding.DecodeString(v.Base64)
		if err != nil {
			return nil, err
		}
		return bson.Binary{v.Type, data}, nil

	case json.DBRef: // DBRef
		var err error
		v.Id, err = ParseJSONValue(v.Id)
		if err != nil {
			return nil, err
		}
		return mgo.DBRef{v.Collection, v.Id, v.Database}, nil

	case json.DBPointer: // DBPointer, for backwards compatibility
		return bson.DBPointer{v.Namespace, v.Id}, nil

	case json.RegExp: // RegExp
		return bson.RegEx{v.Pattern, v.Options}, nil

	case json.Timestamp: // Timestamp
		ts := (int64(v.Seconds) << 32) | int64(v.Increment)
		return bson.MongoTimestamp(ts), nil

	case json.JavaScript: // Javascript
		return bson.JavaScript{v.Code, v.Scope}, nil

	case json.MinKey: // MinKey
		return bson.MinKey, nil

	case json.MaxKey: // MaxKey
		return bson.MaxKey, nil

	case json.Undefined: // undefined
		return bson.Undefined, nil

	default:
		return nil, fmt.Errorf("conversion of JSON value '%v' of type '%T' not supported", v, v)
	}
}
Example #30
0
func (s *oplogSuite) TestNewMongoTimestampBeforeUnixEpoch(c *gc.C) {
	c.Assert(mongo.NewMongoTimestamp(time.Time{}), gc.Equals, bson.MongoTimestamp(0))
}