// RestoreOplog attempts to restore a MongoDB oplog. func (restore *MongoRestore) RestoreOplog() error { log.Logv(log.Always, "replaying oplog") intent := restore.manager.Oplog() if intent == nil { // this should not be reached log.Logv(log.Always, "no oplog file provided, skipping oplog application") return nil } if err := intent.BSONFile.Open(); err != nil { return err } defer intent.BSONFile.Close() // NewBufferlessBSONSource reads each bson document into its own buffer // because bson.Unmarshal currently can't unmarshal binary types without // them referencing the source buffer bsonSource := db.NewDecodedBSONSource(db.NewBufferlessBSONSource(intent.BSONFile)) defer bsonSource.Close() rawOplogEntry := &bson.Raw{} var totalOps int64 var entrySize int oplogProgressor := progress.NewCounter(intent.BSONSize) if restore.ProgressManager != nil { restore.ProgressManager.Attach("oplog", oplogProgressor) defer restore.ProgressManager.Detach("oplog") } session, err := restore.SessionProvider.GetSession() if err != nil { return fmt.Errorf("error establishing connection: %v", err) } defer session.Close() for bsonSource.Next(rawOplogEntry) { entrySize = len(rawOplogEntry.Data) entryAsOplog := db.Oplog{} err = bson.Unmarshal(rawOplogEntry.Data, &entryAsOplog) if err != nil { return fmt.Errorf("error reading oplog: %v", err) } if entryAsOplog.Operation == "n" { //skip no-ops continue } if !restore.TimestampBeforeLimit(entryAsOplog.Timestamp) { log.Logvf( log.DebugLow, "timestamp %v is not below limit of %v; ending oplog restoration", entryAsOplog.Timestamp, restore.oplogLimit, ) break } totalOps++ oplogProgressor.Inc(int64(entrySize)) err = restore.ApplyOps(session, []interface{}{entryAsOplog}) if err != nil { return fmt.Errorf("error applying oplog: %v", err) } } log.Logvf(log.Info, "applied %v ops", totalOps) return nil }
// RestoreOplog attempts to restore a MongoDB oplog. func (restore *MongoRestore) RestoreOplog() error { log.Log(log.Always, "replaying oplog") intent := restore.manager.Oplog() if intent == nil { // this should not be reached log.Log(log.Always, "no oplog file provided, skipping oplog application") return nil } if err := intent.BSONFile.Open(); err != nil { return err } defer intent.BSONFile.Close() // NewBufferlessBSONSource reads each bson document into its own buffer // because bson.Unmarshal currently can't unmarshal binary types without // them referencing the source buffer bsonSource := db.NewDecodedBSONSource(db.NewBufferlessBSONSource(intent.BSONFile)) defer bsonSource.Close() entryArray := make([]interface{}, 0, 1024) rawOplogEntry := &bson.Raw{} var totalOps int64 var entrySize, bufferedBytes int oplogProgressor := progress.NewCounter(intent.BSONSize) bar := progress.Bar{ Name: "oplog", Watching: oplogProgressor, WaitTime: 3 * time.Second, Writer: log.Writer(0), BarLength: progressBarLength, IsBytes: true, } bar.Start() defer bar.Stop() session, err := restore.SessionProvider.GetSession() if err != nil { return fmt.Errorf("error establishing connection: %v", err) } defer session.Close() // To restore the oplog, we iterate over the oplog entries, // filling up a buffer. Once the buffer reaches max document size, // apply the current buffered ops and reset the buffer. for bsonSource.Next(rawOplogEntry) { entrySize = len(rawOplogEntry.Data) if bufferedBytes+entrySize > oplogMaxCommandSize { err = restore.ApplyOps(session, entryArray) if err != nil { return fmt.Errorf("error applying oplog: %v", err) } entryArray = make([]interface{}, 0, 1024) bufferedBytes = 0 } entryAsOplog := db.Oplog{} err = bson.Unmarshal(rawOplogEntry.Data, &entryAsOplog) if err != nil { return fmt.Errorf("error reading oplog: %v", err) } if entryAsOplog.Operation == "n" { //skip no-ops continue } if !restore.TimestampBeforeLimit(entryAsOplog.Timestamp) { log.Logf( log.DebugLow, "timestamp %v is not below limit of %v; ending oplog restoration", entryAsOplog.Timestamp, restore.oplogLimit, ) break } totalOps++ bufferedBytes += entrySize oplogProgressor.Inc(int64(entrySize)) entryArray = append(entryArray, entryAsOplog) } // finally, flush the remaining entries if len(entryArray) > 0 { err = restore.ApplyOps(session, entryArray) if err != nil { return fmt.Errorf("error applying oplog: %v", err) } } log.Logf(log.Info, "applied %v ops", totalOps) return nil }