forked from mongodb/mongo-tools
/
mongooplog.go
148 lines (115 loc) · 3.95 KB
/
mongooplog.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
// Package mongooplog polls operations from the replication oplog of one server, and applies them to another.
package mongooplog
import (
"fmt"
"github.com/mongodb/mongo-tools/common/db"
"github.com/mongodb/mongo-tools/common/log"
"github.com/mongodb/mongo-tools/common/options"
"github.com/mongodb/mongo-tools/common/util"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
"time"
)
// MongoOplog is a container for the user-specified options for running mongooplog.
type MongoOplog struct {
// standard tool options
ToolOptions *options.ToolOptions
// mongooplog-specific options
SourceOptions *SourceOptions
// session provider for the source server
SessionProviderFrom *db.SessionProvider
// session provider for the destination server
SessionProviderTo *db.SessionProvider
}
// Run executes the mongooplog program.
func (mo *MongoOplog) Run() error {
// split up the oplog namespace we are using
oplogDB, oplogColl, err :=
util.SplitAndValidateNamespace(mo.SourceOptions.OplogNS)
if err != nil {
return err
}
// the full oplog namespace needs to be specified
if oplogColl == "" {
return fmt.Errorf("the oplog namespace must specify a collection")
}
log.Logf(log.DebugLow, "using oplog namespace `%v.%v`", oplogDB, oplogColl)
// connect to the destination server
toSession, err := mo.SessionProviderTo.GetSession()
if err != nil {
return fmt.Errorf("error connecting to destination db: %v", err)
}
defer toSession.Close()
toSession.SetSocketTimeout(0)
// purely for logging
destServerStr := mo.ToolOptions.Host
if mo.ToolOptions.Port != "" {
destServerStr = destServerStr + ":" + mo.ToolOptions.Port
}
log.Logf(log.DebugLow, "successfully connected to destination server `%v`", destServerStr)
// connect to the source server
fromSession, err := mo.SessionProviderFrom.GetSession()
if err != nil {
return fmt.Errorf("error connecting to source db: %v", err)
}
defer fromSession.Close()
fromSession.SetSocketTimeout(0)
log.Logf(log.DebugLow, "successfully connected to source server `%v`", mo.SourceOptions.From)
// set slave ok
fromSession.SetMode(mgo.Eventual, true)
// get the tailing cursor for the source server's oplog
tail := buildTailingCursor(fromSession.DB(oplogDB).C(oplogColl),
mo.SourceOptions)
defer tail.Close()
// read the cursor dry, applying ops to the destination
// server in the process
oplogEntry := &db.Oplog{}
res := &db.ApplyOpsResponse{}
log.Log(log.DebugLow, "applying oplog entries...")
for tail.Next(oplogEntry) {
// skip noops
if oplogEntry.Operation == "n" {
log.Logf(log.DebugHigh, "skipping no-op for namespace `%v`", oplogEntry.Namespace)
continue
}
// prepare the op to be applied
opsToApply := []db.Oplog{*oplogEntry}
// apply the operation
err := toSession.Run(bson.M{"applyOps": opsToApply}, res)
if err != nil {
return fmt.Errorf("error applying ops: %v", err)
}
// check the server's response for an issue
if !res.Ok {
return fmt.Errorf("server gave error applying ops: %v", res.ErrMsg)
}
}
// make sure there was no tailing error
if err := tail.Err(); err != nil {
return fmt.Errorf("error querying oplog: %v", err)
}
log.Log(log.DebugLow, "done applying oplog entries")
return nil
}
// get the cursor for the oplog collection, based on the options
// passed in to mongooplog
func buildTailingCursor(oplog *mgo.Collection,
sourceOptions *SourceOptions) *mgo.Iter {
// how many seconds in the past we need
secondsInPast := time.Duration(sourceOptions.Seconds) * time.Second
// the time threshold for oplog queries
threshold := time.Now().Add(-secondsInPast)
// convert to a unix timestamp (seconds since epoch)
thresholdAsUnix := threshold.Unix()
// shift it appropriately, to prepare it to be converted to an
// oplog timestamp
thresholdShifted := uint64(thresholdAsUnix) << 32
// build the oplog query
oplogQuery := bson.M{
"ts": bson.M{
"$gte": bson.MongoTimestamp(thresholdShifted),
},
}
// TODO: wait time
return oplog.Find(oplogQuery).Iter()
}