func waitRdbDump(r io.Reader) <-chan int64 { size := make(chan int64) go func() { var rsp string for { b := []byte{0} if _, err := r.Read(b); err != nil { log.PanicErrorf(err, "read sync response = '%s'", rsp) } if len(rsp) == 0 && b[0] == '\n' { size <- 0 continue } rsp += string(b) if strings.HasSuffix(rsp, "\r\n") { break } } if rsp[0] != '$' { log.Panicf("invalid sync response, rsp = '%s'", rsp) } n, err := strconv.Atoi(rsp[1 : len(rsp)-2]) if err != nil || n <= 0 { log.PanicErrorf(err, "invalid sync response = '%s', n = %d", rsp, n) } size <- int64(n) }() return size }
func openReadFile(name string) (*os.File, int64) { f, err := os.Open(name) if err != nil { log.PanicErrorf(err, "cannot open file-reader '%s'", name) } s, err := f.Stat() if err != nil { log.PanicErrorf(err, "cannot stat file-reader '%s'", name) } return f, s.Size() }
func openReadWriteFile(name string) *os.File { f, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) if err != nil { log.PanicErrorf(err, "cannot open file-readwriter '%s'", name) } return f }
func openWriteFile(name string) *os.File { f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) if err != nil { log.PanicErrorf(err, "cannot open file-writer '%s'", name) } return f }
func openNetConn(target, passwd string) net.Conn { c, err := net.Dial("tcp", target) if err != nil { log.PanicErrorf(err, "cannot connect to '%s'", target) } authPassword(c, passwd) return c }
func (cmd *cmdSync) SendPSyncCmd(master, passwd string) (pipe.Reader, int64) { c := openNetConn(master, passwd) br := bufio.NewReaderSize(c, ReaderBufferSize) bw := bufio.NewWriterSize(c, WriterBufferSize) runid, offset, wait := sendPSyncFullsync(br, bw) log.Infof("psync runid = %s offset = %d, fullsync", runid, offset) var nsize int64 for nsize == 0 { select { case nsize = <-wait: if nsize == 0 { log.Info("+") } case <-time.After(time.Second): log.Info("-") } } piper, pipew := pipe.NewSize(ReaderBufferSize) go func() { defer pipew.Close() p := make([]byte, 8192) for rdbsize := int(nsize); rdbsize != 0; { rdbsize -= iocopy(br, pipew, p, rdbsize) } for { n, err := cmd.PSyncPipeCopy(c, br, bw, offset, pipew) if err != nil { log.PanicErrorf(err, "psync runid = %s, offset = %d, pipe is broken", runid, offset) } offset += n for { time.Sleep(time.Second) c = openNetConnSoft(master, passwd) if c != nil { log.Infof("psync reopen connection, offset = %d", offset) break } else { log.Infof("psync reopen connection, failed") } } authPassword(c, passwd) br = bufio.NewReaderSize(c, ReaderBufferSize) bw = bufio.NewWriterSize(c, WriterBufferSize) sendPSyncContinue(br, bw, runid, offset) } }() return piper, nsize }
func (cmd *cmdSync) SyncCommand(reader *bufio.Reader, target, passwd string) { c := openNetConn(target, passwd) defer c.Close() cr := openRedisConn(target, passwd) defer cr.Close() writer := bufio.NewWriterSize(stats.NewCountWriter(c, &cmd.wbytes), WriterBufferSize) defer flushWriter(writer) go func() { p := make([]byte, ReaderBufferSize) for { iocopy(c, ioutil.Discard, p, len(p)) } }() go func() { var bypass bool = false for { resp := redis.MustDecode(reader) if scmd, args, err := redis.ParseArgs(resp); err != nil { log.PanicError(err, "parse command arguments failed") } else if scmd != "ping" { if scmd == "select" { if len(args) != 1 { log.Panicf("select command len(args) = %d", len(args)) } s := string(args[0]) n, err := parseInt(s, MinDB, MaxDB) if err != nil { log.PanicErrorf(err, "parse db = %s failed", s) } bypass = !acceptDB(uint32(n)) } if bypass || (len(args) > 0 && !acceptKey(args[0])) { cmd.nbypass.Incr() continue } if skipKey(args[0]) { log.Warnf("skip key: %s", args[0]) cmd.ignore.Incr() continue } if aggregateKey(args[0]) && ((scmd == "lpush") || (scmd == "LPUSH")) { log.Infof("Aggregate Key %s", args[0]) for i := 1; i < len(args); i++ { _, err := cr.Do(aggregateCmd, aggregateTarget, args[i]) if err != nil { log.Warnf("SyncAggregate err at: %s %s", args[0], args[i]) //log.PanicError(err, "sync aggregate error") } } } // set 2 sorted set in sync command if set2sortedKey(args[0]) { switch scmd { default: log.Panicf("set2sorted operate %s on key %s err", scmd, args[0]) case "sadd": for i := 1; i < len(args); i++ { _, err := cr.Do("zadd", args[0], 1, args[i]) if err != nil { log.PanicErrorf(err, "set2sorted zadd %s 1 %s", args[0], args[i]) } } case "srem": for i := 1; i < len(args); i++ { _, err := cr.Do("zrem", args[0], args[i]) if err != nil { log.PanicErrorf(err, "set2sorted zrem %s %s", args[0], args[i]) } } case "del": _, err := cr.Do("del", args[0]) if err != nil { log.PanicErrorf(err, "set2sorted del %s", args[0]) } } continue } if sorted2setKey(args[0]) { switch scmd { default: log.Panicf("sorted2set operate %s on key %s err", scmd, args[0]) case "zadd": for i := 1; i < len(args); i++ { if string(args[i]) != "1" { _, err := cr.Do("sadd", args[0], args[i]) if err != nil { log.PanicErrorf(err, "sorted2set sadd %s %s", args[0], args[i]) } } else { continue } } case "zrem": for i := 1; i < len(args); i++ { _, err := cr.Do("srem", args[0], args[i]) if err != nil { log.PanicErrorf(err, "sorted2set srem %s %s", args[0], args[i]) } } case "del": _, err := cr.Do("del", args[0]) if err != nil { log.PanicErrorf(err, "sorted2set del %s", args[0]) } } continue } // Some commands like MSET may have multi keys, but we only use // first for filter } cmd.forward.Incr() redis.MustEncode(writer, resp) flushWriter(writer) } }() for lstat := cmd.Stat(); ; { time.Sleep(time.Second) nstat := cmd.Stat() var b bytes.Buffer fmt.Fprintf(&b, "sync: ") fmt.Fprintf(&b, " +forward=%-6d", nstat.forward-lstat.forward) fmt.Fprintf(&b, " +nbypass=%-6d", nstat.nbypass-lstat.nbypass) fmt.Fprintf(&b, " +nbytes=%d", nstat.wbytes-lstat.wbytes) log.Info(b.String()) lstat = nstat } }
func main() { usage := ` Usage: redis-port decode [--ncpu=N] [--parallel=M] [--input=INPUT] [--output=OUTPUT] redis-port restore [--ncpu=N] [--parallel=M] [--input=INPUT] --target=TARGET [--auth=AUTH] [--extra] [--faketime=FAKETIME] [--filterdb=DB] [--filterkeys=keys] [--skipkeys=keys] [--restorecmd=slotsrestore] [--aggregatetype=type] [--aggregatekeys=keys] [--aggregateTargetKey=key] redis-port dump [--ncpu=N] [--parallel=M] --from=MASTER [--password=PASSWORD] [--output=OUTPUT] [--extra] redis-port sync [--ncpu=N] [--parallel=M] --from=MASTER [--password=PASSWORD] --target=TARGET [--auth=AUTH] [--sockfile=FILE [--filesize=SIZE]] [--filterdb=DB] [--psync] [--force] [--filterkeys=keys] [--skipkeys=keys] [--restorecmd=slotsrestore] [--aggregatetype=type] [--aggregatekeys=keys] [--aggregateTargetKey=key] [--set2sortedkeys=keys] [--sorted2setkeys=keys] Options: -n N, --ncpu=N Set runtime.GOMAXPROCS to N. -p M, --parallel=M Set the number of parallel routines to M. -i INPUT, --input=INPUT Set input file, default is stdin ('/dev/stdin'). -o OUTPUT, --output=OUTPUT Set output file, default is stdout ('/dev/stdout'). -f MASTER, --from=MASTER Set host:port of master redis. -t TARGET, --target=TARGET Set host:port of slave redis. -P PASSWORD, --password=PASSWORD Set redis auth password. -A AUTH, --auth=AUTH Set auth password for target. --faketime=FAKETIME Set current system time to adjust key's expire time. --sockfile=FILE Use FILE to as socket buffer, default is disabled. --filesize=SIZE Set FILE size, default value is 1gb. -e, --extra Set ture to send/receive following redis commands, default is false. --filterdb=DB Filter db = DB, default is *. --filterkeys=keys Filter key in keys, keys is seperated by comma and supports regular expression. --skipkeys=keys Skip key in keys, keys is seperated by comma and supports regular expression. --restorecmd=slotsrestore Restore command, slotsrestore for codis, restore for redis, if the from and target server are the same, use '--restorecmd=del' will delete the keys, togegher with filterkeys, it will delete the keys filtered in the server. --aggregatetype=type Aggregate type: list or set. --aggregatekeys=keys Aggregate key in keys, keys is seperated by comma and supports regular expression. --aggregateTargetKey=key Target key for aggregating. --set2sortedkeys=keys Convert set key in keys to sorted set, keys is seperated by comma and supports regular expression. --sorted2setkeys=keys Convert sorted set key in keys to set, keys is seperated by comma and supports regular expression. --psync Use PSYNC command. --force Use force, do not need to enter "yes", you mustn't use it ,unless you konw what you are doing. ` d, err := docopt.Parse(usage, nil, true, "", false) if err != nil { log.PanicError(err, "parse arguments failed") } if s, ok := d["--ncpu"].(string); ok && s != "" { n, err := parseInt(s, 1, 1024) if err != nil { log.PanicErrorf(err, "parse --ncpu failed") } runtime.GOMAXPROCS(n) } ncpu := runtime.GOMAXPROCS(0) if s, ok := d["--parallel"].(string); ok && s != "" { n, err := parseInt(s, 1, 1024) if err != nil { log.PanicErrorf(err, "parse --parallel failed") } args.parallel = n } if ncpu > args.parallel { args.parallel = ncpu } if args.parallel == 0 { args.parallel = 4 } args.input, _ = d["--input"].(string) args.output, _ = d["--output"].(string) args.from, _ = d["--from"].(string) args.passwd, _ = d["--password"].(string) args.auth, _ = d["--auth"].(string) args.target, _ = d["--target"].(string) args.extra, _ = d["--extra"].(bool) args.psync, _ = d["--psync"].(bool) args.force, _ = d["--force"].(bool) args.sockfile, _ = d["--sockfile"].(string) var input string for { if args.force { goto CONTINUE } fmt.Printf("Are you sure to continue (yes/no)?\n") _, err := fmt.Scanf("%s\r\n", &input) if err != nil { log.PanicError(err, "input yes/no err") } switch input { default: fmt.Printf("Input the wrong value, you should input yes or no") continue case "yes": goto CONTINUE case "no": os.Exit(1) } } CONTINUE: if s, ok := d["--faketime"].(string); ok && s != "" { switch s[0] { case '-', '+': d, err := time.ParseDuration(strings.ToLower(s)) if err != nil { log.PanicError(err, "parse --faketime failed") } args.shift = d case '@': n, err := strconv.ParseInt(s[1:], 10, 64) if err != nil { log.PanicError(err, "parse --faketime failed") } args.shift = time.Duration(n*int64(time.Millisecond) - time.Now().UnixNano()) default: t, err := time.Parse("2006-01-02 15:04:05", s) if err != nil { log.PanicError(err, "parse --faketime failed") } args.shift = time.Duration(t.UnixNano() - time.Now().UnixNano()) } } if s, ok := d["--filterdb"].(string); ok && s != "" && s != "*" { n, err := parseInt(s, MinDB, MaxDB) if err != nil { log.PanicError(err, "parse --filterdb failed") } u := uint32(n) acceptDB = func(db uint32) bool { return db == u } } if s, ok := d["--filterkeys"].(string); ok && s != "" && s != "*" { keys := strings.Split(s, ",") keyRegexps := make([]*regexp.Regexp, len(keys)) for i, key := range keys { keyRegexps[i], err = regexp.Compile(key) if err != nil { log.PanicError(err, "parse --filterkeys failed") } } acceptKey = func(key []byte) bool { for _, reg := range keyRegexps { if reg.Match(key) { return true } } return false } } if s, ok := d["--skipkeys"].(string); ok && s != "" && s != "*" { keys := strings.Split(s, ",") keyRegexps := make([]*regexp.Regexp, len(keys)) for i, key := range keys { keyRegexps[i], err = regexp.Compile(key) if err != nil { log.PanicError(err, "parse --skipkeys failed") } } skipKey = func(key []byte) bool { for _, reg := range keyRegexps { if reg.Match(key) { return true } } return false } } if s, ok := d["--restorecmd"].(string); ok && s != "" { restoreCmd = strings.TrimSpace(s) } if s, ok := d["--aggregatekeys"].(string); ok && s != "" && s != "*" { keys := strings.Split(s, ",") keyRegexps := make([]*regexp.Regexp, len(keys)) for i, key := range keys { keyRegexps[i], err = regexp.Compile(key) if err != nil { log.PanicError(err, "parse --aggregatekeys failed") } } aggregateKey = func(key []byte) bool { for _, reg := range keyRegexps { if reg.Match(key) { return true } } return false } } if s, ok := d["--aggregateTargetKey"].(string); ok && s != "" { aggregateTarget = s } if s, ok := d["--aggregatetype"].(string); ok && s != "" { aggregateType = s switch s { default: aggregateCmd = "lpush" case "list": aggregateCmd = "lpush" case "set": aggregateCmd = "sadd" } } if s, ok := d["--set2sortedkeys"].(string); ok && s != "" && s != "*" { keys := strings.Split(s, ",") keyRegexps := make([]*regexp.Regexp, len(keys)) for i, key := range keys { keyRegexps[i], err = regexp.Compile(key) if err != nil { log.PanicError(err, "parse --set2sortedkeys failed") } } set2sortedKey = func(key []byte) bool { for _, reg := range keyRegexps { if reg.Match(key) { return true } } return false } } if s, ok := d["--sorted2setkeys"].(string); ok && s != "" && s != "*" { keys := strings.Split(s, ",") keyRegexps := make([]*regexp.Regexp, len(keys)) for i, key := range keys { keyRegexps[i], err = regexp.Compile(key) if err != nil { log.PanicError(err, "parse --sorted2setkeys failed") } } sorted2setKey = func(key []byte) bool { for _, reg := range keyRegexps { if reg.Match(key) { return true } } return false } } if s, ok := d["--filesize"].(string); ok && s != "" { if len(args.sockfile) == 0 { log.Panic("please specify --sockfile first") } n, err := bytesize.Parse(s) if err != nil { log.PanicError(err, "parse --filesize failed") } if n <= 0 { log.Panicf("parse --filesize = %d, invalid number", n) } args.filesize = n } else { args.filesize = bytesize.GB } log.Infof("set ncpu = %d, parallel = %d\n", ncpu, args.parallel) switch { case d["decode"].(bool): new(cmdDecode).Main() case d["restore"].(bool): new(cmdRestore).Main() case d["dump"].(bool): new(cmdDump).Main() case d["sync"].(bool): new(cmdSync).Main() } }
func restoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) { var ttlms uint64 if e.ExpireAt != 0 { now := uint64(time.Now().Add(args.shift).UnixNano()) now /= uint64(time.Millisecond) if now >= e.ExpireAt { ttlms = 1 } else { ttlms = e.ExpireAt - now } } toText := func(p []byte) string { var b bytes.Buffer for _, c := range p { switch { case c >= '#' && c <= '~': b.WriteByte(c) default: b.WriteByte('.') } } return b.String() } if aggregateKey(e.Key) { log.Infof("Aggregate key %s", e.Key) o, err := rdb.DecodeDump(e.Value) if err != nil { log.PanicError(err, "decode failed") } switch obj := o.(type) { default: log.Panicf("unknown object %v", o) case rdb.List: for _, ele := range obj { _, err := c.Do(aggregateCmd, aggregateTarget, toText(ele)) if err != nil { log.PanicError(err, "aggregate error") } } case rdb.Set: for _, ele := range obj { _, err := c.Do(aggregateCmd, aggregateTarget, toText(ele)) if err != nil { log.PanicError(err, "aggregate error") } } } } if set2sortedKey(e.Key) { o, err := rdb.DecodeDump(e.Value) if err != nil { log.PanicError(err, "decode failed") } switch obj := o.(type) { default: log.Panicf("set2sorted key %s type is not set err", e.Key) log.Panicf("unknown object %v", o) case rdb.Set: for _, ele := range obj { _, err := c.Do("zadd", e.Key, 1, toText(ele)) if err != nil { log.PanicErrorf(err, "set2sorted zadd %s 1 %s", e.Key, toText(ele)) } } } return } if sorted2setKey(e.Key) { o, err := rdb.DecodeDump(e.Value) if err != nil { log.PanicError(err, "decode failed") } switch obj := o.(type) { default: log.Panicf("sorted2set key %s type is not sorted set err", e.Key) case rdb.ZSet: for _, ele := range obj { _, err := c.Do("sadd", e.Key, toText(ele.Member)) if err != nil { log.PanicErrorf(err, "sorted2set sadd %s %s", e.Key, toText(ele.Member)) } } } return } if (restoreCmd == "del") || (restoreCmd == "DEL") { _, err := redigo.String(c.Do(restoreCmd, e.Key)) if err != nil { log.Warnf("delete key: '%s'", e.Key) } } else { s, err := redigo.String(c.Do(restoreCmd, e.Key, ttlms, e.Value)) if err != nil { log.Warnf("restore error, when '%s' '%s'", restoreCmd, e.Key) //log.PanicError(err, "restore command error") } if s != "OK" { //log.Panicf("restore command response = '%s', should be 'OK'", s) log.Warnf("restore command response = '%s', should be 'OK'", s) } } }
func (cmd *cmdRestore) RestoreCommand(reader *bufio.Reader, target, passwd string) { c := openNetConn(target, passwd) defer c.Close() writer := bufio.NewWriterSize(c, WriterBufferSize) defer flushWriter(writer) go func() { p := make([]byte, ReaderBufferSize) for { iocopy(c, ioutil.Discard, p, len(p)) } }() go func() { var bypass bool = false for { resp := redis.MustDecode(reader) if scmd, args, err := redis.ParseArgs(resp); err != nil { log.PanicError(err, "parse command arguments failed") } else if scmd != "ping" { if scmd == "select" { if len(args) != 1 { log.Panicf("select command len(args) = %d", len(args)) } s := string(args[0]) n, err := parseInt(s, MinDB, MaxDB) if err != nil { log.PanicErrorf(err, "parse db = %s failed", s) } bypass = !acceptDB(uint32(n)) } if skipKey(args[0]) { log.Warnf("skip key: %s", args[0]) cmd.ignore.Incr() continue } if bypass || (len(args) > 0 && !acceptKey(args[0])) { cmd.nbypass.Incr() continue } // added for aggregating list or set if aggregateKey(args[0]) { cr := openRedisConn(target, passwd) defer cr.Close() for i := 1; i < len(args); i++ { _, err := cr.Do(aggregateCmd, aggregateTarget, args[i]) if err != nil { log.PanicError(err, "restore aggregate error") } } } } cmd.forward.Incr() redis.MustEncode(writer, resp) flushWriter(writer) } }() for lstat := cmd.Stat(); ; { time.Sleep(time.Second) nstat := cmd.Stat() var b bytes.Buffer fmt.Fprintf(&b, "restore: ") fmt.Fprintf(&b, " +forward=%-6d", nstat.forward-lstat.forward) fmt.Fprintf(&b, " +nbypass=%-6d", nstat.nbypass-lstat.nbypass) log.Info(b.String()) lstat = nstat } }