func sendPSyncFullsync(br *bufio.Reader, bw *bufio.Writer) (string, int64, <-chan int64) { cmd := redis.NewCommand("psync", "?", -1) if err := redis.Encode(bw, cmd, true); err != nil { log.PanicError(err, "write psync command failed, fullsync") } r, err := redis.Decode(br) if err != nil { log.PanicError(err, "invalid psync response, fullsync") } if e, ok := r.(*redis.Error); ok { log.Panicf("invalid psync response, fullsync, %s", e.Value) } x, err := redis.AsString(r, nil) if err != nil { log.PanicError(err, "invalid psync response, fullsync") } xx := strings.Split(x, " ") if len(xx) != 3 || strings.ToLower(xx[0]) != "fullresync" { log.Panicf("invalid psync response = '%s', should be fullsync", x) } v, err := strconv.ParseInt(xx[2], 10, 64) if err != nil { log.PanicError(err, "parse psync offset failed") } runid, offset := xx[1], v-1 return runid, offset, waitRdbDump(br) }
func waitRdbDump(r io.Reader) <-chan int64 { size := make(chan int64) go func() { var rsp string for { b := []byte{0} if _, err := r.Read(b); err != nil { log.PanicErrorf(err, "read sync response = '%s'", rsp) } if len(rsp) == 0 && b[0] == '\n' { size <- 0 continue } rsp += string(b) if strings.HasSuffix(rsp, "\r\n") { break } } if rsp[0] != '$' { log.Panicf("invalid sync response, rsp = '%s'", rsp) } n, err := strconv.Atoi(rsp[1 : len(rsp)-2]) if err != nil || n <= 0 { log.PanicErrorf(err, "invalid sync response = '%s', n = %d", rsp, n) } size <- int64(n) }() return size }
func selectDB(c redigo.Conn, db uint32) { s, err := redigo.String(c.Do("select", db)) if err != nil { log.PanicError(err, "select command error") } if s != "OK" { log.Panicf("select command response = '%s', should be 'OK'", s) } }
func (cmd *cmdSync) SyncCommand(reader *bufio.Reader, target, passwd string) { c := openNetConn(target, passwd) defer c.Close() writer := bufio.NewWriterSize(stats.NewCountWriter(c, &cmd.wbytes), WriterBufferSize) defer flushWriter(writer) go func() { p := make([]byte, ReaderBufferSize) for { iocopy(c, ioutil.Discard, p, len(p)) } }() go func() { var bypass bool = false for { resp := redis.MustDecode(reader) if scmd, args, err := redis.ParseArgs(resp); err != nil { log.PanicError(err, "parse command arguments failed") } else if scmd != "ping" { if scmd == "select" { if len(args) != 1 { log.Panicf("select command len(args) = %d", len(args)) } s := string(args[0]) n, err := parseInt(s, MinDB, MaxDB) if err != nil { log.PanicErrorf(err, "parse db = %s failed", s) } bypass = !acceptDB(uint32(n)) } if bypass { cmd.nbypass.Incr() continue } } cmd.forward.Incr() redis.MustEncode(writer, resp) flushWriter(writer) } }() for lstat := cmd.Stat(); ; { time.Sleep(time.Second) nstat := cmd.Stat() var b bytes.Buffer fmt.Fprintf(&b, "sync: ") fmt.Fprintf(&b, " +forward=%-6d", nstat.forward-lstat.forward) fmt.Fprintf(&b, " +nbypass=%-6d", nstat.nbypass-lstat.nbypass) fmt.Fprintf(&b, " +nbytes=%d", nstat.wbytes-lstat.wbytes) log.Info(b.String()) lstat = nstat } }
func sendPSyncContinue(br *bufio.Reader, bw *bufio.Writer, runid string, offset int64) { cmd := redis.NewCommand("psync", runid, offset+2) if err := redis.Encode(bw, cmd, true); err != nil { log.PanicError(err, "write psync command failed, continue") } r, err := redis.Decode(br) if err != nil { log.PanicError(err, "invalid psync response, continue") } if e, ok := r.(*redis.Error); ok { log.Panicf("invalid psync response, continue, %s", e.Value) } x, err := redis.AsString(r, nil) if err != nil { log.PanicError(err, "invalid psync response, continue") } xx := strings.Split(x, " ") if len(xx) != 1 || strings.ToLower(xx[0]) != "continue" { log.Panicf("invalid psync response = '%s', should be continue", x) } }
func iocopy(r io.Reader, w io.Writer, p []byte, max int) int { if max <= 0 || len(p) == 0 { log.Panicf("invalid max = %d, len(p) = %d", max, len(p)) } if len(p) > max { p = p[:max] } if n, err := r.Read(p); err != nil { log.PanicError(err, "read error") } else { p = p[:n] } if _, err := w.Write(p); err != nil { log.PanicError(err, "write error") } return len(p) }
func restoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) { var ttlms uint64 if e.ExpireAt != 0 { now := uint64(time.Now().Add(args.shift).UnixNano()) now /= uint64(time.Millisecond) if now >= e.ExpireAt { ttlms = 1 } else { ttlms = e.ExpireAt - now } } s, err := redigo.String(c.Do("slotsrestore", e.Key, ttlms, e.Value)) if err != nil { log.PanicError(err, "restore command error") } if s != "OK" { log.Panicf("restore command response = '%s', should be 'OK'", s) } }
func main() { usage := ` Usage: redis-port decode [--ncpu=N] [--parallel=M] [--input=INPUT] [--output=OUTPUT] redis-port restore [--ncpu=N] [--parallel=M] [--input=INPUT] --target=TARGET [--auth=AUTH] [--extra] [--faketime=FAKETIME] [--filterdb=DB] redis-port dump [--ncpu=N] [--parallel=M] --from=MASTER [--password=PASSWORD] [--output=OUTPUT] [--extra] redis-port sync [--ncpu=N] [--parallel=M] --from=MASTER [--password=PASSWORD] --target=TARGET [--auth=AUTH] [--sockfile=FILE [--filesize=SIZE]] [--filterdb=DB] [--psync] Options: -n N, --ncpu=N Set runtime.GOMAXPROCS to N. -p M, --parallel=M Set the number of parallel routines to M. -i INPUT, --input=INPUT Set input file, default is stdin ('/dev/stdin'). -o OUTPUT, --output=OUTPUT Set output file, default is stdout ('/dev/stdout'). -f MASTER, --from=MASTER Set host:port of master redis. -t TARGET, --target=TARGET Set host:port of slave redis. -P PASSWORD, --password=PASSWORD Set redis auth password. -A AUTH, --auth=AUTH Set auth password for target. --faketime=FAKETIME Set current system time to adjust key's expire time. --sockfile=FILE Use FILE to as socket buffer, default is disabled. --filesize=SIZE Set FILE size, default value is 1gb. -e, --extra Set ture to send/receive following redis commands, default is false. --filterdb=DB Filter db = DB, default is *. --psync Use PSYNC command. ` d, err := docopt.Parse(usage, nil, true, "", false) if err != nil { log.PanicError(err, "parse arguments failed") } if s, ok := d["--ncpu"].(string); ok && s != "" { n, err := parseInt(s, 1, 1024) if err != nil { log.PanicErrorf(err, "parse --ncpu failed") } runtime.GOMAXPROCS(n) } ncpu := runtime.GOMAXPROCS(0) if s, ok := d["--parallel"].(string); ok && s != "" { n, err := parseInt(s, 1, 1024) if err != nil { log.PanicErrorf(err, "parse --parallel failed") } args.parallel = n } if ncpu > args.parallel { args.parallel = ncpu } if args.parallel == 0 { args.parallel = 4 } args.input, _ = d["--input"].(string) args.output, _ = d["--output"].(string) args.from, _ = d["--from"].(string) args.passwd, _ = d["--password"].(string) args.auth, _ = d["--auth"].(string) args.target, _ = d["--target"].(string) args.extra, _ = d["--extra"].(bool) args.psync, _ = d["--psync"].(bool) args.sockfile, _ = d["--sockfile"].(string) if s, ok := d["--faketime"].(string); ok && s != "" { switch s[0] { case '-', '+': d, err := time.ParseDuration(strings.ToLower(s)) if err != nil { log.PanicError(err, "parse --faketime failed") } args.shift = d case '@': n, err := strconv.ParseInt(s[1:], 10, 64) if err != nil { log.PanicError(err, "parse --faketime failed") } args.shift = time.Duration(n*int64(time.Millisecond) - time.Now().UnixNano()) default: t, err := time.Parse("2006-01-02 15:04:05", s) if err != nil { log.PanicError(err, "parse --faketime failed") } args.shift = time.Duration(t.UnixNano() - time.Now().UnixNano()) } } if s, ok := d["--filterdb"].(string); ok && s != "" && s != "*" { n, err := parseInt(s, MinDB, MaxDB) if err != nil { log.PanicError(err, "parse --filterdb failed") } u := uint32(n) acceptDB = func(db uint32) bool { return db == u } } if s, ok := d["--filesize"].(string); ok && s != "" { if len(args.sockfile) == 0 { log.Panic("please specify --sockfile first") } n, err := bytesize.Parse(s) if err != nil { log.PanicError(err, "parse --filesize failed") } if n <= 0 { log.Panicf("parse --filesize = %d, invalid number", n) } args.filesize = n } else { args.filesize = bytesize.GB } log.Infof("set ncpu = %d, parallel = %d\n", ncpu, args.parallel) switch { case d["decode"].(bool): new(cmdDecode).Main() case d["restore"].(bool): new(cmdRestore).Main() case d["dump"].(bool): new(cmdDump).Main() case d["sync"].(bool): new(cmdSync).Main() } }
func (cmd *cmdDecode) decoderMain(ipipe <-chan *rdb.BinEntry, opipe chan<- string) { toText := func(p []byte) string { var b bytes.Buffer for _, c := range p { switch { case c >= '#' && c <= '~': b.WriteByte(c) default: b.WriteByte('.') } } return b.String() } toBase64 := func(p []byte) string { return base64.StdEncoding.EncodeToString(p) } toJson := func(o interface{}) string { b, err := json.Marshal(o) if err != nil { log.PanicError(err, "encode to json failed") } return string(b) } for e := range ipipe { o, err := rdb.DecodeDump(e.Value) if err != nil { log.PanicError(err, "decode failed") } var b bytes.Buffer switch obj := o.(type) { default: log.Panicf("unknown object %v", o) case rdb.String: o := &struct { DB uint32 `json:"db"` Type string `json:"type"` ExpireAt uint64 `json:"expireat"` Key string `json:"key"` Key64 string `json:"key64"` Value64 string `json:"value64"` }{ e.DB, "string", e.ExpireAt, toText(e.Key), toBase64(e.Key), toBase64(obj), } fmt.Fprintf(&b, "%s\n", toJson(o)) case rdb.List: for i, ele := range obj { o := &struct { DB uint32 `json:"db"` Type string `json:"type"` ExpireAt uint64 `json:"expireat"` Key string `json:"key"` Key64 string `json:"key64"` Index int `json:"index"` Value64 string `json:"value64"` }{ e.DB, "list", e.ExpireAt, toText(e.Key), toBase64(e.Key), i, toBase64(ele), } fmt.Fprintf(&b, "%s\n", toJson(o)) } case rdb.Hash: for _, ele := range obj { o := &struct { DB uint32 `json:"db"` Type string `json:"type"` ExpireAt uint64 `json:"expireat"` Key string `json:"key"` Key64 string `json:"key64"` Field string `json:"field"` Field64 string `json:"field64"` Value64 string `json:"value64"` }{ e.DB, "hash", e.ExpireAt, toText(e.Key), toBase64(e.Key), toText(ele.Field), toBase64(ele.Field), toBase64(ele.Value), } fmt.Fprintf(&b, "%s\n", toJson(o)) } case rdb.Set: for _, mem := range obj { o := &struct { DB uint32 `json:"db"` Type string `json:"type"` ExpireAt uint64 `json:"expireat"` Key string `json:"key"` Key64 string `json:"key64"` Member string `json:"member"` Member64 string `json:"member64"` }{ e.DB, "set", e.ExpireAt, toText(e.Key), toBase64(e.Key), toText(mem), toBase64(mem), } fmt.Fprintf(&b, "%s\n", toJson(o)) } case rdb.ZSet: for _, ele := range obj { o := &struct { DB uint32 `json:"db"` Type string `json:"type"` ExpireAt uint64 `json:"expireat"` Key string `json:"key"` Key64 string `json:"key64"` Member string `json:"member"` Member64 string `json:"member64"` Score float64 `json:"score"` }{ e.DB, "zset", e.ExpireAt, toText(e.Key), toBase64(e.Key), toText(ele.Member), toBase64(ele.Member), ele.Score, } fmt.Fprintf(&b, "%s\n", toJson(o)) } } cmd.nentry.Incr() opipe <- b.String() } }