func newRDBLoader(reader *bufio.Reader, rbytes *atomic2.Int64, size int) chan *rdb.BinEntry {
pipe := make(chan *rdb.BinEntry, size)
go func() {
defer close(pipe)
l := rdb.NewLoader(stats.NewCountReader(reader, rbytes))
if err := l.Header(); err != nil {
log.PanicError(err, "parse rdb header error")
}
for {
if entry, err := l.NextBinEntry(); err != nil {
log.PanicError(err, "parse rdb entry error")
} else {
if entry != nil {
pipe <- entry
} else {
if err := l.Footer(); err != nil {
log.PanicError(err, "parse rdb checksum error")
}
return
}
}
}
}()
return pipe
}
func sendPSyncFullsync(br *bufio.Reader, bw *bufio.Writer) (string, int64, <-chan int64) {
cmd := redis.NewCommand("psync", "?", -1)
if err := redis.Encode(bw, cmd, true); err != nil {
log.PanicError(err, "write psync command failed, fullsync")
}
r, err := redis.Decode(br)
if err != nil {
log.PanicError(err, "invalid psync response, fullsync")
}
if e, ok := r.(*redis.Error); ok {
log.Panicf("invalid psync response, fullsync, %s", e.Value)
}
x, err := redis.AsString(r, nil)
if err != nil {
log.PanicError(err, "invalid psync response, fullsync")
}
xx := strings.Split(x, " ")
if len(xx) != 3 || strings.ToLower(xx[0]) != "fullresync" {
log.Panicf("invalid psync response = '%s', should be fullsync", x)
}
v, err := strconv.ParseInt(xx[2], 10, 64)
if err != nil {
log.PanicError(err, "parse psync offset failed")
}
runid, offset := xx[1], v-1
return runid, offset, waitRdbDump(br)
}
func MustParse(s string) int64 {
v, err := Parse(s)
if err != nil {
log.PanicError(err, "parse bytesize failed")
}
return v
}
func MustDecodeFromBytes(p []byte) Resp {
resp, err := DecodeFromBytes(p)
if err != nil {
log.PanicError(err, "decode redis resp from bytes failed")
}
return resp
}
func MustEncodeToBytes(r Resp) []byte {
b, err := EncodeToBytes(r)
if err != nil {
log.PanicError(err, "encode redis resp to bytes failed")
}
return b
}
func MustHandlerTable(o interface{}) map[string]HandlerFunc {
t, err := NewHandlerTable(o)
if err != nil {
log.PanicError(err, "create redis handler map failed")
}
return t
}
func authPassword(c net.Conn, passwd string) {
if passwd == "" {
return
}
_, err := c.Write(redis.MustEncodeToBytes(redis.NewCommand("auth", passwd)))
if err != nil {
log.PanicError(errors.Trace(err), "write auth command failed")
}
var b = make([]byte, 5)
if _, err := io.ReadFull(c, b); err != nil {
log.PanicError(errors.Trace(err), "read auth response failed")
}
if strings.ToUpper(string(b)) != "+OK\r\n" {
log.Panic("auth failed")
}
}
func openSyncConn(target string, passwd string) (net.Conn, <-chan int64) {
c := openNetConn(target, passwd)
if _, err := c.Write(redis.MustEncodeToBytes(redis.NewCommand("sync"))); err != nil {
log.PanicError(errors.Trace(err), "write sync command failed")
}
return c, waitRdbDump(c)
}
func MustDecode(r *bufio.Reader) Resp {
resp, err := Decode(r)
if err != nil {
log.PanicError(err, "decode redis resp failed")
}
return resp
}
func iocopy(r io.Reader, w io.Writer, p []byte, max int) int {
if max <= 0 || len(p) == 0 {
log.Panicf("invalid max = %d, len(p) = %d", max, len(p))
}
if len(p) > max {
p = p[:max]
}
if n, err := r.Read(p); err != nil {
log.PanicError(err, "read error")
} else {
p = p[:n]
}
if _, err := w.Write(p); err != nil {
log.PanicError(err, "write error")
}
return len(p)
}
func selectDB(c redigo.Conn, db uint32) {
s, err := redigo.String(c.Do("select", db))
if err != nil {
log.PanicError(err, "select command error")
}
if s != "OK" {
log.Panicf("select command response = '%s', should be 'OK'", s)
}
}
func (cmd *cmdSync) SyncCommand(reader *bufio.Reader, target, passwd string) {
c := openNetConn(target, passwd)
defer c.Close()
writer := bufio.NewWriterSize(stats.NewCountWriter(c, &cmd.wbytes), WriterBufferSize)
defer flushWriter(writer)
go func() {
p := make([]byte, ReaderBufferSize)
for {
iocopy(c, ioutil.Discard, p, len(p))
}
}()
go func() {
var bypass bool = false
for {
resp := redis.MustDecode(reader)
if scmd, args, err := redis.ParseArgs(resp); err != nil {
log.PanicError(err, "parse command arguments failed")
} else if scmd != "ping" {
if scmd == "select" {
if len(args) != 1 {
log.Panicf("select command len(args) = %d", len(args))
}
s := string(args[0])
n, err := parseInt(s, MinDB, MaxDB)
if err != nil {
log.PanicErrorf(err, "parse db = %s failed", s)
}
bypass = !acceptDB(uint32(n))
}
if bypass {
cmd.nbypass.Incr()
continue
}
}
cmd.forward.Incr()
redis.MustEncode(writer, resp)
flushWriter(writer)
}
}()
for lstat := cmd.Stat(); ; {
time.Sleep(time.Second)
nstat := cmd.Stat()
var b bytes.Buffer
fmt.Fprintf(&b, "sync: ")
fmt.Fprintf(&b, " +forward=%-6d", nstat.forward-lstat.forward)
fmt.Fprintf(&b, " +nbypass=%-6d", nstat.nbypass-lstat.nbypass)
fmt.Fprintf(&b, " +nbytes=%d", nstat.wbytes-lstat.wbytes)
log.Info(b.String())
lstat = nstat
}
}
func sendPSyncContinue(br *bufio.Reader, bw *bufio.Writer, runid string, offset int64) {
cmd := redis.NewCommand("psync", runid, offset+2)
if err := redis.Encode(bw, cmd, true); err != nil {
log.PanicError(err, "write psync command failed, continue")
}
r, err := redis.Decode(br)
if err != nil {
log.PanicError(err, "invalid psync response, continue")
}
if e, ok := r.(*redis.Error); ok {
log.Panicf("invalid psync response, continue, %s", e.Value)
}
x, err := redis.AsString(r, nil)
if err != nil {
log.PanicError(err, "invalid psync response, continue")
}
xx := strings.Split(x, " ")
if len(xx) != 1 || strings.ToLower(xx[0]) != "continue" {
log.Panicf("invalid psync response = '%s', should be continue", x)
}
}
func restoreRdbEntry(c redigo.Conn, e *rdb.BinEntry) {
var ttlms uint64
if e.ExpireAt != 0 {
now := uint64(time.Now().Add(args.shift).UnixNano())
now /= uint64(time.Millisecond)
if now >= e.ExpireAt {
ttlms = 1
} else {
ttlms = e.ExpireAt - now
}
}
s, err := redigo.String(c.Do("slotsrestore", e.Key, ttlms, e.Value))
if err != nil {
log.PanicError(err, "restore command error")
}
if s != "OK" {
log.Panicf("restore command response = '%s', should be 'OK'", s)
}
}
func (cmd *cmdDecode) Main() {
input, output := args.input, args.output
if len(input) == 0 {
input = "/dev/stdin"
}
if len(output) == 0 {
output = "/dev/stdout"
}
log.Infof("decode from '%s' to '%s'\n", input, output)
var readin io.ReadCloser
var nsize int64
if input != "/dev/stdin" {
readin, nsize = openReadFile(input)
defer readin.Close()
} else {
readin, nsize = os.Stdin, 0
}
var saveto io.WriteCloser
if output != "/dev/stdout" {
saveto = openWriteFile(output)
defer saveto.Close()
} else {
saveto = os.Stdout
}
reader := bufio.NewReaderSize(readin, ReaderBufferSize)
writer := bufio.NewWriterSize(saveto, WriterBufferSize)
ipipe := newRDBLoader(reader, &cmd.rbytes, args.parallel*32)
opipe := make(chan string, cap(ipipe))
go func() {
defer close(opipe)
group := make(chan int, args.parallel)
for i := 0; i < cap(group); i++ {
go func() {
defer func() {
group <- 0
}()
cmd.decoderMain(ipipe, opipe)
}()
}
for i := 0; i < cap(group); i++ {
<-group
}
}()
wait := make(chan struct{})
go func() {
defer close(wait)
for s := range opipe {
cmd.wbytes.Add(int64(len(s)))
if _, err := writer.WriteString(s); err != nil {
log.PanicError(err, "write string failed")
}
flushWriter(writer)
}
}()
for done := false; !done; {
select {
case <-wait:
done = true
case <-time.After(time.Second):
}
stat := cmd.Stat()
var b bytes.Buffer
fmt.Fprintf(&b, "decode: ")
if nsize != 0 {
fmt.Fprintf(&b, "total = %d - %12d [%3d%%]", nsize, stat.rbytes, 100*stat.rbytes/nsize)
} else {
fmt.Fprintf(&b, "total = %12d", stat.rbytes)
}
fmt.Fprintf(&b, " write=%-12d", stat.wbytes)
fmt.Fprintf(&b, " entry=%-12d", stat.nentry)
log.Info(b.String())
}
log.Info("decode: done")
}
func MustEncode(w *bufio.Writer, r Resp) {
if err := Encode(w, r, true); err != nil {
log.PanicError(err, "encode redis resp failed")
}
}
func MustNoError(err error) {
if err == nil {
return
}
log.PanicError(err, "error happens, assertion failed")
}
func flushWriter(w *bufio.Writer) {
if err := w.Flush(); err != nil {
log.PanicError(err, "flush error")
}
}
func main() {
usage := `
Usage:
redis-port decode [--ncpu=N] [--parallel=M] [--input=INPUT] [--output=OUTPUT]
redis-port restore [--ncpu=N] [--parallel=M] [--input=INPUT] --target=TARGET [--auth=AUTH] [--extra] [--faketime=FAKETIME] [--filterdb=DB]
redis-port dump [--ncpu=N] [--parallel=M] --from=MASTER [--password=PASSWORD] [--output=OUTPUT] [--extra]
redis-port sync [--ncpu=N] [--parallel=M] --from=MASTER [--password=PASSWORD] --target=TARGET [--auth=AUTH] [--sockfile=FILE [--filesize=SIZE]] [--filterdb=DB] [--psync]
Options:
-n N, --ncpu=N Set runtime.GOMAXPROCS to N.
-p M, --parallel=M Set the number of parallel routines to M.
-i INPUT, --input=INPUT Set input file, default is stdin ('/dev/stdin').
-o OUTPUT, --output=OUTPUT Set output file, default is stdout ('/dev/stdout').
-f MASTER, --from=MASTER Set host:port of master redis.
-t TARGET, --target=TARGET Set host:port of slave redis.
-P PASSWORD, --password=PASSWORD Set redis auth password.
-A AUTH, --auth=AUTH Set auth password for target.
--faketime=FAKETIME Set current system time to adjust key's expire time.
--sockfile=FILE Use FILE to as socket buffer, default is disabled.
--filesize=SIZE Set FILE size, default value is 1gb.
-e, --extra Set ture to send/receive following redis commands, default is false.
--filterdb=DB Filter db = DB, default is *.
--psync Use PSYNC command.
`
d, err := docopt.Parse(usage, nil, true, "", false)
if err != nil {
log.PanicError(err, "parse arguments failed")
}
if s, ok := d["--ncpu"].(string); ok && s != "" {
n, err := parseInt(s, 1, 1024)
if err != nil {
log.PanicErrorf(err, "parse --ncpu failed")
}
runtime.GOMAXPROCS(n)
}
ncpu := runtime.GOMAXPROCS(0)
if s, ok := d["--parallel"].(string); ok && s != "" {
n, err := parseInt(s, 1, 1024)
if err != nil {
log.PanicErrorf(err, "parse --parallel failed")
}
args.parallel = n
}
if ncpu > args.parallel {
args.parallel = ncpu
}
if args.parallel == 0 {
args.parallel = 4
}
args.input, _ = d["--input"].(string)
args.output, _ = d["--output"].(string)
args.from, _ = d["--from"].(string)
args.passwd, _ = d["--password"].(string)
args.auth, _ = d["--auth"].(string)
args.target, _ = d["--target"].(string)
args.extra, _ = d["--extra"].(bool)
args.psync, _ = d["--psync"].(bool)
args.sockfile, _ = d["--sockfile"].(string)
if s, ok := d["--faketime"].(string); ok && s != "" {
switch s[0] {
case '-', '+':
d, err := time.ParseDuration(strings.ToLower(s))
if err != nil {
log.PanicError(err, "parse --faketime failed")
}
args.shift = d
case '@':
n, err := strconv.ParseInt(s[1:], 10, 64)
if err != nil {
log.PanicError(err, "parse --faketime failed")
}
args.shift = time.Duration(n*int64(time.Millisecond) - time.Now().UnixNano())
default:
t, err := time.Parse("2006-01-02 15:04:05", s)
if err != nil {
log.PanicError(err, "parse --faketime failed")
}
args.shift = time.Duration(t.UnixNano() - time.Now().UnixNano())
}
}
if s, ok := d["--filterdb"].(string); ok && s != "" && s != "*" {
n, err := parseInt(s, MinDB, MaxDB)
if err != nil {
log.PanicError(err, "parse --filterdb failed")
}
u := uint32(n)
acceptDB = func(db uint32) bool {
return db == u
}
}
if s, ok := d["--filesize"].(string); ok && s != "" {
if len(args.sockfile) == 0 {
log.Panic("please specify --sockfile first")
}
n, err := bytesize.Parse(s)
if err != nil {
log.PanicError(err, "parse --filesize failed")
}
if n <= 0 {
log.Panicf("parse --filesize = %d, invalid number", n)
}
args.filesize = n
} else {
args.filesize = bytesize.GB
}
log.Infof("set ncpu = %d, parallel = %d\n", ncpu, args.parallel)
switch {
case d["decode"].(bool):
new(cmdDecode).Main()
case d["restore"].(bool):
new(cmdRestore).Main()
case d["dump"].(bool):
new(cmdDump).Main()
case d["sync"].(bool):
new(cmdSync).Main()
}
}
func (cmd *cmdDecode) decoderMain(ipipe <-chan *rdb.BinEntry, opipe chan<- string) {
toText := func(p []byte) string {
var b bytes.Buffer
for _, c := range p {
switch {
case c >= '#' && c <= '~':
b.WriteByte(c)
default:
b.WriteByte('.')
}
}
return b.String()
}
toBase64 := func(p []byte) string {
return base64.StdEncoding.EncodeToString(p)
}
toJson := func(o interface{}) string {
b, err := json.Marshal(o)
if err != nil {
log.PanicError(err, "encode to json failed")
}
return string(b)
}
for e := range ipipe {
o, err := rdb.DecodeDump(e.Value)
if err != nil {
log.PanicError(err, "decode failed")
}
var b bytes.Buffer
switch obj := o.(type) {
default:
log.Panicf("unknown object %v", o)
case rdb.String:
o := &struct {
DB uint32 `json:"db"`
Type string `json:"type"`
ExpireAt uint64 `json:"expireat"`
Key string `json:"key"`
Key64 string `json:"key64"`
Value64 string `json:"value64"`
}{
e.DB, "string", e.ExpireAt, toText(e.Key), toBase64(e.Key),
toBase64(obj),
}
fmt.Fprintf(&b, "%s\n", toJson(o))
case rdb.List:
for i, ele := range obj {
o := &struct {
DB uint32 `json:"db"`
Type string `json:"type"`
ExpireAt uint64 `json:"expireat"`
Key string `json:"key"`
Key64 string `json:"key64"`
Index int `json:"index"`
Value64 string `json:"value64"`
}{
e.DB, "list", e.ExpireAt, toText(e.Key), toBase64(e.Key),
i, toBase64(ele),
}
fmt.Fprintf(&b, "%s\n", toJson(o))
}
case rdb.Hash:
for _, ele := range obj {
o := &struct {
DB uint32 `json:"db"`
Type string `json:"type"`
ExpireAt uint64 `json:"expireat"`
Key string `json:"key"`
Key64 string `json:"key64"`
Field string `json:"field"`
Field64 string `json:"field64"`
Value64 string `json:"value64"`
}{
e.DB, "hash", e.ExpireAt, toText(e.Key), toBase64(e.Key),
toText(ele.Field), toBase64(ele.Field), toBase64(ele.Value),
}
fmt.Fprintf(&b, "%s\n", toJson(o))
}
case rdb.Set:
for _, mem := range obj {
o := &struct {
DB uint32 `json:"db"`
Type string `json:"type"`
ExpireAt uint64 `json:"expireat"`
Key string `json:"key"`
Key64 string `json:"key64"`
Member string `json:"member"`
Member64 string `json:"member64"`
}{
e.DB, "set", e.ExpireAt, toText(e.Key), toBase64(e.Key),
toText(mem), toBase64(mem),
}
fmt.Fprintf(&b, "%s\n", toJson(o))
}
case rdb.ZSet:
for _, ele := range obj {
o := &struct {
DB uint32 `json:"db"`
Type string `json:"type"`
ExpireAt uint64 `json:"expireat"`
Key string `json:"key"`
Key64 string `json:"key64"`
Member string `json:"member"`
Member64 string `json:"member64"`
Score float64 `json:"score"`
}{
e.DB, "zset", e.ExpireAt, toText(e.Key), toBase64(e.Key),
toText(ele.Member), toBase64(ele.Member), ele.Score,
}
fmt.Fprintf(&b, "%s\n", toJson(o))
}
}
cmd.nentry.Incr()
opipe <- b.String()
}
}