func waitRdbDump(r io.Reader) <-chan int64 {
size := make(chan int64)
go func() {
var rsp string
for {
b := []byte{0}
if _, err := r.Read(b); err != nil {
log.PanicErrorf(err, "read sync response = '%s'", rsp)
}
if len(rsp) == 0 && b[0] == '\n' {
size <- 0
continue
}
rsp += string(b)
if strings.HasSuffix(rsp, "\r\n") {
break
}
}
if rsp[0] != '$' {
log.Panicf("invalid sync response, rsp = '%s'", rsp)
}
n, err := strconv.Atoi(rsp[1 : len(rsp)-2])
if err != nil || n <= 0 {
log.PanicErrorf(err, "invalid sync response = '%s', n = %d", rsp, n)
}
size <- int64(n)
}()
return size
}
func openReadFile(name string) (*os.File, int64) {
f, err := os.Open(name)
if err != nil {
log.PanicErrorf(err, "cannot open file-reader '%s'", name)
}
s, err := f.Stat()
if err != nil {
log.PanicErrorf(err, "cannot stat file-reader '%s'", name)
}
return f, s.Size()
}
func openReadWriteFile(name string) *os.File {
f, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600)
if err != nil {
log.PanicErrorf(err, "cannot open file-readwriter '%s'", name)
}
return f
}
func openWriteFile(name string) *os.File {
f, err := os.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
if err != nil {
log.PanicErrorf(err, "cannot open file-writer '%s'", name)
}
return f
}
func encodeRawBytes(w *BufWriter, refs ...interface{}) {
for _, i := range refs {
var err error
switch x := i.(type) {
case byte:
err = w.WriteByte(x)
case ObjectCode:
err = w.WriteByte(byte(x))
case *uint32:
err = w.WriteUvarint(uint64(*x))
case *uint64:
err = w.WriteUvarint(*x)
case *int64:
err = w.WriteVarint(*x)
case *float64:
err = w.WriteFloat64(*x)
case *[]byte:
err = w.WriteVarbytes(*x)
default:
log.Panicf("unsupported type in row value: %+v", x)
}
if err != nil {
log.PanicErrorf(err, "encode raw bytes failed")
}
}
}
func openNetConn(target string) net.Conn {
c, err := net.Dial("tcp", target)
if err != nil {
log.PanicErrorf(err, "cannot connect to '%s'", target)
}
return c
}
func openNetConn(target, passwd string) net.Conn {
c, err := net.Dial("tcp", target)
if err != nil {
log.PanicErrorf(err, "cannot connect to '%s'", target)
}
authPassword(c, passwd)
return c
}
func (cmd *cmdSync) SyncCommand(reader *bufio.Reader, target, passwd string) {
c := openNetConn(target, passwd)
defer c.Close()
writer := bufio.NewWriterSize(stats.NewCountWriter(c, &cmd.wbytes), WriterBufferSize)
defer flushWriter(writer)
go func() {
p := make([]byte, ReaderBufferSize)
for {
iocopy(c, ioutil.Discard, p, len(p))
}
}()
go func() {
var bypass bool = false
for {
resp := redis.MustDecode(reader)
if scmd, args, err := redis.ParseArgs(resp); err != nil {
log.PanicError(err, "parse command arguments failed")
} else if scmd != "ping" {
if scmd == "select" {
if len(args) != 1 {
log.Panicf("select command len(args) = %d", len(args))
}
s := string(args[0])
n, err := parseInt(s, MinDB, MaxDB)
if err != nil {
log.PanicErrorf(err, "parse db = %s failed", s)
}
bypass = !acceptDB(uint32(n))
}
if bypass {
cmd.nbypass.Incr()
continue
}
}
cmd.forward.Incr()
redis.MustEncode(writer, resp)
flushWriter(writer)
}
}()
for lstat := cmd.Stat(); ; {
time.Sleep(time.Second)
nstat := cmd.Stat()
var b bytes.Buffer
fmt.Fprintf(&b, "sync: ")
fmt.Fprintf(&b, " +forward=%-6d", nstat.forward-lstat.forward)
fmt.Fprintf(&b, " +nbypass=%-6d", nstat.nbypass-lstat.nbypass)
fmt.Fprintf(&b, " +nbytes=%d", nstat.wbytes-lstat.wbytes)
log.Info(b.String())
lstat = nstat
}
}
func (cmd *cmdSync) SendPSyncCmd(master, passwd string) (pipe.Reader, int64) {
c := openNetConn(master, passwd)
br := bufio.NewReaderSize(c, ReaderBufferSize)
bw := bufio.NewWriterSize(c, WriterBufferSize)
runid, offset, wait := sendPSyncFullsync(br, bw)
log.Infof("psync runid = %s offset = %d, fullsync", runid, offset)
var nsize int64
for nsize == 0 {
select {
case nsize = <-wait:
if nsize == 0 {
log.Info("+")
}
case <-time.After(time.Second):
log.Info("-")
}
}
piper, pipew := pipe.NewSize(ReaderBufferSize)
go func() {
defer pipew.Close()
p := make([]byte, 8192)
for rdbsize := int(nsize); rdbsize != 0; {
rdbsize -= iocopy(br, pipew, p, rdbsize)
}
for {
n, err := cmd.PSyncPipeCopy(c, br, bw, offset, pipew)
if err != nil {
log.PanicErrorf(err, "psync runid = %s, offset = %d, pipe is broken", runid, offset)
}
offset += n
for {
time.Sleep(time.Second)
c = openNetConnSoft(master, passwd)
if c != nil {
log.Infof("psync reopen connection, offset = %d", offset)
break
} else {
log.Infof("psync reopen connection, failed")
}
}
authPassword(c, passwd)
br = bufio.NewReaderSize(c, ReaderBufferSize)
bw = bufio.NewWriterSize(c, WriterBufferSize)
sendPSyncContinue(br, bw, runid, offset)
}
}()
return piper, nsize
}
func reinit() {
if testbl != nil {
testbl.Close()
testbl = nil
}
const path = "/tmp/testdb-rocksdb"
if err := os.RemoveAll(path); err != nil {
log.PanicErrorf(err, "remove '%s' failed", path)
} else {
conf := rocksdb.NewDefaultConfig()
if testdb, err := rocksdb.Open(path, conf, true, false); err != nil {
log.PanicError(err, "open rocksdb failed")
} else {
testbl = rpdb.New(testdb)
}
}
}
func init() {
const path = "/tmp/testdb2-rocksdb"
if err := os.RemoveAll(path); err != nil {
log.PanicErrorf(err, "remove '%s' failed", path)
} else {
conf := rocksdb.NewDefaultConfig()
if testdb, err := rocksdb.Open(path, conf, true, false); err != nil {
log.PanicError(err, "open rocksdb failed")
} else {
testbl2 = rpdb.New(testdb)
}
}
l, err := net.Listen("tcp", ":0")
if err != nil {
log.PanicError(err, "open listen port failed")
}
port = l.Addr().(*net.TCPAddr).Port
go func() {
server := redis.MustServer(&Handler{})
for {
c, err := l.Accept()
if err != nil {
log.PanicError(err, "accept socket failed")
}
go func() {
defer c.Close()
r, w := bufio.NewReader(c), bufio.NewWriter(c)
s := &fakeSession2{}
for {
if req, err := redis.Decode(r); err != nil {
return
} else {
if rsp, err := server.Dispatch(s, req); err != nil {
return
} else if rsp != nil {
if err := redis.Encode(w, rsp); err != nil {
return
}
}
}
}
}()
}
}()
}
func main() {
usage := `
Usage:
rpdb [--config=CONF] [--create|--repair] [--ncpu=N]
Options:
-n N, --ncpu=N set runtime.GOMAXPROCS to N
-c CONF, --config=CONF specify the config file
--create create if not exists
--repair repair database
`
d, err := docopt.Parse(usage, nil, true, "", false)
if err != nil {
log.PanicErrorf(err, "parse arguments failed")
}
if s, ok := d["--ncpu"].(string); ok && len(s) != 0 {
if n, err := strconv.ParseInt(s, 10, 64); err != nil {
log.PanicErrorf(err, "parse --ncpu failed")
} else if n <= 0 || n > 64 {
log.Panicf("parse --ncpu = %d, only accept [1,64]", n)
} else {
runtime.GOMAXPROCS(int(n))
}
}
args.config, _ = d["--config"].(string)
args.create, _ = d["--create"].(bool)
args.repair, _ = d["--repair"].(bool)
conf := &Config{
DBType: "rocksdb",
DBPath: "testdb-rocksdb",
LevelDB: leveldb.NewDefaultConfig(),
RocksDB: rocksdb.NewDefaultConfig(),
Service: service.NewDefaultConfig(),
}
if args.config != "" {
if err := conf.LoadFromFile(args.config); err != nil {
log.PanicErrorf(err, "load config failed")
}
}
log.Infof("load config\n%s\n\n", conf)
var db store.Database
switch t := strings.ToLower(conf.DBType); t {
default:
log.Panicf("unknown db type = '%s'", conf.DBType)
case "leveldb":
db, err = leveldb.Open(conf.DBPath, conf.LevelDB, args.create, args.repair)
case "rocksdb":
db, err = rocksdb.Open(conf.DBPath, conf.RocksDB, args.create, args.repair)
case "boltdb":
db, err = boltdb.Open(conf.DBPath, conf.BoltDB, args.create, args.repair)
}
if err != nil {
log.PanicErrorf(err, "open database failed")
}
bl := rpdb.New(db)
defer bl.Close()
if args.repair {
return
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
go func() {
for _ = range c {
log.Infof("interrupt and shutdown")
bl.Close()
os.Exit(0)
}
}()
if err := service.Serve(conf.Service, bl); err != nil {
log.ErrorErrorf(err, "service failed")
}
}
func main() {
usage := `
Usage:
redis-port decode [--ncpu=N] [--parallel=M] [--input=INPUT] [--output=OUTPUT]
redis-port restore [--ncpu=N] [--parallel=M] [--input=INPUT] --target=TARGET [--extra] [--faketime=FAKETIME] [--filterdb=DB]
redis-port dump [--ncpu=N] [--parallel=M] --from=MASTER [--output=OUTPUT] [--password=PASSWORD] [--extra]
redis-port sync [--ncpu=N] [--parallel=M] --from=MASTER --target=TARGET [--password=PASSWORD] [--sockfile=FILE [--filesize=SIZE]] [--filterdb=DB] [--psync]
Options:
-n N, --ncpu=N Set runtime.GOMAXPROCS to N.
-p M, --parallel=M Set the number of parallel routines to M.
-i INPUT, --input=INPUT Set input file, default is stdin ('/dev/stdin').
-o OUTPUT, --output=OUTPUT Set output file, default is stdout ('/dev/stdout').
-f MASTER, --from=MASTER Set host:port of master redis.
-t TARGET, --target=TARGET Set host:port of slave redis.
-P PASSWORD, --password=PASSWORD Set redis auth password.
--faketime=FAKETIME Set current system time to adjust key's expire time.
--sockfile=FILE Use FILE to as socket buffer, default is disabled.
--filesize=SIZE Set FILE size, default value is 1gb.
-e, --extra Set ture to send/receive following redis commands, default is false.
--filterdb=DB Filter db = DB, default is *.
--psync Use PSYNC command.
`
d, err := docopt.Parse(usage, nil, true, "", false)
if err != nil {
log.PanicError(err, "parse arguments failed")
}
if s, ok := d["--ncpu"].(string); ok && s != "" {
n, err := parseInt(s, 1, 1024)
if err != nil {
log.PanicErrorf(err, "parse --ncpu failed")
}
runtime.GOMAXPROCS(n)
}
ncpu := runtime.GOMAXPROCS(0)
if s, ok := d["--parallel"].(string); ok && s != "" {
n, err := parseInt(s, 1, 1024)
if err != nil {
log.PanicErrorf(err, "parse --parallel failed")
}
args.parallel = n
}
if ncpu > args.parallel {
args.parallel = ncpu
}
if args.parallel == 0 {
args.parallel = 4
}
args.input, _ = d["--input"].(string)
args.output, _ = d["--output"].(string)
args.from, _ = d["--from"].(string)
args.target, _ = d["--target"].(string)
args.passwd, _ = d["--password"].(string)
args.extra, _ = d["--extra"].(bool)
args.psync, _ = d["--psync"].(bool)
args.sockfile, _ = d["--sockfile"].(string)
if s, ok := d["--faketime"].(string); ok && s != "" {
switch s[0] {
case '-', '+':
d, err := time.ParseDuration(strings.ToLower(s))
if err != nil {
log.PanicError(err, "parse --faketime failed")
}
args.shift = d
case '@':
n, err := strconv.ParseInt(s[1:], 10, 64)
if err != nil {
log.PanicError(err, "parse --faketime failed")
}
args.shift = time.Duration(n*int64(time.Millisecond) - time.Now().UnixNano())
default:
t, err := time.Parse("2006-01-02 15:04:05", s)
if err != nil {
log.PanicError(err, "parse --faketime failed")
}
args.shift = time.Duration(t.UnixNano() - time.Now().UnixNano())
}
}
if s, ok := d["--filterdb"].(string); ok && s != "" && s != "*" {
n, err := parseInt(s, MinDB, MaxDB)
if err != nil {
log.PanicError(err, "parse --filterdb failed")
}
u := uint32(n)
acceptDB = func(db uint32) bool {
return db == u
}
}
if s, ok := d["--filesize"].(string); ok && s != "" {
if len(args.sockfile) == 0 {
log.Panic("please specify --sockfile first")
}
n, err := bytesize.Parse(s)
if err != nil {
log.PanicError(err, "parse --filesize failed")
}
if n <= 0 {
log.Panicf("parse --filesize = %d, invalid number", n)
}
args.filesize = n
} else {
args.filesize = bytesize.GB
}
log.Infof("set ncpu = %d, parallel = %d\n", ncpu, args.parallel)
switch {
case d["decode"].(bool):
new(cmdDecode).Main()
case d["restore"].(bool):
new(cmdRestore).Main()
case d["dump"].(bool):
new(cmdDump).Main()
case d["sync"].(bool):
new(cmdSync).Main()
}
}