Beispiel #1
0
func (cmd *cmdDump) Main() {
	from, output := args.from, args.output
	if len(from) == 0 {
		log.Panic("invalid argument: from")
	}
	if len(output) == 0 {
		output = "/dev/stdout"
	}

	log.Infof("dump from '%s' to '%s'\n", from, output)

	var dumpto io.WriteCloser
	if output != "/dev/stdout" {
		dumpto = openWriteFile(output)
		defer dumpto.Close()
	} else {
		dumpto = os.Stdout
	}

	master, nsize := cmd.SendCmd(from, args.passwd)
	defer master.Close()

	log.Infof("rdb file = %d\n", nsize)

	reader := bufio.NewReaderSize(master, ReaderBufferSize)
	writer := bufio.NewWriterSize(dumpto, WriterBufferSize)

	cmd.DumpRDBFile(reader, writer, nsize)

	if !args.extra {
		return
	}

	cmd.DumpCommand(reader, writer, nsize)
}
Beispiel #2
0
func (cmd *cmdRestore) Main() {
	input, target := args.input, args.target
	if len(target) == 0 {
		log.Panic("invalid argument: target")
	}
	if len(input) == 0 {
		input = "/dev/stdin"
	}

	log.Infof("restore from '%s' to '%s'\n", input, target)

	var readin io.ReadCloser
	var nsize int64
	if input != "/dev/stdin" {
		readin, nsize = openReadFile(input)
		defer readin.Close()
	} else {
		readin, nsize = os.Stdin, 0
	}

	reader := bufio.NewReaderSize(readin, ReaderBufferSize)

	cmd.RestoreRDBFile(reader, target, args.auth, nsize)

	if !args.extra {
		return
	}

	if nsize != 0 && nsize == cmd.rbytes.Get() {
		return
	}

	cmd.RestoreCommand(reader, target, args.auth)
}
Beispiel #3
0
func (cmd *cmdSync) Main() {
	from, target := args.from, args.target
	if len(from) == 0 {
		log.Panic("invalid argument: from")
	}
	if len(target) == 0 {
		log.Panic("invalid argument: target")
	}

	log.Infof("sync from '%s' to '%s'\n", from, target)

	var sockfile *os.File
	if len(args.sockfile) != 0 {
		sockfile = openReadWriteFile(args.sockfile)
		defer sockfile.Close()
	}

	var input io.ReadCloser
	var nsize int64
	if args.psync {
		input, nsize = cmd.SendPSyncCmd(from, args.passwd)
	} else {
		input, nsize = cmd.SendSyncCmd(from, args.passwd)
	}
	defer input.Close()

	log.Infof("rdb file = %d\n", nsize)

	if sockfile != nil {
		r, w := pipe.NewFilePipe(int(args.filesize), sockfile)
		defer r.Close()
		go func(r io.Reader) {
			defer w.Close()
			p := make([]byte, ReaderBufferSize)
			for {
				iocopy(r, w, p, len(p))
			}
		}(input)
		input = r
	}

	reader := bufio.NewReaderSize(input, ReaderBufferSize)

	cmd.SyncRDBFile(reader, target, args.auth, nsize)
	cmd.SyncCommand(reader, target, args.auth)
}
Beispiel #4
0
func authPassword(c net.Conn, passwd string) {
	if passwd == "" {
		return
	}
	_, err := c.Write(redis.MustEncodeToBytes(redis.NewCommand("auth", passwd)))
	if err != nil {
		log.PanicError(errors.Trace(err), "write auth command failed")
	}
	var b = make([]byte, 5)
	if _, err := io.ReadFull(c, b); err != nil {
		log.PanicError(errors.Trace(err), "read auth response failed")
	}
	if strings.ToUpper(string(b)) != "+OK\r\n" {
		log.Panic("auth failed")
	}
}
Beispiel #5
0
func main() {
	usage := `
Usage:
	redis-port decode   [--ncpu=N]  [--parallel=M]  [--input=INPUT]  [--output=OUTPUT]
	redis-port restore  [--ncpu=N]  [--parallel=M]  [--input=INPUT]   --target=TARGET   [--auth=AUTH]  [--extra] [--faketime=FAKETIME]  [--filterdb=DB] 
                        [--filterkeys=keys] [--skipkeys=keys] [--restorecmd=slotsrestore] [--aggregatetype=type] [--aggregatekeys=keys] [--aggregateTargetKey=key] 
	redis-port dump     [--ncpu=N]  [--parallel=M]   --from=MASTER   [--password=PASSWORD]  [--output=OUTPUT]  [--extra]
	redis-port sync     [--ncpu=N]  [--parallel=M]   --from=MASTER   [--password=PASSWORD]   --target=TARGET   [--auth=AUTH]  [--sockfile=FILE [--filesize=SIZE]] [--filterdb=DB] [--psync] [--force]
                        [--filterkeys=keys] [--skipkeys=keys] [--restorecmd=slotsrestore] [--aggregatetype=type] [--aggregatekeys=keys] [--aggregateTargetKey=key] [--set2sortedkeys=keys] [--sorted2setkeys=keys]

Options:
	-n N, --ncpu=N                    Set runtime.GOMAXPROCS to N.
	-p M, --parallel=M                Set the number of parallel routines to M.
	-i INPUT, --input=INPUT           Set input file, default is stdin ('/dev/stdin').
	-o OUTPUT, --output=OUTPUT        Set output file, default is stdout ('/dev/stdout').
	-f MASTER, --from=MASTER          Set host:port of master redis.
	-t TARGET, --target=TARGET        Set host:port of slave redis.
	-P PASSWORD, --password=PASSWORD  Set redis auth password.
	-A AUTH, --auth=AUTH              Set auth password for target.
	--faketime=FAKETIME               Set current system time to adjust key's expire time.
	--sockfile=FILE                   Use FILE to as socket buffer, default is disabled.
	--filesize=SIZE                   Set FILE size, default value is 1gb.
	-e, --extra                       Set ture to send/receive following redis commands, default is false.
	--filterdb=DB                     Filter db = DB, default is *.
    --filterkeys=keys                 Filter key in keys, keys is seperated by comma and supports regular expression.
    --skipkeys=keys                   Skip key in keys, keys is seperated by comma and supports regular expression.
	--restorecmd=slotsrestore		  Restore command, slotsrestore for codis, restore for redis, if the from and target server are the same, use '--restorecmd=del' will delete the keys, togegher with
                                      filterkeys, it will delete the keys filtered in the server. 
    --aggregatetype=type              Aggregate type: list or set.
    --aggregatekeys=keys              Aggregate key in keys, keys is seperated by comma and supports regular expression.
    --aggregateTargetKey=key          Target key for aggregating.
    --set2sortedkeys=keys             Convert set key in keys to sorted set, keys is seperated by comma and supports regular expression.
    --sorted2setkeys=keys             Convert sorted set key in keys to set, keys is seperated by comma and supports regular expression.
	--psync                           Use PSYNC command.
	--force                           Use force, do not need to enter "yes", you mustn't use it ,unless you konw what you are doing.
`
	d, err := docopt.Parse(usage, nil, true, "", false)
	if err != nil {
		log.PanicError(err, "parse arguments failed")
	}

	if s, ok := d["--ncpu"].(string); ok && s != "" {
		n, err := parseInt(s, 1, 1024)
		if err != nil {
			log.PanicErrorf(err, "parse --ncpu failed")
		}
		runtime.GOMAXPROCS(n)
	}
	ncpu := runtime.GOMAXPROCS(0)

	if s, ok := d["--parallel"].(string); ok && s != "" {
		n, err := parseInt(s, 1, 1024)
		if err != nil {
			log.PanicErrorf(err, "parse --parallel failed")
		}
		args.parallel = n
	}
	if ncpu > args.parallel {
		args.parallel = ncpu
	}
	if args.parallel == 0 {
		args.parallel = 4
	}

	args.input, _ = d["--input"].(string)
	args.output, _ = d["--output"].(string)

	args.from, _ = d["--from"].(string)
	args.passwd, _ = d["--password"].(string)
	args.auth, _ = d["--auth"].(string)
	args.target, _ = d["--target"].(string)

	args.extra, _ = d["--extra"].(bool)
	args.psync, _ = d["--psync"].(bool)
	args.force, _ = d["--force"].(bool)
	args.sockfile, _ = d["--sockfile"].(string)

	var input string
	for {
		if args.force {
			goto CONTINUE
		}
		fmt.Printf("Are you sure to continue (yes/no)?\n")
		_, err := fmt.Scanf("%s\r\n", &input)
		if err != nil {
			log.PanicError(err, "input yes/no err")
		}
		switch input {
		default:
			fmt.Printf("Input the wrong value, you should input yes or no")
			continue
		case "yes":
			goto CONTINUE
		case "no":
			os.Exit(1)
		}
	}

CONTINUE:

	if s, ok := d["--faketime"].(string); ok && s != "" {
		switch s[0] {
		case '-', '+':
			d, err := time.ParseDuration(strings.ToLower(s))
			if err != nil {
				log.PanicError(err, "parse --faketime failed")
			}
			args.shift = d
		case '@':
			n, err := strconv.ParseInt(s[1:], 10, 64)
			if err != nil {
				log.PanicError(err, "parse --faketime failed")
			}
			args.shift = time.Duration(n*int64(time.Millisecond) - time.Now().UnixNano())
		default:
			t, err := time.Parse("2006-01-02 15:04:05", s)
			if err != nil {
				log.PanicError(err, "parse --faketime failed")
			}
			args.shift = time.Duration(t.UnixNano() - time.Now().UnixNano())
		}
	}

	if s, ok := d["--filterdb"].(string); ok && s != "" && s != "*" {
		n, err := parseInt(s, MinDB, MaxDB)
		if err != nil {
			log.PanicError(err, "parse --filterdb failed")
		}
		u := uint32(n)
		acceptDB = func(db uint32) bool {
			return db == u
		}
	}

	if s, ok := d["--filterkeys"].(string); ok && s != "" && s != "*" {
		keys := strings.Split(s, ",")

		keyRegexps := make([]*regexp.Regexp, len(keys))
		for i, key := range keys {
			keyRegexps[i], err = regexp.Compile(key)
			if err != nil {
				log.PanicError(err, "parse --filterkeys failed")
			}
		}

		acceptKey = func(key []byte) bool {
			for _, reg := range keyRegexps {
				if reg.Match(key) {
					return true
				}
			}

			return false
		}
	}

	if s, ok := d["--skipkeys"].(string); ok && s != "" && s != "*" {
		keys := strings.Split(s, ",")

		keyRegexps := make([]*regexp.Regexp, len(keys))
		for i, key := range keys {
			keyRegexps[i], err = regexp.Compile(key)
			if err != nil {
				log.PanicError(err, "parse --skipkeys failed")
			}
		}

		skipKey = func(key []byte) bool {
			for _, reg := range keyRegexps {
				if reg.Match(key) {
					return true
				}
			}

			return false
		}
	}

	if s, ok := d["--restorecmd"].(string); ok && s != "" {
		restoreCmd = strings.TrimSpace(s)
	}

	if s, ok := d["--aggregatekeys"].(string); ok && s != "" && s != "*" {
		keys := strings.Split(s, ",")

		keyRegexps := make([]*regexp.Regexp, len(keys))
		for i, key := range keys {
			keyRegexps[i], err = regexp.Compile(key)
			if err != nil {
				log.PanicError(err, "parse --aggregatekeys failed")
			}
		}

		aggregateKey = func(key []byte) bool {
			for _, reg := range keyRegexps {
				if reg.Match(key) {
					return true
				}
			}

			return false
		}
	}

	if s, ok := d["--aggregateTargetKey"].(string); ok && s != "" {
		aggregateTarget = s
	}

	if s, ok := d["--aggregatetype"].(string); ok && s != "" {
		aggregateType = s
		switch s {
		default:
			aggregateCmd = "lpush"
		case "list":
			aggregateCmd = "lpush"
		case "set":
			aggregateCmd = "sadd"
		}
	}

	if s, ok := d["--set2sortedkeys"].(string); ok && s != "" && s != "*" {
		keys := strings.Split(s, ",")

		keyRegexps := make([]*regexp.Regexp, len(keys))
		for i, key := range keys {
			keyRegexps[i], err = regexp.Compile(key)
			if err != nil {
				log.PanicError(err, "parse --set2sortedkeys failed")
			}
		}

		set2sortedKey = func(key []byte) bool {
			for _, reg := range keyRegexps {
				if reg.Match(key) {
					return true
				}
			}

			return false
		}
	}

	if s, ok := d["--sorted2setkeys"].(string); ok && s != "" && s != "*" {
		keys := strings.Split(s, ",")

		keyRegexps := make([]*regexp.Regexp, len(keys))
		for i, key := range keys {
			keyRegexps[i], err = regexp.Compile(key)
			if err != nil {
				log.PanicError(err, "parse --sorted2setkeys failed")
			}
		}

		sorted2setKey = func(key []byte) bool {
			for _, reg := range keyRegexps {
				if reg.Match(key) {
					return true
				}
			}

			return false
		}
	}

	if s, ok := d["--filesize"].(string); ok && s != "" {
		if len(args.sockfile) == 0 {
			log.Panic("please specify --sockfile first")
		}
		n, err := bytesize.Parse(s)
		if err != nil {
			log.PanicError(err, "parse --filesize failed")
		}
		if n <= 0 {
			log.Panicf("parse --filesize = %d, invalid number", n)
		}
		args.filesize = n
	} else {
		args.filesize = bytesize.GB
	}

	log.Infof("set ncpu = %d, parallel = %d\n", ncpu, args.parallel)

	switch {
	case d["decode"].(bool):
		new(cmdDecode).Main()
	case d["restore"].(bool):
		new(cmdRestore).Main()
	case d["dump"].(bool):
		new(cmdDump).Main()
	case d["sync"].(bool):
		new(cmdSync).Main()
	}
}