Exemple #1
0
func (cmd *cmdRestore) RestoreRDBFile(reader *bufio.Reader, target, passwd string, nsize int64) {
	pipe := newRDBLoader(reader, &cmd.rbytes, args.parallel*32)
	wait := make(chan struct{})
	go func() {
		defer close(wait)
		group := make(chan int, args.parallel)
		for i := 0; i < cap(group); i++ {
			go func() {
				defer func() {
					group <- 0
				}()
				c := openRedisConn(target, passwd)
				defer c.Close()
				var lastdb uint32 = 0
				for e := range pipe {
					if !acceptDB(e.DB) || !acceptKey(e.Key) {
						cmd.ignore.Incr()
					} else if skipKey(e.Key) {
						log.Warnf("restore skip key: %s", e.Key)
						cmd.ignore.Incr()
					} else {
						cmd.nentry.Incr()
						if e.DB != lastdb {
							lastdb = e.DB
							selectDB(c, lastdb)
						}
						restoreRdbEntry(c, e)
					}
				}
			}()
		}
		for i := 0; i < cap(group); i++ {
			<-group
		}
	}()

	for done := false; !done; {
		select {
		case <-wait:
			done = true
		case <-time.After(time.Second):
		}
		stat := cmd.Stat()
		var b bytes.Buffer
		if nsize != 0 {
			fmt.Fprintf(&b, "total = %d - %12d [%3d%%]", nsize, stat.rbytes, 100*stat.rbytes/nsize)
		} else {
			fmt.Fprintf(&b, "total = %12d", stat.rbytes)
		}
		fmt.Fprintf(&b, "  entry=%-12d", stat.nentry)
		if stat.ignore != 0 {
			fmt.Fprintf(&b, "  ignore=%-12d", stat.ignore)
		}
		log.Info(b.String())
	}
	log.Info("restore: rdb done")
}
Exemple #2
0
func (cmd *cmdSync) SendPSyncCmd(master, passwd string) (pipe.Reader, int64) {
	c := openNetConn(master, passwd)
	br := bufio.NewReaderSize(c, ReaderBufferSize)
	bw := bufio.NewWriterSize(c, WriterBufferSize)

	runid, offset, wait := sendPSyncFullsync(br, bw)
	log.Infof("psync runid = %s offset = %d, fullsync", runid, offset)

	var nsize int64
	for nsize == 0 {
		select {
		case nsize = <-wait:
			if nsize == 0 {
				log.Info("+")
			}
		case <-time.After(time.Second):
			log.Info("-")
		}
	}

	piper, pipew := pipe.NewSize(ReaderBufferSize)

	go func() {
		defer pipew.Close()
		p := make([]byte, 8192)
		for rdbsize := int(nsize); rdbsize != 0; {
			rdbsize -= iocopy(br, pipew, p, rdbsize)
		}
		for {
			n, err := cmd.PSyncPipeCopy(c, br, bw, offset, pipew)
			if err != nil {
				log.PanicErrorf(err, "psync runid = %s, offset = %d, pipe is broken", runid, offset)
			}
			offset += n
			for {
				time.Sleep(time.Second)
				c = openNetConnSoft(master, passwd)
				if c != nil {
					log.Infof("psync reopen connection, offset = %d", offset)
					break
				} else {
					log.Infof("psync reopen connection, failed")
				}
			}
			authPassword(c, passwd)
			br = bufio.NewReaderSize(c, ReaderBufferSize)
			bw = bufio.NewWriterSize(c, WriterBufferSize)
			sendPSyncContinue(br, bw, runid, offset)
		}
	}()
	return piper, nsize
}
Exemple #3
0
func (cmd *cmdSync) SendSyncCmd(master, passwd string) (net.Conn, int64) {
	c, wait := openSyncConn(master, passwd)
	for {
		select {
		case nsize := <-wait:
			if nsize == 0 {
				log.Info("+")
			} else {
				return c, nsize
			}
		case <-time.After(time.Second):
			log.Info("-")
		}
	}
}
Exemple #4
0
func (cmd *cmdDump) SendCmd(master, passwd string) (net.Conn, int64) {
	c, wait := openSyncConn(master, passwd)
	var nsize int64
	for nsize == 0 {
		select {
		case nsize = <-wait:
			if nsize == 0 {
				log.Info("+")
			}
		case <-time.After(time.Second):
			log.Info("-")
		}
	}
	return c, nsize
}
Exemple #5
0
func (cmd *cmdDump) DumpRDBFile(reader *bufio.Reader, writer *bufio.Writer, nsize int64) {
	var nread atomic2.Int64
	wait := make(chan struct{})
	go func() {
		defer close(wait)
		p := make([]byte, WriterBufferSize)
		for nsize != nread.Get() {
			nstep := int(nsize - nread.Get())
			ncopy := int64(iocopy(reader, writer, p, nstep))
			nread.Add(ncopy)
			flushWriter(writer)
		}
	}()

	for done := false; !done; {
		select {
		case <-wait:
			done = true
		case <-time.After(time.Second):
		}
		n := nread.Get()
		p := 100 * n / nsize
		log.Infof("total = %d - %12d [%3d%%]\n", nsize, n, p)
	}
	log.Info("dump: rdb done")
}
Exemple #6
0
func (cmd *cmdDecode) Main() {
	input, output := args.input, args.output
	if len(input) == 0 {
		input = "/dev/stdin"
	}
	if len(output) == 0 {
		output = "/dev/stdout"
	}

	log.Infof("decode from '%s' to '%s'\n", input, output)

	var readin io.ReadCloser
	var nsize int64
	if input != "/dev/stdin" {
		readin, nsize = openReadFile(input)
		defer readin.Close()
	} else {
		readin, nsize = os.Stdin, 0
	}

	var saveto io.WriteCloser
	if output != "/dev/stdout" {
		saveto = openWriteFile(output)
		defer saveto.Close()
	} else {
		saveto = os.Stdout
	}

	reader := bufio.NewReaderSize(readin, ReaderBufferSize)
	writer := bufio.NewWriterSize(saveto, WriterBufferSize)

	ipipe := newRDBLoader(reader, &cmd.rbytes, args.parallel*32)
	opipe := make(chan string, cap(ipipe))

	go func() {
		defer close(opipe)
		group := make(chan int, args.parallel)
		for i := 0; i < cap(group); i++ {
			go func() {
				defer func() {
					group <- 0
				}()
				cmd.decoderMain(ipipe, opipe)
			}()
		}
		for i := 0; i < cap(group); i++ {
			<-group
		}
	}()

	wait := make(chan struct{})
	go func() {
		defer close(wait)
		for s := range opipe {
			cmd.wbytes.Add(int64(len(s)))
			if _, err := writer.WriteString(s); err != nil {
				log.PanicError(err, "write string failed")
			}
			flushWriter(writer)
		}
	}()

	for done := false; !done; {
		select {
		case <-wait:
			done = true
		case <-time.After(time.Second):
		}
		stat := cmd.Stat()
		var b bytes.Buffer
		fmt.Fprintf(&b, "decode: ")
		if nsize != 0 {
			fmt.Fprintf(&b, "total = %d - %12d [%3d%%]", nsize, stat.rbytes, 100*stat.rbytes/nsize)
		} else {
			fmt.Fprintf(&b, "total = %12d", stat.rbytes)
		}
		fmt.Fprintf(&b, "  write=%-12d", stat.wbytes)
		fmt.Fprintf(&b, "  entry=%-12d", stat.nentry)
		log.Info(b.String())
	}
	log.Info("decode: done")
}
Exemple #7
0
func (cmd *cmdSync) SyncCommand(reader *bufio.Reader, target, passwd string) {
	c := openNetConn(target, passwd)
	defer c.Close()

	cr := openRedisConn(target, passwd)
	defer cr.Close()

	writer := bufio.NewWriterSize(stats.NewCountWriter(c, &cmd.wbytes), WriterBufferSize)
	defer flushWriter(writer)

	go func() {
		p := make([]byte, ReaderBufferSize)
		for {
			iocopy(c, ioutil.Discard, p, len(p))
		}
	}()

	go func() {
		var bypass bool = false
		for {
			resp := redis.MustDecode(reader)
			if scmd, args, err := redis.ParseArgs(resp); err != nil {
				log.PanicError(err, "parse command arguments failed")
			} else if scmd != "ping" {
				if scmd == "select" {
					if len(args) != 1 {
						log.Panicf("select command len(args) = %d", len(args))
					}
					s := string(args[0])
					n, err := parseInt(s, MinDB, MaxDB)
					if err != nil {
						log.PanicErrorf(err, "parse db = %s failed", s)
					}
					bypass = !acceptDB(uint32(n))
				}

				if bypass || (len(args) > 0 && !acceptKey(args[0])) {
					cmd.nbypass.Incr()
					continue
				}

				if skipKey(args[0]) {
					log.Warnf("skip key: %s", args[0])
					cmd.ignore.Incr()
					continue
				}

				if aggregateKey(args[0]) && ((scmd == "lpush") || (scmd == "LPUSH")) {
					log.Infof("Aggregate Key %s", args[0])
					for i := 1; i < len(args); i++ {
						_, err := cr.Do(aggregateCmd, aggregateTarget, args[i])
						if err != nil {
							log.Warnf("SyncAggregate err at: %s %s", args[0], args[i])
							//log.PanicError(err, "sync aggregate error")
						}
					}
				}

				// set 2 sorted set in sync command
				if set2sortedKey(args[0]) {
					switch scmd {
					default:
						log.Panicf("set2sorted operate %s on key %s err", scmd, args[0])
					case "sadd":
						for i := 1; i < len(args); i++ {
							_, err := cr.Do("zadd", args[0], 1, args[i])
							if err != nil {
								log.PanicErrorf(err, "set2sorted zadd %s 1 %s", args[0], args[i])
							}
						}
					case "srem":
						for i := 1; i < len(args); i++ {
							_, err := cr.Do("zrem", args[0], args[i])
							if err != nil {
								log.PanicErrorf(err, "set2sorted zrem %s %s", args[0], args[i])
							}
						}
					case "del":
						_, err := cr.Do("del", args[0])
						if err != nil {
							log.PanicErrorf(err, "set2sorted del %s", args[0])
						}
					}
					continue
				}

				if sorted2setKey(args[0]) {
					switch scmd {
					default:
						log.Panicf("sorted2set operate %s on key %s err", scmd, args[0])
					case "zadd":
						for i := 1; i < len(args); i++ {
							if string(args[i]) != "1" {
								_, err := cr.Do("sadd", args[0], args[i])
								if err != nil {
									log.PanicErrorf(err, "sorted2set sadd %s %s", args[0], args[i])
								}
							} else {
								continue
							}
						}
					case "zrem":
						for i := 1; i < len(args); i++ {
							_, err := cr.Do("srem", args[0], args[i])
							if err != nil {
								log.PanicErrorf(err, "sorted2set srem %s %s", args[0], args[i])
							}
						}
					case "del":
						_, err := cr.Do("del", args[0])
						if err != nil {
							log.PanicErrorf(err, "sorted2set del %s", args[0])
						}
					}
					continue
				}

				// Some commands like MSET may have multi keys, but we only use
				// first for filter
			}
			cmd.forward.Incr()
			redis.MustEncode(writer, resp)
			flushWriter(writer)
		}
	}()

	for lstat := cmd.Stat(); ; {
		time.Sleep(time.Second)
		nstat := cmd.Stat()
		var b bytes.Buffer
		fmt.Fprintf(&b, "sync: ")
		fmt.Fprintf(&b, " +forward=%-6d", nstat.forward-lstat.forward)
		fmt.Fprintf(&b, " +nbypass=%-6d", nstat.nbypass-lstat.nbypass)
		fmt.Fprintf(&b, " +nbytes=%d", nstat.wbytes-lstat.wbytes)
		log.Info(b.String())
		lstat = nstat
	}
}
Exemple #8
0
func (cmd *cmdRestore) RestoreCommand(reader *bufio.Reader, target, passwd string) {
	c := openNetConn(target, passwd)
	defer c.Close()

	writer := bufio.NewWriterSize(c, WriterBufferSize)
	defer flushWriter(writer)

	go func() {
		p := make([]byte, ReaderBufferSize)
		for {
			iocopy(c, ioutil.Discard, p, len(p))
		}
	}()

	go func() {
		var bypass bool = false
		for {
			resp := redis.MustDecode(reader)
			if scmd, args, err := redis.ParseArgs(resp); err != nil {
				log.PanicError(err, "parse command arguments failed")
			} else if scmd != "ping" {
				if scmd == "select" {
					if len(args) != 1 {
						log.Panicf("select command len(args) = %d", len(args))
					}
					s := string(args[0])
					n, err := parseInt(s, MinDB, MaxDB)
					if err != nil {
						log.PanicErrorf(err, "parse db = %s failed", s)
					}
					bypass = !acceptDB(uint32(n))
				}

				if skipKey(args[0]) {
					log.Warnf("skip key: %s", args[0])
					cmd.ignore.Incr()
					continue
				}

				if bypass || (len(args) > 0 && !acceptKey(args[0])) {
					cmd.nbypass.Incr()
					continue
				}

				// added for aggregating list or set
				if aggregateKey(args[0]) {
					cr := openRedisConn(target, passwd)
					defer cr.Close()
					for i := 1; i < len(args); i++ {
						_, err := cr.Do(aggregateCmd, aggregateTarget, args[i])
						if err != nil {
							log.PanicError(err, "restore aggregate error")
						}
					}
				}
			}
			cmd.forward.Incr()
			redis.MustEncode(writer, resp)
			flushWriter(writer)
		}
	}()

	for lstat := cmd.Stat(); ; {
		time.Sleep(time.Second)
		nstat := cmd.Stat()
		var b bytes.Buffer
		fmt.Fprintf(&b, "restore: ")
		fmt.Fprintf(&b, " +forward=%-6d", nstat.forward-lstat.forward)
		fmt.Fprintf(&b, " +nbypass=%-6d", nstat.nbypass-lstat.nbypass)
		log.Info(b.String())
		lstat = nstat
	}
}