Ejemplo n.º 1
0
//获取所有任务的任务列表
func FetchTaskList() (int, *list.List) {
	var stmtIns *sql.Stmt
	var err error
	stmtIns, err = db.Prepare("select task_id, create_time from rdb_parse_task")

	if err != nil {
		logs.Log("fetch task list error in dbutil db.Prepare : " + err.Error())
		return -1, list.New()
	}

	defer stmtIns.Close()

	rows, errs := stmtIns.Query()

	if errs != nil {
		logs.Log("fetch task list error in dbutil stmtIns.Query : " + errs.Error())
		return -2, list.New()
	}

	taskList := list.New()

	var taskId string
	var createTime string
	for rows.Next() {
		var task Task
		rows.Scan(&taskId, &createTime)
		task.TaskId = taskId
		task.CreateTime = createTime
		taskList.PushBack(task)
	}
	return 1, taskList
}
Ejemplo n.º 2
0
//根据taskId和type获取分析结果
func FetchResult(taskId string, types string, limit int, offset int) []string {
	result := make([]string, 0)
	stmtIns, err := db.Prepare("select content from rdb_anly_result where task_id = ? and type = ? order by id desc limit ? offset ? ")

	if err != nil {
		logs.Log("fetch result error in dbutil db.Prepare!" + " : " + err.Error())
		return result
	}

	defer stmtIns.Close()

	rows, errs := stmtIns.Query(taskId, types, limit, offset)
	if errs != nil {
		logs.Log("fetch result error in dbutil stmtIns.Query!" + " : " + errs.Error())
		return result
	}

	var content string

	for rows.Next() {
		rows.Scan(&content)
		result = append(result, content)
	}
	return result
}
Ejemplo n.º 3
0
//删除任务,以及任务的分析结果
func DeleteTask(list *list.List) bool {

	stmtInsTask, err := db.Prepare("delete from rdb_parse_task where task_id = ?")

	if err != nil {
		logs.Log("delete task stmtInsTask error in dbutil : " + err.Error())
		return false
	}

	defer stmtInsTask.Close()

	stmtResult, err := db.Prepare("delete from rdb_anly_result where task_id = ?")
	if err != nil {
		logs.Log("delete dask stmtResult error in dbutil : " + err.Error())
		return false
	}

	defer stmtResult.Close()

	for e := list.Front(); e != nil; e = e.Next() {
		taskId, ok := e.Value.(string)
		if ok {
			stmtInsTask.Exec(taskId)
			stmtResult.Exec(taskId)
		} else {
			logs.Log("type error!")
		}
	}
	return true
}
Ejemplo n.º 4
0
//管道读取
func pipelineAnalyze(inputFile string, param *Parameters) {
	file, err := os.Open(inputFile)

	if err != nil && err != io.EOF {
		logs.Log("file open error in parseRdb pipelineAnalyze : " + err.Error())
		return
	}

	defer file.Close()

	//首先读取一定数量的字节
	buff := make([]byte, (*param).SliceContent)

	n, err := file.Read(buff)
	if err != nil && err != io.EOF {
		logs.Log("file read error in parseRdb pipelineAnalyze : " + err.Error())
		return
	}

	//解析
	var pbuff = &buff
	if n != 0 {
		(*param).OffsetBytes = (*param).OffsetBytes + int64(n)
		parseRedisRdb(pbuff, inputFile, param)
		(*param).FixedInfo = true
	}
	//为了防止文件没有全部读取,再次循环读取,理论上这个地方应该用不到
	for {
		n, err := file.ReadAt(buff, (*param).OffsetBytes)

		if err != nil && err != io.EOF {
			logs.Log("file read error in parseRdb pipelineAnalyze for loop : " + err.Error())
			return
		}

		defer file.Close()

		if n == 0 {
			break
		} else {
			(*param).SliceStart = 0
			(*param).SliceEnd = (*param).SliceStart + 1
			(*param).OffsetBytes = (*param).OffsetBytes + int64(n)
			parseRedisRdb(pbuff, inputFile, param)
			(*param).FixedInfo = true
		}
	}
}
Ejemplo n.º 5
0
/**
这个是解析的入口
pwflag 表示是打印还是写入到文件
0 打印
1 写入到文件
2 写入到数据库
filename 表示要解析的文件路径
*/
func Start(inputFile string, filterLength int, filterKey string) Parameters {

	//读取管道的大小
	conf := config.SetConfig("important.properties")
	sliceContent, _ := strconv.ParseUint(conf.GetValue("pipeline", "size"), 10, 64)
	if sliceContent < 16 {
		sliceContent = 16
		logs.Log("sliceContent < 16")
	}

	var heapIni bool = false

	var param Parameters
	param.Success = false
	param.SliceStart = 0
	param.SliceEnd = 0
	param.Step = []uint64{5, 4}
	param.FileFlag = false
	param.SliceContent = sliceContent
	param.FixedInfo = false
	param.OffsetBytes = 0
	param.Dbnum = -1
	param.ResultMap = make(map[string]int)
	param.ValueFilterLength = filterLength
	param.FilterKey = filterKey
	param.ValueLengthFilterList = list.New()
	param.KeyFilterMap = make(map[string]int)
	param.Heap = &sorting.Heap{}
	param.HeapInit = &heapIni
	pipelineAnalyze(inputFile, &param)

	return param
}
Ejemplo n.º 6
0
//更新任务状态
func UpdateTaskStatus(status int, taskId string) bool {
	stmtIns, err := db.Prepare("update rdb_parse_task set status = ? where task_id = ?")

	if err != nil {
		logs.Log("update task status error in dbutil db.Prepare : " + err.Error())
		return false
	}
	defer stmtIns.Close()

	_, errs := stmtIns.Exec(status, taskId)
	if errs != nil {
		logs.Log("update task status error in dbutil stmtIns.Exec : " + errs.Error())
		return false
	}
	return true
}
Ejemplo n.º 7
0
//存储分析结果
func Store(content, types, taskId string) {

	stmtIns, err := db.Prepare("insert into rdb_anly_result(content, type, task_id, create_time) values (?,?,?,now())")

	if err != nil {
		logs.Log("store error in dbutil db.Prepare : " + err.Error())
		return
	}
	defer stmtIns.Close()

	_, errs := stmtIns.Exec(content, types, taskId)

	if errs != nil {
		logs.Log("store error in dbutil stmtIns.Exec : " + errs.Error())
		return
	}
}
Ejemplo n.º 8
0
//向任务表中插入任务
func InsertTask(host, port, taskId string, filterLength, priority int, filter_key string) bool {

	stmtIns, err := db.Prepare("insert into rdb_parse_task (host, port, filter_length, filter_key, task_id, priority, status, create_time) values (?,?,?,?,?,?,0,now())")

	if err != nil {
		logs.Log("insert task error in dbutil db.Prepare : " + err.Error())
		return false
	}
	defer stmtIns.Close()

	_, errs := stmtIns.Exec(host, port, filterLength, filter_key, taskId, priority)
	if errs != nil {
		logs.Log("insert task error in dbutil stmtIns.Exec : " + errs.Error())
		return false
	}
	return true
}
Ejemplo n.º 9
0
//解析程序
func parseTask(host string, port string, filterLength int, filterKey string, taskId string) {

	//获得机器的ip地址
	ip := utils.GetIpAddress()
	if ip == "unknown" {
		ip = utils.GetIpAddress2()
	}
	logs.Log("machine : " + ip + " processing task : " + taskId)
	//获取rdb文件
	rdbFile := rdbLocation + taskId + ".rdb"

	//如果文件存在则删除
	isExist := exists(rdbFile)
	if isExist == true {
		err := os.Remove(rdbFile)
		if err != nil {
			logs.Log("parseTask remove rdbFile error : " + err.Error())
			return
		}
	}

	flag := syncRdb(host, port, cliLocation, rdbFile, taskId)
	if flag == 1 {
		//还要判断解析是否成功了
		status := start(rdbFile, taskId, filterLength, filterKey)
		if status == true {
			dbutil.UpdateTaskStatus(1, taskId)
		} else {
			dbutil.UpdateTaskStatus(3, taskId)
		}
		//删除文件
		dropFile(rdbFile, taskId)
	} else {
		switch flag {
		case 2: //同步进程失效,文件大小无变化
			logs.Log("task " + taskId + " sync process error, file size no change!")
			dbutil.UpdateTaskStatus(2, taskId)
		case 4: //获得redis密码错误
			dbutil.UpdateTaskStatus(4, taskId)
			logs.Log("task " + taskId + " get remote redis server login password error!")
		}
		//删除文件
		dropFile(rdbFile, taskId)
	}
}
Ejemplo n.º 10
0
/**
net 环境下能够获得访问网络的ip
*/
func GetIpAddress() string {
	conn, err := net.Dial("udp", "www.baidu.com:80")
	if err != nil {
		logs.Log(err.Error())
		return "unknown"
	}
	defer conn.Close()
	return strings.Split(conn.LocalAddr().String(), ":")[0]
}
Ejemplo n.º 11
0
//解析操作完成之后删除文件
func dropFile(file, taskId string) {
	//首先判断存在文件吗
	isExist := exists(file)
	if isExist {
		err := os.Remove(file)
		if err != nil {
			logs.Log("task " + taskId + " remove file error : " + err.Error() + " --- " + file)
		}
	}
}
Ejemplo n.º 12
0
//根据任务id获取单个任务
func FetchTaskById(task_id string) (int, *list.List) {
	stmtIns, err := db.Prepare("select host, port, filter_length, filter_key, task_id, priority, status, create_time from rdb_parse_task where task_id = ?")

	if err != nil {
		logs.Log("fetch task by id error in dbutil db.Prepare : " + err.Error())
		return -1, list.New()
	}

	defer stmtIns.Close()

	rows, errs := stmtIns.Query(task_id)

	if errs != nil {
		logs.Log("fetch task by id error in dbutil stmtIns.Query : " + errs.Error())
		return -2, list.New()
	}

	taskList := list.New()

	var host string
	var port string
	var filterLength int
	var filterKey string
	var taskId string
	var priority int
	var status int
	var createTime string
	for rows.Next() {
		var task Task
		rows.Scan(&host, &port, &filterLength, &filterKey, &taskId, &priority, &status, &createTime)
		task.Host = host
		task.Port = port
		task.FilterLength = filterLength
		task.TaskId = taskId
		task.Priority = priority
		task.Status = status
		task.CreateTime = createTime
		task.FilterKey = filterKey
		taskList.PushBack(task)
	}
	return 1, taskList
}
Ejemplo n.º 13
0
//存储堆排序结果
func StoreSortingResult(h *sorting.Heap, taskId string) {
	for h.Len() > 0 {
		element := heap.Pop(h).(sorting.Element)
		b, err := json.Marshal(element)
		if err != nil {
			logs.Log("json marshal error in StoreValueTypeStat : " + err.Error())
			return
		}
		dbutil.Store(string(b), "2", taskId)
	}
}
Ejemplo n.º 14
0
//为了部署多个实例的需要,修改了startParse
func startParse() {

	//每次只取一个任务,这样保证当存在多个服务时,每个机器都能获得任务
	logs.Log("starting parse!")
	for {
		mark, task := dbutil.FetchNewTask()

		if mark == 1 {
			if task.TaskId == "" {
				time.Sleep(time.Second * 10)
				continue
			} else {
				parseTask(task.Host, task.Port, task.FilterLength, task.FilterKey, task.TaskId)
			}
		} else if mark == -5 {
			logs.Log("task " + task.TaskId + " processing by another machine! ")
		} else {
			logs.Log("fetch task error!")
		}
	}
}
Ejemplo n.º 15
0
func StoreValueLengthFilter(list *list.List, fileId string) {

	for e := list.Front(); e != nil; e = e.Next() {
		str, ok := e.Value.(string)
		if ok {
			dbutil.Store(str, "2", fileId)
		} else {
			logs.Log("list element error in StoreValueLengthFilter list element is not string!")
			continue
		}
	}
}
Ejemplo n.º 16
0
//每过一段时间读取文件的大小,如果文件大小有变化,那么证明正在同步
func checkSize(file string, fileCheck chan bool, taskId string) {
	var size int64 = 0
	var times int = 0 //记录没有同步成功时循环的次数
	var sync int = 0  //记录同步成功时循环的次数
	var zero int = 0  //记录读到文件,但是文件长度为0的次数
	var flag bool = false
	for {
		time.Sleep(time.Second * time.Duration(multiply*45))
		logs.Log("sleep time : " + time.Now().Format("2006-01-02 15:04:05"))
		if times >= 3 {
			fileCheck <- false
			break
		}

		//如果文件不存在,那么继续一次循环
		isExist := exists(file)
		if !isExist {
			times = times + 1
			logs.Log("taskId " + taskId + " times : " + strconv.FormatInt(int64(times), 10))
			continue
		}

		fileInfo, err := os.Stat(file)
		if err != nil {
			fileCheck <- false
			break
		}

		fileSize := fileInfo.Size()
		if fileSize == 0 {
			if zero >= 3 {
				fileCheck <- false
				break
			}
			zero = zero + 1
			logs.Log("taskId " + taskId + " zero : " + strconv.FormatInt(int64(zero), 10))
			continue
		}

		if fileSize != 0 && fileSize > size {
			size = fileSize
			flag = true
			logs.Log("taskId " + taskId + " fileSize : " + strconv.FormatInt(fileSize, 10))
		}
		if flag != true {
			if fileSize != 0 && fileSize == size {
				sync = sync + 1
				logs.Log("taskId " + taskId + " fileSize : " + strconv.FormatInt(fileSize, 10))
				logs.Log("taskId " + taskId + " sync : " + strconv.FormatInt(int64(sync), 10))
			}
		}

		flag = false

		if sync >= 3 {
			fileCheck <- true
			break
		}
	}
}
Ejemplo n.º 17
0
func StoreValueTypeStat(resultMap map[string]int, fileId string) {
	for k, v := range resultMap {
		var value ValueTypeStat
		value.ValueType = k
		value.Num = v
		b, err := json.Marshal(value)
		if err != nil {
			logs.Log("json marshal error in StoreValueTypeStat : " + err.Error())
			return
		}
		dbutil.Store(string(b), "1", fileId)
	}
}
Ejemplo n.º 18
0
func checkOverFlow(array *[]byte, inputFile string, num uint64, flag bool, param *Parameters) {

	//此时管道溢出了,需要重新读取数据
	if (*param).SliceEnd > (*param).SliceContent {
		*array = (*array)[(*param).SliceStart:]
		//1.首先要计算读取多少数据

		var temp []byte
		if (*param).SliceEnd-(*param).SliceContent < (*param).SliceStart {
			temp = make([]byte, (*param).SliceStart)
		} else {
			//如果读取的数据大于管道的大小
			temp = make([]byte, ((*param).SliceEnd-(*param).SliceContent)+1024)
		}

		file, err := os.Open(inputFile)

		if err != nil {
			logs.Log("file open error in checkOverFlow : " + err.Error())
			return
		}

		defer file.Close()

		n, err := file.ReadAt(temp, (*param).OffsetBytes)

		if err != nil && err != io.EOF {
			logs.Log("file read error in checkOverFlow : " + err.Error())
			return
		}

		if n != 0 {
			(*param).OffsetBytes = (*param).OffsetBytes + int64(n)
			*array = append(*array, temp...)
		}
		(*param).SliceEnd = (*param).SliceEnd - (*param).SliceStart
		(*param).SliceStart = 0
	}
}
Ejemplo n.º 19
0
//根据ip地址和端口号获取redis登陆密码
func GetRedisLogPassword(host, port string) string {
	stmtIns, err := db.Prepare("select instance_passwd from instance where instance_ip = ? and instance_port = ?")

	if err != nil {
		logs.Log("get redis log password error in dbutil db.Prepare : " + err.Error())
		return "error"
	}

	defer stmtIns.Close()

	rows, errs := stmtIns.Query(host, port)

	if errs != nil {
		logs.Log("get redis log password error in dbutil stmtIns.Query : " + errs.Error())
		return "error"
	}

	var instancePassword string
	for rows.Next() {
		rows.Scan(&instancePassword)
	}
	return instancePassword
}
Ejemplo n.º 20
0
/**
能够得到所有的ip
*/
func GetIpAddress2() string {
	var ip string
	inter, err := net.InterfaceAddrs()

	if err != nil {
		logs.Log(err.Error())
		return "unknown"
	}

	for _, in := range inter {
		ip += in.String() + " "
	}
	return ip
}
Ejemplo n.º 21
0
//输出过滤的key
func StoreKeyFiltResult(keyMap map[string]int, taskId string) {

	for key, valueL := range keyMap {
		var value ValueLengthFilterData
		value.Key = key
		value.ValueLength = valueL
		b, err := json.Marshal(value)

		if err != nil {
			logs.Log("json marshal error in valueLengthFilter : " + err.Error())
			continue
		}
		dbutil.Store(string(b), "3", taskId)
	}
}
Ejemplo n.º 22
0
func cmdRun(cmd *exec.Cmd, channel chan bool, taskId string) {
	io, err := cmd.StderrPipe()
	if err != nil {
		logs.Log("task " + taskId + " cmdRun StderrPipe() error : " + err.Error())
		channel <- false
		return
	}
	logs.Log("sync cmd.start taskId " + taskId + " " + time.Now().Format("2006-01-02 15:04:05"))
	if err := cmd.Start(); err != nil {
		logs.Log("task " + taskId + " cmdRun Start() error : " + err.Error())
		channel <- false
		return
	}

	message, _ := ioutil.ReadAll(io)

	if err := cmd.Wait(); err != nil {
		logs.Log("task " + taskId + " cmdRun sync rdb file error : " + string(message) + " : " + err.Error())
		channel <- false
		return
	}
	logs.Log("sync cmd.end taskId " + taskId + " " + time.Now().Format("2006-01-02 15:04:05"))
	channel <- true
}
Ejemplo n.º 23
0
//value length 过滤,用来获得value length 大于某个值的key
func valueLengthFilter(key string, valueLength int, filterLength int, valueLengthFilterList *list.List) {

	if valueLength > filterLength {
		var value ValueLengthFilterData
		value.Key = key
		value.ValueLength = valueLength
		b, err := json.Marshal(value)

		if err != nil {
			logs.Log("json marshal error in valueLengthFilter : " + err.Error())
			return
		}
		valueLengthFilterList.PushBack(string(b))
	}
}
Ejemplo n.º 24
0
/**
读取一个字节,按照这个字节获得表示的数字
此方法被: parseHashEncodingValue
		 parseSortedSetEncondingValue
		 parseSetEncodingValue
		 三个方法调用,用来获得value的数量
或者在解压缩时使用,用来获得压缩或者解压缩之后的字节数
*/
func readOneByteAndGetNum(array *[]byte, inputFile string, param *Parameters) uint64 {
	slice := readNextNByte(array, inputFile, 1, param)
	pre, suf := calByte(slice[0])
	var num uint64

	switch pre {
	case 0: //then the next 6 bits represent the length
		num = uint64(suf)
	case 64: //then an additional byte is read from the stream. The combined 14 bits represent the length
		reIndexSliceEnd(array, inputFile, 1, param)
		slice := getArraySlice((*param).SliceStart, (*param).SliceEnd, array)
		//把slice里边的两个字节转换成数字
		slice_value := changeByteToUint64(slice, binary.BigEndian)
		//获取后14个bit代表的数字
		num = slice_value & 0x3FFF

	case 128: //hen the remaining 6 bits are discarded. Additional 4 bytes are read from the stream, and those 4 bytes represent the length
		//读取后4个bytes
		reIndex(array, inputFile, 4, param)
		slice := getArraySlice((*param).SliceStart, (*param).SliceEnd, array)
		//把slice里边的四个字节转换成数字
		num = changeByteToUint64(slice, binary.BigEndian)

	case 192: //then the next object is encoded in a special format. The remaining 6 bits indicate the format
		switch suf {
		case 0: //0 indicates that an 8 bit integer follows.
			reIndex(array, inputFile, 1, param)
			num = changeByteToUint64(getArraySlice((*param).SliceStart, (*param).SliceEnd, array), binary.LittleEndian)
		case 1: //1 indicates that a 16 bit integer follows.
			reIndex(array, inputFile, 2, param)
			num = changeByteToUint64(getArraySlice((*param).SliceStart, (*param).SliceEnd, array), binary.LittleEndian)
		case 2: //2 indicates that a 32 bit integer follows.
			reIndex(array, inputFile, 4, param)
			num = changeByteToUint64(getArraySlice((*param).SliceStart, (*param).SliceEnd, array), binary.LittleEndian)
		case 3, 4: //4 indicates that a compressed string follows.
			logs.Log("error has occurred if you read this sentence. readOneByteAndGetNum")
		}
	}
	return num
}
Ejemplo n.º 25
0
//查看某状态的任务
func taskView(w http.ResponseWriter, r *http.Request) {

	status := r.FormValue("status")

	reg := regexp.MustCompile(status_requ)
	if reg.MatchString(status) == false {
		io.WriteString(w, "need param status, status must be 0-new task ,1-success,2-can not get rdb,3-parse fatal, 4-get remote redis password error or 5-task is running!")
		return
	}

	int_stat, _ := strconv.ParseInt(status, 10, 32)

	mark, list := dbutil.FetchTask(int(int_stat))

	if mark == 1 {
		var listHtml string = "<body><ol>"
		for e := list.Front(); e != nil; e = e.Next() {
			task, ok := e.Value.(dbutil.Task)
			if ok {
				//获取task信息
				host := task.Host
				port := task.Port
				filterLength := task.FilterLength
				filterKey := task.FilterKey
				taskId := task.TaskId
				createTime := task.CreateTime
				listHtml += "<li>" + "host:" + host + ";port:" + port + ";filterLength:" + strconv.FormatInt(int64(filterLength), 10) + ";filterKey:" + filterKey + ";taskId:" + taskId + ";create_time:" + createTime + "</li>"
			} else {
				//task的类型不正确
				return
			}
		}
		listHtml = listHtml + "</ol></body>"
		io.WriteString(w, listHtml)
	} else {
		io.WriteString(w, "error occure !")
		logs.Log("error occure in taskView!")
	}
}
Ejemplo n.º 26
0
//根据taskId查看某一个task
func task(w http.ResponseWriter, r *http.Request) {

	taskId := r.FormValue("taskId")

	mark, list := dbutil.FetchTaskById(taskId)
	if list.Len() == 0 {
		io.WriteString(w, "without this task !")
		return
	}

	if mark == 1 {
		var listHtml string = "<body><ol>"
		for e := list.Front(); e != nil; e = e.Next() {
			task, ok := e.Value.(dbutil.Task)
			if ok {
				//获取task信息
				host := task.Host
				port := task.Port
				filterLength := task.FilterLength
				filterKey := task.FilterKey
				taskId := task.TaskId
				priority := task.Priority
				status := task.Status
				createTime := task.CreateTime
				listHtml += "<li>" + "host:" + host + ";port:" + port + ";filterLength:" + strconv.FormatInt(int64(filterLength), 10) + ";filter_key:" + filterKey + ";taskId:" + taskId +
					";priority:" + strconv.FormatInt(int64(priority), 10) + ";status:" + strconv.FormatInt(int64(status), 10) + ";create_time:" + createTime + "</li>"
			} else {
				//task的类型不正确
				return
			}
			listHtml = listHtml + "</ol></body>"
		}
		io.WriteString(w, listHtml)
	} else {
		io.WriteString(w, "error occure !")
		logs.Log("error occure in taskView!")
	}
}
Ejemplo n.º 27
0
func syncRdb(host string, port string, cliLocation string, rdbFile string, taskId string) int {

	//获取redis登陆密码
	password := dbutil.GetRedisLogPassword(host, port)
	if password == "error" {
		logs.Log("task " + taskId + " syncRdb error can not get redis password")
		return 4
	}

	cmd := exec.Command(cliLocation, "-h", host, "-p", port, "-a", password, "--rdb", rdbFile)

	//同步rdb文件
	cmdChan := make(chan bool, 1)
	go cmdRun(cmd, cmdChan, taskId)

	//检查文件大小变化
	fileChan := make(chan bool, 1)
	go checkSize(rdbFile, fileChan, taskId)

	select {
	case value := <-fileChan:
		if value == false {
			if cmd.Process.Pid != 0 {
				logs.Log("task " + taskId + " sync failed process pid : " + strconv.FormatInt(int64(cmd.Process.Pid), 10))
				cmd.Process.Kill()
			}
			logs.Log("fileChan : false")
			return 2
		} else {
			logs.Log("fileChan : true")
			return 1
		}
	case value := <-cmdChan:
		if value == false {
			logs.Log("cmdChan : false")
			return 2
		} else {
			logs.Log("cmdChan : true")
			return 1
		}
	}
}
Ejemplo n.º 28
0
func main() {

	//	//本地运行
	//	inputFile := "C:/Users/Administrator/Downloads/6410.rdb"
	//	inputFile := "D:/software/work/redis-2.8.19/6383/dump.rdb"
	//	inputFile := "D:/software/1431429678905877955.rdb"
	//	filterLength := 100
	//	taskId := analysis.GetCertificate()
	//	fmt.Println(taskId)
	//	var param parse.Parameters
	//	param = parse.Start(inputFile, filterLength, "")
	//	if param.Success == true {
	//		fmt.Println(param.TotalNum)
	//		//		analysis.StoreValueTypeStat(param.ResultMap, taskId)
	//		//		analysis.StoreValueLengthFilter(param.ValueLengthFilterList, taskId)
	//		//		analysis.OutPut(param.Heap)
	//		analysis.StoreSortingResult(param.Heap, taskId)
	//		analysis.StoreKeyFiltResult(param.KeyFilterMap, taskId)
	//		fmt.Println("success")
	//	} else {
	//		fmt.Println("error")
	//	}
	////////////////////////////////////////////////////////////////////////////////
	defer func() {
		if r := recover(); r != nil {
			logs.Log(r)
		}
	}()
	exitChan := make(chan int)
	signalChan := make(chan os.Signal, 1)
	go func() {
		<-signalChan
		exitChan <- 1
	}()

	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)

	//设置程序使用的cup数量
	cpu := runtime.NumCPU()
	if cpu == 1 {
		cpu = 1
	} else {
		cpu = cpu / 2
	}
	runtime.GOMAXPROCS(cpu)
	go startParse()

	//添加任务并启动解析服务
	http.HandleFunc("/add_task", addTask)
	//查看某种状态的task
	http.HandleFunc("/task_view", taskView)
	//根据taskid查看task
	http.HandleFunc("/task", task)
	//查看value类型统计
	http.HandleFunc("/result/value/type", viewValueTypeStatResult)
	//查看value长度过滤
	http.HandleFunc("/result/value/length", viewValueLengthFilterResult)
	//删除任务
	http.HandleFunc("/delete", delete)
	err := http.ListenAndServe(":"+port, nil)
	if err != nil {
		logs.Log("error in main : " + err.Error())
		os.Exit(-1)
	}
}
Ejemplo n.º 29
0
/**
接受一个redis rdb文件的byte数组,并解析
List Encoding 与 Set Encoding 与 Sorted Set Enconding 的编码方式是一样的
Sorted Set in Ziplist Encoding 与 Ziplist Encoding 的编码方式是一样的
Zipmap encoding are deprecated starting Redis 2.6. Small hashmaps are now encoded using ziplists.
pwflag用来判断是打印还是输出到文件中
*/
func parseRedisRdb(array *[]byte, inputFile string, param *Parameters) {
	//解析固定信息
	//固定信息只能解析一次
	if (*param).FixedInfo == false {
		parseFixedInformation(array, param)
	}
	defer func() {
		if r := recover(); r != nil {
			//设置解析失败
			(*param).Success = false
			//记录log
			logs.Log(r)
		}
	}()
	var flag bool = true

	for flag {
		reIndex(array, inputFile, 1, param)
		slice := getArraySlice((*param).SliceStart, (*param).SliceEnd, array)
		var i uint8 = uint8(slice[0])

		if i == 255 {
			break
		}

		switch i {

		case 0:
			(*param).TotalNum++
			key, _ := parseKeyString(array, inputFile, param)
			value, length := parseStringEncodingValue(array, inputFile, param)
			str := "value type String Encoding: " + key + "--->" + value
			dealResult(key, length, str, 0, param)
		case 1:
			(*param).TotalNum++
			key, _ := parseKeyString(array, inputFile, param)
			length := parseListEncodingValue(array, inputFile, param)
			str := "value type List Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
			dealResult(key, length, str, 1, param)
		case 2:
			(*param).TotalNum++
			key, _ := parseKeyString(array, inputFile, param)
			length := parseSetEncodingValue(array, inputFile, param)
			str := "value type Set Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
			dealResult(key, length, str, 2, param)
		case 3:
			(*param).TotalNum++
			key, _ := parseKeyString(array, inputFile, param)
			length := parseSortedSetEncodingValue(array, inputFile, param)
			str := "value type Sorted Set Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
			dealResult(key, length, str, 3, param)
		case 4:
			(*param).TotalNum++
			key, _ := parseKeyString(array, inputFile, param)
			length := parseHashEncodingValue(array, inputFile, param)
			str := "value type Hash Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
			dealResult(key, length, str, 4, param)

		case 9: //2.6以前的版本有这个
			(*param).TotalNum++
			key, _ := parseKeyString(array, inputFile, param)
			length := parseZipMapEncodingValue(array, inputFile, param)
			str := "value type Zipmap Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
			dealResult(key, length, str, 9, param)
		case 10:
			(*param).TotalNum++
			key, _ := parseKeyString(array, inputFile, param)
			length := parseZiplistEncodingValue(array, inputFile, param)
			str := "value type Ziplist Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
			dealResult(key, length, str, 10, param)
		case 11:
			(*param).TotalNum++
			key, _ := parseKeyString(array, inputFile, param)
			length := parseIntSetEncoding(array, inputFile, param)
			str := "value type Intset Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
			dealResult(key, length, str, 11, param)
		case 12:
			(*param).TotalNum++
			key, _ := parseKeyString(array, inputFile, param)
			length := parseZiplistEncodingValue(array, inputFile, param)
			str := "value type Sorted Set In Ziplist Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
			dealResult(key, length, str, 12, param)
		case 13:
			(*param).TotalNum++
			key, _ := parseKeyString(array, inputFile, param)
			length := parseHashMapInZiplistEncodingValue(array, inputFile, param)
			str := "value type HashMap In Ziplist Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
			dealResult(key, length, str, 13, param)
		case 252: //后边跟的有效期是毫秒(ms)
			parseExpireTimeInMillSeconds(array, inputFile, param)
		case 253: //后边跟的是有效期是秒(s)
			parseExpireTimeInSeconds(array, inputFile, param)
		case 254: //redis 数据库编号
			parseRedisDBNum(array, inputFile, param)
		default:
			logs.Log("error occurred in parseFunc : " + strconv.FormatInt(int64(i), 10))
			(*param).Success = false
			return
		}
	}
	(*param).Success = true
}
Ejemplo n.º 30
0
//获取新的任务
func FetchNewTask() (int, Task) {

	var task Task

	//查询
	stmtQuery, err := db.Prepare("select host, port, filter_length, filter_key, task_id, create_time, update_time from rdb_parse_task where status = 0 order by priority desc limit 1")
	defer stmtQuery.Close()
	if err != nil {
		logs.Log("fetch new task error in dbutil db.stmtQuery Prepare : " + err.Error())
		return -1, task
	}

	rows, errs := stmtQuery.Query()

	if errs != nil {
		logs.Log("fetch new task error in dbutil stmtIns.Query Exec : " + errs.Error())
		return -2, task
	}

	var host string
	var port string
	var filterLength int
	var filterKey string
	var taskId string
	var createTime string
	var updateTime string
	for rows.Next() {
		rows.Scan(&host, &port, &filterLength, &filterKey, &taskId, &createTime, &updateTime)
		task.Host = host
		task.Port = port
		task.FilterLength = filterLength
		task.TaskId = taskId
		task.CreateTime = createTime
		task.UpdateTime = updateTime
		task.FilterKey = filterKey
	}
	if taskId != "" {
		//更改任务状态
		stmtUpdate, errUpdate := db.Prepare("update rdb_parse_task set status = 5 where task_id = ? and update_time = ?")
		defer stmtUpdate.Close()

		if errUpdate != nil {
			logs.Log("fetch new task error in dbutil db.stmtUpdate : " + errUpdate.Error())
			return -3, task
		}

		result, err := stmtUpdate.Exec(taskId, updateTime)
		if err != nil {
			logs.Log("fetch new task error in dbutil db.stmtUpdate Exec : " + err.Error())
			return -4, task
		}

		value, _ := result.RowsAffected()

		//证明记录没有被修改过
		if value == 1 {
			return 1, task
		} else {
			//证明记录已经被修改了,有其它的实例获得了任务
			return -5, task
		}
	} else {
		return 1, task
	}
}