//获取所有任务的任务列表
func FetchTaskList() (int, *list.List) {
var stmtIns *sql.Stmt
var err error
stmtIns, err = db.Prepare("select task_id, create_time from rdb_parse_task")
if err != nil {
logs.Log("fetch task list error in dbutil db.Prepare : " + err.Error())
return -1, list.New()
}
defer stmtIns.Close()
rows, errs := stmtIns.Query()
if errs != nil {
logs.Log("fetch task list error in dbutil stmtIns.Query : " + errs.Error())
return -2, list.New()
}
taskList := list.New()
var taskId string
var createTime string
for rows.Next() {
var task Task
rows.Scan(&taskId, &createTime)
task.TaskId = taskId
task.CreateTime = createTime
taskList.PushBack(task)
}
return 1, taskList
}
//根据taskId和type获取分析结果
func FetchResult(taskId string, types string, limit int, offset int) []string {
result := make([]string, 0)
stmtIns, err := db.Prepare("select content from rdb_anly_result where task_id = ? and type = ? order by id desc limit ? offset ? ")
if err != nil {
logs.Log("fetch result error in dbutil db.Prepare!" + " : " + err.Error())
return result
}
defer stmtIns.Close()
rows, errs := stmtIns.Query(taskId, types, limit, offset)
if errs != nil {
logs.Log("fetch result error in dbutil stmtIns.Query!" + " : " + errs.Error())
return result
}
var content string
for rows.Next() {
rows.Scan(&content)
result = append(result, content)
}
return result
}
//删除任务，以及任务的分析结果
func DeleteTask(list *list.List) bool {
stmtInsTask, err := db.Prepare("delete from rdb_parse_task where task_id = ?")
if err != nil {
logs.Log("delete task stmtInsTask error in dbutil : " + err.Error())
return false
}
defer stmtInsTask.Close()
stmtResult, err := db.Prepare("delete from rdb_anly_result where task_id = ?")
if err != nil {
logs.Log("delete dask stmtResult error in dbutil : " + err.Error())
return false
}
defer stmtResult.Close()
for e := list.Front(); e != nil; e = e.Next() {
taskId, ok := e.Value.(string)
if ok {
stmtInsTask.Exec(taskId)
stmtResult.Exec(taskId)
} else {
logs.Log("type error!")
}
}
return true
}
//管道读取
func pipelineAnalyze(inputFile string, param *Parameters) {
file, err := os.Open(inputFile)
if err != nil && err != io.EOF {
logs.Log("file open error in parseRdb pipelineAnalyze : " + err.Error())
return
}
defer file.Close()
//首先读取一定数量的字节
buff := make([]byte, (*param).SliceContent)
n, err := file.Read(buff)
if err != nil && err != io.EOF {
logs.Log("file read error in parseRdb pipelineAnalyze : " + err.Error())
return
}
//解析
var pbuff = &buff
if n != 0 {
(*param).OffsetBytes = (*param).OffsetBytes + int64(n)
parseRedisRdb(pbuff, inputFile, param)
(*param).FixedInfo = true
}
//为了防止文件没有全部读取，再次循环读取,理论上这个地方应该用不到
for {
n, err := file.ReadAt(buff, (*param).OffsetBytes)
if err != nil && err != io.EOF {
logs.Log("file read error in parseRdb pipelineAnalyze for loop : " + err.Error())
return
}
defer file.Close()
if n == 0 {
break
} else {
(*param).SliceStart = 0
(*param).SliceEnd = (*param).SliceStart + 1
(*param).OffsetBytes = (*param).OffsetBytes + int64(n)
parseRedisRdb(pbuff, inputFile, param)
(*param).FixedInfo = true
}
}
}
/**
这个是解析的入口
pwflag 表示是打印还是写入到文件
0 打印
1 写入到文件
2 写入到数据库
filename 表示要解析的文件路径
*/
func Start(inputFile string, filterLength int, filterKey string) Parameters {
//读取管道的大小
conf := config.SetConfig("important.properties")
sliceContent, _ := strconv.ParseUint(conf.GetValue("pipeline", "size"), 10, 64)
if sliceContent < 16 {
sliceContent = 16
logs.Log("sliceContent < 16")
}
var heapIni bool = false
var param Parameters
param.Success = false
param.SliceStart = 0
param.SliceEnd = 0
param.Step = []uint64{5, 4}
param.FileFlag = false
param.SliceContent = sliceContent
param.FixedInfo = false
param.OffsetBytes = 0
param.Dbnum = -1
param.ResultMap = make(map[string]int)
param.ValueFilterLength = filterLength
param.FilterKey = filterKey
param.ValueLengthFilterList = list.New()
param.KeyFilterMap = make(map[string]int)
param.Heap = &sorting.Heap{}
param.HeapInit = &heapIni
pipelineAnalyze(inputFile, ¶m)
return param
}
//更新任务状态
func UpdateTaskStatus(status int, taskId string) bool {
stmtIns, err := db.Prepare("update rdb_parse_task set status = ? where task_id = ?")
if err != nil {
logs.Log("update task status error in dbutil db.Prepare : " + err.Error())
return false
}
defer stmtIns.Close()
_, errs := stmtIns.Exec(status, taskId)
if errs != nil {
logs.Log("update task status error in dbutil stmtIns.Exec : " + errs.Error())
return false
}
return true
}
//存储分析结果
func Store(content, types, taskId string) {
stmtIns, err := db.Prepare("insert into rdb_anly_result(content, type, task_id, create_time) values (?,?,?,now())")
if err != nil {
logs.Log("store error in dbutil db.Prepare : " + err.Error())
return
}
defer stmtIns.Close()
_, errs := stmtIns.Exec(content, types, taskId)
if errs != nil {
logs.Log("store error in dbutil stmtIns.Exec : " + errs.Error())
return
}
}
//向任务表中插入任务
func InsertTask(host, port, taskId string, filterLength, priority int, filter_key string) bool {
stmtIns, err := db.Prepare("insert into rdb_parse_task (host, port, filter_length, filter_key, task_id, priority, status, create_time) values (?,?,?,?,?,?,0,now())")
if err != nil {
logs.Log("insert task error in dbutil db.Prepare : " + err.Error())
return false
}
defer stmtIns.Close()
_, errs := stmtIns.Exec(host, port, filterLength, filter_key, taskId, priority)
if errs != nil {
logs.Log("insert task error in dbutil stmtIns.Exec : " + errs.Error())
return false
}
return true
}
//解析程序
func parseTask(host string, port string, filterLength int, filterKey string, taskId string) {
//获得机器的ip地址
ip := utils.GetIpAddress()
if ip == "unknown" {
ip = utils.GetIpAddress2()
}
logs.Log("machine : " + ip + " processing task : " + taskId)
//获取rdb文件
rdbFile := rdbLocation + taskId + ".rdb"
//如果文件存在则删除
isExist := exists(rdbFile)
if isExist == true {
err := os.Remove(rdbFile)
if err != nil {
logs.Log("parseTask remove rdbFile error : " + err.Error())
return
}
}
flag := syncRdb(host, port, cliLocation, rdbFile, taskId)
if flag == 1 {
//还要判断解析是否成功了
status := start(rdbFile, taskId, filterLength, filterKey)
if status == true {
dbutil.UpdateTaskStatus(1, taskId)
} else {
dbutil.UpdateTaskStatus(3, taskId)
}
//删除文件
dropFile(rdbFile, taskId)
} else {
switch flag {
case 2: //同步进程失效，文件大小无变化
logs.Log("task " + taskId + " sync process error, file size no change!")
dbutil.UpdateTaskStatus(2, taskId)
case 4: //获得redis密码错误
dbutil.UpdateTaskStatus(4, taskId)
logs.Log("task " + taskId + " get remote redis server login password error!")
}
//删除文件
dropFile(rdbFile, taskId)
}
}
/**
net 环境下能够获得访问网络的ip
*/
func GetIpAddress() string {
conn, err := net.Dial("udp", "www.baidu.com:80")
if err != nil {
logs.Log(err.Error())
return "unknown"
}
defer conn.Close()
return strings.Split(conn.LocalAddr().String(), ":")[0]
}
//解析操作完成之后删除文件
func dropFile(file, taskId string) {
//首先判断存在文件吗
isExist := exists(file)
if isExist {
err := os.Remove(file)
if err != nil {
logs.Log("task " + taskId + " remove file error : " + err.Error() + " --- " + file)
}
}
}
//根据任务id获取单个任务
func FetchTaskById(task_id string) (int, *list.List) {
stmtIns, err := db.Prepare("select host, port, filter_length, filter_key, task_id, priority, status, create_time from rdb_parse_task where task_id = ?")
if err != nil {
logs.Log("fetch task by id error in dbutil db.Prepare : " + err.Error())
return -1, list.New()
}
defer stmtIns.Close()
rows, errs := stmtIns.Query(task_id)
if errs != nil {
logs.Log("fetch task by id error in dbutil stmtIns.Query : " + errs.Error())
return -2, list.New()
}
taskList := list.New()
var host string
var port string
var filterLength int
var filterKey string
var taskId string
var priority int
var status int
var createTime string
for rows.Next() {
var task Task
rows.Scan(&host, &port, &filterLength, &filterKey, &taskId, &priority, &status, &createTime)
task.Host = host
task.Port = port
task.FilterLength = filterLength
task.TaskId = taskId
task.Priority = priority
task.Status = status
task.CreateTime = createTime
task.FilterKey = filterKey
taskList.PushBack(task)
}
return 1, taskList
}
//存储堆排序结果
func StoreSortingResult(h *sorting.Heap, taskId string) {
for h.Len() > 0 {
element := heap.Pop(h).(sorting.Element)
b, err := json.Marshal(element)
if err != nil {
logs.Log("json marshal error in StoreValueTypeStat : " + err.Error())
return
}
dbutil.Store(string(b), "2", taskId)
}
}
//为了部署多个实例的需要，修改了startParse
func startParse() {
//每次只取一个任务，这样保证当存在多个服务时，每个机器都能获得任务
logs.Log("starting parse!")
for {
mark, task := dbutil.FetchNewTask()
if mark == 1 {
if task.TaskId == "" {
time.Sleep(time.Second * 10)
continue
} else {
parseTask(task.Host, task.Port, task.FilterLength, task.FilterKey, task.TaskId)
}
} else if mark == -5 {
logs.Log("task " + task.TaskId + " processing by another machine! ")
} else {
logs.Log("fetch task error!")
}
}
}
func StoreValueLengthFilter(list *list.List, fileId string) {
for e := list.Front(); e != nil; e = e.Next() {
str, ok := e.Value.(string)
if ok {
dbutil.Store(str, "2", fileId)
} else {
logs.Log("list element error in StoreValueLengthFilter list element is not string!")
continue
}
}
}
//每过一段时间读取文件的大小，如果文件大小有变化，那么证明正在同步
func checkSize(file string, fileCheck chan bool, taskId string) {
var size int64 = 0
var times int = 0 //记录没有同步成功时循环的次数
var sync int = 0 //记录同步成功时循环的次数
var zero int = 0 //记录读到文件，但是文件长度为0的次数
var flag bool = false
for {
time.Sleep(time.Second * time.Duration(multiply*45))
logs.Log("sleep time : " + time.Now().Format("2006-01-02 15:04:05"))
if times >= 3 {
fileCheck <- false
break
}
//如果文件不存在，那么继续一次循环
isExist := exists(file)
if !isExist {
times = times + 1
logs.Log("taskId " + taskId + " times : " + strconv.FormatInt(int64(times), 10))
continue
}
fileInfo, err := os.Stat(file)
if err != nil {
fileCheck <- false
break
}
fileSize := fileInfo.Size()
if fileSize == 0 {
if zero >= 3 {
fileCheck <- false
break
}
zero = zero + 1
logs.Log("taskId " + taskId + " zero : " + strconv.FormatInt(int64(zero), 10))
continue
}
if fileSize != 0 && fileSize > size {
size = fileSize
flag = true
logs.Log("taskId " + taskId + " fileSize : " + strconv.FormatInt(fileSize, 10))
}
if flag != true {
if fileSize != 0 && fileSize == size {
sync = sync + 1
logs.Log("taskId " + taskId + " fileSize : " + strconv.FormatInt(fileSize, 10))
logs.Log("taskId " + taskId + " sync : " + strconv.FormatInt(int64(sync), 10))
}
}
flag = false
if sync >= 3 {
fileCheck <- true
break
}
}
}
func StoreValueTypeStat(resultMap map[string]int, fileId string) {
for k, v := range resultMap {
var value ValueTypeStat
value.ValueType = k
value.Num = v
b, err := json.Marshal(value)
if err != nil {
logs.Log("json marshal error in StoreValueTypeStat : " + err.Error())
return
}
dbutil.Store(string(b), "1", fileId)
}
}
func checkOverFlow(array *[]byte, inputFile string, num uint64, flag bool, param *Parameters) {
//此时管道溢出了,需要重新读取数据
if (*param).SliceEnd > (*param).SliceContent {
*array = (*array)[(*param).SliceStart:]
//1.首先要计算读取多少数据
var temp []byte
if (*param).SliceEnd-(*param).SliceContent < (*param).SliceStart {
temp = make([]byte, (*param).SliceStart)
} else {
//如果读取的数据大于管道的大小
temp = make([]byte, ((*param).SliceEnd-(*param).SliceContent)+1024)
}
file, err := os.Open(inputFile)
if err != nil {
logs.Log("file open error in checkOverFlow : " + err.Error())
return
}
defer file.Close()
n, err := file.ReadAt(temp, (*param).OffsetBytes)
if err != nil && err != io.EOF {
logs.Log("file read error in checkOverFlow : " + err.Error())
return
}
if n != 0 {
(*param).OffsetBytes = (*param).OffsetBytes + int64(n)
*array = append(*array, temp...)
}
(*param).SliceEnd = (*param).SliceEnd - (*param).SliceStart
(*param).SliceStart = 0
}
}
//根据ip地址和端口号获取redis登陆密码
func GetRedisLogPassword(host, port string) string {
stmtIns, err := db.Prepare("select instance_passwd from instance where instance_ip = ? and instance_port = ?")
if err != nil {
logs.Log("get redis log password error in dbutil db.Prepare : " + err.Error())
return "error"
}
defer stmtIns.Close()
rows, errs := stmtIns.Query(host, port)
if errs != nil {
logs.Log("get redis log password error in dbutil stmtIns.Query : " + errs.Error())
return "error"
}
var instancePassword string
for rows.Next() {
rows.Scan(&instancePassword)
}
return instancePassword
}
/**
能够得到所有的ip
*/
func GetIpAddress2() string {
var ip string
inter, err := net.InterfaceAddrs()
if err != nil {
logs.Log(err.Error())
return "unknown"
}
for _, in := range inter {
ip += in.String() + " "
}
return ip
}
//输出过滤的key
func StoreKeyFiltResult(keyMap map[string]int, taskId string) {
for key, valueL := range keyMap {
var value ValueLengthFilterData
value.Key = key
value.ValueLength = valueL
b, err := json.Marshal(value)
if err != nil {
logs.Log("json marshal error in valueLengthFilter : " + err.Error())
continue
}
dbutil.Store(string(b), "3", taskId)
}
}
func cmdRun(cmd *exec.Cmd, channel chan bool, taskId string) {
io, err := cmd.StderrPipe()
if err != nil {
logs.Log("task " + taskId + " cmdRun StderrPipe() error : " + err.Error())
channel <- false
return
}
logs.Log("sync cmd.start taskId " + taskId + " " + time.Now().Format("2006-01-02 15:04:05"))
if err := cmd.Start(); err != nil {
logs.Log("task " + taskId + " cmdRun Start() error : " + err.Error())
channel <- false
return
}
message, _ := ioutil.ReadAll(io)
if err := cmd.Wait(); err != nil {
logs.Log("task " + taskId + " cmdRun sync rdb file error : " + string(message) + " : " + err.Error())
channel <- false
return
}
logs.Log("sync cmd.end taskId " + taskId + " " + time.Now().Format("2006-01-02 15:04:05"))
channel <- true
}
//value length 过滤，用来获得value length 大于某个值的key
func valueLengthFilter(key string, valueLength int, filterLength int, valueLengthFilterList *list.List) {
if valueLength > filterLength {
var value ValueLengthFilterData
value.Key = key
value.ValueLength = valueLength
b, err := json.Marshal(value)
if err != nil {
logs.Log("json marshal error in valueLengthFilter : " + err.Error())
return
}
valueLengthFilterList.PushBack(string(b))
}
}
/**
读取一个字节，按照这个字节获得表示的数字
此方法被: parseHashEncodingValue
parseSortedSetEncondingValue
parseSetEncodingValue
三个方法调用，用来获得value的数量
或者在解压缩时使用，用来获得压缩或者解压缩之后的字节数
*/
func readOneByteAndGetNum(array *[]byte, inputFile string, param *Parameters) uint64 {
slice := readNextNByte(array, inputFile, 1, param)
pre, suf := calByte(slice[0])
var num uint64
switch pre {
case 0: //then the next 6 bits represent the length
num = uint64(suf)
case 64: //then an additional byte is read from the stream. The combined 14 bits represent the length
reIndexSliceEnd(array, inputFile, 1, param)
slice := getArraySlice((*param).SliceStart, (*param).SliceEnd, array)
//把slice里边的两个字节转换成数字
slice_value := changeByteToUint64(slice, binary.BigEndian)
//获取后14个bit代表的数字
num = slice_value & 0x3FFF
case 128: //hen the remaining 6 bits are discarded. Additional 4 bytes are read from the stream, and those 4 bytes represent the length
//读取后4个bytes
reIndex(array, inputFile, 4, param)
slice := getArraySlice((*param).SliceStart, (*param).SliceEnd, array)
//把slice里边的四个字节转换成数字
num = changeByteToUint64(slice, binary.BigEndian)
case 192: //then the next object is encoded in a special format. The remaining 6 bits indicate the format
switch suf {
case 0: //0 indicates that an 8 bit integer follows.
reIndex(array, inputFile, 1, param)
num = changeByteToUint64(getArraySlice((*param).SliceStart, (*param).SliceEnd, array), binary.LittleEndian)
case 1: //1 indicates that a 16 bit integer follows.
reIndex(array, inputFile, 2, param)
num = changeByteToUint64(getArraySlice((*param).SliceStart, (*param).SliceEnd, array), binary.LittleEndian)
case 2: //2 indicates that a 32 bit integer follows.
reIndex(array, inputFile, 4, param)
num = changeByteToUint64(getArraySlice((*param).SliceStart, (*param).SliceEnd, array), binary.LittleEndian)
case 3, 4: //4 indicates that a compressed string follows.
logs.Log("error has occurred if you read this sentence. readOneByteAndGetNum")
}
}
return num
}
//查看某状态的任务
func taskView(w http.ResponseWriter, r *http.Request) {
status := r.FormValue("status")
reg := regexp.MustCompile(status_requ)
if reg.MatchString(status) == false {
io.WriteString(w, "need param status, status must be 0-new task ,1-success,2-can not get rdb,3-parse fatal, 4-get remote redis password error or 5-task is running!")
return
}
int_stat, _ := strconv.ParseInt(status, 10, 32)
mark, list := dbutil.FetchTask(int(int_stat))
if mark == 1 {
var listHtml string = "<body><ol>"
for e := list.Front(); e != nil; e = e.Next() {
task, ok := e.Value.(dbutil.Task)
if ok {
//获取task信息
host := task.Host
port := task.Port
filterLength := task.FilterLength
filterKey := task.FilterKey
taskId := task.TaskId
createTime := task.CreateTime
listHtml += "<li>" + "host:" + host + ";port:" + port + ";filterLength:" + strconv.FormatInt(int64(filterLength), 10) + ";filterKey:" + filterKey + ";taskId:" + taskId + ";create_time:" + createTime + "</li>"
} else {
//task的类型不正确
return
}
}
listHtml = listHtml + "</ol></body>"
io.WriteString(w, listHtml)
} else {
io.WriteString(w, "error occure !")
logs.Log("error occure in taskView!")
}
}
//根据taskId查看某一个task
func task(w http.ResponseWriter, r *http.Request) {
taskId := r.FormValue("taskId")
mark, list := dbutil.FetchTaskById(taskId)
if list.Len() == 0 {
io.WriteString(w, "without this task !")
return
}
if mark == 1 {
var listHtml string = "<body><ol>"
for e := list.Front(); e != nil; e = e.Next() {
task, ok := e.Value.(dbutil.Task)
if ok {
//获取task信息
host := task.Host
port := task.Port
filterLength := task.FilterLength
filterKey := task.FilterKey
taskId := task.TaskId
priority := task.Priority
status := task.Status
createTime := task.CreateTime
listHtml += "<li>" + "host:" + host + ";port:" + port + ";filterLength:" + strconv.FormatInt(int64(filterLength), 10) + ";filter_key:" + filterKey + ";taskId:" + taskId +
";priority:" + strconv.FormatInt(int64(priority), 10) + ";status:" + strconv.FormatInt(int64(status), 10) + ";create_time:" + createTime + "</li>"
} else {
//task的类型不正确
return
}
listHtml = listHtml + "</ol></body>"
}
io.WriteString(w, listHtml)
} else {
io.WriteString(w, "error occure !")
logs.Log("error occure in taskView!")
}
}
func syncRdb(host string, port string, cliLocation string, rdbFile string, taskId string) int {
//获取redis登陆密码
password := dbutil.GetRedisLogPassword(host, port)
if password == "error" {
logs.Log("task " + taskId + " syncRdb error can not get redis password")
return 4
}
cmd := exec.Command(cliLocation, "-h", host, "-p", port, "-a", password, "--rdb", rdbFile)
//同步rdb文件
cmdChan := make(chan bool, 1)
go cmdRun(cmd, cmdChan, taskId)
//检查文件大小变化
fileChan := make(chan bool, 1)
go checkSize(rdbFile, fileChan, taskId)
select {
case value := <-fileChan:
if value == false {
if cmd.Process.Pid != 0 {
logs.Log("task " + taskId + " sync failed process pid : " + strconv.FormatInt(int64(cmd.Process.Pid), 10))
cmd.Process.Kill()
}
logs.Log("fileChan : false")
return 2
} else {
logs.Log("fileChan : true")
return 1
}
case value := <-cmdChan:
if value == false {
logs.Log("cmdChan : false")
return 2
} else {
logs.Log("cmdChan : true")
return 1
}
}
}
func main() {
// //本地运行
// inputFile := "C:/Users/Administrator/Downloads/6410.rdb"
// inputFile := "D:/software/work/redis-2.8.19/6383/dump.rdb"
// inputFile := "D:/software/1431429678905877955.rdb"
// filterLength := 100
// taskId := analysis.GetCertificate()
// fmt.Println(taskId)
// var param parse.Parameters
// param = parse.Start(inputFile, filterLength, "")
// if param.Success == true {
// fmt.Println(param.TotalNum)
// // analysis.StoreValueTypeStat(param.ResultMap, taskId)
// // analysis.StoreValueLengthFilter(param.ValueLengthFilterList, taskId)
// // analysis.OutPut(param.Heap)
// analysis.StoreSortingResult(param.Heap, taskId)
// analysis.StoreKeyFiltResult(param.KeyFilterMap, taskId)
// fmt.Println("success")
// } else {
// fmt.Println("error")
// }
////////////////////////////////////////////////////////////////////////////////
defer func() {
if r := recover(); r != nil {
logs.Log(r)
}
}()
exitChan := make(chan int)
signalChan := make(chan os.Signal, 1)
go func() {
<-signalChan
exitChan <- 1
}()
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
//设置程序使用的cup数量
cpu := runtime.NumCPU()
if cpu == 1 {
cpu = 1
} else {
cpu = cpu / 2
}
runtime.GOMAXPROCS(cpu)
go startParse()
//添加任务并启动解析服务
http.HandleFunc("/add_task", addTask)
//查看某种状态的task
http.HandleFunc("/task_view", taskView)
//根据taskid查看task
http.HandleFunc("/task", task)
//查看value类型统计
http.HandleFunc("/result/value/type", viewValueTypeStatResult)
//查看value长度过滤
http.HandleFunc("/result/value/length", viewValueLengthFilterResult)
//删除任务
http.HandleFunc("/delete", delete)
err := http.ListenAndServe(":"+port, nil)
if err != nil {
logs.Log("error in main : " + err.Error())
os.Exit(-1)
}
}
/**
接受一个redis rdb文件的byte数组，并解析
List Encoding 与 Set Encoding 与 Sorted Set Enconding 的编码方式是一样的
Sorted Set in Ziplist Encoding 与 Ziplist Encoding 的编码方式是一样的
Zipmap encoding are deprecated starting Redis 2.6. Small hashmaps are now encoded using ziplists.
pwflag用来判断是打印还是输出到文件中
*/
func parseRedisRdb(array *[]byte, inputFile string, param *Parameters) {
//解析固定信息
//固定信息只能解析一次
if (*param).FixedInfo == false {
parseFixedInformation(array, param)
}
defer func() {
if r := recover(); r != nil {
//设置解析失败
(*param).Success = false
//记录log
logs.Log(r)
}
}()
var flag bool = true
for flag {
reIndex(array, inputFile, 1, param)
slice := getArraySlice((*param).SliceStart, (*param).SliceEnd, array)
var i uint8 = uint8(slice[0])
if i == 255 {
break
}
switch i {
case 0:
(*param).TotalNum++
key, _ := parseKeyString(array, inputFile, param)
value, length := parseStringEncodingValue(array, inputFile, param)
str := "value type String Encoding: " + key + "--->" + value
dealResult(key, length, str, 0, param)
case 1:
(*param).TotalNum++
key, _ := parseKeyString(array, inputFile, param)
length := parseListEncodingValue(array, inputFile, param)
str := "value type List Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
dealResult(key, length, str, 1, param)
case 2:
(*param).TotalNum++
key, _ := parseKeyString(array, inputFile, param)
length := parseSetEncodingValue(array, inputFile, param)
str := "value type Set Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
dealResult(key, length, str, 2, param)
case 3:
(*param).TotalNum++
key, _ := parseKeyString(array, inputFile, param)
length := parseSortedSetEncodingValue(array, inputFile, param)
str := "value type Sorted Set Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
dealResult(key, length, str, 3, param)
case 4:
(*param).TotalNum++
key, _ := parseKeyString(array, inputFile, param)
length := parseHashEncodingValue(array, inputFile, param)
str := "value type Hash Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
dealResult(key, length, str, 4, param)
case 9: //2.6以前的版本有这个
(*param).TotalNum++
key, _ := parseKeyString(array, inputFile, param)
length := parseZipMapEncodingValue(array, inputFile, param)
str := "value type Zipmap Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
dealResult(key, length, str, 9, param)
case 10:
(*param).TotalNum++
key, _ := parseKeyString(array, inputFile, param)
length := parseZiplistEncodingValue(array, inputFile, param)
str := "value type Ziplist Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
dealResult(key, length, str, 10, param)
case 11:
(*param).TotalNum++
key, _ := parseKeyString(array, inputFile, param)
length := parseIntSetEncoding(array, inputFile, param)
str := "value type Intset Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
dealResult(key, length, str, 11, param)
case 12:
(*param).TotalNum++
key, _ := parseKeyString(array, inputFile, param)
length := parseZiplistEncodingValue(array, inputFile, param)
str := "value type Sorted Set In Ziplist Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
dealResult(key, length, str, 12, param)
case 13:
(*param).TotalNum++
key, _ := parseKeyString(array, inputFile, param)
length := parseHashMapInZiplistEncodingValue(array, inputFile, param)
str := "value type HashMap In Ziplist Encoding: " + key + "--->" + strconv.FormatInt(int64(length), 10)
dealResult(key, length, str, 13, param)
case 252: //后边跟的有效期是毫秒(ms)
parseExpireTimeInMillSeconds(array, inputFile, param)
case 253: //后边跟的是有效期是秒(s)
parseExpireTimeInSeconds(array, inputFile, param)
case 254: //redis 数据库编号
parseRedisDBNum(array, inputFile, param)
default:
logs.Log("error occurred in parseFunc : " + strconv.FormatInt(int64(i), 10))
(*param).Success = false
return
}
}
(*param).Success = true
}
//获取新的任务
func FetchNewTask() (int, Task) {
var task Task
//查询
stmtQuery, err := db.Prepare("select host, port, filter_length, filter_key, task_id, create_time, update_time from rdb_parse_task where status = 0 order by priority desc limit 1")
defer stmtQuery.Close()
if err != nil {
logs.Log("fetch new task error in dbutil db.stmtQuery Prepare : " + err.Error())
return -1, task
}
rows, errs := stmtQuery.Query()
if errs != nil {
logs.Log("fetch new task error in dbutil stmtIns.Query Exec : " + errs.Error())
return -2, task
}
var host string
var port string
var filterLength int
var filterKey string
var taskId string
var createTime string
var updateTime string
for rows.Next() {
rows.Scan(&host, &port, &filterLength, &filterKey, &taskId, &createTime, &updateTime)
task.Host = host
task.Port = port
task.FilterLength = filterLength
task.TaskId = taskId
task.CreateTime = createTime
task.UpdateTime = updateTime
task.FilterKey = filterKey
}
if taskId != "" {
//更改任务状态
stmtUpdate, errUpdate := db.Prepare("update rdb_parse_task set status = 5 where task_id = ? and update_time = ?")
defer stmtUpdate.Close()
if errUpdate != nil {
logs.Log("fetch new task error in dbutil db.stmtUpdate : " + errUpdate.Error())
return -3, task
}
result, err := stmtUpdate.Exec(taskId, updateTime)
if err != nil {
logs.Log("fetch new task error in dbutil db.stmtUpdate Exec : " + err.Error())
return -4, task
}
value, _ := result.RowsAffected()
//证明记录没有被修改过
if value == 1 {
return 1, task
} else {
//证明记录已经被修改了，有其它的实例获得了任务
return -5, task
}
} else {
return 1, task
}
}