Example #1
0
func main() {
	gobatsd.LoadConfig()

	spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
	redis, redisErr := redis.NewSynchClientWithSpec(spec)
	if redisErr != nil {
		panic(redisErr)
	}
	datapoints, redisErr := redis.Smembers("datapoints")
	if redisErr != nil {
		panic(redisErr)
	}

	for j := range retentions {
		outfile, err := os.Create(fmt.Sprintf("counters_export_%v.tsv", retentions[j]))
		if err != nil {
			panic(err)
		}

		writer := bufio.NewWriter(outfile)
		for i := range datapoints {
			if datapoints[i][0] == 'c' {
				path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
				fmt.Printf("(%v / %v) Migrating %v:%v from %v\n", i, len(datapoints), string(datapoints[i]), retentions[j], path)
				file, err := os.Open(path)
				if err == nil {
					reader := bufio.NewReader(file)
					linesRead := 0
					for {
						line, err := reader.ReadString('\n')
						linesRead += 1
						if err != nil && err != io.EOF {
							panic(err)
						}
						if err != nil && err == io.EOF {
							break
						}
						if linesRead == 1 {
							continue
						}

						parts := strings.Split(strings.TrimSpace(line), " ")
						value, _ := strconv.ParseFloat(parts[1], 64)
						writer.WriteString(fmt.Sprintf("%v\t%v\t%s\n", string(datapoints[i]), parts[0], gobatsd.EncodeFloat64(value)))
					}
				} else {
					fmt.Printf("%v\n", err)
				}
				file.Close()

				writer.Flush()
			}

		}
		outfile.Close()
		fmt.Printf("Import with: -Dimporttsv.columns=HBASE_ROW_KEY,HBASE_TS_KEY,interval%v:value\n", retentions[j])
	}
}
Example #2
0
func TruncateOnDisk(metric string, since float64) {
	filePath := gobatsd.CalculateFilename(metric, gobatsd.Config.Root)
	file, err := os.Open(filePath)
	tmpfile, writeErr := os.Create(filePath + "tmp")
	if writeErr != nil {
		panic(writeErr)
	}
	if err == nil && writeErr == nil {
		linesWritten := 0
		reader := bufio.NewReader(file)
		writer := bufio.NewWriter(tmpfile)
		for {
			line, err := reader.ReadString('\n')
			if err != nil && err != io.EOF {
				panic(err)
			}
			if err != nil && err == io.EOF {
				break
			}
			//skip the header in v2 files
			if line[0] == 'v' {
				writer.WriteString(line)
			} else {

				parts := strings.Split(strings.TrimSpace(line), " ")
				ts, _ := strconv.ParseFloat(parts[0], 64)
				if ts >= since {
					writer.WriteString(line)
					linesWritten++
				}
			}
		}
		writer.Flush()
		file.Close()
		tmpfile.Close()
		if linesWritten > 0 {
			os.Rename(filePath+"tmp", filePath)
		} else {
			os.Remove(filePath + "tmp")
			os.Remove(filePath)
		}
	}

}
Example #3
0
func main() {
	gobatsd.LoadConfig()

	spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
	redis, redisErr := redis.NewSynchClientWithSpec(spec)
	if redisErr != nil {
		panic(redisErr)
	}
	datapoints, redisErr := redis.Smembers("datapoints")
	if redisErr != nil {
		panic(redisErr)
	}

	for j := range retentions {
		fileNo := 1
		outfile, err := os.Create(fmt.Sprintf("timers_export_%v-%v.tsv", retentions[j], fileNo))
		if err != nil {
			panic(err)
		}

		writer := bufio.NewWriter(outfile)
		for i := range datapoints {
			if i == fileNo*maxPerFile {
				outfile.Close()
				fileNo += 1
				outfile, err = os.Create(fmt.Sprintf("timers_export_%v-%v.tsv", retentions[j], fileNo))
				if err != nil {
					panic(err)
				}
				writer = bufio.NewWriter(outfile)
			}
			if datapoints[i][0] == 't' && (len(datapoints[i]) < 16 || string(datapoints[i][0:15]) != "timers:sysstat.") {
				path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v:2", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
				fmt.Printf("(%v / %v) Migrating %v:%v from %v\n", i, len(datapoints), string(datapoints[i]), retentions[j], path)
				file, err := os.Open(path)
				if err == nil {
					reader := bufio.NewReader(file)
					linesRead := 0
					for {
						line, err := reader.ReadString('\n')
						linesRead += 1
						if err != nil && err != io.EOF {
							panic(err)
						}
						if err != nil && err == io.EOF {
							break
						}
						if linesRead == 1 {
							continue
						}

						parts := strings.Split(strings.TrimSpace(line), " ")
						ts, _ := strconv.ParseInt(parts[0], 10, 64)
						if ts > 1 {
							timerComponents := strings.Split(parts[1], "/")
							count, _ := strconv.ParseFloat(timerComponents[0], 64)
							min, _ := strconv.ParseFloat(timerComponents[1], 64)
							max, _ := strconv.ParseFloat(timerComponents[2], 64)
							median, _ := strconv.ParseFloat(timerComponents[3], 64)
							mean, _ := strconv.ParseFloat(timerComponents[4], 64)
							stddev, _ := strconv.ParseFloat(timerComponents[5], 64)
							percentile_90, _ := strconv.ParseFloat(timerComponents[6], 64)
							percentile_95, _ := strconv.ParseFloat(timerComponents[7], 64)
							percentile_99, _ := strconv.ParseFloat(timerComponents[8], 64)

							writer.WriteString(fmt.Sprintf("%v\t%v\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", string(datapoints[i]), parts[0],
								gobatsd.EncodeFloat64(count), gobatsd.EncodeFloat64(min), gobatsd.EncodeFloat64(max), gobatsd.EncodeFloat64(median),
								gobatsd.EncodeFloat64(mean), gobatsd.EncodeFloat64(stddev), gobatsd.EncodeFloat64(percentile_90), gobatsd.EncodeFloat64(percentile_95),
								gobatsd.EncodeFloat64(percentile_99)))
						}
					}
				} else {
					fmt.Printf("%v\n", err)
				}
				file.Close()

				writer.Flush()
			}

		}
		outfile.Close()
		fmt.Printf("Import with: -Dimporttsv.columns=HBASE_ROW_KEY,HBASE_TS_KEY,interval%v:count\n", retentions[j])
	}
}
Example #4
0
func main() {
	gobatsd.LoadConfig()

	runtime.GOMAXPROCS(runtime.NumCPU())
	spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
	redis, redisErr := redis.NewSynchClientWithSpec(spec)
	if redisErr != nil {
		panic(redisErr)
	}
	datapoints, redisErr := redis.Smembers("datapoints")
	if redisErr != nil {
		panic(redisErr)
	}
	nworkers := 10
	c := make(chan int, nworkers)

	for k := 0; k < nworkers; k++ {
		go func(datapoints [][]byte, c chan int) {
			cli, err := gobatsd.OpenHbaseConnection()
			if err != nil {
				panic(err)
			}
			hbaseClient := cli.(*goh.HClient)
			defer hbaseClient.Close()

			for i := range datapoints {
				if datapoints[i][0] == 'g' && (len(datapoints[i]) < 18 || string(datapoints[i][0:17]) != "gauges:Syslog-NG.") {
					path := gobatsd.CalculateFilename(string(datapoints[i]), gobatsd.Config.Root)
					fmt.Printf("(%v / %v) Migrating %v from %v\n", i, len(datapoints), string(datapoints[i]), path)
					file, err := os.Open(path)
					if err == nil {
						reader := bufio.NewReader(file)
						linesRead := 0
						for {
							line, err := reader.ReadString('\n')
							linesRead += 1
							if err != nil && err != io.EOF {
								panic(err)
							}
							if err != nil && err == io.EOF {
								break
							}
							if linesRead == 1 {
								continue
							}

							parts := strings.Split(strings.TrimSpace(line), " ")
							ts, _ := strconv.ParseInt(parts[0], 10, 64)
							value, _ := strconv.ParseFloat(parts[1], 64)
							if value != float64(0) && ts > yearago {
								mutations := []*Hbase.Mutation{goh.NewMutation("interval0:value", gobatsd.EncodeFloat64(value))}

								err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
								if err != nil {
									fmt.Printf("%v: mutate error -  %v\n", time.Now(), err)
								}
							}
						}
					} else {
						fmt.Printf("%v\n", err)
					}
					/*} else if datapoints[i][0] == 'c' {
					for j := range retentions {
						path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
						fmt.Printf("Migrating %v:%v from %v\n", string(datapoints[i]), retentions[j], path)
						file, err := os.Open(path)
						if err == nil {
							reader := bufio.NewReader(file)
							linesRead := 0
							for {
								line, err := reader.ReadString('\n')
								linesRead += 1
								if err != nil && err != io.EOF {
									panic(err)
								}
								if err != nil && err == io.EOF {
									break
								}
								if linesRead == 1 {
									continue
								}

								parts := strings.Split(strings.TrimSpace(line), " ")
								ts, _ := strconv.ParseInt(parts[0], 10, 64)
								value, _ := strconv.ParseFloat(parts[1], 64)

								mutations := []*Hbase.Mutation{goh.NewMutation(fmt.Sprintf("interval%v:value", retentions[j]), gobatsd.EncodeFloat64(value))}

								err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
								if err != nil {
									err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
									if err != nil {
										fmt.Printf("%v: mutate error -  %v\n", time.Now(), err)
									}
								}
							}
						} else {
							fmt.Printf("%v\n", err)
						}

					}*/
				} else if datapoints[i][0] == 't' && (len(datapoints[i]) < 16 || string(datapoints[i][0:15]) != "timers:sysstat.") {
					for j := range retentions {
						path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v:2", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
						fmt.Printf("(%v / %v) Migrating %v:%v from %v\n", i, len(datapoints), string(datapoints[i]), retentions[j], path)
						file, err := os.Open(path)
						if err == nil {
							reader := bufio.NewReader(file)
							linesRead := 0
							for {
								line, err := reader.ReadString('\n')
								linesRead += 1
								if err != nil && err != io.EOF {
									panic(err)
								}
								if err != nil && err == io.EOF {
									break
								}
								if linesRead == 1 {
									continue
								}

								parts := strings.Split(strings.TrimSpace(line), " ")
								ts, _ := strconv.ParseInt(parts[0], 10, 64)
								timerComponents := strings.Split(parts[1], "/")

								if ts > yearago {
									mutations := make([]*Hbase.Mutation, 0)
									for k, v := range timerHeader {
										val, _ := strconv.ParseFloat(timerComponents[v], 64)
										mutations = append(mutations, goh.NewMutation(fmt.Sprintf("interval%v:%v", retentions[j], k), gobatsd.EncodeFloat64(val)))
									}
									err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
									if err != nil {
										fmt.Printf("%v: mutate error -  %v\n", time.Now(), err)
									}
								}
							}
						} else {
							fmt.Printf("%v\n", err)
						}
					}

				}

			}
			c <- 1
		}(datapoints[k*len(datapoints)/nworkers:(k+1)*len(datapoints)/nworkers-1], c)
	}

	for i := 0; i < nworkers; i++ {
		<-c
	}
}
Example #5
0
func main() {
	gobatsd.LoadConfig()

	spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
	redis, redisErr := redis.NewSynchClientWithSpec(spec)
	if redisErr != nil {
		panic(redisErr)
	}
	datapoints, redisErr := redis.Smembers("datapoints")
	if redisErr != nil {
		panic(redisErr)
	}

	fileNo := 1
	outfile, err := os.Create(fmt.Sprintf("gauges_export-%v.tsv", fileNo))
	if err != nil {
		panic(err)
	}

	writer := bufio.NewWriter(outfile)
	for i := range datapoints {
		if i == fileNo*maxPerFile {
			outfile.Close()
			fileNo += 1
			outfile, err = os.Create(fmt.Sprintf("gauges_export-%v.tsv", fileNo))
			if err != nil {
				panic(err)
			}
			writer = bufio.NewWriter(outfile)
		}
		if datapoints[i][0] == 'g' && (len(datapoints[i]) < 18 || string(datapoints[i][0:17]) != "gauges:Syslog-NG.") {
			path := gobatsd.CalculateFilename(string(datapoints[i]), gobatsd.Config.Root)
			fmt.Printf("(%v / %v) Migrating %v from %v\n", i, len(datapoints), string(datapoints[i]), path)
			file, err := os.Open(path)
			if err == nil {
				reader := bufio.NewReader(file)
				linesRead := 0
				for {
					line, err := reader.ReadString('\n')
					linesRead += 1
					if err != nil && err != io.EOF {
						fmt.Println(err)
					}
					if err != nil && err == io.EOF {
						break
					}
					if linesRead == 1 {
						continue
					}

					parts := strings.Split(strings.TrimSpace(line), " ")
					ts, _ := strconv.ParseInt(parts[0], 10, 64)
					if len(parts) == 2 {
						value, _ := strconv.ParseFloat(parts[1], 64)
						if value != float64(0) && ts > yearago {
							_, err := writer.WriteString(fmt.Sprintf("%v\t%v\t%s\n", string(datapoints[i]), parts[0], gobatsd.EncodeFloat64(value)))
							if err != nil {
								fmt.Println(err)
								outfile.Close()
								fileNo += 1
								outfile, err = os.Create(fmt.Sprintf("sc_gauges_export-%v.tsv", fileNo))
								if err != nil {
									panic(err)
								}
								writer = bufio.NewWriter(outfile)

							}
						}
					}
				}
			} else {
				fmt.Printf("%v\n", err)
			}
			file.Close()

			writer.Flush()
		}

	}
	outfile.Close()
	fmt.Printf("Import with: -Dimporttsv.columns=HBASE_ROW_KEY,HBASE_TS_KEY,interval0:value\n")

}