func main() {
gobatsd.LoadConfig()
spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
redis, redisErr := redis.NewSynchClientWithSpec(spec)
if redisErr != nil {
panic(redisErr)
}
datapoints, redisErr := redis.Smembers("datapoints")
if redisErr != nil {
panic(redisErr)
}
for j := range retentions {
outfile, err := os.Create(fmt.Sprintf("counters_export_%v.tsv", retentions[j]))
if err != nil {
panic(err)
}
writer := bufio.NewWriter(outfile)
for i := range datapoints {
if datapoints[i][0] == 'c' {
path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
fmt.Printf("(%v / %v) Migrating %v:%v from %v\n", i, len(datapoints), string(datapoints[i]), retentions[j], path)
file, err := os.Open(path)
if err == nil {
reader := bufio.NewReader(file)
linesRead := 0
for {
line, err := reader.ReadString('\n')
linesRead += 1
if err != nil && err != io.EOF {
panic(err)
}
if err != nil && err == io.EOF {
break
}
if linesRead == 1 {
continue
}
parts := strings.Split(strings.TrimSpace(line), " ")
value, _ := strconv.ParseFloat(parts[1], 64)
writer.WriteString(fmt.Sprintf("%v\t%v\t%s\n", string(datapoints[i]), parts[0], gobatsd.EncodeFloat64(value)))
}
} else {
fmt.Printf("%v\n", err)
}
file.Close()
writer.Flush()
}
}
outfile.Close()
fmt.Printf("Import with: -Dimporttsv.columns=HBASE_ROW_KEY,HBASE_TS_KEY,interval%v:value\n", retentions[j])
}
}
func TruncateOnDisk(metric string, since float64) {
filePath := gobatsd.CalculateFilename(metric, gobatsd.Config.Root)
file, err := os.Open(filePath)
tmpfile, writeErr := os.Create(filePath + "tmp")
if writeErr != nil {
panic(writeErr)
}
if err == nil && writeErr == nil {
linesWritten := 0
reader := bufio.NewReader(file)
writer := bufio.NewWriter(tmpfile)
for {
line, err := reader.ReadString('\n')
if err != nil && err != io.EOF {
panic(err)
}
if err != nil && err == io.EOF {
break
}
//skip the header in v2 files
if line[0] == 'v' {
writer.WriteString(line)
} else {
parts := strings.Split(strings.TrimSpace(line), " ")
ts, _ := strconv.ParseFloat(parts[0], 64)
if ts >= since {
writer.WriteString(line)
linesWritten++
}
}
}
writer.Flush()
file.Close()
tmpfile.Close()
if linesWritten > 0 {
os.Rename(filePath+"tmp", filePath)
} else {
os.Remove(filePath + "tmp")
os.Remove(filePath)
}
}
}
func main() {
gobatsd.LoadConfig()
spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
redis, redisErr := redis.NewSynchClientWithSpec(spec)
if redisErr != nil {
panic(redisErr)
}
datapoints, redisErr := redis.Smembers("datapoints")
if redisErr != nil {
panic(redisErr)
}
for j := range retentions {
fileNo := 1
outfile, err := os.Create(fmt.Sprintf("timers_export_%v-%v.tsv", retentions[j], fileNo))
if err != nil {
panic(err)
}
writer := bufio.NewWriter(outfile)
for i := range datapoints {
if i == fileNo*maxPerFile {
outfile.Close()
fileNo += 1
outfile, err = os.Create(fmt.Sprintf("timers_export_%v-%v.tsv", retentions[j], fileNo))
if err != nil {
panic(err)
}
writer = bufio.NewWriter(outfile)
}
if datapoints[i][0] == 't' && (len(datapoints[i]) < 16 || string(datapoints[i][0:15]) != "timers:sysstat.") {
path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v:2", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
fmt.Printf("(%v / %v) Migrating %v:%v from %v\n", i, len(datapoints), string(datapoints[i]), retentions[j], path)
file, err := os.Open(path)
if err == nil {
reader := bufio.NewReader(file)
linesRead := 0
for {
line, err := reader.ReadString('\n')
linesRead += 1
if err != nil && err != io.EOF {
panic(err)
}
if err != nil && err == io.EOF {
break
}
if linesRead == 1 {
continue
}
parts := strings.Split(strings.TrimSpace(line), " ")
ts, _ := strconv.ParseInt(parts[0], 10, 64)
if ts > 1 {
timerComponents := strings.Split(parts[1], "/")
count, _ := strconv.ParseFloat(timerComponents[0], 64)
min, _ := strconv.ParseFloat(timerComponents[1], 64)
max, _ := strconv.ParseFloat(timerComponents[2], 64)
median, _ := strconv.ParseFloat(timerComponents[3], 64)
mean, _ := strconv.ParseFloat(timerComponents[4], 64)
stddev, _ := strconv.ParseFloat(timerComponents[5], 64)
percentile_90, _ := strconv.ParseFloat(timerComponents[6], 64)
percentile_95, _ := strconv.ParseFloat(timerComponents[7], 64)
percentile_99, _ := strconv.ParseFloat(timerComponents[8], 64)
writer.WriteString(fmt.Sprintf("%v\t%v\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", string(datapoints[i]), parts[0],
gobatsd.EncodeFloat64(count), gobatsd.EncodeFloat64(min), gobatsd.EncodeFloat64(max), gobatsd.EncodeFloat64(median),
gobatsd.EncodeFloat64(mean), gobatsd.EncodeFloat64(stddev), gobatsd.EncodeFloat64(percentile_90), gobatsd.EncodeFloat64(percentile_95),
gobatsd.EncodeFloat64(percentile_99)))
}
}
} else {
fmt.Printf("%v\n", err)
}
file.Close()
writer.Flush()
}
}
outfile.Close()
fmt.Printf("Import with: -Dimporttsv.columns=HBASE_ROW_KEY,HBASE_TS_KEY,interval%v:count\n", retentions[j])
}
}
func main() {
gobatsd.LoadConfig()
runtime.GOMAXPROCS(runtime.NumCPU())
spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
redis, redisErr := redis.NewSynchClientWithSpec(spec)
if redisErr != nil {
panic(redisErr)
}
datapoints, redisErr := redis.Smembers("datapoints")
if redisErr != nil {
panic(redisErr)
}
nworkers := 10
c := make(chan int, nworkers)
for k := 0; k < nworkers; k++ {
go func(datapoints [][]byte, c chan int) {
cli, err := gobatsd.OpenHbaseConnection()
if err != nil {
panic(err)
}
hbaseClient := cli.(*goh.HClient)
defer hbaseClient.Close()
for i := range datapoints {
if datapoints[i][0] == 'g' && (len(datapoints[i]) < 18 || string(datapoints[i][0:17]) != "gauges:Syslog-NG.") {
path := gobatsd.CalculateFilename(string(datapoints[i]), gobatsd.Config.Root)
fmt.Printf("(%v / %v) Migrating %v from %v\n", i, len(datapoints), string(datapoints[i]), path)
file, err := os.Open(path)
if err == nil {
reader := bufio.NewReader(file)
linesRead := 0
for {
line, err := reader.ReadString('\n')
linesRead += 1
if err != nil && err != io.EOF {
panic(err)
}
if err != nil && err == io.EOF {
break
}
if linesRead == 1 {
continue
}
parts := strings.Split(strings.TrimSpace(line), " ")
ts, _ := strconv.ParseInt(parts[0], 10, 64)
value, _ := strconv.ParseFloat(parts[1], 64)
if value != float64(0) && ts > yearago {
mutations := []*Hbase.Mutation{goh.NewMutation("interval0:value", gobatsd.EncodeFloat64(value))}
err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
if err != nil {
fmt.Printf("%v: mutate error - %v\n", time.Now(), err)
}
}
}
} else {
fmt.Printf("%v\n", err)
}
/*} else if datapoints[i][0] == 'c' {
for j := range retentions {
path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
fmt.Printf("Migrating %v:%v from %v\n", string(datapoints[i]), retentions[j], path)
file, err := os.Open(path)
if err == nil {
reader := bufio.NewReader(file)
linesRead := 0
for {
line, err := reader.ReadString('\n')
linesRead += 1
if err != nil && err != io.EOF {
panic(err)
}
if err != nil && err == io.EOF {
break
}
if linesRead == 1 {
continue
}
parts := strings.Split(strings.TrimSpace(line), " ")
ts, _ := strconv.ParseInt(parts[0], 10, 64)
value, _ := strconv.ParseFloat(parts[1], 64)
mutations := []*Hbase.Mutation{goh.NewMutation(fmt.Sprintf("interval%v:value", retentions[j]), gobatsd.EncodeFloat64(value))}
err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
if err != nil {
err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
if err != nil {
fmt.Printf("%v: mutate error - %v\n", time.Now(), err)
}
}
}
} else {
fmt.Printf("%v\n", err)
}
}*/
} else if datapoints[i][0] == 't' && (len(datapoints[i]) < 16 || string(datapoints[i][0:15]) != "timers:sysstat.") {
for j := range retentions {
path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v:2", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
fmt.Printf("(%v / %v) Migrating %v:%v from %v\n", i, len(datapoints), string(datapoints[i]), retentions[j], path)
file, err := os.Open(path)
if err == nil {
reader := bufio.NewReader(file)
linesRead := 0
for {
line, err := reader.ReadString('\n')
linesRead += 1
if err != nil && err != io.EOF {
panic(err)
}
if err != nil && err == io.EOF {
break
}
if linesRead == 1 {
continue
}
parts := strings.Split(strings.TrimSpace(line), " ")
ts, _ := strconv.ParseInt(parts[0], 10, 64)
timerComponents := strings.Split(parts[1], "/")
if ts > yearago {
mutations := make([]*Hbase.Mutation, 0)
for k, v := range timerHeader {
val, _ := strconv.ParseFloat(timerComponents[v], 64)
mutations = append(mutations, goh.NewMutation(fmt.Sprintf("interval%v:%v", retentions[j], k), gobatsd.EncodeFloat64(val)))
}
err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
if err != nil {
fmt.Printf("%v: mutate error - %v\n", time.Now(), err)
}
}
}
} else {
fmt.Printf("%v\n", err)
}
}
}
}
c <- 1
}(datapoints[k*len(datapoints)/nworkers:(k+1)*len(datapoints)/nworkers-1], c)
}
for i := 0; i < nworkers; i++ {
<-c
}
}
func main() {
gobatsd.LoadConfig()
spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
redis, redisErr := redis.NewSynchClientWithSpec(spec)
if redisErr != nil {
panic(redisErr)
}
datapoints, redisErr := redis.Smembers("datapoints")
if redisErr != nil {
panic(redisErr)
}
fileNo := 1
outfile, err := os.Create(fmt.Sprintf("gauges_export-%v.tsv", fileNo))
if err != nil {
panic(err)
}
writer := bufio.NewWriter(outfile)
for i := range datapoints {
if i == fileNo*maxPerFile {
outfile.Close()
fileNo += 1
outfile, err = os.Create(fmt.Sprintf("gauges_export-%v.tsv", fileNo))
if err != nil {
panic(err)
}
writer = bufio.NewWriter(outfile)
}
if datapoints[i][0] == 'g' && (len(datapoints[i]) < 18 || string(datapoints[i][0:17]) != "gauges:Syslog-NG.") {
path := gobatsd.CalculateFilename(string(datapoints[i]), gobatsd.Config.Root)
fmt.Printf("(%v / %v) Migrating %v from %v\n", i, len(datapoints), string(datapoints[i]), path)
file, err := os.Open(path)
if err == nil {
reader := bufio.NewReader(file)
linesRead := 0
for {
line, err := reader.ReadString('\n')
linesRead += 1
if err != nil && err != io.EOF {
fmt.Println(err)
}
if err != nil && err == io.EOF {
break
}
if linesRead == 1 {
continue
}
parts := strings.Split(strings.TrimSpace(line), " ")
ts, _ := strconv.ParseInt(parts[0], 10, 64)
if len(parts) == 2 {
value, _ := strconv.ParseFloat(parts[1], 64)
if value != float64(0) && ts > yearago {
_, err := writer.WriteString(fmt.Sprintf("%v\t%v\t%s\n", string(datapoints[i]), parts[0], gobatsd.EncodeFloat64(value)))
if err != nil {
fmt.Println(err)
outfile.Close()
fileNo += 1
outfile, err = os.Create(fmt.Sprintf("sc_gauges_export-%v.tsv", fileNo))
if err != nil {
panic(err)
}
writer = bufio.NewWriter(outfile)
}
}
}
}
} else {
fmt.Printf("%v\n", err)
}
file.Close()
writer.Flush()
}
}
outfile.Close()
fmt.Printf("Import with: -Dimporttsv.columns=HBASE_ROW_KEY,HBASE_TS_KEY,interval0:value\n")
}