func main() {
gobatsd.LoadConfig()
fmt.Printf("Starting on port %v, root dir %v\n", gobatsd.Config.Port, gobatsd.Config.Root)
server, err := net.Listen("tcp", ":"+gobatsd.Config.Port)
if err != nil {
panic(err)
}
numClients := 0
hbasePool = gobatsd.MakeHbasePool(10)
for {
client, err := server.Accept()
if client == nil {
fmt.Printf("couldn't accept: %v", err)
continue
}
numClients++
fmt.Printf("Opened connection #%d: %v <-> %v\n", numClients, client.LocalAddr(), client.RemoteAddr())
go func() {
r, err := redis.Dial("tcp", fmt.Sprintf("%v:%v", gobatsd.Config.RedisHost, gobatsd.Config.RedisPort))
if err != nil {
panic(err)
}
defer r.Close()
(&Client{client, r}).Serve()
}()
}
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
gobatsd.LoadConfig()
if gobatsd.ProfileCPU {
cpuprof, err := os.Create(fmt.Sprintf("cpuprof-%v", time.Now().Unix()))
if err != nil {
panic(err)
}
defer cpuprof.Close()
pprof.StartCPUProfile(cpuprof)
defer pprof.StopCPUProfile()
}
processingChannel := clamp.StartExplodingDualServer(":" + gobatsd.Config.Port)
clamp.StartStatsServer(":8124")
gobatsd.SetupDatastore()
fmt.Printf("Starting on port %v\n", gobatsd.Config.Port)
gobatsd.InitializeInternalMetrics()
gaugeChannel = make(chan gobatsd.Datapoint, channelBufferSize)
counterChannel = make(chan gobatsd.Datapoint, channelBufferSize)
timerChannel = make(chan gobatsd.Datapoint, channelBufferSize)
channels := map[string]chan gobatsd.Datapoint{"g": gaugeChannel, "c": counterChannel, "ms": timerChannel}
for i := 0; i < numIncomingMessageProcessors; i++ {
go func(processingChannel chan string) {
for {
message := <-processingChannel
d := gobatsd.ParseDatapointFromString(message)
if ch, ok := channels[d.Datatype]; ok {
ch <- d
}
}
}(processingChannel)
}
go func() {
c := time.Tick(1 * time.Second)
for {
<-c
clamp.StatsChannel <- clamp.Stat{"gaugeChannelSize", fmt.Sprintf("%v", len(gaugeChannel))}
clamp.StatsChannel <- clamp.Stat{"counterChannelSize", fmt.Sprintf("%v", len(counterChannel))}
clamp.StatsChannel <- clamp.Stat{"timerChannelSize", fmt.Sprintf("%v", len(timerChannel))}
}
}()
processDatatype("gauges", gaugeChannel, gobatsd.NewGauge)
processDatatype("timers", timerChannel, gobatsd.NewTimer)
processDatatype("counters", counterChannel, gobatsd.NewCounter)
terminate := make(chan os.Signal)
signal.Notify(terminate, os.Interrupt)
<-terminate
fmt.Printf("Server stopped")
}
func main() {
gobatsd.LoadConfig()
spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
redis, redisErr := redis.NewSynchClientWithSpec(spec)
if redisErr != nil {
panic(redisErr)
}
datapoints, redisErr := redis.Smembers("datapoints")
if redisErr != nil {
panic(redisErr)
}
for j := range retentions {
outfile, err := os.Create(fmt.Sprintf("counters_export_%v.tsv", retentions[j]))
if err != nil {
panic(err)
}
writer := bufio.NewWriter(outfile)
for i := range datapoints {
if datapoints[i][0] == 'c' {
path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
fmt.Printf("(%v / %v) Migrating %v:%v from %v\n", i, len(datapoints), string(datapoints[i]), retentions[j], path)
file, err := os.Open(path)
if err == nil {
reader := bufio.NewReader(file)
linesRead := 0
for {
line, err := reader.ReadString('\n')
linesRead += 1
if err != nil && err != io.EOF {
panic(err)
}
if err != nil && err == io.EOF {
break
}
if linesRead == 1 {
continue
}
parts := strings.Split(strings.TrimSpace(line), " ")
value, _ := strconv.ParseFloat(parts[1], 64)
writer.WriteString(fmt.Sprintf("%v\t%v\t%s\n", string(datapoints[i]), parts[0], gobatsd.EncodeFloat64(value)))
}
} else {
fmt.Printf("%v\n", err)
}
file.Close()
writer.Flush()
}
}
outfile.Close()
fmt.Printf("Import with: -Dimporttsv.columns=HBASE_ROW_KEY,HBASE_TS_KEY,interval%v:value\n", retentions[j])
}
}
func main() {
prof, err := os.Create("/tmp/proxy.prof")
if err != nil {
panic(err)
}
defer prof.Close()
pprof.StartCPUProfile(prof)
defer pprof.StopCPUProfile()
gobatsd.LoadConfig()
fmt.Printf("Starting on port %v\n", gobatsd.Config.Port)
server, err := net.ListenPacket("udp", ":"+gobatsd.Config.Port)
defer server.Close()
if err != nil {
panic(err)
}
destinationAddr, err := net.ResolveUDPAddr("udp", ":8225")
if err != nil {
panic(err)
}
client, err := net.DialUDP("udp", nil, destinationAddr)
if err != nil {
panic(err)
}
//c := TCPRelay(client)
buffer := make([]byte, readLen)
for {
n, _, err := server.ReadFrom(buffer)
if err != nil {
continue
}
client.Write(buffer[0:n])
// c <- buffer[0:n]
}
}
func main() {
gobatsd.LoadConfig()
index := sort.Search(len(gobatsd.Config.Retentions), func(i int) bool { return gobatsd.Config.Retentions[i].Interval == gobatsd.Config.TargetInterval })
if index == len(gobatsd.Config.Retentions) {
if gobatsd.Config.Retentions[0].Interval == gobatsd.Config.TargetInterval {
index = 0
} else {
panic("You must specify the duration you'd like to truncate")
}
}
retention := gobatsd.Config.Retentions[index]
fmt.Printf("Starting truncation for the %v duration.\n", retention.Interval)
spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
redis, err := redis.NewSynchClientWithSpec(spec)
if err != nil {
panic(err)
}
datapoints := make([]string, 0)
if index == 0 {
rawDatapoints, err := redis.Keys("*")
if err != nil {
panic(err)
}
for _, key := range rawDatapoints {
if m, _ := regexp.MatchString("^timers|^counters", string(key)); m {
datapoints = append(datapoints, string(key))
}
}
} else {
rawDatapoints, err := redis.Smembers("datapoints")
if err != nil {
panic(err)
}
for _, key := range rawDatapoints {
if m, _ := regexp.MatchString("^timers|^counters", string(key)); m {
datapoints = append(datapoints, string(key))
}
}
}
since := float64(time.Now().Unix() - retention.Duration)
fmt.Printf("Truncating %v datapoints since %f.\n", len(datapoints), since)
if index == 0 { //Redis truncation
for _, key := range datapoints {
redis.Zremrangebyscore(key, 0.0, since)
}
} else {
nworkers := NWORKERS
if nworkers > len(datapoints) {
nworkers = len(datapoints)
}
c := make(chan int, nworkers)
for i := 0; i < nworkers; i++ {
go func(datapoints []string, i int, c chan int) {
for _, key := range datapoints {
metricName := key + ":" + strconv.FormatInt(retention.Interval, 10)
if m, _ := regexp.MatchString("^timers", metricName); m {
metricName += ":2"
}
TruncateOnDisk(metricName, since)
}
c <- 1
}(datapoints[i*len(datapoints)/nworkers:(i+1)*len(datapoints)/nworkers-1], i, c)
}
for i := 0; i < nworkers; i++ {
<-c
}
}
}
func main() {
gobatsd.LoadConfig()
runtime.GOMAXPROCS(runtime.NumCPU())
spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
redis, redisErr := redis.NewSynchClientWithSpec(spec)
if redisErr != nil {
panic(redisErr)
}
datapoints, redisErr := redis.Smembers("datapoints")
if redisErr != nil {
panic(redisErr)
}
nworkers := 10
c := make(chan int, nworkers)
for k := 0; k < nworkers; k++ {
go func(datapoints [][]byte, c chan int) {
cli, err := gobatsd.OpenHbaseConnection()
if err != nil {
panic(err)
}
hbaseClient := cli.(*goh.HClient)
defer hbaseClient.Close()
for i := range datapoints {
if datapoints[i][0] == 'g' && (len(datapoints[i]) < 18 || string(datapoints[i][0:17]) != "gauges:Syslog-NG.") {
path := gobatsd.CalculateFilename(string(datapoints[i]), gobatsd.Config.Root)
fmt.Printf("(%v / %v) Migrating %v from %v\n", i, len(datapoints), string(datapoints[i]), path)
file, err := os.Open(path)
if err == nil {
reader := bufio.NewReader(file)
linesRead := 0
for {
line, err := reader.ReadString('\n')
linesRead += 1
if err != nil && err != io.EOF {
panic(err)
}
if err != nil && err == io.EOF {
break
}
if linesRead == 1 {
continue
}
parts := strings.Split(strings.TrimSpace(line), " ")
ts, _ := strconv.ParseInt(parts[0], 10, 64)
value, _ := strconv.ParseFloat(parts[1], 64)
if value != float64(0) && ts > yearago {
mutations := []*Hbase.Mutation{goh.NewMutation("interval0:value", gobatsd.EncodeFloat64(value))}
err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
if err != nil {
fmt.Printf("%v: mutate error - %v\n", time.Now(), err)
}
}
}
} else {
fmt.Printf("%v\n", err)
}
/*} else if datapoints[i][0] == 'c' {
for j := range retentions {
path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
fmt.Printf("Migrating %v:%v from %v\n", string(datapoints[i]), retentions[j], path)
file, err := os.Open(path)
if err == nil {
reader := bufio.NewReader(file)
linesRead := 0
for {
line, err := reader.ReadString('\n')
linesRead += 1
if err != nil && err != io.EOF {
panic(err)
}
if err != nil && err == io.EOF {
break
}
if linesRead == 1 {
continue
}
parts := strings.Split(strings.TrimSpace(line), " ")
ts, _ := strconv.ParseInt(parts[0], 10, 64)
value, _ := strconv.ParseFloat(parts[1], 64)
mutations := []*Hbase.Mutation{goh.NewMutation(fmt.Sprintf("interval%v:value", retentions[j]), gobatsd.EncodeFloat64(value))}
err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
if err != nil {
err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
if err != nil {
fmt.Printf("%v: mutate error - %v\n", time.Now(), err)
}
}
}
} else {
fmt.Printf("%v\n", err)
}
}*/
} else if datapoints[i][0] == 't' && (len(datapoints[i]) < 16 || string(datapoints[i][0:15]) != "timers:sysstat.") {
for j := range retentions {
path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v:2", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
fmt.Printf("(%v / %v) Migrating %v:%v from %v\n", i, len(datapoints), string(datapoints[i]), retentions[j], path)
file, err := os.Open(path)
if err == nil {
reader := bufio.NewReader(file)
linesRead := 0
for {
line, err := reader.ReadString('\n')
linesRead += 1
if err != nil && err != io.EOF {
panic(err)
}
if err != nil && err == io.EOF {
break
}
if linesRead == 1 {
continue
}
parts := strings.Split(strings.TrimSpace(line), " ")
ts, _ := strconv.ParseInt(parts[0], 10, 64)
timerComponents := strings.Split(parts[1], "/")
if ts > yearago {
mutations := make([]*Hbase.Mutation, 0)
for k, v := range timerHeader {
val, _ := strconv.ParseFloat(timerComponents[v], 64)
mutations = append(mutations, goh.NewMutation(fmt.Sprintf("interval%v:%v", retentions[j], k), gobatsd.EncodeFloat64(val)))
}
err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
if err != nil {
fmt.Printf("%v: mutate error - %v\n", time.Now(), err)
}
}
}
} else {
fmt.Printf("%v\n", err)
}
}
}
}
c <- 1
}(datapoints[k*len(datapoints)/nworkers:(k+1)*len(datapoints)/nworkers-1], c)
}
for i := 0; i < nworkers; i++ {
<-c
}
}
func main() {
gobatsd.LoadConfig()
spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
redis, redisErr := redis.NewSynchClientWithSpec(spec)
if redisErr != nil {
panic(redisErr)
}
datapoints, redisErr := redis.Smembers("datapoints")
if redisErr != nil {
panic(redisErr)
}
for j := range retentions {
fileNo := 1
outfile, err := os.Create(fmt.Sprintf("timers_export_%v-%v.tsv", retentions[j], fileNo))
if err != nil {
panic(err)
}
writer := bufio.NewWriter(outfile)
for i := range datapoints {
if i == fileNo*maxPerFile {
outfile.Close()
fileNo += 1
outfile, err = os.Create(fmt.Sprintf("timers_export_%v-%v.tsv", retentions[j], fileNo))
if err != nil {
panic(err)
}
writer = bufio.NewWriter(outfile)
}
if datapoints[i][0] == 't' && (len(datapoints[i]) < 16 || string(datapoints[i][0:15]) != "timers:sysstat.") {
path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v:2", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
fmt.Printf("(%v / %v) Migrating %v:%v from %v\n", i, len(datapoints), string(datapoints[i]), retentions[j], path)
file, err := os.Open(path)
if err == nil {
reader := bufio.NewReader(file)
linesRead := 0
for {
line, err := reader.ReadString('\n')
linesRead += 1
if err != nil && err != io.EOF {
panic(err)
}
if err != nil && err == io.EOF {
break
}
if linesRead == 1 {
continue
}
parts := strings.Split(strings.TrimSpace(line), " ")
ts, _ := strconv.ParseInt(parts[0], 10, 64)
if ts > 1 {
timerComponents := strings.Split(parts[1], "/")
count, _ := strconv.ParseFloat(timerComponents[0], 64)
min, _ := strconv.ParseFloat(timerComponents[1], 64)
max, _ := strconv.ParseFloat(timerComponents[2], 64)
median, _ := strconv.ParseFloat(timerComponents[3], 64)
mean, _ := strconv.ParseFloat(timerComponents[4], 64)
stddev, _ := strconv.ParseFloat(timerComponents[5], 64)
percentile_90, _ := strconv.ParseFloat(timerComponents[6], 64)
percentile_95, _ := strconv.ParseFloat(timerComponents[7], 64)
percentile_99, _ := strconv.ParseFloat(timerComponents[8], 64)
writer.WriteString(fmt.Sprintf("%v\t%v\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", string(datapoints[i]), parts[0],
gobatsd.EncodeFloat64(count), gobatsd.EncodeFloat64(min), gobatsd.EncodeFloat64(max), gobatsd.EncodeFloat64(median),
gobatsd.EncodeFloat64(mean), gobatsd.EncodeFloat64(stddev), gobatsd.EncodeFloat64(percentile_90), gobatsd.EncodeFloat64(percentile_95),
gobatsd.EncodeFloat64(percentile_99)))
}
}
} else {
fmt.Printf("%v\n", err)
}
file.Close()
writer.Flush()
}
}
outfile.Close()
fmt.Printf("Import with: -Dimporttsv.columns=HBASE_ROW_KEY,HBASE_TS_KEY,interval%v:count\n", retentions[j])
}
}
func main() {
gobatsd.LoadConfig()
spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
redis, redisErr := redis.NewSynchClientWithSpec(spec)
if redisErr != nil {
panic(redisErr)
}
datapoints, redisErr := redis.Smembers("datapoints")
if redisErr != nil {
panic(redisErr)
}
fileNo := 1
outfile, err := os.Create(fmt.Sprintf("gauges_export-%v.tsv", fileNo))
if err != nil {
panic(err)
}
writer := bufio.NewWriter(outfile)
for i := range datapoints {
if i == fileNo*maxPerFile {
outfile.Close()
fileNo += 1
outfile, err = os.Create(fmt.Sprintf("gauges_export-%v.tsv", fileNo))
if err != nil {
panic(err)
}
writer = bufio.NewWriter(outfile)
}
if datapoints[i][0] == 'g' && (len(datapoints[i]) < 18 || string(datapoints[i][0:17]) != "gauges:Syslog-NG.") {
path := gobatsd.CalculateFilename(string(datapoints[i]), gobatsd.Config.Root)
fmt.Printf("(%v / %v) Migrating %v from %v\n", i, len(datapoints), string(datapoints[i]), path)
file, err := os.Open(path)
if err == nil {
reader := bufio.NewReader(file)
linesRead := 0
for {
line, err := reader.ReadString('\n')
linesRead += 1
if err != nil && err != io.EOF {
fmt.Println(err)
}
if err != nil && err == io.EOF {
break
}
if linesRead == 1 {
continue
}
parts := strings.Split(strings.TrimSpace(line), " ")
ts, _ := strconv.ParseInt(parts[0], 10, 64)
if len(parts) == 2 {
value, _ := strconv.ParseFloat(parts[1], 64)
if value != float64(0) && ts > yearago {
_, err := writer.WriteString(fmt.Sprintf("%v\t%v\t%s\n", string(datapoints[i]), parts[0], gobatsd.EncodeFloat64(value)))
if err != nil {
fmt.Println(err)
outfile.Close()
fileNo += 1
outfile, err = os.Create(fmt.Sprintf("sc_gauges_export-%v.tsv", fileNo))
if err != nil {
panic(err)
}
writer = bufio.NewWriter(outfile)
}
}
}
}
} else {
fmt.Printf("%v\n", err)
}
file.Close()
writer.Flush()
}
}
outfile.Close()
fmt.Printf("Import with: -Dimporttsv.columns=HBASE_ROW_KEY,HBASE_TS_KEY,interval0:value\n")
}