Ejemplo n.º 1
0
func main() {
	gobatsd.LoadConfig()
	fmt.Printf("Starting on port %v, root dir %v\n", gobatsd.Config.Port, gobatsd.Config.Root)

	server, err := net.Listen("tcp", ":"+gobatsd.Config.Port)
	if err != nil {
		panic(err)
	}
	numClients := 0

	hbasePool = gobatsd.MakeHbasePool(10)

	for {
		client, err := server.Accept()
		if client == nil {
			fmt.Printf("couldn't accept: %v", err)
			continue
		}
		numClients++
		fmt.Printf("Opened connection #%d: %v <-> %v\n", numClients, client.LocalAddr(), client.RemoteAddr())

		go func() {
			r, err := redis.Dial("tcp", fmt.Sprintf("%v:%v", gobatsd.Config.RedisHost, gobatsd.Config.RedisPort))
			if err != nil {
				panic(err)
			}
			defer r.Close()

			(&Client{client, r}).Serve()
		}()
	}
}
Ejemplo n.º 2
0
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	gobatsd.LoadConfig()
	if gobatsd.ProfileCPU {
		cpuprof, err := os.Create(fmt.Sprintf("cpuprof-%v", time.Now().Unix()))
		if err != nil {
			panic(err)
		}
		defer cpuprof.Close()
		pprof.StartCPUProfile(cpuprof)
		defer pprof.StopCPUProfile()
	}

	processingChannel := clamp.StartExplodingDualServer(":" + gobatsd.Config.Port)
	clamp.StartStatsServer(":8124")
	gobatsd.SetupDatastore()

	fmt.Printf("Starting on port %v\n", gobatsd.Config.Port)
	gobatsd.InitializeInternalMetrics()

	gaugeChannel = make(chan gobatsd.Datapoint, channelBufferSize)
	counterChannel = make(chan gobatsd.Datapoint, channelBufferSize)
	timerChannel = make(chan gobatsd.Datapoint, channelBufferSize)

	channels := map[string]chan gobatsd.Datapoint{"g": gaugeChannel, "c": counterChannel, "ms": timerChannel}

	for i := 0; i < numIncomingMessageProcessors; i++ {
		go func(processingChannel chan string) {
			for {
				message := <-processingChannel
				d := gobatsd.ParseDatapointFromString(message)
				if ch, ok := channels[d.Datatype]; ok {
					ch <- d
				}
			}
		}(processingChannel)
	}

	go func() {
		c := time.Tick(1 * time.Second)
		for {
			<-c
			clamp.StatsChannel <- clamp.Stat{"gaugeChannelSize", fmt.Sprintf("%v", len(gaugeChannel))}
			clamp.StatsChannel <- clamp.Stat{"counterChannelSize", fmt.Sprintf("%v", len(counterChannel))}
			clamp.StatsChannel <- clamp.Stat{"timerChannelSize", fmt.Sprintf("%v", len(timerChannel))}
		}
	}()

	processDatatype("gauges", gaugeChannel, gobatsd.NewGauge)
	processDatatype("timers", timerChannel, gobatsd.NewTimer)
	processDatatype("counters", counterChannel, gobatsd.NewCounter)

	terminate := make(chan os.Signal)
	signal.Notify(terminate, os.Interrupt)
	<-terminate

	fmt.Printf("Server stopped")

}
Ejemplo n.º 3
0
func main() {
	gobatsd.LoadConfig()

	spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
	redis, redisErr := redis.NewSynchClientWithSpec(spec)
	if redisErr != nil {
		panic(redisErr)
	}
	datapoints, redisErr := redis.Smembers("datapoints")
	if redisErr != nil {
		panic(redisErr)
	}

	for j := range retentions {
		outfile, err := os.Create(fmt.Sprintf("counters_export_%v.tsv", retentions[j]))
		if err != nil {
			panic(err)
		}

		writer := bufio.NewWriter(outfile)
		for i := range datapoints {
			if datapoints[i][0] == 'c' {
				path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
				fmt.Printf("(%v / %v) Migrating %v:%v from %v\n", i, len(datapoints), string(datapoints[i]), retentions[j], path)
				file, err := os.Open(path)
				if err == nil {
					reader := bufio.NewReader(file)
					linesRead := 0
					for {
						line, err := reader.ReadString('\n')
						linesRead += 1
						if err != nil && err != io.EOF {
							panic(err)
						}
						if err != nil && err == io.EOF {
							break
						}
						if linesRead == 1 {
							continue
						}

						parts := strings.Split(strings.TrimSpace(line), " ")
						value, _ := strconv.ParseFloat(parts[1], 64)
						writer.WriteString(fmt.Sprintf("%v\t%v\t%s\n", string(datapoints[i]), parts[0], gobatsd.EncodeFloat64(value)))
					}
				} else {
					fmt.Printf("%v\n", err)
				}
				file.Close()

				writer.Flush()
			}

		}
		outfile.Close()
		fmt.Printf("Import with: -Dimporttsv.columns=HBASE_ROW_KEY,HBASE_TS_KEY,interval%v:value\n", retentions[j])
	}
}
Ejemplo n.º 4
0
func main() {
	prof, err := os.Create("/tmp/proxy.prof")
	if err != nil {
		panic(err)
	}
	defer prof.Close()
	pprof.StartCPUProfile(prof)
	defer pprof.StopCPUProfile()

	gobatsd.LoadConfig()
	fmt.Printf("Starting on port %v\n", gobatsd.Config.Port)

	server, err := net.ListenPacket("udp", ":"+gobatsd.Config.Port)
	defer server.Close()
	if err != nil {
		panic(err)
	}

	destinationAddr, err := net.ResolveUDPAddr("udp", ":8225")
	if err != nil {
		panic(err)
	}
	client, err := net.DialUDP("udp", nil, destinationAddr)
	if err != nil {
		panic(err)
	}

	//c := TCPRelay(client)
	buffer := make([]byte, readLen)

	for {
		n, _, err := server.ReadFrom(buffer)
		if err != nil {
			continue
		}
		client.Write(buffer[0:n])
		//	c <- buffer[0:n]

	}
}
Ejemplo n.º 5
0
func main() {
	gobatsd.LoadConfig()
	index := sort.Search(len(gobatsd.Config.Retentions), func(i int) bool { return gobatsd.Config.Retentions[i].Interval == gobatsd.Config.TargetInterval })
	if index == len(gobatsd.Config.Retentions) {
		if gobatsd.Config.Retentions[0].Interval == gobatsd.Config.TargetInterval {
			index = 0
		} else {
			panic("You must specify the duration you'd like to truncate")
		}
	}
	retention := gobatsd.Config.Retentions[index]
	fmt.Printf("Starting truncation for the %v duration.\n", retention.Interval)
	spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
	redis, err := redis.NewSynchClientWithSpec(spec)
	if err != nil {
		panic(err)
	}

	datapoints := make([]string, 0)
	if index == 0 {
		rawDatapoints, err := redis.Keys("*")
		if err != nil {
			panic(err)
		}

		for _, key := range rawDatapoints {
			if m, _ := regexp.MatchString("^timers|^counters", string(key)); m {
				datapoints = append(datapoints, string(key))
			}
		}
	} else {
		rawDatapoints, err := redis.Smembers("datapoints")
		if err != nil {
			panic(err)
		}

		for _, key := range rawDatapoints {
			if m, _ := regexp.MatchString("^timers|^counters", string(key)); m {
				datapoints = append(datapoints, string(key))
			}
		}

	}
	since := float64(time.Now().Unix() - retention.Duration)
	fmt.Printf("Truncating %v datapoints since %f.\n", len(datapoints), since)

	if index == 0 { //Redis truncation
		for _, key := range datapoints {
			redis.Zremrangebyscore(key, 0.0, since)
		}
	} else {
		nworkers := NWORKERS
		if nworkers > len(datapoints) {
			nworkers = len(datapoints)
		}
		c := make(chan int, nworkers)
		for i := 0; i < nworkers; i++ {
			go func(datapoints []string, i int, c chan int) {
				for _, key := range datapoints {
					metricName := key + ":" + strconv.FormatInt(retention.Interval, 10)
					if m, _ := regexp.MatchString("^timers", metricName); m {
						metricName += ":2"
					}
					TruncateOnDisk(metricName, since)
				}
				c <- 1
			}(datapoints[i*len(datapoints)/nworkers:(i+1)*len(datapoints)/nworkers-1], i, c)

		}

		for i := 0; i < nworkers; i++ {
			<-c
		}

	}
}
Ejemplo n.º 6
0
func main() {
	gobatsd.LoadConfig()

	runtime.GOMAXPROCS(runtime.NumCPU())
	spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
	redis, redisErr := redis.NewSynchClientWithSpec(spec)
	if redisErr != nil {
		panic(redisErr)
	}
	datapoints, redisErr := redis.Smembers("datapoints")
	if redisErr != nil {
		panic(redisErr)
	}
	nworkers := 10
	c := make(chan int, nworkers)

	for k := 0; k < nworkers; k++ {
		go func(datapoints [][]byte, c chan int) {
			cli, err := gobatsd.OpenHbaseConnection()
			if err != nil {
				panic(err)
			}
			hbaseClient := cli.(*goh.HClient)
			defer hbaseClient.Close()

			for i := range datapoints {
				if datapoints[i][0] == 'g' && (len(datapoints[i]) < 18 || string(datapoints[i][0:17]) != "gauges:Syslog-NG.") {
					path := gobatsd.CalculateFilename(string(datapoints[i]), gobatsd.Config.Root)
					fmt.Printf("(%v / %v) Migrating %v from %v\n", i, len(datapoints), string(datapoints[i]), path)
					file, err := os.Open(path)
					if err == nil {
						reader := bufio.NewReader(file)
						linesRead := 0
						for {
							line, err := reader.ReadString('\n')
							linesRead += 1
							if err != nil && err != io.EOF {
								panic(err)
							}
							if err != nil && err == io.EOF {
								break
							}
							if linesRead == 1 {
								continue
							}

							parts := strings.Split(strings.TrimSpace(line), " ")
							ts, _ := strconv.ParseInt(parts[0], 10, 64)
							value, _ := strconv.ParseFloat(parts[1], 64)
							if value != float64(0) && ts > yearago {
								mutations := []*Hbase.Mutation{goh.NewMutation("interval0:value", gobatsd.EncodeFloat64(value))}

								err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
								if err != nil {
									fmt.Printf("%v: mutate error -  %v\n", time.Now(), err)
								}
							}
						}
					} else {
						fmt.Printf("%v\n", err)
					}
					/*} else if datapoints[i][0] == 'c' {
					for j := range retentions {
						path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
						fmt.Printf("Migrating %v:%v from %v\n", string(datapoints[i]), retentions[j], path)
						file, err := os.Open(path)
						if err == nil {
							reader := bufio.NewReader(file)
							linesRead := 0
							for {
								line, err := reader.ReadString('\n')
								linesRead += 1
								if err != nil && err != io.EOF {
									panic(err)
								}
								if err != nil && err == io.EOF {
									break
								}
								if linesRead == 1 {
									continue
								}

								parts := strings.Split(strings.TrimSpace(line), " ")
								ts, _ := strconv.ParseInt(parts[0], 10, 64)
								value, _ := strconv.ParseFloat(parts[1], 64)

								mutations := []*Hbase.Mutation{goh.NewMutation(fmt.Sprintf("interval%v:value", retentions[j]), gobatsd.EncodeFloat64(value))}

								err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
								if err != nil {
									err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
									if err != nil {
										fmt.Printf("%v: mutate error -  %v\n", time.Now(), err)
									}
								}
							}
						} else {
							fmt.Printf("%v\n", err)
						}

					}*/
				} else if datapoints[i][0] == 't' && (len(datapoints[i]) < 16 || string(datapoints[i][0:15]) != "timers:sysstat.") {
					for j := range retentions {
						path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v:2", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
						fmt.Printf("(%v / %v) Migrating %v:%v from %v\n", i, len(datapoints), string(datapoints[i]), retentions[j], path)
						file, err := os.Open(path)
						if err == nil {
							reader := bufio.NewReader(file)
							linesRead := 0
							for {
								line, err := reader.ReadString('\n')
								linesRead += 1
								if err != nil && err != io.EOF {
									panic(err)
								}
								if err != nil && err == io.EOF {
									break
								}
								if linesRead == 1 {
									continue
								}

								parts := strings.Split(strings.TrimSpace(line), " ")
								ts, _ := strconv.ParseInt(parts[0], 10, 64)
								timerComponents := strings.Split(parts[1], "/")

								if ts > yearago {
									mutations := make([]*Hbase.Mutation, 0)
									for k, v := range timerHeader {
										val, _ := strconv.ParseFloat(timerComponents[v], 64)
										mutations = append(mutations, goh.NewMutation(fmt.Sprintf("interval%v:%v", retentions[j], k), gobatsd.EncodeFloat64(val)))
									}
									err = hbaseClient.MutateRowTs(gobatsd.Config.HbaseTable, datapoints[i], mutations, ts, nil)
									if err != nil {
										fmt.Printf("%v: mutate error -  %v\n", time.Now(), err)
									}
								}
							}
						} else {
							fmt.Printf("%v\n", err)
						}
					}

				}

			}
			c <- 1
		}(datapoints[k*len(datapoints)/nworkers:(k+1)*len(datapoints)/nworkers-1], c)
	}

	for i := 0; i < nworkers; i++ {
		<-c
	}
}
Ejemplo n.º 7
0
func main() {
	gobatsd.LoadConfig()

	spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
	redis, redisErr := redis.NewSynchClientWithSpec(spec)
	if redisErr != nil {
		panic(redisErr)
	}
	datapoints, redisErr := redis.Smembers("datapoints")
	if redisErr != nil {
		panic(redisErr)
	}

	for j := range retentions {
		fileNo := 1
		outfile, err := os.Create(fmt.Sprintf("timers_export_%v-%v.tsv", retentions[j], fileNo))
		if err != nil {
			panic(err)
		}

		writer := bufio.NewWriter(outfile)
		for i := range datapoints {
			if i == fileNo*maxPerFile {
				outfile.Close()
				fileNo += 1
				outfile, err = os.Create(fmt.Sprintf("timers_export_%v-%v.tsv", retentions[j], fileNo))
				if err != nil {
					panic(err)
				}
				writer = bufio.NewWriter(outfile)
			}
			if datapoints[i][0] == 't' && (len(datapoints[i]) < 16 || string(datapoints[i][0:15]) != "timers:sysstat.") {
				path := gobatsd.CalculateFilename(fmt.Sprintf("%v:%v:2", string(datapoints[i]), retentions[j]), gobatsd.Config.Root)
				fmt.Printf("(%v / %v) Migrating %v:%v from %v\n", i, len(datapoints), string(datapoints[i]), retentions[j], path)
				file, err := os.Open(path)
				if err == nil {
					reader := bufio.NewReader(file)
					linesRead := 0
					for {
						line, err := reader.ReadString('\n')
						linesRead += 1
						if err != nil && err != io.EOF {
							panic(err)
						}
						if err != nil && err == io.EOF {
							break
						}
						if linesRead == 1 {
							continue
						}

						parts := strings.Split(strings.TrimSpace(line), " ")
						ts, _ := strconv.ParseInt(parts[0], 10, 64)
						if ts > 1 {
							timerComponents := strings.Split(parts[1], "/")
							count, _ := strconv.ParseFloat(timerComponents[0], 64)
							min, _ := strconv.ParseFloat(timerComponents[1], 64)
							max, _ := strconv.ParseFloat(timerComponents[2], 64)
							median, _ := strconv.ParseFloat(timerComponents[3], 64)
							mean, _ := strconv.ParseFloat(timerComponents[4], 64)
							stddev, _ := strconv.ParseFloat(timerComponents[5], 64)
							percentile_90, _ := strconv.ParseFloat(timerComponents[6], 64)
							percentile_95, _ := strconv.ParseFloat(timerComponents[7], 64)
							percentile_99, _ := strconv.ParseFloat(timerComponents[8], 64)

							writer.WriteString(fmt.Sprintf("%v\t%v\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", string(datapoints[i]), parts[0],
								gobatsd.EncodeFloat64(count), gobatsd.EncodeFloat64(min), gobatsd.EncodeFloat64(max), gobatsd.EncodeFloat64(median),
								gobatsd.EncodeFloat64(mean), gobatsd.EncodeFloat64(stddev), gobatsd.EncodeFloat64(percentile_90), gobatsd.EncodeFloat64(percentile_95),
								gobatsd.EncodeFloat64(percentile_99)))
						}
					}
				} else {
					fmt.Printf("%v\n", err)
				}
				file.Close()

				writer.Flush()
			}

		}
		outfile.Close()
		fmt.Printf("Import with: -Dimporttsv.columns=HBASE_ROW_KEY,HBASE_TS_KEY,interval%v:count\n", retentions[j])
	}
}
Ejemplo n.º 8
0
func main() {
	gobatsd.LoadConfig()

	spec := redis.DefaultSpec().Host(gobatsd.Config.RedisHost).Port(gobatsd.Config.RedisPort)
	redis, redisErr := redis.NewSynchClientWithSpec(spec)
	if redisErr != nil {
		panic(redisErr)
	}
	datapoints, redisErr := redis.Smembers("datapoints")
	if redisErr != nil {
		panic(redisErr)
	}

	fileNo := 1
	outfile, err := os.Create(fmt.Sprintf("gauges_export-%v.tsv", fileNo))
	if err != nil {
		panic(err)
	}

	writer := bufio.NewWriter(outfile)
	for i := range datapoints {
		if i == fileNo*maxPerFile {
			outfile.Close()
			fileNo += 1
			outfile, err = os.Create(fmt.Sprintf("gauges_export-%v.tsv", fileNo))
			if err != nil {
				panic(err)
			}
			writer = bufio.NewWriter(outfile)
		}
		if datapoints[i][0] == 'g' && (len(datapoints[i]) < 18 || string(datapoints[i][0:17]) != "gauges:Syslog-NG.") {
			path := gobatsd.CalculateFilename(string(datapoints[i]), gobatsd.Config.Root)
			fmt.Printf("(%v / %v) Migrating %v from %v\n", i, len(datapoints), string(datapoints[i]), path)
			file, err := os.Open(path)
			if err == nil {
				reader := bufio.NewReader(file)
				linesRead := 0
				for {
					line, err := reader.ReadString('\n')
					linesRead += 1
					if err != nil && err != io.EOF {
						fmt.Println(err)
					}
					if err != nil && err == io.EOF {
						break
					}
					if linesRead == 1 {
						continue
					}

					parts := strings.Split(strings.TrimSpace(line), " ")
					ts, _ := strconv.ParseInt(parts[0], 10, 64)
					if len(parts) == 2 {
						value, _ := strconv.ParseFloat(parts[1], 64)
						if value != float64(0) && ts > yearago {
							_, err := writer.WriteString(fmt.Sprintf("%v\t%v\t%s\n", string(datapoints[i]), parts[0], gobatsd.EncodeFloat64(value)))
							if err != nil {
								fmt.Println(err)
								outfile.Close()
								fileNo += 1
								outfile, err = os.Create(fmt.Sprintf("sc_gauges_export-%v.tsv", fileNo))
								if err != nil {
									panic(err)
								}
								writer = bufio.NewWriter(outfile)

							}
						}
					}
				}
			} else {
				fmt.Printf("%v\n", err)
			}
			file.Close()

			writer.Flush()
		}

	}
	outfile.Close()
	fmt.Printf("Import with: -Dimporttsv.columns=HBASE_ROW_KEY,HBASE_TS_KEY,interval0:value\n")

}