func init() {
if conf.StatisticEnable {
var err error
influxClient, err = influx.NewHTTPClient(
influx.HTTPConfig{
Addr: conf.InfluxURL,
Username: conf.InfluxUser,
Password: conf.InfluxPassword,
})
if err != nil {
log.Println("Failed to init influx client: ", err.Error())
os.Exit(1)
}
influxBatchPoints, err = influx.NewBatchPoints(influx.BatchPointsConfig{
Database: conf.InfluxDB,
Precision: "s",
})
if err != nil {
log.Println("Failed to init influx batch points: ", err.Error())
os.Exit(1)
}
go logStatisticTask()
}
}
// Perform the batch write
func (w *Writer) writeEntries(entries []Entry) {
bps, err := client.NewBatchPoints(client.BatchPointsConfig{
Database: w.DB,
RetentionPolicy: "default",
Precision: "s",
// WriteConsistency: string,
})
Checkerr(err)
for _, entry := range entries {
name := "watt" // Measurement
tags := map[string]string{ /*"ted1k",...*/ }
fields := map[string]interface{}{
"value": entry.Watt,
}
pt, err := client.NewPoint(name, tags, fields, entry.Stamp)
Checkerr(err)
bps.AddPoint(pt)
// fmt.Printf("point: %v\n", pt)
}
// TODO(daneroo): retry, if error is timeout?
err = w.con.Write(bps)
Checkerr(err)
}
func TestHistogramWithTags(t *testing.T) {
expectedName := "test_histogram"
expectedTags := map[string]string{
"key1": "value1",
"key2": "value2",
}
expectedFields := []map[string]map[string]interface{}{
{
"test_histogram_p50": {"value": 5.0},
"test_histogram_p90": {"value": 5.0},
"test_histogram_p95": {"value": 5.0},
"test_histogram_p99": {"value": 5.0},
},
{
"test_histogram_p50": {"Test": "Test", "value": 5.0},
"test_histogram_p90": {"Test": "Test", "value": 10.0},
"test_histogram_p95": {"Test": "Test", "value": 10.0},
"test_histogram_p99": {"Test": "Test", "value": 10.0},
},
{
"test_histogram_p50": {"Test": "Test", "value": 5.0},
"test_histogram_p90": {"Test": "Test", "value": 10.0},
"test_histogram_p95": {"Test": "Test", "value": 10.0},
"test_histogram_p99": {"Test": "Test", "value": 10.0},
},
}
quantiles := []int{50, 90, 95, 99}
cl := &mockClient{}
cl.Add(12)
bp, _ := stdinflux.NewBatchPoints(stdinflux.BatchPointsConfig{
Database: "testing",
Precision: "s",
})
tags := []metrics.Field{}
for key, value := range expectedTags {
tags = append(tags, metrics.Field{Key: key, Value: value})
}
triggerChan := make(chan time.Time)
histogram := influxdb.NewHistogramTick(cl, bp, expectedName, tags, triggerChan, 0, 100, 3, quantiles...)
histogram.Observe(5)
histogram = histogram.With(metrics.Field{Key: "Test", Value: "Test"})
histogram.Observe(10)
histogram.Observe(4)
triggerChan <- time.Now()
cl.Wait()
for i := 0; i <= 11; i++ {
actualName := cl.Points[i].Name()
givenName := expectedName + actualName[len(actualName)-4:]
givenPoint := mockPoint{
Name: givenName,
Tags: expectedTags,
Fields: expectedFields[i/4][actualName],
}
comparePoint(t, i, givenPoint, cl.Points[i])
}
}
func syncDeliveryServiceStat(sourceClient influx.Client, targetClient influx.Client, statName string, days int) {
db := "deliveryservice_stats"
bps, _ := influx.NewBatchPoints(influx.BatchPointsConfig{
Database: db,
Precision: "ms",
RetentionPolicy: "monthly",
})
queryString := fmt.Sprintf("select time, cachegroup, cdn, deliveryservice, value from \"monthly\".\"%s\"", statName)
if days > 0 {
queryString += fmt.Sprintf(" where time > now() - %dd", days)
}
fmt.Println("queryString ", queryString)
res, err := queryDB(sourceClient, queryString, db)
if err != nil {
errorMessage = fmt.Sprintf("An error occured getting %s records from sourceDb: %v\n", statName, err)
fmt.Println(errorMessage)
return
}
sourceStats := getDeliveryServiceStats(res)
// get value from target DB
targetRes, err := queryDB(targetClient, queryString, db)
if err != nil {
errorMessage = fmt.Sprintf("An error occured getting %s record from target db: %v\n", statName, err)
fmt.Println(errorMessage)
return
}
targetStats := getDeliveryServiceStats(targetRes)
for ssKey := range sourceStats {
ts := targetStats[ssKey]
ss := sourceStats[ssKey]
if ts.value > ss.value {
//fmt.Printf("target value %v is at least equal to source value %v\n", ts.value, ss.value)
continue //target value is bigger so leave it
}
statTime, _ := time.Parse(time.RFC3339, ss.t)
tags := map[string]string{
"cdn": ss.cdn,
"cachegroup": ss.cacheGroup,
"deliveryservice": ss.deliveryService,
}
fields := map[string]interface{}{
"value": ss.value,
}
pt, err := influx.NewPoint(
statName,
tags,
fields,
statTime,
)
if err != nil {
fmt.Printf("error adding creating point for %v...%v\n", statName, err)
continue
}
bps.AddPoint(pt)
}
targetClient.Write(bps)
}
// InfluxDB writes interest processing time to influxDB.
//
// The data collected can be viewed with: SELECT "value" FROM :name WHERE "name" = ':interest_name'.
func InfluxDB(client influxdb.Client, db, name string, tags map[string]string) mux.Middleware {
return func(next mux.Handler) mux.Handler {
return mux.HandlerFunc(func(w ndn.Sender, i *ndn.Interest) {
before := time.Now()
next.ServeNDN(w, i)
t := make(map[string]string)
for k, v := range tags {
t[k] = v
}
t["name"] = i.Name.String()
pt, _ := influxdb.NewPoint(name, t, map[string]interface{}{
"value": float64(time.Since(before)) / float64(time.Millisecond),
}, time.Now())
bp, _ := influxdb.NewBatchPoints(influxdb.BatchPointsConfig{
Database: db,
})
bp.AddPoint(pt)
err := client.Write(bp)
if err != nil {
log.Println(err)
return
}
})
}
}
func calcDailySummary(now time.Time, config StartupConfig, runningConfig RunningConfig) {
log.Infof("lastSummaryTime is %v", runningConfig.LastSummaryTime)
if runningConfig.LastSummaryTime.Day() != now.Day() {
startTime := now.Truncate(24 * time.Hour).Add(-24 * time.Hour)
endTime := startTime.Add(24 * time.Hour)
log.Info("Summarizing from ", startTime, " (", startTime.Unix(), ") to ", endTime, " (", endTime.Unix(), ")")
// influx connection
influxClient, err := influxConnect(config, runningConfig)
if err != nil {
log.Error("Could not connect to InfluxDb to get daily summary stats!!")
errHndlr(err, ERROR)
return
}
bp, _ := influx.NewBatchPoints(influx.BatchPointsConfig{
Database: "daily_stats",
Precision: "s",
RetentionPolicy: config.DailySummaryRetentionPolicy,
})
calcDailyMaxGbps(influxClient, bp, startTime, endTime, config)
calcDailyBytesServed(influxClient, bp, startTime, endTime, config)
log.Info("Collected daily stats @ ", now)
}
}
// NewResultsPointBatch creates a new batch of points for the results
func (st *StressTest) NewResultsPointBatch() influx.BatchPoints {
bp, _ := influx.NewBatchPoints(influx.BatchPointsConfig{
Database: st.TestDB,
Precision: "ns",
})
return bp
}
func (t *tickData) TickHandler(ctx context.Context, tick *tickRecorder.Tick) error {
var err error
t.log.Infoln("Received data")
tags := map[string]string{"pair": "AUDUSD"}
fields := map[string]interface{}{
"bid": tick.Bid,
"ask": tick.Ask,
"last": tick.Last,
}
point, err := influx.NewPoint("tick_data", tags, fields, time.Unix(0, tick.Time))
bp, err := influx.NewBatchPoints(influx.BatchPointsConfig{
Database: "tick",
Precision: "ns",
})
bp.AddPoint(point)
t.log.Infoln("Created batch point:", bp)
if influxErr := t.influx.Write(bp); influxErr != nil {
t.log.Error(influxErr)
}
return err
}
func createInfluxDBMetrics(ping Ping) (influxdbclient.BatchPoints, error) {
var err error
bp, err := influxdbclient.NewBatchPoints(influxdbclient.BatchPointsConfig{
Database: receiverDatabaseFlag,
Precision: "s",
})
if err != nil {
return nil, err
}
tags := map[string]string{
"origin": ping.origin,
"destination": ping.destination,
}
fields := map[string]interface{}{
"loss": ping.stats.loss,
"min": ping.stats.min,
"avg": ping.stats.avg,
"max": ping.stats.max,
"mdev": ping.stats.mdev,
}
pt, err := influxdbclient.NewPoint("ping", tags, fields, time.Unix(ping.time, 0))
if err != nil {
return nil, err
}
bp.AddPoint(pt)
return bp, nil
}
// WriteTo flushes the buffered content of the metrics to the writer, in an
// Influx BatchPoints format. WriteTo abides best-effort semantics, so
// observations are lost if there is a problem with the write. Clients should be
// sure to call WriteTo regularly, ideally through the WriteLoop helper method.
func (in *Influx) WriteTo(w BatchPointsWriter) (err error) {
bp, err := influxdb.NewBatchPoints(in.conf)
if err != nil {
return err
}
now := time.Now()
in.counters.Reset().Walk(func(name string, lvs lv.LabelValues, values []float64) bool {
fields := fieldsFrom(lvs)
fields["count"] = sum(values)
var p *influxdb.Point
p, err = influxdb.NewPoint(name, in.tags, fields, now)
if err != nil {
return false
}
bp.AddPoint(p)
return true
})
if err != nil {
return err
}
in.gauges.Reset().Walk(func(name string, lvs lv.LabelValues, values []float64) bool {
fields := fieldsFrom(lvs)
fields["value"] = last(values)
var p *influxdb.Point
p, err = influxdb.NewPoint(name, in.tags, fields, now)
if err != nil {
return false
}
bp.AddPoint(p)
return true
})
if err != nil {
return err
}
in.histograms.Reset().Walk(func(name string, lvs lv.LabelValues, values []float64) bool {
fields := fieldsFrom(lvs)
ps := make([]*influxdb.Point, len(values))
for i, v := range values {
fields["value"] = v // overwrite each time
ps[i], err = influxdb.NewPoint(name, in.tags, fields, now)
if err != nil {
return false
}
}
bp.AddPoints(ps)
return true
})
if err != nil {
return err
}
return w.Write(bp)
}
func NewReporter(conf flux.BatchPointsConfig, stats chan flux.BatchPoints, points chan *flux.Point) (*Reporter, error) {
bp, err := flux.NewBatchPoints(conf)
if err != nil {
return nil, err
}
r := &Reporter{
accum: stats,
batch: bp,
conf: conf,
points: points,
}
return r, nil
}
func TestStoreFrontBatcher(t *testing.T) {
sf, _, _ := NewTestStoreFront()
bpconf := influx.BatchPointsConfig{
Database: fmt.Sprintf("_%v", sf.TestName),
Precision: "ns",
}
bp, _ := influx.NewBatchPoints(bpconf)
pt := NewBlankTestPoint()
bp = sf.batcher(pt, bp, bpconf)
if len(bp.Points()) != 1 {
t.Fail()
}
}
func TestStressTestBatcher(t *testing.T) {
sf, _, _ := NewTestStressTest()
bpconf := influx.BatchPointsConfig{
Database: sf.TestDB,
Precision: "ns",
}
bp, _ := influx.NewBatchPoints(bpconf)
pt := NewBlankTestPoint()
bp = sf.batcher(pt, bp)
if len(bp.Points()) != 1 {
t.Fail()
}
}
func (hook *InfluxDBHook) newBatchPoints() (err error) {
// make sure we're only creating new batch points when we don't already have them
if hook.batchP != nil {
return nil
}
hook.batchP, err = influxdb.NewBatchPoints(influxdb.BatchPointsConfig{
Database: hook.database,
Precision: hook.precision,
})
if err != nil {
return err
}
return nil
}
func logStatistic(p *influx.Point) {
influxBatchPoints.AddPoint(p)
if len(influxBatchPoints.Points()) < conf.InfluxBatchPointsCount {
return
}
if err := influxClient.Write(influxBatchPoints); err != nil {
log.Println("Failed to dump point to influxdb: ", err.Error())
}
influxBatchPoints, _ = influx.NewBatchPoints(
influx.BatchPointsConfig{
Database: conf.InfluxDB,
Precision: "s",
})
}
func (a *Reporter) Start() hitman.KillChannel {
done := hitman.NewKillChannel()
tripwire := 1 * time.Second
tic := time.NewTicker(tripwire).C
last := time.Now().UnixNano()
_1000ms := time.Duration(1000)
fuzz := time.Duration(10)
go func() {
for {
select {
case cleaner := <-done:
cleaner.WaitGroup.Done()
fmt.Println("Closing reporter")
return
case p1 := <-a.points:
a.batch.AddPoint(p1)
n := len(a.points)
for i := 0; i < n; i++ {
a.batch.AddPoint(<-a.points)
}
case <-tic:
now := time.Now().UnixNano()
diff := now - last
delta := time.Duration(diff) / time.Millisecond
isOverdue := _1000ms < (delta + fuzz)
if !isOverdue {
continue
}
fmt.Println(len(a.accum), len(a.points), len(a.batch.Points()))
bp := a.batch
newPts, err := flux.NewBatchPoints(a.conf)
if err != nil {
fmt.Println(err)
}
a.batch = newPts
a.accum <- bp
last = time.Now().UnixNano()
}
}
}()
return done
}
// Batches incoming Result.Point and sends them if the batch reaches 5k in sizes
func (sf *StoreFront) batcher(pt *influx.Point, bp influx.BatchPoints, bpconf influx.BatchPointsConfig) influx.BatchPoints {
// If fewer than 5k add point and return
if len(bp.Points()) <= 5000 {
bp.AddPoint(pt)
} else {
// Otherwise send the batch
err := sf.ResultsClient.Write(bp)
// Check error
if err != nil {
log.Fatalf("Error writing performance stats\n error: %v\n", err)
}
// Reset the batch of points
bp, _ = influx.NewBatchPoints(bpconf)
}
return bp
}
func (s *Stats) initClient() (err error) {
log.WithField("Addr", s.influxConfig.Addr).Debug("connecting to influxdb")
s.client, err = influx.NewHTTPClient(s.influxConfig)
if err != nil {
return err
}
defer s.client.Close()
ping, _, err := s.client.Ping(time.Second)
if err != nil {
return err
}
log.WithField("ping", ping).Debug("influx ping")
s.bp, err = influx.NewBatchPoints(influx.BatchPointsConfig{
Database: "dicod",
Precision: "s",
})
return err
}
func TestCounterWithTags(t *testing.T) {
expectedName := "test_counter"
expectedTags := map[string]string{
"key1": "value1",
"key2": "value2",
}
expectedFields := []map[string]interface{}{
{"value": "2"},
{"Test": "Test", "value": "7"},
{"Test": "Test", "value": "10"},
}
cl := &mockClient{}
cl.Add(3)
bp, _ := stdinflux.NewBatchPoints(stdinflux.BatchPointsConfig{
Database: "testing",
Precision: "s",
})
tags := []metrics.Field{}
for key, value := range expectedTags {
tags = append(tags, metrics.Field{Key: key, Value: value})
}
triggerChan := make(chan time.Time)
counter := influxdb.NewCounterTick(cl, bp, expectedName, tags, triggerChan)
counter.Add(2)
counter = counter.With(metrics.Field{Key: "Test", Value: "Test"})
counter.Add(5)
counter.Add(3)
triggerChan <- time.Now()
cl.Wait()
for i := 0; i <= 2; i++ {
givenPoint := mockPoint{
Name: expectedName,
Tags: expectedTags,
Fields: expectedFields[i],
}
comparePoint(t, i, givenPoint, cl.Points[i])
}
}
func (self *InfluxdbStorage) AddStats(infotype string, measurement string, content interface{}) error {
if infotype == Infotypepacket {
//transfer interface into HttpTransaction
httpinstance, ok := content.(*metrics.HttpTransaction)
if !ok {
return errors.New("fail in transformation")
}
influxclient := self.Influxclient
//create bp point write bp point
bp, _ := influxpackage.NewBatchPoints(influxpackage.BatchPointsConfig{
Database: self.Database,
Precision: "us",
})
fields := map[string]interface{}{
"respondtime": httpinstance.Respondtime,
}
tags := map[string]string{
"Srcip": httpinstance.Srcip,
"Srcport": httpinstance.Srcport,
"Destip": httpinstance.Destip,
"Destport": httpinstance.Destport,
// problems in processing nesting problem
// "Requestdetail": httpinstance.Packetdetail.Requestdetail,
// "Responddetail": httpinstance.Packetdetail.Responddetail,
}
fmt.Println("**the measurement**", measurement)
point, err := influxpackage.NewPoint(measurement, tags, fields, time.Now())
if err != nil {
return err
}
fmt.Println("the point name:", point.Name())
bp.AddPoint(point)
influxclient.Write(bp)
}
//common metric info
if infotype == Infotypemetric {
}
return nil
}
// Starts a go routine that listens for Results
func (sf *StoreFront) resultsListen() {
// Make sure databases for results are created
sf.createDatabase(fmt.Sprintf("_%v", sf.TestName))
sf.createDatabase(sf.TestName)
// Listen for Responses
go func() {
// Prepare a BatchPointsConfig
bpconf := influx.BatchPointsConfig{
Database: fmt.Sprintf("_%v", sf.TestName),
Precision: "ns",
}
// Prepare the first batch of points
bp, _ := influx.NewBatchPoints(bpconf)
// TODO: Panics on resp.Tracer.Done() if there are too many 500s in a row
// Loop over ResultsChan
for resp := range sf.ResultsChan {
switch resp.Point.Name() {
// If the done point comes down the channel write the results
case "done":
sf.ResultsClient.Write(bp)
// Decrement the tracer
resp.Tracer.Done()
// By default fall back to the batcher
default:
// Add the StoreFront tags
pt := resp.AddTags(sf.tags())
// Add the point to the batch
bp = sf.batcher(pt, bp, bpconf)
// Decrement the tracer
resp.Tracer.Done()
}
}
}()
}
func sendMetrics(config StartupConfig, runningConfig RunningConfig, bps influx.BatchPoints, retry bool) {
influxClient, err := influxConnect(config, runningConfig)
if err != nil {
if retry {
config.BpsChan <- bps
}
errHndlr(err, ERROR)
return
}
pts := bps.Points()
for len(pts) > 0 {
chunkBps, err := influx.NewBatchPoints(influx.BatchPointsConfig{
Database: bps.Database(),
Precision: bps.Precision(),
RetentionPolicy: bps.RetentionPolicy(),
})
if err != nil {
if retry {
config.BpsChan <- chunkBps
}
errHndlr(err, ERROR)
}
for _, p := range pts[:intMin(config.MaxPublishSize, len(pts))] {
chunkBps.AddPoint(p)
}
pts = pts[intMin(config.MaxPublishSize, len(pts)):]
err = influxClient.Write(chunkBps)
if err != nil {
if retry {
config.BpsChan <- chunkBps
}
errHndlr(err, ERROR)
} else {
log.Info(fmt.Sprintf("Sent %v stats for %v", len(chunkBps.Points()), chunkBps.Database()))
}
}
}
func TestGauge(t *testing.T) {
expectedName := "test_gauge"
expectedTags := map[string]string{}
expectedFields := []map[string]interface{}{
{"value": 2.1},
{"value": 1.0},
{"value": 10.5},
}
cl := &mockClient{}
cl.Add(3)
bp, _ := stdinflux.NewBatchPoints(stdinflux.BatchPointsConfig{
Database: "testing",
Precision: "s",
})
tags := []metrics.Field{}
for key, value := range expectedTags {
tags = append(tags, metrics.Field{Key: key, Value: value})
}
triggerChan := make(chan time.Time)
counter := influxdb.NewGaugeTick(cl, bp, expectedName, tags, triggerChan)
counter.Add(2.1)
counter.Set(1)
counter.Add(9.5)
triggerChan <- time.Now()
cl.Wait()
for i := 0; i <= 2; i++ {
givenPoint := mockPoint{
Name: expectedName,
Tags: expectedTags,
Fields: expectedFields[i],
}
comparePoint(t, i, givenPoint, cl.Points[i])
}
}
func Msg2Series(msgs []Message, database string) client.BatchPoints {
// Create a new point batch
bp, _ := client.NewBatchPoints(client.BatchPointsConfig{
Database: database,
Precision: "s",
})
for _, msg := range msgs {
if msg.Topic == "" && len(msg.Payload) == 0 {
break
}
tokens := strings.Split(msg.Topic, "/")
if len(tokens) < 2 {
break
}
j, err := MsgParse(msg.Payload)
if err != nil {
log.Warn(err)
continue
}
//name := strings.Replace(msg.Topic, "/", ".", -1)
name := strings.Join(tokens[1:], "_")
tags := map[string]string{
"device": tokens[0],
}
pt, err := client.NewPoint(name, tags, j, time.Now())
if err != nil {
break
}
bp.AddPoint(pt)
fmt.Printf("%+v\n", pt)
}
return bp
}
// WriteMetrics is called from a handler to write the eru and qos metrics to the
// influxdb database.
func WriteMetrics(eru, qos float64, trafficPattern string) error {
influxClient, err := createClient()
if err != nil {
log.Println("Error getting the HTTP client for Influx.")
return err
}
// After this function terminates, close the HTTP client connection.
defer influxClient.Close()
bp, _ := client.NewBatchPoints(client.BatchPointsConfig{
Database: databaseName,
// @TODO Does "s" make sense
Precision: "s",
})
tags := map[string]string{
"sm": scalingMethod,
"tp": trafficPattern,
"pit": podInitializationTime,
"ver": version,
}
fields := map[string]interface{}{
"eru": eru,
"qos": qos,
}
pt, err := client.NewPoint(PointName, tags, fields, time.Now())
if err != nil {
log.Println("Error creating the new point.")
return err
}
bp.AddPoint(pt)
return influxClient.Write(bp)
}
func calcCacheValues(trafmonData []byte, cdnName string, sampleTime int64, cacheMap map[string]traffic_ops.Server, config StartupConfig) error {
type CacheStatsJSON struct {
Pp string `json:"pp"`
Date string `json:"date"`
Caches map[string]map[string][]struct {
Index uint64 `json:"index"`
Time int `json:"time"`
Value string `json:"value"`
Span uint64 `json:"span"`
} `json:"caches"`
}
var jData CacheStatsJSON
err := json.Unmarshal(trafmonData, &jData)
if err != nil {
return fmt.Errorf("could not unmarshall cache stats JSON - %v", err)
}
statCount := 0
bps, err := influx.NewBatchPoints(influx.BatchPointsConfig{
Database: "cache_stats",
Precision: "ms",
RetentionPolicy: config.CacheRetentionPolicy,
})
if err != nil {
errHndlr(err, ERROR)
}
for cacheName, cacheData := range jData.Caches {
cache := cacheMap[cacheName]
for statName, statData := range cacheData {
dataKey := statName
dataKey = strings.Replace(dataKey, ".bandwidth", ".kbps", 1)
dataKey = strings.Replace(dataKey, "-", "_", -1)
//Get the stat time and convert to epoch
statTime := strconv.Itoa(statData[0].Time)
msInt, err := strconv.ParseInt(statTime, 10, 64)
if err != nil {
errHndlr(err, ERROR)
}
newTime := time.Unix(0, msInt*int64(time.Millisecond))
//Get the stat value and convert to float
statValue := statData[0].Value
statFloatValue, err := strconv.ParseFloat(statValue, 64)
if err != nil {
statFloatValue = 0.00
}
tags := map[string]string{
"cachegroup": cache.Cachegroup,
"hostname": cacheName,
"cdn": cdnName,
"type": cache.Type,
}
fields := map[string]interface{}{
"value": statFloatValue,
}
pt, err := influx.NewPoint(
dataKey,
tags,
fields,
newTime,
)
if err != nil {
errHndlr(err, ERROR)
continue
}
bps.AddPoint(pt)
statCount++
}
}
config.BpsChan <- bps
log.Info("Collected ", statCount, " cache stats values for ", cdnName, " @ ", sampleTime)
return nil
}
/* the ds json looks like:
{
"deliveryService": {
"linear-gbr-hls-sbr": {
" .us-ma-woburn.kbps": [{
"index": 520281,
"time": 1398893383605,
"value": "0",
"span": 520024
}],
"location.us-de-newcastle.kbps": [{
"index": 520281,
"time": 1398893383605,
"value": "0",
"span": 517707
}],
}
}
*/
func calcDsValues(rascalData []byte, cdnName string, sampleTime int64, config StartupConfig) error {
type DsStatsJSON struct {
Pp string `json:"pp"`
Date string `json:"date"`
DeliveryService map[string]map[string][]struct {
Index uint64 `json:"index"`
Time int `json:"time"`
Value string `json:"value"`
Span uint64 `json:"span"`
} `json:"deliveryService"`
}
var jData DsStatsJSON
err := json.Unmarshal(rascalData, &jData)
if err != nil {
return fmt.Errorf("could not unmarshall deliveryservice stats JSON - %v", err)
}
statCount := 0
bps, _ := influx.NewBatchPoints(influx.BatchPointsConfig{
Database: "deliveryservice_stats",
Precision: "ms",
RetentionPolicy: config.DsRetentionPolicy,
})
for dsName, dsData := range jData.DeliveryService {
for dsMetric, dsMetricData := range dsData {
var cachegroup, statName string
tags := map[string]string{
"deliveryservice": dsName,
"cdn": cdnName,
}
s := strings.Split(dsMetric, ".")
if strings.Contains(dsMetric, "type.") {
cachegroup = "all"
statName = s[2]
tags["type"] = s[1]
} else if strings.Contains(dsMetric, "total.") {
cachegroup, statName = s[0], s[1]
} else {
cachegroup, statName = s[1], s[2]
}
tags["cachegroup"] = cachegroup
//convert stat time to epoch
statTime := strconv.Itoa(dsMetricData[0].Time)
msInt, err := strconv.ParseInt(statTime, 10, 64)
if err != nil {
errHndlr(err, ERROR)
}
newTime := time.Unix(0, msInt*int64(time.Millisecond))
//convert stat value to float
statValue := dsMetricData[0].Value
statFloatValue, err := strconv.ParseFloat(statValue, 64)
if err != nil {
statFloatValue = 0.0
}
fields := map[string]interface{}{
"value": statFloatValue,
}
pt, err := influx.NewPoint(
statName,
tags,
fields,
newTime,
)
if err != nil {
errHndlr(err, ERROR)
continue
}
bps.AddPoint(pt)
statCount++
}
}
config.BpsChan <- bps
log.Info("Collected ", statCount, " deliveryservice stats values for ", cdnName, " @ ", sampleTime)
return nil
}
func calcDailySummary(now time.Time, config StartupConfig, runningConfig RunningConfig) {
log.Infof("lastSummaryTime is %v", runningConfig.LastSummaryTime)
if runningConfig.LastSummaryTime.Day() != now.Day() {
startTime := now.Truncate(24 * time.Hour).Add(-24 * time.Hour)
endTime := startTime.Add(24 * time.Hour)
log.Info("Summarizing from ", startTime, " (", startTime.Unix(), ") to ", endTime, " (", endTime.Unix(), ")")
// influx connection
influxClient, err := influxConnect(config, runningConfig)
if err != nil {
log.Error("Could not connect to InfluxDb to get daily summary stats!!")
errHndlr(err, ERROR)
return
}
//create influxdb query
q := fmt.Sprintf("SELECT sum(value)/6 FROM bandwidth where time > '%s' and time < '%s' group by time(60s), cdn fill(0)", startTime.Format(time.RFC3339), endTime.Format(time.RFC3339))
log.Infof(q)
res, err := queryDB(influxClient, q, "cache_stats")
if err != nil {
errHndlr(err, ERROR)
return
}
bp, _ := influx.NewBatchPoints(influx.BatchPointsConfig{
Database: "daily_stats",
Precision: "s",
RetentionPolicy: config.DailySummaryRetentionPolicy,
})
for _, row := range res[0].Series {
prevtime := startTime
max := float64(0)
bytesServed := float64(0)
cdn := row.Tags["cdn"]
for _, record := range row.Values {
kbps, err := record[1].(json.Number).Float64()
if err != nil {
errHndlr(err, ERROR)
continue
}
sampleTime, err := time.Parse(time.RFC3339, record[0].(string))
if err != nil {
errHndlr(err, ERROR)
continue
}
max = floatMax(max, kbps)
duration := sampleTime.Unix() - prevtime.Unix()
bytesServed += float64(duration) * kbps / 8
prevtime = sampleTime
}
maxGbps := max / 1000000
bytesServedTb := bytesServed / 1000000000
log.Infof("max gbps for cdn %v = %v", cdn, maxGbps)
log.Infof("Tbytes served for cdn %v = %v", cdn, bytesServedTb)
//write daily_maxgbps in traffic_ops
var statsSummary traffic_ops.StatsSummary
statsSummary.CDNName = cdn
statsSummary.DeliveryService = "all"
statsSummary.StatName = "daily_maxgbps"
statsSummary.StatValue = strconv.FormatFloat(maxGbps, 'f', 2, 64)
statsSummary.SummaryTime = now.Format(time.RFC3339)
statsSummary.StatDate = startTime.Format("2006-01-02")
go writeSummaryStats(config, statsSummary)
tags := map[string]string{
"deliveryservice": statsSummary.DeliveryService,
"cdn": statsSummary.CDNName,
}
fields := map[string]interface{}{
"value": maxGbps,
}
pt, err := influx.NewPoint(
statsSummary.StatName,
tags,
fields,
startTime,
)
if err != nil {
errHndlr(err, ERROR)
continue
}
bp.AddPoint(pt)
// write bytes served data to traffic_ops
statsSummary.StatName = "daily_bytesserved"
statsSummary.StatValue = strconv.FormatFloat(bytesServedTb, 'f', 2, 64)
go writeSummaryStats(config, statsSummary)
fields = map[string]interface{}{
"value": bytesServedTb,
}
pt, err = influx.NewPoint(
statsSummary.StatName,
tags,
fields,
startTime,
)
if err != nil {
errHndlr(err, ERROR)
continue
}
bp.AddPoint(pt)
}
config.BpsChan <- bp
log.Info("Collected daily stats @ ", now)
}
}
// NewSender returns a function that will accept datapoints to send to influxdb
func NewSender(
config interface{},
batch client.BatchPointsConfig,
batchSize int,
queueSize int,
flush int,
errFunc func(error),
) (Sender, error) {
if batchSize <= 0 {
batchSize = DefaultBatchSize
}
if queueSize <= 0 {
queueSize = DefaultQueueSize
}
if flush <= 0 {
flush = DefaultFlush
}
var conn client.Client
var err error
switch conf := config.(type) {
case client.HTTPConfig:
conn, err = client.NewHTTPClient(conf)
if err != nil {
return nil, errors.Wrap(err, "error creating HTTPClient")
}
_, _, err = conn.Ping(conf.Timeout)
if err != nil {
return nil, fmt.Errorf("cannot ping influxdb server: %s", conf.Addr)
}
if err := dbCheck(conn, batch.Database); err != nil {
return nil, errors.Wrapf(err, "check for database %s failed", batch.Database)
}
case client.UDPConfig:
conn, err = client.NewUDPClient(conf)
if err != nil {
return nil, errors.Wrap(err, "error creating UDPClient")
}
}
pts := make(chan *client.Point, queueSize)
bp, err := client.NewBatchPoints(batch)
if err != nil {
return nil, errors.Wrap(err, "batchpoints error")
}
go func() {
delay := time.Duration(flush) * time.Second
tick := time.Tick(delay)
count := 0
for {
select {
case p := <-pts:
bp.AddPoint(p)
count++
if count < batchSize {
continue
}
case <-tick:
if len(bp.Points()) == 0 {
continue
}
}
for {
if err := conn.Write(bp); err != nil {
if errFunc != nil {
errFunc(err)
}
time.Sleep(retry)
continue
}
bp, _ = client.NewBatchPoints(batch)
count = 0
break
}
}
}()
return func(key string, tags map[string]string, fields map[string]interface{}, ts time.Time) error {
pt, err := client.NewPoint(key, tags, fields, ts)
if err != nil {
return err
}
pts <- pt
return nil
}, nil
}
func Crawl(searchUrl string, depth int, fetcher Fetcher, redisClient *redis.Client, influxClient influx.Client) {
throttle <- 1
if depth <= 0 {
return
}
fmt.Printf("Depth: %d Crawling: %s\n", depth, searchUrl)
bp, _ := influx.NewBatchPoints(influx.BatchPointsConfig{
Database: "crawler",
Precision: "s",
})
host, err := url.Parse(searchUrl)
// Send this to our redis queue for indexing
if err != nil {
redisClient.LPush("unknown_url_crawler_queue", searchUrl)
} else {
redisClient.LPush(host.Host+"_crawler_queue", searchUrl)
}
urlcache.lock.Lock()
urlcache.m[searchUrl] = crawlError
urlcache.lock.Unlock()
// let's determine how long it is taking to fetch all urls on a page
startFetch := time.Now()
urls, err := fetcher.Fetch(searchUrl)
crawlTime := time.Since(startFetch)
if err != nil {
fmt.Printf("Error fetching results from %s: %s\n", searchUrl, err.Error())
} else {
fmt.Printf("Finished crawling %s in %.2f seconds\n", searchUrl, crawlTime.Seconds())
}
tags := map[string]string{
"domain": host.String(),
}
fields := map[string]interface{}{
"urls_found": len(urls),
"crawl_time": crawlTime.Nanoseconds(),
"total_urls_crawled": len(urlcache.m),
"urls_by_page": len(urls),
}
point, _ := influx.NewPoint(
"crawl_usage",
tags,
fields,
time.Now(),
)
// add data point to influx
bp.AddPoint(point)
if err := influxClient.Write(bp); err != nil {
log.Printf("Unable to write batch point to influxdb: %s\n", err.Error())
}
var wg sync.WaitGroup
for _, u := range urls {
if !urlTest.MatchString(u) {
u = "http://" + host.Host + u
}
urlcache.lock.Lock()
_, crawled := urlcache.m[u]
urlcache.lock.Unlock()
if validURL.MatchString(u) && urlTest.MatchString(u) && !crawled {
wg.Add(1)
go func(u string, depth int, fetcher Fetcher, redisClient *redis.Client, influxClient influx.Client) {
defer wg.Done()
Crawl(u, depth-1, fetcher, redisClient, influxClient)
}(u, depth, fetcher, redisClient, influxClient)
}
}
<-throttle
wg.Wait()
}