func NewRuntimeWare(prefixes []string, trackPageview bool, logInterval ...time.Duration) Middleware { expvar.NewString("at_server_start").Set(time.Now().Format("2006-01-02 15:04:05")) expvar.NewInt("cpu_count").Set(int64(runtime.NumCPU())) ware := &RuntimeWare{ serverStarted: time.Now(), trackPageview: trackPageview, ignoredUrls: prefixes, cQps: ratecounter.NewRateCounter(time.Minute), c4xx: ratecounter.NewRateCounter(5 * time.Minute), c5xx: ratecounter.NewRateCounter(5 * time.Minute), lc: NewLatencyCounter(50), hitsTotal: expvar.NewInt("hits_total"), hitsQps: expvar.NewInt("hits_per_minute"), hits4xx: expvar.NewInt("hits_4xx_per_5min"), hits5xx: expvar.NewInt("hits_5xx_per_5min"), hitsServed: expvar.NewString("latency_recent"), hitsLatMax: expvar.NewString("latency_max"), hitsLatMin: expvar.NewString("latency_min"), hitsLat95: expvar.NewString("latency_p95"), hitsLat50: expvar.NewString("latency_p50"), numGoroutine: expvar.NewInt("goroutine_count"), } if trackPageview { ware.pageviews = expvar.NewMap("hits_pageviews") } if len(logInterval) > 0 && logInterval[0] > 0 { go ware.logSnapshot(logInterval[0]) } return ware }
func (redisServer *RedisServer) Start() error { if redisServer.sender == nil { log.Printf("[%s] No Route is specified for this output", redisServer.name) return nil } // Add the client as a subscriber receiveChan := make(chan *buffer.Event, recvBuffer) redisServer.sender.AddSubscriber(redisServer.name, receiveChan) defer redisServer.sender.DelSubscriber(redisServer.name) allQueues := make([]*RedisQueue, len(redisServer.config.CopyQueues)) // Create Redis queue for index, key := range redisServer.config.CopyQueues { redisQueue := NewRedisQueue(redisServer.config, key) allQueues[index] = redisQueue go redisQueue.Start() } log.Printf("[%s] Started Redis Output Instance", redisServer.name) // Loop events and publish to Redis tick := time.NewTicker(time.Duration(redisFlushInterval) * time.Second) rateCounter := ratecounter.NewRateCounter(1 * time.Second) for { select { case ev := <-receiveChan: rateCounter.Incr(1) var allowed bool allowed = true for key, value := range redisServer.fields { if (*ev.Fields)[key] == nil || ((*ev.Fields)[key] != nil && value != (*ev.Fields)[key].(string)) { allowed = false break } } if allowed { text := *ev.Text for _, queue := range allQueues { queue.data <- text } } case <-tick.C: if rateCounter.Rate() > 0 { log.Printf("[%s] Current Redis input rate: %d/s\n", redisServer.name, rateCounter.Rate()) } case <-redisServer.term: log.Println("RedisServer received term signal") for _, queue := range allQueues { queue.term <- true } return nil } } return nil }
/* * Initializations */ func init() { flag.Parse() status.InputEventCount = expvar.NewInt("input_event_count") status.OutputEventCount = expvar.NewInt("output_event_count") status.ErrorCount = expvar.NewInt("error_count") status.EventCounter = ratecounter.NewRateCounter(5 * time.Second) status.OutputBytesPerSecond = ratecounter.NewRateCounter(5 * time.Second) expvar.Publish("input_events_per_second", expvar.Func(func() interface{} { return float64(status.EventCounter.Rate()) / 5.0 })) expvar.Publish("output_bytes_per_second", expvar.Func(func() interface{} { return float64(status.OutputBytesPerSecond.Rate()) / 5.0 })) expvar.Publish("connection_status", expvar.Func(func() interface{} { res := make(map[string]interface{}, 0) res["last_connect_time"] = status.LastConnectTime res["last_error_text"] = status.LastConnectError res["last_error_time"] = status.ErrorTime if status.IsConnected { res["connected"] = true res["uptime"] = time.Now().Sub(status.LastConnectTime).Seconds() } else { res["connected"] = false res["uptime"] = 0.0 } return res })) expvar.Publish("uptime", expvar.Func(func() interface{} { return time.Now().Sub(status.StartTime).Seconds() })) expvar.Publish("subscribed_events", expvar.Func(func() interface{} { return config.EventTypes })) results = make(chan string, 100) output_errors = make(chan error) status.StartTime = time.Now() }
func redisGet(redisServer *RedisInputServer, consumer *redismq.Consumer) error { consumer.ResetWorking() rateCounter := ratecounter.NewRateCounter(1 * time.Second) for { unacked := consumer.GetUnackedLength() if unacked > 0 { log.Printf("Requeued %d messages\n", unacked) consumer.RequeueWorking() } packages, err := consumer.MultiGet(recvBuffer) if err == nil { numPackages := len(packages) if numPackages > 0 { rateCounter.Incr(int64(numPackages)) err = packages[numPackages-1].MultiAck() if err != nil { log.Println("Failed to ack", err) } } for i := range packages { var ev buffer.Event payload := string(packages[i].Payload) ev.Text = &payload if redisServer.config.JsonDecode { decoder := json.NewDecoder(strings.NewReader(payload)) decoder.UseNumber() err = decoder.Decode(&ev.Fields) if err != nil { continue } } redisServer.receiver.Send(&ev) } } else { log.Printf("Error reading from Redis: %s, sleeping", err) time.Sleep(2 * time.Second) } } return nil }
func (s3Writer *S3Writer) Start() error { if s3Writer.Sender == nil { log.Printf("[%s] No route is specified for this output", s3Writer.name) return nil } // Create file saver fileSaver := new(FileSaver) fileSaver.Config = s3Writer.Config fileSaver.RateCounter = ratecounter.NewRateCounter(1 * time.Second) id := "s3_output" // Add the client as a subscriber receiveChan := make(chan *buffer.Event, recvBuffer) s3Writer.Sender.AddSubscriber(id, receiveChan) defer s3Writer.Sender.DelSubscriber(id) // Loop events and publish to S3 tick := time.NewTicker(time.Duration(s3FlushInterval) * time.Second) go s3Writer.WaitForUpload() for { select { case ev := <-receiveChan: var allowed bool allowed = true for key, value := range s3Writer.fields { if (*ev.Fields)[key] == nil || ((*ev.Fields)[key] != nil && value != (*ev.Fields)[key].(string)) { allowed = false break } } if allowed { fileSaver.WriteToFile(s3Writer.name, ev) } case <-tick.C: s3Writer.InitiateUploadToS3(fileSaver) case <-s3Writer.term: log.Println("S3Writer received term signal") return nil } } return nil }
// AddCount metrics func AddCount(name string) { updLastTime(name, time.Now()) var counter *ratecounter.RateCounter counter, ok := counters[name] if !ok { counter = ratecounter.NewRateCounter(1 * time.Minute) counters[name] = counter } var request *expvar.Int request, ok = requests[name] if !ok { request = expvar.NewInt(name) requests[name] = request } counter.Incr(1) request.Set(counter.Rate()) }
func main() { counter = ratecounter.NewRateCounter(1 * time.Minute) port := os.Getenv("PORT") if port == "" { log.Fatal("$PORT must be set") } var handler httpctx.Handler handler = httpctx.HandlerFunc(requestHandler) handler = userIPMiddleware(handler) handler = requestIDMiddleware(handler) handler = requestCtrMiddleware(handler) ctx := context.Background() svc := &Server{ctx, handler} http.HandleFunc("/", svc.ServeHTTP) log.Fatal(http.ListenAndServe(":"+port, nil)) }
func init() { counter = ratecounter.NewRateCounter(1 * time.Second) rand.Seed(time.Now().UnixNano()) }
func (es *ESServer) Start() error { if es.b == nil { log.Printf("[%s] No Route is specified for this output", es.name) return nil } var client *elastic.Client var err error for { httpClient := http.DefaultClient timeout := 60 * time.Second if es.config.Timeout > 0 { timeout = time.Duration(es.config.Timeout) * time.Second } log.Println("Setting HTTP timeout to", timeout) log.Println("Setting GZIP enabled:", es.config.GzipEnabled) httpClient.Timeout = timeout var infoLogger, errorLogger *log.Logger if es.config.InfoLogEnabled { infoLogger = log.New(os.Stdout, "", log.LstdFlags) } else { infoLogger = log.New(new(DevNull), "", log.LstdFlags) } if es.config.ErrorLogEnabled { errorLogger = log.New(os.Stderr, "", log.LstdFlags) } else { errorLogger = log.New(new(DevNull), "", log.LstdFlags) } client, err = elastic.NewClient(elastic.SetURL(es.hosts...), elastic.SetHttpClient(httpClient), elastic.SetGzip(es.config.GzipEnabled), elastic.SetInfoLog(infoLogger), elastic.SetErrorLog(errorLogger)) if err != nil { log.Printf("Error starting Elasticsearch: %s, will retry", err) time.Sleep(2 * time.Second) continue } es.insertIndexTemplate(client) break } log.Printf("Connected to Elasticsearch") service := elastic.NewBulkService(client) // Add the client as a subscriber receiveChan := make(chan *buffer.Event, esRecvBuffer) es.b.AddSubscriber(es.host, receiveChan) defer es.b.DelSubscriber(es.host) rateCounter := ratecounter.NewRateCounter(1 * time.Second) // Create indexer idx := &Indexer{service, es.config.IndexPrefix, es.config.IndexType, rateCounter, time.Now()} // Loop events and publish to elasticsearch tick := time.NewTicker(time.Duration(esFlushInterval) * time.Second) for { readInputChannel(idx, receiveChan) if len(tick.C) > 0 || len(es.term) > 0 { select { case <-tick.C: idx.flush() case <-es.term: tick.Stop() log.Println("Elasticsearch received term signal") break } } } return nil }
func main() { counter = ratecounter.NewRateCounter(1 * time.Minute) http.HandleFunc("/", increment) http.ListenAndServe(":80", nil) }
package main import ( "fmt" "net/http" "strconv" "time" "github.com/gocraft/health" "github.com/paulbellamy/ratecounter" ) var GlobalRate = ratecounter.NewRateCounter(1 * time.Second) type TykMiddlewareImplementation interface { New() GetConfig() (interface{}, error) ProcessRequest(w http.ResponseWriter, r *http.Request, configuration interface{}) (error, int) // Handles request IsEnabledForSpec() bool GetName() string } func CreateDynamicMiddleware(MiddlewareName string, IsPre, UseSession bool, tykMwSuper *TykMiddleware) func(http.Handler) http.Handler { dMiddleware := &DynamicMiddleware{ TykMiddleware: tykMwSuper, MiddlewareClassName: MiddlewareName, Pre: IsPre, UseSession: UseSession, } return CreateMiddleware(dMiddleware, tykMwSuper)
func init() { n = 0 linecount = 0 counter = ratecounter.NewRateCounter(1 * time.Second) }