func NewRuntimeWare(prefixes []string, trackPageview bool, logInterval ...time.Duration) Middleware {
expvar.NewString("at_server_start").Set(time.Now().Format("2006-01-02 15:04:05"))
expvar.NewInt("cpu_count").Set(int64(runtime.NumCPU()))
ware := &RuntimeWare{
serverStarted: time.Now(),
trackPageview: trackPageview,
ignoredUrls: prefixes,
cQps: ratecounter.NewRateCounter(time.Minute),
c4xx: ratecounter.NewRateCounter(5 * time.Minute),
c5xx: ratecounter.NewRateCounter(5 * time.Minute),
lc: NewLatencyCounter(50),
hitsTotal: expvar.NewInt("hits_total"),
hitsQps: expvar.NewInt("hits_per_minute"),
hits4xx: expvar.NewInt("hits_4xx_per_5min"),
hits5xx: expvar.NewInt("hits_5xx_per_5min"),
hitsServed: expvar.NewString("latency_recent"),
hitsLatMax: expvar.NewString("latency_max"),
hitsLatMin: expvar.NewString("latency_min"),
hitsLat95: expvar.NewString("latency_p95"),
hitsLat50: expvar.NewString("latency_p50"),
numGoroutine: expvar.NewInt("goroutine_count"),
}
if trackPageview {
ware.pageviews = expvar.NewMap("hits_pageviews")
}
if len(logInterval) > 0 && logInterval[0] > 0 {
go ware.logSnapshot(logInterval[0])
}
return ware
}
func (redisServer *RedisServer) Start() error {
if redisServer.sender == nil {
log.Printf("[%s] No Route is specified for this output", redisServer.name)
return nil
}
// Add the client as a subscriber
receiveChan := make(chan *buffer.Event, recvBuffer)
redisServer.sender.AddSubscriber(redisServer.name, receiveChan)
defer redisServer.sender.DelSubscriber(redisServer.name)
allQueues := make([]*RedisQueue, len(redisServer.config.CopyQueues))
// Create Redis queue
for index, key := range redisServer.config.CopyQueues {
redisQueue := NewRedisQueue(redisServer.config, key)
allQueues[index] = redisQueue
go redisQueue.Start()
}
log.Printf("[%s] Started Redis Output Instance", redisServer.name)
// Loop events and publish to Redis
tick := time.NewTicker(time.Duration(redisFlushInterval) * time.Second)
rateCounter := ratecounter.NewRateCounter(1 * time.Second)
for {
select {
case ev := <-receiveChan:
rateCounter.Incr(1)
var allowed bool
allowed = true
for key, value := range redisServer.fields {
if (*ev.Fields)[key] == nil || ((*ev.Fields)[key] != nil && value != (*ev.Fields)[key].(string)) {
allowed = false
break
}
}
if allowed {
text := *ev.Text
for _, queue := range allQueues {
queue.data <- text
}
}
case <-tick.C:
if rateCounter.Rate() > 0 {
log.Printf("[%s] Current Redis input rate: %d/s\n", redisServer.name, rateCounter.Rate())
}
case <-redisServer.term:
log.Println("RedisServer received term signal")
for _, queue := range allQueues {
queue.term <- true
}
return nil
}
}
return nil
}
/*
* Initializations
*/
func init() {
flag.Parse()
status.InputEventCount = expvar.NewInt("input_event_count")
status.OutputEventCount = expvar.NewInt("output_event_count")
status.ErrorCount = expvar.NewInt("error_count")
status.EventCounter = ratecounter.NewRateCounter(5 * time.Second)
status.OutputBytesPerSecond = ratecounter.NewRateCounter(5 * time.Second)
expvar.Publish("input_events_per_second",
expvar.Func(func() interface{} { return float64(status.EventCounter.Rate()) / 5.0 }))
expvar.Publish("output_bytes_per_second",
expvar.Func(func() interface{} { return float64(status.OutputBytesPerSecond.Rate()) / 5.0 }))
expvar.Publish("connection_status",
expvar.Func(func() interface{} {
res := make(map[string]interface{}, 0)
res["last_connect_time"] = status.LastConnectTime
res["last_error_text"] = status.LastConnectError
res["last_error_time"] = status.ErrorTime
if status.IsConnected {
res["connected"] = true
res["uptime"] = time.Now().Sub(status.LastConnectTime).Seconds()
} else {
res["connected"] = false
res["uptime"] = 0.0
}
return res
}))
expvar.Publish("uptime", expvar.Func(func() interface{} {
return time.Now().Sub(status.StartTime).Seconds()
}))
expvar.Publish("subscribed_events", expvar.Func(func() interface{} {
return config.EventTypes
}))
results = make(chan string, 100)
output_errors = make(chan error)
status.StartTime = time.Now()
}
func redisGet(redisServer *RedisInputServer, consumer *redismq.Consumer) error {
consumer.ResetWorking()
rateCounter := ratecounter.NewRateCounter(1 * time.Second)
for {
unacked := consumer.GetUnackedLength()
if unacked > 0 {
log.Printf("Requeued %d messages\n", unacked)
consumer.RequeueWorking()
}
packages, err := consumer.MultiGet(recvBuffer)
if err == nil {
numPackages := len(packages)
if numPackages > 0 {
rateCounter.Incr(int64(numPackages))
err = packages[numPackages-1].MultiAck()
if err != nil {
log.Println("Failed to ack", err)
}
}
for i := range packages {
var ev buffer.Event
payload := string(packages[i].Payload)
ev.Text = &payload
if redisServer.config.JsonDecode {
decoder := json.NewDecoder(strings.NewReader(payload))
decoder.UseNumber()
err = decoder.Decode(&ev.Fields)
if err != nil {
continue
}
}
redisServer.receiver.Send(&ev)
}
} else {
log.Printf("Error reading from Redis: %s, sleeping", err)
time.Sleep(2 * time.Second)
}
}
return nil
}
func (s3Writer *S3Writer) Start() error {
if s3Writer.Sender == nil {
log.Printf("[%s] No route is specified for this output", s3Writer.name)
return nil
}
// Create file saver
fileSaver := new(FileSaver)
fileSaver.Config = s3Writer.Config
fileSaver.RateCounter = ratecounter.NewRateCounter(1 * time.Second)
id := "s3_output"
// Add the client as a subscriber
receiveChan := make(chan *buffer.Event, recvBuffer)
s3Writer.Sender.AddSubscriber(id, receiveChan)
defer s3Writer.Sender.DelSubscriber(id)
// Loop events and publish to S3
tick := time.NewTicker(time.Duration(s3FlushInterval) * time.Second)
go s3Writer.WaitForUpload()
for {
select {
case ev := <-receiveChan:
var allowed bool
allowed = true
for key, value := range s3Writer.fields {
if (*ev.Fields)[key] == nil || ((*ev.Fields)[key] != nil && value != (*ev.Fields)[key].(string)) {
allowed = false
break
}
}
if allowed {
fileSaver.WriteToFile(s3Writer.name, ev)
}
case <-tick.C:
s3Writer.InitiateUploadToS3(fileSaver)
case <-s3Writer.term:
log.Println("S3Writer received term signal")
return nil
}
}
return nil
}
// AddCount metrics
func AddCount(name string) {
updLastTime(name, time.Now())
var counter *ratecounter.RateCounter
counter, ok := counters[name]
if !ok {
counter = ratecounter.NewRateCounter(1 * time.Minute)
counters[name] = counter
}
var request *expvar.Int
request, ok = requests[name]
if !ok {
request = expvar.NewInt(name)
requests[name] = request
}
counter.Incr(1)
request.Set(counter.Rate())
}
func main() {
counter = ratecounter.NewRateCounter(1 * time.Minute)
port := os.Getenv("PORT")
if port == "" {
log.Fatal("$PORT must be set")
}
var handler httpctx.Handler
handler = httpctx.HandlerFunc(requestHandler)
handler = userIPMiddleware(handler)
handler = requestIDMiddleware(handler)
handler = requestCtrMiddleware(handler)
ctx := context.Background()
svc := &Server{ctx, handler}
http.HandleFunc("/", svc.ServeHTTP)
log.Fatal(http.ListenAndServe(":"+port, nil))
}
func init() {
counter = ratecounter.NewRateCounter(1 * time.Second)
rand.Seed(time.Now().UnixNano())
}
func (es *ESServer) Start() error {
if es.b == nil {
log.Printf("[%s] No Route is specified for this output", es.name)
return nil
}
var client *elastic.Client
var err error
for {
httpClient := http.DefaultClient
timeout := 60 * time.Second
if es.config.Timeout > 0 {
timeout = time.Duration(es.config.Timeout) * time.Second
}
log.Println("Setting HTTP timeout to", timeout)
log.Println("Setting GZIP enabled:", es.config.GzipEnabled)
httpClient.Timeout = timeout
var infoLogger, errorLogger *log.Logger
if es.config.InfoLogEnabled {
infoLogger = log.New(os.Stdout, "", log.LstdFlags)
} else {
infoLogger = log.New(new(DevNull), "", log.LstdFlags)
}
if es.config.ErrorLogEnabled {
errorLogger = log.New(os.Stderr, "", log.LstdFlags)
} else {
errorLogger = log.New(new(DevNull), "", log.LstdFlags)
}
client, err = elastic.NewClient(elastic.SetURL(es.hosts...),
elastic.SetHttpClient(httpClient),
elastic.SetGzip(es.config.GzipEnabled),
elastic.SetInfoLog(infoLogger),
elastic.SetErrorLog(errorLogger))
if err != nil {
log.Printf("Error starting Elasticsearch: %s, will retry", err)
time.Sleep(2 * time.Second)
continue
}
es.insertIndexTemplate(client)
break
}
log.Printf("Connected to Elasticsearch")
service := elastic.NewBulkService(client)
// Add the client as a subscriber
receiveChan := make(chan *buffer.Event, esRecvBuffer)
es.b.AddSubscriber(es.host, receiveChan)
defer es.b.DelSubscriber(es.host)
rateCounter := ratecounter.NewRateCounter(1 * time.Second)
// Create indexer
idx := &Indexer{service, es.config.IndexPrefix, es.config.IndexType, rateCounter, time.Now()}
// Loop events and publish to elasticsearch
tick := time.NewTicker(time.Duration(esFlushInterval) * time.Second)
for {
readInputChannel(idx, receiveChan)
if len(tick.C) > 0 || len(es.term) > 0 {
select {
case <-tick.C:
idx.flush()
case <-es.term:
tick.Stop()
log.Println("Elasticsearch received term signal")
break
}
}
}
return nil
}
func main() {
counter = ratecounter.NewRateCounter(1 * time.Minute)
http.HandleFunc("/", increment)
http.ListenAndServe(":80", nil)
}
package main
import (
"fmt"
"net/http"
"strconv"
"time"
"github.com/gocraft/health"
"github.com/paulbellamy/ratecounter"
)
var GlobalRate = ratecounter.NewRateCounter(1 * time.Second)
type TykMiddlewareImplementation interface {
New()
GetConfig() (interface{}, error)
ProcessRequest(w http.ResponseWriter, r *http.Request, configuration interface{}) (error, int) // Handles request
IsEnabledForSpec() bool
GetName() string
}
func CreateDynamicMiddleware(MiddlewareName string, IsPre, UseSession bool, tykMwSuper *TykMiddleware) func(http.Handler) http.Handler {
dMiddleware := &DynamicMiddleware{
TykMiddleware: tykMwSuper,
MiddlewareClassName: MiddlewareName,
Pre: IsPre,
UseSession: UseSession,
}
return CreateMiddleware(dMiddleware, tykMwSuper)
func init() {
n = 0
linecount = 0
counter = ratecounter.NewRateCounter(1 * time.Second)
}
