Exemplo n.º 1
0
func main() {
	region := "us-east-1"
	queueName := "example_queue"
	numFetchers := 3

	// set up an SQS service instance
	// note that you can modify the AWS config used - make your own sqsconsumer.AWSConfigOption
	// or just depend on ~/.aws/... or environment variables and don't pass any opts at all
	s, err := sqsconsumer.SQSServiceForQueue(queueName, sqsconsumer.OptAWSRegion(region))
	if err != nil {
		log.Fatalf("Could not set up queue '%s': %s", queueName, err)
	}

	// set up a context which will gracefully cancel the worker on interrupt
	fetchCtx, cancelFetch := context.WithCancel(context.Background())
	term := make(chan os.Signal, 1)
	signal.Notify(term, os.Interrupt, os.Kill)
	go func() {
		<-term
		log.Println("Starting graceful shutdown")
		cancelFetch()
	}()

	// set up metrics - note TrackMetrics does not run the http server, and uses expvar
	exposeMetrics()
	ms := expvar.NewInt(fmt.Sprintf("%s.success", queueName))
	mf := expvar.NewInt(fmt.Sprintf("%s.fail", queueName))
	mt := expvar.NewFloat(fmt.Sprintf("%s.time", queueName))
	track := middleware.TrackMetrics(ms, mf, mt)

	// wrap the handler
	handler := middleware.ApplyDecoratorsToHandler(processMessage, track)

	// start the consumers
	log.Println("Starting queue consumers")

	wg := &sync.WaitGroup{}
	wg.Add(numFetchers)
	for i := 0; i < numFetchers; i++ {
		go func() {
			// create the consumer and bind it to a queue and processor function
			c := sqsconsumer.NewConsumer(s, handler)

			// start running the consumer with a context that will be cancelled when a graceful shutdown is requested
			c.Run(fetchCtx)

			wg.Done()
		}()
	}

	// wait for all the consumers to exit cleanly
	wg.Wait()
	log.Println("Shutdown complete")
}
Exemplo n.º 2
0
// exposeMetrics adds expvar metrics updated every 5 seconds and runs the HTTP server to expose them.
func exposeMetrics() {
	goroutines := expvar.NewInt("total_goroutines")
	uptime := expvar.NewFloat("process_uptime_seconds")

	start := time.Now()

	go func() {
		for range time.Tick(5 * time.Second) {
			goroutines.Set(int64(runtime.NumGoroutine()))
			uptime.Set(time.Since(start).Seconds())
		}
	}()

	log.Println("Expvars at http://localhost:8123/debug/vars")
	go http.ListenAndServe(":8123", nil)
}
Exemplo n.º 3
0
func TestTrackMetricsMiddleware(t *testing.T) {
	// given a TrackMetrics with known expvar metric names
	successes := expvar.NewInt("success")
	fails := expvar.NewInt("fail")
	timing := expvar.NewFloat("timing")
	m := TrackMetrics(successes, fails, timing)

	// when tracking 9 successes, 6 failures with varied runtimes
	for i := 0; i < 3; i++ {
		m(testHandlerReturnAfterDelay(true, 10*time.Millisecond))(context.Background(), "")
		m(testHandlerReturnAfterDelay(true, 20*time.Millisecond))(context.Background(), "")
		m(testHandlerReturnAfterDelay(true, 50*time.Millisecond))(context.Background(), "")
		m(testHandlerReturnAfterDelay(false, 100*time.Millisecond))(context.Background(), "")
		m(testHandlerReturnAfterDelay(false, 110*time.Millisecond))(context.Background(), "")
	}

	// expvar metrics for success and fail counts should match
	assert.Equal(t, "9", successes.String(), "Success count should match")
	assert.Equal(t, "6", fails.String(), "Failure count should match")

	// expvar metric for timing moving average
	avg, _ := strconv.ParseFloat(timing.String(), 64)
	assert.InDelta(t, 15, avg, 1.5, "Timing average should match (within 10%)")
}
Exemplo n.º 4
0
func main() {
	var inerInt int64 = 10
	pubInt := expvar.NewInt("Int")
	pubInt.Set(inerInt)
	pubInt.Add(2)

	var inerFloat float64 = 1.2
	pubFloat := expvar.NewFloat("Float")
	pubFloat.Set(inerFloat)
	pubFloat.Add(0.1)

	var inerString string = "hello gophers"
	pubString := expvar.NewString("String")
	pubString.Set(inerString)

	pubMap := expvar.NewMap("Map").Init()
	pubMap.Set("Int", pubInt)
	pubMap.Set("Float", pubFloat)
	pubMap.Set("String", pubString)
	pubMap.Add("Int", 1)
	pubMap.Add("NewInt", 123)
	pubMap.AddFloat("Float", 0.5)
	pubMap.AddFloat("NewFloat", 0.9)
	pubMap.Do(kvfunc)

	expvar.Do(kvfunc)

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprint(w, "hello gophers")
	})
	err := http.ListenAndServe(":8080", nil)
	if err != nil {
		panic(err)
	}

}
Exemplo n.º 5
0
		// check for len(tx.Err) > 0
		tx.SetStatus(transaction.StatusIngest)
		fallthrough
	case transaction.StatusIngest:
		tx.Commit(*s.Items, s.FileStore, s.Cache)
	}
	duration := time.Now().Sub(start)
	log.Printf("Finish transaction %s on %s (%s)", tx.ID, tx.ItemID, duration.String())

	xTransactionTime.Add(duration.Seconds())
	xTransactionCount.Add(1)
}

var (
	xTransactionCount = expvar.NewInt("tx.count")
	xTransactionTime  = expvar.NewFloat("tx.seconds")
)

// TxCleaner will loop forever removing old transactions and old orphened
// uploaded files. That means any finished transactions which are older than a
// few days. Both the transaction and any uploaded files referenced by the
// transaction are deleted. This function will never return.
func (s *RESTServer) TxCleaner() {
	for {
		err := s.transactionCleaner()
		if err == nil {
			err = s.fileCleaner()
		}
		if err != nil {
			log.Println("TxCleaner:", err)
		}
Exemplo n.º 6
0
// NewGauge returns a new Gauge backed by an expvar with the given name. It
// should be updated manually; for a callback-based approach, see
// PublishCallbackGauge. Fields are ignored.
func NewGauge(name string) metrics.Gauge {
	return &gauge{expvar.NewFloat(name)}
}
Exemplo n.º 7
0
				BytesReceived: expvar.NewInt("sys.broker.load.bytes_received"),
			},
			SubscriptionsCount: expvar.NewInt("sys.broker.subscriptions.count"),
		},
	},

	// for debug
	NumGoroutine:  expvar.NewInt("numgoroutine"),
	NumCgoCall:    expvar.NewInt("numcgocall"),
	Uptime:        expvar.NewInt("uptime"),
	MemFree:       expvar.NewInt("memfree"),
	MemUsed:       expvar.NewInt("memused"),
	MemActualFree: expvar.NewInt("memactualfree"),
	MemActualUsed: expvar.NewInt("memactualused"),
	MemTotal:      expvar.NewInt("memtotal"),
	LoadOne:       expvar.NewFloat("loadone"),
	LoadFive:      expvar.NewFloat("loadfive"),
	LoadFifteen:   expvar.NewFloat("loadfifteen"),
	CpuUser:       expvar.NewFloat("cpuuser"),
	CpuNice:       expvar.NewFloat("cpunice"),
	CpuSys:        expvar.NewFloat("cpusys"),
	CpuIdle:       expvar.NewFloat("cpuidle"),
	CpuWait:       expvar.NewFloat("cpuwait"),
	CpuIrq:        expvar.NewFloat("cpuirq"),
	CpuSoftIrq:    expvar.NewFloat("cpusoftirq"),
	CpuStolen:     expvar.NewFloat("cpustolen"),
	CpuTotal:      expvar.NewFloat("cputotal"),

	MessageSentPerSec: myexpvar.NewDiffInt("msg_sent_per_sec"),
	ConnectPerSec:     myexpvar.NewDiffInt("connect_per_sec"),
	GoroutinePerConn:  expvar.NewFloat("goroutine_per_conn"),
Exemplo n.º 8
0
	// SetCheck takes an item id and schedules another fixity check at the
	// given time in the future (or past).
	SetCheck(id string, when time.Time) error

	// LookupCheck takes an item id and returns the time of earliest pending
	// fixity check for that item. If no fixity check is pending, returns
	// the zero time.
	LookupCheck(id string) (time.Time, error)
}

var (
	xFixityRunning      = expvar.NewInt("fixity.running")
	xFixityItemsChecked = expvar.NewInt("fixity.check.count")
	xFixityBytesChecked = expvar.NewInt("fixity.check.bytes")
	xFixityDuration     = expvar.NewFloat("fixity.check.seconds")
	xFixityError        = expvar.NewInt("fixity.check.error")
	xFixityMismatch     = expvar.NewInt("fixity.check.mismatch")
)

// StartFixity starts the background goroutines to check item fixity. It
// returns immediately and does not block.
func (s *RESTServer) StartFixity() {
	xFixityRunning.Add(1)

	go s.fixity()

	// should scanfixity run periodically? or only at startup?
	// this will keep running it in a loop with 24 hour rest in between.
	go func() {
		for {
Exemplo n.º 9
0
Arquivo: expvar.go Projeto: qband/down
// NewGauge returns a new Gauge backed by an expvar with the given name. It
// should be updated manually; for a callback-based approach, see
// PublishCallbackGauge. Fields are ignored.
func NewGauge(name string) metrics.Gauge {
	return &gauge{
		name: name,
		v:    expvar.NewFloat(name),
	}
}