Example #1
0
// parameters should be tuned so that:
// can buffer packets for the duration of 1 sync
// buffer no more then needed, esp if we know the queue is slower then the ingest rate
func NewSpool(key, spoolDir string) *Spool {
	dqName := "spool_" + key
	// on our virtualized box i see mean write of around 100 micros upto 250 micros, max up to 200 millis.
	// in 200 millis we can get up to 10k metrics, so let's make that our queueBuffer size
	// for bulk, leaving 500 micros in between every metric should be enough.
	// TODO make all these configurable:
	queueBuffer := 10000
	maxBytesPerFile := int64(200 * 1024 * 1024)
	syncEvery := int64(10000)
	periodSync := 1 * time.Second
	queue := nsqd.NewDiskQueue(dqName, spoolDir, maxBytesPerFile, syncEvery, periodSync).(*nsqd.DiskQueue)

	spoolSleep := time.Duration(500) * time.Microsecond
	unspoolSleep := time.Duration(10) * time.Microsecond
	s := Spool{
		key:             key,
		InRT:            make(chan []byte, 10),
		InBulk:          make(chan []byte),
		Out:             NewSlowChan(queue.ReadChan(), unspoolSleep),
		spoolSleep:      spoolSleep,
		unspoolSleep:    unspoolSleep,
		queue:           queue,
		queueBuffer:     make(chan []byte, queueBuffer),
		durationWrite:   stats.Timer("spool=" + key + ".operation=write"),
		durationBuffer:  stats.Timer("spool=" + key + ".operation=buffer"),
		numBuffered:     stats.Gauge("spool=" + key + ".unit=Metric.status=buffered"),
		numIncomingRT:   stats.Counter("spool=" + key + ".unit=Metric.status=incomingRT"),
		numIncomingBulk: stats.Counter("spool=" + key + ".unit=Metric.status=incomingBulk"),
		shutdownWriter:  make(chan bool),
		shutdownBuffer:  make(chan bool),
	}
	go s.Writer()
	go s.Buffer()
	return &s
}
// NewGrafanaNet creates a special route that writes to a grafana.net datastore
// We will automatically run the route and the destination
// ignores spool for now
func NewGrafanaNet(key, prefix, sub, regex, addr, apiKey, schemasFile string, spool, sslVerify bool, bufSize, flushMaxNum, flushMaxWait, timeout int) (Route, error) {
	m, err := matcher.New(prefix, sub, regex)
	if err != nil {
		return nil, err
	}
	schemas, err := persister.ReadWhisperSchemas(schemasFile)
	if err != nil {
		return nil, err
	}
	var defaultFound bool
	for _, schema := range schemas {
		if schema.Pattern.String() == ".*" {
			defaultFound = true
		}
		if len(schema.Retentions) == 0 {
			return nil, fmt.Errorf("retention setting cannot be empty")
		}
	}
	if !defaultFound {
		// good graphite health (not sure what graphite does if there's no .*
		// but we definitely need to always be able to determine which interval to use
		return nil, fmt.Errorf("storage-conf does not have a default '.*' pattern")
	}

	cleanAddr := util.AddrToPath(addr)

	r := &GrafanaNet{
		baseRoute: baseRoute{sync.Mutex{}, atomic.Value{}, key},
		addr:      addr,
		apiKey:    apiKey,
		buf:       make(chan []byte, bufSize), // takes about 228MB on 64bit
		schemas:   schemas,

		bufSize:      bufSize,
		flushMaxNum:  flushMaxNum,
		flushMaxWait: time.Duration(flushMaxWait) * time.Millisecond,
		timeout:      time.Duration(timeout) * time.Millisecond,
		sslVerify:    sslVerify,

		numErrFlush:       stats.Counter("dest=" + cleanAddr + ".unit=Err.type=flush"),
		numOut:            stats.Counter("dest=" + cleanAddr + ".unit=Metric.direction=out"),
		durationTickFlush: stats.Timer("dest=" + cleanAddr + ".what=durationFlush.type=ticker"),
		durationManuFlush: stats.Timer("dest=" + cleanAddr + ".what=durationFlush.type=manual"),
		tickFlushSize:     stats.Histogram("dest=" + cleanAddr + ".unit=B.what=FlushSize.type=ticker"),
		manuFlushSize:     stats.Histogram("dest=" + cleanAddr + ".unit=B.what=FlushSize.type=manual"),
		numBuffered:       stats.Gauge("dest=" + cleanAddr + ".unit=Metric.what=numBuffered"),
	}

	r.config.Store(baseConfig{*m, make([]*dest.Destination, 0)})
	go r.run()
	return r, nil
}
Example #3
0
func NewConn(addr string, dest *Destination, periodFlush time.Duration, pickle bool) (*Conn, error) {
	raddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, err
	}
	laddr, _ := net.ResolveTCPAddr("tcp", "0.0.0.0")
	conn, err := net.DialTCP("tcp", laddr, raddr)
	if err != nil {
		return nil, err
	}
	cleanAddr := util.AddrToPath(addr)
	connObj := &Conn{
		conn:              conn,
		buffered:          NewWriter(conn, bufio_buffer_size, cleanAddr),
		shutdown:          make(chan bool, 1), // when we write here, HandleData() may not be running anymore to read from the chan
		In:                make(chan []byte, conn_in_buffer),
		dest:              dest,
		up:                true,
		pickle:            pickle,
		checkUp:           make(chan bool),
		updateUp:          make(chan bool),
		flush:             make(chan bool),
		flushErr:          make(chan error),
		periodFlush:       periodFlush,
		keepSafe:          NewKeepSafe(keepsafe_initial_cap, keepsafe_keep_duration),
		numErrTruncated:   stats.Counter("dest=" + cleanAddr + ".unit=Err.type=truncated"),
		numErrWrite:       stats.Counter("dest=" + cleanAddr + ".unit=Err.type=write"),
		numErrFlush:       stats.Counter("dest=" + cleanAddr + ".unit=Err.type=flush"),
		numOut:            stats.Counter("dest=" + cleanAddr + ".unit=Metric.direction=out"),
		durationWrite:     stats.Timer("dest=" + cleanAddr + ".what=durationWrite"),
		durationTickFlush: stats.Timer("dest=" + cleanAddr + ".what=durationFlush.type=ticker"),
		durationManuFlush: stats.Timer("dest=" + cleanAddr + ".what=durationFlush.type=manual"),
		tickFlushSize:     stats.Histogram("dest=" + cleanAddr + ".unit=B.what=FlushSize.type=ticker"),
		manuFlushSize:     stats.Histogram("dest=" + cleanAddr + ".unit=B.what=FlushSize.type=manual"),
		numBuffered:       stats.Gauge("dest=" + cleanAddr + ".unit=Metric.what=numBuffered"),
		numDropBadPickle:  stats.Counter("dest=" + cleanAddr + ".unit=Metric.action=drop.reason=bad_pickle"),
	}

	go connObj.checkEOF()

	go connObj.HandleData()
	go connObj.HandleStatus()
	return connObj, nil
}
Example #4
0
func New(spoolDir string) *Table {
	t := &Table{
		sync.Mutex{},
		atomic.Value{},
		spoolDir,
		stats.Counter("unit=Metric.direction=blacklist"),
		stats.Counter("unit=Metric.direction=unroutable"),
		make(chan []byte),
	}

	t.config.Store(TableConfig{
		make([]rewriter.RW, 0),
		make([]*aggregator.Aggregator, 0),
		make([]*matcher.Matcher, 0),
		make([]route.Route, 0),
	})

	go func() {
		for buf := range t.In {
			t.DispatchAggregate(buf)
		}
	}()
	return t
}
Example #5
0
func init() {
	numIn = stats.Counter("unit=Metric.direction=in")
	numInvalid = stats.Counter("unit=Err.type=invalid")
	numOutOfOrder = stats.Counter("unit=Err.type=out_of_order")
}
func (dest *Destination) setMetrics() {
	dest.numDropNoConnNoSpool = stats.Counter("dest=" + dest.cleanAddr + ".unit=Metric.action=drop.reason=conn_down_no_spool")
	dest.numDropSlowSpool = stats.Counter("dest=" + dest.cleanAddr + ".unit=Metric.action=drop.reason=slow_spool")
	dest.numDropSlowConn = stats.Counter("dest=" + dest.cleanAddr + ".unit=Metric.action=drop.reason=slow_conn")
}