// parameters should be tuned so that:
// can buffer packets for the duration of 1 sync
// buffer no more then needed, esp if we know the queue is slower then the ingest rate
func NewSpool(key, spoolDir string) *Spool {
dqName := "spool_" + key
// on our virtualized box i see mean write of around 100 micros upto 250 micros, max up to 200 millis.
// in 200 millis we can get up to 10k metrics, so let's make that our queueBuffer size
// for bulk, leaving 500 micros in between every metric should be enough.
// TODO make all these configurable:
queueBuffer := 10000
maxBytesPerFile := int64(200 * 1024 * 1024)
syncEvery := int64(10000)
periodSync := 1 * time.Second
queue := nsqd.NewDiskQueue(dqName, spoolDir, maxBytesPerFile, syncEvery, periodSync).(*nsqd.DiskQueue)
spoolSleep := time.Duration(500) * time.Microsecond
unspoolSleep := time.Duration(10) * time.Microsecond
s := Spool{
key: key,
InRT: make(chan []byte, 10),
InBulk: make(chan []byte),
Out: NewSlowChan(queue.ReadChan(), unspoolSleep),
spoolSleep: spoolSleep,
unspoolSleep: unspoolSleep,
queue: queue,
queueBuffer: make(chan []byte, queueBuffer),
durationWrite: stats.Timer("spool=" + key + ".operation=write"),
durationBuffer: stats.Timer("spool=" + key + ".operation=buffer"),
numBuffered: stats.Gauge("spool=" + key + ".unit=Metric.status=buffered"),
numIncomingRT: stats.Counter("spool=" + key + ".unit=Metric.status=incomingRT"),
numIncomingBulk: stats.Counter("spool=" + key + ".unit=Metric.status=incomingBulk"),
shutdownWriter: make(chan bool),
shutdownBuffer: make(chan bool),
}
go s.Writer()
go s.Buffer()
return &s
}
// NewGrafanaNet creates a special route that writes to a grafana.net datastore
// We will automatically run the route and the destination
// ignores spool for now
func NewGrafanaNet(key, prefix, sub, regex, addr, apiKey, schemasFile string, spool, sslVerify bool, bufSize, flushMaxNum, flushMaxWait, timeout int) (Route, error) {
m, err := matcher.New(prefix, sub, regex)
if err != nil {
return nil, err
}
schemas, err := persister.ReadWhisperSchemas(schemasFile)
if err != nil {
return nil, err
}
var defaultFound bool
for _, schema := range schemas {
if schema.Pattern.String() == ".*" {
defaultFound = true
}
if len(schema.Retentions) == 0 {
return nil, fmt.Errorf("retention setting cannot be empty")
}
}
if !defaultFound {
// good graphite health (not sure what graphite does if there's no .*
// but we definitely need to always be able to determine which interval to use
return nil, fmt.Errorf("storage-conf does not have a default '.*' pattern")
}
cleanAddr := util.AddrToPath(addr)
r := &GrafanaNet{
baseRoute: baseRoute{sync.Mutex{}, atomic.Value{}, key},
addr: addr,
apiKey: apiKey,
buf: make(chan []byte, bufSize), // takes about 228MB on 64bit
schemas: schemas,
bufSize: bufSize,
flushMaxNum: flushMaxNum,
flushMaxWait: time.Duration(flushMaxWait) * time.Millisecond,
timeout: time.Duration(timeout) * time.Millisecond,
sslVerify: sslVerify,
numErrFlush: stats.Counter("dest=" + cleanAddr + ".unit=Err.type=flush"),
numOut: stats.Counter("dest=" + cleanAddr + ".unit=Metric.direction=out"),
durationTickFlush: stats.Timer("dest=" + cleanAddr + ".what=durationFlush.type=ticker"),
durationManuFlush: stats.Timer("dest=" + cleanAddr + ".what=durationFlush.type=manual"),
tickFlushSize: stats.Histogram("dest=" + cleanAddr + ".unit=B.what=FlushSize.type=ticker"),
manuFlushSize: stats.Histogram("dest=" + cleanAddr + ".unit=B.what=FlushSize.type=manual"),
numBuffered: stats.Gauge("dest=" + cleanAddr + ".unit=Metric.what=numBuffered"),
}
r.config.Store(baseConfig{*m, make([]*dest.Destination, 0)})
go r.run()
return r, nil
}
func NewConn(addr string, dest *Destination, periodFlush time.Duration, pickle bool) (*Conn, error) {
raddr, err := net.ResolveTCPAddr("tcp", addr)
if err != nil {
return nil, err
}
laddr, _ := net.ResolveTCPAddr("tcp", "0.0.0.0")
conn, err := net.DialTCP("tcp", laddr, raddr)
if err != nil {
return nil, err
}
cleanAddr := util.AddrToPath(addr)
connObj := &Conn{
conn: conn,
buffered: NewWriter(conn, bufio_buffer_size, cleanAddr),
shutdown: make(chan bool, 1), // when we write here, HandleData() may not be running anymore to read from the chan
In: make(chan []byte, conn_in_buffer),
dest: dest,
up: true,
pickle: pickle,
checkUp: make(chan bool),
updateUp: make(chan bool),
flush: make(chan bool),
flushErr: make(chan error),
periodFlush: periodFlush,
keepSafe: NewKeepSafe(keepsafe_initial_cap, keepsafe_keep_duration),
numErrTruncated: stats.Counter("dest=" + cleanAddr + ".unit=Err.type=truncated"),
numErrWrite: stats.Counter("dest=" + cleanAddr + ".unit=Err.type=write"),
numErrFlush: stats.Counter("dest=" + cleanAddr + ".unit=Err.type=flush"),
numOut: stats.Counter("dest=" + cleanAddr + ".unit=Metric.direction=out"),
durationWrite: stats.Timer("dest=" + cleanAddr + ".what=durationWrite"),
durationTickFlush: stats.Timer("dest=" + cleanAddr + ".what=durationFlush.type=ticker"),
durationManuFlush: stats.Timer("dest=" + cleanAddr + ".what=durationFlush.type=manual"),
tickFlushSize: stats.Histogram("dest=" + cleanAddr + ".unit=B.what=FlushSize.type=ticker"),
manuFlushSize: stats.Histogram("dest=" + cleanAddr + ".unit=B.what=FlushSize.type=manual"),
numBuffered: stats.Gauge("dest=" + cleanAddr + ".unit=Metric.what=numBuffered"),
numDropBadPickle: stats.Counter("dest=" + cleanAddr + ".unit=Metric.action=drop.reason=bad_pickle"),
}
go connObj.checkEOF()
go connObj.HandleData()
go connObj.HandleStatus()
return connObj, nil
}
func New(spoolDir string) *Table {
t := &Table{
sync.Mutex{},
atomic.Value{},
spoolDir,
stats.Counter("unit=Metric.direction=blacklist"),
stats.Counter("unit=Metric.direction=unroutable"),
make(chan []byte),
}
t.config.Store(TableConfig{
make([]rewriter.RW, 0),
make([]*aggregator.Aggregator, 0),
make([]*matcher.Matcher, 0),
make([]route.Route, 0),
})
go func() {
for buf := range t.In {
t.DispatchAggregate(buf)
}
}()
return t
}
func init() {
numIn = stats.Counter("unit=Metric.direction=in")
numInvalid = stats.Counter("unit=Err.type=invalid")
numOutOfOrder = stats.Counter("unit=Err.type=out_of_order")
}
func (dest *Destination) setMetrics() {
dest.numDropNoConnNoSpool = stats.Counter("dest=" + dest.cleanAddr + ".unit=Metric.action=drop.reason=conn_down_no_spool")
dest.numDropSlowSpool = stats.Counter("dest=" + dest.cleanAddr + ".unit=Metric.action=drop.reason=slow_spool")
dest.numDropSlowConn = stats.Counter("dest=" + dest.cleanAddr + ".unit=Metric.action=drop.reason=slow_conn")
}