func (q *queue) housekeeping() {
defer func() {
log.Trace("queue[%s] housekeeping done", q.ident())
q.wg.Done()
}()
log.Trace("queue[%s] start housekeeping...", q.ident())
purgeTick := time.NewTicker(q.purgeInterval)
defer purgeTick.Stop()
cursorChkpnt := time.NewTicker(time.Second)
defer cursorChkpnt.Stop()
for {
select {
case <-purgeTick.C:
if err := q.Purge(); err != nil {
log.Error("queue[%s] purge: %s", q.ident(), err)
}
case <-cursorChkpnt.C:
if err := q.cursor.dump(); err != nil {
log.Error("queue[%s] cursor checkpoint: %s", q.ident(), err)
}
case <-q.quit:
return
}
}
}
func (this *controller) RunForever() (err error) {
log.Info("controller[%s] starting", this.Id())
if err = this.orchestrator.RegisterActor(this.Id(), this.Bytes()); err != nil {
return err
}
defer this.orchestrator.ResignActor(this.Id())
if err = manager.Default.Start(); err != nil {
return
}
log.Trace("manager[%s] started", manager.Default.Name())
go this.runWebServer()
jobDispatchQuit := make(chan struct{})
go this.dispatchJobQueues(jobDispatchQuit)
webhookDispatchQuit := make(chan struct{})
go this.dispatchWebhooks(webhookDispatchQuit)
select {
case <-jobDispatchQuit:
log.Warn("dispatchJobQueues quit")
case <-webhookDispatchQuit:
log.Warn("dispatchWebhooks quit")
}
manager.Default.Stop()
log.Trace("manager[%s] stopped", manager.Default.Name())
return
}
func (this *pubStore) watchDeadPartitions() {
ticker := time.NewTicker(time.Minute * 2) // TODO
defer ticker.Stop()
var lastTopics = make(map[string]struct{})
for {
select {
case <-ticker.C:
deadPartitions := manager.Default.DeadPartitions()
for topic, dp := range deadPartitions {
this.markPartitionsDead(topic, dp)
lastTopics[topic] = struct{}{}
}
for lastDeadTopic := range lastTopics {
if _, present := deadPartitions[lastDeadTopic]; !present {
// this topic was marked dead last round, but this round it comes alive
log.Trace("%s come alive again", lastDeadTopic)
this.markPartitionsDead(lastDeadTopic, nil)
delete(lastTopics, lastDeadTopic)
}
}
case <-this.shutdownCh:
return
}
}
}
func (this *subStore) Start() (err error) {
this.subManager = newSubManager()
this.wg.Add(1)
go func() {
defer this.wg.Done()
var remoteAddr string
for {
select {
case <-this.shutdownCh:
log.Trace("sub store[%s] stopped", this.Name())
return
case remoteAddr = <-this.closedConnCh:
this.wg.Add(1)
go func(id string) {
this.subManager.killClient(id)
this.wg.Done()
}(remoteAddr)
}
}
}()
return
}
func (this *pubStore) Start() (err error) {
if ctx.KafkaHome() == "" {
return fmt.Errorf("empty kafka_home in ~/.gafka.cf")
}
if !gio.DirExists(ctx.KafkaHome()) {
return fmt.Errorf("kafka not installed in %s, run 'gk deploy -kfkonly'", ctx.KafkaHome())
}
// warmup: create pools according the current kafka topology
for _, cluster := range meta.Default.ClusterNames() {
this.pubPools[cluster] = newPubPool(this, cluster,
meta.Default.BrokerList(cluster), this.pubPoolsCapcity)
}
this.wg.Add(1)
go func() {
defer this.wg.Done()
for {
select {
case <-meta.Default.RefreshEvent():
this.doRefresh()
case <-this.shutdownCh:
log.Trace("pub store[%s] stopped", this.Name())
return
}
}
}()
return
}
func (f *File) Attr(ctx context.Context, o *fuse.Attr) error {
f.RLock()
defer f.RUnlock()
*o = f.attr
// calculate size
if !f.opened {
if err := f.dir.reconnectKafkaIfNecessary(); err != nil {
return err
}
latestOffset, err := f.dir.GetOffset(f.topic, f.partitionId, sarama.OffsetNewest)
if err != nil {
log.Error(err)
return err
}
oldestOffset, err := f.dir.GetOffset(f.topic, f.partitionId, sarama.OffsetOldest)
if err != nil {
log.Error(err)
return err
}
o.Size = uint64(latestOffset - oldestOffset)
} else {
o.Size = uint64(len(f.content))
}
log.Trace("File Attr, topic=%s, partitionId=%d, size=%d", f.topic, f.partitionId, o.Size)
return nil
}
// Close stops the queue for reading and writing
func (q *queue) Close() error {
close(q.quit)
// wait for pump and housekeeping finish
q.wg.Wait()
q.mu.Lock()
defer q.mu.Unlock()
for _, s := range q.segments {
if err := s.Close(); err != nil {
return err
}
}
q.head = nil
q.tail = nil
q.segments = nil
log.Trace("queue[%s] dumping cursor", q.ident())
if err := q.cursor.dump(); err != nil {
return err
}
q.cursor = nil
return nil
}
func (this *Client) Warmup() {
var (
sess *mgo.Session
err error
t1 = time.Now()
)
for retries := 0; retries < 3; retries++ {
for _, server := range this.selector.ServerList() {
sess, err = this.getConn(server.Uri())
if err != nil {
log.Error("Warmup %v fail: %s", server.Uri(), err)
break
} else {
this.putFreeConn(server.Uri(), sess)
}
}
if err == nil {
break
}
}
if err == nil {
log.Trace("Mongodb warmup within %s: %+v",
time.Since(t1), this.freeconns)
} else {
log.Error("Mongodb failed to warmup within %s: %s",
time.Since(t1), err)
}
}
func (this *profiler) do(callName string, ctx *rpc.Context, format string,
args ...interface{}) {
if this == nil {
return
}
elapsed := time.Since(this.t1)
slow := elapsed > config.Engine.Servants.CallSlowThreshold
if !(slow || this.on) {
return
}
body := fmt.Sprintf(format, args...)
if slow {
svtStats.incCallSlow()
header := fmt.Sprintf("SLOW=%s/%s Q=%s ",
elapsed, time.Since(this.t0), callName)
log.Warn(header + this.truncatedStr(body))
} else if this.on {
header := fmt.Sprintf("T=%s/%s Q=%s ",
elapsed, time.Since(this.t0), callName)
log.Trace(header + this.truncatedStr(body))
}
}
// NewMessage is the supported way to obtain a new Message. This makes
// use of a "slab allocator" which greatly reduces the load on the
// garbage collector.
func NewMessage(size int) *Message {
var ch chan *Message
for _, slabClass := range messagePool { // TODO binary search
if size <= slabClass.maxSize {
ch = slabClass.ch
size = slabClass.maxSize
break
}
}
var msg *Message
select {
case msg = <-ch:
default:
// message pool empty:
// too busy or size greater than largest slab class
log.Trace("allocating message memory pool: %dB", size)
msg = &Message{}
msg.slabSize = size
msg.bodyBuf = make([]byte, 0, msg.slabSize)
}
msg.Body = msg.bodyBuf
return msg
}
// Close must be called before Recycle
func (this *syncProducerClient) Close() {
log.Trace("cluster[%s] closing kafka sync client: %d", this.cluster, this.id)
// will close the producer and the kafka tcp conn
this.SyncProducer.Close()
this.closed = true
}
func (f *File) ReadAll(ctx context.Context) ([]byte, error) {
f.RLock()
defer f.RUnlock()
log.Trace("File ReadAll, topic=%s, partitionId=%d", f.topic, f.partitionId)
return f.content, nil
}
func (f *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
log.Trace("File Release, req=%#v, topic=%s, partitionId=%d", req,
f.topic, f.partitionId)
f.opened = false
close(f.closeCh)
f.content = make([]byte, 0, 16<<10)
return f.consumer.Close()
}
func (this *subServer) waitExit(exit <-chan struct{}) {
<-exit
if this.httpServer != nil {
// HTTP response will have "Connection: close"
this.httpServer.SetKeepAlivesEnabled(false)
// avoid new connections
if err := this.httpListener.Close(); err != nil {
log.Error(err.Error())
}
log.Trace("%s on %s listener closed", this.name, this.httpServer.Addr)
}
if this.httpsServer != nil {
// HTTP response will have "Connection: close"
this.httpsServer.SetKeepAlivesEnabled(false)
// avoid new connections
if err := this.httpsListener.Close(); err != nil {
log.Error(err.Error())
}
log.Trace("%s on %s listener closed", this.name, this.httpsServer.Addr)
}
this.idleConnsLock.Lock()
t := time.Now().Add(time.Millisecond * 100)
for c := range this.idleConns {
c.SetReadDeadline(t)
}
this.idleConnsLock.Unlock()
if this.idleConnsWg.WaitTimeout(Options.SubTimeout) {
log.Warn("%s waiting for all connected client close timeout: %s",
this.name, Options.SubTimeout)
}
this.subMetrics.Flush()
this.timer.Stop()
this.gw.wg.Done()
close(this.closed)
}
func (this *ClientPool) Warmup() {
t1 := time.Now()
for _, client := range this.clients {
client.Warmup()
}
log.Trace("Memcache pool warmup within %s: %+v",
time.Since(t1), this.FreeConnMap())
}
func (this *StandardServerSelector) KickLookupCache(pool string, hintId int) {
if pool != this.conf.LookupPool || hintId == 0 {
return
}
key := this.lookupCacheKey(pool, hintId)
this.lookupCache.Del(key)
log.Trace("lookupCache[%s] kicked", key)
}
func (this *pubPool) asyncProducerFactory() (pool.Resource, error) {
if len(this.brokerList) == 0 {
return nil, store.ErrEmptyBrokers
}
apc := &asyncProducerClient{
rp: this.asyncPool,
cluster: this.cluster,
id: atomic.AddUint64(&this.nextId, 1),
}
var err error
t1 := time.Now()
cf := sarama.NewConfig()
cf.Net.DialTimeout = time.Second * 4
cf.Net.ReadTimeout = time.Second * 4
cf.Net.WriteTimeout = time.Second * 4
cf.Metadata.RefreshFrequency = time.Minute * 10
cf.Metadata.Retry.Max = 3
cf.Metadata.Retry.Backoff = time.Millisecond * 10
cf.Producer.Flush.Frequency = time.Second * 10 // TODO
cf.Producer.Flush.Messages = 1000
cf.Producer.Flush.MaxMessages = 0 // unlimited
cf.Producer.RequiredAcks = sarama.NoResponse
cf.Producer.Partitioner = NewExclusivePartitioner
cf.Producer.Retry.Backoff = time.Millisecond * 10 // gk migrate will trigger this backoff
cf.Producer.Retry.Max = 3
if this.store.compress {
cf.Producer.Compression = sarama.CompressionSnappy
}
cf.ClientID = this.store.hostname
apc.AsyncProducer, err = sarama.NewAsyncProducer(this.brokerList, cf)
if err != nil {
return nil, err
}
log.Trace("cluster[%s] kafka async producer connected[%d]: %+v %s",
this.cluster, apc.id, this.brokerList, time.Since(t1))
// TODO
go func() {
// messages will only be returned here after all retry attempts are exhausted.
for err := range apc.Errors() {
log.Error("cluster[%s] kafka async producer: %v", this.cluster, err)
}
}()
return apc, err
}
func (f *File) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
f.RLock()
defer f.RUnlock()
log.Trace("File Read, req=%#v, topic=%s, partitionId=%d", req,
f.topic, f.partitionId)
resp.Data = resp.Data[:req.Size]
resp.Data = f.content[req.Offset : req.Size+int(req.Offset)]
return nil
}
func (f *File) Open(ctx context.Context, req *fuse.OpenRequest,
resp *fuse.OpenResponse) (fs.Handle, error) {
log.Trace("File Open, req=%#v, topic=%s, partitionId=%d", req,
f.topic, f.partitionId)
// Allow kernel to use buffer cache
resp.Flags &^= fuse.OpenDirectIO
f.opened = true
f.closeCh = make(chan struct{})
go f.readContent()
time.Sleep(time.Second * 2) // TODO
return f, nil
}
// TODO when to close the kafka client?
func (d *Dir) reconnectKafkaIfNecessary() error {
if d.Client != nil {
return nil
}
kfk, err := sarama.NewClient(d.fs.zkcluster.BrokerList(), sarama.NewConfig())
if err != nil {
log.Error(err)
return err
}
d.Client = kfk
log.Trace("kafka %+v connected", d.fs.zkcluster.BrokerList())
return nil
}
func (this *subManager) Stop() {
this.clientMapLock.Lock()
defer this.clientMapLock.Unlock()
var wg sync.WaitGroup
for _, cg := range this.clientMap {
wg.Add(1)
go func(cg *consumergroup.ConsumerGroup) {
cg.Close() // will commit inflight offsets
wg.Done()
}(cg)
}
wg.Wait()
log.Trace("all consumer offsets committed")
}
func (this *zkreg) keepalive() {
for {
select {
case <-this.shutdownCh:
return
case evt := <-this.zkzone.SessionEvents():
// after zk conn lost, zklib will automatically reconnect
// but the ephemeral znodes have to be recreated by ourselves
if evt.State == zklib.StateHasSession {
this.zkzone.CreateEphemeralZnode(this.mypath(), this.data)
log.Trace("registered in zk: %s", this.mypath())
}
}
}
}
func (s *segment) Remove() (err error) {
if s.wfile == nil {
return ErrSegmentNotOpen
}
path := s.wfile.Name()
log.Trace("segment[%s] removed", path)
if err = s.Close(); err != nil {
return
}
if err = os.Remove(path); err != nil {
return
}
return
}
func (this *TFunServer) handleSession(client thrift.TTransport) {
var (
calls int64 // #calls within this session
errs int64 // #errs within this session
t1 = time.Now()
currentSessionN = atomic.AddInt64(&this.activeSessionN, 1)
tcpClient = client.(*thrift.TSocket).Conn().(*net.TCPConn)
remoteAddr = tcpClient.RemoteAddr().String()
processor = this.processorFactory.GetProcessor(client)
inputTransport = this.inputTransportFactory.GetTransport(client)
outputTransport = this.outputTransportFactory.GetTransport(client)
inputProtocol = this.inputProtocolFactory.GetProtocol(inputTransport)
outputProtocol = this.outputProtocolFactory.GetProtocol(outputTransport)
)
atomic.AddInt64(&this.cumSessions, 1)
log.Debug("session[%s]#%d open", remoteAddr, currentSessionN)
if calls, errs = this.serveCalls(tcpClient, remoteAddr, processor,
inputProtocol, outputProtocol); errs > 0 {
atomic.AddInt64(&this.cumCallErrs, errs)
}
atomic.AddInt64(&this.cumCalls, calls)
// server actively closes the socket
if inputTransport != nil {
inputTransport.Close()
}
if outputTransport != nil {
outputTransport.Close()
}
currentSessionN = atomic.AddInt64(&this.activeSessionN, -1) + 1
elapsed := time.Since(t1)
if errs > 0 {
log.Warn("session[%s]#%d %d calls in %s, errs:%d", remoteAddr,
currentSessionN, calls, elapsed, errs)
} else {
log.Trace("session[%s]#%d %d calls in %s", remoteAddr,
currentSessionN, calls, elapsed)
}
}
func watchFaes() {
ch := make(chan []string, 10)
go etclib.WatchService(etclib.SERVICE_FAE, ch)
for {
select {
case <-ch:
endpoints, err := etclib.ServiceEndpoints(etclib.SERVICE_FAE)
if err == nil {
log.Trace("fae endpoints updated: %+v", endpoints)
dumpFaeConfigPhp(endpoints)
} else {
log.Error("fae: %s", err)
}
}
}
log.Warn("fae watcher died")
}
func (f *File) readContent() {
if err := f.reconsume(sarama.OffsetOldest); err != nil {
log.Error(err)
return
}
var msg *sarama.ConsumerMessage
for {
select {
case <-f.closeCh:
log.Trace("File readContent quit, topic=%s, partitionId=%d",
f.topic, f.partitionId)
return
case msg = <-f.consumer.Messages():
f.content = append(f.content, msg.Value...)
f.content = append(f.content, '\n')
}
}
}
func (this *controller) invokeJobExexutor(jobQueue string, wg *sync.WaitGroup, stopper <-chan struct{}) {
defer func() {
wg.Done()
this.JobExecutorN.Add(-1)
}()
var err error
for retries := 0; retries < 3; retries++ {
log.Trace("claiming owner of %s #%d", jobQueue, retries)
if err = this.orchestrator.ClaimResource(this.Id(), zk.PubsubJobQueueOwners, jobQueue); err == nil {
log.Info("claimed owner of %s", jobQueue)
break
} else if err == zk.ErrClaimedByOthers {
log.Error("%s #%d", err, retries)
time.Sleep(time.Second)
} else {
log.Error("%s #%d", err, retries)
return
}
}
if err != nil {
// still err(ErrClaimedByOthers) encountered after max retries
return
}
defer func(q string) {
this.orchestrator.ReleaseResource(this.Id(), zk.PubsubJobQueueOwners, q)
log.Info("de-claimed owner of %s", q)
}(jobQueue)
cluster, err := this.orchestrator.JobQueueCluster(jobQueue)
if err != nil {
log.Error(err)
}
exe := executor.NewJobExecutor(this.shortId, cluster, jobQueue, this.mc, stopper, this.auditor)
exe.Run()
}
func watchMaintain() {
const PATH = "/maintain"
ch := make(chan []string, 10)
go etclib.WatchChildren(PATH, ch)
for {
select {
case <-ch:
kingdoms, err := etclib.Children(PATH)
if err == nil {
log.Trace("maintain kingdoms updated: %+v", kingdoms)
dumpMaintainConfigPhp(kingdoms)
} else {
log.Error("maintain kingdom: %s", err)
}
}
}
log.Warn("maintain watcher died")
}
func (this *controller) invokeWebhookExecutor(topic string, wg *sync.WaitGroup, stopper <-chan struct{}) {
defer func() {
wg.Done()
this.WebhookExecutorN.Add(-1)
}()
hook, err := this.orchestrator.WebhookInfo(topic)
if err != nil {
log.Error("%s: %s", topic, err)
return
}
for retries := 0; retries < 3; retries++ {
log.Trace("claiming owner of %s #%d", topic, retries)
if err = this.orchestrator.ClaimResource(this.Id(), zk.PubsubWebhookOwners, topic); err == nil {
log.Info("claimed owner of %s", topic)
break
} else if err == zk.ErrClaimedByOthers {
log.Error("%s #%d", err, retries)
time.Sleep(time.Second)
} else {
log.Error("%s #%d", err, retries)
return
}
}
if err != nil {
// still err(ErrClaimedByOthers) encountered after max retries
return
}
defer func(topic string) {
this.orchestrator.ReleaseResource(this.Id(), zk.PubsubWebhookOwners, topic)
log.Info("de-claimed owner of %s", topic)
}(topic)
exe := executor.NewWebhookExecutor(this.shortId, hook.Cluster, topic, hook.Endpoints, stopper, this.auditor)
exe.Run()
}
func (d *Dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
d.RLock()
defer d.RUnlock()
log.Trace("Dir Lookup, name=%s", name)
// split the name into topic and partitionId
partitionOffset := -1
for i := len(name) - 1; i > 0; i-- {
if name[i] == '.' {
partitionOffset = i
}
}
if partitionOffset == -1 {
return nil, fuse.ENOENT
}
topic := name[:partitionOffset]
partitionId, _ := strconv.Atoi(name[partitionOffset+1:])
return d.fs.newFile(d, topic, int32(partitionId), os.FileMode(0555)), nil
}