forked from hanguofeng/taiji
/
no_commit_arbiter.go
95 lines (74 loc) · 1.98 KB
/
no_commit_arbiter.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
package main
import (
"sync/atomic"
"time"
"github.com/Shopify/sarama"
"github.com/golang/glog"
)
type NoCommitArbiter struct {
*StartStopControl
// input/output
offsets chan int64
messages chan *sarama.ConsumerMessage
// parent
manager *PartitionManager
// config
config *CallbackItemConfig
arbiterConfig ArbiterConfig
// stat variables
processed uint64
startTime time.Time
}
func NewNoCommitArbiter() Arbiter {
return &NoCommitArbiter{
StartStopControl: NewStartStopControl(),
}
}
func (*NoCommitArbiter) PreferredTransporterWorkerNum(workerNum int) int {
return workerNum
}
func (nca *NoCommitArbiter) OffsetChannel() chan<- int64 {
return nca.offsets
}
func (nca *NoCommitArbiter) MessageChannel() <-chan *sarama.ConsumerMessage {
return nca.manager.GetKafkaPartitionConsumer().Messages()
}
func (nca *NoCommitArbiter) Init(config *CallbackItemConfig, arbiterConfig ArbiterConfig, manager *PartitionManager) error {
nca.manager = manager
nca.config = config
nca.arbiterConfig = arbiterConfig
return nil
}
func (nca *NoCommitArbiter) Run() error {
if err := nca.ensureStart(); err != nil {
return err
}
defer nca.markStop()
nca.offsets = make(chan int64, 256)
// reset stat variables
atomic.StoreUint64(&nca.processed, 0)
nca.startTime = time.Now().Local()
nca.markReady()
arbiterLoop:
for {
select {
case <-nca.WaitForCloseChannel():
glog.V(1).Infof("Stop event triggered [url:%s]", nca.config.Url)
break arbiterLoop
case offset := <-nca.offsets:
glog.V(1).Infof("Read offset from Transporter [topic:%s][partition:%d][url:%s][offset:%d]",
nca.manager.Topic, nca.manager.Partition, nca.config.Url, offset)
atomic.AddUint64(&nca.processed, 1)
}
}
return nil
}
func (nca *NoCommitArbiter) GetStat() interface{} {
result := make(map[string]interface{})
result["processed"] = atomic.LoadUint64(&nca.processed)
result["start_time"] = nca.startTime
return result
}
func init() {
RegisterArbiter("NoCommit", NewNoCommitArbiter)
}