/
job.go
126 lines (114 loc) · 2.5 KB
/
job.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
package redisq
import (
"fmt"
"strconv"
"github.com/garyburd/redigo/redis"
)
type Job interface {
Do() error
}
type JobQ struct {
Name string
pool *redis.Pool
inputQ *Q
consumers []*Q
doneQ *Q
failQ *Q
newJob func() Job
errCh chan error
}
type JobQError struct {
Msg string
Job Job
Err error
}
func (e *JobQError) Error() string {
return e.Msg
}
func NewJobQ(name string, consumerCount int, newJob func() Job, pool *redis.Pool) (*JobQ, error) {
isConflict, err := heartBeatExists(pool, name)
if err != nil {
return nil, err
}
if isConflict {
return nil, fmt.Errorf("job queue %s already exists", name)
}
errCh := make(chan error)
startHeartBeat(pool, name, 5, errCh)
q := &JobQ{
Name: name,
pool: pool,
inputQ: New(name+":input", pool),
consumers: make([]*Q, consumerCount),
doneQ: New(name+":done", pool),
failQ: New(name+":fail", pool),
newJob: newJob,
errCh: errCh,
}
for i := range q.consumers {
q.consumers[i] = New(name+":consumer:"+strconv.Itoa(i), pool)
}
if err := q.resume(); err != nil {
return nil, err
}
return q, nil
}
func (p *JobQ) resume() error {
c := p.pool.Get()
defer c.Close()
cntKey := p.Name + ":consumer:count"
exists, err := redis.Bool(c.Do("EXISTS", cntKey))
if err != nil {
return err
}
if exists {
cnt, err := redis.Int(c.Do("GET", cntKey))
if err != nil {
return err
}
for i := 0; i < cnt; i++ {
n, err := p.consumers[i].Len()
if err != nil {
return err
}
for j := 0; j < n; j++ {
if err := p.consumers[i].PopTo(p.inputQ, nil); err != nil {
return err
}
}
}
}
_, err = c.Do("SET", cntKey, len(p.consumers))
return err
}
func (p *JobQ) Put(job Job) error {
return p.inputQ.Put(job)
}
func (p *JobQ) Consume() <-chan error {
for _, c := range p.consumers {
go p.process(c)
}
return p.errCh
}
func (p *JobQ) process(consumer *Q) {
for {
job := p.newJob()
if err := p.inputQ.PopTo(consumer, &job); err != nil {
p.sendErr("fail to get job from input queue", job, err)
continue
}
if err := job.Do(); err != nil {
p.sendErr("fail to do the job", job, err)
if err := consumer.PopTo(p.failQ, nil); err != nil {
p.sendErr("fail to put job into the fail queue", job, err)
}
continue
}
if err := consumer.PopTo(p.doneQ, nil); err != nil {
p.sendErr("fail to pu job into the done queue", job, err)
}
}
}
func (p *JobQ) sendErr(msg string, job Job, err error) {
p.errCh <- &JobQError{Msg: msg, Job: job, Err: err}
}