/
main.go
351 lines (295 loc) · 14 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
package main
/**
TODO: Run as a service script in version control
TODO: sh file to install it as service
TODO: Do not panic and die
TODO: Inform Admin worker to send mail for first case of 10 continuous GCM error, +15 min same error count (more than half of expected)
DONE: Peaceful quit, wait for all worker to finish before quitting
DONE: What happens if RabbitMQ restarts ? >> Logic for reconnect
DONE: On continuous 10 GCM error, everyworker should hold for 1 minute before trying again
DONE: Multiple trial before discarding and check before re-queue
TODO: % encode vhost name
DONE: To create Queues or not should be configurable as user might not have permission to create queue
TODO: Write test cases
TODO: Setup travis
DONE: Add timestamp to logs
TODO: handle log separately, Don't process it one by one
DONE: Add worker information to logs
TODO: App error should be kept in proper way
DONE: Do not start this app, if its already running (pgrep blitz) Done using listening to port
TODO: Implement multiple types of GCM Error (Not Registed / Invalid....)
DONE: After every database call, goroutine should wait for few seconds, configurable
TODO: Sucess log for database writes should be configurable
DONE: TransactionMinCount from config not working
DONE: Add support for multiple queues with separate API keys
TODO: BUG: Can not Quit application while application is trying to reconnect to rabbitmq
TODO: Combine olog and logger.Printf into single function
DONE: Implement separate log for GCM and Databaseb
DONE: Prevent infinite requeing
DONE: Requeue count from config
TODO: Inform admin if global application based send failure count increaases specified value
TODO: Logs file permission / update user
DONE: Separate folder for error and success / GCM and APN
-- Create channel for APN error
-- Add APN log file to config
-- Make sure folder exists
**/
import (
"flag"
"fmt"
"github.com/go-gomail/gomail"
"github.com/streadway/amqp"
"github.com/streamrail/concurrent-map"
"log"
"os"
"os/exec"
"os/signal"
"strconv"
"syscall"
"time"
)
const VERSION = "0.7"
/**
* In case of error it displays error and exits
* @param err Error object
* @param msg String message to be displayed
*/
func failOnError(err error, msg string) {
if err != nil {
config := loadConfig(false)
sendErrorMail(msg, err, config)
log.Fatalf("FailOnError %s: %s", msg, err)
panic(fmt.Sprintf("%s: %s", msg, err))
}
}
func submitMail(m *gomail.Message, config Configuration) error {
if config.SendMailPath != "" {
cmd := exec.Command(config.SendMailPath, "-t")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
pw, err := cmd.StdinPipe()
if err != nil {
return err
}
err = cmd.Start()
if err != nil {
return err
}
var errs [3]error
_, errs[0] = m.WriteTo(pw)
errs[1] = pw.Close()
errs[2] = cmd.Wait()
for _, err = range errs {
if err != nil {
return err
}
}
return err
}
return nil
}
/**
* initConn() Recursively tries to create connection with RabbitMQ server.
* @param config Hello
*/
func initConn(config Configuration) *amqp.Connection {
olog("Connecting", config.DebugMode)
conn, err := amqp.Dial("amqp://" + config.Rabbit.Username + ":" + config.Rabbit.Password + "@" + config.Rabbit.Host + ":" + strconv.Itoa(config.Rabbit.Port) + "/" + config.Rabbit.Vhost)
if err != nil {
ticker := time.NewTicker(time.Second * time.Duration(config.Rabbit.ReconnectWaitTimeSec))
for range ticker.C {
olog(fmt.Sprintf("Err: %s, Trying to reconnect", err.Error()), config.DebugMode)
conn, err = amqp.Dial("amqp://" + config.Rabbit.Username + ":" + config.Rabbit.Password + "@" + config.Rabbit.Host + ":" + strconv.Itoa(config.Rabbit.Port) + "/" + config.Rabbit.Vhost)
// TODO: Log error in file
if err == nil {
ticker.Stop()
break
}
}
}
return conn
}
var retries_gcm cmap.ConcurrentMap
var retries_apn cmap.ConcurrentMap
func sendErrorMail(msg string, err error, config Configuration) {
errorMessage := config.EmailFailure.Message + "<br>Message: " + msg + "<br> Error: " + err.Error()
m := gomail.NewMessage()
m.SetHeader("From", config.EmailFailure.From)
m.SetHeader("To", config.EmailFailure.To)
m.SetHeader("Subject", config.EmailFailure.Subject)
m.SetBody("text/html", errorMessage)
err = submitMail(m, config)
if err != nil {
log.Fatalf("sendErrorMail %s: %s", msg, err)
panic(fmt.Sprintf("sendErrorMail %s: %s", msg, err))
}
}
func main() {
// CLI arguments
version := flag.Bool("version", false, "Prints current version and exits")
debugModePtr := flag.Bool("debug", false, "Force debug mode")
flag.Parse()
if *version == true {
fmt.Println("Version: " + VERSION)
os.Exit(0)
}
// Load configuration
config := loadConfig(*debugModePtr)
// Check if basis requirements are fulfilled
checkSystem(config)
// A channel which is used to kill all workers
killWorker := make(chan int)
// Open file for loffing file error
ae, err := os.OpenFile(config.Logging.AppErr.FilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
failOnError(err, "Unable to open App Error Log file")
// Create a logger with opened file to log application errors
logger := log.New(ae, "", log.LstdFlags|log.Lshortfile)
logger.Printf("Starting Service...")
retries_gcm = cmap.New()
retries_apn = cmap.New()
// TODO: Get number of worker buffer from config file
// Create buffered channel for writing GCM log
ch_gcm_log := make(chan []byte, 100) // Create a buffered channel so that processor won't block witing for other to write into error log
ch_gcm_log_success := make(chan []byte, 100) // Create a buffered channel so that processor won't block witing for other to write into error log
ch_apn_log := make(chan []byte, 100) // Create a buffered channel so that processor won't block witing for other to write into error log
ch_apn_log_success := make(chan []byte, 100) // Create a buffered channel so that processor won't block witing for other to write into error log
// Create buffered channel for writing db log
ch_db_log := make(chan []byte, 100) // Create a buffered channel so that processor won't block witing for other to write into error log
// Create goroutine for logging gcm error log. Its a separate goroutine to make it non blocking
go logErrToFile(config.Logging.GcmErr.RootPath, ch_gcm_log, config.DebugMode)
if config.Logging.GcmErr.LogSuccess == true {
go logErrToFile(config.Logging.GcmErr.SuccessPath, ch_gcm_log_success, config.DebugMode)
}
// Create goroutine for logging apn error log. Its a separate goroutine to make it non blocking
go logErrToFile(config.Logging.ApnErr.RootPath, ch_apn_log, config.DebugMode)
if config.Logging.ApnErr.LogSuccess == true {
go logErrToFile(config.Logging.ApnErr.SuccessPath, ch_apn_log_success, config.DebugMode)
}
// Create goroutine for logging db error log. Its a separate goroutine to make it non blocking
go logErrToFile(config.Logging.DbErr.RootPath, ch_db_log, config.DebugMode)
// channel for killing status-inactive goroutines
killStatusInactive := make(chan int)
killApnStatusInactive := make(chan int)
// channel for killing token-update goroutines
killTokenUpd := make(chan int)
// channel for receiving ack that token-update goroutine is killed
killTokenUpdAck := make(chan int)
// channel for receiving ack that status-inactive goroutine is killed
killStatusInactiveAck := make(chan int)
killApnStatusInactiveAck := make(chan int)
// Init Connection with RabbitMQ
conn := initConn(config)
defer conn.Close()
// Create channel for reading messages
ch, err := conn.Channel()
failOnError(err, "Failed to open a channel")
defer ch.Close()
// Create Queues if configured to do so.
createQueues(config, ch)
chQuit := make(chan os.Signal, 2)
signal.Notify(chQuit, os.Interrupt, syscall.SIGTERM)
// Function to handle quit singal
go func(chQuit chan os.Signal, config Configuration, killWorker, killStatusInactive, killTokenUpd, killStatusInactiveAck, killTokenUpdAck, killApnStatusInactive, killApnStatusInactiveAck chan int) {
// Wait for quit event
<-chQuit
olog(fmt.Sprintf("Killing all workers"), config.DebugMode)
// TODO: APN All logic to kill APN workers
killAllWorkers(config, killWorker, killStatusInactive, killTokenUpd, killStatusInactiveAck, killTokenUpdAck, killApnStatusInactive, killApnStatusInactiveAck)
// Exit peacefully
os.Exit(1)
}(chQuit, config, killWorker, killStatusInactive, killTokenUpd, killStatusInactiveAck, killTokenUpdAck, killApnStatusInactive, killApnStatusInactiveAck)
olog(fmt.Sprintf("Spinning up workers"), config.DebugMode)
// For all GcmQueues start new goroutines
for i := 0; i < len(config.GcmQueues); i++ {
// For all GCM Queues start workers
for j := 0; j < config.GcmQueues[i].Numworkers; j++ {
go gcm_processor(j, config, conn, config.GcmQueues[i].GcmTokenUpdateQueue, config.GcmQueues[i].GcmStatusInactiveQueue,
config.GcmQueues[i].Name, ch_gcm_log, ch_gcm_log_success, logger, killWorker, config.GcmQueues[i])
}
olog(fmt.Sprintf("Startting workers for tokenUpdate and status_inactive for GCM %s", config.GcmQueues[i].Identifier), config.DebugMode)
go gcm_error_processor_status_inactive(config, conn, config.GcmQueues[i].GcmStatusInactiveQueue, ch_db_log, logger, killStatusInactive, killStatusInactiveAck, config.GcmQueues[i])
go gcm_error_processor_token_update(config, conn, config.GcmQueues[i].GcmTokenUpdateQueue, ch_db_log, logger, killTokenUpd, killTokenUpdAck, config.GcmQueues[i])
}
//For all APN Queues start workers
for i := 0; i < len(config.ApnQueues); i++ {
// For all GCM Queues start workers
for j := 0; j < config.ApnQueues[i].NumWorkers; j++ {
go apn_processor(j, config, conn, config.ApnQueues[i].ApnStatusInactiveQueue,
config.ApnQueues[i].Name, ch_apn_log, ch_apn_log_success, logger, killWorker, config.ApnQueues[i])
}
olog(fmt.Sprintf("Startting workers for status_inactive for APN %s", config.GcmQueues[i].Identifier), config.DebugMode)
go apn_error_processor_status_inactive(config, conn, config.ApnQueues[i].ApnStatusInactiveQueue, ch_db_log, logger, killApnStatusInactive, killApnStatusInactiveAck, config.ApnQueues[i])
}
// If connection is closed restart
reset := conn.NotifyClose(make(chan *amqp.Error))
for range reset {
go restart(reset, config, conn, ch_gcm_log, ch_db_log, ch_apn_log, ch_gcm_log_success, ch_apn_log_success, logger, killWorker, killStatusInactive, killTokenUpd, killStatusInactiveAck, killTokenUpdAck, killApnStatusInactive, killApnStatusInactiveAck)
}
// Run forever until excited using SIGTERM
forever := make(chan bool)
<-forever
}
/**
* It sends kill signal through kill channel to all go routines
*/
func killAllWorkers(config Configuration, killWorker, killStatusInactive, killTokenUpd, killStatusInactiveAck, killTokenUpdAck, killApnStatusInactive, killApnStatusInactiveAck chan int) {
// Kill All GCM workers
for i := 0; i < len(config.GcmQueues); i++ {
for j := 0; j < config.GcmQueues[i].Numworkers; j++ {
killWorker <- 1
}
// Kill status inactive token processor worker for GCM
killStatusInactive <- NeedAck
// Kill update token processor worker for GCM
killTokenUpd <- NeedAck
}
// Kill All APN workers
for i := 0; i < len(config.ApnQueues); i++ {
for j := 0; j < config.ApnQueues[i].NumWorkers; j++ {
killWorker <- 1
}
// Kill status inactive token processor worker for GCM
killApnStatusInactive <- NeedAck
}
// Wait for database goroutines to end
olog("Waiting for GCM Status Inactive service to end", config.DebugMode)
<-killStatusInactiveAck
olog("Waiting for GCM Token Update service to end", config.DebugMode)
<-killTokenUpdAck
olog("Waiting for APN Status Inactive service to end", config.DebugMode)
<-killApnStatusInactiveAck
}
// Function to restart everything
func restart(reset chan *amqp.Error, config Configuration, conn *amqp.Connection, ch_gcm_log, ch_db_log, ch_apn_log, ch_gcm_log_success, ch_apn_log_success chan []byte, logger *log.Logger,
killWorker, killStatusInactive, killTokenUpd, killStatusInactiveAck, killTokenUpdAck, killApnStatusInactive, killApnStatusInactiveAck chan int) {
// Kill all Worker
killAllWorkers(config, killWorker, killStatusInactive, killTokenUpd, killStatusInactiveAck, killTokenUpdAck, killApnStatusInactive, killApnStatusInactiveAck)
conn.Close()
conn = initConn(config)
defer conn.Close()
olog(fmt.Sprintf("Spinning up workers"), config.DebugMode)
// For all GcmQueues start new goroutines
for i := 0; i < len(config.GcmQueues); i++ {
for j := 0; j < config.GcmQueues[i].Numworkers; j++ {
go gcm_processor(j, config, conn, config.GcmQueues[i].GcmTokenUpdateQueue, config.GcmQueues[i].GcmStatusInactiveQueue,
config.GcmQueues[i].Name, ch_gcm_log, ch_gcm_log_success, logger, killWorker, config.GcmQueues[i])
}
go gcm_error_processor_status_inactive(config, conn, config.GcmQueues[i].GcmStatusInactiveQueue, ch_db_log, logger, killStatusInactive, killStatusInactiveAck, config.GcmQueues[i])
go gcm_error_processor_token_update(config, conn, config.GcmQueues[i].GcmTokenUpdateQueue, ch_db_log, logger, killTokenUpd, killTokenUpdAck, config.GcmQueues[i])
}
//For all APN Queues start workers
for i := 0; i < len(config.ApnQueues); i++ {
// For all GCM Queues start workers
for j := 0; j < config.ApnQueues[i].NumWorkers; j++ {
go apn_processor(j, config, conn, config.ApnQueues[i].ApnStatusInactiveQueue,
config.ApnQueues[i].Name, ch_apn_log, ch_apn_log_success, logger, killWorker, config.ApnQueues[i])
}
olog(fmt.Sprintf("Startting workers for status_inactive for APN %s", config.GcmQueues[i].Identifier), config.DebugMode)
go apn_error_processor_status_inactive(config, conn, config.ApnQueues[i].ApnStatusInactiveQueue, ch_db_log, logger, killApnStatusInactive, killApnStatusInactiveAck, config.ApnQueues[i])
}
olog("Starting error processors", config.DebugMode)
reset = conn.NotifyClose(make(chan *amqp.Error))
for range reset {
go restart(reset, config, conn, ch_gcm_log, ch_db_log, ch_apn_log, ch_gcm_log_success, ch_apn_log_success, logger, killWorker, killStatusInactive, killTokenUpd, killStatusInactiveAck, killTokenUpdAck, killApnStatusInactive, killApnStatusInactiveAck)
}
}