// Run puts jex-events in 'events' mode where it listens for events
// on an AMQP exchange, places them into a database, and provides an HTTP
// API on top.
func Run(config *configurate.Configuration, l *log.Logger) {
logger = l
logger.Println("Configuring database connection...")
messaging.Init(logger)
databaser, err := NewDatabaser(config.DBURI)
if err != nil {
logger.Print(err)
os.Exit(-1)
}
logger.Println("Done configuring database connection.")
connErrChan := make(chan messaging.ConnectionError)
consumer := messaging.NewAMQPConsumer(config)
messaging.SetupReconnection(connErrChan, reconnect)
logger.Print("Setting up HTTP")
SetupHTTP(config, databaser)
logger.Print("Done setting up HTTP")
// This is the retry logic that the events mode goes through at start up. It's
// there just in case the AMQP broker isn't up when events mode starts.
randomizer := rand.New(rand.NewSource(time.Now().UnixNano()))
var deliveries <-chan amqp.Delivery
for {
logger.Println("Attempting AMQP connection...")
deliveries, err = consumer.Connect(connErrChan)
if err != nil {
logger.Print(err)
waitFor := randomizer.Intn(10)
logger.Printf("Re-attempting connection in %d seconds", waitFor)
time.Sleep(time.Duration(waitFor) * time.Second)
} else {
logger.Println("Successfully connected to the AMQP broker")
break
}
}
// The actual logic for events mode occurs here.
EventHandler(deliveries, databaser, config.EventURL, config.JEXURL)
}
// Run takes in a configuration and a logger and runs jex-events in 'monitor'
// mode. It watches the configured event_log path for changes and ships to
// another service via AMQP.
func Run(cfg *configurate.Configuration, l *log.Logger) {
logger = l
messaging.Init(logger)
randomizer := rand.New(rand.NewSource(time.Now().UnixNano()))
errChan := make(chan messaging.ConnectionError)
pub := messaging.NewAMQPPublisher(cfg)
messaging.SetupReconnection(errChan, reconnect)
// Handle badness with AMQP at startup.
var err error
for {
logger.Println("Attempting AMQP connection...")
err = pub.Connect(errChan)
if err != nil {
logger.Println(err)
waitFor := randomizer.Intn(10)
logger.Printf("Re-attempting connection in %d seconds", waitFor)
time.Sleep(time.Duration(waitFor) * time.Second)
} else {
logger.Println("Successfully connected to the AMQP broker.")
break
}
}
// First, we need to read the tombstone file if it exists.
var tombstone *Tombstone
if TombstoneExists() {
logger.Printf("Attempting to read tombstone from %s\n", TombstonePath)
tombstone, err = ReadTombstone()
if err != nil {
logger.Println("Couldn't read Tombstone file.")
logger.Println(err)
tombstone = nil
}
logger.Printf("Done reading tombstone file from %s\n", TombstonePath)
} else {
tombstone = nil
}
logDir := filepath.Dir(cfg.EventLog)
logger.Printf("Log directory: %s\n", logDir)
logFilename := filepath.Base(cfg.EventLog)
logger.Printf("Log filename: %s\n", logFilename)
// Now we need to find all of the rotated out log files and parse them for
// potentially missed updates.
logList, err := NewLogfileList(logDir, logFilename)
if err != nil {
logger.Println("Couldn't get list of log files.")
logList = LogfileList{}
}
// We need to sort the rotated log files in order from oldest to newest.
sort.Sort(logList)
// If there aren't any rotated log files or a tombstone file, then there
// isn't a reason to truncate the list of rotated log files. Hopefully, we'd
// trim the list of log files to prevent reprocessing, which could save us
// a significant amount of time at start up.
if len(logList) > 0 && tombstone != nil {
logger.Printf("Slicing log list by inode number %d\n", tombstone.Inode)
logList = logList.SliceByInode(tombstone.Inode)
}
// Iterate through the list of log files, parse them, and ultimately send the
// events out to the AMQP broker. Skip the latest log file, we'll be handling
// that further down.
for _, logFile := range logList {
if logFile.Info.Name() == logFilename { //the current log file will get parsed later
continue
}
logfilePath := path.Join(logFile.BaseDir, logFile.Info.Name())
logger.Printf("Parsing %s\n", logfilePath)
if tombstone != nil {
logfileInode := InodeFromFileInfo(&logFile.Info)
// Inodes need to match and the current position needs to be less than the file size.
if logfileInode == tombstone.Inode && tombstone.CurrentPos < logFile.Info.Size() {
logger.Printf("Tombstoned inode matches %s, starting parse at %d\n", logfilePath, tombstone.CurrentPos)
_, err = ParseEventFile(logfilePath, tombstone.CurrentPos, pub, false)
} else {
logger.Printf("Tombstoned inode does not match %s, starting parse at position 0\n", logfilePath)
_, err = ParseEventFile(logfilePath, 0, pub, false)
}
} else {
logger.Printf("No tombstone found, starting parse at position 0 for %s\n", logfilePath)
_, err = ParseEventFile(logfilePath, 0, pub, false)
}
if err != nil {
logger.Println(err)
}
}
changeDetected := make(chan int)
var startPos int64
d, err := time.ParseDuration("0.5s")
if err != nil {
logger.Println(err)
}
go func() {
logger.Println("Beginning event log monitor goroutine.")
// get the ball rolling...
changeDetected <- 1
err = Path(cfg.EventLog, d, changeDetected)
if err != nil {
logger.Println(err)
}
}()
for {
select {
case <-changeDetected:
//Get the tombstone if it exists.
if TombstoneExists() {
tombstone, err = ReadTombstone()
if err != nil {
logger.Println(err)
}
startPos = tombstone.CurrentPos
// Get the path to the file that the Tombstone was indicating
oldLogs, err := NewLogfileList(logDir, logFilename)
if err != nil {
logger.Println(err)
}
// If the path to the file is different from the configured file the the
// log likely rolled over.
pathFromTombstone := oldLogs.PathFromInode(tombstone.Inode)
if pathFromTombstone != "" && pathFromTombstone != cfg.EventLog {
oldInfo, err := os.Stat(pathFromTombstone)
if err != nil {
logger.Println(err)
}
// Compare the start position to the size of the
// file. If it's less than the size of the file, more of the old file
// needs to be parsed.
if startPos < oldInfo.Size() {
_, err = ParseEventFile(pathFromTombstone, startPos, pub, true)
if err != nil {
logger.Println(err)
}
// Afterwards set the startPos to 0 if it isn't
// already, but ONLY if an old file was parsed first.
startPos = 0
}
}
} else {
// The Tombstone didn't exist, so start from the beginning of the file.
startPos = 0
}
logger.Printf("Parsing %s starting at position %d\n", cfg.EventLog, startPos)
startPos, err = ParseEventFile(cfg.EventLog, startPos, pub, true)
if err != nil {
logger.Println(err)
}
}
}
}
