Example #1
0
// Run puts jex-events in 'events' mode where it listens for events
// on an AMQP exchange, places them into a database, and provides an HTTP
// API on top.
func Run(config *configurate.Configuration, l *log.Logger) {
	logger = l
	logger.Println("Configuring database connection...")
	messaging.Init(logger)
	databaser, err := NewDatabaser(config.DBURI)
	if err != nil {
		logger.Print(err)
		os.Exit(-1)
	}
	logger.Println("Done configuring database connection.")

	connErrChan := make(chan messaging.ConnectionError)
	consumer := messaging.NewAMQPConsumer(config)
	messaging.SetupReconnection(connErrChan, reconnect)
	logger.Print("Setting up HTTP")
	SetupHTTP(config, databaser)
	logger.Print("Done setting up HTTP")

	// This is the retry logic that the events mode goes through at start up. It's
	// there just in case the AMQP broker isn't up when events mode starts.
	randomizer := rand.New(rand.NewSource(time.Now().UnixNano()))
	var deliveries <-chan amqp.Delivery
	for {
		logger.Println("Attempting AMQP connection...")
		deliveries, err = consumer.Connect(connErrChan)
		if err != nil {
			logger.Print(err)
			waitFor := randomizer.Intn(10)
			logger.Printf("Re-attempting connection in %d seconds", waitFor)
			time.Sleep(time.Duration(waitFor) * time.Second)
		} else {
			logger.Println("Successfully connected to the AMQP broker")
			break
		}
	}

	// The actual logic for events mode occurs here.
	EventHandler(deliveries, databaser, config.EventURL, config.JEXURL)
}
Example #2
0
File: clm.go Project: strootman/DE
// Run takes in a configuration and a logger and runs jex-events in 'monitor'
// mode. It watches the configured event_log path for changes and ships to
// another service via AMQP.
func Run(cfg *configurate.Configuration, l *log.Logger) {
	logger = l
	messaging.Init(logger)
	randomizer := rand.New(rand.NewSource(time.Now().UnixNano()))
	errChan := make(chan messaging.ConnectionError)
	pub := messaging.NewAMQPPublisher(cfg)
	messaging.SetupReconnection(errChan, reconnect)

	// Handle badness with AMQP at startup.
	var err error
	for {
		logger.Println("Attempting AMQP connection...")
		err = pub.Connect(errChan)
		if err != nil {
			logger.Println(err)
			waitFor := randomizer.Intn(10)
			logger.Printf("Re-attempting connection in %d seconds", waitFor)
			time.Sleep(time.Duration(waitFor) * time.Second)
		} else {
			logger.Println("Successfully connected to the AMQP broker.")
			break
		}
	}

	// First, we need to read the tombstone file if it exists.
	var tombstone *Tombstone
	if TombstoneExists() {
		logger.Printf("Attempting to read tombstone from %s\n", TombstonePath)
		tombstone, err = ReadTombstone()
		if err != nil {
			logger.Println("Couldn't read Tombstone file.")
			logger.Println(err)
			tombstone = nil
		}
		logger.Printf("Done reading tombstone file from %s\n", TombstonePath)
	} else {
		tombstone = nil
	}

	logDir := filepath.Dir(cfg.EventLog)
	logger.Printf("Log directory: %s\n", logDir)
	logFilename := filepath.Base(cfg.EventLog)
	logger.Printf("Log filename: %s\n", logFilename)

	// Now we need to find all of the rotated out log files and parse them for
	// potentially missed updates.
	logList, err := NewLogfileList(logDir, logFilename)
	if err != nil {
		logger.Println("Couldn't get list of log files.")
		logList = LogfileList{}
	}

	// We need to sort the rotated log files in order from oldest to newest.
	sort.Sort(logList)

	// If there aren't any rotated log files or a tombstone file, then there
	// isn't a reason to truncate the list of rotated log files. Hopefully, we'd
	// trim the list of log files to prevent reprocessing, which could save us
	// a significant amount of time at start up.
	if len(logList) > 0 && tombstone != nil {
		logger.Printf("Slicing log list by inode number %d\n", tombstone.Inode)
		logList = logList.SliceByInode(tombstone.Inode)
	}

	// Iterate through the list of log files, parse them, and ultimately send the
	// events out to the AMQP broker. Skip the latest log file, we'll be handling
	// that further down.
	for _, logFile := range logList {
		if logFile.Info.Name() == logFilename { //the current log file will get parsed later
			continue
		}
		logfilePath := path.Join(logFile.BaseDir, logFile.Info.Name())
		logger.Printf("Parsing %s\n", logfilePath)

		if tombstone != nil {
			logfileInode := InodeFromFileInfo(&logFile.Info)

			// Inodes need to match and the current position needs to be less than the file size.
			if logfileInode == tombstone.Inode && tombstone.CurrentPos < logFile.Info.Size() {
				logger.Printf("Tombstoned inode matches %s, starting parse at %d\n", logfilePath, tombstone.CurrentPos)
				_, err = ParseEventFile(logfilePath, tombstone.CurrentPos, pub, false)
			} else {
				logger.Printf("Tombstoned inode does not match %s, starting parse at position 0\n", logfilePath)
				_, err = ParseEventFile(logfilePath, 0, pub, false)
			}
		} else {
			logger.Printf("No tombstone found, starting parse at position 0 for %s\n", logfilePath)
			_, err = ParseEventFile(logfilePath, 0, pub, false)
		}
		if err != nil {
			logger.Println(err)
		}
	}

	changeDetected := make(chan int)
	var startPos int64

	d, err := time.ParseDuration("0.5s")
	if err != nil {
		logger.Println(err)
	}
	go func() {
		logger.Println("Beginning event log monitor goroutine.")
		// get the ball rolling...
		changeDetected <- 1
		err = Path(cfg.EventLog, d, changeDetected)
		if err != nil {
			logger.Println(err)
		}
	}()

	for {
		select {
		case <-changeDetected:
			//Get the tombstone if it exists.
			if TombstoneExists() {
				tombstone, err = ReadTombstone()
				if err != nil {
					logger.Println(err)
				}
				startPos = tombstone.CurrentPos

				// Get the path to the file that the Tombstone was indicating
				oldLogs, err := NewLogfileList(logDir, logFilename)
				if err != nil {
					logger.Println(err)
				}

				// If the path to the file is different from the configured file the the
				// log likely rolled over.
				pathFromTombstone := oldLogs.PathFromInode(tombstone.Inode)
				if pathFromTombstone != "" && pathFromTombstone != cfg.EventLog {
					oldInfo, err := os.Stat(pathFromTombstone)
					if err != nil {
						logger.Println(err)
					}
					// Compare the start position to the size of the
					// file. If it's less than the size of the file, more of the old file
					// needs to be parsed.
					if startPos < oldInfo.Size() {
						_, err = ParseEventFile(pathFromTombstone, startPos, pub, true)
						if err != nil {
							logger.Println(err)
						}
						// Afterwards set the startPos to 0 if it isn't
						// already, but ONLY if an old file was parsed first.
						startPos = 0
					}
				}
			} else {
				// The Tombstone didn't exist, so start from the beginning of the file.
				startPos = 0
			}

			logger.Printf("Parsing %s starting at position %d\n", cfg.EventLog, startPos)
			startPos, err = ParseEventFile(cfg.EventLog, startPos, pub, true)
			if err != nil {
				logger.Println(err)
			}
		}
	}
}