func main() {
logging.InitializeLogging()
// user needs help
var needHelp bool
// configuration file
var configFile string
// set up flag to point at conf, parse arguments and then verify
flag.BoolVar(&needHelp,
"help",
false,
"Display this dialog")
flag.StringVar(&configFile,
"config",
"",
"JSON configuration file")
flag.Parse()
if needHelp {
flag.Usage()
os.Exit(1)
}
// defult configuration
viper.SetDefault("log-level", "INFO")
viper.SetDefault("broker", "localhost")
viper.SetDefault("queue", "mdreceiver")
// load config
if configFile != "" {
viper.SetConfigFile(configFile)
if parseErr := viper.ReadInConfig(); parseErr != nil {
logging.Log.Criticalf("%v", parseErr)
os.Exit(1)
}
logging.Log.Notice("Config file loaded")
}
logging.ConfigureLogging(viper.GetString("log-level"))
logging.Log.Infof("Log level: %v", viper.GetString("log-level"))
broker := viper.GetString("broker")
queueName := viper.GetString("queue")
// check authentication for desired username
if authErr := authentication.Load(); authErr != nil {
logging.Log.Criticalf("Error in loading authenticators: %v", authErr)
os.Exit(1)
}
if !authentication.AmqpAvailable() {
logging.Log.Critical("Authentication for AMQP is not available")
os.Exit(1)
}
amqpUser := authentication.AmqpUsername()
amqpPassword := authentication.AmqpPassword()
url := "amqp://" + amqpUser + ":" + amqpPassword + "@" + broker
service := dripline.StartService(url, queueName)
if service == nil {
logging.Log.Critical("AMQP service did not start")
os.Exit(1)
}
logging.Log.Info("AMQP service started")
// add .# to the queue name for the subscription
subscriptionKey := queueName + ".#"
if subscribeErr := service.SubscribeToRequests(subscriptionKey); subscribeErr != nil {
logging.Log.Criticalf("Could not subscribe to requests at <%v>: %v", subscriptionKey, subscribeErr)
os.Exit(1)
}
if msiErr := fillMasterSenderInfo(); msiErr != nil {
logging.Log.Criticalf("Could not fill out master sender info: %v", MasterSenderInfo)
os.Exit(1)
}
//context := build.defaultContext()
//os.Exit(2)
receiverLoop:
for {
select {
case request, chanOpen := <-service.Receiver.RequestChan:
if !chanOpen {
logging.Log.Error("Incoming request channel is closed")
break receiverLoop
}
logging.Log.Debug("Received request")
switch request.MsgOp {
case dripline.MOCommand:
var instruction string
if request.Message.Target != queueName {
instruction = strings.TrimPrefix(request.Message.Target, queueName+".")
}
logging.Log.Debugf("Command instruction: %s", instruction)
switch instruction {
case "write_json":
logging.Log.Debug("Received \"write_json\" instruction")
//logging.Log.Warningf("type: %v", reflect.TypeOf(request.Message.Payload))
//logging.Log.Warningf("try printing the payload? \n%v", request.Message.Payload)
payloadAsMap, okPAM := request.Message.Payload.(map[interface{}]interface{})
if !okPAM {
if sendErr := PrepareAndSendReply(service, request, dripline.RCErrDripPayload, "Unable to convert payload to map; aborting message", MasterSenderInfo); sendErr != nil {
break receiverLoop
}
continue receiverLoop
}
filenameIfc, hasFN := payloadAsMap["filename"]
if !hasFN {
if sendErr := PrepareAndSendReply(service, request, dripline.RCErrDripPayload, "No filename present in message; aborting", MasterSenderInfo); sendErr != nil {
break receiverLoop
}
continue receiverLoop
}
thePath, okFP := utility.TryConvertToString(filenameIfc)
if okFP != nil {
if sendErr := PrepareAndSendReply(service, request, dripline.RCErrDripPayload, "Unable to convert filename to string; aborting message", MasterSenderInfo); sendErr != nil {
break receiverLoop
}
continue receiverLoop
}
logging.Log.Debugf("Filename to write: %s", thePath)
dir, _ := filepath.Split(thePath)
// check whether the directory exists
_, dirStatErr := os.Stat(dir)
if dirStatErr != nil && os.IsNotExist(dirStatErr) {
if mkdirErr := os.MkdirAll(dir, os.ModeDir|0775); mkdirErr != nil {
msgText := fmt.Sprintf("Unable to create the directory <%q>", dir)
if sendErr := PrepareAndSendReply(service, request, dripline.RCErrHW, msgText, MasterSenderInfo); sendErr != nil {
break receiverLoop
}
continue receiverLoop
}
// Add a small delay after creating the new directory so that anything (e.g. Hornet) waiting for that directory can react to it before the JSON file is created
time.Sleep(100 * time.Millisecond)
}
contentsIfc, hasContents := payloadAsMap["contents"]
if !hasContents {
msgText := fmt.Sprintf("No file contents present in the message for <%q>", thePath)
if sendErr := PrepareAndSendReply(service, request, dripline.RCErrDripPayload, msgText, MasterSenderInfo); sendErr != nil {
break receiverLoop
}
continue receiverLoop
}
encoded, jsonErr := utility.IfcToJSON(&contentsIfc)
if jsonErr != nil {
msgText := fmt.Sprintf("Unable to convert file contents to JSON for <%q>", thePath)
if sendErr := PrepareAndSendReply(service, request, dripline.RCErrDripPayload, msgText, MasterSenderInfo); sendErr != nil {
break receiverLoop
}
continue receiverLoop
}
theFile, fileErr := os.Create(thePath)
if fileErr != nil {
msgText := fmt.Sprintf("Unable to create the file <%q>", thePath)
if sendErr := PrepareAndSendReply(service, request, dripline.RCErrHW, msgText, MasterSenderInfo); sendErr != nil {
break receiverLoop
}
continue receiverLoop
}
_, writeErr := theFile.Write(encoded)
if writeErr != nil {
theFile.Close()
msgText := fmt.Sprintf("Unable to write to the file <%q>", thePath)
if sendErr := PrepareAndSendReply(service, request, dripline.RCErrHW, msgText, MasterSenderInfo); sendErr != nil {
break receiverLoop
}
continue receiverLoop
}
closeErr := theFile.Close()
if closeErr != nil {
msgText := fmt.Sprintf("Unable to close the file <%q>", thePath)
if sendErr := PrepareAndSendReply(service, request, dripline.RCErrHW, msgText, MasterSenderInfo); sendErr != nil {
break receiverLoop
}
continue receiverLoop
}
msgText := fmt.Sprintf("File written: %q", thePath)
if sendErr := PrepareAndSendReply(service, request, dripline.RCSuccess, msgText, MasterSenderInfo); sendErr != nil {
break receiverLoop
}
default:
message := "Incoming request operation instruction not handled: " + instruction
if sendErr := PrepareAndSendReply(service, request, dripline.RCErrDripMethod, message, MasterSenderInfo); sendErr != nil {
break receiverLoop
}
continue receiverLoop
}
default:
message := "Incoming request operation type not handled: " + strconv.FormatUint(uint64(request.MsgOp), 10)
if sendErr := PrepareAndSendReply(service, request, dripline.RCErrDripMethod, message, MasterSenderInfo); sendErr != nil {
break receiverLoop
}
continue receiverLoop
}
}
}
logging.Log.Info("MdReceiver is finished")
}
func main() {
logging.InitializeLogging()
// user needs help
var needHelp bool
// configuration file
var configFile string
// set up flag to point at conf, parse arguments and then verify
flag.BoolVar(&needHelp,
"help",
false,
"Display this dialog")
flag.StringVar(&configFile,
"config",
"",
"JSON configuration file")
flag.Parse()
if needHelp {
flag.Usage()
os.Exit(1)
}
fmt.Println(" ...~.+=:,.I+... ")
fmt.Println(" ... .+$7II?I?IZ7$??+~?I~.. ")
fmt.Println(" .,.:M=.MINI++I7I??????7I$7I=+7~... ")
fmt.Println(" O7+8.IOIII??7+I+7Z?7$I?I?I+??+$??O?. ")
fmt.Println(" ..8MMMM8I??+?+=I?+I?=7++:7???I?==++??$?+?. ")
fmt.Println(" :MMMMM.:~+?I=++$I++???=+??I==~+++?~I=7+=+~?7. ")
fmt.Println(" MMM..++I~?I?$I$$??+?+~$I++=I?7==++I++I+=?+=~=~. ")
fmt.Println(" .MM..~.7?~==~I+?7=II=8~?++?~?=7+$~?$D?=~7?+?+~=I:. ")
fmt.Println(" .MM.?~~$7+=~?=:+?7I=???I7II?7I++DZ?+=+7===+=??=~+?+. ")
fmt.Println(" .MM..NZ?M87+??+7?++?7=I$I7=+=IMMMMD=+$~=?++?:??=~+77I. ")
fmt.Println(" MMNMMMMMMMMMN$Z++7I7+=+++==MMMI~~+I=?+=?++I+=I?=I7$$?: ")
fmt.Println(" .MMMMMMMMMMMMMM8I+++77++?:?MMM::?~=I?+??II?=+?+=?I++I?Z= ")
fmt.Println(" IDMMMMMMMMMMMMMMM$I?I$7==NMM=++~~=7+=I???7I??8I$+I+?I?I$??. ")
fmt.Println(" .MMMMMMMMMMMMMMMMMMOI7?II~MMM+=7=+~7:+$?+II7I?+~I?++I??I??+I.. ")
fmt.Println(" .MMMMMMMMMMMMMMMMMMMD==:+~NMM+=+?:=+?+~=8=++?$7+I?7+=+?II$?+==. ")
fmt.Println(" .MMMMMMMMMMMMDMMMMMMM8I?:MMM+?+~$,=:??7==???Z?7II$7II7??7++=7:I. ")
fmt.Println(" MMMMMMMMMMMM8NMMMMDMMNMMMM??78~:,+=??~:~II?~+II?II7+I+??7I7=ZI=~. ")
fmt.Println(" MMMMMMMMMMMNNNMMMMNMM$?II==?=?+:==++?I++==+II$I=77I??II?II8$??7I~ ")
fmt.Println(" $M7M8MMMMMMMMMMMMMMMM+?~~==+$~~7:$I+7I??~+++=?7I=?$??I?$III?7?I$7Z ")
fmt.Println(" NDMMMMMMMMMMMMMMMMMMM7???$$I?+III787+?7?????==+++?~??I?8ZI77I?I77?. ")
fmt.Println(" NMMMMMMMMMMMMMMMMMMMMD==??=:~77?I=?77?7+I=?I7=I?~+??77~??+IIZ$7$7$7. ")
fmt.Println(" ZMMMMMMMMMMMMMMMMMMMMM8??+=,=I?IZ+++=7?I+?II???+?+?7?=?I$+IZ$O7$ZI7=. ")
fmt.Println(" MMMMMMMMMMMMMMMMMMM8MMMDIZ7+++?~?+MMMII$?~7II=I?I7ZI=7$?II7$77I$7I7Z ")
fmt.Println(" .NMMMMMMMMDMMMMMMMMMOMO=77?=+++?MMMI=$+III??+IOO+?I?I=???ZI?I7I7$ZZZ?. ")
fmt.Println(" DMMMMMMMMMMMMMMNMMMM7?++$~+OMMMO~II?777III?I$Z=?I7I7?+II+I7II7787O. ")
fmt.Println(" .8MMMMMMMMMMMMN7NMMZ+I=~~IOMMOD/?O=7$7+$7?777I+?I+I?I7Z?7?7I7$8OZ. ")
fmt.Println(" . ..$MMNMMMMMMMMMMMMMNM7+7MMMMMNO+++I??IO77II+8$I?I=+?+I7?+?Z$I7I$$7~, ")
fmt.Println(" .NNMMMMMMMMMMMMMMMMMIMMMMMMZM+?7+?+I+++7ZI7$777??I7I7II777$D8I?+II7Z,. ")
fmt.Println(" .MMMMMMMMMMMMMNMMMMI+~,==?~~7~+++7II7=?II$?IZI7+?II?=IO7?I=+?7Z7O??$D. ")
fmt.Println(".MMNMMM MMMMMMMMMNDMMNMMMZ+~==~::?~I$?I~~II78=7?77?I?II=7??I$II?II$$IZ7$$. ")
fmt.Println(" .O NMMMMMMMMMMD..~NMMM.7=,+~~+?7I?=$+I$7II7I7IIII888$Z$I?II7ZI$7II+$~ ")
fmt.Println(" .~NMNMMMMMDZ. ..MMM.+~=+==+Z7777$7Z77I?7Z$II77ZII$O7I+II?7+7I+~I ")
fmt.Println(" .MM$=. .M .:NMM~.=+?II$III7IIIII$Z?+$O$$?I77777III77I?7?++: ")
fmt.Println(" 8NN. MM .ZMMM$ .:????8887+7III7O$$+Z$$$7Z$O77I??+$=????= ")
fmt.Println(" .MM:. MM. .DMM. ..7+~==?+7$88$$OIOID$7I77???+$7===+I ")
fmt.Println(" ,M. +M, .M,. .~:~,~~=7$Z87I7II7Z7I$I=~~=+=?.. ")
fmt.Println(" ,MMZ,. 7. Z~I+:?++==~7Z:7~=7OZ=+++, ")
fmt.Println(" .MMMMM, ..~:... ....... ")
fmt.Println(" .D?. ?, ")
fmt.Println(" ")
// defult configuration
viper.SetDefault("log-level", "INFO")
viper.SetDefault("maximum-age", "1h")
viper.SetDefault("wait-interval", "10m")
// load config
if configFile != "" {
viper.SetConfigFile(configFile)
if parseErr := viper.ReadInConfig(); parseErr != nil {
logging.Log.Criticalf("%v", parseErr)
os.Exit(1)
}
logging.Log.Notice("Config file loaded")
}
logging.ConfigureLogging(viper.GetString("log-level"))
logging.Log.Infof("Log level: %v", viper.GetString("log-level"))
maxAge := viper.GetDuration("maximum-age")
waitInterval := viper.GetDuration("wait-interval")
rootDirs := viper.GetStringSlice("root-dirs")
if len(rootDirs) == 0 {
logging.Log.Critical("No root directories were provided")
os.Exit(1)
}
// Clean up and check the root directories
for rdInd, rootDir := range rootDirs {
rootDirAbs, rdErr := filepath.Abs(filepath.Clean(rootDir))
if rdErr != nil {
logging.Log.Criticalf("Unable to get absolute form of the root directory <%s>", rootDir)
os.Exit(1)
}
// Do a couple checks on the root directory
rootDirInfo, statErr := os.Stat(rootDirAbs)
if statErr != nil {
logging.Log.Criticalf("Unable to \"Stat\" the root directory <%s>", rootDirAbs)
os.Exit(1)
}
if !rootDirInfo.IsDir() {
logging.Log.Criticalf("Root directory <%s> is not a directory", rootDirAbs)
os.Exit(1)
}
rootDirs[rdInd] = rootDirAbs
}
logging.Log.Notice("Watching for stale directories. Use ctrl-c to exit")
//mainLoop:
for {
// Loop over the contents of rootDirs
// We don't apply processDir() directly to the rootDirs because we don't want to delete rootDir if it's empty
for _, rootDir := range rootDirs {
logging.Log.Debugf("Processing directory <%s>", rootDir)
dirContents, readDirErr := ioutil.ReadDir(rootDir)
if readDirErr != nil {
logging.Log.Criticalf("Unable to read directory <%s>", rootDir)
os.Exit(1)
}
exitOnErrors := false
for _, fileInfo := range dirContents {
logging.Log.Debugf("Directory <%s> is not empty; processing contents", rootDir)
if fileInfo.IsDir() {
if procErr := processDir(fileInfo, rootDir, maxAge); procErr != nil {
logging.Log.Errorf("An error occurred while processing directory <%s>: %v", fileInfo.Name(), procErr)
exitOnErrors = true
}
} // else it's a file; ignore it
}
logging.Log.Debugf("Finished processing <%s>", rootDir)
if exitOnErrors == true {
logging.Log.Critical("Exiting due to directory-processing errors")
break
}
}
// Wait the specified amount of time before running again
time.Sleep(waitInterval)
}
logging.Log.Notice("DungBeetle says: \"My job here is done\"")
}
func main() {
logging.InitializeLogging()
// user needs help
var needHelp bool
// configuration file
var configFile string
// set up flag to point at conf, parse arguments and then verify
flag.BoolVar(&needHelp,
"help",
false,
"display this dialog")
flag.StringVar(&configFile,
"config",
"",
"JSON configuration file")
flag.Parse()
if needHelp {
flag.Usage()
os.Exit(1)
}
// defult configuration
viper.SetDefault("username", "project8")
viper.SetDefault("log-level", "INFO")
// load config
viper.SetConfigFile(configFile)
if parseErr := viper.ReadInConfig(); parseErr != nil {
logging.Log.Criticalf("%v", parseErr)
os.Exit(1)
}
logging.Log.Notice("Config file loaded")
logging.ConfigureLogging(viper.GetString("log-level"))
logging.Log.Infof("Log level: %v", viper.GetString("log-level"))
userName := viper.GetString("username")
if !viper.IsSet("channels") {
logging.Log.Critical("No channel configuration found")
os.Exit(1)
}
// check authentication for desired username
if authErr := authentication.Load(); authErr != nil {
logging.Log.Criticalf("Error in loading authenticators: %v", authErr)
os.Exit(1)
}
if !authentication.SlackAvailable(userName) {
logging.Log.Criticalf("Authentication for user <%s> is not available", userName)
os.Exit(1)
}
authToken := authentication.SlackToken(userName)
logging.Log.Infof("Slack username: %s", userName)
logging.Log.Infof("Slack token: %s", authToken)
// get the slack API object
api := slack.New(authToken)
if api == nil {
logging.Log.Critical("Unable to make a new Slack API")
os.Exit(1)
}
logging.Log.Info("Created Slack API")
// get list of users and then the user ID
userID := ""
users, usersErr := api.GetUsers()
if usersErr != nil {
logging.Log.Criticalf("Unable to get users: %s", usersErr)
os.Exit(1)
} else {
usernameLoop:
for _, user := range users {
if user.Name == userName {
userID = user.ID
break usernameLoop
}
}
}
if userID == "" {
logging.Log.Criticalf("Could not get user ID for user <%s>", userName)
os.Exit(1)
}
logging.Log.Infof("User ID: %s", userID)
// get map of channel IDs
var allChannelMap map[string]string
channels, chanErr := api.GetChannels(true)
if chanErr != nil {
logging.Log.Criticalf("Unable to get channels: %s", chanErr)
os.Exit(1)
} else {
allChannelMap = make(map[string]string, len(channels))
for _, aChan := range channels {
allChannelMap[aChan.Name] = aChan.ID
logging.Log.Debugf("Channel %s has ID %s", aChan.Name, aChan.ID)
}
}
var threadWait sync.WaitGroup
recipientChan := make(chan eventRecipient, 10)
// loop over channels
channelsRaw := viper.GetStringMap("channels")
for channelName, _ := range channelsRaw {
// get channel ID
if channelID, channelExists := allChannelMap[channelName]; channelExists == true {
logging.Log.Infof("(%s) Found request for channel %s", channelID, channelName)
//channelInfo := channelInfoRaw.(map[string](interface{}))
channelConfigName := "channels." + channelName
sizeLimitCN := channelConfigName + ".size-limit"
sizeLimit := -1
monitorSize := false
// size will not be monitored if size-limit is not set in the configuration
if viper.IsSet(sizeLimitCN) {
sizeLimit = viper.GetInt(sizeLimitCN)
monitorSize = viper.GetBool(channelConfigName + ".monitor-size")
}
if sizeLimit < 0 {
logging.Log.Errorf("(%s) Invalid size limit", channelID)
continue
}
doLogging := false // future feature
msgQueue := utility.NewQueue()
var buildHistLock sync.Mutex
histCond := sync.NewCond(&buildHistLock)
if monitorSize || doLogging {
if !monitorStarted {
logging.Log.Notice("Launching monitorSlack")
threadWait.Add(1)
go monitorSlack(api, &threadWait, recipientChan)
monitorStarted = true
}
recipient := eventRecipient{
channelID: channelID,
eventChan: make(chan *slack.MessageEvent, 100),
}
recipientChan <- recipient
logging.Log.Noticef("(%s) Launching monitorChannel", channelID)
threadWait.Add(1)
go monitorChannel(channelID, api, recipient.eventChan, msgQueue, monitorSize, doLogging, histCond, &threadWait)
// If we're monitoring the channel, then we increase the size limit for cleanHistory by 1.
// This is because the latest message will be passed to the monitor as the first message received.
// So the monitor will then remove an old message; so we leave one extra old message to be removed
// once the monitoring begins.
sizeLimit++
}
logging.Log.Noticef("(%s) Launching cleanHistory", channelID)
threadWait.Add(1)
go cleanHistory(channelID, api, msgQueue, sizeLimit, histCond, &threadWait)
} else {
logging.Log.Warningf("Channel <%s> does not exist", channelName)
}
}
logging.Log.Notice("Waiting for threads to finish")
threadWait.Wait()
logging.Log.Notice("Threads complete")
return
}
func main() {
logging.InitializeLogging()
// user needs help
var needHelp bool
// configuration file
var configFile string
// set up flag to point at conf, parse arguments and then verify
flag.BoolVar(&needHelp,
"help",
false,
"Display this dialog")
flag.StringVar(&configFile,
"config",
"",
"JSON configuration file")
flag.Parse()
if needHelp {
flag.Usage()
os.Exit(1)
}
// defult configuration
viper.SetDefault("log-level", "INFO")
viper.SetDefault("broker", "localhost")
viper.SetDefault("wait-interval", "1m")
viper.SetDefault("subscribe-queue", "diopsid-queue")
viper.SetDefault("alerts-queue", "disk_status.machinename")
// load config
if configFile != "" {
viper.SetConfigFile(configFile)
if parseErr := viper.ReadInConfig(); parseErr != nil {
logging.Log.Criticalf("%v", parseErr)
os.Exit(1)
}
logging.Log.Notice("Config file loaded")
}
logging.ConfigureLogging(viper.GetString("log-level"))
logging.Log.Infof("Log level: %v", viper.GetString("log-level"))
wheretolook := viper.GetStringSlice("where-to-look")
if len(wheretolook) == 0 {
logging.Log.Critical("No directories were provided")
os.Exit(1)
}
// computername := viper.GetString("computer-name")
// computername,e := os.Hostname()
// if e != nil {
// logging.Log.Criticalf("Couldn't get the hostname")
// return
// }
broker := viper.GetString("broker")
queueName := viper.GetString("subscribe-queue")
alertsQueueName := viper.GetString("alerts-queue")
waitInterval := viper.GetDuration("wait-interval")
// check authentication for desired username
if authErr := authentication.Load(); authErr != nil {
logging.Log.Criticalf("Error in loading authenticators: %v", authErr)
os.Exit(1)
}
if !authentication.AmqpAvailable() {
logging.Log.Critical("Authentication for AMQP is not available")
os.Exit(1)
}
amqpUser := authentication.AmqpUsername()
amqpPassword := authentication.AmqpPassword()
url := "amqp://" + amqpUser + ":" + amqpPassword + "@" + broker
service := dripline.StartService(url, queueName)
if service == nil {
logging.Log.Critical("AMQP service did not start")
os.Exit(1)
}
logging.Log.Info("AMQP service started")
// add .# to the queue name for the subscription
// the queue name does not have to be the same as the queue where to send the alerts!
// it is better to define a proper queueName in the config file to prevent
// conflict between services subscribing to the same queue (which is not allowed!)
subscriptionKey := queueName + ".#"
if subscribeErr := service.SubscribeToAlerts(subscriptionKey); subscribeErr != nil {
logging.Log.Criticalf("Could not subscribe to alerts at <%v>: %v", subscriptionKey, subscribeErr)
os.Exit(1)
}
if msiErr := fillMasterSenderInfo(); msiErr != nil {
logging.Log.Criticalf("Could not fill out master sender info: %v", MasterSenderInfo)
os.Exit(1)
}
for {
for _, dir := range wheretolook {
alert := dripline.PrepareAlert(alertsQueueName, "application/json", MasterSenderInfo)
disk := DiskUsage(dir)
var payload map[string]interface{}
payload = make(map[string]interface{})
payload["directory"] = dir
payload["all"] = float64(disk.All) / float64(GB)
payload["used"] = float64(disk.Used) / float64(GB)
alert.Message.Payload = payload
e := service.SendAlert(alert)
logging.Log.Infof("Alert sent: [%s] All: %.2f GB Used: %.2f GB", dir, float64(disk.All)/float64(GB), float64(disk.Used)/float64(GB))
if e != nil {
logging.Log.Errorf("Could not send the alert: %v", e)
}
}
logging.Log.Infof("Sleeping now")
time.Sleep(waitInterval)
}
}