// loadCommandsReady walks through the commands ready directory and load the
// commands that are passed their scheduled date. It also delete expired commands.
func loadCommandsReady(ctx Context) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("loadCommandsReady() -> %v", e)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving loadCommandsReady()"}.Debug()
}()
dir, err := os.Open(ctx.Directories.Command.Ready)
dirContent, err := dir.Readdir(-1)
if err != nil {
panic(err)
}
// loop over the content of the directory
for _, DirEntry := range dirContent {
if !DirEntry.Mode().IsRegular() {
// ignore non file
continue
}
filename := ctx.Directories.Command.Ready + "/" + DirEntry.Name()
cmd, err := mig.CmdFromFile(filename)
if err != nil {
panic(err)
}
if time.Now().After(cmd.Action.ExpireAfter) {
// delete expired
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, CommandID: cmd.ID, ActionID: cmd.Action.ID, Desc: fmt.Sprintf("removing expired command '%s'", cmd.Action.Name)}
os.Remove(filename)
} else if time.Now().After(cmd.Action.ValidFrom) {
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, CommandID: cmd.ID, ActionID: cmd.Action.ID, Desc: fmt.Sprintf("launching command '%s'", cmd.Action.Name)}
ctx.Channels.CommandReady <- filename
}
}
dir.Close()
return
}
// loadReturnedCommands walks through the returned commands directory and loads
// the commands into the scheduler
func loadReturnedCommands(ctx Context) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("loadReturnedCommands() -> %v", e)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving loadReturnedCommands()"}.Debug()
}()
dir, err := os.Open(ctx.Directories.Command.Returned)
dirContent, err := dir.Readdir(-1)
if err != nil {
panic(err)
}
// loop over the content of the directory
for _, DirEntry := range dirContent {
if !DirEntry.Mode().IsRegular() {
// ignore non file
continue
}
filename := ctx.Directories.Command.Returned + "/" + DirEntry.Name()
cmd, err := mig.CmdFromFile(filename)
if err != nil {
panic(err)
}
// queue it
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, CommandID: cmd.ID, ActionID: cmd.Action.ID, Desc: fmt.Sprintf("loading returned command '%s'", cmd.Action.Name)}
ctx.Channels.CommandReturned <- filename
}
dir.Close()
return
}
// expireCommands loads commands in the inflight directory
// and terminate the expired ones
func expireCommands(ctx Context) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("expireCommands() -> %v", e)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving expireCommands()"}.Debug()
}()
dir, err := os.Open(ctx.Directories.Command.InFlight)
dirContent, err := dir.Readdir(-1)
if err != nil {
panic(err)
}
// loop over the content of the directory
for _, DirEntry := range dirContent {
if !DirEntry.Mode().IsRegular() {
// ignore non file
continue
}
filename := ctx.Directories.Command.InFlight + "/" + DirEntry.Name()
_, err = os.Stat(filename)
if err != nil {
// file is already gone, probably consumed by a concurrent returning command
// ignore and continue
continue
}
err := waitForFileOrDelete(filename, 3)
if err != nil {
ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("error while reading '%s': %v", filename, err)}.Err()
continue
}
cmd, err := mig.CmdFromFile(filename)
if err != nil {
// failing to load this file, log and skip it
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: fmt.Sprintf("failed to load inflight command file %s", filename)}.Err()
continue
}
if time.Now().After(cmd.Action.ExpireAfter) {
desc := fmt.Sprintf("expiring command '%s' on agent '%s'", cmd.Action.Name, cmd.Agent.Name)
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, CommandID: cmd.ID, ActionID: cmd.Action.ID, Desc: desc}
cmd.Status = "expired"
cmd.FinishTime = time.Now().UTC()
// write it into the returned command directory
data, err := json.Marshal(cmd)
if err != nil {
panic(err)
}
dest := fmt.Sprintf("%s/%.0f-%.0f.json", ctx.Directories.Command.Returned, cmd.Action.ID, cmd.ID)
err = safeWrite(ctx, dest, data)
if err != nil {
panic(err)
}
//ctx.Directories.Command.Returned
os.Remove(filename)
}
}
dir.Close()
return
}
// evaluateInFlightCommand loads commands in the inflight directory
// and terminate the expired ones
func evaluateInFlightCommands(ctx Context) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("evaluateInFlightCommands() -> %v", e)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving evaluateInFlightCommands()"}.Debug()
}()
dir, err := os.Open(ctx.Directories.Command.InFlight)
dirContent, err := dir.Readdir(-1)
if err != nil {
panic(err)
}
// loop over the content of the directory
for _, DirEntry := range dirContent {
if !DirEntry.Mode().IsRegular() {
// ignore non file
continue
}
filename := ctx.Directories.Command.InFlight + "/" + DirEntry.Name()
cmd, err := mig.CmdFromFile(filename)
if err != nil {
panic(err)
}
if time.Now().After(cmd.Action.ExpireAfter) {
desc := fmt.Sprintf("expiring action '%s' on agent '%s'", cmd.Action.Name, cmd.Agent.Name)
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, CommandID: cmd.ID, ActionID: cmd.Action.ID, Desc: desc}
// expired command must be terminated
cmd.Status = "cancelled"
cmd.FinishTime = time.Now().UTC()
// write it into the returned command directory
data, err := json.Marshal(cmd)
if err != nil {
panic(err)
}
dest := fmt.Sprintf("%s/%.0f-%.0f.json", ctx.Directories.Command.Returned, cmd.Action.ID, cmd.ID)
err = safeWrite(ctx, dest, data)
if err != nil {
panic(err)
}
//ctx.Directories.Command.Returned
os.Remove(filename)
}
}
dir.Close()
return
}
// terminateCommand is called when a command result is dropped into ctx.Directories.Command.Returned
// it stores the result of a command and mark it as completed/failed and then
// send a message to the Action completion routine to update the action status
func terminateCommand(cmdPath string, ctx Context) (err error) {
var cmd mig.Command
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("terminateCommand() -> %v", e)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: cmd.Action.ID, CommandID: cmd.ID, Desc: "leaving terminateCommand()"}.Debug()
}()
// load and parse the command. If this fail, skip it and continue.
cmd, err = mig.CmdFromFile(cmdPath)
if err != nil {
panic(err)
}
cmd.FinishTime = time.Now().UTC()
// update command in database
err = ctx.DB.Col.Cmd.Update(bson.M{"id": cmd.ID}, cmd)
if err != nil {
panic(err)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: cmd.Action.ID, CommandID: cmd.ID, Desc: "command updated in database"}.Debug()
// write to disk to Command Done directory
data, err := json.Marshal(cmd)
if err != nil {
panic(err)
}
dest := fmt.Sprintf("%s/%d-%d.json", ctx.Directories.Command.Done, cmd.Action.ID, cmd.ID)
err = safeWrite(ctx, dest, data)
if err != nil {
panic(err)
}
// remove command from inflight dir
inflightPath := fmt.Sprintf("%s/%d-%d.json", ctx.Directories.Command.InFlight, cmd.Action.ID, cmd.ID)
os.Remove(inflightPath)
// remove command from Returned dir
os.Remove(cmdPath)
return
}
// loadReturnedCommands walks through the returned commands directory and loads
// the commands into the scheduler
func loadReturnedCommands(ctx Context) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("loadReturnedCommands() -> %v", e)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving loadReturnedCommands()"}.Debug()
}()
dir, err := os.Open(ctx.Directories.Command.Returned)
dirContent, err := dir.Readdir(-1)
if err != nil {
panic(err)
}
// loop over the content of the directory
for _, DirEntry := range dirContent {
if !DirEntry.Mode().IsRegular() {
// ignore non file
continue
}
filename := ctx.Directories.Command.Returned + "/" + DirEntry.Name()
_, err = os.Stat(filename)
if err != nil {
// file is already gone, probably consumed by the file notifier
// ignore and continue
continue
}
err := waitForFileOrDelete(filename, 3)
if err != nil {
ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("error while reading '%s': %v", filename, err)}.Err()
continue
}
cmd, err := mig.CmdFromFile(filename)
if err != nil {
// failing to load this file, log and skip it
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: fmt.Sprintf("failed to load returned command file %s", filename)}.Err()
continue
}
// queue it
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, CommandID: cmd.ID, ActionID: cmd.Action.ID, Desc: fmt.Sprintf("loading returned command '%s'", cmd.Action.Name)}
ctx.Channels.CommandReturned <- filename
}
dir.Close()
return
}
// returnCommands is called when commands have returned
// it stores the result of a command and mark it as completed/failed and then
// send a message to the Action completion routine to update the action status
func returnCommands(cmdFiles []string, ctx Context) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("terminateCommand() -> %v", e)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving terminateCommand()"}.Debug()
}()
for _, cmdFile := range cmdFiles {
// load and parse the command. If this fail, skip it and continue.
cmd, err := mig.CmdFromFile(cmdFile)
if err != nil {
panic(err)
}
cmd.FinishTime = time.Now().UTC()
if cmd.Status == "sent" {
cmd.Status = "done"
}
// update command in database
go func() {
err = ctx.DB.FinishCommand(cmd)
if err != nil {
desc := fmt.Sprintf("failed to finish command in database: '%v'", err)
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: cmd.Action.ID, CommandID: cmd.ID, Desc: desc}.Err()
} else {
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: cmd.Action.ID, CommandID: cmd.ID, Desc: "command updated in database"}.Debug()
}
}()
// remove command from inflight dir
inflightPath := fmt.Sprintf("%s/%.0f-%.0f.json", ctx.Directories.Command.InFlight, cmd.Action.ID, cmd.ID)
os.Remove(inflightPath)
// remove command from Returned dir
os.Remove(cmdFile)
// pass the command over to the Command Done channel
ctx.Channels.CommandDone <- cmd
}
return
}
func main() {
var err error
var Usage = func() {
fmt.Fprintf(os.Stderr,
"Mozilla InvestiGator Action Verifier\n"+
"usage: %s <-a action file> <-c command file>\n\n"+
"Command line to verify an action *or* command.\n"+
"Options:\n",
os.Args[0])
flag.PrintDefaults()
}
hasaction := false
hascommand := false
// command line options
var actionfile = flag.String("a", "/path/to/action", "Load action from file")
var commandfile = flag.String("c", "/path/to/command", "Load command from file")
var pubring = flag.String("pubring", "/path/to/pubring", "Use pubring at <path>")
flag.Parse()
// if a file is defined, load action from that
if *actionfile != "/path/to/action" {
hasaction = true
}
if *commandfile != "/path/to/command" {
hascommand = true
}
if (hasaction && hascommand) || (!hasaction && !hascommand) {
Usage()
panic(err)
}
var a mig.Action
if hasaction {
a, err = mig.ActionFromFile(*actionfile)
if err != nil {
panic(err)
}
} else {
c, err := mig.CmdFromFile(*commandfile)
if err != nil {
panic(err)
}
a = c.Action
}
fmt.Printf("%s\n", a)
err = a.Validate()
if err != nil {
fmt.Println(err)
}
// find keyring in default location
u, err := user.Current()
if err != nil {
panic(err)
}
if *pubring != "/path/to/pubring" {
// load keyring
var gnupghome string
gnupghome = os.Getenv("GNUPGHOME")
if gnupghome == "" {
gnupghome = "/.gnupg"
}
*pubring = u.HomeDir + gnupghome + "/pubring.gpg"
}
keyring, err := os.Open(*pubring)
if err != nil {
panic(err)
}
defer keyring.Close()
// syntax checking
err = a.VerifySignatures(keyring)
if err != nil {
panic(err)
}
fmt.Println("Valid signature")
}
// updateAction is called when a command has finished and the parent action
// must be updated. It retrieves an action from the database, loops over the
// commands, and if all commands have finished, marks the action as finished.
func updateAction(cmdPath string, ctx Context) (err error) {
var cmd mig.Command
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("updateAction() -> %v", e)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: cmd.Action.ID, CommandID: cmd.ID, Desc: "leaving updateAction()"}.Debug()
}()
// load the command file
cmd, err = mig.CmdFromFile(cmdPath)
if err != nil {
panic(err)
}
// use the action ID from the command file to get the action from the database
var eas []mig.ExtendedAction
actionCursor := ctx.DB.Col.Action.Find(bson.M{"action.id": cmd.Action.ID}).Iter()
err = actionCursor.All(&eas)
if err != nil {
panic(err)
}
if len(eas) > 1 {
err = fmt.Errorf("found multiple actions with same ID")
panic(err)
}
// there is only one entry in the slice, so take the first entry from
ea := eas[0]
switch cmd.Status {
case "succeeded":
ea.Counters.Succeeded++
case "cancelled":
ea.Counters.Cancelled++
case "failed":
ea.Counters.Failed++
case "timeout":
ea.Counters.TimeOut++
default:
err = fmt.Errorf("unknown command status: %s", cmd.Status)
panic(err)
}
// regardless of returned status, increase completion counter
ea.Counters.Completed++
ea.LastUpdateTime = time.Now().UTC()
desc := fmt.Sprintf("updating action '%s': completion=%d/%d, succeeded=%d, cancelled=%d, failed=%d, timeout=%d. duration=%s",
ea.Action.Name, ea.Counters.Completed, ea.Counters.Sent, ea.Counters.Succeeded,
ea.Counters.Cancelled, ea.Counters.Failed, ea.Counters.TimeOut, ea.LastUpdateTime.Sub(ea.StartTime).String())
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: ea.Action.ID, CommandID: cmd.ID, Desc: desc}
// Has the action completed?
if ea.Counters.Completed == ea.Counters.Sent {
// update status and timestamps
ea.Status = "completed"
ea.FinishTime = time.Now().UTC()
duration := ea.FinishTime.Sub(ea.StartTime)
// log
desc = fmt.Sprintf("action has completed in %s", duration.String())
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: ea.Action.ID, CommandID: cmd.ID, Desc: desc}
// TODO write action to Done directory
// landAction()
// delete Action from ctx.Directories.Action.InFlight
actFile := fmt.Sprintf("%d.json", ea.Action.ID)
os.Rename(ctx.Directories.Action.InFlight+"/"+actFile, ctx.Directories.Action.Done+"/"+actFile)
}
// store updated action in database
err = ctx.DB.Col.Action.Update(bson.M{"action.id": ea.Action.ID}, ea)
if err != nil {
panic(err)
}
// remove the command from the spool
os.Remove(cmdPath)
return
}
// sendCommand is called when a command file is created in ctx.Directories.Command.Ready
// it read the command, sends it to the agent via AMQP, and update the DB
func sendCommand(cmdPath string, ctx Context) (err error) {
var cmd mig.Command
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("sendCommand() -> %v", e)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: cmd.Action.ID, Desc: "leaving sendCommand()"}.Debug()
}()
// load and parse the command. If this fail, skip it and continue.
cmd, err = mig.CmdFromFile(cmdPath)
if err != nil {
panic(err)
}
cmd.Status = "sent"
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: cmd.Action.ID, CommandID: cmd.ID, Desc: "command received for sending"}.Debug()
data, err := json.Marshal(cmd)
if err != nil {
panic(err)
}
// build amqp message for sending
msg := amqp.Publishing{
DeliveryMode: amqp.Persistent,
Timestamp: time.Now(),
ContentType: "text/plain",
Body: []byte(data),
}
agtQueue := fmt.Sprintf("mig.agt.%s", cmd.AgentQueueLoc)
// send
err = ctx.MQ.Chan.Publish("mig", agtQueue, true, false, msg)
if err != nil {
panic(err)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: cmd.Action.ID, CommandID: cmd.ID, Desc: "command sent to agent queue"}.Debug()
// write command to InFlight directory
dest := fmt.Sprintf("%s/%d-%d.json", ctx.Directories.Command.InFlight, cmd.Action.ID, cmd.ID)
err = safeWrite(ctx, dest, data)
if err != nil {
panic(err)
}
// remove original command file
err = os.Remove(cmdPath)
if err != nil {
panic(err)
}
// store command in database
err = ctx.DB.Col.Cmd.Insert(cmd)
if err != nil {
panic(err)
}
return
}
func main() {
var err error
var Usage = func() {
fmt.Fprintf(os.Stderr,
"Mozilla InvestiGator Action Verifier\n"+
"usage: %s <-a action file> <-c command file>\n\n"+
"Command line to verify an action *or* command.\n"+
"Options:\n",
os.Args[0])
flag.PrintDefaults()
}
hasaction := false
hascommand := false
homedir := client.FindHomedir()
// command line options
var actionfile = flag.String("a", "/path/to/action", "Load action from file")
var commandfile = flag.String("c", "/path/to/command", "Load command from file")
var config = flag.String("conf", homedir+"/.migrc", "Load configuration from file")
flag.Parse()
conf, err := client.ReadConfiguration(*config)
if err != nil {
panic(err)
}
// if a file is defined, load action from that
if *actionfile != "/path/to/action" {
hasaction = true
}
if *commandfile != "/path/to/command" {
hascommand = true
}
if (hasaction && hascommand) || (!hasaction && !hascommand) {
fmt.Println("[error] either an action file or a command file must be provided")
Usage()
os.Exit(1)
}
var a mig.Action
if hasaction {
a, err = mig.ActionFromFile(*actionfile)
if err != nil {
panic(err)
}
} else {
c, err := mig.CmdFromFile(*commandfile)
if err != nil {
panic(err)
}
a = c.Action
}
err = a.Validate()
if err != nil {
fmt.Println(err)
}
pubringFile, err := os.Open(conf.GPG.Home + "/pubring.gpg")
if err != nil {
panic(err)
}
defer pubringFile.Close()
// syntax checking
err = a.VerifySignatures(pubringFile)
if err != nil {
fmt.Println("[error]", err)
} else {
fmt.Println("Valid signature")
}
}