// invalidAction marks actions that have failed to run
func invalidAction(ctx Context, a mig.Action, origin string) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("invalidAction() -> %v", e)
}
ctx.Channels.Log <- mig.Log{ActionID: a.ID, Desc: "leaving invalidAction()"}.Debug()
}()
// move action to invalid dir
jsonA, err := json.Marshal(a)
if err != nil {
panic(err)
}
dest := fmt.Sprintf("%s/%.0f.json", ctx.Directories.Action.Invalid, a.ID)
err = safeWrite(ctx, dest, jsonA)
if err != nil {
panic(err)
}
// remove the action from its origin
os.Remove(origin)
if err != nil {
panic(err)
}
a.Status = "invalid"
a.LastUpdateTime = time.Now().UTC()
a.FinishTime = time.Now().UTC()
a.Counters.Sent = 0
err = ctx.DB.UpdateAction(a)
if err != nil {
panic(err)
}
desc := fmt.Sprintf("invalidAction(): Action '%s' has been marked as invalid.", a.Name)
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: a.ID, Desc: desc}.Debug()
return
}
// issueKillAction issues an `agentdestroy` action targeted to a specific agent
// and updates the status of the agent in the database
func issueKillAction(agent mig.Agent, ctx Context) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("issueKillAction() -> %v", e)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving issueKillAction()"}.Debug()
}()
// generate an `agentdestroy` action for this agent
killAction := mig.Action{
ID: mig.GenID(),
Name: fmt.Sprintf("Kill agent %s", agent.Name),
Target: fmt.Sprintf("queueloc='%s'", agent.QueueLoc),
ValidFrom: time.Now().Add(-60 * time.Second).UTC(),
ExpireAfter: time.Now().Add(30 * time.Minute).UTC(),
SyntaxVersion: 2,
}
var opparams struct {
PID int `json:"pid"`
Version string `json:"version"`
}
opparams.PID = agent.PID
opparams.Version = agent.Version
killOperation := mig.Operation{
Module: "agentdestroy",
Parameters: opparams,
}
killAction.Operations = append(killAction.Operations, killOperation)
// sign the action with the scheduler PGP key
secring, err := getSecring(ctx)
if err != nil {
panic(err)
}
pgpsig, err := killAction.Sign(ctx.PGP.PrivKeyID, secring)
if err != nil {
panic(err)
}
killAction.PGPSignatures = append(killAction.PGPSignatures, pgpsig)
var jsonAction []byte
jsonAction, err = json.Marshal(killAction)
if err != nil {
panic(err)
}
// write the action to the spool for scheduling
dest := fmt.Sprintf("%s/%.0f.json", ctx.Directories.Action.New, killAction.ID)
err = safeWrite(ctx, dest, jsonAction)
if err != nil {
panic(err)
}
// mark the agent as `destroyed` in the database
err = ctx.DB.MarkAgentDestroyed(agent)
if err != nil {
panic(err)
}
ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("issued kill action for agent '%s' "+
"with PID '%d'", agent.Name, agent.PID)}.Warning()
return
}
// FinishAction updates the action fields to mark it as done
func (db *DB) FinishAction(a mig.Action) (err error) {
a.FinishTime = time.Now()
a.Status = "completed"
_, err = db.c.Exec(`UPDATE actions SET (finishtime, lastupdatetime, status) = ($1, $2, $3) WHERE id=$4`,
a.FinishTime, a.LastUpdateTime, a.Status, a.ID)
if err != nil {
return fmt.Errorf("Failed to update action: '%v'", err)
}
return
}
// updateAction is called with an array of commands that have finished
// Each action that needs updating is processed in a way that reduce IOs
func updateAction(cmds []mig.Command, ctx Context) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("updateAction() -> %v", e)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving updateAction()"}.Debug()
}()
// there may be multiple actions to update, since commands can be mixed,
// so we keep a map of actions
actions := make(map[float64]mig.Action)
for _, cmd := range cmds {
var a mig.Action
// retrieve the action from the DB if we don't already have it mapped
a, ok := actions[cmd.Action.ID]
if !ok {
a, err = ctx.DB.ActionMetaByID(cmd.Action.ID)
if err != nil {
panic(err)
}
}
a.LastUpdateTime = time.Now().UTC()
// store action in the map
actions[a.ID] = a
}
for _, a := range actions {
a.Counters, err = ctx.DB.GetActionCounters(a.ID)
if err != nil {
panic(err)
}
// Has the action completed?
if a.Counters.Done == a.Counters.Sent {
err = landAction(ctx, a)
if err != nil {
panic(err)
}
// delete Action from ctx.Directories.Action.InFlight
actFile := fmt.Sprintf("%.0f.json", a.ID)
os.Rename(ctx.Directories.Action.InFlight+"/"+actFile, ctx.Directories.Action.Done+"/"+actFile)
} else {
// store updated action in database
err = ctx.DB.UpdateRunningAction(a)
if err != nil {
panic(err)
}
desc := fmt.Sprintf("updated action '%s': progress=%d/%d, success=%d, cancelled=%d, expired=%d, failed=%d, timeout=%d, duration=%s",
a.Name, a.Counters.Done, a.Counters.Sent, a.Counters.Success, a.Counters.Cancelled, a.Counters.Expired,
a.Counters.Failed, a.Counters.TimeOut, a.LastUpdateTime.Sub(a.StartTime).String())
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: a.ID, Desc: desc}
}
}
return
}
// flyAction moves an action file to the InFlight directory and
// write it to database
func flyAction(ctx Context, a mig.Action, origin string) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("flyAction() -> %v", e)
}
ctx.Channels.Log <- mig.Log{ActionID: a.ID, Desc: "leaving flyAction()"}.Debug()
}()
// move action to inflight dir
jsonA, err := json.Marshal(a)
if err != nil {
panic(err)
}
dest := fmt.Sprintf("%s/%.0f.json", ctx.Directories.Action.InFlight, a.ID)
err = safeWrite(ctx, dest, jsonA)
if err != nil {
panic(err)
}
// remove the action from its origin
os.Remove(origin)
if err != nil {
panic(err)
}
a.Status = "inflight"
err = ctx.DB.UpdateActionStatus(a)
if err != nil {
panic(err)
}
desc := fmt.Sprintf("flyAction(): Action '%s' is in flight", a.Name)
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: a.ID, Desc: desc}.Debug()
return
}
// SignAction takes a MIG Action, signs it with the key identified in the configuration
// and returns the signed action
func (cli Client) SignAction(a mig.Action) (signed_action mig.Action, err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("SignAction() -> %v", e)
}
}()
secring, err := os.Open(cli.Conf.GPG.Home + "/secring.gpg")
if err != nil {
panic(err)
}
defer secring.Close()
sig, err := a.Sign(cli.Conf.GPG.KeyID, secring)
if err != nil {
panic(err)
}
a.PGPSignatures = append(a.PGPSignatures, sig)
signed_action = a
return
}
// checkActionAuthorization verifies the PGP signatures of a given action
// against the Access Control List of the agent.
func checkActionAuthorization(a mig.Action, ctx *Context) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("checkActionAuthorization() -> %v", e)
}
ctx.Channels.Log <- mig.Log{ActionID: a.ID, Desc: "leaving checkActionAuthorization()"}.Debug()
}()
var keys [][]byte
for _, pk := range PUBLICPGPKEYS {
keys = append(keys, []byte(pk))
}
// get an io.Reader from the public pgp key
keyring, keycount, err := pgp.ArmoredKeysToKeyring(keys)
if err != nil {
panic(err)
}
ctx.Channels.Log <- mig.Log{ActionID: a.ID, Desc: fmt.Sprintf("loaded %d keys", keycount)}.Debug()
// Check the action syntax and signature
err = a.Validate()
if err != nil {
desc := fmt.Sprintf("action validation failed: %v", err)
ctx.Channels.Log <- mig.Log{ActionID: a.ID, Desc: desc}.Err()
panic(desc)
}
// Validate() checks that the action hasn't expired, but we need to
// check the start time ourselves
if time.Now().Before(a.ValidFrom) {
ctx.Channels.Log <- mig.Log{ActionID: a.ID, Desc: "action is scheduled for later"}.Err()
panic("Action ValidFrom date is in the future")
}
// check ACLs, includes verifying signatures
err = a.VerifyACL(ctx.ACL, keyring)
if err != nil {
desc := fmt.Sprintf("action ACL verification failed: %v", err)
ctx.Channels.Log <- mig.Log{ActionID: a.ID, Desc: desc}.Err()
panic(desc)
}
ctx.Channels.Log <- mig.Log{ActionID: a.ID, Desc: "ACL verification succeeded."}.Debug()
return
}
// landAction moves an action file to the Done directory and
// updates it in database
func landAction(ctx Context, a mig.Action) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("landAction() -> %v", e)
}
ctx.Channels.Log <- mig.Log{ActionID: a.ID, Desc: "leaving landAction()"}.Debug()
}()
// update status and timestamps
a.Status = "done"
a.FinishTime = time.Now().UTC()
duration := a.FinishTime.Sub(a.StartTime)
// log
desc := fmt.Sprintf("action has completed in %s", duration.String())
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: a.ID, Desc: desc}
// move action to done dir
jsonA, err := json.Marshal(a)
if err != nil {
panic(err)
}
dest := fmt.Sprintf("%s/%.0f.json", ctx.Directories.Action.Done, a.ID)
err = safeWrite(ctx, dest, jsonA)
if err != nil {
panic(err)
}
// remove the action from its origin
origin := fmt.Sprintf("%s/%.0f.json", ctx.Directories.Action.InFlight, a.ID)
os.Remove(origin)
if err != nil {
panic(err)
}
err = ctx.DB.FinishAction(a)
if err != nil {
panic(err)
}
desc = fmt.Sprintf("landAction(): Action '%s' has landed", a.Name)
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: a.ID, Desc: desc}.Debug()
return
}
// PostAction submits a MIG Action to the API and returns the reflected action with API ID
func (cli Client) PostAction(a mig.Action) (a2 mig.Action, err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("PostAction() -> %v", e)
}
}()
a.SyntaxVersion = mig.ActionVersion
// serialize
ajson, err := json.Marshal(a)
if err != nil {
panic(err)
}
actionstr := string(ajson)
data := url.Values{"action": {actionstr}}
r, err := http.NewRequest("POST", cli.Conf.API.URL+"action/create/", strings.NewReader(data.Encode()))
if err != nil {
panic(err)
}
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, err := cli.Do(r)
if err != nil {
panic(err)
}
defer resp.Body.Close()
if err != nil {
panic(err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
if resp.StatusCode != 202 {
err = fmt.Errorf("error: HTTP %d. action creation failed.", resp.StatusCode)
panic(err)
}
var resource *cljs.Resource
err = json.Unmarshal(body, &resource)
if err != nil {
panic(err)
}
a2, err = ValueToAction(resource.Collection.Items[0].Data[0].Value)
if err != nil {
panic(err)
}
return
}
func main() {
var (
conf client.Configuration
cli client.Client
err error
op mig.Operation
a mig.Action
migrc, show, render, target, expiration, afile string
verbose bool
modargs []string
run interface{}
)
defer func() {
if e := recover(); e != nil {
fmt.Fprintf(os.Stderr, "%v\n", e)
}
}()
homedir := client.FindHomedir()
fs := flag.NewFlagSet("mig flag", flag.ContinueOnError)
fs.Usage = continueOnFlagError
fs.StringVar(&migrc, "c", homedir+"/.migrc", "alternative configuration file")
fs.StringVar(&show, "show", "found", "type of results to show")
fs.StringVar(&render, "render", "text", "results rendering mode")
fs.StringVar(&target, "t", fmt.Sprintf("status='%s' AND mode='daemon'", mig.AgtStatusOnline), "action target")
fs.StringVar(&expiration, "e", "300s", "expiration")
fs.StringVar(&afile, "i", "/path/to/file", "Load action from file")
fs.BoolVar(&verbose, "v", false, "Enable verbose output")
// if first argument is missing, or is help, print help
// otherwise, pass the remainder of the arguments to the module for parsing
// this client is agnostic to module parameters
if len(os.Args) < 2 || os.Args[1] == "help" || os.Args[1] == "-h" || os.Args[1] == "--help" {
usage()
}
if len(os.Args) < 2 || os.Args[1] == "-V" {
fmt.Println(version)
os.Exit(0)
}
// when reading the action from a file, go directly to launch
if os.Args[1] == "-i" {
err = fs.Parse(os.Args[1:])
if err != nil {
panic(err)
}
if afile == "/path/to/file" {
panic("-i flag must take an action file path as argument")
}
a, err = mig.ActionFromFile(afile)
if err != nil {
panic(err)
}
fmt.Fprintf(os.Stderr, "[info] launching action from file, all flags are ignored\n")
goto readytolaunch
}
// arguments parsing works as follow:
// * os.Args[1] must contain the name of the module to launch. we first verify
// that a module exist for this name and then continue parsing
// * os.Args[2:] contains both global options and module parameters. We parse the
// whole []string to extract global options, and module parameters will be left
// unparsed in fs.Args()
// * fs.Args() with the module parameters is passed as a string to the module parser
// which will return a module operation to store in the action
op.Module = os.Args[1]
if _, ok := modules.Available[op.Module]; !ok {
panic("Unknown module " + op.Module)
}
// -- Ugly hack Warning --
// Parse() will fail on the first flag that is not defined, but in our case module flags
// are defined in the module packages and not in this program. Therefore, the flag parse error
// is expected. Unfortunately, Parse() writes directly to stderr and displays the error to
// the user, which confuses them. The right fix would be to prevent Parse() from writing to
// stderr, since that's really the job of the calling program, but in the meantime we work around
// it by redirecting stderr to null before calling Parse(), and put it back to normal afterward.
// for ref, issue is at https://github.com/golang/go/blob/master/src/flag/flag.go#L793
fs.SetOutput(os.NewFile(uintptr(87592), os.DevNull))
err = fs.Parse(os.Args[2:])
fs.SetOutput(nil)
if err != nil {
// ignore the flag not defined error, which is expected because
// module parameters are defined in modules and not in main
if len(err.Error()) > 30 && err.Error()[0:29] == "flag provided but not defined" {
// requeue the parameter that failed
modargs = append(modargs, err.Error()[31:])
} else {
// if it's another error, panic
panic(err)
}
}
for _, arg := range fs.Args() {
modargs = append(modargs, arg)
}
run = modules.Available[op.Module].NewRun()
if _, ok := run.(modules.HasParamsParser); !ok {
fmt.Fprintf(os.Stderr, "[error] module '%s' does not support command line invocation\n", op.Module)
os.Exit(2)
}
op.Parameters, err = run.(modules.HasParamsParser).ParamsParser(modargs)
if err != nil || op.Parameters == nil {
panic(err)
}
// If running against the local target, don't post the action to the MIG API
// but run it locally instead.
if target == "local" {
msg, err := modules.MakeMessage(modules.MsgClassParameters, op.Parameters)
if err != nil {
panic(err)
}
out := run.(modules.Runner).Run(bytes.NewBuffer(msg))
if len(out) == 0 {
panic("got empty results, run failed")
}
if _, ok := run.(modules.HasResultsPrinter); ok {
var modres modules.Result
err := json.Unmarshal([]byte(out), &modres)
if err != nil {
panic(err)
}
outRes, err := run.(modules.HasResultsPrinter).PrintResults(modres, true)
if err != nil {
panic(err)
}
for _, resLine := range outRes {
fmt.Println(resLine)
}
} else {
out = fmt.Sprintf("%s\n", out)
}
os.Exit(0)
}
a.Operations = append(a.Operations, op)
for _, arg := range os.Args[1:] {
a.Name += arg + " "
}
a.Target = target
readytolaunch:
// instanciate an API client
conf, err = client.ReadConfiguration(migrc)
if err != nil {
panic(err)
}
cli, err = client.NewClient(conf, "cmd-"+version)
if err != nil {
panic(err)
}
if verbose {
cli.EnableDebug()
}
// set the validity 60 second in the past to deal with clock skew
a.ValidFrom = time.Now().Add(-60 * time.Second).UTC()
period, err := time.ParseDuration(expiration)
if err != nil {
panic(err)
}
a.ExpireAfter = a.ValidFrom.Add(period)
// add extra 60 seconds taken for clock skew
a.ExpireAfter = a.ExpireAfter.Add(60 * time.Second).UTC()
asig, err := cli.SignAction(a)
if err != nil {
panic(err)
}
a = asig
// evaluate target before launch, give a change to cancel before going out to agents
agents, err := cli.EvaluateAgentTarget(a.Target)
if err != nil {
panic(err)
}
fmt.Fprintf(os.Stderr, "\x1b[33m%d agents will be targeted. ctrl+c to cancel. launching in \x1b[0m", len(agents))
for i := 5; i > 0; i-- {
time.Sleep(1 * time.Second)
fmt.Fprintf(os.Stderr, "\x1b[33m%d\x1b[0m ", i)
}
fmt.Fprintf(os.Stderr, "\x1b[33mGO\n\x1b[0m")
// launch and follow
a, err = cli.PostAction(a)
if err != nil {
panic(err)
}
c := make(chan os.Signal, 1)
done := make(chan bool, 1)
signal.Notify(c, os.Interrupt)
go func() {
err = cli.FollowAction(a)
if err != nil {
panic(err)
}
done <- true
}()
select {
case <-c:
fmt.Fprintf(os.Stderr, "stop following action. agents may still be running. printing available results:\n")
goto printresults
case <-done:
goto printresults
}
printresults:
err = cli.PrintActionResults(a, show, render)
if err != nil {
panic(err)
}
}
// processNewAction is called when a new action is available. It pulls
// the action from the directory, parse it, retrieve a list of targets from
// the backend database, and create individual command for each target.
func processNewAction(actionPath string, ctx Context) (err error) {
var action mig.Action
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("processNewAction() -> %v", e)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: action.ID, Desc: "leaving processNewAction()"}.Debug()
}()
// load the action file
action, err = mig.ActionFromFile(actionPath)
if err != nil {
panic(err)
}
action.StartTime = time.Now()
// generate an action id
if action.ID < 1 {
action.ID = mig.GenID()
}
desc := fmt.Sprintf("new action received: Name='%s' Target='%s' ValidFrom='%s' ExpireAfter='%s'",
action.Name, action.Target, action.ValidFrom, action.ExpireAfter)
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: action.ID, Desc: desc}
// TODO: replace with action.Validate(), to include signature verification
if time.Now().Before(action.ValidFrom) {
// queue new action
desc := fmt.Sprintf("action '%s' is not ready for scheduling", action.Name)
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: action.ID, Desc: desc}.Debug()
return
}
if time.Now().After(action.ExpireAfter) {
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: action.ID, Desc: fmt.Sprintf("action '%s' is expired. invalidating.", action.Name)}
err = invalidAction(ctx, action, actionPath)
if err != nil {
panic(err)
}
return
}
// find target agents for the action
agents, err := ctx.DB.ActiveAgentsByTarget(action.Target)
if err != nil {
panic(err)
}
action.Counters.Sent = len(agents)
if action.Counters.Sent == 0 {
err = fmt.Errorf("No agents found for target '%s'. invalidating action.", action.Target)
err = invalidAction(ctx, action, actionPath)
if err != nil {
panic(err)
}
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: action.ID, Desc: fmt.Sprintf("Found %d target agents", action.Counters.Sent)}
action.Status = "preparing"
inserted, err := ctx.DB.InsertOrUpdateAction(action)
if err != nil {
panic(err)
}
if inserted {
// action was inserted, and not updated, so we need to insert
// the signatures as well
astr, err := action.String()
if err != nil {
panic(err)
}
for _, sig := range action.PGPSignatures {
pubring, err := getPubring(ctx)
if err != nil {
panic(err)
}
fp, err := pgp.GetFingerprintFromSignature(astr, sig, pubring)
if err != nil {
panic(err)
}
inv, err := ctx.DB.InvestigatorByFingerprint(fp)
if err != nil {
panic(err)
}
err = ctx.DB.InsertSignature(action.ID, inv.ID, sig)
if err != nil {
panic(err)
}
}
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: action.ID, Desc: "Action written to database"}.Debug()
// create an array of empty results to serve as default for all commands
emptyResults := make([]modules.Result, len(action.Operations))
created := 0
for _, agent := range agents {
err := createCommand(ctx, action, agent, emptyResults)
if err != nil {
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: action.ID, Desc: "Failed to create commmand on agent" + agent.Name}.Err()
continue
}
created++
}
if created == 0 {
// no command created found
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: action.ID, Desc: "No command created. Invalidating action."}.Err()
err = invalidAction(ctx, action, actionPath)
if err != nil {
panic(err)
}
return nil
}
// move action to flying state
err = flyAction(ctx, action, actionPath)
if err != nil {
panic(err)
}
return
}
// FollowAction continuously loops over an action and prints its completion status in os.Stderr.
// when the action reaches its expiration date, FollowAction prints its final status and returns.
func (cli Client) FollowAction(a mig.Action) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("followAction() -> %v", e)
}
}()
fmt.Fprintf(os.Stderr, "\x1b[34mFollowing action ID %.0f.\x1b[0m", a.ID)
sent := 0
dotter := 0
previousctr := 0
status := ""
attempts := 0
var completion float64
for {
a, _, err = cli.GetAction(a.ID)
if err != nil {
attempts++
time.Sleep(1 * time.Second)
if attempts >= 30 {
panic("failed to retrieve action after 30 seconds. launch may have failed")
}
continue
}
if status == "" {
status = a.Status
}
if status != a.Status {
fmt.Fprintf(os.Stderr, "\x1b[34mstatus=%s\x1b[0m", a.Status)
status = a.Status
}
// exit follower mode if status isn't one we follow,
// or enough commands have returned
// or expiration time has passed
if (status != "pending" && status != "scheduled" && status != "preparing" && status != "inflight") ||
(a.Counters.Done > 0 && a.Counters.Done >= a.Counters.Sent) ||
(time.Now().After(a.ExpireAfter.Add(10 * time.Second))) {
goto finish
break
}
// init counters
if sent == 0 {
if a.Counters.Sent == 0 {
time.Sleep(1 * time.Second)
continue
} else {
sent = a.Counters.Sent
}
}
if a.Counters.Done > 0 && a.Counters.Done > previousctr {
completion = (float64(a.Counters.Done) / float64(a.Counters.Sent)) * 100
if completion < 99.5 {
previousctr = a.Counters.Done
fmt.Fprintf(os.Stderr, "\x1b[34m%.0f%%\x1b[0m", completion)
}
}
fmt.Fprintf(os.Stderr, "\x1b[34m.\x1b[0m")
time.Sleep(2 * time.Second)
dotter++
}
finish:
a, _, err = cli.GetAction(a.ID)
if err != nil {
fmt.Fprintf(os.Stderr, "[error] failed to retrieve action counters\n")
} else {
completion = (float64(a.Counters.Done) / float64(a.Counters.Sent)) * 100
fmt.Fprintf(os.Stderr, "\n\x1b[34m- %2.1f%% done in %s\x1b[0m\n", completion, time.Now().Sub(a.StartTime).String())
}
fmt.Fprintf(os.Stderr, "\x1b[34m")
a.PrintCounters()
fmt.Fprintf(os.Stderr, "\x1b[0m")
return
}
// FollowAction continuously loops over an action and prints its completion status in os.Stderr.
// when the action reaches its expiration date, FollowAction prints its final status and returns.
func (cli Client) FollowAction(a mig.Action, total int) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("followAction() -> %v", e)
}
}()
fmt.Fprintf(os.Stderr, "\x1b[34mFollowing action ID %.0f.\x1b[0m\n", a.ID)
previousctr := 0
status := ""
attempts := 0
var completion float64
bar := pb.New(total)
bar.ShowSpeed = true
bar.SetMaxWidth(80)
bar.Output = os.Stderr
bar.Start()
for {
a, _, err = cli.GetAction(a.ID)
if err != nil {
attempts++
time.Sleep(time.Second)
if attempts >= 30 {
panic("failed to retrieve action after 30 seconds. launch may have failed")
}
continue
}
if status == "" {
status = a.Status
}
// exit follower mode if status isn't one we follow,
// or enough commands have returned
// or expiration time has passed
if (status != "pending" && status != "scheduled" && status != "preparing" && status != "inflight") ||
(a.Counters.Done > 0 && a.Counters.Done >= a.Counters.Sent) ||
(time.Now().After(a.ExpireAfter.Add(10 * time.Second))) {
goto finish
break
}
if a.Counters.Done > 0 && a.Counters.Done > previousctr {
completion = (float64(a.Counters.Done) / float64(a.Counters.Sent)) * 100
if completion < 99.5 {
bar.Add(a.Counters.Done - previousctr)
bar.Update()
previousctr = a.Counters.Done
}
}
time.Sleep(2 * time.Second)
}
finish:
bar.Add(total - previousctr)
bar.Update()
bar.Finish()
a, _, err = cli.GetAction(a.ID)
if err != nil {
fmt.Fprintf(os.Stderr, "[error] failed to retrieve action counters\n")
} else {
completion = (float64(a.Counters.Done) / float64(a.Counters.Sent)) * 100
fmt.Fprintf(os.Stderr, "\x1b[34m%2.1f%% done in %s\x1b[0m\n", completion, time.Now().Sub(a.StartTime).String())
}
fmt.Fprintf(os.Stderr, "\x1b[34m")
a.PrintCounters()
fmt.Fprintf(os.Stderr, "\x1b[0m")
return
}
// createAction receives a signed action in a POST request, validates it,
// and write it into the scheduler spool
func createAction(respWriter http.ResponseWriter, request *http.Request) {
var (
err error
action mig.Action
)
opid := getOpID(request)
loc := fmt.Sprintf("%s%s", ctx.Server.Host, request.URL.String())
resource := cljs.New(loc)
defer func() {
if e := recover(); e != nil {
ctx.Channels.Log <- mig.Log{OpID: opid, ActionID: action.ID, Desc: fmt.Sprintf("%v", e)}.Err()
resource.SetError(cljs.Error{Code: fmt.Sprintf("%.0f", opid), Message: fmt.Sprintf("%v", e)})
respond(http.StatusInternalServerError, resource, respWriter, request)
}
ctx.Channels.Log <- mig.Log{OpID: opid, ActionID: action.ID, Desc: "leaving createAction()"}.Debug()
}()
// parse the POST body into a mig action
err = request.ParseForm()
if err != nil {
panic(err)
}
postAction := request.FormValue("action")
err = json.Unmarshal([]byte(postAction), &action)
if err != nil {
panic(err)
}
ctx.Channels.Log <- mig.Log{OpID: opid, Desc: fmt.Sprintf("Received action for creation '%s'", action)}.Debug()
// Init action fields
action.ID = mig.GenID()
date0 := time.Date(0011, time.January, 11, 11, 11, 11, 11, time.UTC)
date1 := time.Date(9998, time.January, 11, 11, 11, 11, 11, time.UTC)
action.StartTime = date0
action.FinishTime = date1
action.LastUpdateTime = date0
action.Status = "pending"
// load keyring and validate action
keyring, err := getKeyring()
if err != nil {
panic(err)
}
err = action.Validate()
if err != nil {
panic(err)
}
err = action.VerifySignatures(keyring)
if err != nil {
panic(err)
}
ctx.Channels.Log <- mig.Log{OpID: opid, ActionID: action.ID, Desc: "Received new action with valid signature"}
// write action to database
err = ctx.DB.InsertAction(action)
if err != nil {
panic(err)
}
// write signatures to database
astr, err := action.String()
if err != nil {
panic(err)
}
for _, sig := range action.PGPSignatures {
k, err := getKeyring()
if err != nil {
panic(err)
}
fp, err := pgp.GetFingerprintFromSignature(astr, sig, k)
if err != nil {
panic(err)
}
inv, err := ctx.DB.InvestigatorByFingerprint(fp)
if err != nil {
panic(err)
}
err = ctx.DB.InsertSignature(action.ID, inv.ID, sig)
if err != nil {
panic(err)
}
}
ctx.Channels.Log <- mig.Log{OpID: opid, ActionID: action.ID, Desc: "Action written to database"}
err = resource.AddItem(cljs.Item{
Href: fmt.Sprintf("%s/action?actionid=%.0f", ctx.Server.BaseURL, action.ID),
Data: []cljs.Data{{Name: "action ID " + fmt.Sprintf("%.0f", action.ID), Value: action}},
})
if err != nil {
panic(err)
}
// return a 202 Accepted. the action will be processed asynchronously, and may fail later.
respond(http.StatusAccepted, resource, respWriter, request)
}
// actionLauncher prepares an action for launch, either by starting with an empty
// template, or by loading an existing action from the api or the local disk
func actionLauncher(tpl mig.Action, cli client.Client) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("actionLauncher() -> %v", e)
}
}()
var a mig.Action
if tpl.ID == 0 {
fmt.Println("Entering action launcher with empty template")
} else {
// reinit the fields that we don't reuse
a.Name = tpl.Name
a.Target = tpl.Target
a.Description = tpl.Description
a.Threat = tpl.Threat
a.Operations = tpl.Operations
fmt.Printf("Entering action launcher using template '%s'\n", a.Name)
}
hasTimes := false
hasSignatures := false
hasEvaluatedTarget := false
fmt.Println("Type \x1b[32;1mexit\x1b[0m or press \x1b[32;1mctrl+d\x1b[0m to leave. \x1b[32;1mhelp\x1b[0m may help.")
prompt := "\x1b[33;1mlauncher>\x1b[0m "
for {
// completion
var symbols = []string{"addoperation", "deloperation", "exit", "help", "init",
"json", "launch", "listagents", "load", "details", "filechecker", "netstat",
"setname", "settarget", "settimes", "sign", "times"}
readline.Completer = func(query, ctx string) []string {
var res []string
for _, sym := range symbols {
if strings.HasPrefix(sym, query) {
res = append(res, sym)
}
}
return res
}
input, err := readline.String(prompt)
if err == io.EOF {
break
}
if err != nil {
fmt.Println("error: ", err)
break
}
orders := strings.Split(strings.TrimSpace(input), " ")
switch orders[0] {
case "addoperation":
if len(orders) != 2 {
fmt.Println("Wrong arguments. Expects 'addoperation <module_name>'")
fmt.Println("example: addoperation filechecker")
break
}
// attempt to call ParamsCreator from the requested module
// ParamsCreator takes care of retrieving using input
var operation mig.Operation
operation.Module = orders[1]
if _, ok := modules.Available[operation.Module]; ok {
// instanciate and call module parameters creation function
run := modules.Available[operation.Module].NewRun()
if _, ok := run.(modules.HasParamsCreator); !ok {
fmt.Println(operation.Module, "module does not provide a parameters creator.")
fmt.Println("You can write your action by hand and import it using 'load <file>'")
break
}
operation.Parameters, err = run.(modules.HasParamsCreator).ParamsCreator()
if err != nil {
fmt.Printf("Parameters creation failed with error: %v\n", err)
break
}
a.Operations = append(a.Operations, operation)
opjson, err := json.MarshalIndent(operation, "", " ")
if err != nil {
panic(err)
}
fmt.Printf("Inserting %s operation with parameters:\n%s\n", operation.Module, opjson)
} else {
fmt.Println("Module", operation.Module, "is not available in this console...")
fmt.Println("You can write your action by hand and import it using 'load <file>'")
}
case "deloperation":
if len(orders) != 2 {
fmt.Println("Wrong arguments. Expects 'deloperation <opnum>'")
fmt.Println("example: deloperation 0")
break
}
opnum, err := strconv.Atoi(orders[1])
if err != nil || opnum < 0 || opnum > len(a.Operations)-1 {
fmt.Println("error: <opnum> must be a positive integer between 0 and", len(a.Operations)-1)
break
}
a.Operations = append(a.Operations[:opnum], a.Operations[opnum+1:]...)
case "details":
fmt.Printf("ID %.0f\nName %s\nTarget %s\nAuthor %s <%s>\n"+
"Revision %.0f\nURL %s\nThreat Type %s, Level %s, Family %s, Reference %s\n",
a.ID, a.Name, a.Target, a.Description.Author, a.Description.Email,
a.Description.Revision, a.Description.URL,
a.Threat.Type, a.Threat.Level, a.Threat.Family, a.Threat.Ref)
fmt.Printf("%d operations: ", len(a.Operations))
for i, op := range a.Operations {
fmt.Printf("%d=%s; ", i, op.Module)
}
fmt.Printf("\n")
case "exit":
fmt.Printf("exit\n")
goto exit
case "help":
fmt.Printf(`The following orders are available:
addoperation <module> append a new operation of type <module> to the action operations
listagents list agents targetted by an action
deloperation <opnum> remove operation numbered <opnum> from operations array, count starts at zero
details display the action details
exit exit this mode
help show this help
json <pack> show the json of the action
launch <nofollow> launch the action. to return before completion, add "nofollow"
load <path> load an action from a file at <path>
setname <name> set the name of the action
settarget <target> set the target
settimes <start> <stop> set the validity and expiration dates
sign PGP sign the action
times show the various timestamps of the action
`)
case "json":
pack := false
if len(orders) > 1 {
if orders[1] == "pack" {
pack = true
} else {
fmt.Printf("Unknown option '%s'\n", orders[1])
}
}
var ajson []byte
if pack {
ajson, err = json.Marshal(a)
} else {
ajson, err = json.MarshalIndent(a, "", " ")
}
if err != nil {
panic(err)
}
fmt.Printf("%s\n", ajson)
case "launch":
follow := true
if len(orders) > 1 {
if orders[1] == "nofollow" {
follow = false
} else {
fmt.Printf("Unknown option '%s'\n", orders[1])
}
}
if a.Name == "" {
fmt.Println("Action has no name. Define one using 'setname <name>'")
break
}
if a.Target == "" {
fmt.Println("Action has no target. Define one using 'settarget <target>'")
break
}
if !hasEvaluatedTarget {
agents, err := cli.EvaluateAgentTarget(a.Target)
if err != nil {
panic(err)
}
count := len(agents)
if count == 0 {
fmt.Println("0 agents match this target. launch aborted")
break
}
fmt.Printf("%d agents will be targeted by search \"%s\"\n", count, a.Target)
input, err = readline.String("continue? (y/n)> ")
if err != nil {
panic(err)
}
if input != "y" {
fmt.Println("launch aborted")
break
}
}
if !hasTimes {
fmt.Printf("Times are not defined. Setting validity from now until +%s\n", defaultExpiration)
// for immediate execution, set validity one minute in the past
a.ValidFrom = time.Now().Add(-60 * time.Second).UTC()
period, err := time.ParseDuration(defaultExpiration)
if err != nil {
panic(err)
}
a.ExpireAfter = a.ValidFrom.Add(period)
a.ExpireAfter = a.ExpireAfter.Add(60 * time.Second).UTC()
hasTimes = true
}
if !hasSignatures {
asig, err := cli.SignAction(a)
if err != nil {
panic(err)
}
a = asig
hasSignatures = true
}
a, err = cli.PostAction(a)
if err != nil {
panic(err)
}
fmt.Printf("Action '%s' successfully launched with ID '%.0f' on target '%s'\n",
a.Name, a.ID, a.Target)
if follow {
err = cli.FollowAction(a)
if err != nil {
panic(err)
}
}
fmt.Println("")
_ = actionReader(fmt.Sprintf("action %.0f", a.ID), cli)
goto exit
case "listagents":
agents, err := cli.EvaluateAgentTarget(a.Target)
if err != nil {
fmt.Println(err)
break
}
fmt.Println("---- ID ---- + ---- Name -------")
for _, agt := range agents {
fmt.Printf("%20.0f %s\n", agt.ID, agt.Name)
}
case "load":
if len(orders) != 2 {
fmt.Println("Wrong arguments. Expects 'load <path_to_file>'")
break
}
a, err = mig.ActionFromFile(orders[1])
if err != nil {
panic(err)
}
fmt.Printf("Loaded action '%s' from %s\n", a.Name, orders[1])
case "sign":
if !hasTimes {
fmt.Println("Times must be set prior to signing")
break
}
asig, err := cli.SignAction(a)
if err != nil {
panic(err)
}
a = asig
hasSignatures = true
case "setname":
if len(orders) < 2 {
fmt.Println("Wrong arguments. Must be 'setname <some_name>'")
break
}
a.Name = strings.Join(orders[1:], " ")
case "settarget":
if len(orders) < 2 {
fmt.Println("Wrong arguments. Must be 'settarget <some_target_string>'")
break
}
a.Target = strings.Join(orders[1:], " ")
agents, err := cli.EvaluateAgentTarget(a.Target)
if err != nil {
fmt.Println(err)
break
}
fmt.Printf("%d agents will be targetted. To get the list, use 'listagents'\n", len(agents))
hasEvaluatedTarget = true
case "settimes":
// set the dates
if len(orders) != 3 {
fmt.Println(`Invalid times. Expects settimes <start> <stop.)
examples:
settimes 2014-06-30T12:00:00.0Z 2014-06-30T14:00:00.0Z
settimes now +60m
`)
break
}
if orders[1] == "now" {
// for immediate execution, set validity one minute in the past
a.ValidFrom = time.Now().Add(-60 * time.Second).UTC()
period, err := time.ParseDuration(orders[2])
if err != nil {
fmt.Println("Failed to parse duration '%s': %v", orders[2], err)
break
}
a.ExpireAfter = a.ValidFrom.Add(period)
a.ExpireAfter = a.ExpireAfter.Add(60 * time.Second).UTC()
} else {
a.ValidFrom, err = time.Parse("2014-01-01T00:00:00.0Z", orders[1])
if err != nil {
fmt.Println("Failed to parse time '%s': %v", orders[1], err)
break
}
a.ExpireAfter, err = time.Parse("2014-01-01T00:00:00.0Z", orders[2])
if err != nil {
fmt.Println("Failed to parse time '%s': %v", orders[2], err)
break
}
}
hasTimes = true
case "times":
fmt.Printf("Valid from '%s' until '%s'\nStarted on '%s'\n"+
"Last updated '%s'\nFinished on '%s'\n",
a.ValidFrom, a.ExpireAfter, a.StartTime, a.LastUpdateTime, a.FinishTime)
case "":
break
default:
fmt.Printf("Unknown order '%s'. You are in action launcher mode. Try `help`.\n", orders[0])
}
readline.AddHistory(input)
}
exit:
fmt.Printf("\n")
return
}
func main() {
var err error
var Usage = func() {
fmt.Fprintf(os.Stderr,
"Mozilla InvestiGator Action Verifier\n"+
"usage: %s <-a action file> <-c command file>\n\n"+
"Command line to verify an action *or* command.\n"+
"Options:\n",
os.Args[0])
flag.PrintDefaults()
}
hasaction := false
hascommand := false
homedir := client.FindHomedir()
// command line options
var actionfile = flag.String("a", "/path/to/action", "Load action from file")
var commandfile = flag.String("c", "/path/to/command", "Load command from file")
var config = flag.String("conf", homedir+"/.migrc", "Load configuration from file")
var showversion = flag.Bool("V", false, "Show build version and exit")
flag.Parse()
if *showversion {
fmt.Println(mig.Version)
os.Exit(0)
}
conf, err := client.ReadConfiguration(*config)
if err != nil {
panic(err)
}
// if a file is defined, load action from that
if *actionfile != "/path/to/action" {
hasaction = true
}
if *commandfile != "/path/to/command" {
hascommand = true
}
if (hasaction && hascommand) || (!hasaction && !hascommand) {
fmt.Println("[error] either an action file or a command file must be provided")
Usage()
os.Exit(1)
}
var a mig.Action
if hasaction {
a, err = mig.ActionFromFile(*actionfile)
if err != nil {
panic(err)
}
} else {
c, err := mig.CmdFromFile(*commandfile)
if err != nil {
panic(err)
}
a = c.Action
}
err = a.Validate()
if err != nil {
fmt.Println(err)
}
pubringFile, err := os.Open(conf.GPG.Home + "/pubring.gpg")
if err != nil {
panic(err)
}
defer pubringFile.Close()
// syntax checking
err = a.VerifySignatures(pubringFile)
if err != nil {
fmt.Println("[error]", err)
} else {
fmt.Println("Valid signature")
}
}