// invalidAction marks actions that have failed to run
func invalidAction(ctx Context, a mig.Action, origin string) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("invalidAction() -> %v", e)
}
ctx.Channels.Log <- mig.Log{ActionID: a.ID, Desc: "leaving invalidAction()"}.Debug()
}()
// move action to invalid dir
jsonA, err := json.Marshal(a)
if err != nil {
panic(err)
}
dest := fmt.Sprintf("%s/%.0f.json", ctx.Directories.Action.Invalid, a.ID)
err = safeWrite(ctx, dest, jsonA)
if err != nil {
panic(err)
}
// remove the action from its origin
os.Remove(origin)
if err != nil {
panic(err)
}
a.Status = "invalid"
a.LastUpdateTime = time.Now().UTC()
a.FinishTime = time.Now().UTC()
a.Counters.Sent = 0
err = ctx.DB.UpdateAction(a)
if err != nil {
panic(err)
}
desc := fmt.Sprintf("invalidAction(): Action '%s' has been marked as invalid.", a.Name)
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: a.ID, Desc: desc}.Debug()
return
}
// updateAction is called with an array of commands that have finished
// Each action that needs updating is processed in a way that reduce IOs
func updateAction(cmds []mig.Command, ctx Context) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("updateAction() -> %v", e)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving updateAction()"}.Debug()
}()
// there may be multiple actions to update, since commands can be mixed,
// so we keep a map of actions
actions := make(map[float64]mig.Action)
for _, cmd := range cmds {
var a mig.Action
// retrieve the action from the DB if we don't already have it mapped
a, ok := actions[cmd.Action.ID]
if !ok {
a, err = ctx.DB.ActionMetaByID(cmd.Action.ID)
if err != nil {
panic(err)
}
}
a.LastUpdateTime = time.Now().UTC()
// store action in the map
actions[a.ID] = a
// slightly unrelated to updating the action:
// in case the action is about upgrading agents, do some magical stuff
// to continue the upgrade protocol
if cmd.Status == mig.StatusSuccess && len(a.Operations) > 0 {
if a.Operations[0].Module == "upgrade" {
go markUpgradedAgents(cmd, ctx)
}
}
}
for _, a := range actions {
a.Counters, err = ctx.DB.GetActionCounters(a.ID)
if err != nil {
panic(err)
}
// Has the action completed?
if a.Counters.Done == a.Counters.Sent {
err = landAction(ctx, a)
if err != nil {
panic(err)
}
// delete Action from ctx.Directories.Action.InFlight
actFile := fmt.Sprintf("%.0f.json", a.ID)
os.Rename(ctx.Directories.Action.InFlight+"/"+actFile, ctx.Directories.Action.Done+"/"+actFile)
} else {
// store updated action in database
err = ctx.DB.UpdateRunningAction(a)
if err != nil {
panic(err)
}
desc := fmt.Sprintf("updated action '%s': progress=%d/%d, success=%d, cancelled=%d, expired=%d, failed=%d, timeout=%d, duration=%s",
a.Name, a.Counters.Done, a.Counters.Sent, a.Counters.Success, a.Counters.Cancelled, a.Counters.Expired,
a.Counters.Failed, a.Counters.TimeOut, a.LastUpdateTime.Sub(a.StartTime).String())
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: a.ID, Desc: desc}
}
}
return
}
// createAction receives a signed action in a POST request, validates it,
// and write it into the scheduler spool
func createAction(respWriter http.ResponseWriter, request *http.Request) {
var (
err error
action mig.Action
)
opid := getOpID(request)
loc := fmt.Sprintf("%s%s", ctx.Server.Host, request.URL.String())
resource := cljs.New(loc)
defer func() {
if e := recover(); e != nil {
ctx.Channels.Log <- mig.Log{OpID: opid, ActionID: action.ID, Desc: fmt.Sprintf("%v", e)}.Err()
resource.SetError(cljs.Error{Code: fmt.Sprintf("%.0f", opid), Message: fmt.Sprintf("%v", e)})
respond(500, resource, respWriter, request)
}
ctx.Channels.Log <- mig.Log{OpID: opid, ActionID: action.ID, Desc: "leaving createAction()"}.Debug()
}()
// parse the POST body into a mig action
err = request.ParseForm()
if err != nil {
panic(err)
}
postAction := request.FormValue("action")
err = json.Unmarshal([]byte(postAction), &action)
if err != nil {
panic(err)
}
ctx.Channels.Log <- mig.Log{OpID: opid, Desc: fmt.Sprintf("Received action for creation '%s'", action)}.Debug()
// Init action fields
action.ID = mig.GenID()
date0 := time.Date(0011, time.January, 11, 11, 11, 11, 11, time.UTC)
date1 := time.Date(9998, time.January, 11, 11, 11, 11, 11, time.UTC)
action.StartTime = date0
action.FinishTime = date1
action.LastUpdateTime = date0
action.Status = "pending"
// load keyring and validate action
keyring, err := getKeyring()
if err != nil {
panic(err)
}
err = action.Validate()
if err != nil {
panic(err)
}
err = action.VerifySignatures(keyring)
if err != nil {
panic(err)
}
ctx.Channels.Log <- mig.Log{OpID: opid, ActionID: action.ID, Desc: "Received new action with valid signature"}
// write action to database
err = ctx.DB.InsertAction(action)
if err != nil {
panic(err)
}
// write signatures to database
astr, err := action.String()
if err != nil {
panic(err)
}
for _, sig := range action.PGPSignatures {
k, err := getKeyring()
if err != nil {
panic(err)
}
fp, err := pgp.GetFingerprintFromSignature(astr, sig, k)
if err != nil {
panic(err)
}
inv, err := ctx.DB.InvestigatorByFingerprint(fp)
if err != nil {
panic(err)
}
err = ctx.DB.InsertSignature(action.ID, inv.ID, sig)
if err != nil {
panic(err)
}
}
ctx.Channels.Log <- mig.Log{OpID: opid, ActionID: action.ID, Desc: "Action written to database"}
err = resource.AddItem(cljs.Item{
Href: fmt.Sprintf("%s/action?actionid=%.0f", ctx.Server.BaseURL, action.ID),
Data: []cljs.Data{{Name: "action ID " + fmt.Sprintf("%.0f", action.ID), Value: action}},
})
if err != nil {
panic(err)
}
// return a 202 Accepted. the action will be processed asynchronously, and may fail later.
respond(202, resource, respWriter, request)
}
// updateAction is called with an array of commands that have finished
// Each action that needs updating is processed in a way that reduce IOs
func updateAction(cmds []mig.Command, ctx Context) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("updateAction() -> %v", e)
}
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, Desc: "leaving updateAction()"}.Debug()
}()
// there may be multiple actions to update, since commands can be mixed,
// so we keep a map of actions
actions := make(map[float64]mig.Action)
for _, cmd := range cmds {
var a mig.Action
// retrieve the action from the DB if we don't already have it mapped
a, ok := actions[cmd.Action.ID]
if !ok {
a, err = ctx.DB.ActionMetaByID(cmd.Action.ID)
if err != nil {
panic(err)
}
}
// there is only one entry in the slice, so take the first entry from
switch cmd.Status {
case "done":
a.Counters.Done++
case "cancelled":
a.Counters.Cancelled++
case "failed":
a.Counters.Failed++
case "timeout":
a.Counters.TimeOut++
default:
err = fmt.Errorf("unknown command status: %s", cmd.Status)
panic(err)
}
// regardless of returned status, increase completion counter
a.Counters.Returned++
a.LastUpdateTime = time.Now().UTC()
// store action in the map
actions[a.ID] = a
// in case the action is related to upgrading agents, go do stuff
if cmd.Status == "done" {
go markUpgradedAgents(cmd, ctx)
}
}
for _, a := range actions {
// Has the action completed?
if a.Counters.Returned == a.Counters.Sent {
err = landAction(ctx, a)
if err != nil {
panic(err)
}
// delete Action from ctx.Directories.Action.InFlight
actFile := fmt.Sprintf("%.0f.json", a.ID)
os.Rename(ctx.Directories.Action.InFlight+"/"+actFile, ctx.Directories.Action.Done+"/"+actFile)
} else {
// store updated action in database
err = ctx.DB.UpdateRunningAction(a)
if err != nil {
panic(err)
}
desc := fmt.Sprintf("updated action '%s': completion=%d/%d, done=%d, cancelled=%d, failed=%d, timeout=%d, duration=%s",
a.Name, a.Counters.Returned, a.Counters.Sent, a.Counters.Done,
a.Counters.Cancelled, a.Counters.Failed, a.Counters.TimeOut, a.LastUpdateTime.Sub(a.StartTime).String())
ctx.Channels.Log <- mig.Log{OpID: ctx.OpID, ActionID: a.ID, Desc: desc}
}
}
return
}
// createAction receives a signed action in a POST request, validates it,
// and write it into the scheduler spool
func createAction(respWriter http.ResponseWriter, request *http.Request) {
var err error
opid := mig.GenID()
var action mig.Action
loc := fmt.Sprintf("http://%s:%d%s", ctx.Server.IP, ctx.Server.Port, request.URL.String())
resource := cljs.New(loc)
defer func() {
if e := recover(); e != nil {
ctx.Channels.Log <- mig.Log{OpID: opid, ActionID: action.ID, Desc: fmt.Sprintf("%v", e)}.Err()
resource.SetError(cljs.Error{Code: fmt.Sprintf("%.0f", opid), Message: fmt.Sprintf("%v", e)})
respond(500, resource, respWriter, request, opid)
}
ctx.Channels.Log <- mig.Log{OpID: opid, ActionID: action.ID, Desc: "leaving createAction()"}.Debug()
}()
// parse the POST body into a mig action
request.ParseForm()
postAction := request.FormValue("action")
err = json.Unmarshal([]byte(postAction), &action)
if err != nil {
panic(err)
}
ctx.Channels.Log <- mig.Log{OpID: opid, Desc: fmt.Sprintf("Received action for creation '%s'", action)}.Debug()
// Init action fields
action.ID = mig.GenID()
date0 := time.Date(9998, time.January, 11, 11, 11, 11, 11, time.UTC)
action.StartTime = date0
action.FinishTime = date0
action.LastUpdateTime = date0
action.Status = "init"
// load keyring and validate action
keyring, err := os.Open(ctx.PGP.Home + "/pubring.gpg")
if err != nil {
panic(err)
}
defer keyring.Close()
err = action.Validate()
if err != nil {
panic(err)
}
err = action.VerifySignatures(keyring)
if err != nil {
panic(err)
}
ctx.Channels.Log <- mig.Log{OpID: opid, ActionID: action.ID, Desc: "Received new action with valid signature"}
// write action to database
err = ctx.DB.InsertAction(action)
if err != nil {
panic(err)
}
// write signatures to database
astr, err := action.String()
if err != nil {
panic(err)
}
for _, sig := range action.PGPSignatures {
// TODO: opening the keyring in a loop is really ugly. rewind!
k, err := os.Open(ctx.PGP.Home + "/pubring.gpg")
if err != nil {
panic(err)
}
defer k.Close()
fp, err := pgp.GetFingerprintFromSignature(astr, sig, k)
if err != nil {
panic(err)
}
iid, err := ctx.DB.InvestigatorByFingerprint(fp)
if err != nil {
panic(err)
}
err = ctx.DB.InsertSignature(action.ID, iid, sig)
if err != nil {
panic(err)
}
}
ctx.Channels.Log <- mig.Log{OpID: opid, ActionID: action.ID, Desc: "Action written to database"}
// write action to disk
destdir := fmt.Sprintf("%s/%.0f.json", ctx.Directories.Action.New, action.ID)
newAction, err := json.Marshal(action)
if err != nil {
panic(err)
}
err = ioutil.WriteFile(destdir, newAction, 0640)
if err != nil {
panic(err)
}
ctx.Channels.Log <- mig.Log{OpID: opid, ActionID: action.ID, Desc: "Action committed to spool"}
err = resource.AddItem(cljs.Item{
Href: fmt.Sprintf("%s/action?actionid=%.0f", ctx.Server.BaseURL, action.ID),
Data: []cljs.Data{{Name: "action ID " + fmt.Sprintf("%.0f", action.ID), Value: action}},
})
if err != nil {
panic(err)
}
respond(201, resource, respWriter, request, opid)
}