// MarkWorkItemStarted tells Pharos that we've started work on this item.
func MarkWorkItemStarted(ingestState *models.IngestState, _context *context.Context, stage, message string) error {
_context.MessageLog.Info("Telling Pharos we're starting %s for %s/%s",
stage, ingestState.WorkItem.Bucket, ingestState.WorkItem.Name)
utcNow := time.Now().UTC()
ingestState.WorkItem.SetNodeAndPid()
ingestState.WorkItem.Stage = stage
ingestState.WorkItem.StageStartedAt = &utcNow
ingestState.WorkItem.Status = constants.StatusStarted
ingestState.WorkItem.Note = message
resp := _context.PharosClient.WorkItemSave(ingestState.WorkItem)
if resp.Error != nil {
_context.MessageLog.Error("Could not mark WorkItem started for %s for %s/%s: %v",
stage, ingestState.WorkItem.Bucket, ingestState.WorkItem.Name, resp.Error)
return resp.Error
}
ingestState.WorkItem = resp.WorkItem()
return nil
}
// MarkWorkItemFailed tells Pharos that this item failed processing
// due to a fatal error or too many unsuccessful attempts.
func MarkWorkItemFailed(ingestState *models.IngestState, _context *context.Context) error {
_context.MessageLog.Info("Telling Pharos processing failed for %s/%s",
ingestState.WorkItem.Bucket, ingestState.WorkItem.Name)
ingestState.WorkItem.Node = ""
ingestState.WorkItem.Pid = 0
ingestState.WorkItem.StageStartedAt = nil
ingestState.WorkItem.Retry = false
ingestState.WorkItem.NeedsAdminReview = true
ingestState.WorkItem.Status = constants.StatusFailed
ingestState.WorkItem.Note = "Processing failed. " + ingestState.IngestManifest.AllErrorsAsString()
resp := _context.PharosClient.WorkItemSave(ingestState.WorkItem)
if resp.Error != nil {
_context.MessageLog.Error("Could not mark WorkItem failed for %s/%s: %v",
ingestState.WorkItem.Bucket, ingestState.WorkItem.Name, resp.Error)
return resp.Error
}
ingestState.WorkItem = resp.WorkItem()
return nil
}
// RecordWorkItemState saves the WorkItemState for this task. We drop a
// copy into our JSON log as a backup, and update the WorkItemState in
// Pharos, so the next worker knows what to do with this item.
//
// Param activeResult will change, depending on what stage of processing
// we're in. It could be the IngestState.FetchResult, IngestState.RecordResult,
// etc.
func RecordWorkItemState(ingestState *models.IngestState, _context *context.Context, activeResult *models.WorkSummary) {
// Serialize the IngestManifest to JSON, and stuff it into the
// WorkItemState.State. Subsequent workers need this info to
// store the object's files in S3 and Glacier, and to record
// results in Pharos.
err := ingestState.WorkItemState.SetStateFromIngestManifest(ingestState.IngestManifest)
if err != nil {
// If we couldn't serialize the IngestManifest, subsequent workers
// won't have the info they need to process this bag. We'll have to
// requeue this item and start all over.
_context.MessageLog.Error(err.Error())
activeResult.AddError("Could not convert Ingest Manifest "+
"to JSON. This item will have to be re-processed. Error was: %v", err)
} else {
// OK. We serialized the IngestManifest. Dump a copy into the
// file system for backup and troubleshooting, and send a copy
// over to Pharos, so the next worker in the chain (the save worker)
// can access it.
LogJson(ingestState, _context.JsonLog)
resp := _context.PharosClient.WorkItemStateSave(ingestState.WorkItemState)
if resp.Error != nil {
// Could not send a copy of the WorkItemState to Pharos.
// That means subsequent workers won't have the info they
// need to work on this bag. We'll have to start processing
// all over again.
_context.MessageLog.Error(resp.Error.Error())
activeResult.AddError("Could not save WorkItemState "+
"to Pharos. This item will have to be re-processed. Error was: %v", resp.Error)
} else {
// Saved to Pharos!
_context.MessageLog.Info("Saved WorkItemState for WorkItem %d (%s/%s) to Pharos",
ingestState.WorkItem.Id, ingestState.WorkItem.Bucket,
ingestState.WorkItem.Name)
ingestState.WorkItemState = resp.WorkItemState()
}
}
}
// MarkWorkItemSucceeded tells Pharos that this item was processed successfully.
func MarkWorkItemSucceeded(ingestState *models.IngestState, _context *context.Context, nextStage string) error {
_context.MessageLog.Info("Telling Pharos processing can proceed for %s/%s",
ingestState.WorkItem.Bucket, ingestState.WorkItem.Name)
ingestState.WorkItem.Node = ""
ingestState.WorkItem.Pid = 0
ingestState.WorkItem.Retry = true
ingestState.WorkItem.StageStartedAt = nil
ingestState.WorkItem.NeedsAdminReview = false
ingestState.WorkItem.Stage = nextStage
if nextStage == constants.StageCleanup {
ingestState.WorkItem.Status = constants.StatusSuccess
} else {
ingestState.WorkItem.Status = constants.StatusPending
}
ingestState.WorkItem.Note = fmt.Sprintf("Item is ready for %s", nextStage)
resp := _context.PharosClient.WorkItemSave(ingestState.WorkItem)
if resp.Error != nil {
_context.MessageLog.Error("Could not mark WorkItem ready for %s for %s/%s: %v",
nextStage, ingestState.WorkItem.Bucket, ingestState.WorkItem.Name, resp.Error)
return resp.Error
}
ingestState.WorkItem = resp.WorkItem()
return nil
}