func (packager *DPNPackager) finishWithSuccess(manifest *models.DPNIngestManifest) {
// Tell Pharos we're done with this item and save the work state.
packager.Context.MessageLog.Info("Packaging succeeded for %s", manifest.WorkItem.ObjectIdentifier)
manifest.WorkItem.Stage = constants.StageStore
manifest.WorkItem.Status = constants.StatusPending
manifest.WorkItem.Note = "Packaging completed, awaiting storage"
manifest.WorkItem.Node = "" // no worker is working on this now
manifest.WorkItem.Pid = 0 // no process is working on this
manifest.WorkItem.Retry = true // just in case this had been false
SaveWorkItem(packager.Context, manifest, manifest.PackageSummary)
manifest.PackageSummary.Finish()
SaveWorkItemState(packager.Context, manifest, manifest.PackageSummary)
// Delete the working directory where we built the bag
if fileutil.LooksSafeToDelete(manifest.LocalDir, 12, 3) {
os.RemoveAll(manifest.LocalDir)
}
// Push this WorkItem to the next NSQ topic.
packager.Context.MessageLog.Info("Pushing %s (DPN bag %s) to NSQ topic %s",
manifest.IntellectualObject.Identifier, manifest.DPNBag.UUID,
packager.Context.Config.DPN.DPNIngestStoreWorker.NsqTopic)
PushToQueue(packager.Context, manifest, manifest.PackageSummary,
packager.Context.Config.DPN.DPNIngestStoreWorker.NsqTopic)
if manifest.PackageSummary.HasErrors() {
packager.Context.MessageLog.Error(manifest.PackageSummary.Errors[0])
}
// Tell NSQ we're done packaging this.
manifest.NsqMessage.Finish()
}
// DeleteBagFromStaging deletes the bag from the staging area, and releases
// the reserved storage from the volume manager. This deletes both the tarred
// and untarred version of the bag, if they both exist.
func DeleteBagFromStaging(ingestState *models.IngestState, _context *context.Context, activeResult *models.WorkSummary) {
tarFile := ingestState.IngestManifest.Object.IngestTarFilePath
if tarFile != "" && fileutil.FileExists(tarFile) {
_context.MessageLog.Info("Deleting %s", tarFile)
err := os.Remove(tarFile)
if err != nil {
_context.MessageLog.Warning(err.Error())
}
err = _context.VolumeClient.Release(tarFile)
if err != nil {
_context.MessageLog.Warning(err.Error())
}
} else {
_context.MessageLog.Info("Skipping deletion of %s: file does not exist", tarFile)
}
untarredBagPath := ingestState.IngestManifest.Object.IngestUntarredPath
looksSafeToDelete := fileutil.LooksSafeToDelete(untarredBagPath, 12, 3)
if fileutil.FileExists(untarredBagPath) && looksSafeToDelete {
_context.MessageLog.Info("Deleting untarred bag at %s", untarredBagPath)
err := os.RemoveAll(untarredBagPath)
if err != nil {
_context.MessageLog.Warning(err.Error())
}
err = _context.VolumeClient.Release(untarredBagPath)
if err != nil {
_context.MessageLog.Warning(err.Error())
}
} else {
_context.MessageLog.Info("Skipping deletion of untarred bag dir at %s: "+
"Directory does not exist, or is unsafe to delete.", untarredBagPath)
}
}
// Delete temp log dir after tests.
func teardownLoggerTest(config *models.Config) {
absLogDir := config.AbsLogDirectory()
if fileutil.LooksSafeToDelete(absLogDir, 12, 3) {
// Don't call remove all on "/" or "/usr" or anything like that.
os.RemoveAll(absLogDir)
} else {
fmt.Printf("Not deleting log dir '%s' because it looks dangerous.\n"+
"Delete that manually, if you thing it's safe.\n", absLogDir)
}
}
func (copier *DPNCopier) finishWithError(manifest *models.ReplicationManifest) {
xferId := "[unknown]"
fromNode := ""
if manifest.ReplicationTransfer != nil {
xferId = manifest.ReplicationTransfer.ReplicationId
fromNode = manifest.ReplicationTransfer.FromNode
} else if manifest.DPNWorkItem != nil {
xferId = manifest.DPNWorkItem.Identifier
}
if manifest.CopySummary.ErrorIsFatal {
msg := fmt.Sprintf("Xfer %s has fatal error: %s",
xferId, manifest.CopySummary.Errors[0])
copier.Context.MessageLog.Error(msg)
manifest.ReplicationTransfer.Cancelled = true
manifest.ReplicationTransfer.CancelReason = &manifest.CopySummary.Errors[0]
remoteClient := copier.RemoteClients[fromNode]
UpdateReplicationTransfer(copier.Context, remoteClient, manifest)
manifest.NsqMessage.Finish()
} else if manifest.CopySummary.AttemptNumber > copier.Context.Config.DPN.DPNCopyWorker.MaxAttempts {
msg := fmt.Sprintf("Attempt to copy Replication %s failed %d times. %s",
xferId, manifest.CopySummary.AttemptNumber, manifest.CopySummary.Errors[0])
copier.Context.MessageLog.Error(msg)
manifest.ReplicationTransfer.Cancelled = true
manifest.ReplicationTransfer.CancelReason = &msg
remoteClient := copier.RemoteClients[fromNode]
UpdateReplicationTransfer(copier.Context, remoteClient, manifest)
manifest.NsqMessage.Finish()
} else {
msg := fmt.Sprintf("Requeueing xfer %s due to non-fatal error: %s",
xferId, manifest.CopySummary.Errors[0])
copier.Context.MessageLog.Warning(msg)
manifest.NsqMessage.Requeue(1 * time.Minute)
}
// Delete the file, which may not even have been completely copied.
if fileutil.LooksSafeToDelete(manifest.LocalPath, 12, 3) {
os.Remove(manifest.LocalPath)
}
manifest.CopySummary.Finish()
// Tell Pharos what happened, and then dump the JSON to a log file.
*manifest.DPNWorkItem.Note = "Copy failed."
SaveDPNWorkItemState(copier.Context, manifest, manifest.CopySummary)
LogReplicationJson(manifest, copier.Context.JsonLog)
}
func (storer *DPNIngestStorer) finishWithSuccess(manifest *models.DPNIngestManifest) {
storer.Context.MessageLog.Info("Bag %s (%s) stored at %s",
manifest.DPNBag.UUID,
manifest.DPNBag.LocalId,
manifest.StorageURL)
manifest.WorkItem.Note = "Bag copied to long-term storage"
manifest.WorkItem.Stage = constants.StageRecord
manifest.WorkItem.StageStartedAt = nil
manifest.WorkItem.Status = constants.StatusPending
manifest.WorkItem.Node = ""
manifest.WorkItem.Pid = 0
SaveWorkItem(storer.Context, manifest, manifest.StoreSummary)
manifest.StoreSummary.Finish()
SaveWorkItemState(storer.Context, manifest, manifest.StoreSummary)
// Push this WorkItem to the next NSQ topic.
storer.Context.MessageLog.Info("Pushing %s (DPN bag %s) to NSQ topic %s",
manifest.DPNBag.LocalId, manifest.DPNBag.UUID,
storer.Context.Config.DPN.DPNIngestRecordWorker.NsqTopic)
PushToQueue(storer.Context, manifest, manifest.StoreSummary,
storer.Context.Config.DPN.DPNIngestRecordWorker.NsqTopic)
if manifest.PackageSummary.HasErrors() {
storer.Context.MessageLog.Error(manifest.PackageSummary.Errors[0])
}
if fileutil.LooksSafeToDelete(manifest.LocalTarFile, 12, 3) {
err := os.Remove(manifest.LocalTarFile)
if err != nil {
storer.Context.MessageLog.Error("Failed to delete %s after upload",
manifest.LocalTarFile)
}
}
// Tell NSQ we're done storing this.
manifest.NsqMessage.Finish()
}
func (packager *DPNPackager) finishWithError(manifest *models.DPNIngestManifest) {
// Log what happened.
msg := "Ingest failed. See ingest manifest."
packager.Context.MessageLog.Error(manifest.PackageSummary.AllErrorsAsString())
if manifest.PackageSummary.ErrorIsFatal {
msg := fmt.Sprintf("Ingest error for %s is fatal. Will not retry.",
manifest.WorkItem.ObjectIdentifier)
packager.Context.MessageLog.Error(msg)
manifest.NsqMessage.Finish()
manifest.WorkItem.Status = constants.StatusFailed
manifest.WorkItem.Outcome = "Failed due to fatal error"
manifest.WorkItem.Retry = false
manifest.WorkItem.NeedsAdminReview = true
} else if manifest.PackageSummary.AttemptNumber > packager.Context.Config.DPN.DPNPackageWorker.MaxAttempts {
msg := fmt.Sprintf("Giving up on ingest for %s after %d attempts.",
manifest.WorkItem.ObjectIdentifier, manifest.PackageSummary.AttemptNumber)
packager.Context.MessageLog.Error(msg)
manifest.NsqMessage.Finish()
manifest.WorkItem.Status = constants.StatusFailed
manifest.WorkItem.Outcome = "Failed after too many attempts with transient errors"
manifest.WorkItem.Retry = false
manifest.WorkItem.NeedsAdminReview = true
} else {
msg := fmt.Sprintf("Will retry ingest for %s",
manifest.WorkItem.ObjectIdentifier)
packager.Context.MessageLog.Warning(msg)
manifest.NsqMessage.Requeue(1 * time.Minute)
manifest.WorkItem.Status = constants.StatusPending
manifest.WorkItem.Outcome = "Pending retry after transient errors"
manifest.WorkItem.Retry = true
}
// Set the WorkItem fields and save to Pharos
manifest.WorkItem.Note = msg
manifest.WorkItem.Node = "" // no worker is working on this now
manifest.WorkItem.Pid = 0 // no process is working on this
// Delete the folder containing the bag we were building,
// And delete the tar file too, if it exists.
if fileutil.LooksSafeToDelete(manifest.LocalDir, 12, 3) {
err := os.RemoveAll(manifest.LocalDir)
if err != nil {
manifest.PackageSummary.AddError("Could not delete bag directory %s: %v",
manifest.LocalDir, err)
}
}
err := os.Remove(manifest.LocalTarFile)
if err != nil {
manifest.PackageSummary.AddError("Could not delete tar file %s: %v",
manifest.LocalTarFile, err)
}
// Save info to Pharos so the next worker knows what's what.
if manifest.WorkItem != nil {
SaveWorkItem(packager.Context, manifest, manifest.PackageSummary)
}
manifest.PackageSummary.Finish()
if manifest.WorkItemState != nil {
SaveWorkItemState(packager.Context, manifest, manifest.PackageSummary)
}
}
func TestLooksSafeToDelete(t *testing.T) {
assert.True(t, fileutil.LooksSafeToDelete("/mnt/apt/data/some_dir", 15, 3))
assert.False(t, fileutil.LooksSafeToDelete("/usr/local", 12, 3))
}