Example #1
0
func (packager *DPNPackager) finishWithSuccess(manifest *models.DPNIngestManifest) {
	// Tell Pharos we're done with this item and save the work state.
	packager.Context.MessageLog.Info("Packaging succeeded for %s", manifest.WorkItem.ObjectIdentifier)
	manifest.WorkItem.Stage = constants.StageStore
	manifest.WorkItem.Status = constants.StatusPending
	manifest.WorkItem.Note = "Packaging completed, awaiting storage"
	manifest.WorkItem.Node = ""    // no worker is working on this now
	manifest.WorkItem.Pid = 0      // no process is working on this
	manifest.WorkItem.Retry = true // just in case this had been false
	SaveWorkItem(packager.Context, manifest, manifest.PackageSummary)
	manifest.PackageSummary.Finish()
	SaveWorkItemState(packager.Context, manifest, manifest.PackageSummary)

	// Delete the working directory where we built the bag
	if fileutil.LooksSafeToDelete(manifest.LocalDir, 12, 3) {
		os.RemoveAll(manifest.LocalDir)
	}

	// Push this WorkItem to the next NSQ topic.
	packager.Context.MessageLog.Info("Pushing %s (DPN bag %s) to NSQ topic %s",
		manifest.IntellectualObject.Identifier, manifest.DPNBag.UUID,
		packager.Context.Config.DPN.DPNIngestStoreWorker.NsqTopic)
	PushToQueue(packager.Context, manifest, manifest.PackageSummary,
		packager.Context.Config.DPN.DPNIngestStoreWorker.NsqTopic)
	if manifest.PackageSummary.HasErrors() {
		packager.Context.MessageLog.Error(manifest.PackageSummary.Errors[0])
	}

	// Tell NSQ we're done packaging this.
	manifest.NsqMessage.Finish()
}
Example #2
0
// DeleteBagFromStaging deletes the bag from the staging area, and releases
// the reserved storage from the volume manager. This deletes both the tarred
// and untarred version of the bag, if they both exist.
func DeleteBagFromStaging(ingestState *models.IngestState, _context *context.Context, activeResult *models.WorkSummary) {
	tarFile := ingestState.IngestManifest.Object.IngestTarFilePath
	if tarFile != "" && fileutil.FileExists(tarFile) {
		_context.MessageLog.Info("Deleting %s", tarFile)
		err := os.Remove(tarFile)
		if err != nil {
			_context.MessageLog.Warning(err.Error())
		}
		err = _context.VolumeClient.Release(tarFile)
		if err != nil {
			_context.MessageLog.Warning(err.Error())
		}
	} else {
		_context.MessageLog.Info("Skipping deletion of %s: file does not exist", tarFile)
	}

	untarredBagPath := ingestState.IngestManifest.Object.IngestUntarredPath
	looksSafeToDelete := fileutil.LooksSafeToDelete(untarredBagPath, 12, 3)
	if fileutil.FileExists(untarredBagPath) && looksSafeToDelete {
		_context.MessageLog.Info("Deleting untarred bag at %s", untarredBagPath)
		err := os.RemoveAll(untarredBagPath)
		if err != nil {
			_context.MessageLog.Warning(err.Error())
		}
		err = _context.VolumeClient.Release(untarredBagPath)
		if err != nil {
			_context.MessageLog.Warning(err.Error())
		}
	} else {
		_context.MessageLog.Info("Skipping deletion of untarred bag dir at %s: "+
			"Directory does not exist, or is unsafe to delete.", untarredBagPath)
	}

}
Example #3
0
// Delete temp log dir after tests.
func teardownLoggerTest(config *models.Config) {
	absLogDir := config.AbsLogDirectory()
	if fileutil.LooksSafeToDelete(absLogDir, 12, 3) {
		// Don't call remove all on "/" or "/usr" or anything like that.
		os.RemoveAll(absLogDir)
	} else {
		fmt.Printf("Not deleting log dir '%s' because it looks dangerous.\n"+
			"Delete that manually, if you thing it's safe.\n", absLogDir)
	}
}
Example #4
0
func (copier *DPNCopier) finishWithError(manifest *models.ReplicationManifest) {
	xferId := "[unknown]"
	fromNode := ""
	if manifest.ReplicationTransfer != nil {
		xferId = manifest.ReplicationTransfer.ReplicationId
		fromNode = manifest.ReplicationTransfer.FromNode
	} else if manifest.DPNWorkItem != nil {
		xferId = manifest.DPNWorkItem.Identifier
	}
	if manifest.CopySummary.ErrorIsFatal {
		msg := fmt.Sprintf("Xfer %s has fatal error: %s",
			xferId, manifest.CopySummary.Errors[0])
		copier.Context.MessageLog.Error(msg)
		manifest.ReplicationTransfer.Cancelled = true
		manifest.ReplicationTransfer.CancelReason = &manifest.CopySummary.Errors[0]
		remoteClient := copier.RemoteClients[fromNode]
		UpdateReplicationTransfer(copier.Context, remoteClient, manifest)
		manifest.NsqMessage.Finish()
	} else if manifest.CopySummary.AttemptNumber > copier.Context.Config.DPN.DPNCopyWorker.MaxAttempts {
		msg := fmt.Sprintf("Attempt to copy Replication %s failed %d times. %s",
			xferId, manifest.CopySummary.AttemptNumber, manifest.CopySummary.Errors[0])
		copier.Context.MessageLog.Error(msg)
		manifest.ReplicationTransfer.Cancelled = true
		manifest.ReplicationTransfer.CancelReason = &msg
		remoteClient := copier.RemoteClients[fromNode]
		UpdateReplicationTransfer(copier.Context, remoteClient, manifest)
		manifest.NsqMessage.Finish()
	} else {
		msg := fmt.Sprintf("Requeueing xfer %s due to non-fatal error: %s",
			xferId, manifest.CopySummary.Errors[0])
		copier.Context.MessageLog.Warning(msg)
		manifest.NsqMessage.Requeue(1 * time.Minute)
	}
	// Delete the file, which may not even have been completely copied.
	if fileutil.LooksSafeToDelete(manifest.LocalPath, 12, 3) {
		os.Remove(manifest.LocalPath)
	}
	manifest.CopySummary.Finish()

	// Tell Pharos what happened, and then dump the JSON to a log file.
	*manifest.DPNWorkItem.Note = "Copy failed."
	SaveDPNWorkItemState(copier.Context, manifest, manifest.CopySummary)
	LogReplicationJson(manifest, copier.Context.JsonLog)
}
Example #5
0
func (storer *DPNIngestStorer) finishWithSuccess(manifest *models.DPNIngestManifest) {
	storer.Context.MessageLog.Info("Bag %s (%s) stored at %s",
		manifest.DPNBag.UUID,
		manifest.DPNBag.LocalId,
		manifest.StorageURL)

	manifest.WorkItem.Note = "Bag copied to long-term storage"
	manifest.WorkItem.Stage = constants.StageRecord
	manifest.WorkItem.StageStartedAt = nil
	manifest.WorkItem.Status = constants.StatusPending
	manifest.WorkItem.Node = ""
	manifest.WorkItem.Pid = 0
	SaveWorkItem(storer.Context, manifest, manifest.StoreSummary)
	manifest.StoreSummary.Finish()
	SaveWorkItemState(storer.Context, manifest, manifest.StoreSummary)

	// Push this WorkItem to the next NSQ topic.
	storer.Context.MessageLog.Info("Pushing %s (DPN bag %s) to NSQ topic %s",
		manifest.DPNBag.LocalId, manifest.DPNBag.UUID,
		storer.Context.Config.DPN.DPNIngestRecordWorker.NsqTopic)
	PushToQueue(storer.Context, manifest, manifest.StoreSummary,
		storer.Context.Config.DPN.DPNIngestRecordWorker.NsqTopic)
	if manifest.PackageSummary.HasErrors() {
		storer.Context.MessageLog.Error(manifest.PackageSummary.Errors[0])
	}

	if fileutil.LooksSafeToDelete(manifest.LocalTarFile, 12, 3) {
		err := os.Remove(manifest.LocalTarFile)
		if err != nil {
			storer.Context.MessageLog.Error("Failed to delete %s after upload",
				manifest.LocalTarFile)
		}
	}

	// Tell NSQ we're done storing this.
	manifest.NsqMessage.Finish()
}
Example #6
0
func (packager *DPNPackager) finishWithError(manifest *models.DPNIngestManifest) {
	// Log what happened.
	msg := "Ingest failed. See ingest manifest."
	packager.Context.MessageLog.Error(manifest.PackageSummary.AllErrorsAsString())
	if manifest.PackageSummary.ErrorIsFatal {
		msg := fmt.Sprintf("Ingest error for %s is fatal. Will not retry.",
			manifest.WorkItem.ObjectIdentifier)
		packager.Context.MessageLog.Error(msg)
		manifest.NsqMessage.Finish()
		manifest.WorkItem.Status = constants.StatusFailed
		manifest.WorkItem.Outcome = "Failed due to fatal error"
		manifest.WorkItem.Retry = false
		manifest.WorkItem.NeedsAdminReview = true
	} else if manifest.PackageSummary.AttemptNumber > packager.Context.Config.DPN.DPNPackageWorker.MaxAttempts {
		msg := fmt.Sprintf("Giving up on ingest for %s after %d attempts.",
			manifest.WorkItem.ObjectIdentifier, manifest.PackageSummary.AttemptNumber)
		packager.Context.MessageLog.Error(msg)
		manifest.NsqMessage.Finish()
		manifest.WorkItem.Status = constants.StatusFailed
		manifest.WorkItem.Outcome = "Failed after too many attempts with transient errors"
		manifest.WorkItem.Retry = false
		manifest.WorkItem.NeedsAdminReview = true
	} else {
		msg := fmt.Sprintf("Will retry ingest for %s",
			manifest.WorkItem.ObjectIdentifier)
		packager.Context.MessageLog.Warning(msg)
		manifest.NsqMessage.Requeue(1 * time.Minute)
		manifest.WorkItem.Status = constants.StatusPending
		manifest.WorkItem.Outcome = "Pending retry after transient errors"
		manifest.WorkItem.Retry = true
	}

	// Set the WorkItem fields and save to Pharos
	manifest.WorkItem.Note = msg
	manifest.WorkItem.Node = "" // no worker is working on this now
	manifest.WorkItem.Pid = 0   // no process is working on this

	// Delete the folder containing the bag we were building,
	// And delete the tar file too, if it exists.
	if fileutil.LooksSafeToDelete(manifest.LocalDir, 12, 3) {
		err := os.RemoveAll(manifest.LocalDir)
		if err != nil {
			manifest.PackageSummary.AddError("Could not delete bag directory %s: %v",
				manifest.LocalDir, err)
		}
	}
	err := os.Remove(manifest.LocalTarFile)
	if err != nil {
		manifest.PackageSummary.AddError("Could not delete tar file %s: %v",
			manifest.LocalTarFile, err)
	}

	// Save info to Pharos so the next worker knows what's what.
	if manifest.WorkItem != nil {
		SaveWorkItem(packager.Context, manifest, manifest.PackageSummary)
	}
	manifest.PackageSummary.Finish()
	if manifest.WorkItemState != nil {
		SaveWorkItemState(packager.Context, manifest, manifest.PackageSummary)
	}
}
Example #7
0
func TestLooksSafeToDelete(t *testing.T) {
	assert.True(t, fileutil.LooksSafeToDelete("/mnt/apt/data/some_dir", 15, 3))
	assert.False(t, fileutil.LooksSafeToDelete("/usr/local", 12, 3))
}