// buildTransferRequests builds the DPN ReplicationTransfer requests
// that we need to create with the new DPN bag. The ReplicationTransfers
// will be attached to the DPNIngestManifest and saved in our local DPN
// REST server, after we save the DPN bag.
func (recorder *DPNIngestRecorder) buildTransferRequests(manifest *models.DPNIngestManifest) {
	localNode := recorder.getLocalNode(manifest)
	if manifest.RecordSummary.HasErrors() {
		return
	}
	remoteNodes := recorder.chooseReplicationNodes(manifest, localNode)
	if manifest.RecordSummary.HasErrors() {
		return
	}
	for _, toNode := range remoteNodes {
		domain, err := localNode.FQDN()
		if err != nil {
			manifest.RecordSummary.AddError("Can't get FQDN from node %s. APIRoot is %s. "+
				"Error is %v", localNode.Namespace, localNode.APIRoot, err)
			return
		}
		link := fmt.Sprintf("dpn.%s@%s:outbound/%s.tar",
			toNode, domain, manifest.DPNBag.UUID)
		xfer, err := manifest.BuildReplicationTransfer(
			recorder.Context.Config.DPN.LocalNode, toNode, link)
		if err != nil {
			manifest.RecordSummary.AddError("Error building ReplicationTransfer: %v ", err)
			return
		}
		manifest.ReplicationTransfers = append(manifest.ReplicationTransfers, xfer)
	}
}
Beispiel #2
0
// buildDPNBag bags the DPNBag object that will eventually be entered into
// the DPN registry. This is not the bag itself, just the DPN registry entry.
func (packager *DPNPackager) buildDPNBag(manifest *models.DPNIngestManifest) {
	packager.Context.MessageLog.Info("Building DPN bag record for %s",
		manifest.IntellectualObject.Identifier)
	depositingInstitution := packager.getInstitution(manifest)
	if depositingInstitution == nil {
		return
	}
	manifest.DPNBag = models.NewDPNBag(
		manifest.IntellectualObject.Identifier,
		depositingInstitution.DPNUUID,
		packager.Context.Config.DPN.LocalNode)

	// Calculate the sha256 digest of the tag manifest. This is used for
	// validating bag transfers in DPN. Note that we are NOT using a
	// nonce when we call shaHash.Sum(nil). Though the DPN spec allows
	// us to use a nonce, no nodes are using nonces as of late 2016.
	tagManifestFile := filepath.Join(manifest.LocalDir, "tagmanifest-sha256.txt")
	if !fileutil.FileExists(tagManifestFile) {
		manifest.PackageSummary.AddError("Cannot find tag manifest %s", tagManifestFile)
		return
	}
	reader, err := os.Open(tagManifestFile)
	if err != nil {
		manifest.PackageSummary.AddError("Cannot read tag manifest at %s: %v",
			tagManifestFile, err)
		return
	}
	defer reader.Close()
	shaHash := sha256.New()
	io.Copy(shaHash, reader)
	tagManifestDigest := fmt.Sprintf("%x", shaHash.Sum(nil))

	// Now create the MessageDigest for this bag, with the tag manifest
	// checksum that will be used to verify transfers. When a remote
	// node copies this bag to fulfill a replication request, we expect
	// the node to return this fixity value as proof that it received
	// a valid copy of the bag.
	digest := &models.MessageDigest{
		Bag:       manifest.DPNBag.UUID,
		Algorithm: constants.AlgSha256,
		Node:      packager.Context.Config.DPN.LocalNode,
		Value:     tagManifestDigest,
		CreatedAt: time.Now().UTC(),
	}
	manifest.DPNBag.MessageDigests = append(manifest.DPNBag.MessageDigests, digest)

	// Now that we have a valid DPN bag object, we can name the tar file.
	// According to the DPN spec, the tar file name should be the bag's
	// UUID plus a ".tar" extension.
	parentOfBagDir := filepath.Dir(manifest.LocalDir)
	manifest.LocalTarFile = filepath.Join(parentOfBagDir, manifest.DPNBag.UUID+".tar")
}
Beispiel #3
0
func (storer *DPNIngestStorer) copyToLongTermStorage(manifest *models.DPNIngestManifest) {
	manifest.StoreSummary.ClearErrors()
	upload := apt_network.NewS3Upload(
		constants.AWSVirginia,
		storer.Context.Config.DPN.DPNPreservationBucket,
		fmt.Sprintf("%s.tar", manifest.DPNBag.UUID),
		"application/x-tar")
	upload.AddMetadata("from_node", manifest.DPNBag.IngestNode)
	upload.AddMetadata("transfer_id", fmt.Sprintf("None - Ingested Directly at %s", manifest.DPNBag.IngestNode))
	upload.AddMetadata("member", manifest.DPNBag.Member)
	upload.AddMetadata("local_id", manifest.DPNBag.LocalId)
	upload.AddMetadata("version", fmt.Sprintf("%d", manifest.DPNBag.Version))
	reader, err := os.Open(manifest.LocalTarFile)
	if reader != nil {
		defer reader.Close()
	}
	if err != nil {
		manifest.StoreSummary.AddError("Error opening reader for tar file %s: %v",
			manifest.LocalTarFile, err)
		return
	}
	upload.Send(reader)
	if upload.ErrorMessage != "" {
		manifest.StoreSummary.AddError("Error uploading tar file %s: %s",
			manifest.LocalTarFile, upload.ErrorMessage)
		return
	}
	manifest.StorageURL = upload.Response.Location
}
// addEventsToManifest adds PremisEvents to the ingest manifest, if they
// don't already exist.
func (recorder *DPNIngestRecorder) addEventsToManifest(manifest *models.DPNIngestManifest) {
	if manifest.DPNIdentifierEvent == nil {
		event, err := manifest.BuildDPNIdentifierEvent()
		if err != nil {
			manifest.RecordSummary.AddError(err.Error())
		}
		manifest.DPNIdentifierEvent = event
	}
	if manifest.DPNIngestEvent == nil {
		event, err := manifest.BuildDPNIngestEvent()
		if err != nil {
			manifest.RecordSummary.AddError(err.Error())
		}
		manifest.DPNIngestEvent = event
	}
}