// buildTransferRequests builds the DPN ReplicationTransfer requests
// that we need to create with the new DPN bag. The ReplicationTransfers
// will be attached to the DPNIngestManifest and saved in our local DPN
// REST server, after we save the DPN bag.
func (recorder *DPNIngestRecorder) buildTransferRequests(manifest *models.DPNIngestManifest) {
localNode := recorder.getLocalNode(manifest)
if manifest.RecordSummary.HasErrors() {
return
}
remoteNodes := recorder.chooseReplicationNodes(manifest, localNode)
if manifest.RecordSummary.HasErrors() {
return
}
for _, toNode := range remoteNodes {
domain, err := localNode.FQDN()
if err != nil {
manifest.RecordSummary.AddError("Can't get FQDN from node %s. APIRoot is %s. "+
"Error is %v", localNode.Namespace, localNode.APIRoot, err)
return
}
link := fmt.Sprintf("dpn.%s@%s:outbound/%s.tar",
toNode, domain, manifest.DPNBag.UUID)
xfer, err := manifest.BuildReplicationTransfer(
recorder.Context.Config.DPN.LocalNode, toNode, link)
if err != nil {
manifest.RecordSummary.AddError("Error building ReplicationTransfer: %v ", err)
return
}
manifest.ReplicationTransfers = append(manifest.ReplicationTransfers, xfer)
}
}
// buildDPNBag bags the DPNBag object that will eventually be entered into
// the DPN registry. This is not the bag itself, just the DPN registry entry.
func (packager *DPNPackager) buildDPNBag(manifest *models.DPNIngestManifest) {
packager.Context.MessageLog.Info("Building DPN bag record for %s",
manifest.IntellectualObject.Identifier)
depositingInstitution := packager.getInstitution(manifest)
if depositingInstitution == nil {
return
}
manifest.DPNBag = models.NewDPNBag(
manifest.IntellectualObject.Identifier,
depositingInstitution.DPNUUID,
packager.Context.Config.DPN.LocalNode)
// Calculate the sha256 digest of the tag manifest. This is used for
// validating bag transfers in DPN. Note that we are NOT using a
// nonce when we call shaHash.Sum(nil). Though the DPN spec allows
// us to use a nonce, no nodes are using nonces as of late 2016.
tagManifestFile := filepath.Join(manifest.LocalDir, "tagmanifest-sha256.txt")
if !fileutil.FileExists(tagManifestFile) {
manifest.PackageSummary.AddError("Cannot find tag manifest %s", tagManifestFile)
return
}
reader, err := os.Open(tagManifestFile)
if err != nil {
manifest.PackageSummary.AddError("Cannot read tag manifest at %s: %v",
tagManifestFile, err)
return
}
defer reader.Close()
shaHash := sha256.New()
io.Copy(shaHash, reader)
tagManifestDigest := fmt.Sprintf("%x", shaHash.Sum(nil))
// Now create the MessageDigest for this bag, with the tag manifest
// checksum that will be used to verify transfers. When a remote
// node copies this bag to fulfill a replication request, we expect
// the node to return this fixity value as proof that it received
// a valid copy of the bag.
digest := &models.MessageDigest{
Bag: manifest.DPNBag.UUID,
Algorithm: constants.AlgSha256,
Node: packager.Context.Config.DPN.LocalNode,
Value: tagManifestDigest,
CreatedAt: time.Now().UTC(),
}
manifest.DPNBag.MessageDigests = append(manifest.DPNBag.MessageDigests, digest)
// Now that we have a valid DPN bag object, we can name the tar file.
// According to the DPN spec, the tar file name should be the bag's
// UUID plus a ".tar" extension.
parentOfBagDir := filepath.Dir(manifest.LocalDir)
manifest.LocalTarFile = filepath.Join(parentOfBagDir, manifest.DPNBag.UUID+".tar")
}
func (storer *DPNIngestStorer) copyToLongTermStorage(manifest *models.DPNIngestManifest) {
manifest.StoreSummary.ClearErrors()
upload := apt_network.NewS3Upload(
constants.AWSVirginia,
storer.Context.Config.DPN.DPNPreservationBucket,
fmt.Sprintf("%s.tar", manifest.DPNBag.UUID),
"application/x-tar")
upload.AddMetadata("from_node", manifest.DPNBag.IngestNode)
upload.AddMetadata("transfer_id", fmt.Sprintf("None - Ingested Directly at %s", manifest.DPNBag.IngestNode))
upload.AddMetadata("member", manifest.DPNBag.Member)
upload.AddMetadata("local_id", manifest.DPNBag.LocalId)
upload.AddMetadata("version", fmt.Sprintf("%d", manifest.DPNBag.Version))
reader, err := os.Open(manifest.LocalTarFile)
if reader != nil {
defer reader.Close()
}
if err != nil {
manifest.StoreSummary.AddError("Error opening reader for tar file %s: %v",
manifest.LocalTarFile, err)
return
}
upload.Send(reader)
if upload.ErrorMessage != "" {
manifest.StoreSummary.AddError("Error uploading tar file %s: %s",
manifest.LocalTarFile, upload.ErrorMessage)
return
}
manifest.StorageURL = upload.Response.Location
}
// addEventsToManifest adds PremisEvents to the ingest manifest, if they
// don't already exist.
func (recorder *DPNIngestRecorder) addEventsToManifest(manifest *models.DPNIngestManifest) {
if manifest.DPNIdentifierEvent == nil {
event, err := manifest.BuildDPNIdentifierEvent()
if err != nil {
manifest.RecordSummary.AddError(err.Error())
}
manifest.DPNIdentifierEvent = event
}
if manifest.DPNIngestEvent == nil {
event, err := manifest.BuildDPNIngestEvent()
if err != nil {
manifest.RecordSummary.AddError(err.Error())
}
manifest.DPNIngestEvent = event
}
}
