func TestManifestBuildDPNIdentifierEvent(t *testing.T) {
nsqMessage := testutil.MakeNsqMessage("999")
manifest := models.NewDPNIngestManifest(nsqMessage)
manifest.StorageURL = "https://example.com/my_bag.tar"
// Should give error, since DPNBag is nil
event, err := manifest.BuildDPNIdentifierEvent()
require.NotNil(t, err)
manifest.DPNBag = testutil.MakeDPNBag()
event, err = manifest.BuildDPNIdentifierEvent()
require.Nil(t, err)
require.NotNil(t, event)
assert.True(t, util.LooksLikeUUID(event.Identifier))
assert.Equal(t, constants.EventIdentifierAssignment, event.EventType)
assert.False(t, event.DateTime.IsZero())
assert.Equal(t, "Item assigned DPN UUID", event.Detail)
assert.Equal(t, constants.StatusSuccess, event.Outcome)
assert.Equal(t, manifest.DPNBag.UUID, event.OutcomeDetail)
assert.Equal(t, "APTrust exchange using Satori go.uuid", event.Object)
assert.Equal(t, "https://github.com/satori/go.uuid", event.Agent)
assert.True(t, strings.Contains(event.OutcomeInformation, manifest.DPNBag.UUID))
assert.Equal(t, manifest.DPNBag.LocalId, event.IntellectualObjectIdentifier)
}
func TestManifestBuildReplicationTransfer(t *testing.T) {
nsqMessage := testutil.MakeNsqMessage("999")
manifest := models.NewDPNIngestManifest(nsqMessage)
fromNode := "aptrust"
toNode := "chron"
link := "[email protected]:outbound/1234567.tar"
xfer, err := manifest.BuildReplicationTransfer(fromNode, toNode, link)
// No DPNBag attached to manifest. Should get error.
require.NotNil(t, err)
manifest.DPNBag = testutil.MakeDPNBag()
xfer, err = manifest.BuildReplicationTransfer(fromNode, toNode, link)
require.Nil(t, err)
require.NotNil(t, xfer)
assert.Equal(t, fromNode, xfer.FromNode)
assert.Equal(t, toNode, xfer.ToNode)
assert.Equal(t, manifest.DPNBag.UUID, xfer.Bag)
assert.True(t, util.LooksLikeUUID(xfer.ReplicationId))
assert.Equal(t, constants.AlgSha256, xfer.FixityAlgorithm)
assert.Nil(t, xfer.FixityNonce)
assert.Nil(t, xfer.FixityValue)
assert.Equal(t, "rsync", xfer.Protocol)
assert.Equal(t, link, xfer.Link)
assert.False(t, xfer.CreatedAt.IsZero())
assert.False(t, xfer.UpdatedAt.IsZero())
}
// Returns true if the GenericFile IngestUUID is present and looks good.
func (storer *APTStorer) uuidPresent(ingestState *models.IngestState, gf *models.GenericFile) bool {
if !util.LooksLikeUUID(gf.IngestUUID) {
ingestState.IngestManifest.StoreResult.AddError("Cannot save %s to S3/Glacier because "+
"GenericFile.IngestUUID (%s) is missing or invalid",
gf.Identifier, gf.IngestUUID)
ingestState.IngestManifest.StoreResult.ErrorIsFatal = true
return false
}
return true
}
func fetcherTestSpecifics(t *testing.T, ingestManifest *models.IngestManifest) {
obj := ingestManifest.Object
assert.Equal(t, "test.edu/example.edu.tagsample_good", obj.Identifier)
assert.Equal(t, "example.edu.tagsample_good", obj.BagName)
assert.Equal(t, "test.edu", obj.Institution)
assert.NotEqual(t, 0, obj.InstitutionId)
assert.Equal(t, "Tag Sample (Good)", obj.Title)
assert.Equal(t, "Bag of goodies", obj.Description)
assert.Equal(t, "Institution", obj.Access)
assert.Equal(t, "uva-internal-id-0001", obj.AltIdentifier)
assert.Equal(t, "aptrust.receiving.test.test.edu", obj.IngestS3Bucket)
assert.Equal(t, "example.edu.tagsample_good.tar", obj.IngestS3Key)
assert.Equal(t, "manifest-md5.txt", obj.IngestManifests[0])
assert.Equal(t, "manifest-sha256.txt", obj.IngestManifests[1])
assert.Equal(t, "tagmanifest-md5.txt", obj.IngestTagManifests[0])
assert.Equal(t, "tagmanifest-sha256.txt", obj.IngestTagManifests[1])
assert.Empty(t, obj.IngestFilesIgnored)
assert.Equal(t, "example.edu.tagsample_good", obj.IngestTopLevelDirNames[0])
assert.Empty(t, obj.IngestErrorMessage)
assert.Equal(t, 10, len(obj.IngestTags))
assert.Equal(t, "bag-info.txt", obj.IngestTags[5].SourceFile)
assert.Equal(t, "Bag-Group-Identifier", obj.IngestTags[5].Label)
assert.Equal(t, "Charley Horse", obj.IngestTags[5].Value)
assert.Empty(t, obj.PremisEvents)
assert.Equal(t, 16, len(obj.GenericFiles))
gf := obj.GenericFiles[0]
assert.Equal(t, 0, gf.Id)
assert.Equal(t, "test.edu/example.edu.tagsample_good/aptrust-info.txt", gf.Identifier)
assert.Equal(t, 0, gf.IntellectualObjectId)
assert.Equal(t, "test.edu/example.edu.tagsample_good", gf.IntellectualObjectIdentifier)
assert.Equal(t, "text/plain", gf.FileFormat)
assert.EqualValues(t, 45, gf.Size)
assert.EqualValues(t, "0001-01-01T00:00:00Z", gf.FileCreated.Format(time.RFC3339))
assert.Equal(t, "2016-03-21T11:01:51-04:00", gf.FileModified.Format(time.RFC3339))
assert.Empty(t, gf.Checksums)
assert.Empty(t, gf.PremisEvents)
assert.Equal(t, "tag_file", gf.IngestFileType)
assert.Equal(t, "bd8be664c790a9175e9d2fe90b40d502", gf.IngestMd5)
assert.False(t, gf.IngestMd5GeneratedAt.IsZero())
assert.False(t, gf.IngestMd5VerifiedAt.IsZero())
assert.Equal(t, "49dd23cdaf644e60629f01d6ebea770cd1c6229ff89f14a6c030a50c48b6ba27", gf.IngestSha256)
assert.False(t, gf.IngestSha256GeneratedAt.IsZero())
assert.False(t, gf.IngestSha256VerifiedAt.IsZero())
assert.True(t, util.LooksLikeUUID(gf.IngestUUID))
assert.False(t, gf.IngestUUIDGeneratedAt.IsZero())
assert.True(t, gf.IngestStoredAt.IsZero())
assert.True(t, gf.IngestReplicatedAt.IsZero())
assert.True(t, gf.IngestNeedsSave)
assert.EqualValues(t, 502, gf.IngestFileUid)
assert.EqualValues(t, 20, gf.IngestFileGid)
}
// TestInteObjDPNUUID makes sure the DPN UUID was set on the
// IntellectualObject record in Pharos.
func TestInteObjDPNUUID(t *testing.T) {
if !apt_testutil.ShouldRunIntegrationTests() {
t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
}
_context, err := apt_testutil.GetContext("integration.json")
require.Nil(t, err)
for _, s3Key := range apt_testutil.INTEGRATION_GOOD_BAGS[0:7] {
tar := strings.Replace(s3Key, "aptrust.receiving.test.", "", 1)
objIdentifier := strings.Replace(tar, ".tar", "", 1)
resp := _context.PharosClient.IntellectualObjectGet(objIdentifier, false, false)
require.Nil(t, resp.Error)
obj := resp.IntellectualObject()
require.NotNil(t, obj)
// DPNUUID is null in fixture data. It should be set after DPN ingest.
assert.True(t, util.LooksLikeUUID(obj.DPNUUID))
}
}
// Returns the UUID of an existing GenericFile. The UUID is the last component
// of the S3 storage URL. When we are updating an existing GenericFile, we want
// to overwrite the object in S3/Glacier rather than writing a new one and
// leaving the old one hanging around. To overwrite it, we must know its UUID.
func (storer *APTStorer) getUuidOfExistingFile(gfIdentifier string) (string, error) {
storer.Context.MessageLog.Info("Checking Pharos for existing UUID for GenericFile %s",
gfIdentifier)
resp := storer.Context.PharosClient.GenericFileGet(gfIdentifier)
if resp.Error != nil {
return "", resp.Error
}
uuid := ""
existingGenericFile := resp.GenericFile()
if resp.Error != nil {
return "", fmt.Errorf("Pharos cannot find supposedly existing GenericFile '%s'", gfIdentifier)
}
parts := strings.Split(existingGenericFile.URI, "/")
uuid = parts[len(parts)-1]
if !util.LooksLikeUUID(uuid) {
return "", fmt.Errorf("Could not extract UUID from URI %s", existingGenericFile.URI)
}
return uuid, nil
}
func TestLooksLikeUUID(t *testing.T) {
if util.LooksLikeUUID("1552abf5-28f3-46a5-ba63-95302d08e209") == false {
t.Error("That was a valid UUID!")
}
if util.LooksLikeUUID("88198c5a-ec91-4ce1-bfcc-0f607ebdcca3") == false {
t.Error("That was a valid UUID!")
}
if util.LooksLikeUUID("88198C5A-EC91-4CE1-BFCC-0F607EBDCCA3") == false {
t.Error("That was a valid UUID!")
}
if util.LooksLikeUUID("88198c5a-ec91-4ce1-bfcc-0f607ebdccx3") == true {
t.Error("That was not a valid UUID!")
}
if util.LooksLikeUUID("88198c5a-ec91-4ce1-bfcc-0f6c") == true {
t.Error("That was not a valid UUID!")
}
if util.LooksLikeUUID("") == true {
t.Error("That was not a valid UUID! That was an empty string!")
}
}
func recordTestCommon(t *testing.T, bagName string, ingestManifest *models.IngestManifest) {
// Test some basic object properties
assert.NotEmpty(t, ingestManifest.WorkItemId, "WorkItemId should not be empty for %s", bagName)
assert.NotEmpty(t, ingestManifest.S3Bucket, "S3Bucket should not be empty for %s", bagName)
assert.NotEmpty(t, ingestManifest.S3Key, "S3Key should not be empty for %s", bagName)
assert.NotEmpty(t, ingestManifest.ETag, "ETag should not be empty for %s", bagName)
// Make sure the result has some basic info in RecordResult
assert.True(t, ingestManifest.RecordResult.Attempted,
"RecordResult.Attempted should be true for %s", bagName)
assert.True(t, ingestManifest.RecordResult.AttemptNumber > 0,
"RecordResult.AttemptNumber should be > 0 %s", bagName)
assert.NotEmpty(t, ingestManifest.RecordResult.StartedAt,
"RecordResult.StartedAt should not be empty for %s", bagName)
assert.NotEmpty(t, ingestManifest.RecordResult.FinishedAt,
"RecordResult.FinishedAt should not be empty for %s", bagName)
assert.Empty(t, ingestManifest.RecordResult.Errors,
"RecordResult.Errors should be empty for %s", bagName)
assert.True(t, ingestManifest.RecordResult.Retry,
"RecordResult.Retry should be true for %s", bagName)
// Make sure the result has some basic info in CleanupResult
assert.True(t, ingestManifest.CleanupResult.Attempted,
"CleanupResult.Attempted should be true for %s", bagName)
assert.True(t, ingestManifest.CleanupResult.AttemptNumber > 0,
"CleanupResult.AttemptNumber should be > 0 %s", bagName)
assert.NotEmpty(t, ingestManifest.CleanupResult.StartedAt,
"CleanupResult.StartedAt should not be empty for %s", bagName)
assert.NotEmpty(t, ingestManifest.CleanupResult.FinishedAt,
"CleanupResult.FinishedAt should not be empty for %s", bagName)
assert.Empty(t, ingestManifest.CleanupResult.Errors,
"CleanupResult.Errors should be empty for %s", bagName)
assert.True(t, ingestManifest.CleanupResult.Retry,
"CleanupResult.Retry should be true for %s", bagName)
// Make sure our IntellectualObject got all of its PremisEvents
obj := ingestManifest.Object
require.Equal(t, 4, len(obj.PremisEvents))
// Make sure this item was deleted from the receiving bucket
// after ingest completed.
assert.False(t, obj.IngestDeletedFromReceivingAt.IsZero(),
"Object %s was not deleted from receiving bucket", bagName)
assert.Empty(t, obj.IngestErrorMessage)
// Check the object-level events
creationEvents := obj.FindEventsByType(constants.EventCreation)
idEvents := obj.FindEventsByType(constants.EventIdentifierAssignment)
ingestEvents := obj.FindEventsByType(constants.EventIngestion)
accessEvents := obj.FindEventsByType(constants.EventAccessAssignment)
assert.Equal(t, 1, len(accessEvents), "Missing access event for %s", bagName)
assert.Equal(t, 1, len(creationEvents), "Missing creation event for %s", bagName)
assert.Equal(t, 1, len(idEvents), "Missing identifier assignment event for %s", bagName)
assert.Equal(t, 1, len(ingestEvents), "Missing ingest event for %s", bagName)
for _, event := range obj.PremisEvents {
assert.True(t, event.Id > 0, "Event %s was not saved for %s", event.EventType, obj.Identifier)
assert.True(t, event.IntellectualObjectId > 0,
"event.IntellectualObjectId not set for %s %s", event.EventType, obj.Identifier)
assert.False(t, event.DateTime.IsZero(),
"event.DateTime was not set for %s %s", event.EventType, obj.Identifier)
assert.False(t, event.CreatedAt.IsZero(),
"event.CreatedAt was not set for %s %s", event.EventType, obj.Identifier)
assert.False(t, event.UpdatedAt.IsZero(),
"event.UpdatedAt was not set for %s %s", event.EventType, obj.Identifier)
assert.True(t, util.LooksLikeUUID(event.Identifier),
"Identifier for %s %s doesn't look like a UUID", event.EventType, obj.Identifier)
assert.NotEmpty(t, event.EventType, "EventType missing for %s %s", obj.Identifier, event.Identifier)
assert.NotEmpty(t, event.Detail, "Detail is empty for %s %s", event.EventType, obj.Identifier)
assert.NotEmpty(t, event.Outcome, "Outcome is empty for %s %s", event.EventType, obj.Identifier)
assert.NotEmpty(t, event.OutcomeDetail,
"OutcomeDetail is empty for %s %s", event.EventType, obj.Identifier)
assert.NotEmpty(t, event.Object, "Object is empty for %s %s", event.EventType, obj.Identifier)
assert.NotEmpty(t, event.Agent, "Agent is empty for %s %s", event.EventType, obj.Identifier)
assert.NotEmpty(t, event.OutcomeInformation,
"OutcomeInformation is empty for %s %s", event.EventType, obj.Identifier)
assert.Equal(t, obj.Identifier, event.IntellectualObjectIdentifier,
"IntellectualObjectIdentifier is wrong for %s %s", event.EventType, obj.Identifier)
}
for _, gf := range obj.GenericFiles {
// Skip these checks for files that didn't need to be saved.
// Reasons for not needing to be saved:
// 1. File has a non-savable name, according to util.HasSavableName
// 2. File has not changed since last time we ingested this bag.
if !gf.IngestNeedsSave {
continue
}
// Make sure checksums are present
require.Equal(t, 2, len(gf.Checksums),
"Checksums should be %s, found %d for %s", 2, len(gf.Checksums), gf.Identifier)
md5 := gf.GetChecksumByAlgorithm(constants.AlgMd5)
sha256 := gf.GetChecksumByAlgorithm(constants.AlgSha256)
require.NotNil(t, md5, "Missing md5 digest for for %s", gf.Identifier)
require.NotNil(t, sha256, "Missing sha256 digest for for %s", gf.Identifier)
// Make sure that these checksums were saved
assert.True(t, md5.Id > 0, "md5 was not saved for %s", gf.Identifier)
assert.True(t, md5.GenericFileId > 0, "md5.GenericFileId not set for %s", gf.Identifier)
assert.False(t, md5.CreatedAt.IsZero(), "md5.CreatedAt was not set for %s", gf.Identifier)
assert.False(t, md5.UpdatedAt.IsZero(), "md5.UpdatedAt was not set for %s", gf.Identifier)
assert.True(t, sha256.Id > 0, "sha256 was not saved for %s", gf.Identifier)
assert.True(t, sha256.GenericFileId > 0, "sha256.GenericFileId not set for %s", gf.Identifier)
assert.False(t, sha256.CreatedAt.IsZero(), "sha256.CreatedAt was not set for %s", gf.Identifier)
assert.False(t, sha256.UpdatedAt.IsZero(), "sha256.UpdatedAt was not set for %s", gf.Identifier)
// Make sure PremisEvents are present
require.Equal(t, 6, len(gf.PremisEvents),
"PremisEvents count should be %s, found %d for %s", 6, len(gf.PremisEvents), gf.Identifier)
assert.Equal(t, 1, len(gf.FindEventsByType(constants.EventFixityCheck)),
"Missing fixity check event for %s", gf.Identifier)
assert.Equal(t, 1, len(gf.FindEventsByType(constants.EventDigestCalculation)),
"Missing digest calculation event for %s", gf.Identifier)
assert.Equal(t, 2, len(gf.FindEventsByType(constants.EventIdentifierAssignment)),
"Missing identifier assignment event(s) for %s", gf.Identifier)
assert.Equal(t, 1, len(gf.FindEventsByType(constants.EventReplication)),
"Missing replication event for %s", gf.Identifier)
assert.Equal(t, 1, len(gf.FindEventsByType(constants.EventIngestion)),
"Missing ingestion event for %s", gf.Identifier)
for _, event := range gf.PremisEvents {
assert.True(t, event.Id > 0, "Event %s was not saved for %s", event.EventType, gf.Identifier)
assert.True(t, event.IntellectualObjectId > 0,
"event.IntellectualObjectId not set for %s %s", event.EventType, gf.Identifier)
assert.True(t, event.GenericFileId > 0,
"event.GenericFileId not set for %s %s", event.EventType, gf.Identifier)
assert.False(t, event.DateTime.IsZero(),
"event.DateTime was not set for %s %s", event.EventType, gf.Identifier)
assert.False(t, event.CreatedAt.IsZero(),
"event.CreatedAt was not set for %s %s", event.EventType, gf.Identifier)
assert.False(t, event.UpdatedAt.IsZero(),
"event.UpdatedAt was not set for %s %s", event.EventType, gf.Identifier)
assert.True(t, util.LooksLikeUUID(event.Identifier),
"Identifier for %s %s doesn't look like a UUID", event.EventType, gf.Identifier)
assert.NotEmpty(t, event.EventType, "EventType missing for %s %s", gf.Identifier, event.Identifier)
assert.NotEmpty(t, event.Detail, "Detail is empty for %s %s", event.EventType, gf.Identifier)
assert.NotEmpty(t, event.Outcome, "Outcome is empty for %s %s", event.EventType, gf.Identifier)
assert.NotEmpty(t, event.OutcomeDetail,
"OutcomeDetail is empty for %s %s", event.EventType, gf.Identifier)
assert.NotEmpty(t, event.Object, "Object is empty for %s %s", event.EventType, gf.Identifier)
assert.NotEmpty(t, event.Agent, "Agent is empty for %s %s", event.EventType, gf.Identifier)
assert.NotEmpty(t, event.OutcomeInformation,
"OutcomeInformation is empty for %s %s", event.EventType, gf.Identifier)
assert.Equal(t, obj.Identifier, event.IntellectualObjectIdentifier,
"IntellectualObjectIdentifier is wrong for %s %s", event.EventType, gf.Identifier)
assert.Equal(t, gf.Identifier, event.GenericFileIdentifier,
"GenericFileIdentifier is wrong for %s %s", event.EventType, gf.Identifier)
}
}
}