Ejemplo n.º 1
0
func TestManifestBuildDPNIdentifierEvent(t *testing.T) {
	nsqMessage := testutil.MakeNsqMessage("999")
	manifest := models.NewDPNIngestManifest(nsqMessage)
	manifest.StorageURL = "https://example.com/my_bag.tar"

	// Should give error, since DPNBag is nil
	event, err := manifest.BuildDPNIdentifierEvent()
	require.NotNil(t, err)

	manifest.DPNBag = testutil.MakeDPNBag()
	event, err = manifest.BuildDPNIdentifierEvent()
	require.Nil(t, err)
	require.NotNil(t, event)

	assert.True(t, util.LooksLikeUUID(event.Identifier))
	assert.Equal(t, constants.EventIdentifierAssignment, event.EventType)
	assert.False(t, event.DateTime.IsZero())
	assert.Equal(t, "Item assigned DPN UUID", event.Detail)
	assert.Equal(t, constants.StatusSuccess, event.Outcome)
	assert.Equal(t, manifest.DPNBag.UUID, event.OutcomeDetail)
	assert.Equal(t, "APTrust exchange using Satori go.uuid", event.Object)
	assert.Equal(t, "https://github.com/satori/go.uuid", event.Agent)
	assert.True(t, strings.Contains(event.OutcomeInformation, manifest.DPNBag.UUID))
	assert.Equal(t, manifest.DPNBag.LocalId, event.IntellectualObjectIdentifier)
}
Ejemplo n.º 2
0
func TestManifestBuildReplicationTransfer(t *testing.T) {
	nsqMessage := testutil.MakeNsqMessage("999")
	manifest := models.NewDPNIngestManifest(nsqMessage)
	fromNode := "aptrust"
	toNode := "chron"
	link := "[email protected]:outbound/1234567.tar"
	xfer, err := manifest.BuildReplicationTransfer(fromNode, toNode, link)
	// No DPNBag attached to manifest. Should get error.
	require.NotNil(t, err)

	manifest.DPNBag = testutil.MakeDPNBag()
	xfer, err = manifest.BuildReplicationTransfer(fromNode, toNode, link)
	require.Nil(t, err)
	require.NotNil(t, xfer)

	assert.Equal(t, fromNode, xfer.FromNode)
	assert.Equal(t, toNode, xfer.ToNode)
	assert.Equal(t, manifest.DPNBag.UUID, xfer.Bag)
	assert.True(t, util.LooksLikeUUID(xfer.ReplicationId))

	assert.Equal(t, constants.AlgSha256, xfer.FixityAlgorithm)
	assert.Nil(t, xfer.FixityNonce)
	assert.Nil(t, xfer.FixityValue)
	assert.Equal(t, "rsync", xfer.Protocol)
	assert.Equal(t, link, xfer.Link)
	assert.False(t, xfer.CreatedAt.IsZero())
	assert.False(t, xfer.UpdatedAt.IsZero())
}
Ejemplo n.º 3
0
// Returns true if the GenericFile IngestUUID is present and looks good.
func (storer *APTStorer) uuidPresent(ingestState *models.IngestState, gf *models.GenericFile) bool {
	if !util.LooksLikeUUID(gf.IngestUUID) {
		ingestState.IngestManifest.StoreResult.AddError("Cannot save %s to S3/Glacier because "+
			"GenericFile.IngestUUID (%s) is missing or invalid",
			gf.Identifier, gf.IngestUUID)
		ingestState.IngestManifest.StoreResult.ErrorIsFatal = true
		return false
	}
	return true
}
Ejemplo n.º 4
0
func fetcherTestSpecifics(t *testing.T, ingestManifest *models.IngestManifest) {
	obj := ingestManifest.Object
	assert.Equal(t, "test.edu/example.edu.tagsample_good", obj.Identifier)
	assert.Equal(t, "example.edu.tagsample_good", obj.BagName)
	assert.Equal(t, "test.edu", obj.Institution)
	assert.NotEqual(t, 0, obj.InstitutionId)
	assert.Equal(t, "Tag Sample (Good)", obj.Title)
	assert.Equal(t, "Bag of goodies", obj.Description)
	assert.Equal(t, "Institution", obj.Access)
	assert.Equal(t, "uva-internal-id-0001", obj.AltIdentifier)
	assert.Equal(t, "aptrust.receiving.test.test.edu", obj.IngestS3Bucket)
	assert.Equal(t, "example.edu.tagsample_good.tar", obj.IngestS3Key)
	assert.Equal(t, "manifest-md5.txt", obj.IngestManifests[0])
	assert.Equal(t, "manifest-sha256.txt", obj.IngestManifests[1])
	assert.Equal(t, "tagmanifest-md5.txt", obj.IngestTagManifests[0])
	assert.Equal(t, "tagmanifest-sha256.txt", obj.IngestTagManifests[1])
	assert.Empty(t, obj.IngestFilesIgnored)
	assert.Equal(t, "example.edu.tagsample_good", obj.IngestTopLevelDirNames[0])
	assert.Empty(t, obj.IngestErrorMessage)

	assert.Equal(t, 10, len(obj.IngestTags))
	assert.Equal(t, "bag-info.txt", obj.IngestTags[5].SourceFile)
	assert.Equal(t, "Bag-Group-Identifier", obj.IngestTags[5].Label)
	assert.Equal(t, "Charley Horse", obj.IngestTags[5].Value)

	assert.Empty(t, obj.PremisEvents)

	assert.Equal(t, 16, len(obj.GenericFiles))

	gf := obj.GenericFiles[0]
	assert.Equal(t, 0, gf.Id)
	assert.Equal(t, "test.edu/example.edu.tagsample_good/aptrust-info.txt", gf.Identifier)
	assert.Equal(t, 0, gf.IntellectualObjectId)
	assert.Equal(t, "test.edu/example.edu.tagsample_good", gf.IntellectualObjectIdentifier)
	assert.Equal(t, "text/plain", gf.FileFormat)
	assert.EqualValues(t, 45, gf.Size)
	assert.EqualValues(t, "0001-01-01T00:00:00Z", gf.FileCreated.Format(time.RFC3339))
	assert.Equal(t, "2016-03-21T11:01:51-04:00", gf.FileModified.Format(time.RFC3339))
	assert.Empty(t, gf.Checksums)
	assert.Empty(t, gf.PremisEvents)
	assert.Equal(t, "tag_file", gf.IngestFileType)
	assert.Equal(t, "bd8be664c790a9175e9d2fe90b40d502", gf.IngestMd5)
	assert.False(t, gf.IngestMd5GeneratedAt.IsZero())
	assert.False(t, gf.IngestMd5VerifiedAt.IsZero())
	assert.Equal(t, "49dd23cdaf644e60629f01d6ebea770cd1c6229ff89f14a6c030a50c48b6ba27", gf.IngestSha256)
	assert.False(t, gf.IngestSha256GeneratedAt.IsZero())
	assert.False(t, gf.IngestSha256VerifiedAt.IsZero())
	assert.True(t, util.LooksLikeUUID(gf.IngestUUID))
	assert.False(t, gf.IngestUUIDGeneratedAt.IsZero())
	assert.True(t, gf.IngestStoredAt.IsZero())
	assert.True(t, gf.IngestReplicatedAt.IsZero())
	assert.True(t, gf.IngestNeedsSave)
	assert.EqualValues(t, 502, gf.IngestFileUid)
	assert.EqualValues(t, 20, gf.IngestFileGid)
}
// TestInteObjDPNUUID makes sure the DPN UUID was set on the
// IntellectualObject record in Pharos.
func TestInteObjDPNUUID(t *testing.T) {
	if !apt_testutil.ShouldRunIntegrationTests() {
		t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
	}
	_context, err := apt_testutil.GetContext("integration.json")
	require.Nil(t, err)
	for _, s3Key := range apt_testutil.INTEGRATION_GOOD_BAGS[0:7] {
		tar := strings.Replace(s3Key, "aptrust.receiving.test.", "", 1)
		objIdentifier := strings.Replace(tar, ".tar", "", 1)
		resp := _context.PharosClient.IntellectualObjectGet(objIdentifier, false, false)
		require.Nil(t, resp.Error)
		obj := resp.IntellectualObject()
		require.NotNil(t, obj)
		// DPNUUID is null in fixture data. It should be set after DPN ingest.
		assert.True(t, util.LooksLikeUUID(obj.DPNUUID))
	}
}
Ejemplo n.º 6
0
// Returns the UUID of an existing GenericFile. The UUID is the last component
// of the S3 storage URL. When we are updating an existing GenericFile, we want
// to overwrite the object in S3/Glacier rather than writing a new one and
// leaving the old one hanging around. To overwrite it, we must know its UUID.
func (storer *APTStorer) getUuidOfExistingFile(gfIdentifier string) (string, error) {
	storer.Context.MessageLog.Info("Checking Pharos for existing UUID for GenericFile %s",
		gfIdentifier)
	resp := storer.Context.PharosClient.GenericFileGet(gfIdentifier)
	if resp.Error != nil {
		return "", resp.Error
	}
	uuid := ""
	existingGenericFile := resp.GenericFile()
	if resp.Error != nil {
		return "", fmt.Errorf("Pharos cannot find supposedly existing GenericFile '%s'", gfIdentifier)
	}
	parts := strings.Split(existingGenericFile.URI, "/")
	uuid = parts[len(parts)-1]
	if !util.LooksLikeUUID(uuid) {
		return "", fmt.Errorf("Could not extract UUID from URI %s", existingGenericFile.URI)
	}
	return uuid, nil
}
Ejemplo n.º 7
0
func TestLooksLikeUUID(t *testing.T) {
	if util.LooksLikeUUID("1552abf5-28f3-46a5-ba63-95302d08e209") == false {
		t.Error("That was a valid UUID!")
	}
	if util.LooksLikeUUID("88198c5a-ec91-4ce1-bfcc-0f607ebdcca3") == false {
		t.Error("That was a valid UUID!")
	}
	if util.LooksLikeUUID("88198C5A-EC91-4CE1-BFCC-0F607EBDCCA3") == false {
		t.Error("That was a valid UUID!")
	}
	if util.LooksLikeUUID("88198c5a-ec91-4ce1-bfcc-0f607ebdccx3") == true {
		t.Error("That was not a valid UUID!")
	}
	if util.LooksLikeUUID("88198c5a-ec91-4ce1-bfcc-0f6c") == true {
		t.Error("That was not a valid UUID!")
	}
	if util.LooksLikeUUID("") == true {
		t.Error("That was not a valid UUID! That was an empty string!")
	}
}
Ejemplo n.º 8
0
func recordTestCommon(t *testing.T, bagName string, ingestManifest *models.IngestManifest) {
	// Test some basic object properties
	assert.NotEmpty(t, ingestManifest.WorkItemId, "WorkItemId should not be empty for %s", bagName)
	assert.NotEmpty(t, ingestManifest.S3Bucket, "S3Bucket should not be empty for %s", bagName)
	assert.NotEmpty(t, ingestManifest.S3Key, "S3Key should not be empty for %s", bagName)
	assert.NotEmpty(t, ingestManifest.ETag, "ETag should not be empty for %s", bagName)

	// Make sure the result has some basic info in RecordResult
	assert.True(t, ingestManifest.RecordResult.Attempted,
		"RecordResult.Attempted should be true for %s", bagName)
	assert.True(t, ingestManifest.RecordResult.AttemptNumber > 0,
		"RecordResult.AttemptNumber should be > 0 %s", bagName)
	assert.NotEmpty(t, ingestManifest.RecordResult.StartedAt,
		"RecordResult.StartedAt should not be empty for %s", bagName)
	assert.NotEmpty(t, ingestManifest.RecordResult.FinishedAt,
		"RecordResult.FinishedAt should not be empty for %s", bagName)
	assert.Empty(t, ingestManifest.RecordResult.Errors,
		"RecordResult.Errors should be empty for %s", bagName)
	assert.True(t, ingestManifest.RecordResult.Retry,
		"RecordResult.Retry should be true for %s", bagName)

	// Make sure the result has some basic info in CleanupResult
	assert.True(t, ingestManifest.CleanupResult.Attempted,
		"CleanupResult.Attempted should be true for %s", bagName)
	assert.True(t, ingestManifest.CleanupResult.AttemptNumber > 0,
		"CleanupResult.AttemptNumber should be > 0 %s", bagName)
	assert.NotEmpty(t, ingestManifest.CleanupResult.StartedAt,
		"CleanupResult.StartedAt should not be empty for %s", bagName)
	assert.NotEmpty(t, ingestManifest.CleanupResult.FinishedAt,
		"CleanupResult.FinishedAt should not be empty for %s", bagName)
	assert.Empty(t, ingestManifest.CleanupResult.Errors,
		"CleanupResult.Errors should be empty for %s", bagName)
	assert.True(t, ingestManifest.CleanupResult.Retry,
		"CleanupResult.Retry should be true for %s", bagName)

	// Make sure our IntellectualObject got all of its PremisEvents
	obj := ingestManifest.Object
	require.Equal(t, 4, len(obj.PremisEvents))

	// Make sure this item was deleted from the receiving bucket
	// after ingest completed.
	assert.False(t, obj.IngestDeletedFromReceivingAt.IsZero(),
		"Object %s was not deleted from receiving bucket", bagName)
	assert.Empty(t, obj.IngestErrorMessage)

	// Check the object-level events
	creationEvents := obj.FindEventsByType(constants.EventCreation)
	idEvents := obj.FindEventsByType(constants.EventIdentifierAssignment)
	ingestEvents := obj.FindEventsByType(constants.EventIngestion)
	accessEvents := obj.FindEventsByType(constants.EventAccessAssignment)
	assert.Equal(t, 1, len(accessEvents), "Missing access event for %s", bagName)
	assert.Equal(t, 1, len(creationEvents), "Missing creation event for %s", bagName)
	assert.Equal(t, 1, len(idEvents), "Missing identifier assignment event for %s", bagName)
	assert.Equal(t, 1, len(ingestEvents), "Missing ingest event for %s", bagName)

	for _, event := range obj.PremisEvents {
		assert.True(t, event.Id > 0, "Event %s was not saved for %s", event.EventType, obj.Identifier)
		assert.True(t, event.IntellectualObjectId > 0,
			"event.IntellectualObjectId not set for %s %s", event.EventType, obj.Identifier)
		assert.False(t, event.DateTime.IsZero(),
			"event.DateTime was not set for %s %s", event.EventType, obj.Identifier)
		assert.False(t, event.CreatedAt.IsZero(),
			"event.CreatedAt was not set for %s %s", event.EventType, obj.Identifier)
		assert.False(t, event.UpdatedAt.IsZero(),
			"event.UpdatedAt was not set for %s %s", event.EventType, obj.Identifier)

		assert.True(t, util.LooksLikeUUID(event.Identifier),
			"Identifier for %s %s doesn't look like a UUID", event.EventType, obj.Identifier)
		assert.NotEmpty(t, event.EventType, "EventType missing for %s %s", obj.Identifier, event.Identifier)
		assert.NotEmpty(t, event.Detail, "Detail is empty for %s %s", event.EventType, obj.Identifier)
		assert.NotEmpty(t, event.Outcome, "Outcome is empty for %s %s", event.EventType, obj.Identifier)
		assert.NotEmpty(t, event.OutcomeDetail,
			"OutcomeDetail is empty for %s %s", event.EventType, obj.Identifier)
		assert.NotEmpty(t, event.Object, "Object is empty for %s %s", event.EventType, obj.Identifier)
		assert.NotEmpty(t, event.Agent, "Agent is empty for %s %s", event.EventType, obj.Identifier)
		assert.NotEmpty(t, event.OutcomeInformation,
			"OutcomeInformation is empty for %s %s", event.EventType, obj.Identifier)
		assert.Equal(t, obj.Identifier, event.IntellectualObjectIdentifier,
			"IntellectualObjectIdentifier is wrong for %s %s", event.EventType, obj.Identifier)
	}

	for _, gf := range obj.GenericFiles {
		// Skip these checks for files that didn't need to be saved.
		// Reasons for not needing to be saved:
		// 1. File has a non-savable name, according to util.HasSavableName
		// 2. File has not changed since last time we ingested this bag.
		if !gf.IngestNeedsSave {
			continue
		}

		// Make sure checksums are present
		require.Equal(t, 2, len(gf.Checksums),
			"Checksums should be %s, found %d for %s", 2, len(gf.Checksums), gf.Identifier)
		md5 := gf.GetChecksumByAlgorithm(constants.AlgMd5)
		sha256 := gf.GetChecksumByAlgorithm(constants.AlgSha256)
		require.NotNil(t, md5, "Missing md5 digest for for %s", gf.Identifier)
		require.NotNil(t, sha256, "Missing sha256 digest for for %s", gf.Identifier)

		// Make sure that these checksums were saved
		assert.True(t, md5.Id > 0, "md5 was not saved for %s", gf.Identifier)
		assert.True(t, md5.GenericFileId > 0, "md5.GenericFileId not set for %s", gf.Identifier)
		assert.False(t, md5.CreatedAt.IsZero(), "md5.CreatedAt was not set for %s", gf.Identifier)
		assert.False(t, md5.UpdatedAt.IsZero(), "md5.UpdatedAt was not set for %s", gf.Identifier)

		assert.True(t, sha256.Id > 0, "sha256 was not saved for %s", gf.Identifier)
		assert.True(t, sha256.GenericFileId > 0, "sha256.GenericFileId not set for %s", gf.Identifier)
		assert.False(t, sha256.CreatedAt.IsZero(), "sha256.CreatedAt was not set for %s", gf.Identifier)
		assert.False(t, sha256.UpdatedAt.IsZero(), "sha256.UpdatedAt was not set for %s", gf.Identifier)

		// Make sure PremisEvents are present
		require.Equal(t, 6, len(gf.PremisEvents),
			"PremisEvents count should be %s, found %d for %s", 6, len(gf.PremisEvents), gf.Identifier)
		assert.Equal(t, 1, len(gf.FindEventsByType(constants.EventFixityCheck)),
			"Missing fixity check event for %s", gf.Identifier)
		assert.Equal(t, 1, len(gf.FindEventsByType(constants.EventDigestCalculation)),
			"Missing digest calculation event for %s", gf.Identifier)
		assert.Equal(t, 2, len(gf.FindEventsByType(constants.EventIdentifierAssignment)),
			"Missing identifier assignment event(s) for %s", gf.Identifier)
		assert.Equal(t, 1, len(gf.FindEventsByType(constants.EventReplication)),
			"Missing replication event for %s", gf.Identifier)
		assert.Equal(t, 1, len(gf.FindEventsByType(constants.EventIngestion)),
			"Missing ingestion event for %s", gf.Identifier)

		for _, event := range gf.PremisEvents {
			assert.True(t, event.Id > 0, "Event %s was not saved for %s", event.EventType, gf.Identifier)
			assert.True(t, event.IntellectualObjectId > 0,
				"event.IntellectualObjectId not set for %s %s", event.EventType, gf.Identifier)
			assert.True(t, event.GenericFileId > 0,
				"event.GenericFileId not set for %s %s", event.EventType, gf.Identifier)
			assert.False(t, event.DateTime.IsZero(),
				"event.DateTime was not set for %s %s", event.EventType, gf.Identifier)
			assert.False(t, event.CreatedAt.IsZero(),
				"event.CreatedAt was not set for %s %s", event.EventType, gf.Identifier)
			assert.False(t, event.UpdatedAt.IsZero(),
				"event.UpdatedAt was not set for %s %s", event.EventType, gf.Identifier)

			assert.True(t, util.LooksLikeUUID(event.Identifier),
				"Identifier for %s %s doesn't look like a UUID", event.EventType, gf.Identifier)
			assert.NotEmpty(t, event.EventType, "EventType missing for %s %s", gf.Identifier, event.Identifier)
			assert.NotEmpty(t, event.Detail, "Detail is empty for %s %s", event.EventType, gf.Identifier)
			assert.NotEmpty(t, event.Outcome, "Outcome is empty for %s %s", event.EventType, gf.Identifier)
			assert.NotEmpty(t, event.OutcomeDetail,
				"OutcomeDetail is empty for %s %s", event.EventType, gf.Identifier)
			assert.NotEmpty(t, event.Object, "Object is empty for %s %s", event.EventType, gf.Identifier)
			assert.NotEmpty(t, event.Agent, "Agent is empty for %s %s", event.EventType, gf.Identifier)
			assert.NotEmpty(t, event.OutcomeInformation,
				"OutcomeInformation is empty for %s %s", event.EventType, gf.Identifier)
			assert.Equal(t, obj.Identifier, event.IntellectualObjectIdentifier,
				"IntellectualObjectIdentifier is wrong for %s %s", event.EventType, gf.Identifier)
			assert.Equal(t, gf.Identifier, event.GenericFileIdentifier,
				"GenericFileIdentifier is wrong for %s %s", event.EventType, gf.Identifier)
		}
	}
}