func TestFindDPNIngestManifestInLog(t *testing.T) { pathToLogFile, _ := fileutil.RelativeToAbsPath( filepath.Join("testdata", "integration_results", "dpn_package.json")) // Should get a manifest if the item exists // Identifier is the name of the bag's original tar file. manifest, err := testutil.FindDPNIngestManifestInLog(pathToLogFile, "ncsu.1840.16-10.tar") assert.Nil(t, err) assert.NotNil(t, manifest) assert.NotNil(t, manifest.WorkItem) assert.False(t, manifest.PackageSummary.StartedAt.IsZero()) assert.False(t, manifest.ValidateSummary.StartedAt.IsZero()) assert.NotEmpty(t, manifest.LocalDir) assert.NotEmpty(t, manifest.LocalTarFile) // Should get an error if the item does not exist manifest, err = testutil.FindDPNIngestManifestInLog(pathToLogFile, "does_not_exist.tar") assert.NotNil(t, err) assert.Nil(t, manifest) }
// TestIngestStoreItemsAreInStorage makes sure that the items we sent off // to long-term storage in AWS actually made it there. func TestIngestStoreItemsAreInStorage(t *testing.T) { if !apt_testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _context, err := apt_testutil.GetContext("integration.json") require.Nil(t, err, "Could not create context") maxItemsToList := int64(1) // s3List lists bucket contents. s3List := network.NewS3ObjectList( constants.AWSVirginia, _context.Config.DPN.DPNPreservationBucket, maxItemsToList) // s3Head gets metadata about specific objects in S3/Glacier. s3Head := network.NewS3Head(_context.Config.APTrustS3Region, _context.Config.DPN.DPNPreservationBucket) pathToLogFile := filepath.Join(_context.Config.LogDirectory, "dpn_ingest_store.json") for _, s3Key := range apt_testutil.INTEGRATION_GOOD_BAGS[0:7] { parts := strings.Split(s3Key, "/") localTarFileName := parts[1] // APTrust bag name. E.g. "test.edu.test_123.tar" manifest, err := apt_testutil.FindDPNIngestManifestInLog(pathToLogFile, localTarFileName) require.Nil(t, err, "Could not find JSON record for %s", localTarFileName) parts = strings.Split(manifest.StorageURL, "/") dpnTarFileName := parts[len(parts)-1] // DPN bag name: <uuid>.tar s3List.GetList(dpnTarFileName) require.Empty(t, s3List.ErrorMessage) require.EqualValues(t, 1, len(s3List.Response.Contents), "Nothing in S3 for %s", dpnTarFileName) obj := s3List.Response.Contents[0] assert.Equal(t, dpnTarFileName, *obj.Key) // Make sure each item has the expected metadata. // s3Head.Response.Metadata is map[string]*string. s3Head.Head(dpnTarFileName) require.Empty(t, s3Head.ErrorMessage) metadata := s3Head.Response.Metadata require.NotNil(t, metadata, dpnTarFileName) // Notice the Amazon library transforms the first letter of // all our keys to upper case. WTF? require.NotNil(t, metadata["From_node"], dpnTarFileName) require.NotNil(t, metadata["Transfer_id"], dpnTarFileName) require.NotNil(t, metadata["Member"], dpnTarFileName) require.NotNil(t, metadata["Local_id"], dpnTarFileName) require.NotNil(t, metadata["Version"], dpnTarFileName) assert.NotEmpty(t, *metadata["From_node"], dpnTarFileName) assert.NotEmpty(t, *metadata["Transfer_id"], dpnTarFileName) assert.NotEmpty(t, *metadata["Member"], dpnTarFileName) assert.NotEmpty(t, *metadata["Local_id"], dpnTarFileName) assert.NotEmpty(t, *metadata["Version"], dpnTarFileName) } }
// TestIngestRecordJsonLog checks that all expected entries are present // in the dpn_ingest_record.json log. func TestIngestRecordJsonLog(t *testing.T) { if !apt_testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _context, err := apt_testutil.GetContext("integration.json") require.Nil(t, err) pathToLogFile := filepath.Join(_context.Config.LogDirectory, "dpn_ingest_record.json") for _, s3Key := range apt_testutil.INTEGRATION_GOOD_BAGS[0:7] { parts := strings.Split(s3Key, "/") tarFileName := parts[1] manifest, err := apt_testutil.FindDPNIngestManifestInLog(pathToLogFile, tarFileName) require.Nil(t, err) require.NotNil(t, manifest) detail := fmt.Sprintf("%s from JSON log", tarFileName) testIngestRecordManifest(t, _context, manifest, detail) } }