// TestReplicationsMarkedAsStored - make sure that the ReplicationTransfer // records are marked with stored = true on the remote nodes. func TestReplicationsMarkedAsStored(t *testing.T) { if !apt_testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _context, err := apt_testutil.GetContext("integration.json") require.Nil(t, err, "Could not create context") // Get a list of ReplicationTransfers where our local node // is the ToNode, and make sure we marked them all as stored // on the FromNode. localClient, err := network.NewDPNRestClient( _context.Config.DPN.RestClient.LocalServiceURL, _context.Config.DPN.RestClient.LocalAPIRoot, _context.Config.DPN.RestClient.LocalAuthToken, _context.Config.DPN.LocalNode, _context.Config.DPN) require.Nil(t, err) remoteClients, err := localClient.GetRemoteClients() require.Nil(t, err) xferParams := url.Values{} xferParams.Set("to_node", _context.Config.DPN.LocalNode) dpnResp := localClient.ReplicationTransferList(xferParams) require.Nil(t, dpnResp.Error) for _, xfer := range dpnResp.ReplicationTransfers() { remoteClient := remoteClients[xfer.FromNode] require.NotNil(t, remoteClient) resp := remoteClient.ReplicationTransferGet(xfer.ReplicationId) require.Nil(t, resp.Error) remoteXfer := resp.ReplicationTransfer() require.NotNil(t, remoteXfer) assert.True(t, remoteXfer.Stored) } }
// TestItemsAreInLongTermStorage - make sure that each tar file is // stored in our S3 test storage bucket, with correct metadata. func TestItemsAreInLongTermStorage(t *testing.T) { if !apt_testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _context, err := apt_testutil.GetContext("integration.json") require.Nil(t, err, "Could not create context") localClient, err := network.NewDPNRestClient( _context.Config.DPN.RestClient.LocalServiceURL, _context.Config.DPN.RestClient.LocalAPIRoot, _context.Config.DPN.RestClient.LocalAuthToken, _context.Config.DPN.LocalNode, _context.Config.DPN) require.Nil(t, err) // s3List lists bucket contents. s3List := apt_network.NewS3ObjectList( constants.AWSVirginia, _context.Config.DPN.DPNPreservationBucket, int64(100), ) // s3Head gets metadata about specific objects in S3/Glacier. s3Head := apt_network.NewS3Head(_context.Config.APTrustS3Region, _context.Config.DPN.DPNPreservationBucket) for _, identifier := range dpn_testutil.BAG_IDS { resp := localClient.DPNBagGet(identifier) dpnBag := resp.Bag() require.NotNil(t, dpnBag) if dpnBag.IngestNode == _context.Config.DPN.LocalNode { continue // we would not have replicated our own bag } tarFileName := fmt.Sprintf("%s.tar", identifier) s3List.GetList(tarFileName) require.NotEmpty(t, s3List.Response.Contents, "%s not found in S3", tarFileName) object := s3List.Response.Contents[0] fiveMinutesAgo := time.Now().UTC().Add(-5 * time.Minute) require.NotNil(t, object.LastModified) assert.True(t, object.LastModified.After(fiveMinutesAgo)) // Make sure each item has the expected metadata. // s3Head.Response.Metadata is map[string]*string. s3Head.Head(tarFileName) require.Empty(t, s3Head.ErrorMessage) metadata := s3Head.Response.Metadata require.NotNil(t, metadata) // Amazon library transforms first letters of keys to CAPS require.NotNil(t, metadata["From_node"]) require.NotNil(t, metadata["Transfer_id"]) require.NotNil(t, metadata["Member"]) require.NotNil(t, metadata["Local_id"]) require.NotNil(t, metadata["Version"]) assert.NotEmpty(t, *metadata["From_node"]) assert.NotEmpty(t, *metadata["Transfer_id"]) assert.NotEmpty(t, *metadata["Member"]) assert.NotEmpty(t, *metadata["Local_id"]) assert.NotEmpty(t, *metadata["Version"]) } }
// TestIngestStoreTarFilesDeleted tests whether all expected DPN bags // (tar files) have been deleted from the staging area. func TestIngestStoreTarFilesDeleted(t *testing.T) { if !apt_testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _context, err := apt_testutil.GetContext("integration.json") require.Nil(t, err) pattern := filepath.Join(_context.Config.DPN.StagingDirectory, "test.edu", "*.tar") files, err := filepath.Glob(pattern) require.Nil(t, err) assert.Equal(t, 0, len(files)) }
// TestIngestStoreItemsAreInStorage makes sure that the items we sent off // to long-term storage in AWS actually made it there. func TestIngestStoreItemsAreInStorage(t *testing.T) { if !apt_testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _context, err := apt_testutil.GetContext("integration.json") require.Nil(t, err, "Could not create context") maxItemsToList := int64(1) // s3List lists bucket contents. s3List := network.NewS3ObjectList( constants.AWSVirginia, _context.Config.DPN.DPNPreservationBucket, maxItemsToList) // s3Head gets metadata about specific objects in S3/Glacier. s3Head := network.NewS3Head(_context.Config.APTrustS3Region, _context.Config.DPN.DPNPreservationBucket) pathToLogFile := filepath.Join(_context.Config.LogDirectory, "dpn_ingest_store.json") for _, s3Key := range apt_testutil.INTEGRATION_GOOD_BAGS[0:7] { parts := strings.Split(s3Key, "/") localTarFileName := parts[1] // APTrust bag name. E.g. "test.edu.test_123.tar" manifest, err := apt_testutil.FindDPNIngestManifestInLog(pathToLogFile, localTarFileName) require.Nil(t, err, "Could not find JSON record for %s", localTarFileName) parts = strings.Split(manifest.StorageURL, "/") dpnTarFileName := parts[len(parts)-1] // DPN bag name: <uuid>.tar s3List.GetList(dpnTarFileName) require.Empty(t, s3List.ErrorMessage) require.EqualValues(t, 1, len(s3List.Response.Contents), "Nothing in S3 for %s", dpnTarFileName) obj := s3List.Response.Contents[0] assert.Equal(t, dpnTarFileName, *obj.Key) // Make sure each item has the expected metadata. // s3Head.Response.Metadata is map[string]*string. s3Head.Head(dpnTarFileName) require.Empty(t, s3Head.ErrorMessage) metadata := s3Head.Response.Metadata require.NotNil(t, metadata, dpnTarFileName) // Notice the Amazon library transforms the first letter of // all our keys to upper case. WTF? require.NotNil(t, metadata["From_node"], dpnTarFileName) require.NotNil(t, metadata["Transfer_id"], dpnTarFileName) require.NotNil(t, metadata["Member"], dpnTarFileName) require.NotNil(t, metadata["Local_id"], dpnTarFileName) require.NotNil(t, metadata["Version"], dpnTarFileName) assert.NotEmpty(t, *metadata["From_node"], dpnTarFileName) assert.NotEmpty(t, *metadata["Transfer_id"], dpnTarFileName) assert.NotEmpty(t, *metadata["Member"], dpnTarFileName) assert.NotEmpty(t, *metadata["Local_id"], dpnTarFileName) assert.NotEmpty(t, *metadata["Version"], dpnTarFileName) } }
func TestItemsCopiedToStaging(t *testing.T) { // Make sure that each of the expected bags has shown // up in our test staging area. if !testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } for i := 2; i <= 5; i++ { _context, err := testutil.GetContext("integration.json") require.Nil(t, err, "Could not create context") filename := fmt.Sprintf("00000000-0000-4000-a000-00000000000%d.tar", i) path := filepath.Join(_context.Config.DPN.StagingDirectory, filename) assert.True(t, fileutil.FileExists(path), "File %s was not copied", path) } }
// TestIngestStoreWorkItems checks to see if Pharos WorkItems // were updated correctly. func TestIngestStoreWorkItems(t *testing.T) { if !apt_testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _, workItems, err := dpn_testutil.GetDPNWorkItems() require.Nil(t, err) for _, item := range workItems { assert.Equal(t, constants.StageRecord, item.Stage) assert.Equal(t, constants.StatusPending, item.Status) assert.Equal(t, "Bag copied to long-term storage", item.Note) assert.Equal(t, "", item.Node) assert.Equal(t, 0, item.Pid) assert.True(t, item.Retry) } }
// TestIngestRecordWorkItems checks to see if Pharos WorkItems // were updated correctly. func TestIngestRecordWorkItems(t *testing.T) { if !apt_testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _, workItems, err := dpn_testutil.GetDPNWorkItems() require.Nil(t, err) for _, item := range workItems { assert.Equal(t, constants.StageResolve, item.Stage) assert.Equal(t, constants.StatusSuccess, item.Status) assert.Equal(t, "DPN ingest complete", item.Note) assert.Equal(t, "", item.Node) assert.Equal(t, 0, item.Pid) assert.True(t, item.Retry) } }
func TestInstutionsCached(t *testing.T) { if !testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } expected, actual := getBucketReaderOutputs(t) testInstCache(t, expected, actual) testWorkItemsCached(t, expected, actual) testWorkItemsFetched(t, expected, actual) testWorkItemsCreated(t, expected, actual) testWorkItemsQueued(t, expected, actual) testWorkItemsMarkedAsQueued(t, expected, actual) testS3Items(t, expected, actual) testErrors(t, expected, actual) testWarnings(t, expected, actual) }
// TestIngestRecordJsonLog checks that all expected entries are present // in the dpn_ingest_record.json log. func TestIngestRecordJsonLog(t *testing.T) { if !apt_testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _context, err := apt_testutil.GetContext("integration.json") require.Nil(t, err) pathToLogFile := filepath.Join(_context.Config.LogDirectory, "dpn_ingest_record.json") for _, s3Key := range apt_testutil.INTEGRATION_GOOD_BAGS[0:7] { parts := strings.Split(s3Key, "/") tarFileName := parts[1] manifest, err := apt_testutil.FindDPNIngestManifestInLog(pathToLogFile, tarFileName) require.Nil(t, err) require.NotNil(t, manifest) detail := fmt.Sprintf("%s from JSON log", tarFileName) testIngestRecordManifest(t, _context, manifest, detail) } }
// TestInteObjDPNUUID makes sure the DPN UUID was set on the // IntellectualObject record in Pharos. func TestInteObjDPNUUID(t *testing.T) { if !apt_testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _context, err := apt_testutil.GetContext("integration.json") require.Nil(t, err) for _, s3Key := range apt_testutil.INTEGRATION_GOOD_BAGS[0:7] { tar := strings.Replace(s3Key, "aptrust.receiving.test.", "", 1) objIdentifier := strings.Replace(tar, ".tar", "", 1) resp := _context.PharosClient.IntellectualObjectGet(objIdentifier, false, false) require.Nil(t, resp.Error) obj := resp.IntellectualObject() require.NotNil(t, obj) // DPNUUID is null in fixture data. It should be set after DPN ingest. assert.True(t, util.LooksLikeUUID(obj.DPNUUID)) } }
// TestPackageCleanup checks to see whether dpn_package cleaned up // all of the intermediate files created during the bag building // process. Those are directories containing untarred bags. func TestPackageCleanup(t *testing.T) { if !apt_testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _context, err := apt_testutil.GetContext("integration.json") require.Nil(t, err) pattern := filepath.Join(_context.Config.DPN.StagingDirectory, "test.edu", "*") files, err := filepath.Glob(pattern) require.Nil(t, err) // Only the 7 tar file should remain. The 7 working directories // should have been deleted. If anything other than a tar file // remains, some part of cleanup failed. assert.Equal(t, 7, len(files)) for _, file := range files { assert.True(t, strings.HasSuffix(file, ".tar")) } }
// TestIngestStoreItemsQueued checks to see if dpn_ingest_store pushed items // into the dpn_ingest_record NSQ topic. func TestIngestStoreItemsQueued(t *testing.T) { if !apt_testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _context, err := apt_testutil.GetContext("integration.json") require.Nil(t, err, "Could not create context") stats, err := _context.NSQClient.GetStats() require.Nil(t, err) foundTopic := false for _, topic := range stats.Data.Topics { if topic.TopicName == _context.Config.DPN.DPNIngestRecordWorker.NsqTopic { // All 7 stored bags should show up in the record queue foundTopic = true assert.EqualValues(t, uint64(7), topic.MessageCount) } } assert.True(t, foundTopic, "Nothing was queued in %s", _context.Config.DPN.DPNIngestRecordWorker.NsqTopic) }
// test_push_to_dpn is for integration testing only. // It creates a few WorkItems in Pharos asking that // a handful of bags be pushed to DPN. Subsequent // integration tests (such as dpn_queue) depend on // the WorkItems created by this test. func TestPushToDPN(t *testing.T) { if !testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } configFile := filepath.Join("config", "integration.json") config, err := models.LoadConfigFile(configFile) require.Nil(t, err) _context := context.NewContext(config) for _, s3Key := range testutil.INTEGRATION_GOOD_BAGS[0:7] { identifier := strings.Replace(s3Key, "aptrust.receiving.test.", "", 1) identifier = strings.Replace(identifier, ".tar", "", 1) resp := _context.PharosClient.IntellectualObjectPushToDPN(identifier) workItem := resp.WorkItem() require.Nil(t, resp.Error) require.NotNil(t, workItem) _context.MessageLog.Info("Created DPN work item #%d for %s", workItem.Id, workItem.ObjectIdentifier) } }
func TestValidItemInStorageQueue(t *testing.T) { // Make sure the copied DPN bags made it into the validation queue. if !testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _context, err := testutil.GetContext("integration.json") require.Nil(t, err, "Could not create context") stats, err := _context.NSQClient.GetStats() require.Nil(t, err) foundTopic := false for _, topic := range stats.Data.Topics { if topic.TopicName == _context.Config.DPN.DPNReplicationStoreWorker.NsqTopic { // All 4 of the valid bags should appear in the store queue. foundTopic = true assert.EqualValues(t, uint64(4), topic.MessageCount) } } assert.True(t, foundTopic, "Nothing was queued in %s", _context.Config.DPN.DPNReplicationStoreWorker.NsqTopic) }
// TestIngestRecordWorkItemState checks to see if Pharos WorkItemState // records were updated correctly. func TestIngestRecordWorkItemState(t *testing.T) { if !apt_testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } // This actually retrieves DPN related WorkItems, not DPNWorkItems. Hmm... _context, workItems, err := dpn_testutil.GetDPNWorkItems() require.Nil(t, err) for _, item := range workItems { require.NotNil(t, item.WorkItemStateId) resp := _context.PharosClient.WorkItemStateGet(*item.WorkItemStateId) require.Nil(t, resp.Error) workItemState := resp.WorkItemState() require.NotNil(t, workItemState) assert.Equal(t, constants.ActionDPN, workItemState.Action) assert.False(t, workItemState.CreatedAt.IsZero()) assert.False(t, workItemState.UpdatedAt.IsZero()) detail := fmt.Sprintf("%s from Pharos", item.ObjectIdentifier) testIngestRecordWorkItemState(t, _context, workItemState.State, detail) } }
func TestCopyResultSentToRemoteNodes(t *testing.T) { // Query the FromNode of each copied bag to make sure that // we sent a fixity value back to the ingest node. if !testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _context, err := testutil.GetContext("integration.json") require.Nil(t, err, "Could not create context") dpnClient, err := network.NewDPNRestClient( _context.Config.DPN.RestClient.LocalServiceURL, _context.Config.DPN.RestClient.LocalAPIRoot, _context.Config.DPN.RestClient.LocalAuthToken, _context.Config.DPN.LocalNode, _context.Config.DPN) require.Nil(t, err, "Couldn't build DPN REST client: %v", err) remoteClients, err := dpnClient.GetRemoteClients() require.Nil(t, err, "Couldn't build remote DPN clients: %v", err) // These identifiers are in the fixture data for dpn-server. // Key is the FromNode, value is the ReplicationId xferIdentifiers := map[string]string{ "chron": "20000000-0000-4000-a000-000000000007", "hathi": "30000000-0000-4000-a000-000000000001", "sdr": "40000000-0000-4000-a000-000000000013", "tdr": "50000000-0000-4000-a000-000000000019", } for fromNode, identifier := range xferIdentifiers { client := remoteClients[fromNode] require.NotNil(t, client, "No DPN REST client for %s", fromNode) resp := client.ReplicationTransferGet(identifier) require.Nil(t, resp.Error) xfer := resp.ReplicationTransfer() require.NotNil(t, xfer, "ReplicationTransfer %s is missing", identifier) assert.NotEmpty(t, xfer.FixityValue, "Empty fixity for %s", identifier) assert.True(t, xfer.StoreRequested, "StoreRequested should not be false for %s", identifier) } }
// TestDPNWorkItemsCompleted - make sure DPNWorkItems are marked as // completed in Pharos. func TestDPNWorkItemsCompleted(t *testing.T) { if !apt_testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _context, err := apt_testutil.GetContext("integration.json") require.Nil(t, err, "Could not create context") params := url.Values{} params.Set("task", "replication") params.Set("page", "1") params.Set("per_page", "100") resp := _context.PharosClient.DPNWorkItemList(params) require.Nil(t, resp.Error) for _, item := range resp.DPNWorkItems() { require.NotNil(t, item.QueuedAt, "QueuedAt nil for %s", item.Identifier) require.NotNil(t, item.QueuedAt, "CompletedAt nil for %s", item.Identifier) require.NotNil(t, item.State, "State nil for %s", item.Identifier) assert.False(t, item.QueuedAt.IsZero(), "QueuedAt is zero for %s", item.Identifier) assert.False(t, item.CompletedAt.IsZero(), "CompletedAt is zero for %s", item.Identifier) assert.Equal(t, "Bag copied to long-term storage", *item.Note, "Note is incorrect for %s", item.Identifier) assert.True(t, len(*item.State) > 250, "State JSON too short for %s", item.Identifier) } }
func TestFetchResults(t *testing.T) { if !testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } // Load config configFile := filepath.Join("config", "integration.json") config, err := models.LoadConfigFile(configFile) require.Nil(t, err) config.ExpandFilePaths() // Find the log file that apt_fetch created when it was running // with the "config/integration.json" config options. We'll read // that file. pathToJsonLog := filepath.Join(config.LogDirectory, "apt_fetch.json") for _, bagName := range testutil.INTEGRATION_GOOD_BAGS { ingestManifest, err := testutil.FindIngestManifestInLog(pathToJsonLog, bagName) assert.Nil(t, err) if err != nil { continue } fetcherTestCommon(t, bagName, ingestManifest) // TODO: Test WorkItem (stage, status, etc.) below fetcherTestGoodBagResult(t, bagName, ingestManifest) if bagName == "aptrust.receiving.test.test.edu/example.edu.tagsample_good.tar" { fetcherTestSpecifics(t, ingestManifest) } } for _, bagName := range testutil.INTEGRATION_BAD_BAGS { ingestManifest, err := testutil.FindIngestManifestInLog(pathToJsonLog, bagName) assert.Nil(t, err) if err != nil { continue } fetcherTestCommon(t, bagName, ingestManifest) // TODO: Test WorkItem (stage, status, etc.) below fetcherTestBadBagResult(t, bagName, ingestManifest) } }
func TestValidationDPNWorkItems(t *testing.T) { // Make sure that our code updated the DPNWorkItem record for each // ReplicationTransfer. if !testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } _context, err := testutil.GetContext("integration.json") require.Nil(t, err, "Could not create context") params := url.Values{} params.Set("task", "replication") params.Set("page", "1") params.Set("per_page", "100") resp := _context.PharosClient.DPNWorkItemList(params) require.Nil(t, resp.Error) for _, item := range resp.DPNWorkItems() { require.NotNil(t, item.QueuedAt, "QueuedAt nil for %s", item.Identifier) require.NotNil(t, item.Note, "Note nil for %s", item.Identifier) require.NotNil(t, item.State, "State nil for %s", item.Identifier) assert.False(t, item.QueuedAt.IsZero(), "QueuedAt is zero for %s", item.Identifier) assert.Equal(t, "Bag passed validation", *item.Note, "Note is incorrect for %s", item.Identifier) assert.True(t, len(*item.State) > 250, "State JSON too short for %s", item.Identifier) } }
func TestStoreResults(t *testing.T) { if !testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } // Load config configFile := filepath.Join("config", "integration.json") config, err := models.LoadConfigFile(configFile) require.Nil(t, err) config.ExpandFilePaths() // Find the log file that apt_store created when it was running // with the "config/integration.json" config options. We'll read // that file. pathToJsonLog := filepath.Join(config.LogDirectory, "apt_store.json") for _, bagName := range testutil.INTEGRATION_GOOD_BAGS { ingestManifest, err := testutil.FindIngestManifestInLog(pathToJsonLog, bagName) assert.Nil(t, err) if err != nil { continue } // TODO: Test WorkItem (stage, status, etc.) below storeTestCommon(t, bagName, ingestManifest, config) } }
// We should have created one WorkItem for each DPN ingest request. func TestWorkItemsCreatedAndQueued(t *testing.T) { if !testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } expectedIdentifiers := identifiersPushedToDPN() _context, err := testutil.GetContext("integration.json") require.Nil(t, err, "Could not create context") params := url.Values{} params.Set("item_action", "DPN") params.Set("page", "1") params.Set("per_page", "100") resp := _context.PharosClient.WorkItemList(params) require.Nil(t, resp.Error) assert.Equal(t, len(expectedIdentifiers), resp.Count) for _, workItem := range resp.WorkItems() { found := false queued := false currentIdentifier := "" for _, identifier := range expectedIdentifiers { currentIdentifier = identifier if workItem.ObjectIdentifier == identifier { found = true if workItem.QueuedAt != nil && !workItem.QueuedAt.IsZero() { queued = true } break } } assert.True(t, found, "No WorkItem for object %s", currentIdentifier) assert.True(t, queued, "Object %s was not queued", currentIdentifier) } // In addition to checking whether Pharos thinks the items are queued, // let's ask NSQ as well. stats, err := _context.NSQClient.GetStats() require.Nil(t, err) foundFetchTopic := false foundStoreTopic := false foundRecordTopic := false for _, topic := range stats.Data.Topics { if topic.TopicName == _context.Config.FetchWorker.NsqTopic { // We fetch 16 bags in our integration tests. // They're not all valid, but we should have that many in the queue. foundFetchTopic = true assert.EqualValues(t, uint64(16), topic.MessageCount) } else if topic.TopicName == _context.Config.StoreWorker.NsqTopic { // All of the 11 valid bags should have made it into the store topic. foundStoreTopic = true assert.EqualValues(t, uint64(11), topic.MessageCount) } else if topic.TopicName == _context.Config.RecordWorker.NsqTopic { // All of the 11 valid bags should have made it into the record topic. foundRecordTopic = true assert.EqualValues(t, uint64(11), topic.MessageCount) } } assert.True(t, foundFetchTopic, "Nothing was queued in %s", _context.Config.FetchWorker.NsqTopic) assert.True(t, foundStoreTopic, "Nothing was queued in %s", _context.Config.StoreWorker.NsqTopic) assert.True(t, foundRecordTopic, "Nothing was queued in %s", _context.Config.RecordWorker.NsqTopic) }
// We should have created one DPNWorkItem for each replication request // that we synched from the other nodes. func TestDPNWorkItemsCreatedAndQueued(t *testing.T) { if !testutil.ShouldRunIntegrationTests() { t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.") } // Use local DPN REST client to check for all replication requests // where ToNode is our LocalNode. // // Then check Pharos for a DPNWorkItem for each of these replications. // The DPNWorkItem should exist, and should have a QueuedAt timestamp. _context, err := testutil.GetContext("integration.json") require.Nil(t, err, "Could not create context") // Check DPNWorkItems for ReplicationTransfers dpnClient, err := network.NewDPNRestClient( _context.Config.DPN.RestClient.LocalServiceURL, _context.Config.DPN.RestClient.LocalAPIRoot, _context.Config.DPN.RestClient.LocalAuthToken, _context.Config.DPN.LocalNode, _context.Config.DPN) require.Nil(t, err) xferParams := url.Values{} xferParams.Set("to_node", _context.Config.DPN.LocalNode) dpnResp := dpnClient.ReplicationTransferList(xferParams) require.Nil(t, dpnResp.Error) for _, xfer := range dpnResp.ReplicationTransfers() { params := url.Values{} params.Set("identifier", xfer.ReplicationId) params.Set("task", "replication") pharosResp := _context.PharosClient.DPNWorkItemList(params) require.Nil(t, pharosResp.Error) require.Equal(t, 1, pharosResp.Count) dpnWorkItem := pharosResp.DPNWorkItem() require.NotNil(t, dpnWorkItem.QueuedAt) assert.False(t, dpnWorkItem.QueuedAt.IsZero()) assert.Equal(t, xfer.FromNode, dpnWorkItem.RemoteNode) } // Check DPNWorkItems RestoreTransfers xferParams.Set("from_node", _context.Config.DPN.LocalNode) dpnResp = dpnClient.RestoreTransferList(xferParams) require.Nil(t, dpnResp.Error) for _, xfer := range dpnResp.RestoreTransfers() { params := url.Values{} params.Set("identifier", xfer.RestoreId) params.Set("task", "restore") pharosResp := _context.PharosClient.DPNWorkItemList(params) require.Nil(t, pharosResp.Error) require.Equal(t, 1, pharosResp.Count) dpnWorkItem := pharosResp.DPNWorkItem() require.NotNil(t, dpnWorkItem.QueuedAt) assert.False(t, dpnWorkItem.QueuedAt.IsZero()) assert.Equal(t, xfer.ToNode, dpnWorkItem.RemoteNode) } // Check NSQ as well. stats, err := _context.NSQClient.GetStats() require.Nil(t, err) foundPackageTopic := false foundCopyTopic := false foundRestoreTopic := false for _, topic := range stats.Data.Topics { if topic.TopicName == _context.Config.DPN.DPNPackageWorker.NsqTopic { // apps/test_push_to_dpn.go requests that items // testutil.INTEGRATION_GOOD_BAGS[0:7] be sent to DPN, // so we should find seven items in the package queue foundPackageTopic = true assert.EqualValues(t, uint64(7), topic.MessageCount) } else if topic.TopicName == _context.Config.DPN.DPNCopyWorker.NsqTopic { // Fixture data has 4 replications: one from each remote node foundCopyTopic = true assert.EqualValues(t, uint64(4), topic.MessageCount) } else if topic.TopicName == _context.Config.DPN.DPNRestoreWorker.NsqTopic { // Fixture data has 4 restores: one from each remote node foundRestoreTopic = true assert.EqualValues(t, uint64(4), topic.MessageCount) } } assert.True(t, foundPackageTopic, "Nothing was queued in %s", _context.Config.DPN.DPNPackageWorker.NsqTopic) assert.True(t, foundCopyTopic, "Nothing was queued in %s", _context.Config.DPN.DPNCopyWorker.NsqTopic) assert.True(t, foundRestoreTopic, "Nothing was queued in %s", _context.Config.DPN.DPNRestoreWorker.NsqTopic) }