// TestItemsAreInLongTermStorage - make sure that each tar file is
// stored in our S3 test storage bucket, with correct metadata.
func TestItemsAreInLongTermStorage(t *testing.T) {
if !apt_testutil.ShouldRunIntegrationTests() {
t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
}
_context, err := apt_testutil.GetContext("integration.json")
require.Nil(t, err, "Could not create context")
localClient, err := network.NewDPNRestClient(
_context.Config.DPN.RestClient.LocalServiceURL,
_context.Config.DPN.RestClient.LocalAPIRoot,
_context.Config.DPN.RestClient.LocalAuthToken,
_context.Config.DPN.LocalNode,
_context.Config.DPN)
require.Nil(t, err)
// s3List lists bucket contents.
s3List := apt_network.NewS3ObjectList(
constants.AWSVirginia,
_context.Config.DPN.DPNPreservationBucket,
int64(100),
)
// s3Head gets metadata about specific objects in S3/Glacier.
s3Head := apt_network.NewS3Head(_context.Config.APTrustS3Region,
_context.Config.DPN.DPNPreservationBucket)
for _, identifier := range dpn_testutil.BAG_IDS {
resp := localClient.DPNBagGet(identifier)
dpnBag := resp.Bag()
require.NotNil(t, dpnBag)
if dpnBag.IngestNode == _context.Config.DPN.LocalNode {
continue // we would not have replicated our own bag
}
tarFileName := fmt.Sprintf("%s.tar", identifier)
s3List.GetList(tarFileName)
require.NotEmpty(t, s3List.Response.Contents, "%s not found in S3", tarFileName)
object := s3List.Response.Contents[0]
fiveMinutesAgo := time.Now().UTC().Add(-5 * time.Minute)
require.NotNil(t, object.LastModified)
assert.True(t, object.LastModified.After(fiveMinutesAgo))
// Make sure each item has the expected metadata.
// s3Head.Response.Metadata is map[string]*string.
s3Head.Head(tarFileName)
require.Empty(t, s3Head.ErrorMessage)
metadata := s3Head.Response.Metadata
require.NotNil(t, metadata)
// Amazon library transforms first letters of keys to CAPS
require.NotNil(t, metadata["From_node"])
require.NotNil(t, metadata["Transfer_id"])
require.NotNil(t, metadata["Member"])
require.NotNil(t, metadata["Local_id"])
require.NotNil(t, metadata["Version"])
assert.NotEmpty(t, *metadata["From_node"])
assert.NotEmpty(t, *metadata["Transfer_id"])
assert.NotEmpty(t, *metadata["Member"])
assert.NotEmpty(t, *metadata["Local_id"])
assert.NotEmpty(t, *metadata["Version"])
}
}
// NewDPNPackager creates a new DPNPackager.
func NewDPNPackager(_context *context.Context) (*DPNPackager, error) {
localClient, err := network.NewDPNRestClient(
_context.Config.DPN.RestClient.LocalServiceURL,
_context.Config.DPN.RestClient.LocalAPIRoot,
_context.Config.DPN.RestClient.LocalAuthToken,
_context.Config.DPN.LocalNode,
_context.Config.DPN)
if err != nil {
return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
}
remoteClients, err := localClient.GetRemoteClients()
if err != nil {
return nil, err
}
packager := &DPNPackager{
Context: _context,
LocalClient: localClient,
RemoteClients: remoteClients,
}
packager.BagValidationConfig = LoadBagValidationConfig(packager.Context)
workerBufferSize := _context.Config.DPN.DPNPackageWorker.Workers * 4
packager.PackageChannel = make(chan *models.DPNIngestManifest, workerBufferSize)
packager.TarChannel = make(chan *models.DPNIngestManifest, workerBufferSize)
packager.ValidationChannel = make(chan *models.DPNIngestManifest, workerBufferSize)
packager.PostProcessChannel = make(chan *models.DPNIngestManifest, workerBufferSize)
for i := 0; i < _context.Config.DPN.DPNPackageWorker.Workers; i++ {
go packager.buildBag()
go packager.tarBag()
go packager.validate()
go packager.postProcess()
}
return packager, nil
}
// TestReplicationsMarkedAsStored - make sure that the ReplicationTransfer
// records are marked with stored = true on the remote nodes.
func TestReplicationsMarkedAsStored(t *testing.T) {
if !apt_testutil.ShouldRunIntegrationTests() {
t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
}
_context, err := apt_testutil.GetContext("integration.json")
require.Nil(t, err, "Could not create context")
// Get a list of ReplicationTransfers where our local node
// is the ToNode, and make sure we marked them all as stored
// on the FromNode.
localClient, err := network.NewDPNRestClient(
_context.Config.DPN.RestClient.LocalServiceURL,
_context.Config.DPN.RestClient.LocalAPIRoot,
_context.Config.DPN.RestClient.LocalAuthToken,
_context.Config.DPN.LocalNode,
_context.Config.DPN)
require.Nil(t, err)
remoteClients, err := localClient.GetRemoteClients()
require.Nil(t, err)
xferParams := url.Values{}
xferParams.Set("to_node", _context.Config.DPN.LocalNode)
dpnResp := localClient.ReplicationTransferList(xferParams)
require.Nil(t, dpnResp.Error)
for _, xfer := range dpnResp.ReplicationTransfers() {
remoteClient := remoteClients[xfer.FromNode]
require.NotNil(t, remoteClient)
resp := remoteClient.ReplicationTransferGet(xfer.ReplicationId)
require.Nil(t, resp.Error)
remoteXfer := resp.ReplicationTransfer()
require.NotNil(t, remoteXfer)
assert.True(t, remoteXfer.Stored)
}
}
// NewDPNSync creates a new DPNSync object.
func NewDPNSync(_context *context.Context) (*DPNSync, error) {
if _context == nil {
return nil, fmt.Errorf("Param _context cannot be nil.")
}
localClient, err := network.NewDPNRestClient(
_context.Config.DPN.RestClient.LocalServiceURL,
_context.Config.DPN.DPNAPIVersion,
_context.Config.DPN.RestClient.LocalAuthToken,
_context.Config.DPN.LocalNode,
_context.Config.DPN)
if err != nil {
return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
}
remoteClients, err := localClient.GetRemoteClients()
if err != nil {
return nil, fmt.Errorf("Error creating remote DPN REST client: %v", err)
}
results := make(map[string]*models.SyncResult)
for nodeName := range remoteClients {
results[nodeName] = models.NewSyncResult(nodeName)
}
sync := DPNSync{
LocalClient: localClient,
RemoteNodes: make(map[string]*models.Node),
RemoteClients: remoteClients,
Context: _context,
Results: results,
}
return &sync, nil
}
// NewDPNValidator returns a new DPNValidator object.
func NewDPNValidator(_context *context.Context) (*DPNValidator, error) {
localClient, err := network.NewDPNRestClient(
_context.Config.DPN.RestClient.LocalServiceURL,
_context.Config.DPN.RestClient.LocalAPIRoot,
_context.Config.DPN.RestClient.LocalAuthToken,
_context.Config.DPN.LocalNode,
_context.Config.DPN)
if err != nil {
return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
}
remoteClients, err := localClient.GetRemoteClients()
if err != nil {
return nil, err
}
validator := &DPNValidator{
Context: _context,
LocalClient: localClient,
RemoteClients: remoteClients,
}
validator.BagValidationConfig = LoadBagValidationConfig(validator.Context)
workerBufferSize := _context.Config.DPN.DPNValidationWorker.Workers * 4
validator.ValidationChannel = make(chan *models.ReplicationManifest, workerBufferSize)
validator.PostProcessChannel = make(chan *models.ReplicationManifest, workerBufferSize)
for i := 0; i < _context.Config.DPN.DPNValidationWorker.Workers; i++ {
go validator.validate()
go validator.postProcess()
}
return validator, nil
}
// NewDPNQueue creates a new DPNQueue object. Param _context is a Context
// object, and param hours tells the code to examine all Replication,
// Restore and DPN Ingest requests from the past N hours.
func NewDPNQueue(_context *context.Context, hours int) (*DPNQueue, error) {
if _context == nil {
return nil, fmt.Errorf("Param _context cannot be nil.")
}
localClient, err := network.NewDPNRestClient(
_context.Config.DPN.RestClient.LocalServiceURL,
_context.Config.DPN.DPNAPIVersion,
_context.Config.DPN.RestClient.LocalAuthToken,
_context.Config.DPN.LocalNode,
_context.Config.DPN)
if err != nil {
return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
}
remoteClients, err := localClient.GetRemoteClients()
if err != nil {
return nil, fmt.Errorf("Error creating remote DPN REST client: %v", err)
}
sinceWhen := time.Now().UTC().Add(time.Duration(-1*hours) * time.Hour)
_context.MessageLog.Info("Checking records since %d hours ago (%s)",
hours, sinceWhen.Format(time.RFC3339))
dpnQueue := DPNQueue{
LocalClient: localClient,
RemoteNodes: make(map[string]*models.Node),
RemoteClients: remoteClients,
Context: _context,
ExamineItemsSince: sinceWhen,
QueueResult: models.NewQueueResult(),
}
return &dpnQueue, nil
}
// NewDPNReplicationStorer creates a new DPNReplicationStorer object.
func NewDPNReplicationStorer(_context *context.Context) (*DPNReplicationStorer, error) {
localClient, err := network.NewDPNRestClient(
_context.Config.DPN.RestClient.LocalServiceURL,
_context.Config.DPN.RestClient.LocalAPIRoot,
_context.Config.DPN.RestClient.LocalAuthToken,
_context.Config.DPN.LocalNode,
_context.Config.DPN)
if err != nil {
return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
}
remoteClients, err := localClient.GetRemoteClients()
if err != nil {
return nil, err
}
storer := &DPNReplicationStorer{
Context: _context,
LocalClient: localClient,
RemoteClients: remoteClients,
}
workerBufferSize := _context.Config.DPN.DPNReplicationStoreWorker.Workers * 4
storer.StoreChannel = make(chan *models.ReplicationManifest, workerBufferSize)
storer.PostProcessChannel = make(chan *models.ReplicationManifest, workerBufferSize)
for i := 0; i < _context.Config.DPN.DPNReplicationStoreWorker.Workers; i++ {
go storer.store()
go storer.postProcess()
}
return storer, nil
}
func TestRawResponseData(t *testing.T) {
// nodeGetHandler is defined in dpn_rest_client_test.go
testServer := httptest.NewServer(http.HandlerFunc(nodeGetHandler))
defer testServer.Close()
// configFile is defined in dpn_rest_client_test.go
config, err := models.LoadConfigFile(configFile)
require.Nil(t, err)
client, err := network.NewDPNRestClient(
testServer.URL,
"",
"",
config.DPN.LocalNode,
config.DPN)
if err != nil {
t.Error(err)
return
}
resp := client.NodeGet("luna")
// Should be able to call repeatedly without error.
// Incorrect implementation would try to read from
// closed network socket.
for i := 0; i < 3; i++ {
bytes, err := resp.RawResponseData()
assert.NotNil(t, bytes)
assert.NotEmpty(t, bytes)
assert.Nil(t, err)
}
}
func getPostTestClient(t *testing.T) *network.DPNRestClient {
// If you want to debug, change ioutil.Discard to os.Stdout
// to see log output from the client.
config, err := apt_models.LoadConfigFile(filepath.Join("config", "integration.json"))
require.Nil(t, err)
client, err := network.NewDPNRestClient(
config.DPN.RestClient.LocalServiceURL,
config.DPN.RestClient.LocalAPIRoot,
config.DPN.RestClient.LocalAuthToken,
config.DPN.LocalNode,
config.DPN)
require.Nil(t, err)
return client
}
func getClient(t *testing.T) *network.DPNRestClient {
// If you want to debug, change ioutil.Discard to os.Stdout
// to see log output from the client.
config, err := apt_models.LoadConfigFile(configFile)
require.Nil(t, err)
client, err := network.NewDPNRestClient(
config.DPN.RestClient.LocalServiceURL,
config.DPN.RestClient.LocalAPIRoot,
config.DPN.RestClient.LocalAuthToken,
config.DPN.LocalNode,
config.DPN)
if err != nil {
t.Errorf("Error constructing DPN REST client: %v", err)
}
return client
}
func TestCopyResultSentToRemoteNodes(t *testing.T) {
// Query the FromNode of each copied bag to make sure that
// we sent a fixity value back to the ingest node.
if !testutil.ShouldRunIntegrationTests() {
t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
}
_context, err := testutil.GetContext("integration.json")
require.Nil(t, err, "Could not create context")
dpnClient, err := network.NewDPNRestClient(
_context.Config.DPN.RestClient.LocalServiceURL,
_context.Config.DPN.RestClient.LocalAPIRoot,
_context.Config.DPN.RestClient.LocalAuthToken,
_context.Config.DPN.LocalNode,
_context.Config.DPN)
require.Nil(t, err, "Couldn't build DPN REST client: %v", err)
remoteClients, err := dpnClient.GetRemoteClients()
require.Nil(t, err, "Couldn't build remote DPN clients: %v", err)
// These identifiers are in the fixture data for dpn-server.
// Key is the FromNode, value is the ReplicationId
xferIdentifiers := map[string]string{
"chron": "20000000-0000-4000-a000-000000000007",
"hathi": "30000000-0000-4000-a000-000000000001",
"sdr": "40000000-0000-4000-a000-000000000013",
"tdr": "50000000-0000-4000-a000-000000000019",
}
for fromNode, identifier := range xferIdentifiers {
client := remoteClients[fromNode]
require.NotNil(t, client, "No DPN REST client for %s", fromNode)
resp := client.ReplicationTransferGet(identifier)
require.Nil(t, resp.Error)
xfer := resp.ReplicationTransfer()
require.NotNil(t, xfer, "ReplicationTransfer %s is missing", identifier)
assert.NotEmpty(t, xfer.FixityValue, "Empty fixity for %s", identifier)
assert.True(t, xfer.StoreRequested, "StoreRequested should not be false for %s", identifier)
}
}
// We should have created one DPNWorkItem for each replication request
// that we synched from the other nodes.
func TestDPNWorkItemsCreatedAndQueued(t *testing.T) {
if !testutil.ShouldRunIntegrationTests() {
t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
}
// Use local DPN REST client to check for all replication requests
// where ToNode is our LocalNode.
//
// Then check Pharos for a DPNWorkItem for each of these replications.
// The DPNWorkItem should exist, and should have a QueuedAt timestamp.
_context, err := testutil.GetContext("integration.json")
require.Nil(t, err, "Could not create context")
// Check DPNWorkItems for ReplicationTransfers
dpnClient, err := network.NewDPNRestClient(
_context.Config.DPN.RestClient.LocalServiceURL,
_context.Config.DPN.RestClient.LocalAPIRoot,
_context.Config.DPN.RestClient.LocalAuthToken,
_context.Config.DPN.LocalNode,
_context.Config.DPN)
require.Nil(t, err)
xferParams := url.Values{}
xferParams.Set("to_node", _context.Config.DPN.LocalNode)
dpnResp := dpnClient.ReplicationTransferList(xferParams)
require.Nil(t, dpnResp.Error)
for _, xfer := range dpnResp.ReplicationTransfers() {
params := url.Values{}
params.Set("identifier", xfer.ReplicationId)
params.Set("task", "replication")
pharosResp := _context.PharosClient.DPNWorkItemList(params)
require.Nil(t, pharosResp.Error)
require.Equal(t, 1, pharosResp.Count)
dpnWorkItem := pharosResp.DPNWorkItem()
require.NotNil(t, dpnWorkItem.QueuedAt)
assert.False(t, dpnWorkItem.QueuedAt.IsZero())
assert.Equal(t, xfer.FromNode, dpnWorkItem.RemoteNode)
}
// Check DPNWorkItems RestoreTransfers
xferParams.Set("from_node", _context.Config.DPN.LocalNode)
dpnResp = dpnClient.RestoreTransferList(xferParams)
require.Nil(t, dpnResp.Error)
for _, xfer := range dpnResp.RestoreTransfers() {
params := url.Values{}
params.Set("identifier", xfer.RestoreId)
params.Set("task", "restore")
pharosResp := _context.PharosClient.DPNWorkItemList(params)
require.Nil(t, pharosResp.Error)
require.Equal(t, 1, pharosResp.Count)
dpnWorkItem := pharosResp.DPNWorkItem()
require.NotNil(t, dpnWorkItem.QueuedAt)
assert.False(t, dpnWorkItem.QueuedAt.IsZero())
assert.Equal(t, xfer.ToNode, dpnWorkItem.RemoteNode)
}
// Check NSQ as well.
stats, err := _context.NSQClient.GetStats()
require.Nil(t, err)
foundPackageTopic := false
foundCopyTopic := false
foundRestoreTopic := false
for _, topic := range stats.Data.Topics {
if topic.TopicName == _context.Config.DPN.DPNPackageWorker.NsqTopic {
// apps/test_push_to_dpn.go requests that items
// testutil.INTEGRATION_GOOD_BAGS[0:7] be sent to DPN,
// so we should find seven items in the package queue
foundPackageTopic = true
assert.EqualValues(t, uint64(7), topic.MessageCount)
} else if topic.TopicName == _context.Config.DPN.DPNCopyWorker.NsqTopic {
// Fixture data has 4 replications: one from each remote node
foundCopyTopic = true
assert.EqualValues(t, uint64(4), topic.MessageCount)
} else if topic.TopicName == _context.Config.DPN.DPNRestoreWorker.NsqTopic {
// Fixture data has 4 restores: one from each remote node
foundRestoreTopic = true
assert.EqualValues(t, uint64(4), topic.MessageCount)
}
}
assert.True(t, foundPackageTopic, "Nothing was queued in %s",
_context.Config.DPN.DPNPackageWorker.NsqTopic)
assert.True(t, foundCopyTopic, "Nothing was queued in %s",
_context.Config.DPN.DPNCopyWorker.NsqTopic)
assert.True(t, foundRestoreTopic, "Nothing was queued in %s",
_context.Config.DPN.DPNRestoreWorker.NsqTopic)
}