// TestItemsAreInLongTermStorage - make sure that each tar file is
// stored in our S3 test storage bucket, with correct metadata.
func TestItemsAreInLongTermStorage(t *testing.T) {
	if !apt_testutil.ShouldRunIntegrationTests() {
		t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
	}
	_context, err := apt_testutil.GetContext("integration.json")
	require.Nil(t, err, "Could not create context")

	localClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	require.Nil(t, err)

	// s3List lists bucket contents.
	s3List := apt_network.NewS3ObjectList(
		constants.AWSVirginia,
		_context.Config.DPN.DPNPreservationBucket,
		int64(100),
	)
	// s3Head gets metadata about specific objects in S3/Glacier.
	s3Head := apt_network.NewS3Head(_context.Config.APTrustS3Region,
		_context.Config.DPN.DPNPreservationBucket)

	for _, identifier := range dpn_testutil.BAG_IDS {
		resp := localClient.DPNBagGet(identifier)
		dpnBag := resp.Bag()
		require.NotNil(t, dpnBag)
		if dpnBag.IngestNode == _context.Config.DPN.LocalNode {
			continue // we would not have replicated our own bag
		}
		tarFileName := fmt.Sprintf("%s.tar", identifier)
		s3List.GetList(tarFileName)
		require.NotEmpty(t, s3List.Response.Contents, "%s not found in S3", tarFileName)
		object := s3List.Response.Contents[0]
		fiveMinutesAgo := time.Now().UTC().Add(-5 * time.Minute)
		require.NotNil(t, object.LastModified)
		assert.True(t, object.LastModified.After(fiveMinutesAgo))

		// Make sure each item has the expected metadata.
		// s3Head.Response.Metadata is map[string]*string.
		s3Head.Head(tarFileName)
		require.Empty(t, s3Head.ErrorMessage)
		metadata := s3Head.Response.Metadata
		require.NotNil(t, metadata)
		// Amazon library transforms first letters of keys to CAPS
		require.NotNil(t, metadata["From_node"])
		require.NotNil(t, metadata["Transfer_id"])
		require.NotNil(t, metadata["Member"])
		require.NotNil(t, metadata["Local_id"])
		require.NotNil(t, metadata["Version"])

		assert.NotEmpty(t, *metadata["From_node"])
		assert.NotEmpty(t, *metadata["Transfer_id"])
		assert.NotEmpty(t, *metadata["Member"])
		assert.NotEmpty(t, *metadata["Local_id"])
		assert.NotEmpty(t, *metadata["Version"])
	}
}
Example #2
0
// NewDPNPackager creates a new DPNPackager.
func NewDPNPackager(_context *context.Context) (*DPNPackager, error) {
	localClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := localClient.GetRemoteClients()
	if err != nil {
		return nil, err
	}
	packager := &DPNPackager{
		Context:       _context,
		LocalClient:   localClient,
		RemoteClients: remoteClients,
	}
	packager.BagValidationConfig = LoadBagValidationConfig(packager.Context)
	workerBufferSize := _context.Config.DPN.DPNPackageWorker.Workers * 4
	packager.PackageChannel = make(chan *models.DPNIngestManifest, workerBufferSize)
	packager.TarChannel = make(chan *models.DPNIngestManifest, workerBufferSize)
	packager.ValidationChannel = make(chan *models.DPNIngestManifest, workerBufferSize)
	packager.PostProcessChannel = make(chan *models.DPNIngestManifest, workerBufferSize)
	for i := 0; i < _context.Config.DPN.DPNPackageWorker.Workers; i++ {
		go packager.buildBag()
		go packager.tarBag()
		go packager.validate()
		go packager.postProcess()
	}
	return packager, nil
}
// TestReplicationsMarkedAsStored - make sure that the ReplicationTransfer
// records are marked with stored = true on the remote nodes.
func TestReplicationsMarkedAsStored(t *testing.T) {
	if !apt_testutil.ShouldRunIntegrationTests() {
		t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
	}
	_context, err := apt_testutil.GetContext("integration.json")
	require.Nil(t, err, "Could not create context")

	// Get a list of ReplicationTransfers where our local node
	// is the ToNode, and make sure we marked them all as stored
	// on the FromNode.
	localClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	require.Nil(t, err)
	remoteClients, err := localClient.GetRemoteClients()
	require.Nil(t, err)

	xferParams := url.Values{}
	xferParams.Set("to_node", _context.Config.DPN.LocalNode)
	dpnResp := localClient.ReplicationTransferList(xferParams)
	require.Nil(t, dpnResp.Error)
	for _, xfer := range dpnResp.ReplicationTransfers() {
		remoteClient := remoteClients[xfer.FromNode]
		require.NotNil(t, remoteClient)
		resp := remoteClient.ReplicationTransferGet(xfer.ReplicationId)
		require.Nil(t, resp.Error)
		remoteXfer := resp.ReplicationTransfer()
		require.NotNil(t, remoteXfer)
		assert.True(t, remoteXfer.Stored)
	}
}
Example #4
0
// NewDPNSync creates a new DPNSync object.
func NewDPNSync(_context *context.Context) (*DPNSync, error) {
	if _context == nil {
		return nil, fmt.Errorf("Param _context cannot be nil.")
	}
	localClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.DPNAPIVersion,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := localClient.GetRemoteClients()
	if err != nil {
		return nil, fmt.Errorf("Error creating remote DPN REST client: %v", err)
	}
	results := make(map[string]*models.SyncResult)
	for nodeName := range remoteClients {
		results[nodeName] = models.NewSyncResult(nodeName)
	}
	sync := DPNSync{
		LocalClient:   localClient,
		RemoteNodes:   make(map[string]*models.Node),
		RemoteClients: remoteClients,
		Context:       _context,
		Results:       results,
	}
	return &sync, nil
}
Example #5
0
// NewDPNValidator returns a new DPNValidator object.
func NewDPNValidator(_context *context.Context) (*DPNValidator, error) {
	localClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := localClient.GetRemoteClients()
	if err != nil {
		return nil, err
	}
	validator := &DPNValidator{
		Context:       _context,
		LocalClient:   localClient,
		RemoteClients: remoteClients,
	}
	validator.BagValidationConfig = LoadBagValidationConfig(validator.Context)
	workerBufferSize := _context.Config.DPN.DPNValidationWorker.Workers * 4
	validator.ValidationChannel = make(chan *models.ReplicationManifest, workerBufferSize)
	validator.PostProcessChannel = make(chan *models.ReplicationManifest, workerBufferSize)
	for i := 0; i < _context.Config.DPN.DPNValidationWorker.Workers; i++ {
		go validator.validate()
		go validator.postProcess()
	}
	return validator, nil
}
Example #6
0
// NewDPNQueue creates a new DPNQueue object. Param _context is a Context
// object, and param hours tells the code to examine all Replication,
// Restore and DPN Ingest requests from the past N hours.
func NewDPNQueue(_context *context.Context, hours int) (*DPNQueue, error) {
	if _context == nil {
		return nil, fmt.Errorf("Param _context cannot be nil.")
	}
	localClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.DPNAPIVersion,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := localClient.GetRemoteClients()
	if err != nil {
		return nil, fmt.Errorf("Error creating remote DPN REST client: %v", err)
	}
	sinceWhen := time.Now().UTC().Add(time.Duration(-1*hours) * time.Hour)
	_context.MessageLog.Info("Checking records since %d hours ago (%s)",
		hours, sinceWhen.Format(time.RFC3339))
	dpnQueue := DPNQueue{
		LocalClient:       localClient,
		RemoteNodes:       make(map[string]*models.Node),
		RemoteClients:     remoteClients,
		Context:           _context,
		ExamineItemsSince: sinceWhen,
		QueueResult:       models.NewQueueResult(),
	}
	return &dpnQueue, nil
}
// NewDPNReplicationStorer creates a new DPNReplicationStorer object.
func NewDPNReplicationStorer(_context *context.Context) (*DPNReplicationStorer, error) {
	localClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	if err != nil {
		return nil, fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	remoteClients, err := localClient.GetRemoteClients()
	if err != nil {
		return nil, err
	}
	storer := &DPNReplicationStorer{
		Context:       _context,
		LocalClient:   localClient,
		RemoteClients: remoteClients,
	}
	workerBufferSize := _context.Config.DPN.DPNReplicationStoreWorker.Workers * 4
	storer.StoreChannel = make(chan *models.ReplicationManifest, workerBufferSize)
	storer.PostProcessChannel = make(chan *models.ReplicationManifest, workerBufferSize)
	for i := 0; i < _context.Config.DPN.DPNReplicationStoreWorker.Workers; i++ {
		go storer.store()
		go storer.postProcess()
	}
	return storer, nil
}
Example #8
0
func TestRawResponseData(t *testing.T) {
	// nodeGetHandler is defined in dpn_rest_client_test.go
	testServer := httptest.NewServer(http.HandlerFunc(nodeGetHandler))
	defer testServer.Close()

	// configFile is defined in dpn_rest_client_test.go
	config, err := models.LoadConfigFile(configFile)
	require.Nil(t, err)
	client, err := network.NewDPNRestClient(
		testServer.URL,
		"",
		"",
		config.DPN.LocalNode,
		config.DPN)
	if err != nil {
		t.Error(err)
		return
	}
	resp := client.NodeGet("luna")

	// Should be able to call repeatedly without error.
	// Incorrect implementation would try to read from
	// closed network socket.
	for i := 0; i < 3; i++ {
		bytes, err := resp.RawResponseData()
		assert.NotNil(t, bytes)
		assert.NotEmpty(t, bytes)
		assert.Nil(t, err)
	}
}
Example #9
0
func getPostTestClient(t *testing.T) *network.DPNRestClient {
	// If you want to debug, change ioutil.Discard to os.Stdout
	// to see log output from the client.
	config, err := apt_models.LoadConfigFile(filepath.Join("config", "integration.json"))
	require.Nil(t, err)
	client, err := network.NewDPNRestClient(
		config.DPN.RestClient.LocalServiceURL,
		config.DPN.RestClient.LocalAPIRoot,
		config.DPN.RestClient.LocalAuthToken,
		config.DPN.LocalNode,
		config.DPN)
	require.Nil(t, err)
	return client
}
Example #10
0
func getClient(t *testing.T) *network.DPNRestClient {
	// If you want to debug, change ioutil.Discard to os.Stdout
	// to see log output from the client.
	config, err := apt_models.LoadConfigFile(configFile)
	require.Nil(t, err)
	client, err := network.NewDPNRestClient(
		config.DPN.RestClient.LocalServiceURL,
		config.DPN.RestClient.LocalAPIRoot,
		config.DPN.RestClient.LocalAuthToken,
		config.DPN.LocalNode,
		config.DPN)
	if err != nil {
		t.Errorf("Error constructing DPN REST client: %v", err)
	}
	return client
}
Example #11
0
func TestCopyResultSentToRemoteNodes(t *testing.T) {
	// Query the FromNode of each copied bag to make sure that
	// we sent a fixity value back to the ingest node.
	if !testutil.ShouldRunIntegrationTests() {
		t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
	}
	_context, err := testutil.GetContext("integration.json")
	require.Nil(t, err, "Could not create context")
	dpnClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	require.Nil(t, err, "Couldn't build DPN REST client: %v", err)

	remoteClients, err := dpnClient.GetRemoteClients()
	require.Nil(t, err, "Couldn't build remote DPN clients: %v", err)

	// These identifiers are in the fixture data for dpn-server.
	// Key is the FromNode, value is the ReplicationId
	xferIdentifiers := map[string]string{
		"chron": "20000000-0000-4000-a000-000000000007",
		"hathi": "30000000-0000-4000-a000-000000000001",
		"sdr":   "40000000-0000-4000-a000-000000000013",
		"tdr":   "50000000-0000-4000-a000-000000000019",
	}

	for fromNode, identifier := range xferIdentifiers {
		client := remoteClients[fromNode]
		require.NotNil(t, client, "No DPN REST client for %s", fromNode)
		resp := client.ReplicationTransferGet(identifier)
		require.Nil(t, resp.Error)
		xfer := resp.ReplicationTransfer()
		require.NotNil(t, xfer, "ReplicationTransfer %s is missing", identifier)
		assert.NotEmpty(t, xfer.FixityValue, "Empty fixity for %s", identifier)
		assert.True(t, xfer.StoreRequested, "StoreRequested should not be false for %s", identifier)
	}
}
Example #12
0
// We should have created one DPNWorkItem for each replication request
// that we synched from the other nodes.
func TestDPNWorkItemsCreatedAndQueued(t *testing.T) {
	if !testutil.ShouldRunIntegrationTests() {
		t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
	}
	// Use local DPN REST client to check for all replication requests
	// where ToNode is our LocalNode.
	//
	// Then check Pharos for a DPNWorkItem for each of these replications.
	// The DPNWorkItem should exist, and should have a QueuedAt timestamp.
	_context, err := testutil.GetContext("integration.json")
	require.Nil(t, err, "Could not create context")

	// Check DPNWorkItems for ReplicationTransfers
	dpnClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	require.Nil(t, err)
	xferParams := url.Values{}
	xferParams.Set("to_node", _context.Config.DPN.LocalNode)
	dpnResp := dpnClient.ReplicationTransferList(xferParams)
	require.Nil(t, dpnResp.Error)
	for _, xfer := range dpnResp.ReplicationTransfers() {
		params := url.Values{}
		params.Set("identifier", xfer.ReplicationId)
		params.Set("task", "replication")
		pharosResp := _context.PharosClient.DPNWorkItemList(params)
		require.Nil(t, pharosResp.Error)
		require.Equal(t, 1, pharosResp.Count)
		dpnWorkItem := pharosResp.DPNWorkItem()
		require.NotNil(t, dpnWorkItem.QueuedAt)
		assert.False(t, dpnWorkItem.QueuedAt.IsZero())
		assert.Equal(t, xfer.FromNode, dpnWorkItem.RemoteNode)
	}

	// Check DPNWorkItems RestoreTransfers
	xferParams.Set("from_node", _context.Config.DPN.LocalNode)
	dpnResp = dpnClient.RestoreTransferList(xferParams)
	require.Nil(t, dpnResp.Error)
	for _, xfer := range dpnResp.RestoreTransfers() {
		params := url.Values{}
		params.Set("identifier", xfer.RestoreId)
		params.Set("task", "restore")
		pharosResp := _context.PharosClient.DPNWorkItemList(params)
		require.Nil(t, pharosResp.Error)
		require.Equal(t, 1, pharosResp.Count)
		dpnWorkItem := pharosResp.DPNWorkItem()
		require.NotNil(t, dpnWorkItem.QueuedAt)
		assert.False(t, dpnWorkItem.QueuedAt.IsZero())
		assert.Equal(t, xfer.ToNode, dpnWorkItem.RemoteNode)
	}

	// Check NSQ as well.
	stats, err := _context.NSQClient.GetStats()
	require.Nil(t, err)
	foundPackageTopic := false
	foundCopyTopic := false
	foundRestoreTopic := false
	for _, topic := range stats.Data.Topics {
		if topic.TopicName == _context.Config.DPN.DPNPackageWorker.NsqTopic {
			// apps/test_push_to_dpn.go requests that items
			// testutil.INTEGRATION_GOOD_BAGS[0:7] be sent to DPN,
			// so we should find seven items in the package queue
			foundPackageTopic = true
			assert.EqualValues(t, uint64(7), topic.MessageCount)
		} else if topic.TopicName == _context.Config.DPN.DPNCopyWorker.NsqTopic {
			// Fixture data has 4 replications: one from each remote node
			foundCopyTopic = true
			assert.EqualValues(t, uint64(4), topic.MessageCount)
		} else if topic.TopicName == _context.Config.DPN.DPNRestoreWorker.NsqTopic {
			// Fixture data has 4 restores: one from each remote node
			foundRestoreTopic = true
			assert.EqualValues(t, uint64(4), topic.MessageCount)
		}
	}
	assert.True(t, foundPackageTopic, "Nothing was queued in %s",
		_context.Config.DPN.DPNPackageWorker.NsqTopic)
	assert.True(t, foundCopyTopic, "Nothing was queued in %s",
		_context.Config.DPN.DPNCopyWorker.NsqTopic)
	assert.True(t, foundRestoreTopic, "Nothing was queued in %s",
		_context.Config.DPN.DPNRestoreWorker.NsqTopic)
}