Example #1
0
// queueIngestRequests queues ingest requests that are sitting in Pharos'
// WorkItems table. Users request DPN ingest directly in Pharos, and that's
// the only place those requests exist. (You won't find them in the DPN REST
// server.) We want to queue DPN ingest requests that have no QueuedAt timestamp.
// The DPN REST server will have no record of these items until we finish
// ingesting them into DPN.
func (dpnQueue *DPNQueue) queueIngestRequests() {
	pageNumber := 1
	params := dpnQueue.ingestParams(pageNumber)
	for {
		resp := dpnQueue.Context.PharosClient.WorkItemList(params)
		if resp.Error != nil {
			dpnQueue.err("Error getting ingest WorkItems from Pharos: %v", resp.Error)
			break
		}
		ingestItems := resp.WorkItems()
		for _, ingestItem := range ingestItems {
			queueItem := models.NewQueueItem(ingestItem.ObjectIdentifier)
			queueItem.ItemId = ingestItem.Id
			if ingestItem.QueuedAt == nil || ingestItem.QueuedAt.IsZero() {
				dpnQueue.queueIngest(ingestItem)
			}
			queueItem.QueuedAt = *ingestItem.QueuedAt
			dpnQueue.QueueResult.AddIngest(queueItem)
		}
		if resp.Next == nil {
			break
		} else {
			pageNumber += 1
			params = dpnQueue.ingestParams(pageNumber)
		}
	}
}
Example #2
0
// queueRestoreRequests collects RestoreTransfer requests from
// the local DPN server and if necessary 1) creates a DPNWorkItem record
// in our Pharos server for the replication request, and 2) creates an
// entry in NSQ telling our replication workers to copy the bag.
//
// We query our local DPN node after synching data from other nodes, and
// we're looking for RestoreTransfers where the from_node is our node.
// We want to skip transfers that are cancelled or already finished. We
// also want to skip transfers where accepted is true, because
// those transfers are already in progress.
func (dpnQueue *DPNQueue) queueRestoreRequests() {
	pageNumber := 1
	params := dpnQueue.restoreParams(pageNumber)
	for {
		dpnResp := dpnQueue.LocalClient.RestoreTransferList(params)
		if dpnResp.Error != nil {
			dpnQueue.err("Error getting RestoreTransfers from local node: %v", dpnResp.Error)
			break
		}
		xfers := dpnResp.RestoreTransfers()
		for _, xfer := range xfers {
			queueItem := models.NewQueueItem(xfer.RestoreId)
			dpnWorkItem := dpnQueue.getOrCreateWorkItem(xfer.RestoreId, xfer.ToNode, constants.DPNTaskRestore)
			queueItem.ItemId = dpnWorkItem.Id
			if dpnWorkItem.QueuedAt == nil || dpnWorkItem.QueuedAt.IsZero() {
				dpnQueue.queueTransfer(dpnWorkItem, constants.DPNTaskRestore)
			}
			queueItem.QueuedAt = *dpnWorkItem.QueuedAt
			dpnQueue.QueueResult.AddRestore(queueItem)
		}
		if dpnResp.Next == nil {
			break
		} else {
			pageNumber += 1
			params = dpnQueue.restoreParams(pageNumber)
		}
	}
}
Example #3
0
func TestAddAndFindIngest(t *testing.T) {
	result := models.NewQueueResult()
	require.NotNil(t, result)
	assert.Empty(t, result.Ingests)
	result.AddIngest(models.NewQueueItem("ingest1"))
	assert.Equal(t, 1, len(result.Ingests))
	item := result.FindIngest("ingest1")
	require.NotNil(t, item)
	assert.Equal(t, "ingest1", item.Identifier)
	assert.Nil(t, result.FindIngest("does not exist"))
}
Example #4
0
func TestAddAndFindReplication(t *testing.T) {
	result := models.NewQueueResult()
	require.NotNil(t, result)
	assert.Empty(t, result.Replications)
	result.AddReplication(models.NewQueueItem("replication1"))
	assert.Equal(t, 1, len(result.Replications))
	item := result.FindReplication("replication1")
	require.NotNil(t, item)
	assert.Equal(t, "replication1", item.Identifier)
	assert.Nil(t, result.FindReplication("does not exist"))
}
Example #5
0
func TestNewQueueItem(t *testing.T) {
	item := models.NewQueueItem("aptrust")
	require.NotNil(t, item)
	assert.Equal(t, "aptrust", item.Identifier)
}