// queueIngestRequests queues ingest requests that are sitting in Pharos'
// WorkItems table. Users request DPN ingest directly in Pharos, and that's
// the only place those requests exist. (You won't find them in the DPN REST
// server.) We want to queue DPN ingest requests that have no QueuedAt timestamp.
// The DPN REST server will have no record of these items until we finish
// ingesting them into DPN.
func (dpnQueue *DPNQueue) queueIngestRequests() {
pageNumber := 1
params := dpnQueue.ingestParams(pageNumber)
for {
resp := dpnQueue.Context.PharosClient.WorkItemList(params)
if resp.Error != nil {
dpnQueue.err("Error getting ingest WorkItems from Pharos: %v", resp.Error)
break
}
ingestItems := resp.WorkItems()
for _, ingestItem := range ingestItems {
queueItem := models.NewQueueItem(ingestItem.ObjectIdentifier)
queueItem.ItemId = ingestItem.Id
if ingestItem.QueuedAt == nil || ingestItem.QueuedAt.IsZero() {
dpnQueue.queueIngest(ingestItem)
}
queueItem.QueuedAt = *ingestItem.QueuedAt
dpnQueue.QueueResult.AddIngest(queueItem)
}
if resp.Next == nil {
break
} else {
pageNumber += 1
params = dpnQueue.ingestParams(pageNumber)
}
}
}
// queueRestoreRequests collects RestoreTransfer requests from
// the local DPN server and if necessary 1) creates a DPNWorkItem record
// in our Pharos server for the replication request, and 2) creates an
// entry in NSQ telling our replication workers to copy the bag.
//
// We query our local DPN node after synching data from other nodes, and
// we're looking for RestoreTransfers where the from_node is our node.
// We want to skip transfers that are cancelled or already finished. We
// also want to skip transfers where accepted is true, because
// those transfers are already in progress.
func (dpnQueue *DPNQueue) queueRestoreRequests() {
pageNumber := 1
params := dpnQueue.restoreParams(pageNumber)
for {
dpnResp := dpnQueue.LocalClient.RestoreTransferList(params)
if dpnResp.Error != nil {
dpnQueue.err("Error getting RestoreTransfers from local node: %v", dpnResp.Error)
break
}
xfers := dpnResp.RestoreTransfers()
for _, xfer := range xfers {
queueItem := models.NewQueueItem(xfer.RestoreId)
dpnWorkItem := dpnQueue.getOrCreateWorkItem(xfer.RestoreId, xfer.ToNode, constants.DPNTaskRestore)
queueItem.ItemId = dpnWorkItem.Id
if dpnWorkItem.QueuedAt == nil || dpnWorkItem.QueuedAt.IsZero() {
dpnQueue.queueTransfer(dpnWorkItem, constants.DPNTaskRestore)
}
queueItem.QueuedAt = *dpnWorkItem.QueuedAt
dpnQueue.QueueResult.AddRestore(queueItem)
}
if dpnResp.Next == nil {
break
} else {
pageNumber += 1
params = dpnQueue.restoreParams(pageNumber)
}
}
}
func TestAddAndFindIngest(t *testing.T) {
result := models.NewQueueResult()
require.NotNil(t, result)
assert.Empty(t, result.Ingests)
result.AddIngest(models.NewQueueItem("ingest1"))
assert.Equal(t, 1, len(result.Ingests))
item := result.FindIngest("ingest1")
require.NotNil(t, item)
assert.Equal(t, "ingest1", item.Identifier)
assert.Nil(t, result.FindIngest("does not exist"))
}
func TestAddAndFindReplication(t *testing.T) {
result := models.NewQueueResult()
require.NotNil(t, result)
assert.Empty(t, result.Replications)
result.AddReplication(models.NewQueueItem("replication1"))
assert.Equal(t, 1, len(result.Replications))
item := result.FindReplication("replication1")
require.NotNil(t, item)
assert.Equal(t, "replication1", item.Identifier)
assert.Nil(t, result.FindReplication("does not exist"))
}
func TestNewQueueItem(t *testing.T) {
item := models.NewQueueItem("aptrust")
require.NotNil(t, item)
assert.Equal(t, "aptrust", item.Identifier)
}