Ejemplo n.º 1
0
// WorkItemSave saves a WorkItem record to Pharos. If the WorkItems's ID
// is zero, this performs a POST to create a new record. For non-zero IDs, this
// performs a PUT to update the existing record. The response object
// will include a new copy of the WorkItem if it was saved successfully.
func (client *PharosClient) WorkItemSave(obj *models.WorkItem) *PharosResponse {
	// Set up the response object
	resp := NewPharosResponse(PharosWorkItem)
	resp.workItems = make([]*models.WorkItem, 1)

	// URL and method
	relativeUrl := fmt.Sprintf("/api/%s/items/", client.apiVersion)
	httpMethod := "POST"
	if obj.Id > 0 {
		// URL should look like /api/v2/items/46956/
		relativeUrl = fmt.Sprintf("%s%d/", relativeUrl, obj.Id)
		httpMethod = "PUT"
	}
	absoluteUrl := client.BuildUrl(relativeUrl)

	// Prepare the JSON data
	postData, err := obj.SerializeForPharos()
	if err != nil {
		resp.Error = err
	}

	// Run the request
	client.DoRequest(resp, httpMethod, absoluteUrl, bytes.NewBuffer(postData))
	if resp.Error != nil {
		return resp
	}

	// Parse the JSON from the response body
	workItem := &models.WorkItem{}
	resp.Error = json.Unmarshal(resp.data, workItem)
	if resp.Error == nil {
		resp.workItems[0] = workItem
	}
	return resp
}
Ejemplo n.º 2
0
func TestIsStoring(t *testing.T) {
	workItem := models.WorkItem{
		Action: "Ingest",
		Stage:  "Store",
		Status: "Started",
	}
	assert.True(t, workItem.IsStoring())
	workItem.Status = "Pending"
	assert.False(t, workItem.IsStoring())

	workItem.Status = "Started"
	workItem.Stage = "Record"
	assert.False(t, workItem.IsStoring())
}
Ejemplo n.º 3
0
// queueIngest adds the id of a DPN Ingest WorkItem to NSQ and records info about
// when the item was queued in WorkItem.QueuedAt, which is saved to Pharos.
func (dpnQueue *DPNQueue) queueIngest(workItem *apt_models.WorkItem) {
	// Put the item into NSQ. First step for DPN ingest is DPNPackage.
	err := dpnQueue.Context.NSQClient.Enqueue(
		dpnQueue.Context.Config.DPN.DPNPackageWorker.NsqTopic,
		workItem.Id)
	if err != nil {
		dpnQueue.err("Error queueing DPNWorkItem %d for ingest: %v", workItem.Id, err)
	} else {
		// Let Pharos know this item has been queued
		utcNow := time.Now().UTC()
		workItem.QueuedAt = &utcNow
		resp := dpnQueue.Context.PharosClient.WorkItemSave(workItem)
		if resp.Error != nil {
			dpnQueue.err("Error updating WorkItem %d for ingest: %v",
				workItem.Id, resp.Error)
			return
		}
		workItem = resp.WorkItem()
	}
}
Ejemplo n.º 4
0
func (reader *APTBucketReader) markAsQueued(workItem *models.WorkItem) *models.WorkItem {
	utcNow := time.Now().UTC()
	workItem.QueuedAt = &utcNow
	resp := reader.Context.PharosClient.WorkItemSave(workItem)
	if resp.Error != nil {
		errMsg := fmt.Sprintf("Error setting QueuedAt for WorkItem with id %d: %v",
			workItem.Id, resp.Error)
		reader.Context.MessageLog.Debug("%s", resp.Request.URL.String())
		reader.Context.MessageLog.Error(errMsg)
		if reader.stats != nil {
			reader.stats.AddError(errMsg)
		}
		return nil
	}
	if resp.Response.StatusCode != 200 {
		reader.processPharosError(resp)
		return nil
	}
	if reader.stats != nil {
		reader.stats.AddWorkItem("WorkItemsMarkedAsQueued", workItem)
	}
	return resp.WorkItem()
}
Ejemplo n.º 5
0
func TestWorkItemShouldTryIngest(t *testing.T) {
	workItem := models.WorkItem{
		Action: "Ingest",
		Stage:  "Receive",
		Status: "Pending",
		Retry:  true,
	}

	// Test stages
	assert.True(t, workItem.ShouldTryIngest())

	workItem.Stage = "Fetch"
	assert.True(t, workItem.ShouldTryIngest())

	workItem.Stage = "Unpack"
	assert.True(t, workItem.ShouldTryIngest())

	workItem.Stage = "Validate"
	assert.True(t, workItem.ShouldTryIngest())

	workItem.Stage = "Record"
	assert.False(t, workItem.ShouldTryIngest())

	// Test Store/Pending and Store/Started
	workItem.Stage = "Store"
	workItem.Status = "Started"
	assert.False(t, workItem.ShouldTryIngest())

	workItem.Stage = "Store"
	workItem.Status = "Pending"
	assert.False(t, workItem.ShouldTryIngest())

	// Test Retry = false
	workItem.Status = "Started"
	workItem.Retry = false

	workItem.Stage = "Receive"
	assert.False(t, workItem.ShouldTryIngest())

	workItem.Stage = "Fetch"
	assert.False(t, workItem.ShouldTryIngest())

	workItem.Stage = "Unpack"
	assert.False(t, workItem.ShouldTryIngest())

	workItem.Stage = "Validate"
	assert.False(t, workItem.ShouldTryIngest())

	workItem.Stage = "Record"
	assert.False(t, workItem.ShouldTryIngest())
}
Ejemplo n.º 6
0
func TestWorkItemHasBeenStored(t *testing.T) {
	workItem := models.WorkItem{
		Action: "Ingest",
		Stage:  "Record",
		Status: "Success",
	}
	assert.True(t, workItem.HasBeenStored())

	workItem.Stage = constants.StageCleanup
	assert.True(t, workItem.HasBeenStored())

	workItem.Stage = constants.StageStore
	workItem.Status = constants.StatusPending
	assert.True(t, workItem.HasBeenStored())

	workItem.Stage = constants.StageStore
	workItem.Status = constants.StatusStarted
	assert.False(t, workItem.HasBeenStored())

	workItem.Stage = constants.StageFetch
	assert.False(t, workItem.HasBeenStored())

	workItem.Stage = constants.StageUnpack
	assert.False(t, workItem.HasBeenStored())

	workItem.Stage = constants.StageValidate
	assert.False(t, workItem.HasBeenStored())
}