func AppendLogChunk(db database.Database, artifact *model.Artifact, logChunk *model.LogChunk) *HttpError {
	if artifact.State != model.APPENDING {
		return NewHttpError(http.StatusBadRequest, fmt.Sprintf("Unexpected artifact state: %s", artifact.State))
	}

	if logChunk.Size <= 0 {
		return NewHttpError(http.StatusBadRequest, "Invalid chunk size %d", logChunk.Size)
	}

	if logChunk.Content == "" {
		return NewHttpError(http.StatusBadRequest, "Empty content string")
	}

	if int64(len(logChunk.Content)) != logChunk.Size {
		return NewHttpError(http.StatusBadRequest, "Content length does not match indicated size")
	}

	// Find previous chunk in DB - append only
	if nextByteOffset, err := db.GetLastByteSeenForArtifact(artifact.Id); err != nil {
		return NewHttpError(http.StatusInternalServerError, "Error while checking for previous byte range: %s", err)
	} else if nextByteOffset != logChunk.ByteOffset {
		return NewHttpError(http.StatusBadRequest, "Overlapping ranges detected, expected offset: %d, actual offset: %d", nextByteOffset, logChunk.ByteOffset)
	}

	logChunk.ArtifactId = artifact.Id

	// Expand artifact size - redundant after above change.
	if artifact.Size < logChunk.ByteOffset+logChunk.Size {
		artifact.Size = logChunk.ByteOffset + logChunk.Size
		if err := db.UpdateArtifact(artifact); err != nil {
			return NewHttpError(http.StatusInternalServerError, err.Error())
		}
	}

	if err := db.InsertLogChunk(logChunk); err != nil {
		return NewHttpError(http.StatusBadRequest, "Error updating log chunk: %s", err)
	}
	return nil
}
Ejemplo n.º 2
0
// CreateArtifact creates a new artifact in a open bucket.
//
// If an artifact with the same name already exists in the same bucket, we attempt to rename the
// artifact by adding a suffix.
// If the request specifies a chunked artifact, the size field is ignored and always set to zero.
// If the request is for a streamed artifact, size is mandatory.
// A relative path field may be specified to preserve the original file name and path. If no path is
// specified, the original artifact name is used by default.
func CreateArtifact(req createArtifactReq, bucket *model.Bucket, db database.Database) (*model.Artifact, *HttpError) {
	if len(req.Name) == 0 {
		return nil, NewHttpError(http.StatusBadRequest, "Artifact name not provided")
	}

	if bucket.State != model.OPEN {
		return nil, NewHttpError(http.StatusBadRequest, "Bucket is already closed")
	}

	artifact := new(model.Artifact)

	artifact.Name = req.Name
	artifact.BucketId = bucket.Id
	artifact.DateCreated = time.Now()

	if req.DeadlineMins == 0 {
		artifact.DeadlineMins = DEFAULT_DEADLINE
	} else {
		artifact.DeadlineMins = req.DeadlineMins
	}

	if req.Chunked {
		artifact.State = model.APPENDING
	} else {
		if req.Size == 0 {
			return nil, NewHttpError(http.StatusBadRequest, "Cannot create a new upload artifact without size.")
		} else if req.Size > MaxArtifactSizeBytes {
			return nil, NewHttpError(http.StatusRequestEntityTooLarge, fmt.Sprintf("Entity '%s' (size %d) is too large (limit %d)", req.Name, req.Size, MaxArtifactSizeBytes))
		}
		artifact.Size = req.Size
		artifact.State = model.WAITING_FOR_UPLOAD
	}

	if req.RelativePath == "" {
		// Use artifact name provided as default relativePath
		artifact.RelativePath = req.Name
	} else {
		artifact.RelativePath = req.RelativePath
	}

	// Attempt to insert artifact and retry with a different name if it fails.
	if err := db.InsertArtifact(artifact); err != nil {
		for attempt := 1; attempt <= MaxDuplicateFileNameResolutionAttempts; attempt++ {
			// Unable to create new artifact - if an artifact already exists, the above insert failed
			// because of a collision.
			if _, err := db.GetArtifactByName(bucket.Id, artifact.Name); err != nil {
				// This could be a transient DB error (down/unreachable), in which case we expect the client
				// to retry. There is no value in attempting alternate artifact names.
				//
				// We have no means of verifying there was a name collision - bail with an internal error.
				return nil, NewHttpError(http.StatusInternalServerError, err.Error())
			}

			// File name collision - attempt to resolve
			artifact.Name = fmt.Sprintf(DuplicateArtifactNameFormat, req.Name, randString(5))
			if err := db.InsertArtifact(artifact); err == nil {
				return artifact, nil
			}
		}

		return nil, NewHttpError(http.StatusInternalServerError, "Exceeded retry limit avoiding duplicates")
	}

	return artifact, nil
}
Ejemplo n.º 3
0
// AppendLogChunk appends a logchunk to an artifact.
// If the logchunk position does not match the current end of artifact, an error is returned.
// An exception to this is made when the last seen logchunk is repeated, which is silently ignored
// without an error.
func AppendLogChunk(ctx context.Context, db database.Database, artifact *model.Artifact, logChunkReq *createLogChunkReq) *HttpError {
	if artifact.State != model.APPENDING {
		return NewHttpError(http.StatusBadRequest, fmt.Sprintf("Unexpected artifact state: %s", artifact.State))
	}

	if logChunkReq.Size <= 0 {
		return NewHttpError(http.StatusBadRequest, "Invalid chunk size %d", logChunkReq.Size)
	}

	var contentBytes []byte
	if len(logChunkReq.Bytes) != 0 {
		// If request sent Bytes, use Bytes.
		if int64(len(logChunkReq.Bytes)) != logChunkReq.Size {
			return NewHttpError(http.StatusBadRequest, "Content length %d does not match indicated size %d", len(logChunkReq.Bytes), logChunkReq.Size)
		}
		contentBytes = logChunkReq.Bytes
	} else {
		// Otherwise, allow Content, for now.
		if len(logChunkReq.Content) == 0 {
			return NewHttpError(http.StatusBadRequest, "Empty content string")
		}

		if int64(len(logChunkReq.Content)) != logChunkReq.Size {
			return NewHttpError(http.StatusBadRequest, "Content length %d does not match indicated size %d", len(logChunkReq.Content), logChunkReq.Size)
		}
		contentBytes = []byte(logChunkReq.Content)
	}

	// Find previous chunk in DB - append only
	nextByteOffset := artifact.Size
	if nextByteOffset != logChunkReq.ByteOffset {
		// There is a possibility the previous logchunk is being retried - we need to handle cases where
		// a server/proxy time out caused the client not to get an ACK when it successfully uploaded the
		// previous logchunk, due to which it is retrying.
		//
		// This is a best-effort check - if we encounter DB errors or any mismatch in the chunk
		// contents, we ignore this test and claim that a range mismatch occured.
		if nextByteOffset != 0 && nextByteOffset == logChunkReq.ByteOffset+logChunkReq.Size {
			if prevLogChunk, err := db.GetLastLogChunkSeenForArtifact(artifact.Id); err == nil {
				if prevLogChunk != nil && prevLogChunk.ByteOffset == logChunkReq.ByteOffset && prevLogChunk.Size == logChunkReq.Size && bytes.Equal(prevLogChunk.ContentBytes, contentBytes) {
					sentry.ReportMessage(ctx, fmt.Sprintf("Received duplicate chunk for artifact %v of size %d at byte %d", artifact.Id, logChunkReq.Size, logChunkReq.ByteOffset))
					return nil
				}
			}
		}

		return NewHttpError(http.StatusBadRequest, "Overlapping ranges detected, expected offset: %d, actual offset: %d", nextByteOffset, logChunkReq.ByteOffset)
	}

	// Expand artifact size - redundant after above change.
	if artifact.Size < logChunkReq.ByteOffset+logChunkReq.Size {
		artifact.Size = logChunkReq.ByteOffset + logChunkReq.Size
		if err := db.UpdateArtifact(artifact); err != nil {
			return NewHttpError(http.StatusInternalServerError, err.Error())
		}
	}

	logChunk := &model.LogChunk{
		ArtifactId:   artifact.Id,
		ByteOffset:   logChunkReq.ByteOffset,
		ContentBytes: contentBytes,
		Size:         logChunkReq.Size,
	}

	if err := db.InsertLogChunk(logChunk); err != nil {
		return NewHttpError(http.StatusBadRequest, "Error updating log chunk: %s", err)
	}
	return nil
}