func AppendLogChunk(db database.Database, artifact *model.Artifact, logChunk *model.LogChunk) *HttpError {
if artifact.State != model.APPENDING {
return NewHttpError(http.StatusBadRequest, fmt.Sprintf("Unexpected artifact state: %s", artifact.State))
}
if logChunk.Size <= 0 {
return NewHttpError(http.StatusBadRequest, "Invalid chunk size %d", logChunk.Size)
}
if logChunk.Content == "" {
return NewHttpError(http.StatusBadRequest, "Empty content string")
}
if int64(len(logChunk.Content)) != logChunk.Size {
return NewHttpError(http.StatusBadRequest, "Content length does not match indicated size")
}
// Find previous chunk in DB - append only
if nextByteOffset, err := db.GetLastByteSeenForArtifact(artifact.Id); err != nil {
return NewHttpError(http.StatusInternalServerError, "Error while checking for previous byte range: %s", err)
} else if nextByteOffset != logChunk.ByteOffset {
return NewHttpError(http.StatusBadRequest, "Overlapping ranges detected, expected offset: %d, actual offset: %d", nextByteOffset, logChunk.ByteOffset)
}
logChunk.ArtifactId = artifact.Id
// Expand artifact size - redundant after above change.
if artifact.Size < logChunk.ByteOffset+logChunk.Size {
artifact.Size = logChunk.ByteOffset + logChunk.Size
if err := db.UpdateArtifact(artifact); err != nil {
return NewHttpError(http.StatusInternalServerError, err.Error())
}
}
if err := db.InsertLogChunk(logChunk); err != nil {
return NewHttpError(http.StatusBadRequest, "Error updating log chunk: %s", err)
}
return nil
}
// AppendLogChunk appends a logchunk to an artifact.
// If the logchunk position does not match the current end of artifact, an error is returned.
// An exception to this is made when the last seen logchunk is repeated, which is silently ignored
// without an error.
func AppendLogChunk(ctx context.Context, db database.Database, artifact *model.Artifact, logChunkReq *createLogChunkReq) *HttpError {
if artifact.State != model.APPENDING {
return NewHttpError(http.StatusBadRequest, fmt.Sprintf("Unexpected artifact state: %s", artifact.State))
}
if logChunkReq.Size <= 0 {
return NewHttpError(http.StatusBadRequest, "Invalid chunk size %d", logChunkReq.Size)
}
var contentBytes []byte
if len(logChunkReq.Bytes) != 0 {
// If request sent Bytes, use Bytes.
if int64(len(logChunkReq.Bytes)) != logChunkReq.Size {
return NewHttpError(http.StatusBadRequest, "Content length %d does not match indicated size %d", len(logChunkReq.Bytes), logChunkReq.Size)
}
contentBytes = logChunkReq.Bytes
} else {
// Otherwise, allow Content, for now.
if len(logChunkReq.Content) == 0 {
return NewHttpError(http.StatusBadRequest, "Empty content string")
}
if int64(len(logChunkReq.Content)) != logChunkReq.Size {
return NewHttpError(http.StatusBadRequest, "Content length %d does not match indicated size %d", len(logChunkReq.Content), logChunkReq.Size)
}
contentBytes = []byte(logChunkReq.Content)
}
// Find previous chunk in DB - append only
nextByteOffset := artifact.Size
if nextByteOffset != logChunkReq.ByteOffset {
// There is a possibility the previous logchunk is being retried - we need to handle cases where
// a server/proxy time out caused the client not to get an ACK when it successfully uploaded the
// previous logchunk, due to which it is retrying.
//
// This is a best-effort check - if we encounter DB errors or any mismatch in the chunk
// contents, we ignore this test and claim that a range mismatch occured.
if nextByteOffset != 0 && nextByteOffset == logChunkReq.ByteOffset+logChunkReq.Size {
if prevLogChunk, err := db.GetLastLogChunkSeenForArtifact(artifact.Id); err == nil {
if prevLogChunk != nil && prevLogChunk.ByteOffset == logChunkReq.ByteOffset && prevLogChunk.Size == logChunkReq.Size && bytes.Equal(prevLogChunk.ContentBytes, contentBytes) {
sentry.ReportMessage(ctx, fmt.Sprintf("Received duplicate chunk for artifact %v of size %d at byte %d", artifact.Id, logChunkReq.Size, logChunkReq.ByteOffset))
return nil
}
}
}
return NewHttpError(http.StatusBadRequest, "Overlapping ranges detected, expected offset: %d, actual offset: %d", nextByteOffset, logChunkReq.ByteOffset)
}
// Expand artifact size - redundant after above change.
if artifact.Size < logChunkReq.ByteOffset+logChunkReq.Size {
artifact.Size = logChunkReq.ByteOffset + logChunkReq.Size
if err := db.UpdateArtifact(artifact); err != nil {
return NewHttpError(http.StatusInternalServerError, err.Error())
}
}
logChunk := &model.LogChunk{
ArtifactId: artifact.Id,
ByteOffset: logChunkReq.ByteOffset,
ContentBytes: contentBytes,
Size: logChunkReq.Size,
}
if err := db.InsertLogChunk(logChunk); err != nil {
return NewHttpError(http.StatusBadRequest, "Error updating log chunk: %s", err)
}
return nil
}