// ComputeSignatureToWriter is a variant of ComputeSignature that writes hashes
// to a callback
func ComputeSignatureToWriter(container *tlc.Container, pool wsync.Pool, consumer *state.Consumer, sigWriter wsync.SignatureWriter) error {
var err error
defer func() {
if pErr := pool.Close(); pErr != nil && err == nil {
err = errors.Wrap(pErr, 1)
}
}()
sctx := mksync()
totalBytes := container.Size
fileOffset := int64(0)
onRead := func(count int64) {
consumer.Progress(float64(fileOffset+count) / float64(totalBytes))
}
for fileIndex, f := range container.Files {
consumer.ProgressLabel(f.Path)
fileOffset = f.Offset
var reader io.Reader
reader, err = pool.GetReader(int64(fileIndex))
if err != nil {
return errors.Wrap(err, 1)
}
cr := counter.NewReaderCallback(onRead, reader)
err = sctx.CreateSignature(int64(fileIndex), cr, sigWriter)
if err != nil {
return errors.Wrap(err, 1)
}
}
if err != nil {
return errors.Wrap(err, 1)
}
return nil
}
func (ru *ResumableUpload) trySendBytes(buf []byte, offset int64, isLast bool) error {
buflen := int64(len(buf))
ru.Debugf("uploading chunk of %d bytes", buflen)
body := bytes.NewReader(buf)
countingReader := counter.NewReaderCallback(func(count int64) {
ru.UploadedBytes = offset + count
if ru.OnProgress != nil {
ru.OnProgress()
}
}, body)
req, err := http.NewRequest("PUT", ru.uploadURL, countingReader)
if err != nil {
// does not include HTTP errors, more like golang API usage errors
return errors.Wrap(err, 1)
}
start := offset
end := start + buflen - 1
contentRange := fmt.Sprintf("bytes %d-%d/*", offset, end)
if isLast {
contentRange = fmt.Sprintf("bytes %d-%d/%d", offset, end, offset+buflen)
}
req.Header.Set("content-range", contentRange)
req.ContentLength = buflen
ru.Debugf("uploading %d-%d, last? %v, content-length set to %d", start, end, isLast, req.ContentLength)
startTime := time.Now()
res, err := ru.httpClient.Do(req)
if err != nil {
ru.Debugf("while uploading %d-%d: \n%s", start, end, err.Error())
return &netError{err, GcsUnknown}
}
ru.Debugf("server replied in %s, with status %s", time.Since(startTime), res.Status)
for k, v := range res.Header {
ru.Debugf("[Reply header] %s: %s", k, v)
}
if buflen != int64(len(buf)) {
// see https://github.com/itchio/butler/issues/71#issuecomment-243081797
return &netError{fmt.Errorf("send buffer size changed while we were uploading"), GcsResume}
}
status := interpretGcsStatusCode(res.StatusCode)
if status == GcsUploadComplete && isLast {
ru.Debugf("upload complete!")
return nil
}
if status == GcsNeedQuery {
ru.Debugf("need to query upload status (HTTP %s)", res.Status)
statusRes, err := ru.queryStatus()
if err != nil {
// this happens after we retry the query a few times
return err
}
if statusRes.StatusCode == 308 {
ru.Debugf("got upload status, trying to resume")
res = statusRes
status = GcsResume
} else {
status = interpretGcsStatusCode(statusRes.StatusCode)
err = fmt.Errorf("expected upload status, got HTTP %s (%s) instead", statusRes.Status, status)
ru.Debugf(err.Error())
return err
}
}
if status == GcsResume {
expectedOffset := offset + buflen
rangeHeader := res.Header.Get("Range")
if rangeHeader == "" {
ru.Debugf("commit failed (null range), retrying")
return &retryError{committedBytes: 0}
}
committedRange, err := parseRangeHeader(rangeHeader)
if err != nil {
return err
}
ru.Debugf("got resume, expectedOffset: %d, committedRange: %s", expectedOffset, committedRange)
if committedRange.start != 0 {
return fmt.Errorf("upload failed: beginning not committed somehow (committed range: %s)", committedRange)
}
if committedRange.end == expectedOffset {
ru.Debugf("commit succeeded (%d blocks stored)", buflen/gcsChunkSize)
return nil
} else {
committedBytes := committedRange.end - offset
if committedBytes < 0 {
return fmt.Errorf("upload failed: committed negative bytes somehow (committed range: %s, expectedOffset: %d)", committedRange, expectedOffset)
}
if committedBytes > 0 {
ru.Debugf("commit partially succeeded (committed %d / %d byte, %d blocks)", committedBytes, buflen, committedBytes/gcsChunkSize)
return &retryError{committedBytes}
} else {
ru.Debugf("commit failed (retrying %d blocks)", buflen/gcsChunkSize)
return &retryError{committedBytes}
}
}
}
return fmt.Errorf("got HTTP %d (%s)", res.StatusCode, status)
}
func ExtractZip(readerAt io.ReaderAt, size int64, dir string, settings ExtractSettings) (*ExtractResult, error) {
dirCount := 0
regCount := 0
symlinkCount := 0
reader, err := zip.NewReader(readerAt, size)
if err != nil {
return nil, errors.Wrap(err, 1)
}
var totalSize int64
for _, file := range reader.File {
totalSize += int64(file.UncompressedSize64)
}
var doneSize uint64
var lastDoneIndex int = -1
func() {
if settings.ResumeFrom == "" {
return
}
resBytes, resErr := ioutil.ReadFile(settings.ResumeFrom)
if resErr != nil {
if !errors.Is(resErr, os.ErrNotExist) {
settings.Consumer.Warnf("Couldn't read resume file: %s", resErr.Error())
}
return
}
lastDone64, resErr := strconv.ParseInt(string(resBytes), 10, 64)
if resErr != nil {
settings.Consumer.Warnf("Couldn't parse resume file: %s", resErr.Error())
return
}
lastDoneIndex = int(lastDone64)
settings.Consumer.Infof("Resuming from file %d", lastDoneIndex)
}()
warnedAboutWrite := false
writeProgress := func(fileIndex int) {
if settings.ResumeFrom == "" {
return
}
payload := fmt.Sprintf("%d", fileIndex)
wErr := ioutil.WriteFile(settings.ResumeFrom, []byte(payload), 0644)
if wErr != nil {
if !warnedAboutWrite {
warnedAboutWrite = true
settings.Consumer.Warnf("Couldn't save resume file: %s", wErr.Error())
}
return
}
}
defer func() {
if settings.ResumeFrom == "" {
return
}
rErr := os.Remove(settings.ResumeFrom)
if rErr != nil {
settings.Consumer.Warnf("Couldn't remove resume file: %s", rErr.Error())
}
}()
if settings.OnUncompressedSizeKnown != nil {
settings.OnUncompressedSizeKnown(totalSize)
}
windows := runtime.GOOS == "windows"
for fileIndex, file := range reader.File {
if fileIndex <= lastDoneIndex {
settings.Consumer.Debugf("Skipping file %d")
doneSize += file.UncompressedSize64
settings.Consumer.Progress(float64(doneSize) / float64(totalSize))
continue
}
err = func() error {
rel := file.Name
filename := path.Join(dir, filepath.FromSlash(rel))
info := file.FileInfo()
mode := info.Mode()
if info.IsDir() {
err = Mkdir(filename)
if err != nil {
return errors.Wrap(err, 1)
}
dirCount++
} else if mode&os.ModeSymlink > 0 && !windows {
fileReader, fErr := file.Open()
if fErr != nil {
return errors.Wrap(fErr, 1)
}
defer fileReader.Close()
linkname, lErr := ioutil.ReadAll(fileReader)
lErr = Symlink(string(linkname), filename, settings.Consumer)
if lErr != nil {
return errors.Wrap(lErr, 1)
}
symlinkCount++
} else {
regCount++
fileReader, fErr := file.Open()
if fErr != nil {
return errors.Wrap(fErr, 1)
}
defer fileReader.Close()
settings.Consumer.Debugf("extract %s", filename)
countingReader := counter.NewReaderCallback(func(offset int64) {
currentSize := int64(doneSize) + offset
settings.Consumer.Progress(float64(currentSize) / float64(totalSize))
}, fileReader)
err = CopyFile(filename, os.FileMode(mode&LuckyMode|ModeMask), countingReader)
if err != nil {
return errors.Wrap(err, 1)
}
}
return nil
}()
if err != nil {
return nil, errors.Wrap(err, 1)
}
doneSize += file.UncompressedSize64
settings.Consumer.Progress(float64(doneSize) / float64(totalSize))
writeProgress(fileIndex)
}
return &ExtractResult{
Dirs: dirCount,
Files: regCount,
Symlinks: symlinkCount,
}, nil
}
// WritePatch outputs a pwr patch to patchWriter
func (dctx *DiffContext) WritePatch(patchWriter io.Writer, signatureWriter io.Writer) error {
if dctx.Compression == nil {
return errors.Wrap(fmt.Errorf("No compression settings specified, bailing out"), 1)
}
// signature header
rawSigWire := wire.NewWriteContext(signatureWriter)
err := rawSigWire.WriteMagic(SignatureMagic)
if err != nil {
return errors.Wrap(err, 1)
}
err = rawSigWire.WriteMessage(&SignatureHeader{
Compression: dctx.Compression,
})
if err != nil {
return errors.Wrap(err, 1)
}
sigWire, err := CompressWire(rawSigWire, dctx.Compression)
if err != nil {
return errors.Wrap(err, 1)
}
err = sigWire.WriteMessage(dctx.SourceContainer)
if err != nil {
return errors.Wrap(err, 1)
}
// patch header
rawPatchWire := wire.NewWriteContext(patchWriter)
err = rawPatchWire.WriteMagic(PatchMagic)
if err != nil {
return errors.Wrap(err, 1)
}
header := &PatchHeader{
Compression: dctx.Compression,
}
err = rawPatchWire.WriteMessage(header)
if err != nil {
return errors.Wrap(err, 1)
}
patchWire, err := CompressWire(rawPatchWire, dctx.Compression)
if err != nil {
return errors.Wrap(err, 1)
}
err = patchWire.WriteMessage(dctx.TargetContainer)
if err != nil {
return errors.Wrap(err, 1)
}
err = patchWire.WriteMessage(dctx.SourceContainer)
if err != nil {
return errors.Wrap(err, 1)
}
sourceBytes := dctx.SourceContainer.Size
fileOffset := int64(0)
onSourceRead := func(count int64) {
dctx.Consumer.Progress(float64(fileOffset+count) / float64(sourceBytes))
}
sigWriter := makeSigWriter(sigWire)
opsWriter := makeOpsWriter(patchWire, dctx)
diffContext := mksync()
signContext := mksync()
blockLibrary := wsync.NewBlockLibrary(dctx.TargetSignature)
targetContainerPathToIndex := make(map[string]int64)
for index, f := range dctx.TargetContainer.Files {
targetContainerPathToIndex[f.Path] = int64(index)
}
// re-used messages
syncHeader := &SyncHeader{}
syncDelimiter := &SyncOp{
Type: SyncOp_HEY_YOU_DID_IT,
}
pool := dctx.Pool
defer func() {
if fErr := pool.Close(); fErr != nil && err == nil {
err = errors.Wrap(fErr, 1)
}
}()
for fileIndex, f := range dctx.SourceContainer.Files {
dctx.Consumer.ProgressLabel(f.Path)
dctx.Consumer.Debug(fmt.Sprintf("%s (%s)", f.Path, humanize.IBytes(uint64(f.Size))))
fileOffset = f.Offset
syncHeader.Reset()
syncHeader.FileIndex = int64(fileIndex)
err = patchWire.WriteMessage(syncHeader)
if err != nil {
return errors.Wrap(err, 1)
}
var sourceReader io.Reader
sourceReader, err = pool.GetReader(int64(fileIndex))
if err != nil {
return errors.Wrap(err, 1)
}
// / differ
// source file +
// \ signer
diffReader, diffWriter := io.Pipe()
signReader, signWriter := io.Pipe()
done := make(chan bool)
errs := make(chan error)
var preferredFileIndex int64 = -1
if oldIndex, ok := targetContainerPathToIndex[f.Path]; ok {
preferredFileIndex = oldIndex
}
go diffFile(diffContext, dctx, blockLibrary, diffReader, opsWriter, preferredFileIndex, errs, done)
go signFile(signContext, fileIndex, signReader, sigWriter, errs, done)
go func() {
defer func() {
if dErr := diffWriter.Close(); dErr != nil {
errs <- errors.Wrap(dErr, 1)
}
}()
defer func() {
if sErr := signWriter.Close(); sErr != nil {
errs <- errors.Wrap(sErr, 1)
}
}()
mw := io.MultiWriter(diffWriter, signWriter)
sourceReadCounter := counter.NewReaderCallback(onSourceRead, sourceReader)
_, cErr := io.Copy(mw, sourceReadCounter)
if cErr != nil {
errs <- errors.Wrap(cErr, 1)
}
}()
// wait until all are done
// or an error occurs
for c := 0; c < 2; c++ {
select {
case wErr := <-errs:
return errors.Wrap(wErr, 1)
case <-done:
}
}
err = patchWire.WriteMessage(syncDelimiter)
if err != nil {
return errors.Wrap(err, 1)
}
}
err = patchWire.Close()
if err != nil {
return errors.Wrap(err, 1)
}
err = sigWire.Close()
if err != nil {
return errors.Wrap(err, 1)
}
return nil
}
