コード例 #1
0
ファイル: dl.go プロジェクト: itchio/butler
func appendAllToFile(src io.Reader, dest string, existingBytes int64, totalBytes int64) error {
	out, _ := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	defer out.Close()

	prevPercent := 0.0
	comm.StartProgress()

	onWrite := func(bytesDownloaded int64) {
		bytesWritten := existingBytes + bytesDownloaded
		percent := float64(bytesWritten) / float64(totalBytes)
		if math.Abs(percent-prevPercent) < 0.0001 {
			return
		}

		prevPercent = percent
		comm.Progress(percent)
	}
	counter := counter.NewWriterCallback(onWrite, out)

	_, err := io.Copy(counter, src)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	comm.EndProgress()
	return nil
}
コード例 #2
0
ファイル: healer_archive.go プロジェクト: itchio/butler
func (ah *ArchiveHealer) healOne(sourcePool wsync.Pool, targetPool wsync.WritablePool, fileIndex int64, chunkHealed chunkHealedFunc) error {
	var err error
	var reader io.Reader
	var writer io.WriteCloser

	reader, err = sourcePool.GetReader(fileIndex)
	if err != nil {
		return err
	}

	writer, err = targetPool.GetWriter(fileIndex)
	if err != nil {
		return err
	}

	lastCount := int64(0)
	cw := counter.NewWriterCallback(func(count int64) {
		chunk := count - lastCount
		chunkHealed(chunk)
		lastCount = count
	}, writer)

	_, err = io.Copy(cw, reader)
	if err != nil {
		return err
	}

	err = writer.Close()
	if err != nil {
		return err
	}

	return err
}
コード例 #3
0
ファイル: copy.go プロジェクト: itchio/wharf
// CopyContainer copies from one container to the other. Combined with fspool
// and blockpool, it can be used to split a container into blocks or join it back
// into regular files.
func CopyContainer(container *tlc.Container, outPool wsync.WritablePool, inPool wsync.Pool, consumer *state.Consumer) error {
	copyFile := func(byteOffset int64, fileIndex int64) error {
		r, err := inPool.GetReader(fileIndex)
		if err != nil {
			return err
		}

		w, err := outPool.GetWriter(fileIndex)
		if err != nil {
			return err
		}

		cw := counter.NewWriterCallback(func(count int64) {
			alpha := float64(byteOffset+count) / float64(container.Size)
			consumer.Progress(alpha)
		}, w)

		_, err = io.Copy(cw, r)
		if err != nil {
			return err
		}

		err = w.Close()
		if err != nil {
			return err
		}

		return nil
	}

	byteOffset := int64(0)

	for fileIndex, f := range container.Files {
		consumer.ProgressLabel(f.Path)

		err := copyFile(byteOffset, int64(fileIndex))
		if err != nil {
			return err
		}

		byteOffset += f.Size
	}

	return nil
}
コード例 #4
0
ファイル: resumable.go プロジェクト: itchio/butler
func NewResumableUpload(uploadURL string, done chan bool, errs chan error, settings ResumableUploadSettings) (*ResumableUpload, error) {
	ru := &ResumableUpload{}
	ru.MaxChunkGroup = settings.MaxChunkGroup
	if ru.MaxChunkGroup == 0 {
		ru.MaxChunkGroup = 64
	}
	ru.uploadURL = uploadURL
	ru.id = seed
	seed++
	ru.consumer = settings.Consumer
	ru.httpClient = timeout.NewClient(resumableConnectTimeout, resumableIdleTimeout)

	pipeR, pipeW := io.Pipe()

	ru.pipeWriter = pipeW

	bufferSize := settings.BufferSize
	if bufferSize == 0 {
		bufferSize = 32 * 1024 * 1024
	}

	bufferedWriter := bufio.NewWriterSize(pipeW, bufferSize)
	ru.bufferedWriter = bufferedWriter

	onWrite := func(count int64) {
		// ru.Debugf("onwrite %d", count)
		ru.TotalBytes = count
		if ru.OnProgress != nil {
			ru.OnProgress()
		}
	}
	ru.writeCounter = counter.NewWriterCallback(onWrite, bufferedWriter)

	go ru.uploadChunks(pipeR, done, errs)

	return ru, nil
}
コード例 #5
0
ファイル: cp.go プロジェクト: itchio/butler
func doCp(srcPath string, destPath string, resume bool) error {
	src, err := eos.Open(srcPath)
	if err != nil {
		return err
	}

	defer src.Close()

	dir := filepath.Dir(destPath)
	err = os.MkdirAll(dir, 0755)
	if err != nil {
		return err
	}

	flags := os.O_CREATE | os.O_WRONLY

	dest, err := os.OpenFile(destPath, flags, 0644)
	if err != nil {
		return err
	}

	defer dest.Close()

	stats, err := src.Stat()
	if err != nil {
		return err
	}

	totalBytes := int64(stats.Size())
	startOffset := int64(0)

	if resume {
		startOffset, err = dest.Seek(0, os.SEEK_END)
		if err != nil {
			return err
		}

		if startOffset == 0 {
			comm.Logf("Downloading %s", humanize.IBytes(uint64(totalBytes)))
		} else if startOffset > totalBytes {
			comm.Logf("Existing data too big (%s > %s), starting over", humanize.IBytes(uint64(startOffset)), humanize.IBytes(uint64(totalBytes)))
		} else if startOffset == totalBytes {
			comm.Logf("All %s already there", humanize.IBytes(uint64(totalBytes)))
			return nil
		}

		comm.Logf("Resuming at %s / %s", humanize.IBytes(uint64(startOffset)), humanize.IBytes(uint64(totalBytes)))

		_, err = src.Seek(startOffset, os.SEEK_SET)
		if err != nil {
			return err
		}
	} else {
		comm.Logf("Downloading %s", humanize.IBytes(uint64(totalBytes)))
	}

	start := time.Now()

	comm.Progress(float64(startOffset) / float64(totalBytes))
	comm.StartProgressWithTotalBytes(totalBytes)

	cw := counter.NewWriterCallback(func(count int64) {
		alpha := float64(startOffset+count) / float64(totalBytes)
		comm.Progress(alpha)
	}, dest)

	copiedBytes, err := io.Copy(cw, src)
	if err != nil {
		return err
	}
	comm.EndProgress()

	totalDuration := time.Since(start)
	prettyStartOffset := humanize.IBytes(uint64(startOffset))
	prettySize := humanize.IBytes(uint64(copiedBytes))
	perSecond := humanize.IBytes(uint64(float64(totalBytes-startOffset) / totalDuration.Seconds()))
	comm.Statf("%s + %s copied @ %s/s\n", prettyStartOffset, prettySize, perSecond)

	return nil
}
コード例 #6
0
ファイル: apply.go プロジェクト: itchio/butler
func (actx *ApplyContext) lazilyPatchFile(sctx *wsync.Context, targetContainer *tlc.Container, targetPool wsync.Pool, outputContainer *tlc.Container, outputPool wsync.WritablePool,
	fileIndex int64, onSourceWrite counter.CountCallback, ops chan wsync.Operation, inplace bool) (written int64, transposition *Transposition, err error) {

	var writer io.WriteCloser

	defer func() {
		if writer != nil {
			cErr := writer.Close()
			if cErr != nil && err == nil {
				err = cErr
			}
		}
	}()

	var realops chan wsync.Operation

	errs := make(chan error, 1)
	first := true

	for op := range ops {
		if first {
			first = false

			// if the first operation is a blockrange that copies an
			// entire file from target into a file from source that has
			// the same name and size, then it's a no-op!
			if inplace && op.Type == wsync.OpBlockRange && op.BlockIndex == 0 {
				outputFile := outputContainer.Files[fileIndex]
				targetFile := targetContainer.Files[op.FileIndex]
				numOutputBlocks := ComputeNumBlocks(outputFile.Size)

				if op.BlockSpan == numOutputBlocks &&
					outputFile.Size == targetFile.Size {
					transposition = &Transposition{
						TargetPath: targetFile.Path,
						OutputPath: outputFile.Path,
					}
				}
			}

			if transposition != nil {
				errs <- nil
			} else {
				realops = make(chan wsync.Operation)

				writer, err = outputPool.GetWriter(fileIndex)
				if err != nil {
					return 0, nil, errors.Wrap(err, 1)
				}
				writeCounter := counter.NewWriterCallback(onSourceWrite, writer)

				go func() {
					failFast := true
					if actx.WoundsConsumer != nil {
						failFast = false
					}

					applyErr := sctx.ApplyPatchFull(writeCounter, targetPool, realops, failFast)
					if applyErr != nil {
						errs <- applyErr
						return
					}

					written = writeCounter.Count()
					errs <- nil
				}()
			}
		}

		// if not a transposition, relay errors
		if transposition == nil {
			select {
			case cErr := <-errs:
				// if we get an error here, ApplyPatch failed so we no longer need to close realops
				if cErr != nil {
					return 0, nil, errors.Wrap(cErr, 1)
				}
			case realops <- op:
				// muffin
			}
		}
	}

	if transposition == nil {
		// realops may be nil if the file was empty (0 ops)
		if realops != nil {
			close(realops)
		} else {
			// if we had 0 ops, signal no errors occured
			errs <- nil
		}
	}

	err = <-errs
	if err != nil {
		return 0, nil, errors.Wrap(err, 1)
	}

	return
}
コード例 #7
0
ファイル: validator.go プロジェクト: itchio/wharf
func (vctx *ValidatorContext) validate(target string, signature *SignatureInfo, fileIndices chan int64,
	errs chan error, onProgress onProgressFunc, cancelled chan struct{}) {

	var retErr error

	targetPool, err := pools.New(signature.Container, target)
	if err != nil {
		errs <- err
		return
	}

	defer func() {
		err := targetPool.Close()
		if err != nil {
			retErr = errors.Wrap(err, 1)
			return
		}

		errs <- retErr
	}()

	aggregateOut := make(chan *Wound)
	relayDone := make(chan bool)
	go func() {
		for w := range aggregateOut {
			vctx.Wounds <- w
		}
		relayDone <- true
	}()

	wounds := AggregateWounds(aggregateOut, MaxWoundSize)
	defer func() {
		// signal no more wounds are going to be sent
		close(wounds)
		// wait for all of them to be relayed
		<-relayDone
	}()

	validatingPool := &ValidatingPool{
		Pool:      nullpool.New(signature.Container),
		Container: signature.Container,
		Signature: signature,

		Wounds: wounds,
	}

	doOne := func(fileIndex int64) error {
		file := signature.Container.Files[fileIndex]

		var reader io.Reader
		reader, err = targetPool.GetReader(fileIndex)
		if err != nil {
			if os.IsNotExist(err) {
				// whole file is missing
				wound := &Wound{
					Kind:  WoundKind_FILE,
					Index: fileIndex,
					Start: 0,
					End:   file.Size,
				}
				onProgress(file.Size)

				select {
				case wounds <- wound:
				case <-cancelled:
				}
				return nil
			}
			return err
		}

		var writer io.WriteCloser
		writer, err = validatingPool.GetWriter(fileIndex)
		if err != nil {
			return err
		}

		defer writer.Close()

		lastCount := int64(0)
		countingWriter := counter.NewWriterCallback(func(count int64) {
			delta := count - lastCount
			onProgress(delta)
			lastCount = count
		}, writer)

		var writtenBytes int64
		writtenBytes, err = io.Copy(countingWriter, reader)
		if err != nil {
			return err
		}

		if writtenBytes != file.Size {
			onProgress(file.Size - writtenBytes)
			wound := &Wound{
				Kind:  WoundKind_FILE,
				Index: fileIndex,
				Start: writtenBytes,
				End:   file.Size,
			}

			select {
			case wounds <- wound:
			case <-cancelled:
			}
		}

		return nil
	}

	for {
		select {
		case fileIndex, ok := <-fileIndices:
			if !ok {
				// no more work
				return
			}

			err := doOne(fileIndex)
			if err != nil {
				if retErr == nil {
					retErr = err
				}
				return
			}
		case <-cancelled:
			// cancelled
			return
		}
	}
}