func Test_CopyContainer(t *testing.T) {
mainDir, err := ioutil.TempDir("", "copycontainer")
assert.NoError(t, err)
defer os.RemoveAll(mainDir)
src := path.Join(mainDir, "src")
dst := path.Join(mainDir, "dst")
makeTestDir(t, src, testDirSettings{
seed: 0x91738,
entries: []testDirEntry{
{path: "subdir/file-1", seed: 0x1},
{path: "file-1", seed: 0x2},
{path: "file-2", seed: 0x3},
},
})
container, err := tlc.WalkAny(src, nil)
assert.NoError(t, err)
inPool := fspool.New(container, src)
outPool := fspool.New(container, dst)
err = CopyContainer(container, outPool, inPool, &state.Consumer{})
assert.NoError(t, err)
assert.NoError(t, inPool.Close())
}
func doSign(output string, signature string, compression pwr.CompressionSettings, fixPerms bool) error {
comm.Opf("Creating signature for %s", output)
startTime := time.Now()
container, err := tlc.WalkAny(output, filterPaths)
if err != nil {
return errors.Wrap(err, 1)
}
pool, err := pools.New(container, output)
if err != nil {
return errors.Wrap(err, 1)
}
if fixPerms {
container.FixPermissions(pool)
}
signatureWriter, err := os.Create(signature)
if err != nil {
return errors.Wrap(err, 1)
}
rawSigWire := wire.NewWriteContext(signatureWriter)
rawSigWire.WriteMagic(pwr.SignatureMagic)
rawSigWire.WriteMessage(&pwr.SignatureHeader{
Compression: &compression,
})
sigWire, err := pwr.CompressWire(rawSigWire, &compression)
if err != nil {
return errors.Wrap(err, 1)
}
sigWire.WriteMessage(container)
comm.StartProgress()
err = pwr.ComputeSignatureToWriter(container, pool, comm.NewStateConsumer(), func(hash wsync.BlockHash) error {
return sigWire.WriteMessage(&pwr.BlockHash{
WeakHash: hash.WeakHash,
StrongHash: hash.StrongHash,
})
})
comm.EndProgress()
if err != nil {
return errors.Wrap(err, 1)
}
err = sigWire.Close()
if err != nil {
return errors.Wrap(err, 1)
}
prettySize := humanize.IBytes(uint64(container.Size))
perSecond := humanize.IBytes(uint64(float64(container.Size) / time.Since(startTime).Seconds()))
comm.Statf("%s (%s) @ %s/s\n", prettySize, container.Stats(), perSecond)
return nil
}
func doWalk(path string, out chan walkResult, errs chan error, fixPerms bool) {
container, err := tlc.WalkAny(path, filterPaths)
if err != nil {
errs <- errors.Wrap(err, 1)
return
}
pool, err := pools.New(container, path)
if err != nil {
errs <- errors.Wrap(err, 1)
return
}
result := walkResult{
container: container,
pool: pool,
}
if fixPerms {
result.container.FixPermissions(result.pool)
}
out <- result
}
func runPatchingScenario(t *testing.T, scenario patchScenario) {
log := func(format string, args ...interface{}) {
t.Logf("[%s] %s", scenario.name, fmt.Sprintf(format, args...))
}
log("Scenario start")
mainDir, err := ioutil.TempDir("", "patch-cycle")
assert.NoError(t, err)
assert.NoError(t, os.MkdirAll(mainDir, 0755))
defer os.RemoveAll(mainDir)
v1 := filepath.Join(mainDir, "v1")
makeTestDir(t, v1, scenario.v1)
v2 := filepath.Join(mainDir, "v2")
makeTestDir(t, v2, scenario.v2)
compression := &CompressionSettings{}
compression.Algorithm = CompressionAlgorithm_NONE
sourceContainer, err := tlc.WalkAny(v2, nil)
assert.NoError(t, err)
consumer := &state.Consumer{}
patchBuffer := new(bytes.Buffer)
signatureBuffer := new(bytes.Buffer)
func() {
targetContainer, dErr := tlc.WalkAny(v1, nil)
assert.NoError(t, dErr)
targetPool := fspool.New(targetContainer, v1)
targetSignature, dErr := ComputeSignature(targetContainer, targetPool, consumer)
assert.NoError(t, dErr)
pool := fspool.New(sourceContainer, v2)
dctx := &DiffContext{
Compression: compression,
Consumer: consumer,
SourceContainer: sourceContainer,
Pool: pool,
TargetContainer: targetContainer,
TargetSignature: targetSignature,
}
assert.NoError(t, dctx.WritePatch(patchBuffer, signatureBuffer))
}()
v1Before := filepath.Join(mainDir, "v1Before")
cpDir(t, v1, v1Before)
v1After := filepath.Join(mainDir, "v1After")
woundsPath := filepath.Join(mainDir, "wounds.pww")
if scenario.extraTests {
log("Making sure before-path folder doesn't validate")
signature, sErr := ReadSignature(bytes.NewReader(signatureBuffer.Bytes()))
assert.NoError(t, sErr)
assert.Error(t, AssertValid(v1Before, signature))
runExtraTest := func(setup SetupFunc) error {
assert.NoError(t, os.RemoveAll(woundsPath))
assert.NoError(t, os.RemoveAll(v1Before))
cpDir(t, v1, v1Before)
actx := &ApplyContext{
TargetPath: v1Before,
OutputPath: v1Before,
InPlace: true,
Consumer: consumer,
}
if setup != nil {
setup(actx)
}
patchReader := bytes.NewReader(patchBuffer.Bytes())
aErr := actx.ApplyPatch(patchReader)
if aErr != nil {
return aErr
}
if actx.Signature == nil {
vErr := AssertValid(v1Before, signature)
if vErr != nil {
return vErr
}
}
return nil
}
func() {
log("In-place with failing vet")
var NotVettingError = errors.New("not vetting this")
pErr := runExtraTest(func(actx *ApplyContext) {
actx.VetApply = func(actx *ApplyContext) error {
return NotVettingError
}
})
assert.Error(t, pErr)
assert.True(t, errors.Is(pErr, NotVettingError))
}()
func() {
log("In-place with signature (failfast, passing)")
assert.NoError(t, runExtraTest(func(actx *ApplyContext) {
actx.Signature = signature
}))
}()
func() {
log("In-place with signature (failfast, failing)")
assert.Error(t, runExtraTest(func(actx *ApplyContext) {
actx.Signature = signature
makeTestDir(t, v1Before, *scenario.corruptions)
}))
}()
func() {
log("In-place with signature (wounds, passing)")
assert.NoError(t, runExtraTest(func(actx *ApplyContext) {
actx.Signature = signature
actx.WoundsPath = woundsPath
}))
_, sErr := os.Lstat(woundsPath)
assert.Error(t, sErr)
assert.True(t, os.IsNotExist(sErr))
}()
func() {
log("In-place with signature (wounds, failing)")
assert.NoError(t, runExtraTest(func(actx *ApplyContext) {
actx.Signature = signature
actx.WoundsPath = woundsPath
makeTestDir(t, v1Before, *scenario.corruptions)
}))
_, sErr := os.Lstat(woundsPath)
assert.NoError(t, sErr)
}()
}
log("Applying to other directory, with separate check")
assert.NoError(t, os.RemoveAll(v1Before))
cpDir(t, v1, v1Before)
func() {
actx := &ApplyContext{
TargetPath: v1Before,
OutputPath: v1After,
Consumer: consumer,
}
patchReader := bytes.NewReader(patchBuffer.Bytes())
aErr := actx.ApplyPatch(patchReader)
assert.NoError(t, aErr)
assert.Equal(t, 0, actx.Stats.DeletedFiles, "deleted files (other dir)")
assert.Equal(t, 0, actx.Stats.DeletedDirs, "deleted dirs (other dir)")
assert.Equal(t, 0, actx.Stats.DeletedSymlinks, "deleted symlinks (other dir)")
assert.Equal(t, 0, actx.Stats.MovedFiles, "moved files (other dir)")
assert.Equal(t, len(sourceContainer.Files), actx.Stats.TouchedFiles, "touched files (other dir)")
assert.Equal(t, 0, actx.Stats.NoopFiles, "noop files (other dir)")
signature, sErr := ReadSignature(bytes.NewReader(signatureBuffer.Bytes()))
assert.NoError(t, sErr)
assert.NoError(t, AssertValid(v1After, signature))
}()
log("Applying in-place")
testAll := func(setup SetupFunc) {
assert.NoError(t, os.RemoveAll(v1After))
assert.NoError(t, os.RemoveAll(v1Before))
cpDir(t, v1, v1Before)
func() {
actx := &ApplyContext{
TargetPath: v1Before,
OutputPath: v1Before,
InPlace: true,
Consumer: consumer,
}
if setup != nil {
setup(actx)
}
patchReader := bytes.NewReader(patchBuffer.Bytes())
aErr := actx.ApplyPatch(patchReader)
assert.NoError(t, aErr)
assert.Equal(t, scenario.deletedFiles, actx.Stats.DeletedFiles, "deleted files (in-place)")
assert.Equal(t, scenario.deletedSymlinks, actx.Stats.DeletedSymlinks, "deleted symlinks (in-place)")
assert.Equal(t, scenario.deletedDirs+scenario.leftDirs, actx.Stats.DeletedDirs, "deleted dirs (in-place)")
assert.Equal(t, scenario.touchedFiles, actx.Stats.TouchedFiles, "touched files (in-place)")
assert.Equal(t, scenario.movedFiles, actx.Stats.MovedFiles, "moved files (in-place)")
assert.Equal(t, len(sourceContainer.Files)-scenario.touchedFiles-scenario.movedFiles, actx.Stats.NoopFiles, "noop files (in-place)")
signature, sErr := ReadSignature(bytes.NewReader(signatureBuffer.Bytes()))
assert.NoError(t, sErr)
assert.NoError(t, AssertValid(v1Before, signature))
}()
if scenario.intermediate != nil {
log("Applying in-place with %d intermediate files", len(scenario.intermediate.entries))
assert.NoError(t, os.RemoveAll(v1After))
assert.NoError(t, os.RemoveAll(v1Before))
cpDir(t, v1, v1Before)
makeTestDir(t, v1Before, *scenario.intermediate)
func() {
actx := &ApplyContext{
TargetPath: v1Before,
OutputPath: v1Before,
InPlace: true,
Consumer: consumer,
}
if setup != nil {
setup(actx)
}
patchReader := bytes.NewReader(patchBuffer.Bytes())
aErr := actx.ApplyPatch(patchReader)
assert.NoError(t, aErr)
assert.Equal(t, scenario.deletedFiles, actx.Stats.DeletedFiles, "deleted files (in-place w/intermediate)")
assert.Equal(t, scenario.deletedDirs, actx.Stats.DeletedDirs, "deleted dirs (in-place w/intermediate)")
assert.Equal(t, scenario.deletedSymlinks, actx.Stats.DeletedSymlinks, "deleted symlinks (in-place w/intermediate)")
assert.Equal(t, scenario.touchedFiles, actx.Stats.TouchedFiles, "touched files (in-place w/intermediate)")
assert.Equal(t, scenario.noopFiles, actx.Stats.NoopFiles, "noop files (in-place w/intermediate)")
assert.Equal(t, scenario.leftDirs, actx.Stats.LeftDirs, "left dirs (in-place w/intermediate)")
signature, sErr := ReadSignature(bytes.NewReader(signatureBuffer.Bytes()))
assert.NoError(t, sErr)
assert.NoError(t, AssertValid(v1Before, signature))
}()
}
}
testAll(nil)
if scenario.testBrokenRename {
testAll(func(actx *ApplyContext) {
actx.debugBrokenRename = true
})
}
}
func doDiff(target string, source string, patch string, compression pwr.CompressionSettings) error {
var err error
startTime := time.Now()
targetSignature := &pwr.SignatureInfo{}
targetSignature.Container, err = tlc.WalkAny(target, filterPaths)
if err != nil {
// Signature file perhaps?
var signatureReader io.ReadCloser
signatureReader, err = eos.Open(target)
if err != nil {
return errors.Wrap(err, 1)
}
targetSignature, err = pwr.ReadSignature(signatureReader)
if err != nil {
if errors.Is(err, wire.ErrFormat) {
return fmt.Errorf("unrecognized target %s (not a container, not a signature file)", target)
}
return errors.Wrap(err, 1)
}
comm.Opf("Read signature from %s", target)
err = signatureReader.Close()
if err != nil {
return errors.Wrap(err, 1)
}
} else {
// Container (dir, archive, etc.)
comm.Opf("Hashing %s", target)
comm.StartProgress()
var targetPool wsync.Pool
targetPool, err = pools.New(targetSignature.Container, target)
if err != nil {
return errors.Wrap(err, 1)
}
targetSignature.Hashes, err = pwr.ComputeSignature(targetSignature.Container, targetPool, comm.NewStateConsumer())
comm.EndProgress()
if err != nil {
return errors.Wrap(err, 1)
}
{
prettySize := humanize.IBytes(uint64(targetSignature.Container.Size))
perSecond := humanize.IBytes(uint64(float64(targetSignature.Container.Size) / time.Since(startTime).Seconds()))
comm.Statf("%s (%s) @ %s/s\n", prettySize, targetSignature.Container.Stats(), perSecond)
}
}
startTime = time.Now()
var sourceContainer *tlc.Container
sourceContainer, err = tlc.WalkAny(source, filterPaths)
if err != nil {
return errors.Wrap(err, 1)
}
var sourcePool wsync.Pool
sourcePool, err = pools.New(sourceContainer, source)
if err != nil {
return errors.Wrap(err, 1)
}
patchWriter, err := os.Create(patch)
if err != nil {
return errors.Wrap(err, 1)
}
defer patchWriter.Close()
signaturePath := patch + ".sig"
signatureWriter, err := os.Create(signaturePath)
if err != nil {
return errors.Wrap(err, 1)
}
defer signatureWriter.Close()
patchCounter := counter.NewWriter(patchWriter)
signatureCounter := counter.NewWriter(signatureWriter)
dctx := &pwr.DiffContext{
SourceContainer: sourceContainer,
Pool: sourcePool,
TargetContainer: targetSignature.Container,
TargetSignature: targetSignature.Hashes,
Consumer: comm.NewStateConsumer(),
Compression: &compression,
}
comm.Opf("Diffing %s", source)
comm.StartProgress()
err = dctx.WritePatch(patchCounter, signatureCounter)
if err != nil {
return errors.Wrap(err, 1)
}
comm.EndProgress()
totalDuration := time.Since(startTime)
{
prettySize := humanize.IBytes(uint64(sourceContainer.Size))
perSecond := humanize.IBytes(uint64(float64(sourceContainer.Size) / totalDuration.Seconds()))
comm.Statf("%s (%s) @ %s/s\n", prettySize, sourceContainer.Stats(), perSecond)
}
if *diffArgs.verify {
tmpDir, err := ioutil.TempDir("", "pwr")
if err != nil {
return errors.Wrap(err, 1)
}
defer os.RemoveAll(tmpDir)
apply(patch, target, tmpDir, false, signaturePath, "")
}
{
prettyPatchSize := humanize.IBytes(uint64(patchCounter.Count()))
percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes)
relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size)
prettyFreshSize := humanize.IBytes(uint64(dctx.FreshBytes))
comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize)
comm.Statf("%s patch (%.2f%% of the full size) in %s", prettyPatchSize, relToNew, totalDuration)
}
return nil
}
func Test_ArchiveHealer(t *testing.T) {
mainDir, err := ioutil.TempDir("", "archivehealer")
assert.NoError(t, err)
defer os.RemoveAll(mainDir)
archivePath := filepath.Join(mainDir, "archive.zip")
archiveWriter, err := os.Create(archivePath)
assert.NoError(t, err)
defer archiveWriter.Close()
targetDir := filepath.Join(mainDir, "target")
assert.NoError(t, os.MkdirAll(targetDir, 0755))
zw := zip.NewWriter(archiveWriter)
numFiles := 16
fakeData := []byte{1, 2, 3, 4}
nameFor := func(index int) string {
return fmt.Sprintf("file-%d", index)
}
pathFor := func(index int) string {
return filepath.Join(targetDir, nameFor(index))
}
for i := 0; i < numFiles; i++ {
writer, cErr := zw.Create(nameFor(i))
assert.NoError(t, cErr)
_, cErr = writer.Write(fakeData)
assert.NoError(t, cErr)
}
assert.NoError(t, zw.Close())
container, err := tlc.WalkAny(archivePath, nil)
assert.NoError(t, err)
healAll := func() Healer {
healer, err := NewHealer(fmt.Sprintf("archive,%s", archivePath), targetDir)
assert.NoError(t, err)
wounds := make(chan *Wound)
done := make(chan bool)
go func() {
err := healer.Do(container, wounds)
assert.NoError(t, err)
done <- true
}()
for i := 0; i < numFiles; i++ {
wounds <- &Wound{
Kind: WoundKind_FILE,
Index: int64(i),
Start: 0,
End: 1,
}
}
close(wounds)
<-done
return healer
}
assertAllFilesHealed := func() {
for i := 0; i < numFiles; i++ {
data, err := ioutil.ReadFile(pathFor(i))
assert.NoError(t, err)
assert.Equal(t, fakeData, data)
}
}
t.Logf("...with no files present")
healer := healAll()
assert.Equal(t, int64(numFiles), healer.TotalCorrupted())
assert.Equal(t, int64(numFiles*len(fakeData)), healer.TotalHealed())
assertAllFilesHealed()
t.Logf("...with one file too long")
assert.NoError(t, ioutil.WriteFile(pathFor(3), bytes.Repeat(fakeData, 4), 0644))
healer = healAll()
assert.Equal(t, int64(numFiles), healer.TotalCorrupted())
assert.Equal(t, int64(numFiles*len(fakeData)), healer.TotalHealed())
assertAllFilesHealed()
t.Logf("...with one file too short")
assert.NoError(t, ioutil.WriteFile(pathFor(7), fakeData[:1], 0644))
healer = healAll()
assert.Equal(t, int64(numFiles), healer.TotalCorrupted())
assert.Equal(t, int64(numFiles*len(fakeData)), healer.TotalHealed())
assertAllFilesHealed()
t.Logf("...with one file slightly corrupted")
corruptedFakeData := append([]byte{}, fakeData...)
corruptedFakeData[2] = 255
assert.NoError(t, ioutil.WriteFile(pathFor(9), corruptedFakeData, 0644))
healer = healAll()
assert.Equal(t, int64(numFiles), healer.TotalCorrupted())
assert.Equal(t, int64(numFiles*len(fakeData)), healer.TotalHealed())
assertAllFilesHealed()
}