コード例 #1
0
ファイル: blobs_test.go プロジェクト: voidException/bazil
func TestWriteSaveAndReadLarge(t *testing.T) {
	const chunkSize = 4096
	const fanout = 2
	chunkStore := &mock.InMemory{}
	// just enough to span multiple chunks
	greeting := bytes.Repeat(GREETING, chunkSize/len(GREETING)+1)

	var saved *blobs.Manifest
	{
		blob, err := blobs.Open(chunkStore, &blobs.Manifest{
			Type:      "footype",
			ChunkSize: chunkSize,
			Fanout:    fanout,
		})
		if err != nil {
			t.Fatalf("cannot open blob: %v", err)
		}
		n, err := blob.WriteAt(greeting, 0)
		if err != nil {
			t.Fatalf("unexpected write error: %v", err)
		}
		if g, e := n, len(greeting); g != e {
			t.Errorf("unexpected write length: %v != %v", g, e)
		}
		if g, e := blob.Size(), uint64(len(greeting)); g != e {
			t.Errorf("unexpected manifest size: %v != %v", g, e)
		}
		saved, err = blob.Save()
		if err != nil {
			t.Fatalf("unexpected error from Save: %v", err)
		}
	}

	t.Logf("saved manifest: %+v", saved)
	b, err := blobs.Open(chunkStore, saved)
	if err != nil {
		t.Fatalf("cannot open saved blob: %v", err)
	}
	// do +1 to trigger us seeing EOF too
	buf := make([]byte, len(greeting)+1)
	n, err := b.ReadAt(buf, 0)
	if err != io.EOF {
		t.Errorf("expected read EOF: %v", err)
	}
	if g, e := n, len(greeting); g != e {
		t.Errorf("unexpected read length: %v != %v", g, e)
	}
	buf = buf[:n]
	if !bytes.Equal(greeting, buf) {
		t.Errorf("unexpected read data: %q", buf)
	}
}
コード例 #2
0
ファイル: fuse.go プロジェクト: jgluck/bazil
func (d fuseDir) Lookup(name string, intr fusefs.Intr) (fusefs.Node, fuse.Error) {
	de, err := d.reader.Lookup(name)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, fuse.ENOENT
		}
		return nil, fmt.Errorf("snap lookup error: %v", err)
	}

	switch {
	case de.Type.File != nil:
		manifest := de.Type.File.Manifest.ToBlob("file")
		blob, err := blobs.Open(d.chunkStore, manifest)
		if err != nil {
			return nil, fmt.Errorf("snap file blob open error: %v", err)
		}
		child := fuseFile{
			rat: blob,
			de:  de,
		}
		return child, nil

	case de.Type.Dir != nil:
		child, err := Open(d.chunkStore, de.Type.Dir)
		if err != nil {
			return nil, fmt.Errorf("snap dir FUSE serving error: %v", err)
		}
		return child, nil

	default:
		return nil, fmt.Errorf("unknown entry in tree, %v", de.Type.GetValue())
	}
}
コード例 #3
0
ファイル: blobs_test.go プロジェクト: voidException/bazil
func TestWriteAndSaveLarge(t *testing.T) {
	const chunkSize = 4096
	const fanout = 64
	chunkStore := &mock.InMemory{}
	blob, err := blobs.Open(chunkStore, &blobs.Manifest{
		Type:      "footype",
		ChunkSize: chunkSize,
		Fanout:    fanout,
	})
	if err != nil {
		t.Fatalf("cannot open blob: %v", err)
	}
	n, err := blob.WriteAt(bytes.Join([][]byte{
		bytes.Repeat([]byte{'x'}, chunkSize),
		bytes.Repeat([]byte{'y'}, chunkSize),
	}, []byte{}), 0)
	if err != nil {
		t.Fatalf("unexpected write error: %v", err)
	}
	if g, e := n, 2*chunkSize; g != e {
		t.Errorf("unexpected write length: %v != %v", g, e)
	}

	saved, err := blob.Save()
	if err != nil {
		t.Fatalf("unexpected error from Save: %v", err)
	}
	if g, e := saved.Root.String(), "9f3f6815c7680f98e00fe9ab5edc85ba3f4ceb657b9562c35b5a865d970ea3270bab8c7aa3162cbaaa966ad84330f34a22aa9539b4c416f858c35c0775482665"; g != e {
		t.Errorf("unexpected key: %q != %q", g, e)
	}
	if g, e := saved.Size, uint64(chunkSize+chunkSize); g != e {
		t.Errorf("unexpected size: %v != %v", g, e)
	}
}
コード例 #4
0
ファイル: serve_test.go プロジェクト: voidException/bazil
func setup_dir(t testing.TB, chunkStore chunks.Store, dirents []*wire.Dirent) *wire.Dirent {
	blob, err := blobs.Open(
		chunkStore,
		blobs.EmptyManifest("dir"),
	)
	if err != nil {
		t.Fatalf("unexpected blob open error: %v", err)
	}
	w := snap.NewWriter(blob)
	for _, de := range dirents {
		err := w.Add(de)
		if err != nil {
			t.Fatalf("unexpected add error: %v", err)
		}
	}
	manifest, err := blob.Save()
	if err != nil {
		t.Fatalf("unexpected save error: %v", err)
	}
	var de wire.Dirent
	de.Type.Dir = &wire.Dir{
		Manifest: wirecas.FromBlob(manifest),
	}
	return &de
}
コード例 #5
0
ファイル: blobs_test.go プロジェクト: voidException/bazil
func TestWriteSaveLoopAndRead(t *testing.T) {
	const chunkSize = 4096
	const fanout = 2
	chunkStore := &mock.InMemory{}
	blob, err := blobs.Open(chunkStore, &blobs.Manifest{
		Type:      "footype",
		ChunkSize: chunkSize,
		Fanout:    fanout,
	})
	if err != nil {
		t.Fatalf("cannot open blob: %v", err)
	}
	// not exactly sure where this magic number comes from :(
	greeting := bytes.Repeat(GREETING, 40330)

	var prev *cas.Key
	for i := 0; i <= 2; i++ {
		n, err := blob.WriteAt(greeting, 0)
		if err != nil {
			t.Fatalf("unexpected write error: %v", err)
		}
		if g, e := n, len(greeting); g != e {
			t.Errorf("unexpected write length: %v != %v", g, e)
		}
		if g, e := blob.Size(), uint64(len(greeting)); g != e {
			t.Errorf("unexpected manifest size: %v != %v", g, e)
		}
		saved, err := blob.Save()
		if err != nil {
			t.Fatalf("unexpected error from Save: %v", err)
		}
		t.Logf("saved %v size=%d", saved.Root, saved.Size)
		if prev != nil {
			if g, e := saved.Root, *prev; g != e {
				t.Errorf("unexpected key: %q != %q", g, e)
			}
		}
		tmp := saved.Root
		prev = &tmp
	}

	// do +1 to trigger us seeing EOF too
	buf := make([]byte, len(greeting)+1)
	n, err := blob.ReadAt(buf, 0)
	if err != io.EOF {
		t.Errorf("expected read EOF: %v", err)
	}
	if g, e := n, len(greeting); g != e {
		t.Errorf("unexpected read length: %v != %v", g, e)
	}
	buf = buf[:n]
	if !bytes.Equal(greeting, buf) {
		// assumes len > 100, which we know is true
		t.Errorf("unexpected read data %q..%q", buf[:100], buf[len(buf)-100:])
	}
}
コード例 #6
0
ファイル: blobs_test.go プロジェクト: voidException/bazil
func emptyBlob(t testing.TB, chunkStore chunks.Store) *blobs.Blob {
	blob, err := blobs.Open(
		chunkStore,
		blobs.EmptyManifest("footype"),
	)
	if err != nil {
		t.Fatalf("cannot open blob: %v", err)
	}
	return blob
}
コード例 #7
0
ファイル: blobs_test.go プロジェクト: voidException/bazil
func TestOpenNoType(t *testing.T) {
	_, err := blobs.Open(mock.NeverUsed{}, &blobs.Manifest{
		// no Type
		ChunkSize: blobs.MinChunkSize,
		Fanout:    2,
	})
	if g, e := err, blobs.MissingType; g != e {
		t.Fatalf("bad error: %v != %v", g, e)
	}
}
コード例 #8
0
ファイル: fuse.go プロジェクト: som-snytt/bazil
// Serve this snapshot with FUSE, with this object store.
func Open(chunkStore chunks.Store, de *wire.Dirent) (fusefs.Node, error) {
	switch {
	case de.File != nil:
		manifest, err := de.File.Manifest.ToBlob("file")
		if err != nil {
			return nil, err
		}
		blob, err := blobs.Open(chunkStore, manifest)
		if err != nil {
			return nil, fmt.Errorf("snap file blob open error: %v", err)
		}
		child := fuseFile{
			rat: blob,
			de:  de,
		}
		return child, nil

	case de.Dir != nil:
		manifest, err := de.Dir.Manifest.ToBlob("dir")
		if err != nil {
			return nil, err
		}
		blob, err := blobs.Open(chunkStore, manifest)
		if err != nil {
			return nil, err
		}
		r, err := NewReader(blob, de.Dir.Align)
		if err != nil {
			return nil, err
		}
		child := fuseDir{
			chunkStore: chunkStore,
			reader:     r,
		}
		return child, nil

	default:
		return nil, fmt.Errorf("unknown entry in tree, %v", de)
	}
}
コード例 #9
0
ファイル: blobs_test.go プロジェクト: voidException/bazil
func TestWriteTruncateZero(t *testing.T) {
	const chunkSize = 4096
	const fanout = 64
	blob, err := blobs.Open(&mock.InMemory{}, &blobs.Manifest{
		Type:      "footype",
		ChunkSize: chunkSize,
		Fanout:    fanout,
	})
	if err != nil {
		t.Fatalf("cannot open blob: %v", err)
	}

	n, err := blob.WriteAt(bytes.Join([][]byte{
		bytes.Repeat([]byte{'x'}, chunkSize),
		bytes.Repeat([]byte{'y'}, chunkSize),
	}, []byte{}), 0)
	if err != nil {
		t.Fatalf("unexpected write error: %v", err)
	}
	if g, e := n, 2*chunkSize; g != e {
		t.Errorf("unexpected write length: %v != %v", g, e)
	}

	_, err = blob.Save()
	if err != nil {
		t.Fatalf("unexpected error from Save: %v", err)
	}

	err = blob.Truncate(0)
	if err != nil {
		t.Fatalf("unexpected Truncate error: %v", err)
	}

	if g, e := blob.Size(), uint64(0); g != e {
		t.Errorf("unexpected manifest size: %v != %v", g, e)
	}

	saved, err := blob.Save()
	if err != nil {
		t.Errorf("unexpected error from Save: %v", err)
	}
	if g, e := saved.Root, cas.Empty; g != e {
		t.Errorf("unexpected key: %v != %v", g, e)
	}
	if g, e := saved.Size, uint64(0); g != e {
		t.Errorf("unexpected size: %v != %v", g, e)
	}
}
コード例 #10
0
ファイル: fuse.go プロジェクト: jgluck/bazil
// Serve this snapshot with FUSE, with this object store.
func Open(chunkStore chunks.Store, dir *wire.Dir) (fusefs.Node, error) {
	manifest := dir.Manifest.ToBlob("dir")
	blob, err := blobs.Open(chunkStore, manifest)
	if err != nil {
		return nil, err
	}
	r, err := NewReader(blob, dir.Align)
	if err != nil {
		return nil, err
	}
	node := fuseDir{
		chunkStore: chunkStore,
		reader:     r,
	}
	return node, nil
}
コード例 #11
0
ファイル: serve_test.go プロジェクト: voidException/bazil
func setup_greeting(t testing.TB, chunkStore chunks.Store) *blobs.Manifest {
	blob, err := blobs.Open(
		chunkStore,
		blobs.EmptyManifest("file"),
	)
	if err != nil {
		t.Fatalf("unexpected blob open error: %v", err)
	}
	_, err = blob.WriteAt([]byte(GREETING), 0)
	if err != nil {
		t.Fatalf("unexpected write error: %v", err)
	}
	manifest, err := blob.Save()
	if err != nil {
		t.Fatalf("unexpected save error: %v", err)
	}
	return manifest
}
コード例 #12
0
ファイル: dir.go プロジェクト: jgluck/bazil
func (d *dir) Create(req *fuse.CreateRequest, resp *fuse.CreateResponse, intr fs.Intr) (fs.Node, fs.Handle, fuse.Error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	// TODO check for duplicate name

	switch req.Mode & os.ModeType {
	case 0:
		var child node
		err := d.fs.db.Update(func(tx *bolt.Tx) error {
			bucket := d.fs.bucket(tx).Bucket(bucketInode)
			if bucket == nil {
				return errors.New("inode bucket is missing")
			}
			inode, err := inodes.Allocate(bucket)
			if err != nil {
				return err
			}

			manifest := blobs.EmptyManifest("file")
			blob, err := blobs.Open(d.fs.chunkStore, manifest)
			if err != nil {
				return fmt.Errorf("blob open problem: %v", err)
			}
			child = &file{
				inode:  inode,
				name:   req.Name,
				parent: d,
				blob:   blob,
			}
			d.active[req.Name] = child

			return d.saveInternal(tx, req.Name, child)
			// TODO clean up active on error
		})
		if err != nil {
			return nil, nil, err
		}
		return child, child, nil
	default:
		return nil, nil, fuse.EPERM
	}
}
コード例 #13
0
ファイル: blobs_test.go プロジェクト: voidException/bazil
func TestSparseRead(t *testing.T) {
	const chunkSize = 4096
	blob, err := blobs.Open(
		&mock.InMemory{},
		&blobs.Manifest{
			Type:      "footype",
			Size:      100,
			ChunkSize: chunkSize,
			Fanout:    2,
		},
	)
	buf := make([]byte, 10)
	n, err := blob.ReadAt(buf, 3)
	if err != nil {
		t.Errorf("unexpected read error: %v", err)
	}
	if g, e := n, 10; g != e {
		t.Errorf("expected to read 0 bytes: %v != %v", g, e)
	}
}
コード例 #14
0
ファイル: dir.go プロジェクト: jgluck/bazil
func (d *dir) reviveNode(de *wire.Dirent, name string) (node, error) {
	switch {
	case de.Type.Dir != nil:
		return d.reviveDir(de, name)

	case de.Type.File != nil:
		manifest := de.Type.File.Manifest.ToBlob("file")
		blob, err := blobs.Open(d.fs.chunkStore, manifest)
		if err != nil {
			return nil, err
		}
		child := &file{
			inode:  de.Inode,
			name:   name,
			parent: d,
			blob:   blob,
		}
		return child, nil
	}

	return nil, fmt.Errorf("dirent unknown type: %v", de.GetValue())
}
コード例 #15
0
ファイル: blobs_test.go プロジェクト: voidException/bazil
func TestWriteSaveAndRead(t *testing.T) {
	chunkStore := &mock.InMemory{}
	var saved *blobs.Manifest
	{
		blob := emptyBlob(t, chunkStore)
		n, err := blob.WriteAt(GREETING, 0)
		if err != nil {
			t.Fatalf("unexpected write error: %v", err)
		}
		if g, e := n, len(GREETING); g != e {
			t.Errorf("unexpected write length: %v != %v", g, e)
		}
		if g, e := blob.Size(), uint64(len(GREETING)); g != e {
			t.Errorf("unexpected manifest size: %v != %v", g, e)
		}
		saved, err = blob.Save()
		if err != nil {
			t.Fatalf("unexpected error from Save: %v", err)
		}
	}

	b, err := blobs.Open(chunkStore, saved)
	if err != nil {
		t.Fatalf("cannot open saved blob: %v", err)
	}
	// do +1 to trigger us seeing EOF too
	buf := make([]byte, len(GREETING)+1)
	n, err := b.ReadAt(buf, 0)
	if err != io.EOF {
		t.Errorf("expected read EOF: %v", err)
	}
	if g, e := n, len(GREETING); g != e {
		t.Errorf("unexpected read length: %v != %v", g, e)
	}
	buf = buf[:n]
	if !bytes.Equal(GREETING, buf) {
		t.Errorf("unexpected read data: %q", buf)
	}
}
コード例 #16
0
ファイル: blobs_test.go プロジェクト: voidException/bazil
func TestWriteSparse(t *testing.T) {
	const chunkSize = 4096
	chunkStore := &mock.InMemory{}
	blob, err := blobs.Open(chunkStore, &blobs.Manifest{
		Type:      "footype",
		ChunkSize: chunkSize,
		Fanout:    2,
	})
	if err != nil {
		t.Fatalf("cannot open blob: %v", err)
	}

	// note: gap after end of first chunk
	n, err := blob.WriteAt([]byte{'x'}, chunkSize+3)
	if err != nil {
		t.Fatalf("unexpected write error: %v", err)
	}
	if g, e := n, 1; g != e {
		t.Errorf("unexpected write length: %v != %v", g, e)
	}
	if g, e := blob.Size(), uint64(chunkSize)+3+1; g != e {
		t.Errorf("unexpected manifest size: %v != %v", g, e)
	}

	// read exactly a chunksize to access only the hole
	buf := make([]byte, 1)
	n, err = blob.ReadAt(buf, 0)
	if err != nil {
		t.Fatalf("unexpected read error: %v", err)
	}
	if g, e := n, len(buf); g != e {
		t.Errorf("unexpected read length: %v != %v", g, e)
	}
	buf = buf[:n]
	if !bytes.Equal([]byte{0}, buf) {
		t.Errorf("unexpected read data: %q", buf)
	}
}
コード例 #17
0
ファイル: blobs_test.go プロジェクト: voidException/bazil
func TestWriteTruncateGrow(t *testing.T) {
	const chunkSize = 4096
	const fanout = 64
	chunkStore := &mock.InMemory{}
	blob, err := blobs.Open(chunkStore, &blobs.Manifest{
		Type:      "footype",
		ChunkSize: chunkSize,
		Fanout:    fanout,
	})
	if err != nil {
		t.Fatalf("cannot open blob: %v", err)
	}

	n, err := blob.WriteAt(GREETING, 0)
	if err != nil {
		t.Fatalf("unexpected write error: %v", err)
	}
	if g, e := n, len(GREETING); g != e {
		t.Errorf("unexpected write length: %v != %v", g, e)
	}
	if g, e := blob.Size(), uint64(len(GREETING)); g != e {
		t.Errorf("unexpected manifest size: %v != %v", g, e)
	}

	_, err = blob.Save()
	if err != nil {
		t.Fatalf("unexpected error from Save: %v", err)
	}

	// grow enough to need a new chunk
	const newSize = chunkSize + 3
	err = blob.Truncate(newSize)
	if err != nil {
		t.Fatalf("unexpected Truncate error: %v", err)
	}

	if g, e := blob.Size(), uint64(newSize); g != e {
		t.Errorf("unexpected manifest size: %v != %v", g, e)
	}

	// do +1 to trigger us seeing EOF too
	buf := make([]byte, newSize+1)
	n, err = blob.ReadAt(buf, 0)
	if err != io.EOF {
		t.Errorf("expected read EOF: %v", err)
	}
	if g, e := n, newSize; g != e {
		t.Errorf("unexpected read length: %v != %v", g, e)
	}
	buf = buf[:n]
	want := bytes.Join([][]byte{
		GREETING,
		make([]byte, newSize-len(GREETING)),
	}, []byte{})
	if g, e := buf, want; !bytes.Equal(g, e) {
		t.Errorf("unexpected read data: %q != %q", g, e)
	}

	saved, err := blob.Save()
	if err != nil {
		t.Fatalf("unexpected error from Save: %v", err)
	}
	if g, e := saved.Size, uint64(newSize); g != e {
		t.Errorf("unexpected size: %v != %v", g, e)
	}
	{
		blob, err := blobs.Open(chunkStore, saved)
		if err != nil {
			t.Fatalf("cannot open saved blob: %v", err)
		}
		buf := make([]byte, newSize+1)
		n, err = blob.ReadAt(buf, 0)
		if err != io.EOF {
			t.Errorf("expected read EOF: %v", err)
		}
		if g, e := n, newSize; g != e {
			t.Errorf("unexpected read length: %v != %v", g, e)
		}
		buf = buf[:n]
		want := bytes.Join([][]byte{
			GREETING,
			make([]byte, newSize-len(GREETING)),
		}, []byte{})
		if g, e := buf, want; !bytes.Equal(g, e) {
			t.Errorf("unexpected read data: %q != %q", g, e)
		}
	}
}
コード例 #18
0
ファイル: dir.go プロジェクト: jgluck/bazil
// snapshot records a snapshot of the directory and stores it in wde
func (d *dir) snapshot(tx *bolt.Tx, out *wiresnap.Dir, intr fs.Intr) error {
	// NOT HOLDING THE LOCK, accessing database snapshot ONLY

	// TODO move bucket lookup to caller?
	bucket := d.fs.bucket(tx).Bucket(bucketDir)
	if bucket == nil {
		return errors.New("dir bucket missing")
	}

	manifest := blobs.EmptyManifest("dir")
	blob, err := blobs.Open(d.fs.chunkStore, manifest)
	if err != nil {
		return err
	}
	w := snap.NewWriter(blob)

	c := bucket.Cursor()
	prefix := pathToKey(d.inode, "")
	for k, v := c.Seek(prefix); k != nil; k, v = c.Next() {
		if !bytes.HasPrefix(k, prefix) {
			// past the end of the directory
			break
		}

		name := string(k[len(prefix):])
		de, err := d.unmarshalDirent(v)
		if err != nil {
			return err
		}
		sde := wiresnap.Dirent{
			Name: name,
		}
		switch {
		case de.Type.File != nil:
			// TODO d.reviveNode would do blobs.Open and that's a bit
			// too much work; rework the apis
			sde.Type.File = &wiresnap.File{
				Manifest: de.Type.File.Manifest,
			}
		case de.Type.Dir != nil:
			child, err := d.reviveDir(de, name)
			if err != nil {
				return err
			}
			sde.Type.Dir = &wiresnap.Dir{}
			err = child.snapshot(tx, sde.Type.Dir, intr)
			if err != nil {
				return err
			}
		default:
			return errors.New("TODO")
		}
		err = w.Add(&sde)
		if err != nil {
			return err
		}
	}

	manifest, err = blob.Save()
	if err != nil {
		return err
	}
	out.Manifest = wirecas.FromBlob(manifest)
	out.Align = w.Align()
	return nil
}
コード例 #19
0
ファイル: blobs_test.go プロジェクト: voidException/bazil
func TestWriteTruncateShrink(t *testing.T) {
	const chunkSize = 4096
	const fanout = 64
	chunkStore := &mock.InMemory{}
	blob, err := blobs.Open(chunkStore, &blobs.Manifest{
		Type:      "footype",
		ChunkSize: chunkSize,
		Fanout:    fanout,
	})
	if err != nil {
		t.Fatalf("cannot open blob: %v", err)
	}

	n, err := blob.WriteAt(bytes.Join([][]byte{
		bytes.Repeat([]byte{'x'}, chunkSize),
		bytes.Repeat([]byte{'y'}, chunkSize),
	}, []byte{}), 0)
	if err != nil {
		t.Fatalf("unexpected write error: %v", err)
	}
	if g, e := n, 2*chunkSize; g != e {
		t.Errorf("unexpected write length: %v != %v", g, e)
	}

	_, err = blob.Save()
	if err != nil {
		t.Fatalf("unexpected error from Save: %v", err)
	}

	// shrink enough to need less depth in tree
	const newSize = 5
	err = blob.Truncate(newSize)
	if err != nil {
		t.Fatalf("unexpected Truncate error: %v", err)
	}

	if g, e := blob.Size(), uint64(newSize); g != e {
		t.Errorf("unexpected manifest size: %v != %v", g, e)
	}

	// do +1 to trigger us seeing EOF too
	buf := make([]byte, newSize+1)
	n, err = blob.ReadAt(buf, 0)
	if err != io.EOF {
		t.Errorf("expected read EOF: %v", err)
	}
	if g, e := n, newSize; g != e {
		t.Errorf("unexpected read length: %v != %v", g, e)
	}
	buf = buf[:n]
	if g, e := buf, []byte("xxxxx"); !bytes.Equal(g, e) {
		t.Errorf("unexpected read data: %q != %q", g, e)
	}

	saved, err := blob.Save()
	if err != nil {
		t.Fatalf("unexpected error from Save: %v", err)
	}
	if g, e := saved.Size, uint64(newSize); g != e {
		t.Errorf("unexpected size: %v != %v", g, e)
	}
	{
		blob, err := blobs.Open(chunkStore, saved)
		if err != nil {
			t.Fatalf("cannot open saved blob: %v", err)
		}
		buf := make([]byte, newSize+1)
		n, err = blob.ReadAt(buf, 0)
		if err != io.EOF {
			t.Errorf("expected read EOF: %v", err)
		}
		if g, e := n, newSize; g != e {
			t.Errorf("unexpected read length: %v != %v", g, e)
		}
		buf = buf[:n]
		if g, e := buf, []byte("xxxxx"); !bytes.Equal(g, e) {
			t.Errorf("unexpected read data: %q != %q", g, e)
		}
	}
}
コード例 #20
0
ファイル: quickblobs_test.go プロジェクト: som-snytt/bazil
func testCompareBoth(t *testing.T, saveEvery int) {
	f, err := ioutil.TempFile("", "baziltest-")
	if err != nil {
		t.Fatalf("tempfile error: %v")
	}
	defer f.Close()

	blob, err := blobs.Open(&mock.InMemory{},
		&blobs.Manifest{
			Type:      "footype",
			ChunkSize: blobs.MinChunkSize,
			Fanout:    2,
		},
	)
	if err != nil {
		t.Fatalf("cannot open blob: %v", err)
	}

	if seed == 0 {
		seed = uint64(entropy.Seed())
	}
	t.Logf("Seed is %d", seed)
	qconf := quick.Config{
		Rand: rand.New(rand.NewSource(int64(seed))),
	}

	count := 0
	got := func(isWrite bool, off int64, size int, writeSeed int64) (num int, read []byte, err error) {
		if off < 0 {
			off = -off
		}
		off = off % (10 * 1024 * 1024)

		if size < 0 {
			size = -size
		}
		size = size % (10 * 1024)

		if isWrite {
			count++
			if saveEvery > 0 && count%saveEvery == 0 {
				_, err := blob.Save()
				if err != nil {
					return 0, nil, err
				}
			}

			p := make([]byte, size)
			NewRandReader(writeSeed).Read(p)
			t.Logf("write %d@%d", len(p), off)
			n, err := blob.WriteAt(p, off)
			return n, nil, err
		} else {
			p := make([]byte, size)
			t.Logf("read %d@%d", len(p), off)
			n, err := blob.ReadAt(p, off)

			// http://golang.org/pkg/io/#ReaderAt says "If the n = len(p)
			// bytes returned by ReadAt are at the end of the input
			// source, ReadAt may return either err == EOF or err ==
			// nil." Unify the result
			if n == len(p) && err == io.EOF {
				err = nil
			}

			return n, p, err
		}
	}

	exp := func(isWrite bool, off int64, size int, writeSeed int64) (num int, read []byte, err error) {
		if off < 0 {
			off = -off
		}
		off = off % (10 * 1024 * 1024)

		if size < 0 {
			size = -size
		}
		size = size % (10 * 1024)

		if isWrite {
			p := make([]byte, size)
			NewRandReader(writeSeed).Read(p)
			n, err := f.WriteAt(p, off)
			return n, nil, err
		} else {
			p := make([]byte, size)
			n, err := f.ReadAt(p, off)

			// http://golang.org/pkg/io/#ReaderAt says "If the n = len(p)
			// bytes returned by ReadAt are at the end of the input
			// source, ReadAt may return either err == EOF or err ==
			// nil." Unify the result
			if n == len(p) && err == io.EOF {
				err = nil
			}

			return n, p, err
		}
	}

	if err := quick.CheckEqual(got, exp, &qconf); err != nil {
		t.Error(err)
	}
}
コード例 #21
0
func TestSyncPull(t *testing.T) {
	tmp := tempdir.New(t)
	defer tmp.Cleanup()
	app1 := bazfstestutil.NewApp(t, tmp.Subdir("app1"))
	defer app1.Close()
	app2 := bazfstestutil.NewApp(t, tmp.Subdir("app2"))
	defer app2.Close()

	var wg sync.WaitGroup
	defer wg.Wait()
	web1 := httptest.ServeHTTP(t, &wg, app1)
	defer web1.Close()

	pub1 := (*peer.PublicKey)(app1.Keys.Sign.Pub)
	pub2 := (*peer.PublicKey)(app2.Keys.Sign.Pub)

	var volID db.VolumeID
	sharingKey := [32]byte{42, 42, 42, 13}
	const volumeName = "foo"

	setup1 := func(tx *db.Tx) error {
		sharingKey, err := tx.SharingKeys().Add("testkey", &sharingKey)
		if err != nil {
			return err
		}
		v, err := tx.Volumes().Create(volumeName, "local", sharingKey)
		if err != nil {
			return err
		}
		v.VolumeID(&volID)
		p, err := tx.Peers().Make(pub2)
		if err != nil {
			return err
		}
		if err := p.Volumes().Allow(v); err != nil {
			return err
		}
		if err := p.Storage().Allow("local"); err != nil {
			return err
		}
		return nil
	}
	if err := app1.DB.Update(setup1); err != nil {
		t.Fatalf("app1 setup: %v", err)
	}

	setup2 := func(tx *db.Tx) error {
		sharingKey, err := tx.SharingKeys().Add("testkey", &sharingKey)
		if err != nil {
			return err
		}
		v, err := tx.Volumes().Add(volumeName, &volID, "local", sharingKey)
		if err != nil {
			return err
		}
		v.VolumeID(&volID)
		p, err := tx.Peers().Make(pub1)
		if err != nil {
			return err
		}
		if err := v.Storage().Add("jdoe", "peerkey:"+pub1.String(), sharingKey); err != nil {
			return err
		}
		if err := p.Locations().Set(web1.Addr().String()); err != nil {
			return err
		}
		return nil
	}
	if err := app2.DB.Update(setup2); err != nil {
		t.Fatalf("app2 setup location: %v", err)
	}

	var chunkStore2 chunks.Store
	openKV := func(tx *db.Tx) error {
		// This cannot be combined into setup2 because OpenKV/DialPeer
		// starts its own transaction, and wouldn't see the
		// uncommitted peer.
		v, err := tx.Volumes().GetByVolumeID(&volID)
		if err != nil {
			return err
		}
		kvstore, err := app2.OpenKV(tx, v.Storage())
		if err != nil {
			return err
		}
		chunkStore2 = kvchunks.New(kvstore)
		return nil
	}
	if err := app2.DB.View(openKV); err != nil {
		t.Fatalf("cannot open storage for app2: %v", err)
	}

	const testFileName = "greeting"
	const testFileContent = "hello, world"
	func() {
		mnt := bazfstestutil.Mounted(t, app1, volumeName)
		defer mnt.Close()
		if err := ioutil.WriteFile(path.Join(mnt.Dir, testFileName), []byte(testFileContent), 0644); err != nil {
			t.Fatalf("cannot create file: %v", err)
		}
	}()

	client, err := app2.DialPeer(pub1)
	if err != nil {
		t.Fatalf("dial: %v", err)
	}
	defer client.Close()

	volIDBuf, err := volID.MarshalBinary()
	if err != nil {
		t.Fatalf("marshal volume id: %v", err)
	}
	ctx := context.Background()
	stream, err := client.VolumeSyncPull(ctx, &wire.VolumeSyncPullRequest{
		VolumeID: volIDBuf,
	})
	if err != nil {
		t.Fatalf("sync failed: %v", err)
	}

	item, err := stream.Recv()
	if err != nil {
		t.Fatalf("sync stream failed: %v", err)
	}
	if g, e := item.Error, wire.VolumeSyncPullItem_SUCCESS; g != e {
		t.Errorf("unexpected error: %v != %v", g, e)
	}
	if g, e := item.Peers, map[uint32][]byte{
		0: pub1[:],
		1: pub2[:],
	}; !reflect.DeepEqual(g, e) {
		t.Errorf("bad peers: %v != %v", g, e)
	}

	wantFiles := map[string]func(*wire.Dirent){
		testFileName: func(de *wire.Dirent) {
			if de.File == nil {
				t.Errorf("wrong type for %q, not a file: %v", de.Name, de)
				return
			}

			var c clock.Clock
			if err := c.UnmarshalBinary(de.Clock); err != nil {
				t.Errorf("invalid clock for %q: %v", de.Name, err)
				return
			}
			if g, e := c.String(), `{sync{0:1} mod{0:1} create{0:1}}`; g != e {
				t.Errorf("wrong clock for %q: %v != %v", de.Name, g, e)
				return
			}

			// verify file contents
			manifest, err := de.File.Manifest.ToBlob("file")
			if err != nil {
				t.Errorf("cannot open manifest for %q: %v", de.Name, err)
				return
			}
			blob, err := blobs.Open(chunkStore2, manifest)
			if err != nil {
				t.Errorf("cannot open blob for %q: %v", de.Name, err)
				return
			}
			r := io.NewSectionReader(blob, 0, int64(blob.Size()))
			buf, err := ioutil.ReadAll(r)
			if err != nil {
				t.Errorf("cannot read blob for %q: %v", de.Name, err)
				return
			}
			if g, e := string(buf), testFileContent; g != e {
				t.Errorf("wrong content for %q: %q != %q", de.Name, g, e)
			}
		},
	}
	for _, de := range item.Children {
		fn, ok := wantFiles[de.Name]
		if !ok {
			t.Errorf("unexpected direntry: %q", de.Name)
			continue
		}
		fn(de)
		delete(wantFiles, de.Name)
	}
	for name, _ := range wantFiles {
		t.Errorf("missing direntry: %q", name)
	}

	item, err = stream.Recv()
	if err != io.EOF {
		t.Errorf("expected eof, got error %v, item=%v", err, item)
	}
}