Beispiel #1
0
// CreateDirectory initializes a new Directory at the given location, suitable
// for OpenDirectory. It will return an error if one already exists.
func CreateDirectory(dir string) error {
	dirs := []string{
		dir,
		filepath.Join(dir, "data"),
		filepath.Join(dir, "quarantine"),
	}
	for _, d := range dirs {
		err := os.Mkdir(d, 0777)
		if err != nil && !os.IsExist(err) {
			return err
		}
	}

	f, err := os.OpenFile(filepath.Join(dir, "uuid"),
		os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
	if err != nil {
		return err
	}

	_, err = f.Write([]byte(uuid.Fmt(uuid.Gen4())))
	if err != nil {
		f.Close()
		return err
	}

	return f.Close()
}
Beispiel #2
0
func NewMockStore(size int64) *MockStore {
	m := &MockStore{
		contents: make(map[string]storeEntry, 128),
		uuid:     uuid.Gen4(),
		size:     size,
	}
	m.cond = sync.NewCond(&m.mu)
	return m
}
Beispiel #3
0
func TestLayerFileGetSetRemove(t *testing.T) {
	db := ram.New()

	err := db.RunTx(func(ctx kvl.Ctx) error {
		l, err := Open(ctx)
		if err != nil {
			return err
		}

		fileA := File{
			Path:         "path to a",
			Size:         4,
			SHA256:       sha256.Sum256([]byte("abcd")),
			WriteTime:    time.Now().Unix(),
			PrefixID:     uuid.Gen4(),
			DataChunks:   2,
			MappingValue: 0,
			Locations:    [][16]byte{uuid.Gen4(), uuid.Gen4()},
		}

		err = l.SetFile(&fileA)
		if err != nil {
			t.Errorf("Couldn't SetFile(%#v): %v", fileA, err)
			return err
		}

		f, err := l.GetFile("path to a")
		if err != nil {
			t.Errorf("Couldn't GetFile(path to a): %v", err)
			return err
		}

		if !reflect.DeepEqual(f, &fileA) {
			t.Errorf("GetFile(path to a) returned %#v, but wanted %#v",
				f, &fileA)
		}

		f, err = l.GetFile("nonexistent")
		if err != nil {
			t.Errorf("Couldn't GetFile(nonexistent): %v", err)
			return err
		}

		if f != nil {
			t.Errorf("GetFile(nonexistent) returned %#v, but wanted nil", f)
		}

		err = l.RemoveFilePath("nonexistent")
		if err != kvl.ErrNotFound {
			t.Errorf("RemoveFilePath(nonexistent) returned %v, but wanted %v",
				err, kvl.ErrNotFound)
			if err != nil {
				return err
			}
		}

		err = l.RemoveFilePath("path to a")
		if err != nil {
			t.Errorf("Couldn't RemoveFilePath(path to a): %v",
				err)
			return err
		}

		f, err = l.GetFile("path to a")
		if err != nil {
			t.Errorf("Couldn't GetFile(path to a): %v", err)
			return err
		}

		if f != nil {
			t.Errorf("GetFile(path to a) returned %#v, but wnated nil", f)
		}

		return nil
	})
	if err != nil {
		t.Errorf("Couldn't run transaction: %v", err)
	}
}
Beispiel #4
0
func TestLayerWALConcurrent(t *testing.T) {
	db := ram.New()

	err := db.RunTx(func(ctx kvl.Ctx) error {
		l, err := Open(ctx)
		if err != nil {
			return err
		}

		id := uuid.Gen4()
		out, err := l.WALCheck(id)
		if err != nil {
			return err
		}
		if out != false {
			t.Errorf("WALCheck returned %v on an empty database", out)
		}

		for i := 0; i < 5; i++ {
			err = l.WALMark(id)
			if err != nil {
				return err
			}

			out, err = l.WALCheck(id)
			if err != nil {
				return err
			}
			if out != true {
				t.Errorf("WALCheck returned %v after marking %v times", out, i+1)
			}
		}

		for i := 0; i < 5; i++ {
			err = l.WALClear(id)
			if err != nil {
				return err
			}

			out, err = l.WALCheck(id)
			if err != nil {
				return err
			}
			if out != (i < 4) {
				t.Errorf("WALCheck returned %v after unmarking %v times (should have %v marks)", out, i+1, 5-(i+1))
			}
		}

		out, err = l.WALCheck(id)
		if err != nil {
			return err
		}
		if out != false {
			t.Errorf("WALCheck returned %v on an empty database after unmarking", out)
		}

		return nil
	})
	if err != nil {
		t.Fatalf("Couldn't run transaction: %v", err)
	}
}
Beispiel #5
0
func TestLayerLocationContents(t *testing.T) {
	db := ram.New()

	err := db.RunTx(func(ctx kvl.Ctx) error {
		l, err := Open(ctx)
		if err != nil {
			return err
		}

		locA := uuid.Gen4()
		locB := uuid.Gen4()
		locC := uuid.Gen4()

		files := []File{
			{
				Path:         "file-a",
				Size:         1,
				SHA256:       sha256.Sum256([]byte{'a'}),
				WriteTime:    time.Now().Unix(),
				PrefixID:     uuid.Gen4(),
				DataChunks:   2,
				MappingValue: 0,
				Locations:    [][16]byte{locA, locB, locC},
			},
			{
				Path:         "file-b",
				Size:         1,
				SHA256:       sha256.Sum256([]byte{'b'}),
				WriteTime:    time.Now().Unix(),
				PrefixID:     uuid.Gen4(),
				DataChunks:   2,
				MappingValue: 0,
				Locations:    [][16]byte{locB, locA, locC},
			},
			{
				Path:         "file-c",
				Size:         1,
				SHA256:       sha256.Sum256([]byte{'c'}),
				WriteTime:    time.Now().Unix(),
				PrefixID:     uuid.Gen4(),
				DataChunks:   2,
				MappingValue: 0,
				Locations:    [][16]byte{locC, locA},
			},
		}

		for _, file := range files {
			err := l.SetFile(&file)
			if err != nil {
				t.Errorf("Couldn't SetFile: %v", err)
				return err
			}
		}

		localKeyFor := func(file File, idx int) string {
			return fmt.Sprintf("%v_%x_%v",
				uuid.Fmt(file.PrefixID), file.SHA256[:8], idx)
		}

		tests := []struct {
			ID      [16]byte
			Entries []string
		}{
			{
				ID: locA,
				Entries: []string{
					localKeyFor(files[0], 0),
					localKeyFor(files[1], 1),
					localKeyFor(files[2], 1),
				},
			},
			{
				ID: locB,
				Entries: []string{
					localKeyFor(files[0], 1),
					localKeyFor(files[1], 0),
				},
			},
			{
				ID: locC,
				Entries: []string{
					localKeyFor(files[0], 2),
					localKeyFor(files[1], 2),
					localKeyFor(files[2], 0),
				},
			},
		}

		for _, test := range tests {
			sort.Strings(test.Entries)

			names, err := l.GetLocationContents(test.ID, "", 0)
			if err != nil {
				t.Errorf("Couldn't GetLocationContents(%#v, \"\", 0): %v",
					uuid.Fmt(test.ID), err)
				return err
			}

			if !reflect.DeepEqual(names, test.Entries) {
				t.Errorf("GetLocationContents(%#v, \"\", 0) returned %#v, wanted %#v",
					uuid.Fmt(test.ID), names, test.Entries)
			}

			names = nil
			after := ""
			for {
				localNames, err := l.GetLocationContents(test.ID, after, 1)
				if err != nil {
					t.Errorf("Couldn't GetLocationContents(%#v, %#v, %v): %v",
						uuid.Fmt(test.ID), after, 1, err)
					return err
				}

				if len(localNames) > 1 {
					t.Errorf("GetLocationContents(%#v, %#v, %v) = %#v, too many results",
						uuid.Fmt(test.ID), after, 1, localNames)
				}

				if len(localNames) == 0 {
					break
				}

				names = append(names, localNames...)
				after = localNames[0]
			}

			if !reflect.DeepEqual(names, test.Entries) {
				t.Errorf("GetLocationContents on %v one by one returned %#v, wanted %#v",
					uuid.Fmt(test.ID), names, test.Entries)
			}
		}

		return nil
	})
	if err != nil {
		t.Errorf("Couldn't run transaction: %v", err)
	}
}
Beispiel #6
0
func TestLayerFileList(t *testing.T) {
	db := ram.New()

	err := db.RunTx(func(ctx kvl.Ctx) error {
		l, err := Open(ctx)
		if err != nil {
			return err
		}

		for c := 'a'; c <= 'z'; c++ {
			f := File{
				Path:         string(c),
				Size:         1,
				SHA256:       sha256.Sum256([]byte{byte(c)}),
				WriteTime:    time.Now().Unix(),
				PrefixID:     uuid.Gen4(),
				DataChunks:   1,
				MappingValue: 0,
				Locations:    [][16]byte{uuid.Gen4()},
			}

			err := l.SetFile(&f)
			if err != nil {
				t.Errorf("Couldn't SetFile: %v", err)
				return err
			}
		}

		tests := []struct {
			After string
			Limit int
			Paths []string
		}{
			{"", 2, []string{"a", "b"}},
			{"a", 4, []string{"b", "c", "d", "e"}},
			{"azz", 3, []string{"b", "c", "d"}},
			{"q", 0, []string{"r", "s", "t", "u", "v", "w", "x", "y", "z"}},
			{"w", 6, []string{"x", "y", "z"}},
		}

		for _, test := range tests {
			fs, err := l.ListFiles(test.After, test.Limit)
			if err != nil {
				t.Errorf("Couldn't ListFiles(%#v, %v): %v",
					test.After, test.Limit, err)
				continue
			}

			bad := false
			if len(fs) != len(test.Paths) {
				bad = true
			} else {
				for i, f := range fs {
					if f.Path != test.Paths[i] {
						bad = true
						break
					}
				}
			}

			if bad {
				t.Errorf("ListFiles(%#v, %v) returned %#v, but wanted paths %v",
					test.After, test.Limit, fs, test.Paths)
			}
		}

		return nil
	})
	if err != nil {
		t.Errorf("Couldn't run transaction: %v", err)
	}
}
Beispiel #7
0
func TestLayerFileGetByLocation(t *testing.T) {
	db := ram.New()

	err := db.RunTx(func(ctx kvl.Ctx) error {
		l, err := Open(ctx)
		if err != nil {
			return err
		}

		idA := [16]byte{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
		idB := [16]byte{2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
		idC := [16]byte{3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}

		fileA := File{
			Path:         "path to a",
			Size:         4,
			SHA256:       sha256.Sum256([]byte("abcd")),
			WriteTime:    time.Now().Unix(),
			PrefixID:     uuid.Gen4(),
			DataChunks:   2,
			MappingValue: 0,
			Locations:    [][16]byte{idB, idC},
		}
		fileB := File{
			Path:         "path to b",
			Size:         12,
			SHA256:       sha256.Sum256([]byte("goodbye dude")),
			WriteTime:    time.Now().Unix(),
			PrefixID:     uuid.Gen4(),
			DataChunks:   1,
			MappingValue: 0,
			Locations:    [][16]byte{idA, idB, idC},
		}

		err = l.SetFile(&fileA)
		if err != nil {
			t.Errorf("Couldn't SetFile(%#v): %v", fileA, err)
			return err
		}

		err = l.SetFile(&fileB)
		if err != nil {
			t.Errorf("Couldn't SetFile(%#v): %v", fileB, err)
			return err
		}

		fs, err := l.GetFilesByLocation(idB, 1)
		if err != nil {
			t.Errorf("Couldn't GetFilesByLocation: %v", err)
			return err
		}

		if len(fs) != 1 || !reflect.DeepEqual(fs[0], fileA) {
			t.Errorf("GetFilesByLocation(%v, 1) returned %#v, but wanted %#v",
				uuid.Fmt(idB), fs, []File{fileA})
		}

		fs, err = l.GetFilesByLocation(idA, 3)
		if err != nil {
			t.Errorf("Couldn't GetFilesByLocation: %v", err)
			return err
		}

		if len(fs) != 1 || !reflect.DeepEqual(fs[0], fileB) {
			t.Errorf("GetFilesByLocation(%v, 1) returned %#v, but wanted %#v",
				uuid.Fmt(idA), fs, []File{fileB})
		}

		fs, err = l.GetFilesByLocation(idC, 3)
		if err != nil {
			t.Errorf("Couldn't GetFilesByLocation: %v", err)
			return err
		}

		if len(fs) != 2 ||
			!reflect.DeepEqual(fs[0], fileA) ||
			!reflect.DeepEqual(fs[1], fileB) {

			t.Errorf("GetFilesByLocation(%v, 3) returned %#v, but wanted %#v",
				uuid.Fmt(idC), fs, []File{fileA, fileB})
		}

		return nil
	})
	if err != nil {
		t.Errorf("Couldn't run transaction: %v", err)
	}
}
Beispiel #8
0
func (m *Multi) CAS(key string, from, to store.CASV, cancel <-chan struct{}) error {
	var file *meta.File
	prefixid := uuid.Gen4()

	if to.Present {
		// check before doing work; additionally, add a WAL entry
		err := m.db.RunTx(func(ctx kvl.Ctx) error {
			layer, err := meta.Open(ctx)
			if err != nil {
				return err
			}

			oldFile, err := layer.GetFile(key)
			if err != nil {
				return err
			}

			if !from.Any {
				if from.Present {
					if oldFile == nil {
						return store.ErrCASFailure
					}
					if oldFile.SHA256 != from.SHA256 {
						return store.ErrCASFailure
					}
				} else {
					if oldFile != nil {
						return store.ErrCASFailure
					}
				}
			}

			err = layer.WALMark(prefixid)
			if err != nil {
				return err
			}

			return nil
		})
		if err != nil {
			return err
		}

		file, err = m.writeChunks(key, to.Data, to.SHA256, prefixid)
		if err != nil {
			return err
		}
	}

	var oldFile *meta.File
	err := m.db.RunTx(func(ctx kvl.Ctx) error {
		layer, err := meta.Open(ctx)
		if err != nil {
			return err
		}

		oldFile, err = layer.GetFile(key)
		if err != nil {
			return err
		}

		if !from.Any {
			if from.Present {
				if oldFile == nil {
					return store.ErrCASFailure
				}
				if oldFile.SHA256 != from.SHA256 {
					return store.ErrCASFailure
				}
			} else {
				if oldFile != nil {
					return store.ErrCASFailure
				}
			}
		}

		if to.Present {
			err = layer.SetFile(file)
			if err != nil {
				return err
			}

			err = layer.WALClear(prefixid)
			if err != nil {
				return err
			}
		} else {
			if from.Present || from.Any {
				err = layer.RemoveFilePath(key)
				if err == kvl.ErrNotFound {
					if !from.Any {
						// internal inconsistency
						return store.ErrCASFailure
					}
				} else if err != nil {
					return err
				}
			}
		}

		return nil
	})
	if err != nil {
		if file != nil {
			m.asyncDeletions <- file
		}
		return err
	}

	if oldFile != nil {
		m.asyncDeletions <- oldFile
	}

	return nil
}