Example #1
0
func checkUUID(addr string, wantID [16]byte) {
	resp, err := http.Get(urlJoin(addr, "/data/?mode=uuid"))
	if err != nil {
		dieUnknown("Couldn't get uuid: %v", err)
	}

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		dieUnknown("Got response code %v from uuid request",
			resp.StatusCode)
	}

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		dieUnknown("Couldn't read response for uuid: %v", err)
	}

	resp.Body.Close()

	id, err := uuid.Parse(string(data))
	if err != nil {
		dieUnknown("Couldn't parse uuid from server: %v", err)
	}

	if id != wantID {
		fmt.Printf("wanted id %v, got id %v\n",
			uuid.Fmt(wantID), uuid.Fmt(id))
		os.Exit(2)
	}
}
Example #2
0
func (l *Layer) WALClear(id [16]byte) error {
	err := l.LinearizeOn(uuid.Fmt(id))
	if err != nil {
		return err
	}

	key := tuple.MustAppend(nil, "wal2", id)
	p, err := l.inner.Get(key)
	if err != nil && err != kvl.ErrNotFound {
		return err
	}

	timestamps, err := walParse(p.Value)
	if err != nil {
		return err
	}

	if len(timestamps) == 0 {
		return errors.New("WAL does not have locks on that id")
	}

	// Remove most recent timestamp.
	//
	// In the case of hung/stopped processes that marked an id but never
	// cleared it, this effectively assumes that the hung one is the *oldest*
	// one, making the WAL lock unneccessarily longer than it should be. This
	// is not ideal, but is safe.
	timestamps = timestamps[:len(timestamps)-1]

	if len(timestamps) == 0 {
		return l.inner.Delete(key)
	}

	return l.inner.Set(kvl.Pair{key, walDump(timestamps)})
}
Example #3
0
// CreateDirectory initializes a new Directory at the given location, suitable
// for OpenDirectory. It will return an error if one already exists.
func CreateDirectory(dir string) error {
	dirs := []string{
		dir,
		filepath.Join(dir, "data"),
		filepath.Join(dir, "quarantine"),
	}
	for _, d := range dirs {
		err := os.Mkdir(d, 0777)
		if err != nil && !os.IsExist(err) {
			return err
		}
	}

	f, err := os.OpenFile(filepath.Join(dir, "uuid"),
		os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
	if err != nil {
		return err
	}

	_, err = f.Write([]byte(uuid.Fmt(uuid.Gen4())))
	if err != nil {
		f.Close()
		return err
	}

	return f.Close()
}
Example #4
0
func TestHTTPCommon(t *testing.T) {
	mock := storetests.NewMockStore(0)

	srv := httptest.NewServer(NewServer(mock))
	defer srv.Close()

	client, err := NewClient(srv.URL + "/")
	if err != nil {
		t.Fatalf("Couldn't initialize client: %v", err)
	}

	storetests.TestStore(t, client)

	if client.UUID() != mock.UUID() {
		t.Errorf("client UUID %v does not match directory UUID %v",
			uuid.Fmt(client.UUID()), uuid.Fmt(mock.UUID()))
	}
}
Example #5
0
func (h *Handler) serveUUIDs(w http.ResponseWriter, r *http.Request) {
	var resp bytes.Buffer
	for _, st := range h.stores {
		id := st.UUID()
		if id != [16]byte{} {
			fmt.Fprintf(&resp, "%v\n", uuid.Fmt(id))
		}
	}

	w.WriteHeader(http.StatusOK)
	w.Write(resp.Bytes())
}
Example #6
0
func (l *Layer) WALCheck(id [16]byte) (bool, error) {
	err := l.LinearizeOn(uuid.Fmt(id))
	if err != nil {
		return false, err
	}

	_, err = l.inner.Get(tuple.MustAppend(nil, "wal2", id))
	if err != nil {
		if err == kvl.ErrNotFound {
			err = nil
		}
		return false, err
	}
	return true, nil
}
Example #7
0
func (ds *Directory) hashcheckLoop() error {
	for {
		_, bad := ds.hashstep()
		if bad != 0 {
			log.Printf("Found %v bad items hash check on %v\n",
				bad, uuid.Fmt(ds.UUID()))
		}

		select {
		case <-time.After(5 * time.Second):
		case <-ds.tomb.Dying():
			return nil
		}
	}
}
Example #8
0
func chunkServer() {
	loadConfigOrDie()
	debug.SetGCPercent(config.GCPercent)

	stores := make([]store.Store, len(config.Chunk.Dirs))
	for i := range config.Chunk.Dirs {
		dir := config.Chunk.Dirs[i]
		construct := func() store.Store {
			log.Printf("Trying to open store at %v", dir)

			start := time.Now()

			ds, err := storedir.OpenDirectory(
				dir,
				config.Chunk.Scrubber.SleepPerFile.Duration,
				config.Chunk.Scrubber.SleepPerByte.Duration,
			)

			if err != nil {
				log.Printf("Couldn't open store at %v: %v", dir, err)
				return nil
			}

			dur := time.Now().Sub(start)

			log.Printf("Store at %v opened with UUID %v in %v",
				dir, uuid.Fmt(ds.UUID()), dur)

			return ds
		}
		stores[i] = store.NewRetryStore(construct, time.Second*15)
	}

	var h http.Handler
	var err error
	h, err = chunkserver.New(stores)
	if err != nil {
		log.Fatalf("Couldn't initialize handler: %v", err)
	}

	h = httputil.NewLimitParallelism(config.Chunk.ParallelRequests, h)
	h = httputil.AddDebugHandlers(h, config.Chunk.Debug)
	if !config.Chunk.DisableHTTPLogging {
		h = httputil.LogHTTPRequests(h)
	}
	serveOrDie(config.Chunk.Listen, h)
}
Example #9
0
func (l *Layer) WALMark(id [16]byte) error {
	err := l.LinearizeOn(uuid.Fmt(id))
	if err != nil {
		return err
	}

	key := tuple.MustAppend(nil, "wal2", id)
	p, err := l.inner.Get(key)
	if err != nil && err != kvl.ErrNotFound {
		return err
	}

	timestamps, err := walParse(p.Value)
	if err != nil {
		return err
	}

	timestamps = append(timestamps, time.Now().Unix())

	return l.inner.Set(kvl.Pair{key, walDump(timestamps)})
}
Example #10
0
func (f *File) indexPairs() []kvl.Pair {
	ret := make([]kvl.Pair, 0, len(f.Locations)*2+1)

	for idx, loc := range f.Locations {
		localKey := fmt.Sprintf("%v_%x_%v",
			uuid.Fmt(f.PrefixID), f.SHA256[:8], idx)

		ret = append(ret, kvl.Pair{
			tuple.MustAppend(nil, "file", "location", loc, f.Path),
			nil,
		}, kvl.Pair{
			tuple.MustAppend(nil, "locationlist", loc, localKey),
			nil,
		})
	}

	ret = append(ret, kvl.Pair{
		tuple.MustAppend(nil, "file", "prefix", f.PrefixID),
		[]byte(f.Path),
	})

	return ret
}
Example #11
0
func (h *Server) serveUUID(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
	w.Write([]byte(uuid.Fmt(h.store.UUID())))
}
Example #12
0
func (l *Layer) WALClearOld() error {
	var rang kvl.RangeQuery
	rang.Low, rang.High = keys.PrefixRange(tuple.MustAppend(nil, "wal2"))
	rang.Limit = 100

	for {
		ps, err := l.inner.Range(rang)
		if err != nil {
			return err
		}

		now := time.Now().Unix()

		for _, p := range ps {
			var typ string
			var id [16]byte
			err = tuple.UnpackInto(p.Key, &typ, &id)
			if err != nil {
				return err
			}

			timestamps, err := walParse(p.Value)
			if err != nil {
				return err
			}

			newestAge := time.Duration(now-timestamps[len(timestamps)-1]) * time.Second
			oldestAge := time.Duration(now-timestamps[0]) * time.Second

			if newestAge < -walUnsafeFutureAge {
				log.Printf("WARNING: WAL entry %v is %v in the future! Skipping it.",
					uuid.Fmt(id), newestAge)
			} else if oldestAge > walUnsafeOldAge {
				log.Printf("WARNING: WAL entry %v is %v in the past! Skipping it.",
					uuid.Fmt(id), oldestAge)
			} else if oldestAge > walExpireAge {
				log.Printf("Removing expired WAL entry %v (%v old)",
					uuid.Fmt(id), oldestAge)

				// Remove the oldest timestamp (only); we'll get to the other timestamps
				// in future passes if needed.
				timestamps = timestamps[1:]

				if len(timestamps) == 0 {
					err = l.inner.Delete(p.Key)
					if err != nil {
						return err
					}
				} else {
					err = l.inner.Set(kvl.Pair{p.Key, walDump(timestamps)})
					if err != nil {
						return err
					}
				}
			}
		}

		if len(ps) < rang.Limit {
			break
		}

		rang.Low = ps[len(ps)-1].Key
	}

	return nil
}
Example #13
0
func TestLayerFileGetByLocation(t *testing.T) {
	db := ram.New()

	err := db.RunTx(func(ctx kvl.Ctx) error {
		l, err := Open(ctx)
		if err != nil {
			return err
		}

		idA := [16]byte{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
		idB := [16]byte{2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
		idC := [16]byte{3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}

		fileA := File{
			Path:         "path to a",
			Size:         4,
			SHA256:       sha256.Sum256([]byte("abcd")),
			WriteTime:    time.Now().Unix(),
			PrefixID:     uuid.Gen4(),
			DataChunks:   2,
			MappingValue: 0,
			Locations:    [][16]byte{idB, idC},
		}
		fileB := File{
			Path:         "path to b",
			Size:         12,
			SHA256:       sha256.Sum256([]byte("goodbye dude")),
			WriteTime:    time.Now().Unix(),
			PrefixID:     uuid.Gen4(),
			DataChunks:   1,
			MappingValue: 0,
			Locations:    [][16]byte{idA, idB, idC},
		}

		err = l.SetFile(&fileA)
		if err != nil {
			t.Errorf("Couldn't SetFile(%#v): %v", fileA, err)
			return err
		}

		err = l.SetFile(&fileB)
		if err != nil {
			t.Errorf("Couldn't SetFile(%#v): %v", fileB, err)
			return err
		}

		fs, err := l.GetFilesByLocation(idB, 1)
		if err != nil {
			t.Errorf("Couldn't GetFilesByLocation: %v", err)
			return err
		}

		if len(fs) != 1 || !reflect.DeepEqual(fs[0], fileA) {
			t.Errorf("GetFilesByLocation(%v, 1) returned %#v, but wanted %#v",
				uuid.Fmt(idB), fs, []File{fileA})
		}

		fs, err = l.GetFilesByLocation(idA, 3)
		if err != nil {
			t.Errorf("Couldn't GetFilesByLocation: %v", err)
			return err
		}

		if len(fs) != 1 || !reflect.DeepEqual(fs[0], fileB) {
			t.Errorf("GetFilesByLocation(%v, 1) returned %#v, but wanted %#v",
				uuid.Fmt(idA), fs, []File{fileB})
		}

		fs, err = l.GetFilesByLocation(idC, 3)
		if err != nil {
			t.Errorf("Couldn't GetFilesByLocation: %v", err)
			return err
		}

		if len(fs) != 2 ||
			!reflect.DeepEqual(fs[0], fileA) ||
			!reflect.DeepEqual(fs[1], fileB) {

			t.Errorf("GetFilesByLocation(%v, 3) returned %#v, but wanted %#v",
				uuid.Fmt(idC), fs, []File{fileA, fileB})
		}

		return nil
	})
	if err != nil {
		t.Errorf("Couldn't run transaction: %v", err)
	}
}
Example #14
0
func (m *Multi) scrubLocationsStep() (bool, error) {
	uuidCmp := func(a, b [16]byte) int {
		for i := 0; i < 16; i++ {
			if a[i] < b[i] {
				return -1
			} else if a[i] > b[i] {
				return 1
			}
		}
		return 0
	}

	var thisLoc meta.Location
	var haveLoc bool
	var from string
	var wantFiles []string
	err := m.db.RunTx(func(ctx kvl.Ctx) error {
		from = ""
		wantFiles = nil
		haveLoc = false

		layer, err := meta.Open(ctx)
		if err != nil {
			return err
		}

		pos, err := layer.GetConfig("scrublocationpos")
		if err != nil {
			return err
		}

		// parse errors okay, since will be the zero UUID
		since, _ := uuid.Parse(string(pos))

		allLocations, err := layer.AllLocations()
		if err != nil {
			return err
		}

		if len(allLocations) == 0 {
			// no locations exist
			return nil
		}

		for {
			// get the Location with the lowest UUID greater than since
			haveLoc = false
			for _, loc := range allLocations {
				if uuidCmp(loc.UUID, since) >= 0 && (!haveLoc || uuidCmp(thisLoc.UUID, loc.UUID) > 0) {
					thisLoc = loc
					haveLoc = true
				}
			}

			if haveLoc {
				break
			}

			// all locations handled already, try again from the top
			if since == [16]byte{} {
				panic("locations exist but couldn't find one")
			}
			since = [16]byte{}
		}

		// set since to the next highest UUID
		since = thisLoc.UUID
		for i := 15; i >= 0; i-- {
			since[i]++
			if since[i] != 0 {
				break
			}
		}

		err = layer.SetConfig("scrublocationpos", []byte(uuid.Fmt(since)))
		if err != nil {
			return err
		}

		// now get the file list in the location we've chosen, resuming at the
		// saved position (per location)

		fromB, err := layer.GetConfig(
			"scrublocationpos-" + uuid.Fmt(thisLoc.UUID))
		if err != nil {
			return err
		}
		from = string(fromB)

		files, err := layer.GetLocationContents(thisLoc.UUID, from,
			scrubLocationsCount)
		if err != nil {
			return err
		}

		var newLocPos string
		if len(files) > 0 {
			newLocPos = files[len(files)-1]
		} else {
			newLocPos = ""
		}
		err = layer.SetConfig(
			"scrublocationpos-"+uuid.Fmt(thisLoc.UUID), []byte(newLocPos))
		if err != nil {
			return err
		}

		wantFiles = files

		return nil
	})
	if err != nil {
		return false, err
	}

	if !haveLoc {
		// no chunk servers
		return true, nil
	}

	wantFilesMap := make(map[string]struct{}, len(wantFiles))
	for _, f := range wantFiles {
		wantFilesMap[f] = struct{}{}
	}

	st := m.finder.StoreFor(thisLoc.UUID)
	if st == nil {
		log.Printf("Couldn't scrubLocation %v, it is offline",
			uuid.Fmt(thisLoc.UUID))
		return false, nil
	}

	haveFiles, err := st.List(from, scrubLocationsCount, nil)
	if err != nil {
		log.Printf("Couldn't List from %v: %v", uuid.Fmt(thisLoc.UUID), err)
		return false, nil
	}

	haveFilesMap := make(map[string]struct{}, len(haveFiles))
	for _, f := range haveFiles {
		haveFilesMap[f] = struct{}{}
	}

	checkHaveFiles := make([]string, 0, len(haveFiles))
	for _, f := range haveFiles {
		if len(wantFiles) < scrubLocationsCount || f <= wantFiles[len(wantFiles)-1] {
			checkHaveFiles = append(checkHaveFiles, f)
		}
	}

	checkWantFiles := make([]string, 0, len(wantFiles))
	for _, f := range wantFiles {
		if len(haveFiles) < scrubLocationsCount || f <= haveFiles[len(haveFiles)-1] {
			checkWantFiles = append(checkWantFiles, f)
		}
	}

	for _, have := range checkHaveFiles {
		if _, ok := wantFilesMap[have]; !ok {
			pid, err := prefixIDFromLocalKey(have)
			if err != nil {
				log.Printf("Couldn't figure out PrefixID from localKey(%#v): %v",
					have, err)

				err = st.CAS(have, store.AnyV, store.MissingV, nil)
				if err != nil {
					log.Printf("Couldn't delete extraneous chunk %v from %v: %v",
						have, uuid.Fmt(thisLoc.UUID), err)
					continue
				}

				continue
			}

			var inWAL bool
			err = m.db.RunTx(func(ctx kvl.Ctx) error {
				layer, err := meta.Open(ctx)
				if err != nil {
					return err
				}

				inWAL, err = layer.WALCheck(pid)
				if err != nil {
					return err
				}

				if !inWAL {
					// check to see if the write finished
					inWAL, err = layer.LocationShouldHave(thisLoc.UUID, have)
					if err != nil {
						return err
					}
				}

				return nil
			})
			if err != nil {
				log.Printf("Couldn't check WAL for PrefixID %v: %v",
					uuid.Fmt(pid), err)
				continue
			}

			if inWAL {
				log.Printf("extraneous chunk %v on %v, but is in WAL, skipping",
					have, uuid.Fmt(thisLoc.UUID))
				continue
			}

			log.Printf("deleting extraneous chunk %v on %v",
				have, uuid.Fmt(thisLoc.UUID))
			err = st.CAS(have, store.AnyV, store.MissingV, nil)
			if err != nil {
				log.Printf("Couldn't delete extraneous chunk %v from %v: %v",
					have, uuid.Fmt(thisLoc.UUID), err)
				continue
			}
		}
	}

	for _, want := range checkWantFiles {
		if _, ok := haveFilesMap[want]; !ok {
			log.Printf("missing chunk %v on %v", want, uuid.Fmt(thisLoc.UUID))

			pid, err := prefixIDFromLocalKey(want)
			if err != nil {
				log.Printf("Couldn't figure out PrefixID from localKey(%#v): %v",
					want, err)
				continue
			}

			var inWAL bool
			var path string
			err = m.db.RunTx(func(ctx kvl.Ctx) error {
				path = ""

				layer, err := meta.Open(ctx)
				if err != nil {
					return err
				}

				inWAL, err = layer.WALCheck(pid)
				if err != nil {
					return err
				}

				if !inWAL {
					path, err = layer.PathForPrefixID(pid)
					if err != nil {
						return err
					}
				}

				return nil
			})
			if err != nil {
				log.Printf("Couldn't get path for PrefixID %v: %v",
					uuid.Fmt(pid), err)
				continue
			}

			if inWAL {
				log.Printf("skipping rebuild of %v, prefix in WAL", path)
				continue
			}

			err = m.rebuild(path)
			if err != nil {
				log.Printf("Couldn't rebuild %v: %v", path, err)
				continue
			}

			log.Printf("successfully rebuilt %v", path)
		}
	}

	if thisLoc.Dead && len(wantFiles) > 0 {
		// dead stores should be cleared

		for _, want := range wantFiles {
			pid, err := prefixIDFromLocalKey(want)
			if err != nil {
				// already logged in a loop above
				continue
			}

			var inWAL bool
			var path string
			err = m.db.RunTx(func(ctx kvl.Ctx) error {
				path = ""

				layer, err := meta.Open(ctx)
				if err != nil {
					return err
				}

				inWAL, err = layer.WALCheck(pid)
				if err != nil {
					return err
				}

				if !inWAL {
					path, err = layer.PathForPrefixID(pid)
					if err != nil {
						return err
					}
				}

				return nil
			})
			if err != nil {
				log.Printf("Couldn't check wal/path for PrefixID %v: %v",
					uuid.Fmt(pid), err)
				continue
			}

			if inWAL {
				log.Printf("skipping rebuild of PrefixID %v on dead store, prefix in WAL", uuid.Fmt(pid))
				continue
			}

			err = m.rebuild(path)
			if err != nil {
				log.Printf("Couldn't rebuild %v: %v", path, err)
				continue
			}

			log.Printf("successfully rebuilt %v", path)
		}
	}

	return len(wantFiles) == 0, nil
}
Example #15
0
func (m *Multi) rebalanceFile(f meta.File, finderEntries map[[16]byte]FinderEntry) (bool, error) {
	// search for the lowest free location that this file is stored on
	var minI int
	var minF int64
	var minS store.Store
	for i, l := range f.Locations {
		fe, ok := finderEntries[l]
		if !ok {
			continue
		}

		if minS == nil || minF > fe.Free {
			minS = fe.Store
			minF = fe.Free
			minI = i
		}
	}

	if minS == nil {
		return false, nil
	}

	// search for the highest free location that this file is NOT stored on
	var maxF int64
	var maxS store.Store
	for id, fe := range finderEntries {
		if fe.Dead {
			continue
		}

		found := false
		for _, locid := range f.Locations {
			if locid == id {
				found = true
				break
			}
		}
		if found {
			continue
		}

		if maxS == nil || maxF < fe.Free {
			maxF = fe.Free
			maxS = fe.Store
		}
	}

	if maxS == nil {
		return false, nil
	}

	if maxF-minF < rebalanceMinDifference {
		return false, nil
	}

	// we should move chunk minI on minS to maxS

	err := m.db.RunTx(func(ctx kvl.Ctx) error {
		l, err := meta.Open(ctx)
		if err != nil {
			return err
		}

		return l.WALMark(f.PrefixID)
	})
	if err != nil {
		return false, err
	}
	defer func() {
		// TODO: how to handle errors here?
		m.db.RunTx(func(ctx kvl.Ctx) error {
			l, err := meta.Open(ctx)
			if err != nil {
				return err
			}

			return l.WALClear(f.PrefixID)
		})
	}()

	localKey := localKeyFor(&f, minI)
	data, st, err := minS.Get(localKey, store.GetOptions{})
	if err != nil {
		return false, err
	}

	setCASV := store.CASV{
		Present: true,
		SHA256:  st.SHA256,
		Data:    data,
	}

	err = maxS.CAS(localKey, store.MissingV, setCASV, nil)
	if err != nil {
		return false, err
	}

	newF := f
	newF.Locations = make([][16]byte, len(f.Locations))
	copy(newF.Locations, f.Locations)
	newF.Locations[minI] = maxS.UUID()

	err = m.db.RunTx(func(ctx kvl.Ctx) error {
		l, err := meta.Open(ctx)
		if err != nil {
			return err
		}

		f2, err := l.GetFile(f.Path)
		if err != nil {
			return err
		}

		if f2.PrefixID != f.PrefixID {
			return errModifiedDuringBalance
		}

		if len(f2.Locations) != len(f.Locations) {
			return errModifiedDuringBalance
		}

		for i, floc := range f.Locations {
			if floc != f2.Locations[i] {
				return errModifiedDuringBalance
			}
		}

		err = l.SetFile(&newF)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		maxS.CAS(localKey, setCASV, store.MissingV, nil) // ignore error
		return false, err
	}

	err = minS.CAS(localKey, setCASV, store.MissingV, nil)
	if err != nil {
		log.Printf("Couldn't remove rebalanced chunk from old location %v: %v",
			uuid.Fmt(maxS.UUID()), err)
	}

	fe := finderEntries[minS.UUID()]
	fe.Free += int64(len(data))
	finderEntries[minS.UUID()] = fe

	fe = finderEntries[maxS.UUID()]
	fe.Free -= int64(len(data))
	finderEntries[maxS.UUID()] = fe

	return true, nil
}
Example #16
0
func localKeyFor(file *meta.File, idx int) string {
	return fmt.Sprintf("%v_%x_%v",
		uuid.Fmt(file.PrefixID), file.SHA256[:8], idx)
}
Example #17
0
func (h *Handler) serveStores(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		var req storesRequest
		err := json.NewDecoder(r.Body).Decode(&req)
		if err != nil {
			httputil.RespondJSONError(w, err.Error(), http.StatusBadRequest)
			return
		}

		switch req.Operation {
		case "rescan":
			err = h.finder.Rescan()
			if err != nil {
				httputil.RespondJSONError(w, fmt.Sprintf("Couldn't rescan: %v", err),
					http.StatusInternalServerError)
				return
			}

		case "scan":
			err = h.finder.Scan(req.URL)
			if err != nil {
				httputil.RespondJSONError(w, fmt.Sprintf("Couldn't scan %v: %v", req.URL, err),
					http.StatusBadRequest)
				return
			}

		case "dead", "undead":
			id, err := uuid.Parse(req.UUID)
			if err != nil {
				httputil.RespondJSONError(w, "Couldn't parse UUID", http.StatusBadRequest)
				return
			}

			err = h.db.RunTx(func(ctx kvl.Ctx) error {
				layer, err := meta.Open(ctx)
				if err != nil {
					return err
				}

				loc, err := layer.GetLocation(id)
				if err != nil {
					return err
				}

				if loc == nil {
					return kvl.ErrNotFound
				}

				loc.Dead = req.Operation == "dead"

				return layer.SetLocation(*loc)
			})
			if err != nil {
				if err == kvl.ErrNotFound {
					httputil.RespondJSONError(w, "No store with that UUID",
						http.StatusBadRequest)
					return
				}
				httputil.RespondJSONError(w, err.Error(), http.StatusInternalServerError)
				return
			}

		case "delete":
			id, err := uuid.Parse(req.UUID)
			if err != nil {
				httputil.RespondJSONError(w, "Couldn't parse UUID", http.StatusBadRequest)
				return
			}

			st := h.finder.StoreFor(id)
			if st != nil {
				httputil.RespondJSONError(w, "UUID currently connected", http.StatusBadRequest)
				return
			}

			err = h.db.RunTx(func(ctx kvl.Ctx) error {
				layer, err := meta.Open(ctx)
				if err != nil {
					return err
				}

				loc, err := layer.GetLocation(id)
				if err != nil {
					return err
				}

				if loc == nil {
					return kvl.ErrNotFound
				}

				return layer.DeleteLocation(*loc)
			})
			if err != nil {
				if err == kvl.ErrNotFound {
					httputil.RespondJSONError(w, "No store with that UUID",
						http.StatusBadRequest)
					return
				}
				httputil.RespondJSONError(w, err.Error(), http.StatusInternalServerError)
				return
			}

		default:
			httputil.RespondJSONError(w, "unsupported operation", http.StatusBadRequest)
			return
		}
	} else if r.Method != "GET" {
		w.Header().Set("Allow", "GET, POST")
		httputil.RespondJSONError(w, "bad method", http.StatusMethodNotAllowed)
		return
	}

	finderEntries := h.finder.Stores()

	ret := make([]storesResponseEntry, 0, 10)
	err := h.db.RunReadTx(func(ctx kvl.Ctx) error {
		ret = ret[:0]

		layer, err := meta.Open(ctx)
		if err != nil {
			return err
		}

		locs, err := layer.AllLocations()
		if err != nil {
			return err
		}

		for _, loc := range locs {
			fe, connected := finderEntries[loc.UUID]

			ret = append(ret, storesResponseEntry{
				UUID:      uuid.Fmt(loc.UUID),
				URL:       loc.URL,
				Name:      loc.Name,
				Dead:      loc.Dead,
				Connected: connected,
				LastSeen:  time.Unix(loc.LastSeen, 0).UTC(),
				Free:      fe.Free,
			})
		}

		return nil
	})
	if err != nil {
		httputil.RespondJSONError(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("content-type", "application/json; charset=utf-8")
	w.WriteHeader(http.StatusOK)
	json.NewEncoder(w).Encode(ret)
}
Example #18
0
func TestLayerLocationContents(t *testing.T) {
	db := ram.New()

	err := db.RunTx(func(ctx kvl.Ctx) error {
		l, err := Open(ctx)
		if err != nil {
			return err
		}

		locA := uuid.Gen4()
		locB := uuid.Gen4()
		locC := uuid.Gen4()

		files := []File{
			{
				Path:         "file-a",
				Size:         1,
				SHA256:       sha256.Sum256([]byte{'a'}),
				WriteTime:    time.Now().Unix(),
				PrefixID:     uuid.Gen4(),
				DataChunks:   2,
				MappingValue: 0,
				Locations:    [][16]byte{locA, locB, locC},
			},
			{
				Path:         "file-b",
				Size:         1,
				SHA256:       sha256.Sum256([]byte{'b'}),
				WriteTime:    time.Now().Unix(),
				PrefixID:     uuid.Gen4(),
				DataChunks:   2,
				MappingValue: 0,
				Locations:    [][16]byte{locB, locA, locC},
			},
			{
				Path:         "file-c",
				Size:         1,
				SHA256:       sha256.Sum256([]byte{'c'}),
				WriteTime:    time.Now().Unix(),
				PrefixID:     uuid.Gen4(),
				DataChunks:   2,
				MappingValue: 0,
				Locations:    [][16]byte{locC, locA},
			},
		}

		for _, file := range files {
			err := l.SetFile(&file)
			if err != nil {
				t.Errorf("Couldn't SetFile: %v", err)
				return err
			}
		}

		localKeyFor := func(file File, idx int) string {
			return fmt.Sprintf("%v_%x_%v",
				uuid.Fmt(file.PrefixID), file.SHA256[:8], idx)
		}

		tests := []struct {
			ID      [16]byte
			Entries []string
		}{
			{
				ID: locA,
				Entries: []string{
					localKeyFor(files[0], 0),
					localKeyFor(files[1], 1),
					localKeyFor(files[2], 1),
				},
			},
			{
				ID: locB,
				Entries: []string{
					localKeyFor(files[0], 1),
					localKeyFor(files[1], 0),
				},
			},
			{
				ID: locC,
				Entries: []string{
					localKeyFor(files[0], 2),
					localKeyFor(files[1], 2),
					localKeyFor(files[2], 0),
				},
			},
		}

		for _, test := range tests {
			sort.Strings(test.Entries)

			names, err := l.GetLocationContents(test.ID, "", 0)
			if err != nil {
				t.Errorf("Couldn't GetLocationContents(%#v, \"\", 0): %v",
					uuid.Fmt(test.ID), err)
				return err
			}

			if !reflect.DeepEqual(names, test.Entries) {
				t.Errorf("GetLocationContents(%#v, \"\", 0) returned %#v, wanted %#v",
					uuid.Fmt(test.ID), names, test.Entries)
			}

			names = nil
			after := ""
			for {
				localNames, err := l.GetLocationContents(test.ID, after, 1)
				if err != nil {
					t.Errorf("Couldn't GetLocationContents(%#v, %#v, %v): %v",
						uuid.Fmt(test.ID), after, 1, err)
					return err
				}

				if len(localNames) > 1 {
					t.Errorf("GetLocationContents(%#v, %#v, %v) = %#v, too many results",
						uuid.Fmt(test.ID), after, 1, localNames)
				}

				if len(localNames) == 0 {
					break
				}

				names = append(names, localNames...)
				after = localNames[0]
			}

			if !reflect.DeepEqual(names, test.Entries) {
				t.Errorf("GetLocationContents on %v one by one returned %#v, wanted %#v",
					uuid.Fmt(test.ID), names, test.Entries)
			}
		}

		return nil
	})
	if err != nil {
		t.Errorf("Couldn't run transaction: %v", err)
	}
}