func main() {
addr := flag.String("addr", "", "host:port of slime-proxy to check")
uuidFlag := flag.String("uuid", "", "uuid of metadata store to verify")
warnUnconnected := flag.Int("warn", 0,
"WARN if number of undead unconnected stores exceeds this number")
critUnconnected := flag.Int("crit", 0,
"CRIT if number of undead unconnected stores exceeds this number")
flag.Parse()
if *addr == "" ||
*uuidFlag == "" ||
*warnUnconnected == 0 ||
*critUnconnected == 0 {
dieUnknown("Incorrect invocation, all arguments are required.\n" +
"Run with -h for usage.")
}
wantID, err := uuid.Parse(*uuidFlag)
if err != nil {
dieUnknown("Couldn't parse uuid given: %v", err)
}
checkUUID(*addr, wantID)
checkConnectivity(*addr, *warnUnconnected, *critUnconnected)
os.Exit(0)
}
func checkUUID(addr string, wantID [16]byte) {
resp, err := http.Get(urlJoin(addr, "/data/?mode=uuid"))
if err != nil {
dieUnknown("Couldn't get uuid: %v", err)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
dieUnknown("Got response code %v from uuid request",
resp.StatusCode)
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
dieUnknown("Couldn't read response for uuid: %v", err)
}
resp.Body.Close()
id, err := uuid.Parse(string(data))
if err != nil {
dieUnknown("Couldn't parse uuid from server: %v", err)
}
if id != wantID {
fmt.Printf("wanted id %v, got id %v\n",
uuid.Fmt(wantID), uuid.Fmt(id))
os.Exit(2)
}
}
func prefixIDFromLocalKey(key string) ([16]byte, error) {
parts := strings.SplitN(key, "_", 2)
if len(parts) != 2 {
return [16]byte{}, errors.New("not enough key components")
}
return uuid.Parse(parts[0])
}
func handleStoreStoreOperation(operation string, target string) error {
_, err := uuid.Parse(target)
if err != nil {
target, err = resolveStoreUUID(target)
if err != nil {
return err
}
}
var list []storeResponse
return jsonPost(conf.Base+"stores", map[string]string{
"operation": operation,
"uuid": target,
}, &list)
}
func (ds *Directory) Available() bool {
ds.mu.RLock()
defer ds.mu.RUnlock()
data, err := ioutil.ReadFile(filepath.Join(ds.Dir, "uuid"))
if err != nil {
return false
}
thatUUID, err := uuid.Parse(string(data))
if err != nil {
return false
}
return thatUUID == ds.uuid
}
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/uuids":
h.serveUUIDs(w, r)
case "/":
h.serveRoot(w, r)
default:
parts := strings.SplitN(r.URL.Path, "/", 3)
if len(parts) != 3 {
http.Error(w, "bad url", http.StatusBadRequest)
return
}
uuidStr := parts[1]
subObject := parts[2]
uuid, err := uuid.Parse(uuidStr)
if err != nil {
http.Error(w, "bad uuid", http.StatusBadRequest)
return
}
var subHandler *storehttp.Server
for i, st := range h.stores {
if st.UUID() == uuid {
subHandler = h.handlers[i]
break
}
}
if subHandler == nil {
http.Error(w, "no such uuid", http.StatusNotFound)
return
}
r.URL.Path = "/" + subObject
subHandler.ServeHTTP(w, r)
}
}
func openDirectoryImpl(dir string, perFileWait, perByteWait time.Duration, disableBackgroundLoops bool) (*Directory, error) {
data, err := ioutil.ReadFile(filepath.Join(dir, "uuid"))
if err != nil {
return nil, err
}
myUUID, err := uuid.Parse(string(data))
if err != nil {
return nil, err
}
host, _ := os.Hostname()
ds := &Directory{
Dir: dir,
uuid: myUUID,
name: host + ":" + dir,
perFileWait: perFileWait,
perByteWait: perByteWait,
minSplitSize: 500,
maxSplitSize: 2000,
}
err = ds.loadSplitsAndRecover()
if err != nil {
return nil, err
}
ds.tomb.Go(func() error {
if !disableBackgroundLoops {
ds.tomb.Go(ds.hashcheckLoop)
ds.tomb.Go(ds.resplitLoop)
}
return nil
})
return ds, nil
}
func (cc *Client) loadUUID() error {
resp, err := cc.startReq("GET", cc.url+"?mode=uuid", nil, nil, nil)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return httputil.ReadResponseAsError(resp)
}
// 37 is the length of a formatted UUID plus one (to avoid false positives)
data, err := ioutil.ReadAll(io.LimitReader(resp.Body, 37))
if err != nil {
return err
}
cc.uuid, err = uuid.Parse(string(data))
if err != nil {
return err
}
return nil
}
func (h *Handler) serveStores(w http.ResponseWriter, r *http.Request) {
if r.Method == "POST" {
var req storesRequest
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil {
httputil.RespondJSONError(w, err.Error(), http.StatusBadRequest)
return
}
switch req.Operation {
case "rescan":
err = h.finder.Rescan()
if err != nil {
httputil.RespondJSONError(w, fmt.Sprintf("Couldn't rescan: %v", err),
http.StatusInternalServerError)
return
}
case "scan":
err = h.finder.Scan(req.URL)
if err != nil {
httputil.RespondJSONError(w, fmt.Sprintf("Couldn't scan %v: %v", req.URL, err),
http.StatusBadRequest)
return
}
case "dead", "undead":
id, err := uuid.Parse(req.UUID)
if err != nil {
httputil.RespondJSONError(w, "Couldn't parse UUID", http.StatusBadRequest)
return
}
err = h.db.RunTx(func(ctx kvl.Ctx) error {
layer, err := meta.Open(ctx)
if err != nil {
return err
}
loc, err := layer.GetLocation(id)
if err != nil {
return err
}
if loc == nil {
return kvl.ErrNotFound
}
loc.Dead = req.Operation == "dead"
return layer.SetLocation(*loc)
})
if err != nil {
if err == kvl.ErrNotFound {
httputil.RespondJSONError(w, "No store with that UUID",
http.StatusBadRequest)
return
}
httputil.RespondJSONError(w, err.Error(), http.StatusInternalServerError)
return
}
case "delete":
id, err := uuid.Parse(req.UUID)
if err != nil {
httputil.RespondJSONError(w, "Couldn't parse UUID", http.StatusBadRequest)
return
}
st := h.finder.StoreFor(id)
if st != nil {
httputil.RespondJSONError(w, "UUID currently connected", http.StatusBadRequest)
return
}
err = h.db.RunTx(func(ctx kvl.Ctx) error {
layer, err := meta.Open(ctx)
if err != nil {
return err
}
loc, err := layer.GetLocation(id)
if err != nil {
return err
}
if loc == nil {
return kvl.ErrNotFound
}
return layer.DeleteLocation(*loc)
})
if err != nil {
if err == kvl.ErrNotFound {
httputil.RespondJSONError(w, "No store with that UUID",
http.StatusBadRequest)
return
}
httputil.RespondJSONError(w, err.Error(), http.StatusInternalServerError)
return
}
default:
httputil.RespondJSONError(w, "unsupported operation", http.StatusBadRequest)
return
}
} else if r.Method != "GET" {
w.Header().Set("Allow", "GET, POST")
httputil.RespondJSONError(w, "bad method", http.StatusMethodNotAllowed)
return
}
finderEntries := h.finder.Stores()
ret := make([]storesResponseEntry, 0, 10)
err := h.db.RunReadTx(func(ctx kvl.Ctx) error {
ret = ret[:0]
layer, err := meta.Open(ctx)
if err != nil {
return err
}
locs, err := layer.AllLocations()
if err != nil {
return err
}
for _, loc := range locs {
fe, connected := finderEntries[loc.UUID]
ret = append(ret, storesResponseEntry{
UUID: uuid.Fmt(loc.UUID),
URL: loc.URL,
Name: loc.Name,
Dead: loc.Dead,
Connected: connected,
LastSeen: time.Unix(loc.LastSeen, 0).UTC(),
Free: fe.Free,
})
}
return nil
})
if err != nil {
httputil.RespondJSONError(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("content-type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(ret)
}
func (m *Multi) scrubLocationsStep() (bool, error) {
uuidCmp := func(a, b [16]byte) int {
for i := 0; i < 16; i++ {
if a[i] < b[i] {
return -1
} else if a[i] > b[i] {
return 1
}
}
return 0
}
var thisLoc meta.Location
var haveLoc bool
var from string
var wantFiles []string
err := m.db.RunTx(func(ctx kvl.Ctx) error {
from = ""
wantFiles = nil
haveLoc = false
layer, err := meta.Open(ctx)
if err != nil {
return err
}
pos, err := layer.GetConfig("scrublocationpos")
if err != nil {
return err
}
// parse errors okay, since will be the zero UUID
since, _ := uuid.Parse(string(pos))
allLocations, err := layer.AllLocations()
if err != nil {
return err
}
if len(allLocations) == 0 {
// no locations exist
return nil
}
for {
// get the Location with the lowest UUID greater than since
haveLoc = false
for _, loc := range allLocations {
if uuidCmp(loc.UUID, since) >= 0 && (!haveLoc || uuidCmp(thisLoc.UUID, loc.UUID) > 0) {
thisLoc = loc
haveLoc = true
}
}
if haveLoc {
break
}
// all locations handled already, try again from the top
if since == [16]byte{} {
panic("locations exist but couldn't find one")
}
since = [16]byte{}
}
// set since to the next highest UUID
since = thisLoc.UUID
for i := 15; i >= 0; i-- {
since[i]++
if since[i] != 0 {
break
}
}
err = layer.SetConfig("scrublocationpos", []byte(uuid.Fmt(since)))
if err != nil {
return err
}
// now get the file list in the location we've chosen, resuming at the
// saved position (per location)
fromB, err := layer.GetConfig(
"scrublocationpos-" + uuid.Fmt(thisLoc.UUID))
if err != nil {
return err
}
from = string(fromB)
files, err := layer.GetLocationContents(thisLoc.UUID, from,
scrubLocationsCount)
if err != nil {
return err
}
var newLocPos string
if len(files) > 0 {
newLocPos = files[len(files)-1]
} else {
newLocPos = ""
}
err = layer.SetConfig(
"scrublocationpos-"+uuid.Fmt(thisLoc.UUID), []byte(newLocPos))
if err != nil {
return err
}
wantFiles = files
return nil
})
if err != nil {
return false, err
}
if !haveLoc {
// no chunk servers
return true, nil
}
wantFilesMap := make(map[string]struct{}, len(wantFiles))
for _, f := range wantFiles {
wantFilesMap[f] = struct{}{}
}
st := m.finder.StoreFor(thisLoc.UUID)
if st == nil {
log.Printf("Couldn't scrubLocation %v, it is offline",
uuid.Fmt(thisLoc.UUID))
return false, nil
}
haveFiles, err := st.List(from, scrubLocationsCount, nil)
if err != nil {
log.Printf("Couldn't List from %v: %v", uuid.Fmt(thisLoc.UUID), err)
return false, nil
}
haveFilesMap := make(map[string]struct{}, len(haveFiles))
for _, f := range haveFiles {
haveFilesMap[f] = struct{}{}
}
checkHaveFiles := make([]string, 0, len(haveFiles))
for _, f := range haveFiles {
if len(wantFiles) < scrubLocationsCount || f <= wantFiles[len(wantFiles)-1] {
checkHaveFiles = append(checkHaveFiles, f)
}
}
checkWantFiles := make([]string, 0, len(wantFiles))
for _, f := range wantFiles {
if len(haveFiles) < scrubLocationsCount || f <= haveFiles[len(haveFiles)-1] {
checkWantFiles = append(checkWantFiles, f)
}
}
for _, have := range checkHaveFiles {
if _, ok := wantFilesMap[have]; !ok {
pid, err := prefixIDFromLocalKey(have)
if err != nil {
log.Printf("Couldn't figure out PrefixID from localKey(%#v): %v",
have, err)
err = st.CAS(have, store.AnyV, store.MissingV, nil)
if err != nil {
log.Printf("Couldn't delete extraneous chunk %v from %v: %v",
have, uuid.Fmt(thisLoc.UUID), err)
continue
}
continue
}
var inWAL bool
err = m.db.RunTx(func(ctx kvl.Ctx) error {
layer, err := meta.Open(ctx)
if err != nil {
return err
}
inWAL, err = layer.WALCheck(pid)
if err != nil {
return err
}
if !inWAL {
// check to see if the write finished
inWAL, err = layer.LocationShouldHave(thisLoc.UUID, have)
if err != nil {
return err
}
}
return nil
})
if err != nil {
log.Printf("Couldn't check WAL for PrefixID %v: %v",
uuid.Fmt(pid), err)
continue
}
if inWAL {
log.Printf("extraneous chunk %v on %v, but is in WAL, skipping",
have, uuid.Fmt(thisLoc.UUID))
continue
}
log.Printf("deleting extraneous chunk %v on %v",
have, uuid.Fmt(thisLoc.UUID))
err = st.CAS(have, store.AnyV, store.MissingV, nil)
if err != nil {
log.Printf("Couldn't delete extraneous chunk %v from %v: %v",
have, uuid.Fmt(thisLoc.UUID), err)
continue
}
}
}
for _, want := range checkWantFiles {
if _, ok := haveFilesMap[want]; !ok {
log.Printf("missing chunk %v on %v", want, uuid.Fmt(thisLoc.UUID))
pid, err := prefixIDFromLocalKey(want)
if err != nil {
log.Printf("Couldn't figure out PrefixID from localKey(%#v): %v",
want, err)
continue
}
var inWAL bool
var path string
err = m.db.RunTx(func(ctx kvl.Ctx) error {
path = ""
layer, err := meta.Open(ctx)
if err != nil {
return err
}
inWAL, err = layer.WALCheck(pid)
if err != nil {
return err
}
if !inWAL {
path, err = layer.PathForPrefixID(pid)
if err != nil {
return err
}
}
return nil
})
if err != nil {
log.Printf("Couldn't get path for PrefixID %v: %v",
uuid.Fmt(pid), err)
continue
}
if inWAL {
log.Printf("skipping rebuild of %v, prefix in WAL", path)
continue
}
err = m.rebuild(path)
if err != nil {
log.Printf("Couldn't rebuild %v: %v", path, err)
continue
}
log.Printf("successfully rebuilt %v", path)
}
}
if thisLoc.Dead && len(wantFiles) > 0 {
// dead stores should be cleared
for _, want := range wantFiles {
pid, err := prefixIDFromLocalKey(want)
if err != nil {
// already logged in a loop above
continue
}
var inWAL bool
var path string
err = m.db.RunTx(func(ctx kvl.Ctx) error {
path = ""
layer, err := meta.Open(ctx)
if err != nil {
return err
}
inWAL, err = layer.WALCheck(pid)
if err != nil {
return err
}
if !inWAL {
path, err = layer.PathForPrefixID(pid)
if err != nil {
return err
}
}
return nil
})
if err != nil {
log.Printf("Couldn't check wal/path for PrefixID %v: %v",
uuid.Fmt(pid), err)
continue
}
if inWAL {
log.Printf("skipping rebuild of PrefixID %v on dead store, prefix in WAL", uuid.Fmt(pid))
continue
}
err = m.rebuild(path)
if err != nil {
log.Printf("Couldn't rebuild %v: %v", path, err)
continue
}
log.Printf("successfully rebuilt %v", path)
}
}
return len(wantFiles) == 0, nil
}