func walkOnce( dst farm.Selecter, wait waiter, src <-chan []string, maxSize int, instr instrumentation.WalkInstrumentation, ) { defer func(t time.Time) { log.Printf("single walk complete, %s", time.Since(t)) }(time.Now()) for batch := range src { log.Printf("walk: received batch of %d, requesting tokens", len(batch)) wait.Wait(int64(len(batch))) log.Printf("walk: received tokens, performing Select") dst.SelectOffset(batch, 0, maxSize) instr.WalkKeys(len(batch)) log.Printf("walk: performed Select, waiting for next batch") } }
func handleSelect(selecter farm.Selecter) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { began := time.Now() if err := r.ParseForm(); err != nil { respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err) return } var keys [][]byte if err := json.NewDecoder(r.Body).Decode(&keys); err != nil { respondError(w, r.Method, r.URL.String(), http.StatusBadRequest, err) return } keyStrings := make([]string, len(keys)) for i := range keys { keyStrings[i] = string(keys[i]) } var ( offset, offsetGiven = parseInt(r.Form, "offset", 0) startStr, startGiven = parseStr(r.Form, "start", "") stopStr, stopGiven = parseStr(r.Form, "stop", "") limit, _ = parseInt(r.Form, "limit", 10) coalesce, _ = parseBool(r.Form, "coalesce", false) ) switch { case !offsetGiven && (startGiven || stopGiven): // SelectRange. `coalesce` has no impact on the request, only the // handling of the response. var ( start = common.Cursor{Score: math.MaxFloat64} stop = common.Cursor{Score: 0} ) if startGiven { if err := start.Parse(startStr); err != nil { respondError(w, r.Method, r.URL.String(), http.StatusBadRequest, err) return } } if stopGiven { if err := stop.Parse(stopStr); err != nil { respondError(w, r.Method, r.URL.String(), http.StatusBadRequest, err) return } } results, err := selecter.SelectRange(keyStrings, start, stop, limit) if err != nil { respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err) return } //cursorResults := addCursor(results) if coalesce { respondSelected(w, flatten(results, 0, limit), time.Since(began)) return } respondSelected(w, results, time.Since(began)) return case !startGiven && !stopGiven: // SelectOffset. The offset/limit may be altered by `coalesce`. var ( selectOffset = offset selectLimit = limit ) if coalesce { selectOffset = 0 selectLimit = offset + limit } results, err := selecter.SelectOffset(keyStrings, selectOffset, selectLimit) if err != nil { respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err) return } //cursorResults := addCursor(results) if coalesce { respondSelected(w, flatten(results, offset, limit), time.Since(began)) return } respondSelected(w, results, time.Since(began)) return case offsetGiven && (startGiven || stopGiven): respondError(w, r.Method, r.URL.String(), http.StatusBadRequest, fmt.Errorf("cannot specify both offset and start/stop")) return default: panic("unreachable") } } }