Esempio n. 1
0
func walkOnce(
	dst farm.Selecter,
	wait waiter,
	src <-chan []string,
	maxSize int,
	instr instrumentation.WalkInstrumentation,
) {
	for batch := range src {
		log.Printf("walk: received batch of %d, requesting tokens", len(batch))
		wait.Wait(int64(len(batch)))
		log.Printf("walk: received tokens, performing Select")
		dst.Select(batch, 0, maxSize)
		instr.WalkKeys(len(batch))
		log.Printf("walk: performed Select, waiting for next batch")
	}
}
Esempio n. 2
0
func handleSelect(selecter farm.Selecter) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		began := time.Now()

		if err := r.ParseForm(); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
			return
		}
		offset := parseInt(r.Form, "offset", 0)
		limit := parseInt(r.Form, "limit", 10)
		coalesce := parseBool(r.Form, "coalesce", false)

		var keys [][]byte
		defer r.Body.Close()
		if err := json.NewDecoder(r.Body).Decode(&keys); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusBadRequest, err)
			return
		}

		keyStrings := make([]string, len(keys))
		for i := range keys {
			keyStrings[i] = string(keys[i])
		}

		var records interface{}
		if coalesce {
			// We need to Select from 0 to offset+limit, flatten the map to a
			// single ordered slice, and then cut off the last limit elements.
			m, err := selecter.Select(keyStrings, 0, offset+limit)
			if err != nil {
				respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
				return
			}
			records = flatten(m, offset, limit)
		} else {
			// We can directly Select using the given offset and limit.
			m, err := selecter.Select(keyStrings, offset, limit)
			if err != nil {
				respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
				return
			}
			records = m
		}

		respondSelected(w, keys, offset, limit, records, time.Since(began))
	}
}
Esempio n. 3
0
func walkOnce(
	dst farm.Selecter,
	wait waiter,
	src <-chan []string,
	maxSize int,
	instr instrumentation.WalkInstrumentation,
) {
	defer func(t time.Time) { log.Printf("single walk complete, %s", time.Since(t)) }(time.Now())
	for batch := range src {
		log.Printf("walk: received batch of %d, requesting tokens", len(batch))
		wait.Wait(int64(len(batch)))
		log.Printf("walk: received tokens, performing Select")
		dst.SelectOffset(batch, 0, maxSize)
		instr.WalkKeys(len(batch))
		log.Printf("walk: performed Select, waiting for next batch")
	}
}
Esempio n. 4
0
func handleSelect(selecter farm.Selecter) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		began := time.Now()

		if err := r.ParseForm(); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
			return
		}

		var keys [][]byte
		if err := json.NewDecoder(r.Body).Decode(&keys); err != nil {
			respondError(w, r.Method, r.URL.String(), http.StatusBadRequest, err)
			return
		}

		keyStrings := make([]string, len(keys))
		for i := range keys {
			keyStrings[i] = string(keys[i])
		}

		var (
			offset, offsetGiven  = parseInt(r.Form, "offset", 0)
			startStr, startGiven = parseStr(r.Form, "start", "")
			stopStr, stopGiven   = parseStr(r.Form, "stop", "")
			limit, _             = parseInt(r.Form, "limit", 10)
			coalesce, _          = parseBool(r.Form, "coalesce", false)
		)

		switch {
		case !offsetGiven && (startGiven || stopGiven):
			// SelectRange. `coalesce` has no impact on the request, only the
			// handling of the response.

			var (
				start = common.Cursor{Score: math.MaxFloat64}
				stop  = common.Cursor{Score: 0}
			)

			if startGiven {
				if err := start.Parse(startStr); err != nil {
					respondError(w, r.Method, r.URL.String(), http.StatusBadRequest, err)
					return
				}
			}

			if stopGiven {
				if err := stop.Parse(stopStr); err != nil {
					respondError(w, r.Method, r.URL.String(), http.StatusBadRequest, err)
					return
				}
			}

			results, err := selecter.SelectRange(keyStrings, start, stop, limit)
			if err != nil {
				respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
				return
			}

			//cursorResults := addCursor(results)

			if coalesce {
				respondSelected(w, flatten(results, 0, limit), time.Since(began))
				return
			}

			respondSelected(w, results, time.Since(began))
			return

		case !startGiven && !stopGiven:
			// SelectOffset. The offset/limit may be altered by `coalesce`.
			var (
				selectOffset = offset
				selectLimit  = limit
			)

			if coalesce {
				selectOffset = 0
				selectLimit = offset + limit
			}

			results, err := selecter.SelectOffset(keyStrings, selectOffset, selectLimit)
			if err != nil {
				respondError(w, r.Method, r.URL.String(), http.StatusInternalServerError, err)
				return
			}

			//cursorResults := addCursor(results)

			if coalesce {
				respondSelected(w, flatten(results, offset, limit), time.Since(began))
				return
			}

			respondSelected(w, results, time.Since(began))
			return

		case offsetGiven && (startGiven || stopGiven):
			respondError(w, r.Method, r.URL.String(), http.StatusBadRequest, fmt.Errorf("cannot specify both offset and start/stop"))
			return

		default:
			panic("unreachable")
		}
	}
}