Example #1
0
func maybeStartQuery(queryid, src, query string) bool {
	stateMu.Lock()
	defer stateMu.Unlock()
	querystate, running := state[queryid]
	// XXX: Starting a new query while there may still be clients reading that
	// query is not a great idea. Best fix may be to make getEvent() use a
	// querystate instead of the string identifier.
	if !running || time.Since(querystate.started) > 30*time.Minute {
		// See if we can garbage-collect old queries.
		if !running && len(state) >= 10 {
			log.Printf("Trying to garbage collect queries (currently %d)\n", len(state))
			for queryid, s := range state {
				if len(state) < 10 {
					break
				}
				if !s.done {
					continue
				}
				for _, state := range s.perBackend {
					state.tempFile.Close()
				}
				delete(state, queryid)
			}
			log.Printf("Garbage collection done. %d queries remaining", len(state))
		}
		backends := strings.Split(*common.SourceBackends, ",")
		state[queryid] = queryState{
			started:        time.Now(),
			query:          query,
			newEvent:       sync.NewCond(&sync.Mutex{}),
			filesTotal:     make([]int, len(backends)),
			filesProcessed: make([]int, len(backends)),
			filesMu:        &sync.Mutex{},
			perBackend:     make([]*perBackendState, len(backends)),
			tempFilesMu:    &sync.Mutex{},
		}

		activeQueries.Add(1)

		var err error
		dir := filepath.Join(*queryResultsPath, queryid)
		if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil {
			log.Printf("[%s] could not create %q: %v\n", queryid, dir, err)
			failQuery(queryid)
			return false
		}

		// TODO: it’d be so much better if we would correctly handle ESPACE errors
		// in the code below (and above), but for that we need to carefully test it.
		ensureEnoughSpaceAvailable()

		for i := 0; i < len(backends); i++ {
			state[queryid].filesTotal[i] = -1
			path := filepath.Join(dir, fmt.Sprintf("unsorted_%d.capnproto", i))
			f, err := os.Create(path)
			if err != nil {
				log.Printf("[%s] could not create %q: %v\n", queryid, path, err)
				failQuery(queryid)
				return false
			}
			state[queryid].perBackend[i] = &perBackendState{
				packagePool:    stringpool.NewStringPool(),
				tempFile:       f,
				tempFileWriter: bufio.NewWriterSize(f, 65536),
				allPackages:    make(map[string]bool),
			}
		}
		log.Printf("initial results = %v\n", state[queryid])

		// Rewrite the query into a query for source backends.
		fakeUrl, err := url.Parse("?" + query)
		if err != nil {
			log.Fatal(err)
		}
		rewritten := search.RewriteQuery(*fakeUrl)
		type streamingRequest struct {
			Query string
			URL   string
		}
		request := streamingRequest{
			Query: rewritten.Query().Get("q"),
			URL:   rewritten.String(),
		}
		log.Printf("[%s] querying for %q\n", queryid, request.Query)
		sourceQuery, err := json.Marshal(&request)
		if err != nil {
			log.Fatal(err)
		}

		for idx, backend := range backends {
			go queryBackend(queryid, backend, idx, sourceQuery)
		}
		return false
	}

	return true
}
Example #2
0
// maybeStartQuery starts a specified query if that query does not already
// exist. Returns whether the query existed and any errors during query
// creation.
func maybeStartQuery(queryid, src, query string) (bool, error) {
	if queryExists(queryid) {
		return true, nil
	}

	querystate := queryState{
		started:        time.Now(),
		query:          query,
		newEvent:       sync.NewCond(&sync.Mutex{}),
		filesTotal:     make([]int, len(common.SourceBackendStubs)),
		filesProcessed: make([]int, len(common.SourceBackendStubs)),
		filesMu:        &sync.Mutex{},
		perBackend:     make([]*perBackendState, len(common.SourceBackendStubs)),
		tempFilesMu:    &sync.Mutex{},
	}

	dir := filepath.Join(*queryResultsPath, queryid)
	if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil {
		return false, fmt.Errorf("could not create %q: %v", dir, err)
	}

	// TODO: it’d be so much better if we would correctly handle ESPACE errors
	// in the code below (and above), but for that we need to carefully test it.
	ensureEnoughSpaceAvailable()

	for i := 0; i < len(common.SourceBackendStubs); i++ {
		querystate.filesTotal[i] = -1
		path := filepath.Join(dir, fmt.Sprintf("unsorted_%d.pb", i))
		f, err := os.Create(path)
		if err != nil {
			return false, fmt.Errorf("could not create %q: %v", path, err)
		}
		querystate.perBackend[i] = &perBackendState{
			packagePool:    stringpool.NewStringPool(),
			tempFile:       f,
			tempFileWriter: bufio.NewWriterSize(f, 65536),
			allPackages:    make(map[string]bool),
		}
	}
	log.Printf("querystate = %v\n", querystate)

	// Rewrite the query into a query for source backends.
	fakeUrl, err := url.Parse("?" + query)
	if err != nil {
		log.Fatal(err)
	}
	rewritten := search.RewriteQuery(*fakeUrl)
	searchRequest := &pb.SearchRequest{
		Query:        rewritten.Query().Get("q"),
		RewrittenUrl: rewritten.String(),
	}
	log.Printf("[%s] querying for %+v\n", queryid, searchRequest)
	if err := startQuery(queryid, querystate); err != nil {
		// Another goroutine must have raced us since we called queryExists().
		return true, nil
	}
	for idx, backend := range common.SourceBackendStubs {
		go queryBackend(queryid, src, backend, idx, searchRequest)
	}
	return false, nil
}