// Returns a bleve.IndexAlias that represents all the PIndexes for the
// index, including perhaps bleve remote client PIndexes.
//
// TODO: Perhaps need a tighter check around indexUUID, as the current
// implementation might have a race where old pindexes with a matching
// (but invalid) indexUUID might be hit.
//
// TODO: If this returns an error, perhaps the caller somewhere up the
// chain should close the cancelCh to help stop any other inflight
// activities.
func bleveIndexAlias(mgr *cbgt.Manager, indexName, indexUUID string,
	ensureCanRead bool, consistencyParams *cbgt.ConsistencyParams,
	cancelCh <-chan bool) (bleve.IndexAlias, error) {
	alias := bleve.NewIndexAlias()

	err := bleveIndexTargets(mgr, indexName, indexUUID, ensureCanRead,
		consistencyParams, cancelCh, alias)
	if err != nil {
		return nil, err
	}

	return alias, nil
}
Beispiel #2
0
// NewIndex returns an Index for the given start and end time, with the requested shards. It
// returns an error if an index already exists at the path.
func NewIndex(path string, startTime, endTime time.Time, numShards int) (*Index, error) {
	indexName := startTime.UTC().Format(indexNameLayout)
	indexPath := filepath.Join(path, indexName)
	durationPath := filepath.Join(indexPath, endTimeFileName)

	// Create the directory for the index, if it doesn't already exist.
	if _, err := os.Stat(indexPath); err == nil && os.IsNotExist(err) {
		return nil, fmt.Errorf("index already exists at %s", indexPath)
	}
	if err := os.MkdirAll(indexPath, 0755); err != nil {
		return nil, err
	}

	// Insert the file with the duration information.
	f, err := os.Create(durationPath)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	_, err = f.WriteString(endTime.UTC().Format(indexNameLayout))
	if err != nil {
		return nil, err
	}

	// Create the shards.
	shards := make([]*Shard, 0, numShards)
	for n := 0; n < numShards; n++ {
		s := NewShard(filepath.Join(indexPath, strconv.Itoa(n)))
		if err := s.Open(); err != nil {
			return nil, err
		}
		shards = append(shards, s)
	}

	// Create alias for searching.
	alias := bleve.NewIndexAlias()
	for _, s := range shards {
		alias.Add(s.b)
	}

	// Index is ready to go.
	return &Index{
		path:      indexPath,
		Shards:    shards,
		Alias:     alias,
		startTime: startTime,
		endTime:   endTime,
	}, nil
}
Beispiel #3
0
func UpdateAlias(alias string, add, remove []string) error {
	indexNameMappingLock.Lock()
	defer indexNameMappingLock.Unlock()

	index, exists := indexNameMapping[alias]
	if !exists {
		// new alias
		if len(remove) > 0 {
			return fmt.Errorf("cannot remove indexes from a new alias")
		}
		indexes := make([]bleve.Index, len(add))
		for i, addIndexName := range add {
			addIndex, indexExists := indexNameMapping[addIndexName]
			if !indexExists {
				return fmt.Errorf("index named '%s' does not exist", addIndexName)
			}
			indexes[i] = addIndex
		}
		indexAlias := bleve.NewIndexAlias(indexes...)
		indexNameMapping[alias] = indexAlias
	} else {
		// something with this name already exists
		indexAlias, isAlias := index.(bleve.IndexAlias)
		if !isAlias {
			return fmt.Errorf("'%s' is not an alias", alias)
		}
		// build list of add indexes
		addIndexes := make([]bleve.Index, len(add))
		for i, addIndexName := range add {
			addIndex, indexExists := indexNameMapping[addIndexName]
			if !indexExists {
				return fmt.Errorf("index named '%s' does not exist", addIndexName)
			}
			addIndexes[i] = addIndex
		}
		// build list of remove indexes
		removeIndexes := make([]bleve.Index, len(remove))
		for i, removeIndexName := range remove {
			removeIndex, indexExists := indexNameMapping[removeIndexName]
			if !indexExists {
				return fmt.Errorf("index named '%s' does not exist", removeIndexName)
			}
			removeIndexes[i] = removeIndex
		}
		indexAlias.Swap(addIndexes, removeIndexes)
	}
	return nil
}
Beispiel #4
0
// Returns a bleve.IndexAlias that represents all the PIndexes for the
// index, including perhaps bleve remote client PIndexes.
//
// TODO: Perhaps need a tighter check around indexUUID, as the current
// implementation might have a race where old pindexes with a matching
// (but invalid) indexUUID might be hit.
//
// TODO: If this returns an error, perhaps the caller somewhere up the
// chain should close the cancelCh to help stop any other inflight
// activities.
func bleveIndexAlias(mgr *cbgt.Manager, indexName, indexUUID string,
	ensureCanRead bool, consistencyParams *cbgt.ConsistencyParams,
	cancelCh <-chan bool) (bleve.IndexAlias, error) {
	planPIndexNodeFilter := cbgt.PlanPIndexNodeOk
	if ensureCanRead {
		planPIndexNodeFilter = cbgt.PlanPIndexNodeCanRead
	}

	localPIndexes, remotePlanPIndexes, err :=
		mgr.CoveringPIndexes(indexName, indexUUID, planPIndexNodeFilter,
			"queries")
	if err != nil {
		return nil, fmt.Errorf("bleve: bleveIndexAlias, err: %v", err)
	}

	alias := bleve.NewIndexAlias()

	for _, remotePlanPIndex := range remotePlanPIndexes {
		baseURL := "http://" + remotePlanPIndex.NodeDef.HostPort +
			"/api/pindex/" + remotePlanPIndex.PlanPIndex.Name
		alias.Add(&IndexClient{
			QueryURL:    baseURL + "/query",
			CountURL:    baseURL + "/count",
			Consistency: consistencyParams,
			// TODO: Propagate auth to remote client.
		})
	}

	// TODO: Should kickoff remote queries concurrently before we wait.

	err = cbgt.ConsistencyWaitGroup(indexName, consistencyParams,
		cancelCh, localPIndexes,
		func(localPIndex *cbgt.PIndex) error {
			bindex, ok := localPIndex.Impl.(bleve.Index)
			if !ok || bindex == nil ||
				!strings.HasPrefix(localPIndex.IndexType, "bleve") {
				return fmt.Errorf("bleve: wrong type, localPIndex: %#v",
					localPIndex)
			}
			alias.Add(bindex)
			return nil
		})
	if err != nil {
		return nil, err
	}

	return alias, nil
}
Beispiel #5
0
// The indexName/indexUUID is for a user-defined index alias.
//
// TODO: One day support user-defined aliases for non-bleve indexes.
func bleveIndexAliasForUserIndexAlias(mgr *cbgt.Manager,
	indexName, indexUUID string, ensureCanRead bool,
	consistencyParams *cbgt.ConsistencyParams,
	cancelCh <-chan bool) (
	bleve.IndexAlias, error) {
	alias := bleve.NewIndexAlias()

	indexDefs, _, err := cbgt.CfgGetIndexDefs(mgr.Cfg())
	if err != nil {
		return nil, fmt.Errorf("alias: could not get indexDefs,"+
			" indexName: %s, err: %v", indexName, err)
	}

	num := 0

	var fillAlias func(aliasName, aliasUUID string) error

	fillAlias = func(aliasName, aliasUUID string) error {
		aliasDef := indexDefs.IndexDefs[aliasName]
		if aliasDef == nil {
			return fmt.Errorf("alias: could not get aliasDef,"+
				" aliasName: %s, indexName: %s",
				aliasName, indexName)
		}
		if aliasDef.Type != "alias" {
			return fmt.Errorf("alias: not alias type: %s,"+
				" aliasName: %s, indexName: %s",
				aliasDef.Type, aliasName, indexName)
		}
		if aliasUUID != "" &&
			aliasUUID != aliasDef.UUID {
			return fmt.Errorf("alias: mismatched aliasUUID: %s,"+
				" aliasDef.UUID: %s, aliasName: %s, indexName: %s",
				aliasUUID, aliasDef.UUID, aliasName, indexName)
		}

		params := AliasParams{}
		err := json.Unmarshal([]byte(aliasDef.Params), &params)
		if err != nil {
			return fmt.Errorf("alias: could not parse aliasDef.Params: %s,"+
				" aliasName: %s, indexName: %s",
				aliasDef.Params, aliasName, indexName)
		}

		for targetName, targetSpec := range params.Targets {
			if num > maxAliasTargets {
				return fmt.Errorf("alias: too many alias targets,"+
					" perhaps there's a cycle,"+
					" aliasName: %s, indexName: %s",
					aliasName, indexName)
			}
			targetDef := indexDefs.IndexDefs[targetName]
			if targetDef == nil {
				return fmt.Errorf("alias: the alias depends upon"+
					" a target index that does not exist,"+
					" targetName: %q, aliasName: %q",
					targetName, aliasName)
			}
			if targetSpec.IndexUUID != "" &&
				targetSpec.IndexUUID != targetDef.UUID {
				return fmt.Errorf("alias: mismatched targetSpec.UUID: %s,"+
					" targetDef.UUID: %s, targetName: %s,"+
					" aliasName: %s, indexName: %s",
					targetSpec.IndexUUID, targetDef.UUID, targetName,
					aliasName, indexName)
			}

			// TODO: Convert to registered callbacks instead of if-else-if.
			if targetDef.Type == "alias" {
				err = fillAlias(targetName, targetSpec.IndexUUID)
				if err != nil {
					return err
				}
			} else if strings.HasPrefix(targetDef.Type, "bleve") {
				subAlias, err := bleveIndexAlias(mgr, targetName,
					targetSpec.IndexUUID, ensureCanRead,
					consistencyParams, cancelCh)
				if err != nil {
					return err
				}
				alias.Add(subAlias)
				num += 1
			} else {
				return fmt.Errorf("alias: unsupported target type: %s,"+
					" targetName: %s, aliasName: %s, indexName: %s",
					targetDef.Type, targetName, aliasName, indexName)
			}
		}

		return nil
	}

	err = fillAlias(indexName, indexUUID)
	if err != nil {
		return nil, err
	}

	return alias, nil
}
Beispiel #6
0
func main() {
	var err error

	appName := path.Base(os.Args[0])
	cfg := cli.New(appName, "CAIT", fmt.Sprintf(license, appName, cait.Version), cait.Version)
	cfg.UsageText = fmt.Sprintf(usage, appName)
	cfg.DescriptionText = fmt.Sprintf(description, appName, appName)
	cfg.OptionsText = "OPTIONS\n"

	flag.Parse()
	if showHelp == true {
		fmt.Println(cfg.Usage())
		os.Exit(0)
	}
	if showVersion == true {
		fmt.Println(cfg.Version())
		os.Exit(0)
	}

	if showLicense == true {
		fmt.Println(cfg.License())
		os.Exit(0)
	}

	siteURL = check(cfg, "site_url", cfg.MergeEnv("site_url", siteURL))
	serviceURL, err := url.Parse(siteURL)
	if err != nil {
		log.Fatal(err)
	}
	htdocsDir = check(cfg, "htdocs", cfg.MergeEnv("htdocs", htdocsDir))
	bleveNames = check(cfg, "bleve", cfg.MergeEnv("bleve", bleveNames))
	templatesDir = check(cfg, "templates", cfg.MergeEnv("templates", templatesDir))
	webhookPath = cfg.MergeEnv("webhook_path", webhookPath)
	webhookSecret = cfg.MergeEnv("webhook_secret", webhookSecret)
	webhookCommand = cfg.MergeEnv("webhook_command", webhookCommand)

	templateName := path.Join(templatesDir, "advanced-search.html")
	advancedPage, err = ioutil.ReadFile(templateName)
	if err != nil {
		log.Fatalf("Can't read templates, e.g. %s, %s", templateName, err)
	}
	templateName = path.Join(templatesDir, "basic-search.html")
	basicPage, err = ioutil.ReadFile(templateName)
	if err != nil {
		log.Fatalf("Can't read %s, %s", templateName, err)
	}

	handleSignals()

	// Wake up our search engine
	indexList := strings.Split(bleveNames, ":")
	availableIndex := false
	if enableSearch == true {
		for i := 0; i < len(indexList) && availableIndex == false; i++ {
			indexName := indexList[i]
			log.Printf("Opening %q", indexName)
			index, err = bleve.OpenUsing(indexName, map[string]interface{}{
				"read_only": true,
			})
			if err != nil {
				log.Printf("Can't open Bleve index %q, %s, trying next index", indexName, err)
			} else {
				indexAlias = bleve.NewIndexAlias(index)
				availableIndex = true
			}
		}
		if availableIndex == false {
			log.Fatalf("No index available %s", bleveNames)
		}
		defer index.Close()
	}

	// Send static file request to the default handler,
	// search routes are handled by middleware customRoutes()
	http.Handle("/", http.FileServer(http.Dir(htdocsDir)))

	log.Printf("%s %s\n", appName, cait.Version)
	log.Printf("Listening on %s\n", serviceURL.String())
	err = http.ListenAndServe(serviceURL.Host, requestLogger(customRoutes(http.DefaultServeMux)))
	if err != nil {
		log.Fatal(err)
	}
}
Beispiel #7
0
// OpenIndex opens an existing index, at the given path.
func OpenIndex(path string) (*Index, error) {
	fi, err := os.Stat(path)
	if err != nil {
		return nil, fmt.Errorf("failed to access index at %s", path)
	}
	if !fi.IsDir() {
		return nil, fmt.Errorf("index %s path is not a directory", path)
	}

	// Get the start time and end time.
	startTime, err := time.Parse(indexNameLayout, fi.Name())
	if err != nil {
		return nil, fmt.Errorf("unable to determine start time of index: %s", err.Error())
	}

	var endTime time.Time
	if f, err := os.Open(filepath.Join(path, endTimeFileName)); err != nil {
		return nil, fmt.Errorf("unable to open end time file for index: %s", err.Error())
	} else {
		defer f.Close()
		r := bufio.NewReader(f)
		if s, err := r.ReadString('\n'); err != nil && err != io.EOF {
			return nil, fmt.Errorf("unable to determine end time of index: %s", err.Error())
		} else {
			endTime, err = time.Parse(indexNameLayout, s)
			if err != nil {
				return nil, fmt.Errorf("unable to parse end time from '%s': %s", s, err.Error())
			}
		}
	}

	// Get an index directory listing.
	d, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	fis, err := d.Readdir(0)
	if err != nil {
		return nil, err
	}

	// Open the shards.
	shards := make([]*Shard, 0)
	for _, fi := range fis {
		if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
			continue
		}
		s := NewShard(filepath.Join(path, fi.Name()))
		if err := s.Open(); err != nil {
			return nil, fmt.Errorf("shard open fail: %s", err.Error())
		}
		shards = append(shards, s)
	}

	// Create alias for searching.
	alias := bleve.NewIndexAlias()
	for _, s := range shards {
		alias.Add(s.b)
	}

	// Index is ready to go.
	return &Index{
		path:      path,
		Shards:    shards,
		Alias:     alias,
		startTime: startTime,
		endTime:   endTime,
	}, nil
}