// Returns a bleve.IndexAlias that represents all the PIndexes for the
// index, including perhaps bleve remote client PIndexes.
//
// TODO: Perhaps need a tighter check around indexUUID, as the current
// implementation might have a race where old pindexes with a matching
// (but invalid) indexUUID might be hit.
//
// TODO: If this returns an error, perhaps the caller somewhere up the
// chain should close the cancelCh to help stop any other inflight
// activities.
func bleveIndexAlias(mgr *cbgt.Manager, indexName, indexUUID string,
ensureCanRead bool, consistencyParams *cbgt.ConsistencyParams,
cancelCh <-chan bool) (bleve.IndexAlias, error) {
alias := bleve.NewIndexAlias()
err := bleveIndexTargets(mgr, indexName, indexUUID, ensureCanRead,
consistencyParams, cancelCh, alias)
if err != nil {
return nil, err
}
return alias, nil
}
// NewIndex returns an Index for the given start and end time, with the requested shards. It
// returns an error if an index already exists at the path.
func NewIndex(path string, startTime, endTime time.Time, numShards int) (*Index, error) {
indexName := startTime.UTC().Format(indexNameLayout)
indexPath := filepath.Join(path, indexName)
durationPath := filepath.Join(indexPath, endTimeFileName)
// Create the directory for the index, if it doesn't already exist.
if _, err := os.Stat(indexPath); err == nil && os.IsNotExist(err) {
return nil, fmt.Errorf("index already exists at %s", indexPath)
}
if err := os.MkdirAll(indexPath, 0755); err != nil {
return nil, err
}
// Insert the file with the duration information.
f, err := os.Create(durationPath)
if err != nil {
return nil, err
}
defer f.Close()
_, err = f.WriteString(endTime.UTC().Format(indexNameLayout))
if err != nil {
return nil, err
}
// Create the shards.
shards := make([]*Shard, 0, numShards)
for n := 0; n < numShards; n++ {
s := NewShard(filepath.Join(indexPath, strconv.Itoa(n)))
if err := s.Open(); err != nil {
return nil, err
}
shards = append(shards, s)
}
// Create alias for searching.
alias := bleve.NewIndexAlias()
for _, s := range shards {
alias.Add(s.b)
}
// Index is ready to go.
return &Index{
path: indexPath,
Shards: shards,
Alias: alias,
startTime: startTime,
endTime: endTime,
}, nil
}
func UpdateAlias(alias string, add, remove []string) error {
indexNameMappingLock.Lock()
defer indexNameMappingLock.Unlock()
index, exists := indexNameMapping[alias]
if !exists {
// new alias
if len(remove) > 0 {
return fmt.Errorf("cannot remove indexes from a new alias")
}
indexes := make([]bleve.Index, len(add))
for i, addIndexName := range add {
addIndex, indexExists := indexNameMapping[addIndexName]
if !indexExists {
return fmt.Errorf("index named '%s' does not exist", addIndexName)
}
indexes[i] = addIndex
}
indexAlias := bleve.NewIndexAlias(indexes...)
indexNameMapping[alias] = indexAlias
} else {
// something with this name already exists
indexAlias, isAlias := index.(bleve.IndexAlias)
if !isAlias {
return fmt.Errorf("'%s' is not an alias", alias)
}
// build list of add indexes
addIndexes := make([]bleve.Index, len(add))
for i, addIndexName := range add {
addIndex, indexExists := indexNameMapping[addIndexName]
if !indexExists {
return fmt.Errorf("index named '%s' does not exist", addIndexName)
}
addIndexes[i] = addIndex
}
// build list of remove indexes
removeIndexes := make([]bleve.Index, len(remove))
for i, removeIndexName := range remove {
removeIndex, indexExists := indexNameMapping[removeIndexName]
if !indexExists {
return fmt.Errorf("index named '%s' does not exist", removeIndexName)
}
removeIndexes[i] = removeIndex
}
indexAlias.Swap(addIndexes, removeIndexes)
}
return nil
}
// Returns a bleve.IndexAlias that represents all the PIndexes for the
// index, including perhaps bleve remote client PIndexes.
//
// TODO: Perhaps need a tighter check around indexUUID, as the current
// implementation might have a race where old pindexes with a matching
// (but invalid) indexUUID might be hit.
//
// TODO: If this returns an error, perhaps the caller somewhere up the
// chain should close the cancelCh to help stop any other inflight
// activities.
func bleveIndexAlias(mgr *cbgt.Manager, indexName, indexUUID string,
ensureCanRead bool, consistencyParams *cbgt.ConsistencyParams,
cancelCh <-chan bool) (bleve.IndexAlias, error) {
planPIndexNodeFilter := cbgt.PlanPIndexNodeOk
if ensureCanRead {
planPIndexNodeFilter = cbgt.PlanPIndexNodeCanRead
}
localPIndexes, remotePlanPIndexes, err :=
mgr.CoveringPIndexes(indexName, indexUUID, planPIndexNodeFilter,
"queries")
if err != nil {
return nil, fmt.Errorf("bleve: bleveIndexAlias, err: %v", err)
}
alias := bleve.NewIndexAlias()
for _, remotePlanPIndex := range remotePlanPIndexes {
baseURL := "http://" + remotePlanPIndex.NodeDef.HostPort +
"/api/pindex/" + remotePlanPIndex.PlanPIndex.Name
alias.Add(&IndexClient{
QueryURL: baseURL + "/query",
CountURL: baseURL + "/count",
Consistency: consistencyParams,
// TODO: Propagate auth to remote client.
})
}
// TODO: Should kickoff remote queries concurrently before we wait.
err = cbgt.ConsistencyWaitGroup(indexName, consistencyParams,
cancelCh, localPIndexes,
func(localPIndex *cbgt.PIndex) error {
bindex, ok := localPIndex.Impl.(bleve.Index)
if !ok || bindex == nil ||
!strings.HasPrefix(localPIndex.IndexType, "bleve") {
return fmt.Errorf("bleve: wrong type, localPIndex: %#v",
localPIndex)
}
alias.Add(bindex)
return nil
})
if err != nil {
return nil, err
}
return alias, nil
}
// The indexName/indexUUID is for a user-defined index alias.
//
// TODO: One day support user-defined aliases for non-bleve indexes.
func bleveIndexAliasForUserIndexAlias(mgr *cbgt.Manager,
indexName, indexUUID string, ensureCanRead bool,
consistencyParams *cbgt.ConsistencyParams,
cancelCh <-chan bool) (
bleve.IndexAlias, error) {
alias := bleve.NewIndexAlias()
indexDefs, _, err := cbgt.CfgGetIndexDefs(mgr.Cfg())
if err != nil {
return nil, fmt.Errorf("alias: could not get indexDefs,"+
" indexName: %s, err: %v", indexName, err)
}
num := 0
var fillAlias func(aliasName, aliasUUID string) error
fillAlias = func(aliasName, aliasUUID string) error {
aliasDef := indexDefs.IndexDefs[aliasName]
if aliasDef == nil {
return fmt.Errorf("alias: could not get aliasDef,"+
" aliasName: %s, indexName: %s",
aliasName, indexName)
}
if aliasDef.Type != "alias" {
return fmt.Errorf("alias: not alias type: %s,"+
" aliasName: %s, indexName: %s",
aliasDef.Type, aliasName, indexName)
}
if aliasUUID != "" &&
aliasUUID != aliasDef.UUID {
return fmt.Errorf("alias: mismatched aliasUUID: %s,"+
" aliasDef.UUID: %s, aliasName: %s, indexName: %s",
aliasUUID, aliasDef.UUID, aliasName, indexName)
}
params := AliasParams{}
err := json.Unmarshal([]byte(aliasDef.Params), ¶ms)
if err != nil {
return fmt.Errorf("alias: could not parse aliasDef.Params: %s,"+
" aliasName: %s, indexName: %s",
aliasDef.Params, aliasName, indexName)
}
for targetName, targetSpec := range params.Targets {
if num > maxAliasTargets {
return fmt.Errorf("alias: too many alias targets,"+
" perhaps there's a cycle,"+
" aliasName: %s, indexName: %s",
aliasName, indexName)
}
targetDef := indexDefs.IndexDefs[targetName]
if targetDef == nil {
return fmt.Errorf("alias: the alias depends upon"+
" a target index that does not exist,"+
" targetName: %q, aliasName: %q",
targetName, aliasName)
}
if targetSpec.IndexUUID != "" &&
targetSpec.IndexUUID != targetDef.UUID {
return fmt.Errorf("alias: mismatched targetSpec.UUID: %s,"+
" targetDef.UUID: %s, targetName: %s,"+
" aliasName: %s, indexName: %s",
targetSpec.IndexUUID, targetDef.UUID, targetName,
aliasName, indexName)
}
// TODO: Convert to registered callbacks instead of if-else-if.
if targetDef.Type == "alias" {
err = fillAlias(targetName, targetSpec.IndexUUID)
if err != nil {
return err
}
} else if strings.HasPrefix(targetDef.Type, "bleve") {
subAlias, err := bleveIndexAlias(mgr, targetName,
targetSpec.IndexUUID, ensureCanRead,
consistencyParams, cancelCh)
if err != nil {
return err
}
alias.Add(subAlias)
num += 1
} else {
return fmt.Errorf("alias: unsupported target type: %s,"+
" targetName: %s, aliasName: %s, indexName: %s",
targetDef.Type, targetName, aliasName, indexName)
}
}
return nil
}
err = fillAlias(indexName, indexUUID)
if err != nil {
return nil, err
}
return alias, nil
}
func main() {
var err error
appName := path.Base(os.Args[0])
cfg := cli.New(appName, "CAIT", fmt.Sprintf(license, appName, cait.Version), cait.Version)
cfg.UsageText = fmt.Sprintf(usage, appName)
cfg.DescriptionText = fmt.Sprintf(description, appName, appName)
cfg.OptionsText = "OPTIONS\n"
flag.Parse()
if showHelp == true {
fmt.Println(cfg.Usage())
os.Exit(0)
}
if showVersion == true {
fmt.Println(cfg.Version())
os.Exit(0)
}
if showLicense == true {
fmt.Println(cfg.License())
os.Exit(0)
}
siteURL = check(cfg, "site_url", cfg.MergeEnv("site_url", siteURL))
serviceURL, err := url.Parse(siteURL)
if err != nil {
log.Fatal(err)
}
htdocsDir = check(cfg, "htdocs", cfg.MergeEnv("htdocs", htdocsDir))
bleveNames = check(cfg, "bleve", cfg.MergeEnv("bleve", bleveNames))
templatesDir = check(cfg, "templates", cfg.MergeEnv("templates", templatesDir))
webhookPath = cfg.MergeEnv("webhook_path", webhookPath)
webhookSecret = cfg.MergeEnv("webhook_secret", webhookSecret)
webhookCommand = cfg.MergeEnv("webhook_command", webhookCommand)
templateName := path.Join(templatesDir, "advanced-search.html")
advancedPage, err = ioutil.ReadFile(templateName)
if err != nil {
log.Fatalf("Can't read templates, e.g. %s, %s", templateName, err)
}
templateName = path.Join(templatesDir, "basic-search.html")
basicPage, err = ioutil.ReadFile(templateName)
if err != nil {
log.Fatalf("Can't read %s, %s", templateName, err)
}
handleSignals()
// Wake up our search engine
indexList := strings.Split(bleveNames, ":")
availableIndex := false
if enableSearch == true {
for i := 0; i < len(indexList) && availableIndex == false; i++ {
indexName := indexList[i]
log.Printf("Opening %q", indexName)
index, err = bleve.OpenUsing(indexName, map[string]interface{}{
"read_only": true,
})
if err != nil {
log.Printf("Can't open Bleve index %q, %s, trying next index", indexName, err)
} else {
indexAlias = bleve.NewIndexAlias(index)
availableIndex = true
}
}
if availableIndex == false {
log.Fatalf("No index available %s", bleveNames)
}
defer index.Close()
}
// Send static file request to the default handler,
// search routes are handled by middleware customRoutes()
http.Handle("/", http.FileServer(http.Dir(htdocsDir)))
log.Printf("%s %s\n", appName, cait.Version)
log.Printf("Listening on %s\n", serviceURL.String())
err = http.ListenAndServe(serviceURL.Host, requestLogger(customRoutes(http.DefaultServeMux)))
if err != nil {
log.Fatal(err)
}
}
// OpenIndex opens an existing index, at the given path.
func OpenIndex(path string) (*Index, error) {
fi, err := os.Stat(path)
if err != nil {
return nil, fmt.Errorf("failed to access index at %s", path)
}
if !fi.IsDir() {
return nil, fmt.Errorf("index %s path is not a directory", path)
}
// Get the start time and end time.
startTime, err := time.Parse(indexNameLayout, fi.Name())
if err != nil {
return nil, fmt.Errorf("unable to determine start time of index: %s", err.Error())
}
var endTime time.Time
if f, err := os.Open(filepath.Join(path, endTimeFileName)); err != nil {
return nil, fmt.Errorf("unable to open end time file for index: %s", err.Error())
} else {
defer f.Close()
r := bufio.NewReader(f)
if s, err := r.ReadString('\n'); err != nil && err != io.EOF {
return nil, fmt.Errorf("unable to determine end time of index: %s", err.Error())
} else {
endTime, err = time.Parse(indexNameLayout, s)
if err != nil {
return nil, fmt.Errorf("unable to parse end time from '%s': %s", s, err.Error())
}
}
}
// Get an index directory listing.
d, err := os.Open(path)
if err != nil {
return nil, err
}
fis, err := d.Readdir(0)
if err != nil {
return nil, err
}
// Open the shards.
shards := make([]*Shard, 0)
for _, fi := range fis {
if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
continue
}
s := NewShard(filepath.Join(path, fi.Name()))
if err := s.Open(); err != nil {
return nil, fmt.Errorf("shard open fail: %s", err.Error())
}
shards = append(shards, s)
}
// Create alias for searching.
alias := bleve.NewIndexAlias()
for _, s := range shards {
alias.Add(s.b)
}
// Index is ready to go.
return &Index{
path: path,
Shards: shards,
Alias: alias,
startTime: startTime,
endTime: endTime,
}, nil
}
