func showReindexProgress(current, total int64) {
percent := (float64(current) / float64(total)) * 100
if int(percent) > reindexProgress {
reindexProgress = int(percent)
logger.Info("Reindexing... %d%% [Time elapsed: %s]", reindexProgress, time.Since(reindexStart).String())
}
}
func showExportProgress(current, total int64) {
percent := (float64(current) / float64(total)) * 100
if int(percent) > exportProgress {
exportProgress = int(percent)
logger.Info("Exporting... %d%% [Time elapsed: %s]", exportProgress, time.Since(exportStart).String())
}
}
func getESClient(esURL string) (*elastic.Client, error) {
esClient, err := elastic.NewClient(
elastic.SetURL(esURL),
elastic.SetSniff(false),
elastic.SetErrorLog(logger.DefaultLogger.Handlers[0].(*logger.DefaultHandler).ErrorLogger),
elastic.SetInfoLog(logger.DefaultLogger.Handlers[0].(*logger.DefaultHandler).DebugLogger),
elastic.SetTraceLog(logger.DefaultLogger.Handlers[0].(*logger.DefaultHandler).DebugLogger),
)
if err != nil {
return esClient, err
}
esVersion, err := esClient.ElasticsearchVersion(esURL)
if err != nil {
logger.Fatal("Error getting ES version: %+v", err.Error())
}
logger.Info("Connected in Elasticsearch <%s>, version %s", esURL, esVersion)
return esClient, err
}
func main() {
flag.StringVar(&fromHost, "from-host", defaultElasticSearch, "elastic search host to get data from")
flag.StringVar(&fromIndex, "index", "", "name of index to reindex/copy")
flag.StringVar(&toHost, "to-host", defaultElasticSearch, "elastic search host to get data from")
flag.StringVar(&toIndex, "new-index", "", "name of new-index")
flag.StringVar(&newMapping, "new-mapping", "", "path to new mapping file of new-index")
flag.IntVar(&bulkSize, "bulk-size", 500, "amount of data to get in each request")
flag.BoolVar(&forceYes, "force-yes", false, "even if destination alias already exists continue reindexing")
flag.BoolVar(&forceNo, "force-no", false, "if destination alias already exists abort reindexing")
flag.Parse()
if fromIndex == "" {
logger.Error("The `-index` parameters are required to reindex/copy")
flag.Usage()
os.Exit(1)
}
// Read new mapping file
if newMapping != "" {
mappingBytes, err := ioutil.ReadFile(newMapping)
if err != nil {
logger.Fatal("Error reading mapping file: %+v", err.Error())
}
mappingContent = string(mappingBytes)
logger.Debug("New mapping of %s:", newMapping)
logger.Debug(mappingContent)
}
// Set default toIndex
if toIndex == "" {
toIndex = fromIndex + "-" + uuid.New()[24:]
}
// Connect to clients
fromClient, err := getESClient(fromHost)
if err != nil {
logger.Fatal("Error connecting to `%s`: %+v", fromHost, err.Error())
}
toClient, err := getESClient(toHost)
if err != nil {
logger.Fatal("Error connecting to `%s`: %+v", toHost, err.Error())
}
// Verify if fromIndex exists
exists, err := fromClient.IndexExists(fromIndex).Do()
if err != nil {
logger.Fatal("Error verifying if index <%s> exists: %+v", fromIndex, err.Error())
}
if !exists {
logger.Fatal("The index <%s> doesn't exists, we need a valid index or alias", fromIndex)
}
// Verify if toIndex already exists
exists, err = toClient.IndexExists(toIndex).Do()
if err != nil {
logger.Fatal("Error verifying if index <%s> exists: %+v", toIndex, err.Error())
}
// If toIndex don't exists we need create it
if !exists {
indexService := toClient.CreateIndex(toIndex)
// If -new-mapping was not provided use original mapping
if newMapping == "" {
mapping, err := fromClient.GetMapping().Index(fromIndex).Do()
if err != nil {
logger.Fatal("Error getting mapping of index <%s>", fromIndex)
}
for _, v := range mapping {
mapping = v.(map[string]interface{})
break
}
resp, err := fromClient.IndexGetSettings().Index(fromIndex).Do()
if err != nil {
logger.Fatal("Error getting settings of index <%s>", fromIndex)
}
for _, v := range resp {
mapping["settings"] = v.Settings
break
}
indexService.BodyJson(mapping)
} else {
indexService.BodyString(mappingContent)
}
createNewIndex, err := indexService.Do()
if err != nil {
logger.Fatal("Error creating new index <%s>: %+v", toIndex, err.Error())
}
if !createNewIndex.Acknowledged {
logger.Fatal("Was not possible create new index <%s>", toIndex)
}
logger.Info("New index <%s> was created!", toIndex)
} else if forceNo {
logger.Fatal("Index <%s> already exists in destination server", toIndex)
} else if !forceYes {
if !askForConfirmation(fmt.Sprintf("Index <%s> already exists, do you want index all documents without change the current mapping? (yes/no) ", toIndex)) {
os.Exit(0)
}
}
// Reindex fromIndex to toIndex
reindexer := fromClient.Reindex(fromIndex, toIndex)
reindexer.TargetClient(toClient)
reindexer.Progress(showReindexProgress)
if bulkSize > 0 {
reindexer.BulkSize(bulkSize)
}
logger.Info("Starting reindexing...")
reindexStart = time.Now()
resp, err := reindexer.Do()
if err != nil {
logger.Fatal("Error trying reindexing: %+v", err.Error())
}
logger.Info("Reindexed was completed in <%s>, %d documents successed and %d failed", time.Since(reindexStart), resp.Success, resp.Failed)
if len(resp.Errors) > 0 {
logger.Warn("We get errors in some documents...")
for _, respItem := range resp.Errors {
logger.Error("Index[%s] Type[%s] Id[%s]: %s", respItem.Index, respItem.Type, respItem.Id, respItem.Error)
}
}
// If index is a alias, update its reference
aliasesService := toClient.Aliases()
aliases, err := aliasesService.Do()
if err != nil {
logger.Fatal("Error getting aliases: %+v", err.Error())
}
indices := aliases.IndicesByAlias(fromIndex)
if len(indices) > 0 {
aliasService := toClient.Alias()
for _, index := range indices {
aliasService.Remove(index, fromIndex)
}
_, err = aliasService.Add(toIndex, fromIndex).Do()
if err != nil {
logger.Fatal("Error updating alias <%s>: %+v", fromIndex, err.Error())
}
logger.Info("Alias <%s>: %+v was removed and now point to: <%s>", fromIndex, indices, toIndex)
}
}
func main() {
flag.StringVar(&host, "host", defaultElasticSearch, "elastic search host to get data from")
flag.StringVar(&index, "index", "", "name of index to export")
flag.StringVar(&indexType, "type", "", "name of type inside of <index> to export [optional]")
flag.StringVar(&fieldlist, "fieldlist", "", "list of fields to export")
flag.StringVar(&output, "output", "", "name of file to output")
flag.Parse()
if host == "" || index == "" || indexType == "" || fieldlist == "" || output == "" {
logger.Error("Missing some parameters")
flag.Usage()
os.Exit(1)
}
// Connect to client
client, err := getESClient(host)
if err != nil {
logger.Fatal("Error connecting to `%s`: %+v", host, err.Error())
}
// Verify if index exists
exists, err := client.IndexExists(index).Do()
if err != nil {
logger.Fatal("Error verifying if index <%s> exists: %+v", index, err.Error())
}
if !exists {
logger.Fatal("The index <%s> doesn't exists, we need a valid index or alias", index)
}
// Verify if type exists
exists, err = client.TypeExists().Index(index).Type(indexType).Do()
if err != nil {
logger.Fatal("Error verifying if type <%s/%s> exists: %+v", index, indexType, err.Error())
}
if !exists {
logger.Fatal("The type <%s/%s> doesn't exists, we need a valid type", index, indexType)
}
fields = strings.Split(fieldlist, ",")
if len(fields) == 0 || fields[0] == "" {
logger.Fatal("Fields informed is invalid")
}
file, err := os.Create(output)
if err != nil {
logger.Fatal("Cannot create output file[%s]: %s", output, err.Error())
}
csvWriter := csv.NewWriter(file)
csvWriter.Comma = ';'
// Export index/type to output
exporter := NewExporter(client, index).Size(10).BulkSize(1000)
if indexType != "" {
exporter = exporter.Type(indexType)
}
exporter.Fields(fields...)
exporter.Writer(csvWriter)
exporter.Progress(showExportProgress)
// Implement HERE your search quey
query := elastic.NewMatchAllQuery()
exporter.Query(query)
logger.Info("Starting exporting to <%s>...", output)
exportStart = time.Now()
resp, err := exporter.Do()
if err != nil {
logger.Fatal("Error trying exporting: %+v", err.Error())
}
logger.Info("Exported was completed in <%s>, %d documents successed and %d failed", time.Since(exportStart), resp.Success, resp.Failed)
if len(resp.Errors) > 0 {
logger.Warn("We get errors in some documents...")
for _, respItem := range resp.Errors {
logger.Error("Index[%s] Type[%s] Id[%s]: %s", respItem.Index, respItem.Type, respItem.Id, respItem.Error)
}
}
}