예제 #1
0
func showReindexProgress(current, total int64) {
	percent := (float64(current) / float64(total)) * 100
	if int(percent) > reindexProgress {
		reindexProgress = int(percent)
		logger.Info("Reindexing... %d%% [Time elapsed: %s]", reindexProgress, time.Since(reindexStart).String())
	}
}
예제 #2
0
func showExportProgress(current, total int64) {
	percent := (float64(current) / float64(total)) * 100
	if int(percent) > exportProgress {
		exportProgress = int(percent)
		logger.Info("Exporting... %d%% [Time elapsed: %s]", exportProgress, time.Since(exportStart).String())
	}
}
예제 #3
0
func getESClient(esURL string) (*elastic.Client, error) {
	esClient, err := elastic.NewClient(
		elastic.SetURL(esURL),
		elastic.SetSniff(false),
		elastic.SetErrorLog(logger.DefaultLogger.Handlers[0].(*logger.DefaultHandler).ErrorLogger),
		elastic.SetInfoLog(logger.DefaultLogger.Handlers[0].(*logger.DefaultHandler).DebugLogger),
		elastic.SetTraceLog(logger.DefaultLogger.Handlers[0].(*logger.DefaultHandler).DebugLogger),
	)

	if err != nil {
		return esClient, err
	}

	esVersion, err := esClient.ElasticsearchVersion(esURL)
	if err != nil {
		logger.Fatal("Error getting ES version: %+v", err.Error())
	}
	logger.Info("Connected in Elasticsearch <%s>, version %s", esURL, esVersion)

	return esClient, err
}
예제 #4
0
func main() {
	flag.StringVar(&fromHost, "from-host", defaultElasticSearch, "elastic search host to get data from")
	flag.StringVar(&fromIndex, "index", "", "name of index to reindex/copy")
	flag.StringVar(&toHost, "to-host", defaultElasticSearch, "elastic search host to get data from")
	flag.StringVar(&toIndex, "new-index", "", "name of new-index")
	flag.StringVar(&newMapping, "new-mapping", "", "path to new mapping file of new-index")
	flag.IntVar(&bulkSize, "bulk-size", 500, "amount of data to get in each request")
	flag.BoolVar(&forceYes, "force-yes", false, "even if destination alias already exists continue reindexing")
	flag.BoolVar(&forceNo, "force-no", false, "if destination alias already exists abort reindexing")

	flag.Parse()

	if fromIndex == "" {
		logger.Error("The `-index` parameters are required to reindex/copy")

		flag.Usage()
		os.Exit(1)
	}

	// Read new mapping file
	if newMapping != "" {
		mappingBytes, err := ioutil.ReadFile(newMapping)
		if err != nil {
			logger.Fatal("Error reading mapping file: %+v", err.Error())
		}

		mappingContent = string(mappingBytes)
		logger.Debug("New mapping of %s:", newMapping)
		logger.Debug(mappingContent)
	}

	// Set default toIndex
	if toIndex == "" {
		toIndex = fromIndex + "-" + uuid.New()[24:]
	}

	// Connect to clients
	fromClient, err := getESClient(fromHost)
	if err != nil {
		logger.Fatal("Error connecting to `%s`: %+v", fromHost, err.Error())
	}

	toClient, err := getESClient(toHost)
	if err != nil {
		logger.Fatal("Error connecting to `%s`: %+v", toHost, err.Error())
	}

	// Verify if fromIndex exists
	exists, err := fromClient.IndexExists(fromIndex).Do()
	if err != nil {
		logger.Fatal("Error verifying if index <%s> exists: %+v", fromIndex, err.Error())
	}
	if !exists {
		logger.Fatal("The index <%s> doesn't exists, we need a valid index or alias", fromIndex)
	}

	// Verify if toIndex already exists
	exists, err = toClient.IndexExists(toIndex).Do()
	if err != nil {
		logger.Fatal("Error verifying if index <%s> exists: %+v", toIndex, err.Error())
	}

	// If toIndex don't exists we need create it
	if !exists {
		indexService := toClient.CreateIndex(toIndex)
		// If -new-mapping was not provided use original mapping
		if newMapping == "" {
			mapping, err := fromClient.GetMapping().Index(fromIndex).Do()
			if err != nil {
				logger.Fatal("Error getting mapping of index <%s>", fromIndex)
			}

			for _, v := range mapping {
				mapping = v.(map[string]interface{})
				break
			}

			resp, err := fromClient.IndexGetSettings().Index(fromIndex).Do()
			if err != nil {
				logger.Fatal("Error getting settings of index <%s>", fromIndex)
			}

			for _, v := range resp {
				mapping["settings"] = v.Settings
				break
			}

			indexService.BodyJson(mapping)
		} else {
			indexService.BodyString(mappingContent)
		}

		createNewIndex, err := indexService.Do()
		if err != nil {
			logger.Fatal("Error creating new index <%s>: %+v", toIndex, err.Error())
		}
		if !createNewIndex.Acknowledged {
			logger.Fatal("Was not possible create new index <%s>", toIndex)
		}

		logger.Info("New index <%s> was created!", toIndex)
	} else if forceNo {
		logger.Fatal("Index <%s> already exists in destination server", toIndex)
	} else if !forceYes {
		if !askForConfirmation(fmt.Sprintf("Index <%s> already exists, do you want index all documents without change the current mapping? (yes/no) ", toIndex)) {
			os.Exit(0)
		}
	}

	// Reindex fromIndex to toIndex
	reindexer := fromClient.Reindex(fromIndex, toIndex)
	reindexer.TargetClient(toClient)
	reindexer.Progress(showReindexProgress)

	if bulkSize > 0 {
		reindexer.BulkSize(bulkSize)
	}

	logger.Info("Starting reindexing...")
	reindexStart = time.Now()
	resp, err := reindexer.Do()
	if err != nil {
		logger.Fatal("Error trying reindexing: %+v", err.Error())
	}

	logger.Info("Reindexed was completed in <%s>, %d documents successed and %d failed", time.Since(reindexStart), resp.Success, resp.Failed)

	if len(resp.Errors) > 0 {
		logger.Warn("We get errors in some documents...")

		for _, respItem := range resp.Errors {
			logger.Error("Index[%s] Type[%s] Id[%s]: %s", respItem.Index, respItem.Type, respItem.Id, respItem.Error)
		}
	}

	// If index is a alias, update its reference
	aliasesService := toClient.Aliases()
	aliases, err := aliasesService.Do()
	if err != nil {
		logger.Fatal("Error getting aliases: %+v", err.Error())
	}

	indices := aliases.IndicesByAlias(fromIndex)
	if len(indices) > 0 {
		aliasService := toClient.Alias()
		for _, index := range indices {
			aliasService.Remove(index, fromIndex)
		}
		_, err = aliasService.Add(toIndex, fromIndex).Do()
		if err != nil {
			logger.Fatal("Error updating alias <%s>:  %+v", fromIndex, err.Error())
		}

		logger.Info("Alias <%s>: %+v was removed and now point to: <%s>", fromIndex, indices, toIndex)
	}
}
예제 #5
0
func main() {
	flag.StringVar(&host, "host", defaultElasticSearch, "elastic search host to get data from")
	flag.StringVar(&index, "index", "", "name of index to export")
	flag.StringVar(&indexType, "type", "", "name of type inside of <index> to export [optional]")
	flag.StringVar(&fieldlist, "fieldlist", "", "list of fields to export")
	flag.StringVar(&output, "output", "", "name of file to output")

	flag.Parse()

	if host == "" || index == "" || indexType == "" || fieldlist == "" || output == "" {
		logger.Error("Missing some parameters")

		flag.Usage()
		os.Exit(1)
	}

	// Connect to client
	client, err := getESClient(host)
	if err != nil {
		logger.Fatal("Error connecting to `%s`: %+v", host, err.Error())
	}

	// Verify if index exists
	exists, err := client.IndexExists(index).Do()
	if err != nil {
		logger.Fatal("Error verifying if index <%s> exists: %+v", index, err.Error())
	}
	if !exists {
		logger.Fatal("The index <%s> doesn't exists, we need a valid index or alias", index)
	}

	// Verify if type exists
	exists, err = client.TypeExists().Index(index).Type(indexType).Do()
	if err != nil {
		logger.Fatal("Error verifying if type <%s/%s> exists: %+v", index, indexType, err.Error())
	}
	if !exists {
		logger.Fatal("The type <%s/%s> doesn't exists, we need a valid type", index, indexType)
	}

	fields = strings.Split(fieldlist, ",")
	if len(fields) == 0 || fields[0] == "" {
		logger.Fatal("Fields informed is invalid")
	}

	file, err := os.Create(output)
	if err != nil {
		logger.Fatal("Cannot create output file[%s]: %s", output, err.Error())
	}

	csvWriter := csv.NewWriter(file)
	csvWriter.Comma = ';'

	// Export index/type to output
	exporter := NewExporter(client, index).Size(10).BulkSize(1000)
	if indexType != "" {
		exporter = exporter.Type(indexType)
	}
	exporter.Fields(fields...)
	exporter.Writer(csvWriter)
	exporter.Progress(showExportProgress)

	// Implement HERE your search quey
	query := elastic.NewMatchAllQuery()
	exporter.Query(query)

	logger.Info("Starting exporting to <%s>...", output)
	exportStart = time.Now()

	resp, err := exporter.Do()
	if err != nil {
		logger.Fatal("Error trying exporting: %+v", err.Error())
	}

	logger.Info("Exported was completed in <%s>, %d documents successed and %d failed", time.Since(exportStart), resp.Success, resp.Failed)

	if len(resp.Errors) > 0 {
		logger.Warn("We get errors in some documents...")

		for _, respItem := range resp.Errors {
			logger.Error("Index[%s] Type[%s] Id[%s]: %s", respItem.Index, respItem.Type, respItem.Id, respItem.Error)
		}
	}
}