예제 #1
0
// askForConfirmation uses Scanln to parse user input. A user must type in "yes" or "no" and
// then press enter. It has fuzzy matching, so "y", "Y", "yes", "YES", and "Yes" all count as
// confirmations. If the input is not recognized, it will ask again. The function does not return
// until it gets a valid response from the user. Typically, you should use fmt to print out a question
// before calling askForConfirmation. E.g. fmt.Println("WARNING: Are you sure? (yes/no)")
func askForConfirmation(message string) bool {
	fmt.Printf(message)

	var response string
	_, err := fmt.Scanln(&response)
	if err != nil {
		logger.Fatal("Cannot read from stdin: %s", err.Error())
	}

	yesResponses := []string{"y", "Y", "yes", "Yes", "YES"}
	noResponses := []string{"n", "N", "no", "No", "NO"}

	if containsString(yesResponses, response) {
		return true
	} else if containsString(noResponses, response) {
		return false
	} else {
		return askForConfirmation(message)
	}
}
예제 #2
0
func getESClient(esURL string) (*elastic.Client, error) {
	esClient, err := elastic.NewClient(
		elastic.SetURL(esURL),
		elastic.SetSniff(false),
		elastic.SetErrorLog(logger.DefaultLogger.Handlers[0].(*logger.DefaultHandler).ErrorLogger),
		elastic.SetInfoLog(logger.DefaultLogger.Handlers[0].(*logger.DefaultHandler).DebugLogger),
		elastic.SetTraceLog(logger.DefaultLogger.Handlers[0].(*logger.DefaultHandler).DebugLogger),
	)

	if err != nil {
		return esClient, err
	}

	esVersion, err := esClient.ElasticsearchVersion(esURL)
	if err != nil {
		logger.Fatal("Error getting ES version: %+v", err.Error())
	}
	logger.Info("Connected in Elasticsearch <%s>, version %s", esURL, esVersion)

	return esClient, err
}
예제 #3
0
func main() {
	flag.Parse()

	machineFile := flag.Arg(0)
	if machineFile == "" {
		logger.Fatal("You need to pass a machine definition file, type: %s <machine.yml>\n", os.Args[0])
	}

	machineContent, err := ioutil.ReadFile(machineFile)
	if err != nil {
		logger.Fatal("Error open machine file: %s", err.Error())
	}

	var machineConfig machine.Machine
	err = yaml.Unmarshal(machineContent, &machineConfig)
	if err != nil {
		logger.Fatal("Error reading machine file: %s", err.Error())
	}

	auth, err := AwsAuth()
	if err != nil {
		logger.Fatal("Error reading aws credentials: %s", err.Error())
	}

	if machineConfig.Instance.AvailableZone == "" {
		if machineConfig.Instance.DefaultAvailableZone == "" {
			logger.Fatal("Cannot create machine, instance.availablezone is missing")
		} else {
			machineConfig.Instance.AvailableZone = machineConfig.Instance.DefaultAvailableZone
		}
	}

	err = machine.Get(&machineConfig, auth)
	if err != nil {
		logger.Fatal("Error getting machine: %s", err.Error())
	}
}
예제 #4
0
func main() {
	flag.StringVar(&fromHost, "from-host", defaultElasticSearch, "elastic search host to get data from")
	flag.StringVar(&fromIndex, "index", "", "name of index to reindex/copy")
	flag.StringVar(&toHost, "to-host", defaultElasticSearch, "elastic search host to get data from")
	flag.StringVar(&toIndex, "new-index", "", "name of new-index")
	flag.StringVar(&newMapping, "new-mapping", "", "path to new mapping file of new-index")
	flag.IntVar(&bulkSize, "bulk-size", 500, "amount of data to get in each request")
	flag.BoolVar(&forceYes, "force-yes", false, "even if destination alias already exists continue reindexing")
	flag.BoolVar(&forceNo, "force-no", false, "if destination alias already exists abort reindexing")

	flag.Parse()

	if fromIndex == "" {
		logger.Error("The `-index` parameters are required to reindex/copy")

		flag.Usage()
		os.Exit(1)
	}

	// Read new mapping file
	if newMapping != "" {
		mappingBytes, err := ioutil.ReadFile(newMapping)
		if err != nil {
			logger.Fatal("Error reading mapping file: %+v", err.Error())
		}

		mappingContent = string(mappingBytes)
		logger.Debug("New mapping of %s:", newMapping)
		logger.Debug(mappingContent)
	}

	// Set default toIndex
	if toIndex == "" {
		toIndex = fromIndex + "-" + uuid.New()[24:]
	}

	// Connect to clients
	fromClient, err := getESClient(fromHost)
	if err != nil {
		logger.Fatal("Error connecting to `%s`: %+v", fromHost, err.Error())
	}

	toClient, err := getESClient(toHost)
	if err != nil {
		logger.Fatal("Error connecting to `%s`: %+v", toHost, err.Error())
	}

	// Verify if fromIndex exists
	exists, err := fromClient.IndexExists(fromIndex).Do()
	if err != nil {
		logger.Fatal("Error verifying if index <%s> exists: %+v", fromIndex, err.Error())
	}
	if !exists {
		logger.Fatal("The index <%s> doesn't exists, we need a valid index or alias", fromIndex)
	}

	// Verify if toIndex already exists
	exists, err = toClient.IndexExists(toIndex).Do()
	if err != nil {
		logger.Fatal("Error verifying if index <%s> exists: %+v", toIndex, err.Error())
	}

	// If toIndex don't exists we need create it
	if !exists {
		indexService := toClient.CreateIndex(toIndex)
		// If -new-mapping was not provided use original mapping
		if newMapping == "" {
			mapping, err := fromClient.GetMapping().Index(fromIndex).Do()
			if err != nil {
				logger.Fatal("Error getting mapping of index <%s>", fromIndex)
			}

			for _, v := range mapping {
				mapping = v.(map[string]interface{})
				break
			}

			resp, err := fromClient.IndexGetSettings().Index(fromIndex).Do()
			if err != nil {
				logger.Fatal("Error getting settings of index <%s>", fromIndex)
			}

			for _, v := range resp {
				mapping["settings"] = v.Settings
				break
			}

			indexService.BodyJson(mapping)
		} else {
			indexService.BodyString(mappingContent)
		}

		createNewIndex, err := indexService.Do()
		if err != nil {
			logger.Fatal("Error creating new index <%s>: %+v", toIndex, err.Error())
		}
		if !createNewIndex.Acknowledged {
			logger.Fatal("Was not possible create new index <%s>", toIndex)
		}

		logger.Info("New index <%s> was created!", toIndex)
	} else if forceNo {
		logger.Fatal("Index <%s> already exists in destination server", toIndex)
	} else if !forceYes {
		if !askForConfirmation(fmt.Sprintf("Index <%s> already exists, do you want index all documents without change the current mapping? (yes/no) ", toIndex)) {
			os.Exit(0)
		}
	}

	// Reindex fromIndex to toIndex
	reindexer := fromClient.Reindex(fromIndex, toIndex)
	reindexer.TargetClient(toClient)
	reindexer.Progress(showReindexProgress)

	if bulkSize > 0 {
		reindexer.BulkSize(bulkSize)
	}

	logger.Info("Starting reindexing...")
	reindexStart = time.Now()
	resp, err := reindexer.Do()
	if err != nil {
		logger.Fatal("Error trying reindexing: %+v", err.Error())
	}

	logger.Info("Reindexed was completed in <%s>, %d documents successed and %d failed", time.Since(reindexStart), resp.Success, resp.Failed)

	if len(resp.Errors) > 0 {
		logger.Warn("We get errors in some documents...")

		for _, respItem := range resp.Errors {
			logger.Error("Index[%s] Type[%s] Id[%s]: %s", respItem.Index, respItem.Type, respItem.Id, respItem.Error)
		}
	}

	// If index is a alias, update its reference
	aliasesService := toClient.Aliases()
	aliases, err := aliasesService.Do()
	if err != nil {
		logger.Fatal("Error getting aliases: %+v", err.Error())
	}

	indices := aliases.IndicesByAlias(fromIndex)
	if len(indices) > 0 {
		aliasService := toClient.Alias()
		for _, index := range indices {
			aliasService.Remove(index, fromIndex)
		}
		_, err = aliasService.Add(toIndex, fromIndex).Do()
		if err != nil {
			logger.Fatal("Error updating alias <%s>:  %+v", fromIndex, err.Error())
		}

		logger.Info("Alias <%s>: %+v was removed and now point to: <%s>", fromIndex, indices, toIndex)
	}
}
예제 #5
0
func main() {
	flag.StringVar(&host, "host", defaultElasticSearch, "elastic search host to get data from")
	flag.StringVar(&index, "index", "", "name of index to export")
	flag.StringVar(&indexType, "type", "", "name of type inside of <index> to export [optional]")
	flag.StringVar(&fieldlist, "fieldlist", "", "list of fields to export")
	flag.StringVar(&output, "output", "", "name of file to output")

	flag.Parse()

	if host == "" || index == "" || indexType == "" || fieldlist == "" || output == "" {
		logger.Error("Missing some parameters")

		flag.Usage()
		os.Exit(1)
	}

	// Connect to client
	client, err := getESClient(host)
	if err != nil {
		logger.Fatal("Error connecting to `%s`: %+v", host, err.Error())
	}

	// Verify if index exists
	exists, err := client.IndexExists(index).Do()
	if err != nil {
		logger.Fatal("Error verifying if index <%s> exists: %+v", index, err.Error())
	}
	if !exists {
		logger.Fatal("The index <%s> doesn't exists, we need a valid index or alias", index)
	}

	// Verify if type exists
	exists, err = client.TypeExists().Index(index).Type(indexType).Do()
	if err != nil {
		logger.Fatal("Error verifying if type <%s/%s> exists: %+v", index, indexType, err.Error())
	}
	if !exists {
		logger.Fatal("The type <%s/%s> doesn't exists, we need a valid type", index, indexType)
	}

	fields = strings.Split(fieldlist, ",")
	if len(fields) == 0 || fields[0] == "" {
		logger.Fatal("Fields informed is invalid")
	}

	file, err := os.Create(output)
	if err != nil {
		logger.Fatal("Cannot create output file[%s]: %s", output, err.Error())
	}

	csvWriter := csv.NewWriter(file)
	csvWriter.Comma = ';'

	// Export index/type to output
	exporter := NewExporter(client, index).Size(10).BulkSize(1000)
	if indexType != "" {
		exporter = exporter.Type(indexType)
	}
	exporter.Fields(fields...)
	exporter.Writer(csvWriter)
	exporter.Progress(showExportProgress)

	// Implement HERE your search quey
	query := elastic.NewMatchAllQuery()
	exporter.Query(query)

	logger.Info("Starting exporting to <%s>...", output)
	exportStart = time.Now()

	resp, err := exporter.Do()
	if err != nil {
		logger.Fatal("Error trying exporting: %+v", err.Error())
	}

	logger.Info("Exported was completed in <%s>, %d documents successed and %d failed", time.Since(exportStart), resp.Success, resp.Failed)

	if len(resp.Errors) > 0 {
		logger.Warn("We get errors in some documents...")

		for _, respItem := range resp.Errors {
			logger.Error("Index[%s] Type[%s] Id[%s]: %s", respItem.Index, respItem.Type, respItem.Id, respItem.Error)
		}
	}
}
예제 #6
0
func main() {
	flag.Parse()

	clusterFile := flag.Arg(0)
	if clusterFile == "" {
		logger.Fatal("You need to pass the cluster file, type: %s <cluster-file.yml>\n", os.Args[0])
	}

	clusterContent, err := ioutil.ReadFile(clusterFile)
	if err != nil {
		logger.Fatal("Error open cluster file: %s", err.Error())
	}

	var clusters Clusters
	err = yaml.Unmarshal(clusterContent, &clusters)
	if err != nil {
		logger.Fatal("Error reading cluster file: %s", err.Error())
	}

	if clusters.Default.AvailableZone == "" {
		if clusters.Default.DefaultAvailableZone != "" {
			clusters.Default.AvailableZone = clusters.Default.DefaultAvailableZone
		}
	}

	// First verify if I can open all machine files
	machines := make([]Cluster, len(clusters.Clusters))
	for key := range clusters.Clusters {
		clusterConfig := &clusters.Clusters[key]

		machineContent, err := ioutil.ReadFile(clusterConfig.Machine)
		if err != nil {
			logger.Fatal("Error open machine file: %s", err.Error())
		}

		var machineConfig machine.Machine
		err = yaml.Unmarshal(machineContent, &machineConfig)
		if err != nil {
			logger.Fatal("Error reading machine file: %s", err.Error())
		}

		// Verify if cloud-config file exists
		if machineConfig.Instance.CloudConfig != "" {
			_, err := os.Stat(machineConfig.Instance.CloudConfig)
			if err != nil {
				logger.Fatal("Error reading cloud-config: %s", err.Error())
			}
		}

		// Set default values of cluster to machine
		if machineConfig.Instance.ImageID == "" {
			machineConfig.Instance.ImageID = clusters.Default.ImageID
		}
		if machineConfig.Instance.Region == "" {
			machineConfig.Instance.Region = clusters.Default.Region
		}
		if machineConfig.Instance.KeyName == "" {
			machineConfig.Instance.KeyName = clusters.Default.KeyName
		}
		if len(machineConfig.Instance.SecurityGroups) == 0 {
			machineConfig.Instance.SecurityGroups = clusters.Default.SecurityGroups
		}
		if machineConfig.Instance.SubnetID == "" {
			machineConfig.Instance.SubnetID = clusters.Default.SubnetID
		}

		if machineConfig.Instance.AvailableZone == "" {
			if machineConfig.Instance.DefaultAvailableZone != "" {
				machineConfig.Instance.AvailableZone = machineConfig.Instance.DefaultAvailableZone
			} else {
				machineConfig.Instance.AvailableZone = clusters.Default.AvailableZone
			}
		}

		for _, tag := range clusters.Default.Tags {
			addTag := true
			for _, instanceTag := range machineConfig.Instance.Tags {
				if strings.EqualFold(instanceTag.Key, tag.Key) {
					addTag = false
				}
			}

			if addTag {
				machineConfig.Instance.Tags = append(machineConfig.Instance.Tags, tag)
			}

			addTag = true
			for k, volume := range machineConfig.Volumes {
				for _, volumeTag := range volume.Tags {
					if strings.EqualFold(volumeTag.Key, tag.Key) {
						addTag = false
					}

				}

				if addTag {
					machineConfig.Volumes[k].Tags = append(machineConfig.Volumes[k].Tags, tag)
				}
			}
		}

		machines[key] = Cluster{Machine: machineConfig, Nodes: clusterConfig.Nodes}
	}

	auth, err := AwsAuth()
	if err != nil {
		logger.Fatal("Error reading aws credentials: %s", err.Error())
	}

	machine.SetLogger(ioutil.Discard, "", 0)

	for key, clusterConfig := range machines {
		fmt.Printf("================ Running machines of %d. cluster ================\n", key+1)

		for i := 1; i <= clusterConfig.Nodes; i++ {
			machineConfig := clusterConfig.Machine
			machineConfig.Volumes = make([]volume.Volume, len(machineConfig.Volumes))

			// append machine number to name of instance
			machineConfig.Instance.Name += fmt.Sprintf("-%d", i)

			// append machine number to name of volume
			for key := range clusterConfig.Machine.Volumes {
				volumeRef := &clusterConfig.Machine.Volumes[key]

				volumeConfig := *volumeRef
				volumeConfig.Name += fmt.Sprintf("-%d", i)
				machineConfig.Volumes[key] = volumeConfig
			}

			fmt.Printf("Running machine: %s\n", machineConfig.Instance.Name)
			err = machine.Get(&machineConfig, auth)
			if err != nil {
				logger.Fatal("Error getting machine: %s", err.Error())
			}

			fmt.Printf("Machine Id <%s>, IP Address <%s>\n", machineConfig.Instance.ID, machineConfig.Instance.PrivateIPAddress)
			if i < clusterConfig.Nodes {
				fmt.Println("----------------------------------")
			}
		}
	}
	fmt.Println("================================================================")
}