func checkClient() {
// Create a client and connect to http://192.1.199.81:9200
// client, err := elastic.NewClient(elastic.SetURL("http://192.1.199.81:9200"))
client, err := elastic.NewClient(elastic.SetURL("http://127.0.0.1:9200"))
if err != nil {
// Handle error
panic(err)
}
// Ping the Elasticsearch server to get e.g. the version number
info, code, err := client.Ping().Do()
if err != nil {
// Handle error
panic(err)
}
fmt.Printf("Elasticsearch returned with code %d and version %s\n", code, info.Version.Number)
// Getting the ES version number
esversion, err := client.ElasticsearchVersion("http://127.0.0.1:9200")
if err != nil {
// Handle error
panic(err)
}
fmt.Printf("Elasticsearch version %s\n", esversion)
}
func runIndices(cmd *Command, args []string) {
var pattern = ""
if len(args) > 0 {
pattern = args[0]
}
// Get a client
client, err := elastic.NewClient(elastic.SetURL(esUrl))
if err != nil {
log.Fatal("%v", err)
}
indices, err := client.IndexNames()
if err != nil {
log.Fatal("%v", err)
}
// Sort by default
sort.Strings(indices)
for _, index := range indices {
if len(pattern) > 0 {
matched, err := regexp.MatchString(pattern, index)
if err != nil {
log.Fatal("invalid pattern")
}
if matched {
fmt.Println(index)
}
} else {
fmt.Println(index)
}
}
}
func main() {
// Parse arguments
kingpin.Version(Version)
kingpin.CommandLine.Help = "Search for logs in a Logstash Elasticsearch index."
kingpin.Parse()
errorLog = log.New(os.Stderr, "ERROR ", log.Ltime|log.Lshortfile)
if *verboseFlag || *debugFlag {
infoLog = log.New(os.Stderr, "INFO ", log.Ltime|log.Lshortfile)
}
if *debugFlag {
debugLog = log.New(os.Stderr, "TRACE ", log.Ltime|log.Lshortfile)
}
// Connect to the Elasticsearch cluster
logInfo("Creating client...\n")
client, err := elastic.NewClient(
elastic.SetURL((*urlFlag).String()),
elastic.SetSniff(false),
elastic.SetHealthcheck(false),
elastic.SetErrorLog(errorLog),
elastic.SetInfoLog(infoLog),
elastic.SetTraceLog(debugLog))
exitIfErr(err)
if *tailFlag {
Tail(client)
} else {
Search(client)
}
}
func (es *ElasticSearch) Conn() error {
client, err := elastic.NewClient(
elastic.SetURL(es.uri),
elastic.SetSniff(false),
elastic.SetHealthcheckInterval(10*time.Second),
elastic.SetMaxRetries(5))
if err != nil {
// Handle error
return err
}
exists, err := client.IndexExists(es.index).Do()
if err != nil {
// Handle error
return err
}
if !exists {
// Create a new index.
createIndex, err := client.CreateIndex(es.index).Do()
if err != nil {
// Handle error
return err
}
if !createIndex.Acknowledged {
// Not acknowledged
}
}
es.client = client
return nil
}
func elasticInsert() {
// Create a client and connect to http://127.0.0.1:9200
client, err := elastic.NewClient(elastic.SetURL("http://127.0.0.1:9200"))
if err != nil {
// Handle error
panic(err)
}
// Use the IndexExists service to check if a specified index exists.
exists, err := client.IndexExists("products").Do()
if err != nil {
// Handle error
panic(err)
}
if !exists {
// Create a new index.
createIndex, err := client.CreateIndex("products").Do()
if err != nil {
// Handle error
panic(err)
}
if !createIndex.Acknowledged {
// Not acknowledged
}
}
var product = "name"
// Index a product (using JSON serialization)
put1, err := client.Index().
Index("products").
Type("product").
Id("1").
BodyJson(product).
Do()
if err != nil {
// Handle error
panic(err)
}
fmt.Printf("Indexed product %s to index %s, type %s\n", put1.Id, put1.Index, put1.Type)
//Get product with specified ID
get1, err := client.Get().
Index("products").
Type("product").
Id("1").
Do()
if err != nil {
// Handle error
panic(err)
}
if get1.Found {
fmt.Printf("Got document %s in version %d from index %s, type %s\n", get1.Id, get1.Version, get1.Index, get1.Type)
}
}
func elasticDel() {
client, err := elastic.NewClient(elastic.SetURL("http://127.0.0.1:9200"))
// Delete the index again
_, err = client.DeleteIndex("products").Do()
if err != nil {
// Handle error
panic(err)
}
}
func ExampleClient_NewClient_cluster() {
// Obtain a client for an Elasticsearch cluster of two nodes,
// running on 10.0.1.1 and 10.0.1.2.
client, err := elastic.NewClient(elastic.SetURL("http://10.0.1.1:9200", "http://10.0.1.2:9200"))
if err != nil {
// Handle error
panic(err)
}
_ = client
}
// InitClient sets up the elastic client. If the client has already been
// initalized it is a noop
func (e LogstashElasticHosts) InitClient() error {
if lsClient == nil {
var err error
lsClient, err = elastic.NewClient(elastic.SetURL(e...), elastic.SetMaxRetries(10))
if err != nil {
return err
}
}
return nil
}
func elasticQuery() {
client, err := elastic.NewClient(elastic.SetURL("http://127.0.0.1:9200"))
// Flush to make sure the documents got written.
_, err = client.Flush().Index("products").Do()
if err != nil {
panic(err)
}
// Search with a term query
termQuery := elastic.NewTermQuery("name", "annual_fee")
searchResult, err := client.Search().
Index("products"). // search in index "products"
Query(&termQuery). // specify the query
Sort("name", true). // sort by "name" field, ascending
From(0).Size(10). // take documents 0-9
Pretty(true). // pretty print request and response JSON
Do() // execute
if err != nil {
// Handle error
panic(err)
}
// searchResult is of type SearchResult and returns hits, suggestions,
// and all kinds of other information from Elasticsearch.
fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)
// // TotalHits is another convenience function that works even when something goes wrong.
fmt.Printf("Found a total of %d products\n", searchResult.TotalHits())
// Here's how you iterate through results with full control over each step.
if searchResult.Hits != nil {
fmt.Printf("Found a total of %d products\n", searchResult.Hits.TotalHits)
// Iterate through results
for _, hit := range searchResult.Hits.Hits {
// hit.Index contains the name of the index
// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
var p models.Product
err := json.Unmarshal(*hit.Source, &p)
if err != nil {
// Deserialization failed
}
// Work with product
fmt.Printf("The new product is %s: %s\n", p.Name, p.Description)
}
} else {
// No hits
fmt.Print("Found no products\n")
}
}
func (shell *Shell) refreshClient() {
url := fmt.Sprint("http://", shell.prompt.Host, ":", strconv.Itoa(shell.prompt.Port), "/")
util.LogInfo(fmt.Sprint("Connecting to ", url, "..."))
client, err := elastic.NewClient(
elastic.SetURL(url),
)
if err == nil {
shell.client = client
} else {
util.LogError(err.Error())
}
}
func (f *Factory) New(uri *url.URL) bridge.RegistryAdapter {
urls := "http://127.0.0.1:9200"
if uri.Host != "" {
urls = "http://" + uri.Host
}
client, err := elasticapi.NewClient(elasticapi.SetURL(urls))
if err != nil {
log.Fatal("elastic: ", uri.Scheme)
}
return &ElasticAdapter{client: client}
}
func NewElasticHistory(host string, index string) *ElasticHistory {
var err error
history := new(ElasticHistory)
history.client, err = elastic.NewClient(elastic.SetURL(host))
history.incomingMessages = make(chan *Message)
history.index = strings.Replace(index, "#", "", -1)
if err != nil {
log.Panicf("Failed to connect to elasticsearch : %e", err)
}
go history.incomingLoop()
return history
}
func (a *Appbase) setupClient() error {
var err error
a.client, err = elastic.NewClient(
elastic.SetURL(a.uri.String()),
elastic.SetSniff(false),
)
if err != nil {
return err
}
a.bulkService = a.client.Bulk().Index(a.appName).Type(a.typename)
return nil
}
func CreateElasticSearchConfig(uri *url.URL) (*ElasticSearchConfig, error) {
var esConfig ElasticSearchConfig
opts, err := url.ParseQuery(uri.RawQuery)
if err != nil {
return nil, fmt.Errorf("failed to parser url's query string: %s", err)
}
// set the index for es,the default value is "heapster"
esConfig.Index = ESIndex
if len(opts["index"]) > 0 {
esConfig.Index = opts["index"][0]
}
// If the ES cluster needs authentication, the username and secret
// should be set in sink config.Else, set the Authenticate flag to false
esConfig.NeedAuthen = false
if len(opts["esUserName"]) > 0 && len(opts["esUserSecret"]) > 0 {
esConfig.EsUserName = opts["esUserName"][0]
esConfig.NeedAuthen = true
}
// set the URL endpoints of the ES's nodes. Notice that
// when sniffing is enabled, these URLs are used to initially sniff the
// cluster on startup.
if len(opts["nodes"]) < 1 {
return nil, fmt.Errorf("There is no node assigned for connecting ES cluster")
}
esConfig.EsNodes = append(esConfig.EsNodes, opts["nodes"]...)
glog.V(2).Infof("configing elasticsearch sink with ES's nodes - %v", esConfig.EsNodes)
var client *(elastic.Client)
if esConfig.NeedAuthen == false {
client, err = elastic.NewClient(elastic.SetURL(esConfig.EsNodes...))
} else {
client, err = elastic.NewClient(elastic.SetBasicAuth(esConfig.EsUserName, esConfig.EsUserSecret), elastic.SetURL(esConfig.EsNodes...))
}
if err != nil {
return nil, fmt.Errorf("failed to create ElasticSearch client: %v", err)
}
esConfig.EsClient = client
glog.V(2).Infof("elasticsearch sink configure successfully")
return &esConfig, nil
}
func processor(ic <-chan string) {
es, err := elastic.NewClient(elastic.SetURL("http://testbox02.chinacloudapp.cn:9200"), elastic.SetSniff(false))
if err != nil {
log.Println(err)
}
for item := range ic {
if item == "" {
break
}
var m media
if err = json.Unmarshal([]byte(item), &m); err == nil {
es_push(es, "fsmedia", "media", m)
}
}
if es != nil {
es.Flush().Do()
}
}
func TestCreateElasticSearchConfig(t *testing.T) {
url, err := url.Parse("?nodes=https://foo.com:20468&nodes=https://bar.com:20468&esUserName=test&esUserSecret=password&maxRetries=10&startupHealthcheckTimeout=30&sniff=false&healthCheck=false")
if err != nil {
t.Fatalf("Error when parsing URL: %s", err.Error())
}
config, err := CreateElasticSearchConfig(url)
if err != nil {
t.Fatalf("Error when creating config: %s", err.Error())
}
expectedClient, err := elastic.NewClient(
elastic.SetURL("https://foo.com:20468", "https://bar.com:20468"),
elastic.SetBasicAuth("test", "password"),
elastic.SetMaxRetries(10),
elastic.SetHealthcheckTimeoutStartup(30*time.Second),
elastic.SetSniff(false), elastic.SetHealthcheck(false))
if err != nil {
t.Fatalf("Error when creating client: %s", err.Error())
}
actualClientRefl := reflect.ValueOf(config.EsClient).Elem()
expectedClientRefl := reflect.ValueOf(expectedClient).Elem()
if actualClientRefl.FieldByName("basicAuthUsername").String() != expectedClientRefl.FieldByName("basicAuthUsername").String() {
t.Fatalf("basicAuthUsername is not equal")
}
if actualClientRefl.FieldByName("basicAuthUsername").String() != expectedClientRefl.FieldByName("basicAuthUsername").String() {
t.Fatalf("basicAuthUsername is not equal")
}
if actualClientRefl.FieldByName("maxRetries").Int() != expectedClientRefl.FieldByName("maxRetries").Int() {
t.Fatalf("maxRetries is not equal")
}
if actualClientRefl.FieldByName("healthcheckTimeoutStartup").Int() != expectedClientRefl.FieldByName("healthcheckTimeoutStartup").Int() {
t.Fatalf("healthcheckTimeoutStartup is not equal")
}
if actualClientRefl.FieldByName("snifferEnabled").Bool() != expectedClientRefl.FieldByName("snifferEnabled").Bool() {
t.Fatalf("snifferEnabled is not equal")
}
if actualClientRefl.FieldByName("healthcheckEnabled").Bool() != expectedClientRefl.FieldByName("healthcheckEnabled").Bool() {
t.Fatalf("healthcheckEnabled is not equal")
}
}
func ExampleClient_NewClient_manyOptions() {
// Obtain a client for an Elasticsearch cluster of two nodes,
// running on 10.0.1.1 and 10.0.1.2. Do not run the sniffer.
// Set the healthcheck interval to 10s. When requests fail,
// retry 5 times. Print error messages to os.Stderr and informational
// messages to os.Stdout.
client, err := elastic.NewClient(
elastic.SetURL("http://10.0.1.1:9200", "http://10.0.1.2:9200"),
elastic.SetSniff(false),
elastic.SetHealthcheckInterval(10*time.Second),
elastic.SetMaxRetries(5),
elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)),
elastic.SetInfoLog(log.New(os.Stdout, "", log.LstdFlags)))
if err != nil {
// Handle error
panic(err)
}
_ = client
}
/**
*音乐查询接口1a
*/
func MusicHandler(w http.ResponseWriter, r *http.Request) {
client, err := elastic.NewClient(elastic.SetURL(urles), elastic.SetMaxRetries(10))
if err != nil {
fmt.Fprint(w, "ES联接有问题", err)
} else {
jsdata := r.FormValue("jsondata")
var conds Conditions
err := json.Unmarshal([]byte(jsdata), &conds)
if err != nil {
fmt.Println("格式存在问题")
fmt.Fprint(w, "格式存在问题")
} else if len(conds.UserKey) <= 0 {
fmt.Fprint(w, "请出示你的Key")
} else {
fmt.Println(conds.ArtisName, conds.Special, conds.Name)
fmt.Fprint(w, getData(*client, conds))
}
}
}
func main() {
client, err := elastic.NewClient(elastic.SetURL("http://testbox02.chinacloudapp.cn:9200"), elastic.SetSniff(false))
drop_index(client, "fsmedia")
create_index(client, "fsmedia")
file, err := os.Open("e:/medias.json")
panic_error(err)
defer file.Close()
item_chan := make(chan string)
const pcount = 8
for i := 0; i < pcount; i++ {
go processor(item_chan)
}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
txt := scanner.Text()
item_chan <- txt
}
for i := 0; i < pcount; i++ {
item_chan <- ""
}
}
// CreateElasticSearchConfig creates an ElasticSearch configuration struct
// which contains an ElasticSearch client for later use
func CreateElasticSearchConfig(uri *url.URL) (*ElasticSearchConfig, error) {
var esConfig ElasticSearchConfig
opts, err := url.ParseQuery(uri.RawQuery)
if err != nil {
return nil, fmt.Errorf("failed to parser url's query string: %s", err)
}
// set the index for es,the default value is "heapster"
esConfig.Index = ESIndex
if len(opts["index"]) > 0 {
esConfig.Index = opts["index"][0]
}
// Set the URL endpoints of the ES's nodes. Notice that when sniffing is
// enabled, these URLs are used to initially sniff the cluster on startup.
if len(opts["nodes"]) < 1 {
return nil, fmt.Errorf("There is no node assigned for connecting ES cluster")
}
startupFns := []elastic.ClientOptionFunc{elastic.SetURL(opts["nodes"]...)}
// If the ES cluster needs authentication, the username and secret
// should be set in sink config.Else, set the Authenticate flag to false
if len(opts["esUserName"]) > 0 && len(opts["esUserSecret"]) > 0 {
startupFns = append(startupFns, elastic.SetBasicAuth(opts["esUserName"][0], opts["esUserSecret"][0]))
}
if len(opts["maxRetries"]) > 0 {
maxRetries, err := strconv.Atoi(opts["maxRetries"][0])
if err != nil {
return nil, fmt.Errorf("Failed to parse URL's maxRetries value into an int")
}
startupFns = append(startupFns, elastic.SetMaxRetries(maxRetries))
}
if len(opts["healthCheck"]) > 0 {
healthCheck, err := strconv.ParseBool(opts["healthCheck"][0])
if err != nil {
return nil, fmt.Errorf("Failed to parse URL's healthCheck value into a bool")
}
startupFns = append(startupFns, elastic.SetHealthcheck(healthCheck))
}
if len(opts["sniff"]) > 0 {
sniff, err := strconv.ParseBool(opts["sniff"][0])
if err != nil {
return nil, fmt.Errorf("Failed to parse URL's sniff value into a bool")
}
startupFns = append(startupFns, elastic.SetSniff(sniff))
}
if len(opts["startupHealthcheckTimeout"]) > 0 {
timeout, err := time.ParseDuration(opts["startupHealthcheckTimeout"][0] + "s")
if err != nil {
return nil, fmt.Errorf("Failed to parse URL's startupHealthcheckTimeout: %s", err.Error())
}
startupFns = append(startupFns, elastic.SetHealthcheckTimeoutStartup(timeout))
}
esConfig.EsClient, err = elastic.NewClient(startupFns...)
if err != nil {
return nil, fmt.Errorf("failed to create ElasticSearch client: %v", err)
}
glog.V(2).Infof("elasticsearch sink configure successfully")
return &esConfig, nil
}
Long: `This will take a single check result and create a key based upon the host name and
the check name. This key will remain consistent so that only the latest status will be available in
the index. This is designed to allow the creation of current dashboards from Kibana or Dashing.`,
Run: func(sensupluginses *cobra.Command, args []string) {
// read in the event data from the sensu server
sensuEvent := new(sensuhandler.SensuEvent)
sensuEvent = sensuEvent.AcquireSensuEvent()
// set the environment this is running in (prd, dev,stg)
sensuEnv = sensuEnv.SetSensuEnv()
// Create a client
client, err := elastic.NewClient(
elastic.SetURL("http://" + esHost + ":" + esPort),
)
if err != nil {
syslogLog.WithFields(logrus.Fields{
"check": "sensupluginses",
"client": host,
"version": version.AppVersion(),
"error": err,
"esHost": esHost,
"esPort": esPort,
}).Error(`Could not create an elasticsearch client`)
}
// Check to see if the index exists and if not create it
if client.IndexExists(esIndex) == nil { // need to test to make sure this does what I want
func runReindex(cmd *Command, args []string) {
if len(args) < 2 {
cmd.printUsage()
os.Exit(1)
}
sourceIndex := args[0]
targetIndex := args[1]
if sourceURL == "" {
sourceURL = esUrl
}
if targetURL == "" {
targetURL = esUrl
}
// Get a client
sourceClient, err := elastic.NewClient(elastic.SetURL(sourceURL))
if err != nil {
log.Fatalf("%v", err)
}
targetClient := sourceClient
if sourceURL != targetURL {
targetClient, err = elastic.NewClient(elastic.SetURL(targetURL))
if err != nil {
log.Fatalf("%v", err)
}
}
// Check if source index exists. Stop if it doesn't.
exists, err := sourceClient.IndexExists(sourceIndex).Do()
if err != nil {
log.Fatalf("%v", err)
}
if !exists {
os.Exit(0)
}
// Check if the target index exists. If it doesn't, create it with
// the number of shards/replicas specified.
exists, err = targetClient.IndexExists(targetIndex).Do()
if err != nil {
log.Fatalf("%v", err)
}
if !exists {
settings := make(map[string]interface{})
if shards > 0 || replicas >= 0 {
ixs := make(map[string]interface{})
if shards > 0 {
ixs["number_of_shards"] = shards
}
if replicas >= 0 {
ixs["number_of_replicas"] = replicas
}
settings["index"] = ixs
}
_, err = targetClient.CreateIndex(targetIndex).BodyJson(settings).Do()
if err != nil {
log.Fatalf("%v", err)
}
} else if shards > 0 || replicas >= 0 {
fmt.Fprint(os.Stderr, "Shards and/or replicas will not be changed on an existing index\n")
}
// Progress callback
progress := func(current, total int64) {
if verbose {
var percent int64
if total > 0 {
percent = 100 * current / total
}
fmt.Fprintf(os.Stderr, "Reindexing %9d of %9d (%3d%%)\r", current, total, percent)
}
}
// Use the Elastic Reindexer
ix := elastic.NewReindexer(sourceClient, sourceIndex, elastic.CopyToTargetIndex(targetIndex))
ix = ix.TargetClient(targetClient)
if bulkSize > 0 {
ix = ix.BulkSize(bulkSize)
}
ix = ix.Scroll("5m")
ix = ix.Progress(progress)
ix = ix.StatsOnly(true)
res, err := ix.Do()
if err != nil {
log.Fatalf("%v", err)
}
fmt.Fprintf(os.Stderr, "%d successful and %d failed request(s)\n", res.Success, res.Failed)
}
func (t *TestCase) setup() error {
var errorlogger *log.Logger
if t.errorlogfile != "" {
f, err := os.OpenFile(t.errorlogfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0664)
if err != nil {
return err
}
errorlogger = log.New(f, "", log.Ltime|log.Lmicroseconds|log.Lshortfile)
}
var infologger *log.Logger
if t.infologfile != "" {
f, err := os.OpenFile(t.infologfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0664)
if err != nil {
return err
}
infologger = log.New(f, "", log.LstdFlags)
}
// Trace request and response details like this
var tracelogger *log.Logger
if t.tracelogfile != "" {
f, err := os.OpenFile(t.tracelogfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0664)
if err != nil {
return err
}
tracelogger = log.New(f, "", log.LstdFlags)
}
client, err := elastic.NewClient(
elastic.SetURL(t.nodes...),
elastic.SetErrorLog(errorlogger),
elastic.SetInfoLog(infologger),
elastic.SetTraceLog(tracelogger),
elastic.SetMaxRetries(t.maxRetries),
elastic.SetSniff(t.sniff),
elastic.SetSnifferInterval(t.snifferInterval),
elastic.SetHealthcheck(t.healthcheck),
elastic.SetHealthcheckInterval(t.healthcheckInterval))
if err != nil {
// Handle error
return err
}
t.client = client
// Use the IndexExists service to check if a specified index exists.
exists, err := t.client.IndexExists(t.index).Do()
if err != nil {
return err
}
if exists {
deleteIndex, err := t.client.DeleteIndex(t.index).Do()
if err != nil {
return err
}
if !deleteIndex.Acknowledged {
return errors.New("delete index not acknowledged")
}
}
// Create a new index.
createIndex, err := t.client.CreateIndex(t.index).Do()
if err != nil {
return err
}
if !createIndex.Acknowledged {
return errors.New("create index not acknowledged")
}
// Index a tweet (using JSON serialization)
tweet1 := Tweet{User: "******", Message: "Take Five", Retweets: 0}
_, err = t.client.Index().
Index(t.index).
Type("tweet").
Id("1").
BodyJson(tweet1).
Do()
if err != nil {
return err
}
// Index a second tweet (by string)
tweet2 := `{"user" : "olivere", "message" : "It's a Raggy Waltz"}`
_, err = t.client.Index().
Index(t.index).
Type("tweet").
Id("2").
BodyString(tweet2).
Do()
if err != nil {
return err
}
// Flush to make sure the documents got written.
_, err = t.client.Flush().Index(t.index).Do()
if err != nil {
return err
}
return nil
}