コード例 #1
0
ファイル: state.go プロジェクト: ameihm0912/geomodel
func (e *esStateService) readObject(objid string) (o *object, err error) {
	conn := elastigo.NewConn()
	defer conn.Close()
	conn.Domain = e.stateDomain

	template := `{
		"query": {
			"term": {
				"object_id": "%v"
			}
		}
	}`
	tempbuf := fmt.Sprintf(template, objid)
	res, err := conn.Search(e.stateIndex, "geomodel_state", nil, tempbuf)
	if err != nil {
		return o, err
	}
	if res.Hits.Len() == 0 {
		return nil, nil
	}
	if res.Hits.Len() > 1 {
		return nil, fmt.Errorf("consistency failure, more than one object matched")
	}
	o = &object{}
	err = json.Unmarshal(*res.Hits.Hits[0].Source, o)
	if err != nil {
		return nil, err
	}

	return o, nil
}
コード例 #2
0
ファイル: elastic.go プロジェクト: sdgoij/TheDistributedBay
// Initializes a new elastic search handler.
func NewElastic(host string, index string) (*Elastic, error) {
	c := elastigo.NewConn()
	c.Domain = host
	b := c.NewBulkIndexer(10)
	b.Start()
	return &Elastic{c, b, index}, nil
}
コード例 #3
0
func InitElasticsearch() error {
	es = elastigo.NewConn()
	es.Domain = setting.Config.ElasticsearchDomain // needs to be configurable obviously
	es.Port = strconv.Itoa(setting.Config.ElasticsearchPort)
	if setting.Config.ElasticsearchUser != "" && setting.Config.ElasticsearchPasswd != "" {
		es.Username = setting.Config.ElasticsearchUser
		es.Password = setting.Config.ElasticsearchPasswd
	}
	if exists, err := es.ExistsIndex("definitions", "metric", nil); err != nil && err.Error() != "record not found" {
		return err
	} else {
		if !exists {
			_, err = es.CreateIndex("definitions")
			if err != nil {
				return err
			}
		}
		esopts := elastigo.MappingOptions{}

		err = es.PutMapping("definitions", "metric", MetricDefinition{}, esopts)
		if err != nil {
			return err
		}
	}

	return nil
}
コード例 #4
0
ファイル: configuration.go プロジェクト: AlexsJones/crashmat
func (c *Configuration) StartElasticSearch() {

	address := c.Json.Elastic.HostAddress

	if !c.Json.RunLocalMode {
		if os.Getenv("CRASHMAT_ELASTICHOSTADDRESS") != "" {
			log.Println("Using environmental for CRASHMAT_ELASTICHOSTADDRESS")
			address = os.Getenv("CRASHMAT_ELASTICHOSTADDRESS")
		}
	}
	if c.Json.Elastic.DropAllOnStartUp {
		log.Println("Purging => ", address+"crashmat")
		out, err := exec.Command("curl", "-XDELETE", address+"crashmat").Output()
		if err != nil {
			log.Println(err)
		}
		fmt.Printf("%s", out)

		log.Println("Creating => ", address+"crashmat")
		out, err = exec.Command("curl", "-XPUT", address+"crashmat").Output()
		if err != nil {
			log.Println(err)
		}
	}

	elasticConnection := elastigo.NewConn()

	elasticConnection.SetFromUrl(address)

	ElasticConnection = elasticConnection
	c.ElasticConnection = elasticConnection
}
コード例 #5
0
// NewServer returns a new instance of Server built from a config.
func NewServer(c *Config, buildInfo *BuildInfo) (*Server, error) {

	elasticsearch := elastigo.NewConn()
	elasticsearch.SetFromUrl(c.Meta.ElasticSearchUrl)

	db := gocql.NewCluster(c.Database.CassandraNodes...)
	db.Keyspace = c.Database.CassandraKeyspace
	db.NumConns = c.Database.CassandraConns
	db.Discovery = gocql.DiscoveryConfig{
		DcFilter:   "",
		RackFilter: "",
		Sleep:      30 * time.Second,
	}

	s := &Server{
		buildInfo: *buildInfo,
		err:       make(chan error),
		closing:   make(chan struct{}),

		Hostname:      c.Meta.Hostname,
		BindAddress:   c.Meta.BindAddress,
		elasticsearch: elasticsearch,
		cassandra:     db,
	}

	// Append services.
	//s.appendMongoService(c.Mongo)

	s.appendMetricsReportingService(c.Meta)
	s.appendHTTPDService(c.HTTPD)
	s.appendRegistrationService(c.Registration, c.RegMeta)
	return s, nil
}
コード例 #6
0
ファイル: elasticstore.go プロジェクト: reduxdj/grafana
func Init() {
	host := setting.Cfg.Section("elasticsearch").Key("host").MustString("localhost")
	port := setting.Cfg.Section("elasticsearch").Key("port").MustInt64(9200)
	es = elastigo.NewConn()
	es.Domain = host
	es.Port = strconv.FormatInt(port, 10)
}
コード例 #7
0
ファイル: start_1.go プロジェクト: jacqui/elastigo
func main() {
	c := elastigo.NewConn()
	log.SetFlags(log.LstdFlags)
	flag.Parse()

	fmt.Println("host = ", *host)
	// Set the Elasticsearch Host to Connect to
	c.Domain = *host

	// Index a document
	_, err := c.Index("testindex", "user", "docid_1", nil, `{"name":"bob"}`)
	exitIfErr(err)

	// Index a doc using a map of values
	_, err = c.Index("testindex", "user", "docid_2", nil, map[string]string{"name": "venkatesh"})
	exitIfErr(err)

	// Index a doc using Structs
	_, err = c.Index("testindex", "user", "docid_3", nil, MyUser{"wanda", 22})
	exitIfErr(err)

	// Search Using Raw json String
	searchJson := `{
	    "query" : {
	        "term" : { "Name" : "wanda" }
	    }
	}`
	out, err := c.Search("testindex", "user", nil, searchJson)
	if len(out.Hits.Hits) == 1 {
		fmt.Println("%v", out.Hits.Hits[0].Source)
	}
	exitIfErr(err)

}
コード例 #8
0
ファイル: russhes.go プロジェクト: CodepopeWorking/RuSShes
func main() {
	kingpin.Parse()

	r := Rss2{}

	response, err := http.DefaultClient.Get(*feedurl)
	if err != nil {
		log.Fatal(err)
	}

	xmlContent, err := ioutil.ReadAll(response.Body)
	err = xml.Unmarshal(xmlContent, &r)
	if err != nil {
		log.Fatal(err)
	}

	client := elastigo.NewConn()

	if *esurl != "" {
		client.SetFromUrl(*esurl)
	}

	for _, item := range r.ItemList {
		jsonValue, _ := json.MarshalIndent(item, "", "    ")
		if *esurl != "" {
			client.Index(*esindex, *estype, item.GUID, nil, jsonValue)
		} else {
			fmt.Println(string(jsonValue))
		}
	}
}
コード例 #9
0
// The simplest usage of background bulk indexing
func ExampleBulkIndexer_simple() {
	c := elastigo.NewConn()

	indexer := c.NewBulkIndexerErrors(10, 60)
	indexer.Start()
	indexer.Index("twitter", "user", "1", "", "", nil, `{"name":"bob"}`, true)
	indexer.Stop()
}
コード例 #10
0
ファイル: elasticsearch.go プロジェクト: razorinc/skydive
func GetInstance(addr string, port int) *ElasticSearchStorage {
	c := elastigo.NewConn()
	c.Domain = addr
	c.Port = strconv.FormatInt(int64(port), 10)

	storage := &ElasticSearchStorage{connection: c}
	return storage
}
コード例 #11
0
// connects to an NSQ topic and emits each message into streamtools.
func (b *ToElasticsearch) Run() {
	var err error
	var esIndex string
	var esType string

	conn := elastigo.NewConn()

	host := "localhost"
	port := "9200"

	for {
		select {
		case ruleI := <-b.inrule:
			host, err = util.ParseString(ruleI, "Host")
			if err != nil {
				b.Error(err)
				break
			}

			port, err = util.ParseString(ruleI, "Port")
			if err != nil {
				b.Error(err)
				break
			}

			esIndex, err = util.ParseString(ruleI, "Index")
			if err != nil {
				b.Error(err)
				break
			}

			esType, err = util.ParseString(ruleI, "Type")
			if err != nil {
				b.Error(err)
				break
			}

			conn.Domain = host
			conn.Port = port

		case msg := <-b.in:
			_, err := conn.Index(esIndex, esType, "", nil, msg)
			if err != nil {
				b.Error(err)
			}
		case <-b.quit:
			return
		case c := <-b.queryrule:
			c <- map[string]interface{}{
				"Host":  host,
				"Port":  port,
				"Index": esIndex,
				"Type":  esType,
			}
		}
	}
}
コード例 #12
0
ファイル: bloomdb.go プロジェクト: untoldone/bloomapi
func (bdb *BloomDatabase) SearchConnection() *elastigo.Conn {
	if bdb.sharedSearch == nil {
		conn := elastigo.NewConn()
		conn.SetHosts(bdb.searchHosts)
		bdb.sharedSearch = conn
	}

	return bdb.sharedSearch
}
コード例 #13
0
ファイル: state.go プロジェクト: ameihm0912/geomodel
func (e *esStateService) writeObject(o object) (err error) {
	conn := elastigo.NewConn()
	defer conn.Close()
	conn.Domain = e.stateDomain
	_, err = conn.Index(e.stateIndex, "geomodel_state", o.ObjectID, nil, o)
	if err != nil {
		return err
	}
	return nil
}
コード例 #14
0
ファイル: eventdef.go プロジェクト: splaspood/raintank-metric
func InitElasticsearch() error {
	es = elastigo.NewConn()
	es.Domain = setting.Config.ElasticsearchDomain // needs to be configurable obviously
	es.Port = strconv.Itoa(setting.Config.ElasticsearchPort)
	if setting.Config.ElasticsearchUser != "" && setting.Config.ElasticsearchPasswd != "" {
		es.Username = setting.Config.ElasticsearchUser
		es.Password = setting.Config.ElasticsearchPasswd
	}

	return nil
}
コード例 #15
0
ファイル: ess.go プロジェクト: vindalu/vindalu
/*
   Create the index if it does not exist.
   Optionally apply a mapping if mapping file is supplied.
*/
func NewExtendedEssConn(esshost string, essport int) *ExtendedEssConn {

	ed := ExtendedEssConn{
		Conn: elastigo.NewConn(),
	}

	ed.Domain = esshost
	ed.Port = fmt.Sprintf("%d", essport)

	return &ed
}
コード例 #16
0
func main() {
	c := elastigo.NewConn()
	log.SetFlags(log.LstdFlags)
	flag.Parse()

	// Trace all requests
	c.RequestTracer = func(method, url, body string) {
		log.Printf("Requesting %s %s", method, url)
		log.Printf("Request body: %s", body)
	}

	fmt.Println("host = ", *host)
	// Set the Elasticsearch Host to Connect to
	c.Domain = *host

	// Index a document
	_, err := c.Index("oldindex", "product", "docid_1", nil, `{"name":"bob"}`)
	exitIfErr(err)

	// Index a doc using a map of values
	_, err = c.Index("oldindex", "product", "docid_2", nil, map[string]string{"name": "venkatesh"})
	exitIfErr(err)

	// Index a doc using Structs
	_, err = c.Index("testindex", "user", "docid_3", nil, map[string]string{"name": "wena"})
	exitIfErr(err)

	// "name":"peanuts","description":"Honey Roasted peanuts","permalink":"",
	//             "tax_category_id":0,"shipping_category_id":0,"deleted_at":"0001-01-01T00:00:00Z","meta_description":"",
	//             "meta_keywords":"","position":0,"is_featured":false,"can_discount":false,"distributor_only_membership":false

	// product := models.Product{ 59001, "peanuts", "Honey Roasted peanuts", "Good taste food"}
	//
	//
	// // Index a doc using Structs
	// _, err = c.Index("pdtindex", "product", "producid_3", nil, product)
	//     fmt.Println(product)

	//exitIfErr(err)

	// Search Using Raw json String
	searchJson := `{
	    "query" : {
	        "term" : { "Name" : "venkatesh" }
	    }
	}`
	out, err := c.Search("oldindex", "product", nil, searchJson)

	if len(out.Hits.Hits) == 1 {
		fmt.Println("%v", out.Hits.Hits[0].Source)
	}
	exitIfErr(err)

}
コード例 #17
0
ファイル: es.go プロジェクト: sundy-li/dbstattest
func NewESClient() DbClient {
	conn := elastigo.NewConn()
	host := eshost1
	conn.Domain = host

	batchconn := goes.NewConnection(host, "9200")
	return &ESClient{
		conn:      conn,
		batchconn: batchconn,
	}
}
コード例 #18
0
func (elasticSearchClient *ElasticSearchClient) GetConnection() *elastigo.Conn {
	if elasticSearchClient.connection != nil {
		return elasticSearchClient.connection
	} else {
		c := elastigo.NewConn()
		c.SetHosts(elasticSearchClient.host)
		c.SetPort(strconv.Itoa(elasticSearchClient.port))

		elasticSearchClient.connection = c
		return elasticSearchClient.connection
	}
}
コード例 #19
0
ファイル: slack.go プロジェクト: asim/archiver
// Consumes messages from NSQ and publishes to ElasticSearch
func NSQToES() error {
	nc, err := nsq.NewConsumer(Topic, Channel, nsq.NewConfig())
	if err != nil {
		return err
	}
	tn := &nsqToES{
		conn: el.NewConn(),
	}
	tn.conn.Domain = ESAddress

	nc.AddConcurrentHandlers(tn, NumHandlers)
	return nc.ConnectToNSQD(NSQAddress)
}
コード例 #20
0
ファイル: elasticsearch.go プロジェクト: pombredanne/gollum-1
// Configure initializes this producer with values from a plugin config.
func (prod *ElasticSearch) Configure(conf core.PluginConfig) error {
	err := prod.ProducerBase.Configure(conf)
	if err != nil {
		return err
	}

	prod.SetStopCallback(prod.close)

	defaultServer := []string{"localhost"}
	numConnections := conf.GetInt("Connections", 6)
	retrySec := conf.GetInt("RetrySec", 5)

	prod.conn = elastigo.NewConn()
	prod.conn.Hosts = conf.GetStringArray("Servers", defaultServer)
	prod.conn.Domain = conf.GetString("Domain", prod.conn.Hosts[0])
	prod.conn.ClusterDomains = prod.conn.Hosts
	prod.conn.Port = strconv.Itoa(conf.GetInt("Port", 9200))
	prod.conn.Username = conf.GetString("User", "")
	prod.conn.Password = conf.GetString("Password", "")

	prod.indexer = prod.conn.NewBulkIndexerErrors(numConnections, retrySec)
	prod.indexer.BufferDelayMax = time.Duration(conf.GetInt("BatchTimeoutSec", 5)) * time.Second
	prod.indexer.BulkMaxBuffer = conf.GetInt("BatchSizeByte", 32768)
	prod.indexer.BulkMaxDocs = conf.GetInt("BatchMaxCount", 128)

	prod.indexer.Sender = func(buf *bytes.Buffer) error {
		_, err := prod.conn.DoCommand("POST", "/_bulk", nil, buf)
		if err != nil {
			Log.Error.Print("ElasticSearch response error - ", err)
		}
		return err
	}

	prod.index = conf.GetStreamMap("Index", "")
	prod.msgType = conf.GetStreamMap("Type", "log")
	prod.msgTTL = conf.GetString("TTL", "")
	prod.dayBasedIndex = conf.GetBool("DayBasedIndex", false)

	prod.counters = make(map[string]*int64)
	prod.lastMetricUpdate = time.Now()

	for _, index := range prod.index {
		shared.Metric.New(elasticMetricMessages + index)
		shared.Metric.New(elasticMetricMessagesSec + index)
		prod.counters[index] = new(int64)
	}

	prod.SetCheckFuseCallback(prod.isClusterUp)
	return nil
}
コード例 #21
0
ファイル: elasticsearch.go プロジェクト: fdebonneval/skydive
func New() (*ElasticSearchStorage, error) {
	c := elastigo.NewConn()

	elasticonfig := strings.Split(config.GetConfig().GetString("storage.elasticsearch"), ":")
	if len(elasticonfig) != 2 {
		return nil, ErrBadConfig
	}
	c.Domain = elasticonfig[0]
	c.Port = elasticonfig[1]

	storage := &ElasticSearchStorage{connection: c}
	storage.started.Store(false)

	return storage, nil
}
コード例 #22
0
ファイル: eventdef.go プロジェクト: aglagla/raintank-metric
func InitElasticsearch(addr, user, pass string) error {
	es = elastigo.NewConn()
	parts := strings.Split(addr, ":")
	if len(parts) != 2 {
		return fmt.Errorf("invalid tcp addr %q", addr)
	}
	es.Domain = parts[0]
	es.Port = parts[1]
	if user != "" && pass != "" {
		es.Username = user
		es.Password = pass
	}

	return nil
}
コード例 #23
0
ファイル: esswrapper.go プロジェクト: euforia/ess-go-wrapper
/*
   Create the index if it does not exist.
   Optionally apply a mapping if mapping file is supplied.
*/
func NewEssWrapper(esshost string, essport int, index string, mappingfile ...string) (*EssWrapper, error) {

	ed := EssWrapper{conn: elastigo.NewConn(), Index: index}
	ed.conn.Domain = esshost
	ed.conn.Port = fmt.Sprintf("%d", essport)

	if !ed.IndexExists() {
		if len(mappingfile) > 0 {
			log.V(9).Infof("Initializing with mapping file: %#v\n", mappingfile[0])
			return &ed, ed.initializeIndex(mappingfile[0])
		} else {
			return &ed, ed.initializeIndex("")
		}
	}
	return &ed, nil
}
コード例 #24
0
ファイル: client.go プロジェクト: nplanel/skydive
func NewElasticSearchClient(addr string, port string, maxConns int, retrySeconds int, bulkMaxDocs int) (*ElasticSearchClient, error) {
	c := elastigo.NewConn()

	c.Domain = addr
	c.Port = port

	indexer := c.NewBulkIndexerErrors(maxConns, retrySeconds)
	if bulkMaxDocs <= 0 {
		indexer.BulkMaxDocs = bulkMaxDocs
	}

	client := &ElasticSearchClient{
		connection: c,
		indexer:    indexer,
	}

	client.started.Store(false)
	return client, nil
}
コード例 #25
0
ファイル: elasticsearch.go プロジェクト: lebauce/skydive
func New() (*ElasticSearchStorage, error) {
	c := elastigo.NewConn()

	elasticonfig := strings.Split(config.GetConfig().GetString("storage.elasticsearch"), ":")
	if len(elasticonfig) != 2 {
		return nil, ErrBadConfig
	}
	c.Domain = elasticonfig[0]
	c.Port = elasticonfig[1]

	storage := &ElasticSearchStorage{connection: c}

	err := storage.initialize()
	if err != nil {
		return nil, err
	}

	return storage, nil
}
コード例 #26
0
// The inspecting the response
func ExampleBulkIndexer_responses() {
	c := elastigo.NewConn()

	indexer := c.NewBulkIndexer(10)
	// Create a custom Sender Func, to allow inspection of response/error
	indexer.Sender = func(buf *bytes.Buffer) error {
		// @buf is the buffer of docs about to be written
		respJson, err := c.DoCommand("POST", "/_bulk", nil, buf)
		if err != nil {
			// handle it better than this
			fmt.Println(string(respJson))
		}
		return err
	}
	indexer.Start()
	for i := 0; i < 20; i++ {
		indexer.Index("twitter", "user", strconv.Itoa(i), "", "", nil, `{"name":"bob"}`, true)
	}
	indexer.Stop()
}
コード例 #27
0
ファイル: datastore.go プロジェクト: krets/infra-inventory
/*
   Create the index if it does not exist.
   Optionally apply a mapping if mapping file is supplied.
*/
func NewElasticsearchDatastore(esshost string, essport int, index string, mappingfile ...string) (*ElasticsearchDatastore, error) {

	ed := ElasticsearchDatastore{
		Conn:         elastigo.NewConn(),
		Index:        index,
		VersionIndex: index + "_versions",
	}

	ed.Conn.Domain = esshost
	ed.Conn.Port = fmt.Sprintf("%d", essport)

	if !ed.IndexExists() {
		if len(mappingfile) > 0 {
			log.V(9).Infof("Initializing with mapping file: %#v\n", mappingfile[0])
			return &ed, ed.initializeIndex(mappingfile[0])
		} else {
			return &ed, ed.initializeIndex("")
		}
	}
	return &ed, nil
}
コード例 #28
0
ファイル: client.go プロジェクト: Sean-Der/elastigo
// for testing
func main() {
	flag.Parse()
	log.SetFlags(log.Ltime | log.Lshortfile)

	c := elastigo.NewConn()
	c.Domain = *eshost
	response, _ := c.Index("twitter", "tweet", "1", nil, NewTweet("kimchy", "Search is cool"))
	c.Flush()
	log.Printf("Index OK: %v", response.Ok)
	searchresponse, err := c.Search("twitter", "tweet", nil, "{\"query\" : {\"term\" : { \"user\" : \"kimchy\" }}}")
	if err != nil {
		log.Println("error during search:" + err.Error())
		log.Fatal(err)
	}
	// try marshalling to tweet type
	var t Tweet
	bytes, err := searchresponse.Hits.Hits[0].Source.MarshalJSON()
	if err != nil {
		log.Fatalf("err calling marshalJson:%v", err)
	}
	json.Unmarshal(bytes, t)
	log.Printf("Search Found: %s", t)
	response, _ = c.Get("twitter", "tweet", "1", nil)
	log.Printf("Get: %v", response.Exists)
	exists, _ := c.Exists("twitter", "tweet", "1", nil)
	log.Printf("Exists: %v", exists)
	c.Flush()
	countResponse, _ := c.Count("twitter", "tweet", nil, nil)

	log.Printf("Count: %v", countResponse.Count)
	response, _ = c.Delete("twitter", "tweet", "1", map[string]interface{}{"version": -1, "routing": ""})
	log.Printf("Delete OK: %v", response.Ok)
	response, _ = c.Get("twitter", "tweet", "1", nil)
	log.Printf("Get: %v", response.Exists)

	healthResponse, _ := c.Health()
	log.Printf("Health: %v", healthResponse.Status)

	c.UpdateSettings("transient", "discovery.zen.minimum_master_nodes", 2)
}
コード例 #29
0
ファイル: elasticsearch.go プロジェクト: HiveHQ/transporter
func (e *Elasticsearch) setupClient() {
	// set up the client, we need host(s), port, username, password, and scheme
	client := elastigo.NewConn()

	if e.uri.User != nil {
		client.Username = e.uri.User.Username()
		if password, set := e.uri.User.Password(); set {
			client.Password = password
		}
	}

	// we might have a port in the host bit
	hostBits := strings.Split(e.uri.Host, ":")
	if len(hostBits) > 1 {
		client.SetPort(hostBits[1])
	}

	client.SetHosts(strings.Split(hostBits[0], ","))
	client.Protocol = e.uri.Scheme

	e.indexer = client.NewBulkIndexerErrors(10, 60)
}
コード例 #30
0
func NewElasticsearchDatastore(cfg *config.DatastoreConfig, logger *simplelog.Logger) (*ElasticsearchDatastore, error) {
	//logger.Info.Printf("HERE\n")
	var (
		ed        = ElasticsearchDatastore{logger: logger}
		err       error
		idxExists = false
	)

	typeCfg, err := ed.parseTypeConfig(cfg.TypeConfig)
	if err != nil {
		return &ed, err
	}

	ed.index = typeCfg.Index

	ed.conn = elastigo.NewConn()
	ed.conn.Domain = typeCfg.Host
	ed.conn.Port = fmt.Sprintf("%d", typeCfg.Port)

	//ed.conn.ExistsIndex(index, _type, args)
	logger.Trace.Printf("Checking if index exists...\n")
	idxExists, err = ed.conn.ExistsIndex(ed.index, "", nil)
	//logger.Error.Println("exists=%s; err=%s", idxExists, err)

	//logger.Error.Printf("%s\n", idxExists)

	if err != nil {
		if err.Error() == "record not found" {
			idxExists = true
		} else {
			return &ed, fmt.Errorf("Failed index check: %s", err)
		}
	}

	if !idxExists {
		return &ed, ed.initializeIndex(typeCfg.MappingFile)
	}
	return &ed, nil
}