Beispiel #1
0
func TestTopology(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping topology tests in short mode, because they require Elasticsearch")
	}

	api.Domain = "localhost"
	api.Port = "9200"

	_, _ = core.Delete("packetbeat-topology", "server-ip", "", nil)
	var publisher1 PublisherType = PublisherType{name: "proxy1"}
	var publisher2 PublisherType = PublisherType{name: "proxy2"}
	var publisher3 PublisherType = PublisherType{name: "proxy3"}

	publisher1.PublishTopology("10.1.0.4")
	publisher2.PublishTopology("10.1.0.9", "fe80::4e8d:79ff:fef2:de6a")
	publisher3.PublishTopology("10.1.0.10")

	// give some time to Elasticsearch to add the IPs
	time.Sleep(1 * time.Second)

	publisher1.UpdateTopology()
	publisher2.UpdateTopology()
	publisher3.UpdateTopology()

	name2 := publisher1.GetServerName("10.1.0.9")
	if name2 != "proxy2" {
		t.Error("Failed to update proxy2 in topology: name=%s", name2)
	}

	name2 = publisher3.GetServerName("10.1.0.9")
	if name2 != "proxy2" {
		t.Error("Failed to update proxy2 in topology: name=%s", name2)
	}

	publisher1.PublishTopology("10.1.0.4")
	publisher2.PublishTopology("10.1.0.9")
	publisher3.PublishTopology("192.168.1.2")

	// give some time to Elasticsearch to add the IPs
	time.Sleep(1 * time.Second)

	publisher1.UpdateTopology()
	publisher2.UpdateTopology()
	publisher3.UpdateTopology()

	name3 := publisher1.GetServerName("192.168.1.2")
	if name3 != "proxy3" {
		t.Error("Failed to add a new IP")
	}

	name3 = publisher1.GetServerName("10.1.0.10")
	if name3 != "" {
		t.Error("Failed to delete old IP of proxy3: %s", name3)
	}

	name2 = publisher1.GetServerName("fe80::4e8d:79ff:fef2:de6a")
	if name2 != "" {
		t.Error("Failed to delete old IP of proxy2: %s", name2)
	}
}
Beispiel #2
0
// for testing
func main() {
	flag.Parse()
	log.SetFlags(log.Ltime | log.Lshortfile)
	api.Domain = *eshost
	response, _ := core.Index(true, "twitter", "tweet", "1", NewTweet("kimchy", "Search is cool"))
	indices.Flush()
	log.Printf("Index OK: %v", response.Ok)
	searchresponse, err := core.SearchRequest(true, "twitter", "tweet", "{\"query\" : {\"term\" : { \"user\" : \"kimchy\" }}}", "", 0)
	if err != nil {
		log.Println("error during search:" + err.Error())
		log.Fatal(err)
	}
	// try marshalling to tweet type
	var t Tweet
	json.Unmarshal(searchresponse.Hits.Hits[0].Source, t)
	log.Printf("Search Found: %s", t)
	response, _ = core.Get(true, "twitter", "tweet", "1")
	log.Printf("Get: %v", response.Exists)
	exists, _ := core.Exists(true, "twitter", "tweet", "1")
	log.Printf("Exists: %v", exists)
	indices.Flush()
	countResponse, _ := core.Count(true, "twitter", "tweet")
	log.Printf("Count: %v", countResponse.Count)
	response, _ = core.Delete(true, "twitter", "tweet", "1", -1, "")
	log.Printf("Delete OK: %v", response.Ok)
	response, _ = core.Get(true, "twitter", "tweet", "1")
	log.Printf("Get: %v", response.Exists)

	healthResponse, _ := cluster.Health(true)
	log.Printf("Health: %v", healthResponse.Status)

	cluster.State("transient", "discovery.zen.minimum_master_nodes", 2)

}
Beispiel #3
0
func TestTopology(t *testing.T) {

	api.Domain = "10.0.50.4"
	api.Port = "9200"

	_, _ = core.Delete("packetbeat-topology", "server-ip", "", nil)
	var publisher1 PublisherType = PublisherType{name: "proxy1", mother_host: api.Domain, mother_port: api.Port}
	var publisher2 PublisherType = PublisherType{name: "proxy2", mother_host: api.Domain, mother_port: api.Port}
	var publisher3 PublisherType = PublisherType{name: "proxy3", mother_host: api.Domain, mother_port: api.Port}

	publisher1.PublishTopology("10.1.0.4")
	publisher2.PublishTopology("10.1.0.9", "fe80::4e8d:79ff:fef2:de6a")
	publisher3.PublishTopology("10.1.0.10")

	// give some time to Elasticsearch to add the IPs
	time.Sleep(1 * time.Second)

	publisher1.UpdateTopology()
	publisher2.UpdateTopology()
	publisher3.UpdateTopology()

	name2 := publisher1.GetServerName("10.1.0.9")
	if name2 != "proxy2" {
		t.Error("Failed to update proxy2 in topology: name=%s", name2)
	}

	name2 = publisher3.GetServerName("10.1.0.9")
	if name2 != "proxy2" {
		t.Error("Failed to update proxy2 in topology: name=%s", name2)
	}

	publisher1.PublishTopology("10.1.0.4")
	publisher2.PublishTopology("10.1.0.9")
	publisher3.PublishTopology("192.168.1.2")

	// give some time to Elasticsearch to add the IPs
	time.Sleep(1 * time.Second)

	publisher1.UpdateTopology()
	publisher2.UpdateTopology()
	publisher3.UpdateTopology()

	name3 := publisher1.GetServerName("192.168.1.2")
	if name3 != "proxy3" {
		t.Error("Failed to add a new IP")
	}

	name3 = publisher1.GetServerName("10.1.0.10")
	if name3 != "" {
		t.Error("Failed to delete old IP of proxy3: %s", name3)
	}

	name2 = publisher1.GetServerName("fe80::4e8d:79ff:fef2:de6a")
	if name2 != "" {
		t.Error("Failed to delete old IP of proxy2: %s", name2)
	}
}
Beispiel #4
0
func (d *Datastore) Delete(key ds.Key) (err error) {
	id := d.KeyHash(key)
	res, err := core.Delete(false, d.Index(key), key.Type(), id, 0, "")
	if err != nil {
		return err
	}
	if !res.Ok {
		return fmt.Errorf("Elasticsearch response: NOT OK. %v", res)
	}
	return nil
}
Beispiel #5
0
func DeleteGeoJotHandler(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	_, err := core.Delete(true, es_index, es_index_type, vars["id"], 1, "")
	if err != nil {
		serveError(w, err)
	}

	// Respond with a 204 indicating success, but no content
	w.WriteHeader(http.StatusNoContent)

}
// for testing
func main() {
	flag.Parse()
	log.SetFlags(log.Ltime | log.Lshortfile)
	api.Domain = *eshost
	core.VerboseLogging = true
	response, _ := core.Index("twitter", "tweet", "1", nil, NewTweet("kimchy", "Search is cool"))
	indices.Flush()
	log.Printf("Index OK: %v", response.Ok)
	searchresponse, err := core.SearchRequest("twitter", "tweet", nil, "{\"query\" : {\"term\" : { \"user\" : \"kimchy\" }}}")
	if err != nil {
		log.Println("error during search:" + err.Error())
		log.Fatal(err)
	}
	// try marshalling to tweet type
	var t Tweet
	bytes, err := searchresponse.Hits.Hits[0].Source.MarshalJSON()
	if err != nil {
		log.Fatalf("err calling marshalJson:%v", err)
	}
	json.Unmarshal(bytes, t)
	log.Printf("Search Found: %s", t)
	response, _ = core.Get("twitter", "tweet", "1", nil)
	log.Printf("Get: %v", response.Exists)
	exists, _ := core.Exists("twitter", "tweet", "1", nil)
	log.Printf("Exists: %v", exists)
	indices.Flush()
	countResponse, _ := core.Count("twitter", "tweet", nil)
	log.Printf("Count: %v", countResponse.Count)
	response, _ = core.Delete("twitter", "tweet", "1", map[string]interface{}{"version": -1, "routing": ""})
	log.Printf("Delete OK: %v", response.Ok)
	response, _ = core.Get("twitter", "tweet", "1", nil)
	log.Printf("Get: %v", response.Exists)

	healthResponse, _ := cluster.Health()
	log.Printf("Health: %v", healthResponse.Status)

	cluster.UpdateSettings("transient", "discovery.zen.minimum_master_nodes", 2)

}
Beispiel #7
0
func (publisher *PublisherType) PublishTopology(params ...string) error {

	var localAddrs []string = params

	if len(params) == 0 {
		addrs, err := LocalAddrs()
		if err != nil {
			ERR("Getting local IP addresses fails with: %s", err)
			return err
		}
		localAddrs = addrs
	}

	// delete old IP addresses
	searchJson := fmt.Sprintf("{query: {term: {name: %s}}}", strconv.Quote(publisher.name))
	res, err := core.SearchRequest("packetbeat-topology", "server-ip", nil, searchJson)
	if err == nil {
		for _, server := range res.Hits.Hits {

			var top Topology
			err = json.Unmarshal([]byte(*server.Source), &top)
			if err != nil {
				ERR("Failed to unmarshal json data: %s", err)
			}
			if !stringInSlice(top.Ip, localAddrs) {
				res, err := core.Delete("packetbeat-topology", "server-ip" /*id*/, top.Ip, nil)
				if err != nil {
					ERR("Failed to delete the old IP address from packetbeat-topology")
				}
				if !res.Ok {
					ERR("Fail to delete old topology entry")
				}
			}

		}
	}

	// add new IP addresses
	for _, addr := range localAddrs {

		// check if the IP is already in the elasticsearch, before adding it
		found, err := core.Exists("packetbeat-topology", "server-ip" /*id*/, addr, nil)
		if err != nil {
			ERR("core.Exists fails with: %s", err)
		} else {

			if !found {
				res, err := core.Index("packetbeat-topology", "server-ip" /*id*/, addr, nil,
					Topology{publisher.name, addr})
				if err != nil {
					return err
				}
				if !res.Ok {
					ERR("Fail to add new topology entry")
				}
			}
		}
	}

	DEBUG("publish", "Topology: name=%s, ips=%s", publisher.name, strings.Join(localAddrs, " "))

	// initialize local topology map
	publisher.TopologyMap = make(map[string]string)

	return nil
}
//创建索引
func CreateIndex(message string) {
	//db, err := sql.Open("mysql", "root:123456@tcp(localhost:3306)/jacktest?charset=utf8")
	strSQL, table, constring, dbtype, isfiel, filepath, filekey, primarykey, otype, appindex := ReadIndexTypeForSQL(message)
	if dbtype == 0 {
		db, er = sql.Open("mysql", constring)

	}
	if dbtype == 1 {
		db, er = sql.Open("mgodbc", constring)
	}
	if er != nil {
		panic(er.Error())
	}
	defer db.Close()

	fmt.Println(strSQL)
	rows, err := db.Query(strSQL)
	if err != nil {
		panic(err.Error())
	}
	columns, err := rows.Columns()
	if err != nil {
		panic(err.Error())
	}
	values := make([]sql.RawBytes, len(columns))
	scanArgs := make([]interface{}, len(values))

	for i := range values {
		scanArgs[i] = &values[i]
	}
	fmt.Println(ESAPI)
	api.Domain = ESAPI
	s := map[string]interface{}{}
	var pid string
	for rows.Next() {
		err = rows.Scan(scanArgs...)
		if err != nil {
			panic(err.Error())
		}
		var value string
		for i, col := range values {
			//获取表格里数据
			if col == nil {
				value = "NULL"
			} else {
				value = string(col)
			}
			if columns[i] == primarykey {
				pid = string(col)
			}

			//当有文件标识列时,将文件进行base64编码
			if columns[i] == filekey {
				f, _ := ioutil.ReadFile(filepath + "/" + string(col))
				str := base64.StdEncoding.EncodeToString(f)
				s["file"] = str
			} else {
				s[columns[i]] = value
			}
			fmt.Println(columns[i] + ":" + value)
		}
		if otype != 2 {
			if isfiel == 0 {
				core.Index(true, strings.ToLower(appindex), strings.ToLower(table), pid, s)
			} else {
				core.Index(true, strings.ToLower(appindex), strings.ToLower(table), pid, s)
			}
		} else {
			if isfiel == 0 {
				fmt.Println("Delete")
				core.Delete(true, strings.ToLower(appindex), strings.ToLower(table), pid, 0, "")
			} else {
				core.Delete(true, strings.ToLower(appindex), "attachment", pid, 0, "")
			}
		}

		fmt.Println("-----------------------------------")
	}
}