func TestTopology(t *testing.T) {
if testing.Short() {
t.Skip("Skipping topology tests in short mode, because they require Elasticsearch")
}
api.Domain = "localhost"
api.Port = "9200"
_, _ = core.Delete("packetbeat-topology", "server-ip", "", nil)
var publisher1 PublisherType = PublisherType{name: "proxy1"}
var publisher2 PublisherType = PublisherType{name: "proxy2"}
var publisher3 PublisherType = PublisherType{name: "proxy3"}
publisher1.PublishTopology("10.1.0.4")
publisher2.PublishTopology("10.1.0.9", "fe80::4e8d:79ff:fef2:de6a")
publisher3.PublishTopology("10.1.0.10")
// give some time to Elasticsearch to add the IPs
time.Sleep(1 * time.Second)
publisher1.UpdateTopology()
publisher2.UpdateTopology()
publisher3.UpdateTopology()
name2 := publisher1.GetServerName("10.1.0.9")
if name2 != "proxy2" {
t.Error("Failed to update proxy2 in topology: name=%s", name2)
}
name2 = publisher3.GetServerName("10.1.0.9")
if name2 != "proxy2" {
t.Error("Failed to update proxy2 in topology: name=%s", name2)
}
publisher1.PublishTopology("10.1.0.4")
publisher2.PublishTopology("10.1.0.9")
publisher3.PublishTopology("192.168.1.2")
// give some time to Elasticsearch to add the IPs
time.Sleep(1 * time.Second)
publisher1.UpdateTopology()
publisher2.UpdateTopology()
publisher3.UpdateTopology()
name3 := publisher1.GetServerName("192.168.1.2")
if name3 != "proxy3" {
t.Error("Failed to add a new IP")
}
name3 = publisher1.GetServerName("10.1.0.10")
if name3 != "" {
t.Error("Failed to delete old IP of proxy3: %s", name3)
}
name2 = publisher1.GetServerName("fe80::4e8d:79ff:fef2:de6a")
if name2 != "" {
t.Error("Failed to delete old IP of proxy2: %s", name2)
}
}
// for testing
func main() {
flag.Parse()
log.SetFlags(log.Ltime | log.Lshortfile)
api.Domain = *eshost
response, _ := core.Index(true, "twitter", "tweet", "1", NewTweet("kimchy", "Search is cool"))
indices.Flush()
log.Printf("Index OK: %v", response.Ok)
searchresponse, err := core.SearchRequest(true, "twitter", "tweet", "{\"query\" : {\"term\" : { \"user\" : \"kimchy\" }}}", "", 0)
if err != nil {
log.Println("error during search:" + err.Error())
log.Fatal(err)
}
// try marshalling to tweet type
var t Tweet
json.Unmarshal(searchresponse.Hits.Hits[0].Source, t)
log.Printf("Search Found: %s", t)
response, _ = core.Get(true, "twitter", "tweet", "1")
log.Printf("Get: %v", response.Exists)
exists, _ := core.Exists(true, "twitter", "tweet", "1")
log.Printf("Exists: %v", exists)
indices.Flush()
countResponse, _ := core.Count(true, "twitter", "tweet")
log.Printf("Count: %v", countResponse.Count)
response, _ = core.Delete(true, "twitter", "tweet", "1", -1, "")
log.Printf("Delete OK: %v", response.Ok)
response, _ = core.Get(true, "twitter", "tweet", "1")
log.Printf("Get: %v", response.Exists)
healthResponse, _ := cluster.Health(true)
log.Printf("Health: %v", healthResponse.Status)
cluster.State("transient", "discovery.zen.minimum_master_nodes", 2)
}
func TestTopology(t *testing.T) {
api.Domain = "10.0.50.4"
api.Port = "9200"
_, _ = core.Delete("packetbeat-topology", "server-ip", "", nil)
var publisher1 PublisherType = PublisherType{name: "proxy1", mother_host: api.Domain, mother_port: api.Port}
var publisher2 PublisherType = PublisherType{name: "proxy2", mother_host: api.Domain, mother_port: api.Port}
var publisher3 PublisherType = PublisherType{name: "proxy3", mother_host: api.Domain, mother_port: api.Port}
publisher1.PublishTopology("10.1.0.4")
publisher2.PublishTopology("10.1.0.9", "fe80::4e8d:79ff:fef2:de6a")
publisher3.PublishTopology("10.1.0.10")
// give some time to Elasticsearch to add the IPs
time.Sleep(1 * time.Second)
publisher1.UpdateTopology()
publisher2.UpdateTopology()
publisher3.UpdateTopology()
name2 := publisher1.GetServerName("10.1.0.9")
if name2 != "proxy2" {
t.Error("Failed to update proxy2 in topology: name=%s", name2)
}
name2 = publisher3.GetServerName("10.1.0.9")
if name2 != "proxy2" {
t.Error("Failed to update proxy2 in topology: name=%s", name2)
}
publisher1.PublishTopology("10.1.0.4")
publisher2.PublishTopology("10.1.0.9")
publisher3.PublishTopology("192.168.1.2")
// give some time to Elasticsearch to add the IPs
time.Sleep(1 * time.Second)
publisher1.UpdateTopology()
publisher2.UpdateTopology()
publisher3.UpdateTopology()
name3 := publisher1.GetServerName("192.168.1.2")
if name3 != "proxy3" {
t.Error("Failed to add a new IP")
}
name3 = publisher1.GetServerName("10.1.0.10")
if name3 != "" {
t.Error("Failed to delete old IP of proxy3: %s", name3)
}
name2 = publisher1.GetServerName("fe80::4e8d:79ff:fef2:de6a")
if name2 != "" {
t.Error("Failed to delete old IP of proxy2: %s", name2)
}
}
func (d *Datastore) Delete(key ds.Key) (err error) {
id := d.KeyHash(key)
res, err := core.Delete(false, d.Index(key), key.Type(), id, 0, "")
if err != nil {
return err
}
if !res.Ok {
return fmt.Errorf("Elasticsearch response: NOT OK. %v", res)
}
return nil
}
func DeleteGeoJotHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
_, err := core.Delete(true, es_index, es_index_type, vars["id"], 1, "")
if err != nil {
serveError(w, err)
}
// Respond with a 204 indicating success, but no content
w.WriteHeader(http.StatusNoContent)
}
// for testing
func main() {
flag.Parse()
log.SetFlags(log.Ltime | log.Lshortfile)
api.Domain = *eshost
core.VerboseLogging = true
response, _ := core.Index("twitter", "tweet", "1", nil, NewTweet("kimchy", "Search is cool"))
indices.Flush()
log.Printf("Index OK: %v", response.Ok)
searchresponse, err := core.SearchRequest("twitter", "tweet", nil, "{\"query\" : {\"term\" : { \"user\" : \"kimchy\" }}}")
if err != nil {
log.Println("error during search:" + err.Error())
log.Fatal(err)
}
// try marshalling to tweet type
var t Tweet
bytes, err := searchresponse.Hits.Hits[0].Source.MarshalJSON()
if err != nil {
log.Fatalf("err calling marshalJson:%v", err)
}
json.Unmarshal(bytes, t)
log.Printf("Search Found: %s", t)
response, _ = core.Get("twitter", "tweet", "1", nil)
log.Printf("Get: %v", response.Exists)
exists, _ := core.Exists("twitter", "tweet", "1", nil)
log.Printf("Exists: %v", exists)
indices.Flush()
countResponse, _ := core.Count("twitter", "tweet", nil)
log.Printf("Count: %v", countResponse.Count)
response, _ = core.Delete("twitter", "tweet", "1", map[string]interface{}{"version": -1, "routing": ""})
log.Printf("Delete OK: %v", response.Ok)
response, _ = core.Get("twitter", "tweet", "1", nil)
log.Printf("Get: %v", response.Exists)
healthResponse, _ := cluster.Health()
log.Printf("Health: %v", healthResponse.Status)
cluster.UpdateSettings("transient", "discovery.zen.minimum_master_nodes", 2)
}
func (publisher *PublisherType) PublishTopology(params ...string) error {
var localAddrs []string = params
if len(params) == 0 {
addrs, err := LocalAddrs()
if err != nil {
ERR("Getting local IP addresses fails with: %s", err)
return err
}
localAddrs = addrs
}
// delete old IP addresses
searchJson := fmt.Sprintf("{query: {term: {name: %s}}}", strconv.Quote(publisher.name))
res, err := core.SearchRequest("packetbeat-topology", "server-ip", nil, searchJson)
if err == nil {
for _, server := range res.Hits.Hits {
var top Topology
err = json.Unmarshal([]byte(*server.Source), &top)
if err != nil {
ERR("Failed to unmarshal json data: %s", err)
}
if !stringInSlice(top.Ip, localAddrs) {
res, err := core.Delete("packetbeat-topology", "server-ip" /*id*/, top.Ip, nil)
if err != nil {
ERR("Failed to delete the old IP address from packetbeat-topology")
}
if !res.Ok {
ERR("Fail to delete old topology entry")
}
}
}
}
// add new IP addresses
for _, addr := range localAddrs {
// check if the IP is already in the elasticsearch, before adding it
found, err := core.Exists("packetbeat-topology", "server-ip" /*id*/, addr, nil)
if err != nil {
ERR("core.Exists fails with: %s", err)
} else {
if !found {
res, err := core.Index("packetbeat-topology", "server-ip" /*id*/, addr, nil,
Topology{publisher.name, addr})
if err != nil {
return err
}
if !res.Ok {
ERR("Fail to add new topology entry")
}
}
}
}
DEBUG("publish", "Topology: name=%s, ips=%s", publisher.name, strings.Join(localAddrs, " "))
// initialize local topology map
publisher.TopologyMap = make(map[string]string)
return nil
}
//创建索引
func CreateIndex(message string) {
//db, err := sql.Open("mysql", "root:123456@tcp(localhost:3306)/jacktest?charset=utf8")
strSQL, table, constring, dbtype, isfiel, filepath, filekey, primarykey, otype, appindex := ReadIndexTypeForSQL(message)
if dbtype == 0 {
db, er = sql.Open("mysql", constring)
}
if dbtype == 1 {
db, er = sql.Open("mgodbc", constring)
}
if er != nil {
panic(er.Error())
}
defer db.Close()
fmt.Println(strSQL)
rows, err := db.Query(strSQL)
if err != nil {
panic(err.Error())
}
columns, err := rows.Columns()
if err != nil {
panic(err.Error())
}
values := make([]sql.RawBytes, len(columns))
scanArgs := make([]interface{}, len(values))
for i := range values {
scanArgs[i] = &values[i]
}
fmt.Println(ESAPI)
api.Domain = ESAPI
s := map[string]interface{}{}
var pid string
for rows.Next() {
err = rows.Scan(scanArgs...)
if err != nil {
panic(err.Error())
}
var value string
for i, col := range values {
//获取表格里数据
if col == nil {
value = "NULL"
} else {
value = string(col)
}
if columns[i] == primarykey {
pid = string(col)
}
//当有文件标识列时，将文件进行base64编码
if columns[i] == filekey {
f, _ := ioutil.ReadFile(filepath + "/" + string(col))
str := base64.StdEncoding.EncodeToString(f)
s["file"] = str
} else {
s[columns[i]] = value
}
fmt.Println(columns[i] + ":" + value)
}
if otype != 2 {
if isfiel == 0 {
core.Index(true, strings.ToLower(appindex), strings.ToLower(table), pid, s)
} else {
core.Index(true, strings.ToLower(appindex), strings.ToLower(table), pid, s)
}
} else {
if isfiel == 0 {
fmt.Println("Delete")
core.Delete(true, strings.ToLower(appindex), strings.ToLower(table), pid, 0, "")
} else {
core.Delete(true, strings.ToLower(appindex), "attachment", pid, 0, "")
}
}
fmt.Println("-----------------------------------")
}
}