func TestUrlGenerationNoIndices(t *testing.T) {
expectedUrl := "/_cluster/health?level=cluster&pretty=true&timeout=30s&wait_for_nodes=1&wait_for_relocating_shards=1&wait_for_status=green"
indices := []string{}
url, err := getHealthUrl(true, "cluster", "green", "30s", 1, 1, indices...)
u.Assert(err == nil, t, fmt.Sprintf("err was not nil: %v", err))
u.Assert(url == expectedUrl, t, fmt.Sprintf("TestUrlGeneration Should get %s, instead got %s", expectedUrl, url))
}
func TestUrlGenerationNoWaits(t *testing.T) {
expectedUrl := "/_cluster/health/Indice1,Indice2,Indice3?level=cluster&pretty=true&timeout=30s&wait_for_status=green"
indices := []string{"Indice1", "Indice2", "Indice3"}
url, err := getHealthUrl(true, "cluster", "green", "30s", 0, 0, indices...)
u.Assert(err == nil, t, fmt.Sprintf("err was not nil: %v", err))
u.Assert(url == expectedUrl, t, fmt.Sprintf("TestUrlGeneration Should get %s, instead got %s", expectedUrl, url))
}
func TestSearchFields(t *testing.T) {
// same as terms, search using fields:
// how many different docs have jasmine in repository.name?
qry := Search("github").Query(
Query().Fields("repository.name", "jasmine", "", ""),
)
out, _ := qry.Result()
u.Assert(out.Hits.Len() == 4, t, "Should have 4 docs %v", out.Hits.Len())
u.Assert(out.Hits.Total == 4, t, "Should have 4 total= %v", out.Hits.Total)
}
func TestSearchRequest(t *testing.T) {
qry := map[string]interface{}{
"query": map[string]interface{}{
"wildcard": map[string]string{"actor": "a*"},
},
}
out, err := core.SearchRequest(true, "github", "", qry, "", 0)
//log.Println(out)
u.Assert(&out != nil && err == nil, t, "Should get docs")
u.Assert(out.Hits.Total == 616 && out.Hits.Len() == 10, t, "Should have 616 hits but was %v", out.Hits.Total)
}
func TestSearchTerm(t *testing.T) {
// ok, now lets try searching with term query (specific field/term)
qry := Search("github").Query(
Query().Term("repository.name", "jasmine"),
)
out, _ := qry.Result()
// how many different docs have jasmine in repository.name?
u.Assert(out.Hits.Len() == 4, t, "Should have 4 docs %v", out.Hits.Len())
u.Assert(out.Hits.Total == 4, t, "Should have 4 total= %v", out.Hits.Total)
}
func TestSearchRange(t *testing.T) {
// now lets filter by a subset of the total time
out, _ := Search("github").Size("25").Query(
Query().Range(
Range().Field("created_at").From("2012-12-10T15:00:00-08:00").To("2012-12-10T15:10:00-08:00"),
).Search("add"),
).Result()
u.Assert(out != nil && &out.Hits != nil, t, "Must not have nil results, or hits")
u.Assert(out.Hits.Len() == 25, t, "Should have 25 docs %v", out.Hits.Len())
u.Assert(out.Hits.Total == 92, t, "Should have total=92 but was %v", out.Hits.Total)
}
func TestBulkErrors(t *testing.T) {
// lets set a bad port, and hope we get a connection refused error?
api.Port = "27845"
defer func() {
api.Port = "9200"
}()
BulkDelaySeconds = 1
indexor := NewBulkIndexorErrors(10, 1)
done := make(chan bool)
indexor.Run(done)
errorCt := 0
go func() {
for i := 0; i < 20; i++ {
date := time.Unix(1257894000, 0)
data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0)}
indexor.Index("users", "user", strconv.Itoa(i), "", &date, data)
}
}()
for errBuf := range indexor.ErrorChannel {
errorCt++
u.Debug(errBuf.Err)
break
}
u.Assert(errorCt > 0, t, "ErrorCt should be > 0 %d", errorCt)
}
func TestSearchSimple(t *testing.T) {
// searching without faceting
qry := Search("github").Pretty().Query(
Query().Search("add"),
)
out, _ := qry.Result()
// how many different docs used the word "add"
u.Assert(out.Hits.Len() == 10, t, "Should have 10 docs %v", out.Hits.Len())
u.Assert(out.Hits.Total == 494, t, "Should have 494 total= %v", out.Hits.Total)
// now the same result from a "Simple" search
out, _ = Search("github").Search("add").Result()
u.Assert(out.Hits.Len() == 10, t, "Should have 10 docs %v", out.Hits.Len())
u.Assert(out.Hits.Total == 494, t, "Should have 494 total= %v", out.Hits.Total)
}
func TestSearchFilterQuery(t *testing.T) {
// compound query + filter with query being wildcard
out, _ := Search("github").Size("25").Query(
Query().Fields("repository.name", "jas*", "", ""),
).Filter(
Filter().Terms("repository.has_wiki", true),
).Result()
if out == nil || &out.Hits == nil {
t.Fail()
return
}
u.Assert(out.Hits.Len() == 7, t, "Should have 7 docs %v", out.Hits.Len())
u.Assert(out.Hits.Total == 7, t, "Should have total=7 but was %v", out.Hits.Total)
}
func TestSearchMissingExists(t *testing.T) {
// search for docs that are missing repository.name
qry := Search("github").Filter(
Filter().Exists("repository.name"),
)
out, _ := qry.Result()
u.Assert(out.Hits.Len() == 10, t, "Should have 10 docs %v", out.Hits.Len())
u.Assert(out.Hits.Total == 7695, t, "Should have 7695 total= %v", out.Hits.Total)
qry = Search("github").Filter(
Filter().Missing("repository.name"),
)
out, _ = qry.Result()
u.Assert(out.Hits.Len() == 10, t, "Should have 10 docs %v", out.Hits.Len())
u.Assert(out.Hits.Total == 389, t, "Should have 389 total= %v", out.Hits.Total)
}
func TestBulkUpdate(t *testing.T) {
InitTests(true)
api.Port = "9200"
indexor := NewBulkIndexor(3)
indexor.BulkSendor = func(buf *bytes.Buffer) error {
messageSets += 1
totalBytesSent += buf.Len()
buffers = append(buffers, buf)
u.Debug(string(buf.Bytes()))
return BulkSend(buf)
}
done := make(chan bool)
indexor.Run(done)
date := time.Unix(1257894000, 0)
user := map[string]interface{}{
"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0), "count": 1,
}
// Lets make sure the data is in the index ...
_, err := Index(true, "users", "user", "5", user)
// script and params
data := map[string]interface{}{
"script": "ctx._source.count += 2",
}
err = indexor.Update("users", "user", "5", "", &date, data)
// So here's the deal. Flushing does seem to work, you just have to give the
// channel a moment to recieve the message ...
// <- time.After(time.Millisecond * 20)
// indexor.Flush()
done <- true
WaitFor(func() bool {
return len(buffers) > 0
}, 5)
u.Assert(BulkErrorCt == 0 && err == nil, t, "Should not have any errors %v", err)
response, err := Get(true, "users", "user", "5")
u.Assert(err == nil, t, "Should not have any errors %v", err)
newCount := response.Source.(map[string]interface{})["count"]
u.Assert(newCount.(float64) == 3, t, "Should have update count: %#v ... %#v", response.Source.(map[string]interface{})["count"], response)
}
func TestSearchSortOrder(t *testing.T) {
// ok, now lets try sorting by repository watchers descending
qry := Search("github").Pretty().Query(
Query().All(),
).Sort(
Sort("repository.watchers").Desc(),
)
out, _ := qry.Result()
// how many different docs used the word "add", during entire time range
u.Assert(out.Hits.Len() == 10, t, "Should have 10 docs %v", out.Hits.Len())
u.Assert(out.Hits.Total == 8084, t, "Should have 8084 total= %v", out.Hits.Total)
h1 := u.NewJsonHelper(out.Hits.Hits[0].Source)
u.Assert(h1.Int("repository.watchers") == 41377, t, "Should have 41377 watchers= %v", h1.Int("repository.watchers"))
// ascending
out, _ = Search("github").Pretty().Query(
Query().All(),
).Sort(
Sort("repository.watchers"),
).Result()
// how many different docs used the word "add", during entire time range
u.Assert(out.Hits.Len() == 10, t, "Should have 10 docs %v", out.Hits.Len())
u.Assert(out.Hits.Total == 8084, t, "Should have 8084 total= %v", out.Hits.Total)
h2 := u.NewJsonHelper(out.Hits.Hits[0].Source)
u.Assert(h2.Int("repository.watchers") == 0, t, "Should have 0 watchers= %v", h2.Int("repository.watchers"))
// sort descending with search
out, _ = Search("github").Pretty().Size("5").Query(
Query().Search("python"),
).Sort(
Sort("repository.watchers").Desc(),
).Result()
//log.Println(out)
//log.Println(err)
// how many different docs used the word "add", during entire time range
u.Assert(out.Hits.Len() == 5, t, "Should have 5 docs %v", out.Hits.Len())
u.Assert(out.Hits.Total == 734, t, "Should have 734 total= %v", out.Hits.Total)
h3 := u.NewJsonHelper(out.Hits.Hits[0].Source)
u.Assert(h3.Int("repository.watchers") == 8659, t, "Should have 8659 watchers= %v", h3.Int("repository.watchers"))
}
func TestLRUTtlEvict(t *testing.T) {
// ttl = 1 second, 100 ms heartbeats, should check 10 times
l := NewTimeEvictLru(10000, 200*time.Millisecond, 50*time.Millisecond)
var evictCt int = 0
l.EvictCallback = func(key string, v Value) {
evictCt++
//u.Debug("evicting ", key)
}
for i := 0; i < 100; i++ {
istr := strconv.FormatInt(int64(i), 10)
l.Set("key"+istr, &CacheValue{i})
}
u.WaitFor(func() bool {
return l.Len() == 0
}, 10)
u.Assert(l.Len() == 0, t, "should have evicted all items but still have %d", l.Len())
u.Assert(evictCt == 100, t, "Should have evicted 100 items but have %d", evictCt)
}
func TestSearchFacetRange(t *testing.T) {
// ok, now lets try facet but on actor field with a range
qry := Search("github").Pretty().Facet(
Facet().Fields("actor").Size("500"),
).Query(
Query().Search("add"),
)
out, err := qry.Result()
u.Assert(out != nil && err == nil, t, "Should have output")
if out == nil {
t.Fail()
return
}
//log.Println(string(out.Facets))
h := u.NewJsonHelper(out.Facets)
// how many different docs used the word "add", during entire time range
u.Assert(h.Int("actor.total") == 521, t, "Should have 521 results %v", h.Int("actor.total"))
// make sure size worked
u.Assert(len(h.List("actor.terms")) == 366, t, "Should have 366 unique userids, %v", len(h.List("actor.terms")))
// ok, repeat but with a range showing different results
qry = Search("github").Pretty().Facet(
Facet().Fields("actor").Size("500"),
).Query(
Query().Range(
Range().Field("created_at").From("2012-12-10T15:00:00-08:00").To("2012-12-10T15:10:00-08:00"),
).Search("add"),
)
out, err = qry.Result()
u.Assert(out != nil && err == nil, t, "Should have output")
if out == nil {
t.Fail()
return
}
//log.Println(string(out.Facets))
h = u.NewJsonHelper(out.Facets)
// how many different events used the word "add", during time range?
u.Assert(h.Int("actor.total") == 97, t, "Should have 97 results %v", h.Int("actor.total"))
// make sure size worked
u.Assert(len(h.List("actor.terms")) == 71, t, "Should have 71 event types, %v", len(h.List("actor.terms")))
}
func TestBulkIndexorBasic(t *testing.T) {
InitTests(true)
indexor := NewBulkIndexor(3)
indexor.BulkSendor = func(buf *bytes.Buffer) error {
messageSets += 1
totalBytesSent += buf.Len()
buffers = append(buffers, buf)
u.Debug(string(buf.Bytes()))
return BulkSend(buf)
}
done := make(chan bool)
indexor.Run(done)
date := time.Unix(1257894000, 0)
data := map[string]interface{}{"name": "smurfs", "age": 22, "date": time.Unix(1257894000, 0)}
err := indexor.Index("users", "user", "1", "", &date, data)
WaitFor(func() bool {
return len(buffers) > 0
}, 5)
// part of request is url, so lets factor that in
//totalBytesSent = totalBytesSent - len(*eshost)
u.Assert(len(buffers) == 1, t, "Should have sent one operation but was %d", len(buffers))
u.Assert(BulkErrorCt == 0 && err == nil, t, "Should not have any errors %v", err)
u.Assert(totalBytesSent == 145, t, "Should have sent 135 bytes but was %v", totalBytesSent)
err = indexor.Index("users", "user", "2", "", nil, data)
<-time.After(time.Millisecond * 10) // we need to wait for doc to hit send channel
// this will test to ensure that Flush actually catches a doc
indexor.Flush()
totalBytesSent = totalBytesSent - len(*eshost)
u.Assert(err == nil, t, "Should have nil error =%v", err)
u.Assert(len(buffers) == 2, t, "Should have another buffer ct=%d", len(buffers))
u.Assert(BulkErrorCt == 0, t, "Should not have any errors %d", BulkErrorCt)
u.Assert(u.CloseInt(totalBytesSent, 257), t, "Should have sent 257 bytes but was %v", totalBytesSent)
}
func TestUrlGenerationBadWaitForStatus(t *testing.T) {
indices := []string{"Indice1", "Indice2", "Indice3"}
_, err := getHealthUrl(true, "cluster", "Wait_For_Status", "Timeout", 1, 1, indices...)
u.Assert(err != nil, t, fmt.Sprintf("Call should have failed with bad wait_for_status parameter: %v", err))
}
func TestUrlGenerationNoTimeout(t *testing.T) {
expectedUrl := "/Index/Type/Id?op_type=create&parent=Parent&percolate=Percolate&routing=Routing×tamp=TimeStamp&ttl=10&version=1"
url, err := GetIndexUrl("Index", "Type", "Id", "Parent", 1, "create", "Routing", "TimeStamp", 10, "Percolate", "")
u.Assert(err == nil, t, "err was not nil")
u.Assert(url == expectedUrl, t, fmt.Sprintf("TestUrlGenerationNoTimeout Should get %s, instead got %s", expectedUrl, url))
}
func TestUrlGenerationNoType(t *testing.T) {
_, err := GetIndexUrl("Index", "", "Id", "Parent", 1, "create", "Routing", "TimeStamp", 10, "Percolate", "Timeout")
u.Assert(err != nil, t, "err should have been returned")
}
func TestSearchFacetOne(t *testing.T) {
/*
A faceted search for what "type" of events there are
- since we are not specifying an elasticsearch type it searches all ()
{
"terms" : {
"_type" : "terms",
"missing" : 0,
"total" : 7561,
"other" : 0,
"terms" : [ {
"term" : "pushevent",
"count" : 4185
}, {
"term" : "createevent",
"count" : 786
}.....]
}
}
*/
qry := Search("github").Pretty().Facet(
Facet().Fields("type").Size("25"),
).Query(
Query().All(),
).Size("1")
out, err := qry.Result()
//log.Println(string(out.Facets))
u.Debug(out)
u.Assert(out != nil && err == nil, t, "Should have output")
if out == nil {
t.Fail()
return
}
h := u.NewJsonHelper(out.Facets)
u.Assert(h.Int("type.total") == 8084, t, "Should have 8084 results %v", h.Int("type.total"))
u.Assert(len(h.List("type.terms")) == 16, t, "Should have 16 event types, %v", len(h.List("type.terms")))
// Now, lets try changing size to 10
qry.FacetVal.Size("10")
out, err = qry.Result()
h = u.NewJsonHelper(out.Facets)
// still same doc count
u.Assert(h.Int("type.total") == 8084, t, "Should have 8084 results %v", h.Int("type.total"))
// make sure size worked
u.Assert(len(h.List("type.terms")) == 10, t, "Should have 10 event types, %v", len(h.List("type.terms")))
// now, lets add a type (out of the 16)
out, _ = Search("github").Type("IssueCommentEvent").Pretty().Facet(
Facet().Fields("type").Size("25"),
).Query(
Query().All(),
).Result()
h = u.NewJsonHelper(out.Facets)
//log.Println(string(out.Facets))
// still same doc count
u.Assert(h.Int("type.total") == 685, t, "Should have 685 results %v", h.Int("type.total"))
// we should only have one facettype because we limited to one type
u.Assert(len(h.List("type.terms")) == 1, t, "Should have 1 event types, %v", len(h.List("type.terms")))
// now, add a second type (chained)
out, _ = Search("github").Type("IssueCommentEvent").Type("PushEvent").Pretty().Facet(
Facet().Fields("type").Size("25"),
).Query(
Query().All(),
).Result()
h = u.NewJsonHelper(out.Facets)
//log.Println(string(out.Facets))
// still same doc count
u.Assert(h.Int("type.total") == 4941, t, "Should have 4941 results %v", h.Int("type.total"))
// make sure we now have 2 types
u.Assert(len(h.List("type.terms")) == 2, t, "Should have 2 event types, %v", len(h.List("type.terms")))
//and instead of faceting on type, facet on userid
// now, add a second type (chained)
out, _ = Search("github").Type("IssueCommentEvent,PushEvent").Pretty().Facet(
Facet().Fields("actor").Size("500"),
).Query(
Query().All(),
).Result()
h = u.NewJsonHelper(out.Facets)
// still same doc count
u.Assert(h.Int("actor.total") == 5158, t, "Should have 5158 results %v", h.Int("actor.total"))
// make sure size worked
u.Assert(len(h.List("actor.terms")) == 500, t, "Should have 500 users, %v", len(h.List("actor.terms")))
}
func TestSearchRequestQueryString(t *testing.T) {
out, err := core.SearchUri("github", "", "actor:a*", "", 0)
//log.Println(out)
u.Assert(&out != nil && err == nil, t, "Should get docs")
u.Assert(out.Hits.Total == 616, t, "Should have 616 hits but was %v", out.Hits.Total)
}