func Exists(pretty bool, index string, _type string, id string) (bool, error) {
var url string
var response map[string]interface{}
if len(_type) > 0 {
url = fmt.Sprintf("/%s/%s/%s?fields=_id%s", index, _type, id, api.Pretty(pretty))
} else {
url = fmt.Sprintf("/%s/%s?fields=_id%s", index, id, api.Pretty(pretty))
}
req, err := api.ElasticSearchRequest("HEAD", url)
if err != nil {
fmt.Println(err)
}
httpStatusCode, _, err := req.Do(&response)
if err != nil {
return false, err
}
if httpStatusCode == 404 {
return false, err
} else {
return true, err
}
}
// This loads test data from github archives (~6700 docs)
func LoadTestData() {
docCt := 0
errCt := 0
indexor := NewBulkIndexor(20)
indexor.BulkSendor = func(buf *bytes.Buffer) error {
log.Printf("Sent %d bytes total %d docs sent", buf.Len(), docCt)
req, err := api.ElasticSearchRequest("POST", "/_bulk")
if err != nil {
errCt += 1
log.Println("ERROR: ", err)
return err
}
req.SetBody(buf)
res, err := http.DefaultClient.Do((*http.Request)(req))
if err != nil {
errCt += 1
log.Println("ERROR: ", err)
return err
}
if res.StatusCode != 200 {
log.Printf("Not 200! %d \n", res.StatusCode)
}
return err
}
done := make(chan bool)
indexor.Run(done)
resp, err := http.Get("http://data.githubarchive.org/2012-12-10-15.json.gz")
if err != nil || resp == nil {
panic("Could not download data")
}
defer resp.Body.Close()
if err != nil {
log.Println(err)
return
}
gzReader, err := gzip.NewReader(resp.Body)
defer gzReader.Close()
if err != nil {
panic(err)
}
r := bufio.NewReader(gzReader)
var ge GithubEvent
docsm := make(map[string]bool)
h := md5.New()
for {
line, err := r.ReadBytes('\n')
if err != nil && err != io.EOF {
log.Println("FATAL: could not read line? ", err)
} else if err != nil {
indexor.Flush()
break
}
if err := json.Unmarshal(line, &ge); err == nil {
// create an "ID"
h.Write(line)
id := fmt.Sprintf("%x", h.Sum(nil))
if _, ok := docsm[id]; ok {
log.Println("HM, already exists? ", ge.Url)
}
docsm[id] = true
indexor.Index("github", ge.Type, id, "", &ge.Created, line)
docCt++
//log.Println(docCt, " ", string(line))
//os.Exit(1)
} else {
log.Println("ERROR? ", string(line))
}
}
if errCt != 0 {
log.Println("FATAL, could not load ", errCt)
}
// lets wait a bit to ensure that elasticsearch finishes?
time.Sleep(time.Second * 5)
if len(docsm) != docCt {
panic(fmt.Sprintf("Docs didn't match? %d:%d", len(docsm), docCt))
}
}