Beispiel #1
0
// Exists allows caller to check for the existance of a document using HEAD
func Exists(pretty bool, index string, _type string, id string) (bool, error) {

	var url string

	if len(_type) > 0 {
		url = fmt.Sprintf("/%s/%s/%s?fields=_id%s", index, _type, id, api.Pretty(pretty))
	} else {
		url = fmt.Sprintf("/%s/%s?fields=_id%s", index, id, api.Pretty(pretty))
	}

	req, err := api.ElasticSearchRequest("HEAD", url)

	if err != nil {
		fmt.Println(err)
	}

	httpStatusCode, _, err := req.Do(nil)

	if err != nil {
		return false, err
	}
	if httpStatusCode == http.StatusOK {
		return true, err
	} else {
		return false, err
	}
}
Beispiel #2
0
// This loads test data from github archives (~6700 docs)
func LoadTestData() {
	docCt := 0
	errCt := 0
	indexer := NewBulkIndexer(20)
	indexer.BulkSendor = func(buf *bytes.Buffer) error {
		log.Printf("Sent %d bytes total %d docs sent", buf.Len(), docCt)
		req, err := api.ElasticSearchRequest("POST", "/_bulk")
		if err != nil {
			errCt += 1
			log.Println("ERROR: ", err)
			return err
		}
		req.SetBody(buf)
		res, err := http.DefaultClient.Do((*http.Request)(req))
		if err != nil {
			errCt += 1
			log.Println("ERROR: ", err)
			return err
		}
		if res.StatusCode != 200 {
			log.Printf("Not 200! %d \n", res.StatusCode)
		}
		return err
	}
	done := make(chan bool)
	indexer.Run(done)
	resp, err := http.Get("http://data.githubarchive.org/2012-12-10-15.json.gz")
	if err != nil || resp == nil {
		panic("Could not download data")
	}
	defer resp.Body.Close()
	if err != nil {
		log.Println(err)
		return
	}
	gzReader, err := gzip.NewReader(resp.Body)
	defer gzReader.Close()
	if err != nil {
		panic(err)
	}
	r := bufio.NewReader(gzReader)
	var ge GithubEvent
	docsm := make(map[string]bool)
	h := md5.New()
	for {
		line, err := r.ReadBytes('\n')
		if err != nil && err != io.EOF {
			log.Println("FATAL:  could not read line? ", err)
		} else if err != nil {
			indexer.Flush()
			break
		}
		if err := json.Unmarshal(line, &ge); err == nil {
			// create an "ID"
			h.Write(line)
			id := fmt.Sprintf("%x", h.Sum(nil))
			if _, ok := docsm[id]; ok {
				log.Println("HM, already exists? ", ge.Url)
			}
			docsm[id] = true
			indexer.Index("github", ge.Type, id, "", &ge.Created, line)
			docCt++
			//log.Println(docCt, " ", string(line))
			//os.Exit(1)
		} else {
			log.Println("ERROR? ", string(line))
		}

	}
	if errCt != 0 {
		log.Println("FATAL, could not load ", errCt)
	}
	// lets wait a bit to ensure that elasticsearch finishes?
	time.Sleep(time.Second * 5)
	if len(docsm) != docCt {
		panic(fmt.Sprintf("Docs didn't match?   %d:%d", len(docsm), docCt))
	}
}