// Search searches the index for the given query, and returns documents, // the total number of results, or an error if something went wrong func (i *Index) Search(q query.Query) (docs []index.Document, total int, err error) { query := solr.NewQuery() query.Q(q.Term) query.AddParam("cache", "false") //query.Start(int(q.Paging.Offset)) //query.Rows(int(q.Paging.Num)) s := i.si.Search(query) r, err := s.Result(nil) if err != nil { return nil, 0, err } ret := make([]index.Document, 0, len(r.Results.Docs)) for _, d := range r.Results.Docs { doc := index.NewDocument(d.Get("id").(string), 1.0) for k, v := range d { if reflect.TypeOf(v).Kind() == reflect.Slice { v = v.([]interface{})[0] } if k != "id" { doc.Set(k, v) } } ret = append(ret, doc) } return ret, r.Results.NumFound, nil }
func TestIndex(t *testing.T) { // todo: run redisearch automatically //t.SkipNow() md := index.NewMetadata().AddField(index.NewTextField("title", 1.0)). AddField(index.NewNumericField("score")) idx := NewIndex("localhost:6379", "testung", md) docs := []index.Document{ index.NewDocument("doc1", 0.1).Set("title", "hello world").Set("score", 1), index.NewDocument("doc2", 1.0).Set("title", "foo bar hello").Set("score", 2), } assert.NoError(t, idx.Drop()) assert.NoError(t, idx.Create()) assert.NoError(t, idx.Index(docs, nil)) q := query.NewQuery(idx.name, "hello world") docs, total, err := idx.Search(*q) assert.NoError(t, err) assert.True(t, total > 0) assert.Len(t, docs, 1) assert.Equal(t, docs[0].Id, "doc1") assert.Equal(t, docs[0].Properties["title"], "hello world") q = query.NewQuery(idx.name, "hello") docs, total, err = idx.Search(*q) assert.NoError(t, err) assert.Equal(t, 2, total) assert.Len(t, docs, 2) assert.Equal(t, docs[0].Id, "doc2") assert.Equal(t, docs[1].Id, "doc1") }
func TestDistributedIndex(t *testing.T) { // todo: run redisearch automatically //st.SkipNow() md := index.NewMetadata().AddField(index.NewTextField("title", 1.0)). AddField(index.NewNumericField("score")) idx := NewDistributedIndex("dtest", []string{"localhost:6379"}, 2, md) docs := []index.Document{ index.NewDocument("doc1", 0.1).Set("title", "hello world").Set("score", 1), index.NewDocument("doc2", 1.0).Set("title", "foo bar hello").Set("score", 2), } assert.NoError(t, idx.Drop()) assert.NoError(t, idx.Create()) assert.NoError(t, idx.Index(docs, nil)) q := query.NewQuery("dtest", "hello world") docs, total, err := idx.Search(*q) assert.NoError(t, err) assert.True(t, total > 0) assert.Len(t, docs, 1) assert.Equal(t, docs[0].Id, "doc1") assert.Equal(t, docs[0].Properties["title"], "hello world") q = query.NewQuery("dtest", "hello") docs, total, err = idx.Search(*q) t.Log(docs, total, err) assert.NoError(t, err) assert.Equal(t, 2, total) assert.Len(t, docs, 2) assert.Equal(t, docs[0].Id, "doc2") assert.Equal(t, docs[1].Id, "doc1") suggs := []index.Suggestion{} for i := 0; i < 100; i++ { suggs = append(suggs, index.Suggestion{fmt.Sprintf("suggestion %d", i), float64(i)}) } assert.NoError(t, idx.AddTerms(suggs...)) suggs, err = idx.Suggest("sugg", 10, false) assert.NoError(t, err) fmt.Println(suggs) assert.Len(t, suggs, 10) }
func TestIndex(t *testing.T) { // todo: run redisearch automatically md := index.NewMetadata().AddField(index.NewTextField("title", 1.0)). AddField(index.NewNumericField("score")) idx, err := NewIndex("http://localhost:8983/solr", "testung", md) assert.NoError(t, err) docs := []index.Document{} for i := 0; i < 100; i++ { docs = append(docs, index.NewDocument(fmt.Sprintf("doc%d", i), 0.1).Set("title", "hello world").Set("body", "lorem ipsum foo bar")) //index.NewDocument("doc2", 1.0).Set("title", "foo bar hello").Set("score", 2), } assert.NoError(t, idx.Drop()) // assert.NoError(t, idx.Create()) assert.NoError(t, idx.Index(docs, nil)) q := query.NewQuery("testung", "hello world") docs, total, err := idx.Search(*q) assert.NoError(t, err) assert.True(t, total == 100) assert.Len(t, docs, int(q.Paging.Num)) assert.Equal(t, docs[0].Id, "doc0") assert.Equal(t, docs[0].Properties["title"], "hello world") }
// Search searches the index for the given query, and returns documents, // the total number of results, or an error if something went wrong func (i *Index) Search(q query.Query) ([]index.Document, int, error) { eq := elastic.NewQueryStringQuery(q.Term) res, err := i.conn.Search(i.name).Type("doc"). Query(eq). From(q.Paging.Offset). Size(q.Paging.Num). Do() if err != nil { return nil, 0, err } ret := make([]index.Document, 0, q.Paging.Num) for _, h := range res.Hits.Hits { if h != nil { d := index.NewDocument(h.Id, float32(*h.Score)) if err := json.Unmarshal(*h.Source, &d.Properties); err == nil { ret = append(ret, d) } } } return ret, int(res.TotalHits()), err }
func (wr *WikipediaAbstractsReader) Read(r io.Reader) (<-chan index.Document, error) { dec := xml.NewDecoder(r) ch := make(chan index.Document) go func() { tok, err := dec.RawToken() props := map[string]string{} var currentText string for err != io.EOF { switch t := tok.(type) { case xml.CharData: if len(t) > 1 { currentText += string(t) } case xml.EndElement: name := t.Name.Local if name == "title" || name == "url" || name == "abstract" { props[name] = currentText } else if name == "doc" { id := path.Base(props["url"]) if len(id) > 0 { title := strings.TrimPrefix(strings.TrimSpace(props["title"]), "Wikipedia: ") body := strings.TrimSpace(props["abstract"]) //fmt.Println(title) if filter(title, body) { doc := index.NewDocument(id, wr.score(id)). Set("title", title). Set("body", body). Set("url", strings.TrimSpace(props["url"])) ch <- doc } } props = map[string]string{} } currentText = "" } tok, err = dec.RawToken() } fmt.Println("error: ", err) close(ch) }() return ch, nil }
// convert the result from a redis query to a proper Document object func loadDocument(id, sc, fields interface{}) (index.Document, error) { score, err := strconv.ParseFloat(string(sc.([]byte)), 64) if err != nil { return index.Document{}, fmt.Errorf("Could not parse score: %s", err) } doc := index.NewDocument(string(id.([]byte)), float32(score)) lst := fields.([]interface{}) for i := 0; i < len(lst); i += 2 { prop := string(lst[i].([]byte)) var val interface{} switch v := lst[i+1].(type) { case []byte: val = string(v) default: val = v } doc = doc.Set(prop, val) } return doc, nil }
func TestPaging(t *testing.T) { md := index.NewMetadata().AddField(index.NewTextField("title", 1.0)). AddField(index.NewNumericField("score")) idx := NewDistributedIndex("td", []string{"localhost:6379"}, 4, md) assert.NoError(t, idx.Drop()) assert.NoError(t, idx.Create()) N := 100 docs := make([]index.Document, 0, N) for i := 0; i < N; i++ { docs = append(docs, index.NewDocument(fmt.Sprintf("doc%d", i), float32(i)/100).Set("title", fmt.Sprintf("hello world title%d", i)).Set("score", i)) } assert.NoError(t, idx.Index(docs, nil)) q := query.NewQuery("td", "hello").Limit(10, 10) docs, total, err := idx.Search(*q) assert.NoError(t, err) assert.Len(t, docs, 10) assert.Equal(t, docs[0].Id, "doc89") assert.Equal(t, N, total) q = query.NewQuery("td", "title80").Limit(0, 1) docs, total, err = idx.Search(*q) assert.Len(t, docs, 1) assert.Equal(t, docs[0].Id, "doc80") assert.Equal(t, 1, total) q = query.NewQuery("td", "title80").Limit(5, 1) docs, total, err = idx.Search(*q) assert.NoError(t, err) assert.Len(t, docs, 0) }