func (ie *ItemElastic) MultiGet(indexType string, itemIds []string) ([]*model.Item, error) {
endpoint, err := ie.funcEndpoint(ie.index, indexType, "_mget")
result := make([]*model.Item, len(itemIds))
if err != nil {
return nil, err
}
mget := map[string][]string{
"ids": itemIds,
}
response := struct {
Docs []ElasticModel `json:"docs"`
}{}
err = ie.handler.SendRAW("GET", endpoint, &mget, &response)
if err != nil {
return nil, err
}
eitem := response.Docs
for i, _ := range eitem {
if eitem[i].Found {
var item model.Item
err = NewModelFromRaw(eitem[i].Source, &item)
item.Id = eitem[i].Id
item.Version = eitem[i].Version
result[i] = &item
}
}
return result, nil
}
func (ie *ItemElastic) Get(indexType string, itemId string) (*model.Item, error) {
var item model.Item
endpoint, err := ie.funcEndpoint(ie.index, indexType, itemId)
if err != nil {
return nil, err
}
es, err := ie.handler.Get(endpoint)
if err != nil {
return nil, err
}
err = NewModelFromRaw(es.Source, &item)
if err != nil {
return nil, err
}
// populate Id and Version from ElasticSearch
item.Id = es.Id
item.Version = es.Version
return &item, nil
}
// Scrapping logic from the document
func DocumentScrap(jobId string, selector ScrapSelector, doc *goquery.Document, items chan ItemResult) {
rdata := NewRedisScrapdata()
defer func() {
if r := recover(); r != nil {
log.Printf("ERROR: DocumentScrap Panic applying selectors: '%v'", r)
rdata.ScrapLogWrite("ERROR: bad CSS Selector, please review the syntax")
}
}()
sel := doc.Find(selector.Base)
for i := range sel.Nodes {
s := sel.Eq(i)
var err error
item := model.Item{}
item.ScrapUrl = selector.Url
item.ScrapTags = selector.ScrapTags
item.Link = SanitizeURL(item.ScrapUrl, extractText(s, selector.Link), selector.LinkPathLimit)
item.Id, err = extractId(s, selector, item.Link)
item.Image = SanitizeURL(item.ScrapUrl, extractText(s, selector.Image), 0)
item.Title = extractText(s, selector.Title)
item.Description = extractText(s, selector.Description)
item.Price = extractFloat(s, selector.Price)
item.Currency = extractCurrency(s, selector.Price)
item.Stars = extractFloat(s, selector.Stars)
item.Categories = extractText(s, selector.Categories)
item.LastScrap = time.Now().Format(time.RFC3339)
items <- ItemResult{
JobId: jobId,
Item: item,
Err: err,
}
}
}