Beispiel #1
0
func (ie *ItemElastic) MultiGet(indexType string, itemIds []string) ([]*model.Item, error) {
	endpoint, err := ie.funcEndpoint(ie.index, indexType, "_mget")
	result := make([]*model.Item, len(itemIds))

	if err != nil {
		return nil, err
	}

	mget := map[string][]string{
		"ids": itemIds,
	}

	response := struct {
		Docs []ElasticModel `json:"docs"`
	}{}

	err = ie.handler.SendRAW("GET", endpoint, &mget, &response)
	if err != nil {
		return nil, err
	}

	eitem := response.Docs
	for i, _ := range eitem {
		if eitem[i].Found {
			var item model.Item
			err = NewModelFromRaw(eitem[i].Source, &item)
			item.Id = eitem[i].Id
			item.Version = eitem[i].Version
			result[i] = &item
		}
	}

	return result, nil

}
Beispiel #2
0
func (ie *ItemElastic) Get(indexType string, itemId string) (*model.Item, error) {
	var item model.Item

	endpoint, err := ie.funcEndpoint(ie.index, indexType, itemId)
	if err != nil {
		return nil, err
	}

	es, err := ie.handler.Get(endpoint)
	if err != nil {
		return nil, err
	}

	err = NewModelFromRaw(es.Source, &item)
	if err != nil {
		return nil, err
	}

	// populate Id and Version from ElasticSearch
	item.Id = es.Id
	item.Version = es.Version

	return &item, nil
}
Beispiel #3
0
// Scrapping logic from the document
func DocumentScrap(jobId string, selector ScrapSelector, doc *goquery.Document, items chan ItemResult) {
	rdata := NewRedisScrapdata()

	defer func() {
		if r := recover(); r != nil {
			log.Printf("ERROR: DocumentScrap Panic applying selectors: '%v'", r)
			rdata.ScrapLogWrite("ERROR: bad CSS Selector, please review the syntax")
		}
	}()

	sel := doc.Find(selector.Base)
	for i := range sel.Nodes {
		s := sel.Eq(i)
		var err error
		item := model.Item{}
		item.ScrapUrl = selector.Url
		item.ScrapTags = selector.ScrapTags

		item.Link = SanitizeURL(item.ScrapUrl, extractText(s, selector.Link), selector.LinkPathLimit)
		item.Id, err = extractId(s, selector, item.Link)
		item.Image = SanitizeURL(item.ScrapUrl, extractText(s, selector.Image), 0)
		item.Title = extractText(s, selector.Title)
		item.Description = extractText(s, selector.Description)
		item.Price = extractFloat(s, selector.Price)
		item.Currency = extractCurrency(s, selector.Price)
		item.Stars = extractFloat(s, selector.Stars)
		item.Categories = extractText(s, selector.Categories)

		item.LastScrap = time.Now().Format(time.RFC3339)

		items <- ItemResult{
			JobId: jobId,
			Item:  item,
			Err:   err,
		}
	}

}