// Scrapping logic from the document func DocumentScrap(jobId string, selector ScrapSelector, doc *goquery.Document, items chan ItemResult) { rdata := NewRedisScrapdata() defer func() { if r := recover(); r != nil { log.Printf("ERROR: DocumentScrap Panic applying selectors: '%v'", r) rdata.ScrapLogWrite("ERROR: bad CSS Selector, please review the syntax") } }() sel := doc.Find(selector.Base) for i := range sel.Nodes { s := sel.Eq(i) var err error item := model.Item{} item.ScrapUrl = selector.Url item.ScrapTags = selector.ScrapTags item.Link = SanitizeURL(item.ScrapUrl, extractText(s, selector.Link), selector.LinkPathLimit) item.Id, err = extractId(s, selector, item.Link) item.Image = SanitizeURL(item.ScrapUrl, extractText(s, selector.Image), 0) item.Title = extractText(s, selector.Title) item.Description = extractText(s, selector.Description) item.Price = extractFloat(s, selector.Price) item.Currency = extractCurrency(s, selector.Price) item.Stars = extractFloat(s, selector.Stars) item.Categories = extractText(s, selector.Categories) item.LastScrap = time.Now().Format(time.RFC3339) items <- ItemResult{ JobId: jobId, Item: item, Err: err, } } }