func updateAuctionsByIdList(appCtx *lib.AppContext, idList []*backfillTarget) { aidriver := NewAuctionItemDriver(appCtx) assocdriver := NewAssocCrawlerKeywordAuctionItemDriver(appCtx) crawler := yauction.NewCrawler(appCtx.NewHttpClient()) auctionKeys := make([]*ds.Key, 0) auctions := make([]*yauction.AuctionItem, 0) assocKeys := make([]*ds.Key, 0) assocs := make([]*yauction.AssocCrawlerKeywordAuctionItem, 0) for _, target := range idList { auction, err := crawler.GetAuction(target.AuctionId) if err != nil { target.Error = err continue } auctionKeys = append(auctionKeys, aidriver.NewKey( auction.Id, 0, nil, )) auctions = append(auctions, auction) assocKeys = append(assocKeys, assocdriver.NewKey( fmt.Sprintf("%s-%s", target.Keyword, auction.Id), 0, nil, )) assocs = append(assocs, &yauction.AssocCrawlerKeywordAuctionItem{ Keyword: target.Keyword, AuctionId: auction.Id, }) } aidriver.PutMulti(auctionKeys, auctions) assocdriver.PutMulti(assocKeys, assocs) }
func NewShowDriver(appCtx *lib.AppContext) *ShowDriver { return &ShowDriver{ appCtx.NewDriver("Show"), NewEventDriver(appCtx), } }
func crawlClosedKeyword(appCtx *lib.AppContext, keyword string) ([]string, error) { var ckeyword yauction.CrawlerKeyword var logger = appCtx.Logger ckdriver := NewCrawlerKeywordDriver(appCtx) if err := ckdriver.Get( ckdriver.NewKey(keyword, 0, nil), &ckeyword, ); err != nil { return nil, err } iter := yauction.NewAuctionClosedSearchIterator( yauction.NewCrawler(appCtx.NewHttpClient()), keyword, ) logger.Info("Start Backfilling for %q", keyword) ids, err := iterateClosedSearchKeyword(iter, logger) if err != nil { return nil, err } logger.Info("Found %d AuctionIds", len(ids)) // Group by 20 updateGroups := make([][]*backfillTarget, 0) updateIdList := make([]*backfillTarget, 0) for _, id := range ids { updateIdList = append(updateIdList, &backfillTarget{ AuctionId: id, Keyword: keyword, }) if len(updateIdList) == 20 { updateGroups = append(updateGroups, updateIdList) updateIdList = make([]*backfillTarget, 0) } } if len(updateIdList) > 0 { updateGroups = append(updateGroups, updateIdList) } var wg sync.WaitGroup for i, group := range updateGroups { wg.Add(1) go func(j int, idList []*backfillTarget) { updateAuctionsByIdList(appCtx, idList) wg.Done() }(i, group) } wg.Wait() result := make([]string, 0) numErrors := 0 for _, group := range updateGroups { for _, target := range group { if target.Error != nil { numErrors += 1 } else { result = append(result, target.AuctionId) } } } logger.Info( "Finished to update %d auctions, %d errors.", len(result), numErrors, ) if numErrors > 0 { return result, fmt.Errorf("Keyword Crawling failed with %d errors", numErrors) } else { return result, nil } }
func crawlExpiredAuctions(appCtx *lib.AppContext) error { var auctions []yauction.AuctionItem var logger = appCtx.Logger _now := now() aidriver := NewAuctionItemDriver(appCtx) q := aidriver.NewQuery() q = q.Filter("EndAt <", _now).Filter("Status =", yauction.STATUS_OPEN).Limit(50) keys, err := q.GetAll(&auctions) if err != nil { return err } if len(auctions) == 0 { logger.Info("No auctions need to be checked") return nil } logger.Debug("Found %d expired auctions", len(auctions)) var wg sync.WaitGroup errorList := make([]error, len(auctions)) for i, auction := range auctions { wg.Add(1) go func(j int, origAuction *yauction.AuctionItem) { crawler := yauction.NewCrawler(appCtx.NewHttpClient()) _auction, err := crawler.GetAuction(auction.Id) if err == nil { if _auction.Status == yauction.STATUS_DELETED { origAuction.Status = yauction.STATUS_DELETED } else { origAuction.Title = _auction.Title origAuction.Url = _auction.Url origAuction.ImageUrl = _auction.ImageUrl origAuction.Seller = _auction.Seller origAuction.InitPrice = _auction.InitPrice origAuction.CurrentPrice = _auction.CurrentPrice origAuction.BidOrBuy = _auction.BidOrBuy origAuction.Quantity = _auction.Quantity origAuction.Bids = _auction.Bids origAuction.StartAt = _auction.StartAt origAuction.EndAt = _auction.EndAt origAuction.UpdatedAt = now() origAuction.Status = _auction.Status } _, err = aidriver.Put(keys[j], origAuction) } errorList[j] = err wg.Done() }(i, &auction) } wg.Wait() numErrors := 0 numUpdates := 0 for i, auction := range auctions { err = errorList[i] if err != nil { logger.Error( "An error occurred while updating an auction %q: %v", auction.Id, errorList[i], ) numErrors += 1 continue } else { numUpdates += 1 } } logger.Info( "Finished to update %d auctions, %d errors.", numUpdates, numErrors, ) if numErrors > 0 { return fmt.Errorf("Crawling expired auctions failed with %d errors", numErrors) } else { return nil } }
func crawlKeywords(appCtx *lib.AppContext) error { var keywords []yauction.CrawlerKeyword var logger = appCtx.Logger _now := now() ckdriver := NewCrawlerKeywordDriver(appCtx) q := ckdriver.NewQuery() q = q.Filter("EndAt >=", _now) _, err := q.GetAll(&keywords) if err != nil { return err } // TODO: optimize query result to remove keywords crawled within an hour. if len(keywords) == 0 { logger.Info("No keywords need to be checked") return nil } logger.Debug("%d Keywords start to crawl", len(keywords)) var wg sync.WaitGroup auctionList := make([][]*yauction.AuctionItem, len(keywords)) errorList := make([]error, len(keywords)) for i, keyword := range keywords { wg.Add(1) iter := yauction.NewAuctionSearchIterator( yauction.NewCrawler(appCtx.NewHttpClient()), keyword.Keyword, ) go func(j int, iter *yauction.AuctionSearchIterator) { list, err := iterateSearchKeyword(iter, logger) if err == yauction.ErrNoMoreIds { err = nil // Update Auctions aidriver := NewAuctionItemDriver(appCtx) assocdriver := NewAssocCrawlerKeywordAuctionItemDriver(appCtx) auctionKeys := make([]*ds.Key, len(list)) assocKeys := make([]*ds.Key, len(list)) assocs := make([]*yauction.AssocCrawlerKeywordAuctionItem, len(list)) for k, auction := range list { auctionKeys[k] = aidriver.NewKey(auction.Id, 0, nil) assocKeys[k] = assocdriver.NewKey( fmt.Sprintf("%s-%s", iter.Keyword, auction.Id), 0, nil, ) assocs[k] = &yauction.AssocCrawlerKeywordAuctionItem{ Keyword: iter.Keyword, AuctionId: auction.Id, } } _, err = aidriver.PutMulti(auctionKeys, list) if err == nil { _, err = assocdriver.PutMulti(assocKeys, assocs) if err == nil { auctionList[j] = list } } } errorList[j] = err wg.Done() }(i, iter) } wg.Wait() numErrors := 0 numUpdates := 0 for i, list := range auctionList { err = errorList[i] if err != nil { logger.Error( "An error occurred while crawling the keyword %q: %v", keywords[i].Keyword, errorList[i], ) numErrors += 1 continue } else { numUpdates += len(list) } } logger.Info( "Finished to crawl %d keywords. Found %d AuctionItems, %d errors.", len(keywords), numUpdates, numErrors, ) if numErrors > 0 { return fmt.Errorf("Keyword Crawling failed with %d errors", numErrors) } else { return nil } }
func NewAmebloEntryDriver(appCtx *lib.AppContext) *AmebloEntryDriver { return &AmebloEntryDriver{appCtx.NewDriver("AmebloEntry")} }
func NewEventDriver(appCtx *lib.AppContext) *EventDriver { return &EventDriver{appCtx.NewDriver("Event")} }
func NewCrawlerKeywordDriver(appCtx *lib.AppContext) *CrawlerKeywordDriver { return &CrawlerKeywordDriver{ appCtx.NewDriverWithNamespace("CrawlerKeyword", "yauction"), } }
func NewAuctionItemDriver(appCtx *lib.AppContext) *AuctionItemDriver { return &AuctionItemDriver{appCtx.NewDriverWithNamespace("AuctionItem", "yauction")} }
func NewAssocCrawlerKeywordAuctionItemDriver(appCtx *lib.AppContext) *AssocCrawlerKeywordAuctionItemDriver { return &AssocCrawlerKeywordAuctionItemDriver{ appCtx.NewDriverWithNamespace("AssocCrawlerKeywordAuctionItem", "yauction"), } }
func NewPostDriver(appCtx *lib.AppContext) *PostDriver { return &PostDriver{appCtx.NewDriver("Post")} }
func NewAmebloRefDriver(appCtx *lib.AppContext) *AmebloRefDriver { return &AmebloRefDriver{appCtx.NewDriver("AmebloRef")} }