Ejemplo n.º 1
0
// validPaths validates path to folders, if necessery then
// it will remover and recreate folders
func validPaths(conf *config) (err error) {
	workDir := conf.workDir
	tmpPrefix := conf.tmpPrefix
	removeFirst := !conf.sendOnly
	_, err = os.Stat(workDir)
	dirExists := err == nil
	if removeFirst && dirExists && !strings.HasPrefix(workDir, tmpPrefix) {
		return utils.Errorf("Dest folder '%s' already exists is not in tmp folder so need to be removed manually\n", workDir)
	}
	if removeFirst && dirExists {
		if err = os.RemoveAll(workDir); err != nil {
			return
		}
	}
	if err = os.MkdirAll(workDir, os.ModePerm); err != nil {
		return
	}
	return os.MkdirAll(filepath.Join(workDir, "images"), os.ModePerm)
}
Ejemplo n.º 2
0
func dumpAsXML(conf *config) (count int, err error) {
	workDir := conf.workDir
	exportId := fmt.Sprintf("%s_00.xml", time.Now().Format(DateFormat))
	exportXMLName := filepath.Join(workDir, exportId)

	// sync channels
	imagesChan = make(chan error)
	inOffersChan := make(chan *db.Offer, 5)
	outOffersChan := make(chan *xml.Offer)
	// connect to db
	dbConn := &db.DB{}
	if err = dbConn.Connect(DbUser, DbPass, DbHost, DbName); err != nil {
		err = utils.Errorf("Database connection %v", err)
		return
	}
	defer dbConn.Close()
	// specials are common for all offers
	specials := db.NewSpecials(dbConn)
	if err = specials.GetAll(); err != nil {
		return
	}
	specialsMap := specials.AsMap()
	// start workers
	for i := 0; i < noOfWorkers; i++ {
		go worker(inOffersChan, outOffersChan, specialsMap)
	}
	// start getting offers
	offers := db.NewOffers(dbConn)
	if err = offers.GetAllIds(); err != nil {
		return
	}
	go func() {
		for _, offerId := range offers.Ids() {
			offer := db.NewOffer(dbConn, offerId)
			inOffersChan <- offer
		}
	}()
	step := 25
	total := len(offers.Ids())
	counter := 0
	// collect offers from workers
	xmlOffers := xml.NewOffers()
	for range offers.Ids() {
		xmlOffer := <-outOffersChan
		xmlOffers.Add(xmlOffer)
		counter += 1
		if (counter % step) == 0 {
			log.Printf("Completed %d/%d\n", counter, total)
		}
	}
	log.Printf("Completed %d/%d\n", counter, total)
	log.Printf("Wait for images\n")
	// wait for all images finish download
	for range offers.Ids() {
		<-imagesChan
	}
	system := xml.NewSystem("1", "date", "hour", "www")
	head := xml.NewHeader(system, xmlOffers)
	if err = xml.Write(head, exportXMLName); err != nil {
		return
	}
	if err = ioutil.WriteFile(filepath.Join(workDir, "last.id"), []byte(exportId), os.ModePerm); err != nil {
		return
	}
	count = len(offers.Ids())
	return
}