Beispiel #1
0
func main() {
	flag.Parse()
	var bucketname string

	// TODO, remove channel? not using it or waiter
	uploads := make(chan FileUpload, 1)
	waiter := &sync.WaitGroup{}

	directory := flag.Arg(0)

	if *bucket == "" {
		bucketname = directory
	} else {
		bucketname = *bucket
	}

	fmt.Println("Uploading to bucket named: ", bucketname)
	fmt.Println("Publicly visible:", *public)
	s3bucket := sss.GetBucket(sss.Auth(), sss.Region, bucketname)

	err := filepath.Walk(directory, makeVisitor(uploads, s3bucket, waiter, args{
		force:          *force,
		mimetype:       *mimetype,
		newer:          *newer,
		newermetamtime: *newermetamtime,
		prefix:         *prefix,
		public:         *public,
		sse:            *sse,
	}))

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	waiter.Wait()
	// fmt.Printf("filepatxh.Walk() returned %v\n", err)
}
Beispiel #2
0
func main() {
	log.Println("Starting")

	runtime.GOMAXPROCS(12) //runtime.NumCPU())

	jobs := make(chan Job, 1)
	done := make(chan bool)

	err := db.Dial()
	if err != nil {
		panic(err)
	}
	errors := db.ErrorCollection()
	urls := db.UrlCollection()
	uc, _ := urls.Count()
	resources := db.ResourceCollection()
	allurls := db.AllUrls()
	result := db.Url{}
	s3bucket := sss.GetBucket(auth(), sss.Region, sss.BucketName)
	log.Printf("Got bucket: %v\n", s3bucket.Name)
	go func() {
		counter := 1
		processed := 1
		for allurls.Next(&result) {
			seen := db.Seen(result.Url)
			counter += 1
			if seen == false {
				job := Job{}
				j, r := fetcher.Get(result.Url)
				job.UrlInfo = j
				job.Body = r
				jobs <- job
				processed += 1
			}
		}
		fmt.Printf("goroutine counted: %i processed unseen: %i\n", counter, processed)
	}()

	for i := 0; i < workers; i++ {
		go func() {
			for job := range jobs {
				if job.UrlInfo.Status_Code == 200 {
					err = s3bucket.Put(
						job.UrlInfo.Path, job.Body, job.UrlInfo.Content_Type, s3.PublicRead)
					if err != nil {
						log.Printf("Failed to put file for: %s\nError%v\n", job.UrlInfo.Url, err)
						//log.Printf("JOB %v\n", job.Body)
						errors.Insert(&job.UrlInfo)
					} else {
						err = resources.Insert(&job.UrlInfo)
						if err != nil {
							fmt.Printf("%s\n", err)
						}
					}
				} else {
					err = resources.Insert(&job.UrlInfo)
					if err != nil {
						fmt.Printf("%s\n", err)
					}
				}
			}
		}()
		done <- true
	}
	for i := 0; i < workers; i++ {
		// block until all workers are done
		<-done
	}
	log.Println(uc)
	log.Println("Finished")
}