func main() {
flag.Parse()
var bucketname string
// TODO, remove channel? not using it or waiter
uploads := make(chan FileUpload, 1)
waiter := &sync.WaitGroup{}
directory := flag.Arg(0)
if *bucket == "" {
bucketname = directory
} else {
bucketname = *bucket
}
fmt.Println("Uploading to bucket named: ", bucketname)
fmt.Println("Publicly visible:", *public)
s3bucket := sss.GetBucket(sss.Auth(), sss.Region, bucketname)
err := filepath.Walk(directory, makeVisitor(uploads, s3bucket, waiter, args{
force: *force,
mimetype: *mimetype,
newer: *newer,
newermetamtime: *newermetamtime,
prefix: *prefix,
public: *public,
sse: *sse,
}))
if err != nil {
fmt.Println(err)
os.Exit(1)
}
waiter.Wait()
// fmt.Printf("filepatxh.Walk() returned %v\n", err)
}
func main() {
log.Println("Starting")
runtime.GOMAXPROCS(12) //runtime.NumCPU())
jobs := make(chan Job, 1)
done := make(chan bool)
err := db.Dial()
if err != nil {
panic(err)
}
errors := db.ErrorCollection()
urls := db.UrlCollection()
uc, _ := urls.Count()
resources := db.ResourceCollection()
allurls := db.AllUrls()
result := db.Url{}
s3bucket := sss.GetBucket(auth(), sss.Region, sss.BucketName)
log.Printf("Got bucket: %v\n", s3bucket.Name)
go func() {
counter := 1
processed := 1
for allurls.Next(&result) {
seen := db.Seen(result.Url)
counter += 1
if seen == false {
job := Job{}
j, r := fetcher.Get(result.Url)
job.UrlInfo = j
job.Body = r
jobs <- job
processed += 1
}
}
fmt.Printf("goroutine counted: %i processed unseen: %i\n", counter, processed)
}()
for i := 0; i < workers; i++ {
go func() {
for job := range jobs {
if job.UrlInfo.Status_Code == 200 {
err = s3bucket.Put(
job.UrlInfo.Path, job.Body, job.UrlInfo.Content_Type, s3.PublicRead)
if err != nil {
log.Printf("Failed to put file for: %s\nError%v\n", job.UrlInfo.Url, err)
//log.Printf("JOB %v\n", job.Body)
errors.Insert(&job.UrlInfo)
} else {
err = resources.Insert(&job.UrlInfo)
if err != nil {
fmt.Printf("%s\n", err)
}
}
} else {
err = resources.Insert(&job.UrlInfo)
if err != nil {
fmt.Printf("%s\n", err)
}
}
}
}()
done <- true
}
for i := 0; i < workers; i++ {
// block until all workers are done
<-done
}
log.Println(uc)
log.Println("Finished")
}