func NewProxy(min_pool int64, logger *log.WOFLogger) *Proxy {
api_client := api.NewAPIClient()
pool := pool.NewLIFOPool()
// See notes in RefillPool() for details
size := 10
refill := make(chan bool, 10)
for i := 0; i < size; i++ {
refill <- true
}
// Possibly also keep global stats on number of fetches
// and cache hits/misses/etc
proxy := Proxy{
logger: logger,
client: api_client,
pool: pool,
minpool: min_pool,
refill: refill,
}
return &proxy
}
func main() {
p := pool.NewLIFOPool()
f := pool.PoolInt{Int: int64(123)}
p.Push(f)
v, _ := p.Pop()
fmt.Printf("%d", v.IntValue())
}
func NewSync(creds *credentials.Credentials, region string, acl string, bucket string, prefix string, procs int, debug bool, logger *log.WOFLogger) *Sync {
runtime.GOMAXPROCS(procs)
workpool, _ := tunny.CreatePoolGeneric(procs).Open()
retries := pool.NewLIFOPool()
cfg := aws.NewConfig()
cfg.WithRegion(region)
if creds != nil {
cfg.WithCredentials(creds)
}
sess := session.New(cfg)
svc := s3.New(sess)
ttp := new(time.Duration)
return &Sync{
Service: svc,
ACL: acl,
Bucket: bucket,
Prefix: prefix,
WorkPool: *workpool,
Debug: debug,
Dryrun: false,
Logger: logger,
Scheduled: 0,
Completed: 0,
Skipped: 0,
Error: 0,
Success: 0,
Retried: 0,
TimeToProcess: ttp,
Retries: retries,
MaxRetries: 25.0, // maybe allow this to be user-defined ?
}
}
func NewWOFClone(source string, dest string, procs int, logger *log.WOFLogger) (*WOFClone, error) {
// https://golang.org/src/net/http/filetransport.go
u, err := url.Parse(source)
if err != nil {
return nil, err
}
var cl *http.Client
if u.Scheme == "file" {
root := u.Path
if !strings.HasSuffix(root, "/") {
root = root + "/"
}
/*
Pay attention to what's going here. Absent tweaking the URL to
fetch in the 'Fetch' method the following will not work. In
order to make this working *without* tweaking the URL you would
need to specifiy the root as '/' which just seems like a bad
idea. The fear of blindly opening up the root level directory on
the file system in this context may seem a bit premature (not to
mention silly) but measure twice and all that good stuff...
See also: https://code.google.com/p/go/issues/detail?id=2113
(20160112/thisisaaronland)
*/
t := &http.Transport{}
t.RegisterProtocol("file", http.NewFileTransport(http.Dir(root)))
cl = &http.Client{Transport: t}
} else {
cl = &http.Client{}
}
runtime.GOMAXPROCS(procs)
workpool, _ := tunny.CreatePoolGeneric(procs).Open()
retries := pool.NewLIFOPool()
/*
This gets triggered in the 'Process' function to ensure that
we don't exit out of 'CloneMetaFile' before all the goroutines
to write new files to disk actually finish ... you know, writing
to disk (20160606/thisisaaronland)
*/
writesync := new(sync.WaitGroup)
ch := make(chan bool)
c := WOFClone{
Success: 0,
Error: 0,
Skipped: 0,
Source: source,
Dest: dest,
Logger: logger,
MaxRetries: 25.0, // maybe allow this to be user-defined ?
client: cl,
workpool: workpool,
writesync: writesync,
retries: retries,
timer: time.Now(),
done: ch,
}
go func(c *WOFClone) {
for {
select {
case <-c.done:
break
case <-time.After(1 * time.Second):
c.Status()
}
}
}(&c)
return &c, nil
}