コード例 #1
0
func NewProxy(min_pool int64, logger *log.WOFLogger) *Proxy {

	api_client := api.NewAPIClient()
	pool := pool.NewLIFOPool()

	// See notes in RefillPool() for details

	size := 10
	refill := make(chan bool, 10)

	for i := 0; i < size; i++ {
		refill <- true
	}

	// Possibly also keep global stats on number of fetches
	// and cache hits/misses/etc

	proxy := Proxy{
		logger:  logger,
		client:  api_client,
		pool:    pool,
		minpool: min_pool,
		refill:  refill,
	}

	return &proxy
}
コード例 #2
0
func main() {

	p := pool.NewLIFOPool()

	f := pool.PoolInt{Int: int64(123)}

	p.Push(f)
	v, _ := p.Pop()

	fmt.Printf("%d", v.IntValue())
}
コード例 #3
0
ファイル: s3.go プロジェクト: whosonfirst/go-whosonfirst-s3
func NewSync(creds *credentials.Credentials, region string, acl string, bucket string, prefix string, procs int, debug bool, logger *log.WOFLogger) *Sync {

	runtime.GOMAXPROCS(procs)

	workpool, _ := tunny.CreatePoolGeneric(procs).Open()
	retries := pool.NewLIFOPool()

	cfg := aws.NewConfig()
	cfg.WithRegion(region)

	if creds != nil {
		cfg.WithCredentials(creds)
	}

	sess := session.New(cfg)

	svc := s3.New(sess)

	ttp := new(time.Duration)

	return &Sync{
		Service:       svc,
		ACL:           acl,
		Bucket:        bucket,
		Prefix:        prefix,
		WorkPool:      *workpool,
		Debug:         debug,
		Dryrun:        false,
		Logger:        logger,
		Scheduled:     0,
		Completed:     0,
		Skipped:       0,
		Error:         0,
		Success:       0,
		Retried:       0,
		TimeToProcess: ttp,
		Retries:       retries,
		MaxRetries:    25.0, // maybe allow this to be user-defined ?
	}
}
コード例 #4
0
func NewWOFClone(source string, dest string, procs int, logger *log.WOFLogger) (*WOFClone, error) {

	// https://golang.org/src/net/http/filetransport.go

	u, err := url.Parse(source)

	if err != nil {
		return nil, err
	}

	var cl *http.Client

	if u.Scheme == "file" {

		root := u.Path

		if !strings.HasSuffix(root, "/") {
			root = root + "/"
		}

		/*
			Pay attention to what's going here. Absent tweaking the URL to
			fetch in the 'Fetch' method the following will not work. In
			order to make this working *without* tweaking the URL you would
			need to specifiy the root as '/' which just seems like a bad
			idea. The fear of blindly opening up the root level directory on
			the file system in this context may seem a bit premature (not to
			mention silly) but measure twice and all that good stuff...
			See also: https://code.google.com/p/go/issues/detail?id=2113
			(20160112/thisisaaronland)
		*/

		t := &http.Transport{}
		t.RegisterProtocol("file", http.NewFileTransport(http.Dir(root)))

		cl = &http.Client{Transport: t}
	} else {
		cl = &http.Client{}
	}

	runtime.GOMAXPROCS(procs)

	workpool, _ := tunny.CreatePoolGeneric(procs).Open()
	retries := pool.NewLIFOPool()

	/*

		This gets triggered in the 'Process' function to ensure that
		we don't exit out of 'CloneMetaFile' before all the goroutines
		to write new files to disk actually finish ... you know, writing
		to disk (20160606/thisisaaronland)
	*/

	writesync := new(sync.WaitGroup)

	ch := make(chan bool)

	c := WOFClone{
		Success:    0,
		Error:      0,
		Skipped:    0,
		Source:     source,
		Dest:       dest,
		Logger:     logger,
		MaxRetries: 25.0, // maybe allow this to be user-defined ?
		client:     cl,
		workpool:   workpool,
		writesync:  writesync,
		retries:    retries,
		timer:      time.Now(),
		done:       ch,
	}

	go func(c *WOFClone) {

		for {
			select {

			case <-c.done:
				break
			case <-time.After(1 * time.Second):
				c.Status()
			}
		}
	}(&c)

	return &c, nil
}