Golang NewUrlGetter Examples

Example #1

File: entrail.go Project: odeke-em/vidler

func getAndMatch(uri string, pattern *regexp.Regexp) chan *uriMatchChanPair {
	// TODO: Read in the value of n
	retries := uint32(3)

	getter := expb.NewUrlGetter(uri, retries)
	chanOPair := make(chan *uriMatchChanPair)

	go func() {
		expb.ExponentialBackOff(getter, func(resp interface{}, err error) {
			defer close(chanOPair)

			lines := responseStringer(resp, err)

			matchesChan := extractAndMatch(lines, pattern)
			uriChan := extractAndMatch(lines, uriRegCompile)

			chanOPair <- &uriMatchChanPair{uriChan: uriChan, matchesChan: matchesChan}
		})
	}()

	return chanOPair
}

Example #2

File: main.go Project: YuhsiangLu/drive

func main() {
	backer := expb.NewUrlGetter("https://golang.org/pkg/net/httpx", 5)
	expb.ExponentialBackOff(backer, consume)
	fmt.Println("expb")
}