func TestPriorityChannel(t *testing.T) {
go timeoutTest(t, 11*time.Second)
p := newPriorityChannel()
done := make(chan struct{})
u1, _ := url.Parse("http://www.google.fr/1")
u2, _ := url.Parse("http://www.google.fr/2")
u3, _ := url.Parse("http://www.yahoo.fr/1")
u4, _ := url.Parse("http://www.yahoo.fr/2")
u5, _ := url.Parse("http://www.bing.fr/1")
u6, _ := url.Parse("http://www.bing.fr/2")
go func() {
for u := range p.Out {
hostName, _ := url.StripWWW(u.Url.Host)
t.Log(u, hostName)
p.doneWithHost(hostName)
}
done <- struct{}{}
}()
p.addUrl(frontier.UrlRequest{*u1, frontier.NoopCtxFunc}, 2)
p.addUrl(frontier.UrlRequest{*u2, frontier.NoopCtxFunc}, 2)
p.addUrl(frontier.UrlRequest{*u3, frontier.NoopCtxFunc}, 2)
p.addUrl(frontier.UrlRequest{*u4, frontier.NoopCtxFunc}, 2)
p.addUrl(frontier.UrlRequest{*u5, frontier.NoopCtxFunc}, 2)
p.addUrl(frontier.UrlRequest{*u6, frontier.NoopCtxFunc}, 2)
p.Stop()
<-done
}
func (s *Store) HostIgnoreWWW(hostName string) error {
symbHost, hadWWW := url.StripWWW(hostName)
if !hadWWW {
symbHost = "www." + hostName
}
return s.createSymbHostLink(hostName, symbHost)
}
// set options attached to an host :
//
// - how many concurrent connections are allowed
//
// - delay between connections
func (c *Limiter) SetHostOptions(hostName string, accessDelay time.Duration, maxConnections int) {
hostName, _ = url.StripWWW(hostName)
pool := c.getHostPool(hostName)
pool.accessDelay = accessDelay
pool.maxToken = maxConnections
pool.token = maxConnections
}
func (c *Limiter) getHostPool(host string) *hostPool {
host, _ = url.StripWWW(host)
if c.hosts[host] == nil {
c.hosts[host] = newHostPool(20, c.defaultAccessDelay, c.maxConnectionsPerHost)
}
return c.hosts[host]
}
func (c *Limiter) checkHost(hostName string) {
hostName, _ = url.StripWWW(hostName)
host := c.hosts[hostName]
if !host.IsEmpty() && host.GetToken() {
req := host.waitingRequest.Pop()
req.ready <- struct{}{}
}
}
func (p *priorityChannel) newHost(hostName string) *hostInfo {
h := hostInfo{name: hostName, priority: 9, delay: defaultDelay}
hostStrip, _ := url.StripWWW(hostName)
if d, ok := p.hostDelay[hostStrip]; ok {
h.delay = d
}
heap.Push(p.OutQueue, &h)
return &h
}
func (p *priorityChannel) addUrl(request frontier.UrlRequest, priority int) {
p.Lock()
defer p.Unlock()
hostName, _ := url.StripWWW(request.Url.Host)
host := p.OutQueue.getHost(hostName)
if host == nil {
p.count += 1
host = p.newHost(hostName)
p.dchan.pushElement(delayedElement{hostName, time.Now()})
}
host.urls[priority] = append(host.urls[priority], request)
host.count += 1
p.count += 1
if priority < host.priority {
p.OutQueue.UpdatePriority(hostName, priority)
}
}
func bindHostToCtx(ctx context.Context, host string) context.Context {
host, _ = url.StripWWW(host)
return context.WithValue(ctx, ctxKey, host)
}
func (p *priorityChannel) SetHostDelay(hostName string, delay time.Duration) {
hostName, _ = url.StripWWW(hostName)
p.hostDelay[hostName] = delay
}