Esempio n. 1
0
func testMatchFail(t *testing.T, target string, matchInputs ...string) {
	mm, err := mimetools.NewMatcher(matchInputs)
	if err != nil {
		t.Error(err)
	}

	q, err := mm.Match(target)
	if q {
		t.Errorf("Errantly matched against target: %v, inputs: %v", target, matchInputs)
	}
}
Esempio n. 2
0
// Start begins processing assuming that the datastore and any handlers have
// been set. This is a blocking call (run in a goroutine if you want to do
// other things)
//
// You cannot change the datastore or handlers after starting.
func (fm *FetchManager) Start() {
	log4go.Info("Starting FetchManager")
	if fm.Datastore == nil {
		panic("Cannot start a FetchManager without a datastore")
	}
	if fm.Handler == nil {
		panic("Cannot start a FetchManager without a handler")
	}
	if fm.started {
		panic("Cannot start a FetchManager multiple times")
	}

	mm, err := mimetools.NewMatcher(Config.AcceptFormats)
	fm.acceptFormats = mm
	if err != nil {
		panic(fmt.Errorf("mimetools.NewMatcher failed to initialize: %v", err))
	}

	fm.started = true

	if fm.Transport == nil {
		// Set fm.Transport == http.DefaultTransport, but create a new one; we
		// want to override Dial but don't want to globally override it in
		// http.DefaultTransport.
		fm.Transport = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 10 * time.Second,
		}
	}
	t, ok := fm.Transport.(*http.Transport)
	if ok {
		t.Dial = DNSCachingDial(t.Dial, Config.MaxDNSCacheEntries)
	} else {
		log4go.Info("Given an non-http transport, not using dns caching")
	}

	numFetchers := Config.NumSimultaneousFetchers
	fm.fetchers = make([]*fetcher, numFetchers)
	for i := 0; i < numFetchers; i++ {
		f := newFetcher(fm)
		fm.fetchers[i] = f
		fm.fetchWait.Add(1)
		go func() {
			f.start()
			fm.fetchWait.Done()
		}()
	}
	fm.fetchWait.Wait()
}