예제 #1
0
파일: client.go 프로젝트: fanyang01/crawler
// NewStdClient returns a new standard client that uses the provided HTTP
// client to do HTTP requests. cacheSize specifies the maxmium size(in
// bytes) of the HTTP cache pool. If cacheSize <= 0, HTTP caching will be
// disabled.
func NewStdClient(client *http.Client, cacheSize int) *StdClient {
	if client == nil {
		client = DefaultHTTPClient
	}
	c := &StdClient{client: client}
	if cacheSize > 0 {
		c.cache = cache.NewPool(cacheSize)
	}
	return c
}
예제 #2
0
func TestClientCache(t *testing.T) {
	assert := assert.New(t)
	checkErr := func(err error) {
		if err != nil {
			t.Error(err)
			t.FailNow()
		}
	}
	client := &StdClient{
		client: &http.Client{},
		cache:  cache.NewPool(1 << 20),
	}

	magic := "00000"
	count := 0
	notModified := 0
	mux := http.NewServeMux()
	mux.Handle("/normal", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		count++
		if etag := r.Header.Get("If-None-Match"); etag == magic {
			notModified++
			w.WriteHeader(http.StatusNotModified)
			return
		}
		w.Header().Set("Cache-Control", "max-age=1")
		w.Header().Set("ETag", magic)
		fmt.Fprint(w, magic)
	}))
	mux.Handle("/revalidate", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		count++
		if etag := r.Header.Get("If-None-Match"); etag == magic {
			notModified++
			w.WriteHeader(http.StatusNotModified)
			return
		}
		w.Header().Set("Cache-Control", "no-cache")
		w.Header().Set("ETag", magic)
		fmt.Fprint(w, magic)
	}))
	ts := httptest.NewServer(mux)

	var cnt int
	var notMod int
	f := func(u string, content string, server bool, nomod bool) {
		hreq, err := http.NewRequest("GET", ts.URL+u, nil)
		checkErr(err)
		req := &Request{
			Request: hreq,
		}
		r, err := client.Do(req)
		checkErr(err)
		b, _ := ioutil.ReadAll(r.Body)
		assert.Equal(content, string(b))
		if server {
			cnt++
			if nomod {
				notMod++
			}
		}
		assert.Equal(count, cnt)
		assert.Equal(notModified, notMod)
	}

	f("/normal", magic, true, false)
	f("/normal", magic, false, false) // should access cache rather than server
	f("/revalidate", magic, true, false)
	f("/revalidate", magic, true, true)
	time.Sleep(500 * time.Millisecond)
	magic = "22222"
	f("/normal", "00000", false, false)
	f("/revalidate", magic, true, false)
	time.Sleep(500 * time.Millisecond)
	f("/normal", magic, true, false)
	f("/normal", magic, false, false)
	f("/revalidate", magic, true, true)
	f("/revalidate", magic, true, true)
}