func main() {
log.SetFlags(0)
var cmd, url, addr, cachedir string
parseArgs(&cmd, &url, &addr, &cachedir)
cache, err := cdc.OpenCache(cachedir)
if err != nil {
log.Fatal(err)
}
if cmd == "list" {
for _, url := range cache.URLs() {
fmt.Printf("%d %s\n", cache.GetAddr(url), url)
}
} else {
entry := openEntry(cache, url, addr, cachedir)
if cmd == "header" {
printHeader(entry)
} else if cmd == "body" {
printBody(entry)
} else {
log.Fatalf("unknown command: %s", cmd)
}
}
}
func withContext(fn func(url string)) {
cache, err := cdc.OpenCache("../../testcache")
if err != nil {
log.Fatal(err)
}
handler := CacheHandler(cache)
server := httptest.NewServer(handler)
defer server.Close()
fn(server.URL)
}
func TestCrawl(t *testing.T) {
cache, err := cdc.OpenCache("testcache")
if err != nil {
t.Fatal(err)
}
_, err = cache.OpenURL("http://foo.com")
if err != cdc.ErrNotFound {
t.Fatalf("got: %v, want: %v", err, cdc.ErrNotFound)
}
for _, url := range cache.URLs() {
addr := cache.GetAddr(url)
if !addr.Initialized() {
t.Fatal("got: addr not initialized")
}
entry, err := cache.OpenURL(url)
if err != nil {
t.Fatal(err)
}
if url != entry.URL() {
t.Fatalf("got: %s, want: %s", entry.URL(), url)
}
header, err := entry.Header()
if err != nil {
t.Fatal(err)
}
if len(header) == 0 {
t.Fatal("got: empty header")
}
clength := header.Get("Content-Length")
nlength, err := strconv.ParseInt(clength, 10, 64)
if err != nil {
t.Fatal(err)
}
body, err := entry.Body()
if err != nil {
t.Fatal(err)
}
n, err := io.Copy(ioutil.Discard, body)
if err != nil {
t.Fatal(err)
}
body.Close()
if n != nlength {
t.Fatalf("got: %d, want: %d", n, nlength)
}
}
}
// Example gets an entry from the cache and prints to stdout.
func Example() {
cache, err := cdc.OpenCache("testcache")
if err != nil {
log.Fatal(err)
}
entry, err := cache.OpenURL("https://golang.org/doc/gopher/pkg.png")
if err != nil {
log.Fatal(err)
}
fmt.Println(entry.URL())
header, err := entry.Header()
if err != nil {
log.Fatal(err)
}
for _, key := range []string{"Status", "Content-Length", "Content-Type"} {
fmt.Printf("%s: %s\n", key, header.Get(key))
}
body, err := entry.Body()
if err != nil {
log.Fatal(err)
}
defer body.Close()
config, err := png.DecodeConfig(body)
if err != nil {
log.Fatal(err)
}
fmt.Printf("PNG image data, %d x %d\n", config.Width, config.Height)
// Output:
// https://golang.org/doc/gopher/pkg.png
// Status: 200
// Content-Length: 5409
// Content-Type: image/png
// PNG image data, 83 x 120
}
func main() {
if len(os.Args) != 2 {
log.SetFlags(0)
log.Fatal(usage)
}
cache, err := cdc.OpenCache(os.Args[1])
if err != nil {
log.Fatal(err)
}
handler := CacheHandler(cache)
http.Handle("/", handler)
http.HandleFunc("/favicon.ico", http.NotFound)
http.HandleFunc("/favicon.png", http.NotFound)
http.HandleFunc("/opensearch.xml", http.NotFound)
err = http.ListenAndServe(":8000", nil)
if err != nil {
log.Fatal(err)
}
}
