Example #1
0
File: main.go Project: schorlet/cdc
func main() {
	log.SetFlags(0)

	var cmd, url, addr, cachedir string
	parseArgs(&cmd, &url, &addr, &cachedir)

	cache, err := cdc.OpenCache(cachedir)
	if err != nil {
		log.Fatal(err)
	}

	if cmd == "list" {
		for _, url := range cache.URLs() {
			fmt.Printf("%d %s\n", cache.GetAddr(url), url)
		}

	} else {
		entry := openEntry(cache, url, addr, cachedir)

		if cmd == "header" {
			printHeader(entry)

		} else if cmd == "body" {
			printBody(entry)

		} else {
			log.Fatalf("unknown command: %s", cmd)
		}
	}
}
Example #2
0
func withContext(fn func(url string)) {
	cache, err := cdc.OpenCache("../../testcache")
	if err != nil {
		log.Fatal(err)
	}

	handler := CacheHandler(cache)
	server := httptest.NewServer(handler)
	defer server.Close()

	fn(server.URL)
}
Example #3
0
func TestCrawl(t *testing.T) {
	cache, err := cdc.OpenCache("testcache")
	if err != nil {
		t.Fatal(err)
	}

	_, err = cache.OpenURL("http://foo.com")
	if err != cdc.ErrNotFound {
		t.Fatalf("got: %v, want: %v", err, cdc.ErrNotFound)
	}

	for _, url := range cache.URLs() {
		addr := cache.GetAddr(url)
		if !addr.Initialized() {
			t.Fatal("got: addr not initialized")
		}

		entry, err := cache.OpenURL(url)
		if err != nil {
			t.Fatal(err)
		}
		if url != entry.URL() {
			t.Fatalf("got: %s, want: %s", entry.URL(), url)
		}

		header, err := entry.Header()
		if err != nil {
			t.Fatal(err)
		}
		if len(header) == 0 {
			t.Fatal("got: empty header")
		}
		clength := header.Get("Content-Length")
		nlength, err := strconv.ParseInt(clength, 10, 64)
		if err != nil {
			t.Fatal(err)
		}

		body, err := entry.Body()
		if err != nil {
			t.Fatal(err)
		}
		n, err := io.Copy(ioutil.Discard, body)
		if err != nil {
			t.Fatal(err)
		}
		body.Close()
		if n != nlength {
			t.Fatalf("got: %d, want: %d", n, nlength)
		}
	}
}
Example #4
0
// Example gets an entry from the cache and prints to stdout.
func Example() {
	cache, err := cdc.OpenCache("testcache")
	if err != nil {
		log.Fatal(err)
	}

	entry, err := cache.OpenURL("https://golang.org/doc/gopher/pkg.png")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(entry.URL())

	header, err := entry.Header()
	if err != nil {
		log.Fatal(err)
	}
	for _, key := range []string{"Status", "Content-Length", "Content-Type"} {
		fmt.Printf("%s: %s\n", key, header.Get(key))
	}

	body, err := entry.Body()
	if err != nil {
		log.Fatal(err)
	}
	defer body.Close()

	config, err := png.DecodeConfig(body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("PNG image data, %d x %d\n", config.Width, config.Height)

	// Output:
	// https://golang.org/doc/gopher/pkg.png
	// Status: 200
	// Content-Length: 5409
	// Content-Type: image/png
	// PNG image data, 83 x 120
}
Example #5
0
File: main.go Project: schorlet/cdc
func main() {
	if len(os.Args) != 2 {
		log.SetFlags(0)
		log.Fatal(usage)
	}

	cache, err := cdc.OpenCache(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}

	handler := CacheHandler(cache)
	http.Handle("/", handler)

	http.HandleFunc("/favicon.ico", http.NotFound)
	http.HandleFunc("/favicon.png", http.NotFound)
	http.HandleFunc("/opensearch.xml", http.NotFound)

	err = http.ListenAndServe(":8000", nil)
	if err != nil {
		log.Fatal(err)
	}
}