/
blackbox_test.go
78 lines (65 loc) · 1.68 KB
/
blackbox_test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
package cybergrab_test
import (
"github.com/nightowlware/cybergrab"
"fmt"
"log"
"os"
"os/signal"
"runtime"
"strings"
"syscall"
"testing"
)
type simpleCrawlPolicy struct{}
func (scp simpleCrawlPolicy) ShouldDownload(url string) bool {
return strings.HasSuffix(url, "jpg") ||
strings.HasSuffix(url, "gif") ||
strings.HasSuffix(url, "pdg")
}
func (scp simpleCrawlPolicy) ShouldCrawl(url string) bool {
return !scp.ShouldDownload(url)
}
/////////////////////////////////////////////////////////////
// TestMain is a setup function for the rest of the
// tests in this file.
func TestMain(m *testing.M) {
// setup a signal handler so that "Ctrl-\" shows a stack-trace.
go func() {
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGQUIT)
buf := make([]byte, 1<<20)
for {
<-sigs
runtime.Stack(buf, true)
log.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf)
}
}()
os.Exit(m.Run())
}
func TestMaxGoRoutines(t *testing.T) {
var spider cybergrab.Spider
spider, err := cybergrab.NewSpider(simpleCrawlPolicy{}, cybergrab.MAX_DOWNLOADS, cybergrab.MAX_WORKERS)
if err == nil {
spider.Crawl("http://www.salami.com")
} else {
fmt.Println(err)
}
}
func TestMaxWorkers(t *testing.T) {
var spider cybergrab.Spider
spider, err := cybergrab.NewSpider(simpleCrawlPolicy{}, 100, cybergrab.MAX_WORKERS)
if err == nil {
spider.Crawl("http://www.quickmeme.com")
} else {
fmt.Println(err)
}
}
func TestMaxDownloads(t *testing.T) {
var spider cybergrab.Spider
spider, err := cybergrab.NewSpider(simpleCrawlPolicy{}, cybergrab.MAX_DOWNLOADS, 1000)
if err == nil {
spider.Crawl("http://www.spam.com")
} else {
fmt.Println(err)
}
}