//
// Generate Fixtures
//
func spoofData() {
if console.DS != nil {
console.DS.Close()
console.DS = nil
}
modifyConfigControllers()
console.SpoofData()
ds, err := console.NewCqlModel()
if err != nil {
panic(fmt.Errorf("Failed to start data source: %v", err))
}
console.DS = ds
console.BuildRender()
}
func getDs(t *testing.T) *console.CqlModel {
modifyConfigDataSource()
initdb.Do(func() {
cluster := gocql.NewCluster(walker.Config.Cassandra.Hosts...)
db, err := cluster.CreateSession()
if err != nil {
panic(err)
}
// Just want to make sure no one makes a mistake with this code
if walker.Config.Cassandra.Keyspace == "walker" {
panic("Not allowed to spoof the walker keyspace")
}
err = db.Query(fmt.Sprintf("DROP KEYSPACE IF EXISTS %s", walker.Config.Cassandra.Keyspace)).Exec()
if err != nil {
panic(fmt.Errorf("Failed to drop %s keyspace: %v", walker.Config.Cassandra.Keyspace, err))
}
err = walker.CreateCassandraSchema()
if err != nil {
t.Fatalf(err.Error())
}
db.Close()
})
ds, err := console.NewCqlModel()
if err != nil {
panic(err)
}
db := ds.Db
//
// Clear out the tables first
//
tables := []string{"links", "segments", "domain_info"}
for _, table := range tables {
err := db.Query(fmt.Sprintf(`TRUNCATE %v`, table)).Exec()
if err != nil {
t.Fatalf("Failed to truncate table %v: %v", table, err)
}
}
//
// Insert some data
//
insertDomainInfo := `INSERT INTO domain_info (dom) VALUES (?)`
insertDomainToCrawl := `INSERT INTO domain_info (dom, claim_tok, claim_time, dispatched) VALUES (?, ?, ?, true)`
insertSegment := `INSERT INTO segments (dom, subdom, path, proto) VALUES (?, ?, ?, ?)`
insertLink := `INSERT INTO links (dom, subdom, path, proto, time, stat, err, robot_ex) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
queries := []*gocql.Query{
db.Query(insertDomainToCrawl, "test.com", gocql.UUID{}, testTime),
db.Query(insertLink, "test.com", "", "/page1.html", "http", walker.NotYetCrawled, 200, "", false),
db.Query(insertLink, "test.com", "", "/page2.html", "http", walker.NotYetCrawled, 200, "", false),
db.Query(insertLink, "test.com", "", "/page3.html", "http", walker.NotYetCrawled, 404, "", false),
db.Query(insertLink, "test.com", "", "/page4.html", "http", walker.NotYetCrawled, 200, "An Error", false),
db.Query(insertLink, "test.com", "", "/page5.html", "http", walker.NotYetCrawled, 200, "", true),
db.Query(insertLink, "test.com", "sub", "/page6.html", "http", walker.NotYetCrawled, 200, "", false),
db.Query(insertLink, "test.com", "sub", "/page7.html", "https", walker.NotYetCrawled, 200, "", false),
db.Query(insertLink, "test.com", "sub", "/page8.html", "https", walker.NotYetCrawled, 200, "", false),
db.Query(insertSegment, "test.com", "", "/page1.html", "http"),
db.Query(insertSegment, "test.com", "", "/page2.html", "http"),
db.Query(insertDomainInfo, "foo.com"),
db.Query(insertLink, "foo.com", "sub", "/page1.html", "http", fooTime, 200, "", false),
db.Query(insertLink, "foo.com", "sub", "/page2.html", "http", fooTime, 200, "", false),
db.Query(insertDomainInfo, "bar.com"),
db.Query(insertDomainToCrawl, "baz.com", bazUuid, testTime),
db.Query(insertLink, "baz.com", "sub", "/page1.html", "http", bazLinkHistoryInit[0].CrawlTime, 200, "", false),
db.Query(insertLink, "baz.com", "sub", "/page1.html", "http", bazLinkHistoryInit[1].CrawlTime, 200, "", false),
db.Query(insertLink, "baz.com", "sub", "/page1.html", "http", bazLinkHistoryInit[2].CrawlTime, 200, "", false),
db.Query(insertLink, "baz.com", "sub", "/page1.html", "http", bazLinkHistoryInit[3].CrawlTime, 200, "", false),
db.Query(insertLink, "baz.com", "sub", "/page1.html", "http", bazLinkHistoryInit[4].CrawlTime, 200, "", false),
db.Query(insertLink, "baz.com", "sub", "/page1.html", "http", bazLinkHistoryInit[5].CrawlTime, 200, "", false),
db.Query(insertSegment, "baz.com", "sub", "page1.html", "http"),
}
for _, q := range queries {
err := q.Exec()
if err != nil {
t.Fatalf("Failed to insert test data: %v\nQuery: %v", err, q)
}
}
//
// Need to record the order that the test.com urls come off on
//
itr := db.Query("SELECT dom, subdom, path, proto FROM links WHERE dom = 'test.com'").Iter()
var domain, subdomain, path, protocol string
testComLinkOrder = nil
for itr.Scan(&domain, &subdomain, &path, &protocol) {
u, _ := walker.CreateURL(domain, subdomain, path, protocol, walker.NotYetCrawled)
urlString := u.String()
linfo, gotLinfo := testComLinkHash[urlString]
if !gotLinfo {
panic(fmt.Errorf("testComLinkOrder can't find url: %v", urlString))
}
testComLinkOrder = append(testComLinkOrder, linfo)
}
err = itr.Close()
if err != nil {
panic(fmt.Errorf("testComLinkOrder iterator error: %v", err))
}
//
// Need to record order for baz
//
itr = db.Query("SELECT time FROM links WHERE dom = 'baz.com'").Iter()
var crawlTime time.Time
bazLinkHistoryOrder = nil
for itr.Scan(&crawlTime) {
bestIndex := -1
var bestDiff int64 = 99999999
for i := range bazLinkHistoryInit {
e := &bazLinkHistoryInit[i]
delta := crawlTime.Unix() - e.CrawlTime.Unix()
if delta < 0 {
delta = -delta
}
if delta < bestDiff {
bestIndex = i
bestDiff = delta
}
}
if bestIndex < 0 {
panic("UNEXPECTED ERROR")
}
bazLinkHistoryOrder = append(bazLinkHistoryOrder, bazLinkHistoryInit[bestIndex])
}
err = itr.Close()
if err != nil {
panic(fmt.Errorf("bazLinkHistoryOrder iterator error: %v", err))
}
itr = db.Query("SELECT dom, subdom, path, proto FROM links").Iter()
var foundBaz = false
var beforeBazComLink *walker.URL = nil
for itr.Scan(&domain, &subdomain, &path, &protocol) {
url, err := walker.CreateURL(domain, subdomain, path, protocol, walker.NotYetCrawled)
if err != nil {
panic(err)
}
if domain == "baz.com" {
foundBaz = true
break
}
beforeBazComLink = url
}
if !foundBaz {
panic("Unable to find domain before baz.com")
}
err = itr.Close()
if err != nil {
panic(fmt.Errorf("beforeBazCom link iterator error: %v", err))
}
if beforeBazComLink == nil {
bazSeed = ""
} else {
bazSeed = beforeBazComLink.String()
}
return ds
}