// Start is the function that starts up the crawler for the CCFS
func Start() {
fmt.Printf("Crawler Starting\n")
queuedTargets = make(map[string]bool)
targetQueue = make(chan target, 100)
go processQueue()
http.HandleFunc("/crawler/", webCrawlerHandler)
http.HandleFunc("/index/", webIndexHandler)
http.HandleFunc("/search/", webSearchHandler)
go seedQueue(interfaces.GetLocalSeed())
fmt.Println("Seeding crawler with " + interfaces.GetLocalSeed())
}
//testing push with new origin
func startFSintegration() {
//log.SetFlags(log.Lshortfile) //gives filename for every log statement
mountpoint := "mountpoint"
err := os.MkdirAll(mountpoint, 0777)
if err != nil {
log.Printf("Unable to create directory in mountpoint: %s", err)
return
}
c, err := fuse.Mount(mountpoint)
if err != nil {
log.Printf("Unable to mount mountpoint: %s", err)
return
}
//defer profile.Start(profile.CPUProfile).Stop()
interfaces.KeyLocalSeed()
instance = fsFromHKIDString(interfaces.GetLocalSeed(), mountpoint)
fs.Serve(c, instance)
}
func (command) Command(command string) {
switch command {
case "crawl":
go processQueue()
seedQueue(interfaces.GetLocalSeed())
case "dump":
blobIndex = nil
commitIndex = nil
tagIndex = nil
textIndex = nil
nameSegmentIndex = nil
queuedTargets = make(map[string]bool)
targetQueue = make(chan target, 100)
default:
fmt.Printf("Crawler Command Line\n")
return
}
}