Example #1
0
// Start is the function that starts up the crawler for the CCFS
func Start() {
	fmt.Printf("Crawler Starting\n")
	queuedTargets = make(map[string]bool)
	targetQueue = make(chan target, 100)
	go processQueue()
	http.HandleFunc("/crawler/", webCrawlerHandler)
	http.HandleFunc("/index/", webIndexHandler)
	http.HandleFunc("/search/", webSearchHandler)
	go seedQueue(interfaces.GetLocalSeed())
	fmt.Println("Seeding crawler with " + interfaces.GetLocalSeed())
}
Example #2
0
//testing push with new origin
func startFSintegration() {
	//log.SetFlags(log.Lshortfile) //gives filename for every log statement
	mountpoint := "mountpoint"
	err := os.MkdirAll(mountpoint, 0777)
	if err != nil {
		log.Printf("Unable to create directory in mountpoint: %s", err)
		return
	}
	c, err := fuse.Mount(mountpoint)
	if err != nil {
		log.Printf("Unable to mount mountpoint: %s", err)
		return
	}

	//defer profile.Start(profile.CPUProfile).Stop()

	interfaces.KeyLocalSeed()
	instance = fsFromHKIDString(interfaces.GetLocalSeed(), mountpoint)
	fs.Serve(c, instance)
}
Example #3
0
func (command) Command(command string) {
	switch command {
	case "crawl":
		go processQueue()
		seedQueue(interfaces.GetLocalSeed())

	case "dump":
		blobIndex = nil
		commitIndex = nil
		tagIndex = nil
		textIndex = nil
		nameSegmentIndex = nil
		queuedTargets = make(map[string]bool)
		targetQueue = make(chan target, 100)

	default:
		fmt.Printf("Crawler Command Line\n")
		return
	}

}