Exemplo n.º 1
0
func main() {
	flag.Parse()
	log = logging.NewFromFlags()

	// Let's go find some mongo.
	mdb, err := db.Connect("localhost")
	if err != nil {
		log.Fatal("Oh no: %v", err)
	}
	defer mdb.Session.Close()
	qc := quotes.Collection(mdb, log)

	// A communication channel of Quotes.
	quotes := make(chan *quotes.Quote)
	rows := make(chan []interface{})

	// Function to feed rows into the rows channel.
	row_feeder := func(sth *sqlite3.Statement, row ...interface{}) {
		rows <- row
	}

	// Function to execute a query on the SQLite db.
	db_query := func(dbh *sqlite3.Database) {
		n, err := dbh.Execute("SELECT * FROM Quotes;", row_feeder)
		if err == nil {
			log.Info("Read %d rows from database.\n", n)
		} else {
			log.Error("DB error: %s\n", err)
		}
	}

	// Open up the quote database in a goroutine and feed rows
	// in on the input_rows channel.
	go func() {
		sqlite3.Session(*file, db_query)
		// once we've done the query, close the channel to indicate this
		close(rows)
	}()

	// Another goroutine to munge the rows into quotes.
	// This was originally done inside the SQLite callbacks, but
	// cgo or sqlite3 obscures runtime panics and makes fail happen.
	go func() {
		for row := range rows {
			parseQuote(row, quotes)
		}
		close(quotes)
	}()

	// And finally...
	count := 0
	for quote := range quotes {
		// ... push each quote into mongo
		err = qc.Insert(quote)
		if err != nil {
			log.Error("Awww: %v\n", err)
		} else {
			if count%1000 == 0 {
				fmt.Printf("%d...", count)
			}
			count++
		}
	}
	fmt.Println("done.")
	log.Info("Inserted %d quotes.\n", count)
}
Exemplo n.º 2
0
Arquivo: main.go Projeto: b33f/sp0rkle
func main() {
	flag.Parse()
	log := logging.NewFromFlags()
	reg := event.NewRegistry()

	if *server == "" {
		//Don't call log.Fatal as we don't want a backtrace in this case
		log.Error("--server option required. \nOptions are:\n")
		flag.PrintDefaults()
		os.Exit(1)
	}

	// Connect to mongo
	db, err := db.Connect("localhost")
	if err != nil {
		log.Fatal("mongo dial failed: %v\n", err)
	}
	defer db.Session.Close()

	// Initialise the factoid driver (which currently acts as a plugin mgr too).
	fd := factdriver.FactoidDriver(db, log)

	// Configure IRC client
	irc := client.Client(*nick, "boing", "not really sp0rkle", reg, log)
	irc.SSL = *ssl

	// Initialise bot state
	bot := bot.Bot(irc, fd, log)
	bot.AddChannels(strings.Split(*channels, ","))

	// Add drivers
	bot.AddDriver(bot)
	bot.AddDriver(fd)
	bot.AddDriver(calcdriver.CalcDriver(log))
	bot.AddDriver(decisiondriver.DecisionDriver(log))
	bot.AddDriver(quotedriver.QuoteDriver(db, log))
	bot.AddDriver(netdriver.NetDriver(log))
	bot.AddDriver(seendriver.SeenDriver(db, log))
	bot.AddDriver(urldriver.UrlDriver(db, log))

	// Register everything (including http handlers)
	bot.RegisterAll()

	// Start up the HTTP server
	go http.ListenAndServe(*httpPort, nil)

	// Connect loop.
	quit := false
	for !quit {
		if err := irc.Connect(*server); err != nil {
			log.Fatal("Connection error: %s", err)
		}
		quit = <-bot.Quit
	}
	if bot.ReExec() {
		// Calling syscall.Exec probably means deferred functions won't get
		// called, so disconnect from mongodb first for politeness' sake.
		db.Session.Close()
		// If sp0rkle was run from PATH, we need to do that lookup manually.
		fq, _ := exec.LookPath(os.Args[0])
		log.Warn("Re-executing sp0rkle with args '%v'.", os.Args)
		err := syscall.Exec(fq, os.Args, os.Environ())
		if err != nil {
			// hmmmmmm
			log.Fatal("Couldn't re-exec sp0rkle: %v", err)
		}
	}
}
Exemplo n.º 3
0
func main() {
	flag.Parse()
	log = logging.NewFromFlags()

	// Let's go find some mongo.
	mdb, err := db.Connect("localhost")
	if err != nil {
		log.Fatal("Oh no: %v", err)
	}
	defer mdb.Session.Close()
	uc := urls.Collection(mdb, log)

	work := make(chan *urls.Url)
	quit := make(chan bool)
	urls := make(chan *urls.Url)
	rows := make(chan []interface{})
	failed := 0

	// If we're checking, spin up some workers
	if *check {
		for i := 1; i <= *workq; i++ {
			go func(n int) {
				count := 0
				for u := range work {
					count++
					log.Debug("w%02d r%04d: Fetching '%s'", n, count, u.Url)
					res, err := http.Head(u.Url)
					log.Debug("w%02d r%04d: Response '%s'", n, count, res.Status)
					if err == nil && res.StatusCode == 200 {
						urls <- u
					} else {
						failed++
					}
				}
				quit <- true
			}(i)
		}
	}

	// Function to feed rows into the rows channel.
	row_feeder := func(sth *sqlite3.Statement, row ...interface{}) {
		rows <- row
	}

	// Function to execute a query on the SQLite db.
	db_query := func(dbh *sqlite3.Database) {
		n, err := dbh.Execute("SELECT * FROM urls;", row_feeder)
		if err == nil {
			log.Info("Read %d rows from database.\n", n)
		} else {
			log.Error("DB error: %s\n", err)
		}
	}

	// Open up the URL database in a goroutine and feed rows
	// in on the input_rows channel.
	go func() {
		sqlite3.Session(*file, db_query)
		// once we've done the query, close the channel to indicate this
		close(rows)
	}()

	// Another goroutine to munge the rows into Urls and optionally send
	// them to the pool of checker goroutines.
	go func() {
		for row := range rows {
			u := parseUrl(row)
			if *check {
				work <- u
			} else {
				urls <- u
			}
		}
		if *check {
			// Close work channel and wait for all workers to quit.
			close(work)
			for i := 0; i < *workq; i++ {
				<-quit
			}
		}
		close(urls)
	}()

	// And finally...
	count := 0
	for u := range urls {
		// ... push each url into mongo
		err = uc.Insert(u)
		if err != nil {
			log.Error("Awww: %v\n", err)
		} else {
			if count%1000 == 0 {
				fmt.Printf("%d...", count)
			}
			count++
		}
	}
	fmt.Println("done.")
	if *check {
		log.Info("Dropped %d non-200 urls.", failed)
	}
	log.Info("Inserted %d urls.", count)
}