func connect(c *irc.Conn, server string, password string, exiting chan bool) {
err := c.ConnectTo(server, password)
if err != nil {
log.Error(err.Error())
log.Info("Unable to connect to server")
exiting <- true
} else {
log.Info(c.String())
}
}
// Parse mode strings for a Nick.
func (nk *Nick) ParseModes(modes string) {
var modeop bool // true => add mode, false => remove mode
for i := 0; i < len(modes); i++ {
switch m := modes[i]; m {
case '+':
modeop = true
case '-':
modeop = false
case 'B':
nk.Modes.Bot = modeop
case 'i':
nk.Modes.Invisible = modeop
case 'o':
nk.Modes.Oper = modeop
case 'w':
nk.Modes.WallOps = modeop
case 'x':
nk.Modes.HiddenHost = modeop
case 'z':
nk.Modes.SSL = modeop
default:
logging.Info("Nick.ParseModes(): unknown mode char %c", m)
}
}
}
func Init() {
bot.Command(urbanDictionary, "ud", "ud <term> -- "+
"Look up <term> on UrbanDictionary.")
mcConf = conf.Ns("mc")
srv := mcConf.String(mcServer)
if srv != "" {
if st, err := pollServer(srv); err == nil {
logging.Info("Starting MC poller for '%s'", srv)
bot.Poll(st)
bot.Handle(func(ctx *bot.Context) {
st.Topic(ctx)
}, "332")
} else {
logging.Error("Not starting MC poller: %v", err)
}
}
bot.Command(mcSet, "mc set", "mc set <key> <value> -- "+
"Set minecraft server polling config vars.")
// TODO(fluffle): Polling can only be en/disabled at reconnect.
// bot.Command(mcPoll, "mc poll", "mc poll start|stop -- "+
// "Enable or disable minecraft server polling.")
if *githubToken != "" {
rc = reminders.Init()
gh = githubClient()
bot.Handle(githubWatcher, client.PRIVMSG)
bot.Command(githubCreateIssue, "file bug:", "file bug: <title>. "+
"<descriptive body> -- Files a bug on GitHub. Abusers will be hurt.")
bot.Command(githubCreateIssue, "file bug", "file bug <title>. "+
"<descriptive body> -- Files a bug on GitHub. Abusers will be hurt.")
bot.Command(githubCreateIssue, "report bug", "report bug <title>. "+
"<descriptive body> -- Files a bug on GitHub. Abusers will be hurt.")
bot.Command(githubUpdateIssue, "update bug #", "update bug #<number> "+
"<comment> -- Adds a comment to bug <number>. Abusers will be hurt.")
}
if push.Enabled() {
pc = pushes.Init()
bot.Command(pushEnable, "push enable", "push enable -- "+
"Start the OAuth flow to enable pushbullet notifications.")
bot.Command(pushDisable, "push disable", "push disable -- "+
"Disable pushbullet notifications and delete tokens.")
bot.Command(pushConfirm, "push auth", "push auth <pin> -- "+
"Confirm pushed PIN to finish pushbullet auth dance.")
bot.Command(pushAddAlias, "push add alias", "push add alias -- "+
"Add a push alias for your nick.")
bot.Command(pushDelAlias, "push del alias", "push del alias -- "+
"Delete a push alias for your nick.")
http.HandleFunc("/oauth/auth", pushAuthHTTP)
http.HandleFunc("/oauth/device", pushDeviceHTTP)
http.HandleFunc("/oauth/success", pushSuccessHTTP)
http.HandleFunc("/oauth/failure", pushFailureHTTP)
}
}
func rebuild(ctx *Context) {
if !check_rebuilder("rebuild", ctx) {
return
}
// Ok, we should be good to rebuild now.
logging.Info("Beginning rebuild")
ctx.conn.Notice(ctx.Nick, "Beginning rebuild")
cmd := exec.Command("go", "get", "-u", "github.com/fluffle/sp0rkle")
out, err := cmd.CombinedOutput()
logging.Info("Output from go get:\n%s", out)
if err != nil {
ctx.conn.Notice(ctx.Nick, fmt.Sprintf("Rebuild failed: %s", err))
for _, l := range strings.Split(string(out), "\n") {
ctx.conn.Notice(ctx.Nick, l)
}
return
}
bot.servers.Shutdown(true)
}
func (fc *Collection) GetAll(key string) []*Factoid {
// Insisting GetAll isn't used to get every key is probably a good idea
if key == "" {
return nil
}
res := make([]*Factoid, 0, 10)
if err := fc.Find(lookup(key)).All(&res); err == nil {
logging.Info("res = %#v", res)
return res
}
return nil
}
// Connect the IRC connection object to "host[:port]" which should be either
// a hostname or an IP address, with an optional port. To enable explicit SSL
// on the connection to the IRC server, set Conn.SSL to true before calling
// Connect(). The port will default to 6697 if ssl is enabled, and 6667
// otherwise. You can also provide an optional connect password.
func (conn *Conn) Connect(host string, pass ...string) error {
if conn.Connected {
return errors.New(fmt.Sprintf(
"irc.Connect(): already connected to %s, cannot connect to %s",
conn.Host, host))
}
if conn.SSL {
if !hasPort(host) {
host += ":6697"
}
logging.Info("irc.Connect(): Connecting to %s with SSL.", host)
if s, err := tls.Dial("tcp", host, conn.SSLConfig); err == nil {
conn.sock = s
} else {
return err
}
} else {
if !hasPort(host) {
host += ":6667"
}
logging.Info("irc.Connect(): Connecting to %s without SSL.", host)
if s, err := net.Dial("tcp", host); err == nil {
conn.sock = s
} else {
return err
}
}
conn.Host = host
if len(pass) > 0 {
conn.password = pass[0]
}
conn.Connected = true
conn.postConnect()
conn.ED.Dispatch(INIT, conn, &Line{})
return nil
}
func (conn *Conn) Connect() error {
conn.mu.Lock()
defer conn.mu.Unlock()
if conn.cfg.Server == "" {
return fmt.Errorf("irc.Connect(): cfg.Server must be non-empty")
}
if conn.connected {
return fmt.Errorf("irc.Connect(): Cannot connect to %s, already connected.", conn.cfg.Server)
}
if conn.cfg.SSL {
if !hasPort(conn.cfg.Server) {
conn.cfg.Server += ":6697"
}
logging.Info("irc.Connect(): Connecting to %s with SSL.", conn.cfg.Server)
if s, err := tls.Dial("tcp", conn.cfg.Server, conn.cfg.SSLConfig); err == nil {
conn.sock = s
} else {
return err
}
} else {
if !hasPort(conn.cfg.Server) {
conn.cfg.Server += ":6667"
}
logging.Info("irc.Connect(): Connecting to %s without SSL.", conn.cfg.Server)
if s, err := net.Dial("tcp", conn.cfg.Server); err == nil {
conn.sock = s
} else {
return err
}
}
conn.connected = true
conn.postConnect()
conn.dispatch(&Line{Cmd: REGISTER})
return nil
}
func (conn *Conn) shutdown() {
// Guard against double-call of shutdown() if we get an error in send()
// as calling sock.Close() will cause recv() to recieve EOF in readstring()
if conn.Connected {
logging.Info("irc.shutdown(): Disconnected from server.")
conn.ED.Dispatch(DISCONNECTED, conn, &Line{})
conn.Connected = false
conn.sock.Close()
conn.cSend <- true
conn.cLoop <- true
conn.cPing <- true
// reinit datastructures ready for next connection
// do this here rather than after runLoop()'s for due to race
conn.initialise()
}
}
func (conn *Conn) shutdown() {
// Guard against double-call of shutdown() if we get an error in send()
// as calling sock.Close() will cause recv() to receive EOF in readstring()
conn.mu.Lock()
defer conn.mu.Unlock()
if !conn.connected {
return
}
logging.Info("irc.shutdown(): Disconnected from server.")
conn.dispatch(&Line{Cmd: DISCONNECTED})
conn.connected = false
conn.sock.Close()
close(conn.die)
// reinit datastructures ready for next connection
// do this here rather than after runLoop()'s for due to race
conn.initialise()
}
func main() {
flag.Parse()
logging.InitFromFlags()
// Initialise bot state
bot.Init()
// Connect to mongo
db.Init()
defer db.Close()
// Add drivers
calcdriver.Init()
decisiondriver.Init()
factdriver.Init()
karmadriver.Init()
markovdriver.Init()
quotedriver.Init()
reminddriver.Init()
seendriver.Init()
urldriver.Init()
// Start up the HTTP server
go http.ListenAndServe(*httpPort, nil)
// Connect the bot to IRC and wait; reconnects are handled automatically.
// If we get true back from the bot, re-exec the (rebuilt) binary.
if bot.Connect() {
// Calling syscall.Exec probably means deferred functions won't get
// called, so disconnect from mongodb first for politeness' sake.
db.Close()
// If sp0rkle was run from PATH, we need to do that lookup manually.
fq, _ := exec.LookPath(os.Args[0])
logging.Warn("Re-executing sp0rkle with args '%v'.", os.Args)
err := syscall.Exec(fq, os.Args, os.Environ())
if err != nil {
// hmmmmmm
logging.Fatal("Couldn't re-exec sp0rkle: %v", err)
}
}
logging.Info("Shutting down cleanly.")
}
func connected(ctx *Context) {
// Set bot mode to keep people informed.
ctx.conn.Mode(ctx.Me(), "+B")
if GetSecret(*oper) != "" {
up := strings.SplitN(*oper, ":", 2)
if len(up) == 2 {
ctx.conn.Oper(up[0], up[1])
}
}
if GetSecret(*vhost) != "" {
up := strings.SplitN(*vhost, ":", 2)
if len(up) == 2 {
ctx.conn.VHost(up[0], up[1])
}
}
for _, c := range strings.Split(*channels, ",") {
logging.Info("Joining %s on startup.\n", c)
ctx.conn.Join(c)
}
}
func (b *boltDatabase) doBackup() {
fn := path.Join(b.dir, fmt.Sprintf("sp0rkle.boltdb.%s.gz",
time.Now().Format("2006-01-02.15:04")))
fh, err := os.OpenFile(fn, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
logging.Error("Could not create backup file %q: %v", fn, err)
return
}
fz := gzip.NewWriter(fh)
defer fz.Close()
err = b.db.View(func(tx *bolt.Tx) error {
return tx.Copy(fz)
})
if err != nil {
logging.Error("Could not write backup file %q: %v", fn, err)
os.Remove(fn)
return
}
logging.Info("Wrote backup to %q.", fn)
}
func (rc *Collection) LoadAndPrune() []*Reminder {
// First, drop any reminders where RemindAt < time.Now()
ci, err := rc.RemoveAll(bson.M{"$and": []bson.M{
{"remindat": bson.M{"$lt": time.Now()}},
{"tell": false},
}})
if err != nil {
logging.Error("Pruning reminders returned error: %v", err)
}
if ci.Removed > 0 {
logging.Info("Removed %d old reminders", ci.Removed)
}
// Now, load the remainder; the db is just used for persistence
q := rc.Find(bson.M{"tell": false})
ret := make([]*Reminder, 0)
if err := q.All(&ret); err != nil {
logging.Error("Loading reminders returned error: %v", err)
return nil
}
return ret
}
func main() {
flag.Parse()
logging.InitFromFlags()
// Let's go find some mongo.
db.Init()
defer db.Close()
qc := quotes.Init()
// A communication channel of Quotes.
quotes := make(chan *quotes.Quote)
rows := make(chan []interface{})
// Function to feed rows into the rows channel.
row_feeder := func(sth *sqlite3.Statement, row ...interface{}) {
rows <- row
}
// Function to execute a query on the SQLite db.
db_query := func(dbh *sqlite3.Database) {
n, err := dbh.Execute("SELECT * FROM Quotes;", row_feeder)
if err == nil {
logging.Info("Read %d rows from database.\n", n)
} else {
logging.Error("DB error: %s\n", err)
}
}
// Open up the quote database in a goroutine and feed rows
// in on the input_rows channel.
go func() {
sqlite3.Session(*file, db_query)
// once we've done the query, close the channel to indicate this
close(rows)
}()
// Another goroutine to munge the rows into quotes.
// This was originally done inside the SQLite callbacks, but
// cgo or sqlite3 obscures runtime panics and makes fail happen.
go func() {
for row := range rows {
parseQuote(row, quotes)
}
close(quotes)
}()
// And finally...
count := 0
var err error
for quote := range quotes {
// ... push each quote into mongo
err = qc.Insert(quote)
if err != nil {
logging.Error("Awww: %v\n", err)
} else {
if count%1000 == 0 {
fmt.Printf("%d...", count)
}
count++
}
}
fmt.Println("done.")
logging.Info("Inserted %d quotes.\n", count)
}
func bot_disconnected(line *base.Line) {
// The read from this channel is in connectLoop
disconnected <- true
logging.Info("Disconnected...")
}
func loadModules(conn *irc.Conn) (modules []IModule) {
// Load each module in the modules directory
scripts, err := ioutil.ReadDir("modules")
if err != nil {
log.Fatal(err.Error())
}
// Read base module script
file, err := os.Open("module.js")
if err != nil {
log.Fatal("Error reading base module script:", err)
}
defer file.Close()
baseScript, err := ioutil.ReadAll(file)
if err != nil {
log.Fatal("Error reading base module script:", err)
}
for _, fileInfo := range scripts {
if !fileInfo.IsDir() {
if !strings.HasSuffix(fileInfo.Name(), "js") {
continue
}
filename := "modules/" + fileInfo.Name()
file, err := os.Open(filename)
if err != nil {
log.Error("Error loading module:", err)
continue
}
defer file.Close()
script, err := ioutil.ReadAll(file)
if err != nil {
log.Error("Error loading module:", err)
continue
}
module := NewModule(fileInfo.Name(), conn)
// Init module with base script and its own script
ret, err := module.Init(v8.NewContext(), string(baseScript)+string(script))
if err != nil {
log.Error("Error loading module: %s\n%s", err, ret)
continue
}
// Create file watcher
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Error("Error creating file watcher for %s:", fileInfo.Name(), err)
}
watcher.Watch(filename)
go func(filename string) {
for {
select {
case ev := <-watcher.Event:
watcher.Watch(filename) // Make sure we continue to watch the file at this location
if !ev.IsModify() {
// Only reload the module on a modification event
continue
}
case err := <-watcher.Error:
log.Error("Error watching file: %s", filename, err)
continue
}
// Reload script
log.Info("Reloading %s", filename)
file, err := os.Open(filename)
if err != nil {
log.Error("Error loading module:", err)
continue
}
defer file.Close()
script, err := ioutil.ReadAll(file)
if err != nil {
log.Error("Error loading module:", err)
continue
}
ret, err := module.Init(v8.NewContext(), string(baseScript)+string(script))
if err != nil {
log.Error("Error reloading module: %s\n%s", err, ret)
}
}
}(filename)
log.Info("Loaded module %s", fileInfo.Name())
modules = append(modules, &module)
}
}
return
}
func main() {
exiting := make(chan bool)
// Set up signal handlers
quitting := make(chan os.Signal, 1)
signal.Notify(quitting, os.Interrupt)
signal.Notify(quitting, os.Kill)
// Parse flags
server := flag.String("server", "", "Hostname and/or port (e.g. 'localhost:6667')")
gonkNick := flag.String("nick", "gonk", "Nickname used for the connection")
password := flag.String("password", "", "Server password")
ssl := flag.Bool("ssl", false, "Use SSL")
verifyCert := flag.Bool("verify-ssl", true, "Verify SSL certificate")
flag.Parse()
// User must specify server
if *server == "" {
printUsageAndExit()
}
// Setup IRC client
c := irc.SimpleClient(*gonkNick, *gonkNick)
if *ssl {
c.Config().SSL = true
}
if !*verifyCert {
c.Config().SSLConfig = &tls.Config{InsecureSkipVerify: true}
}
// Load modules and connect them to the IRC client
modules := loadModules(c)
c.HandleFunc(irc.CONNECTED, func(conn *irc.Conn, line *irc.Line) {
// Join all specified channels upon connecting
for i := 0; i < len(flag.Args()); i++ {
channel := fmt.Sprintf("#%s", flag.Arg(i))
log.Info("Joining %s", channel)
conn.Join(channel)
conn.Privmsg(channel, "*"+strings.ToUpper(*gonkNick)+"*")
}
})
c.HandleFunc(irc.DISCONNECTED, func(conn *irc.Conn, line *irc.Line) {
log.Info("Disconnected from server; attempting to reconnect")
go connect(c, *server, *password, exiting)
})
c.HandleFunc("privmsg", func(conn *irc.Conn, line *irc.Line) {
// Determine reply target
target := line.Args[0]
if target == conn.Me().Nick {
// Reply via PM
target = line.Nick
}
text := strings.Join(line.Args[1:], "")
go func() {
for _, module := range modules {
if target == line.Nick || strings.HasPrefix(text, conn.Me().Nick) {
// Received a PM or addressed directly in a channel
if module.Respond(target, text, line.Nick) {
break
}
} else {
if module.Hear(target, text, line.Nick) {
break
}
}
}
}()
})
go func() {
connect(c, *server, *password, exiting)
}()
go func() {
<-quitting
c.Quit("*" + strings.ToUpper(*gonkNick) + "*")
exiting <- true
}()
<-exiting
log.Info("Shutting down")
}
func main() {
flag.Parse()
logging.InitFromFlags()
golog.Init()
// Slightly more random than 1.
rand.Seed(time.Now().UnixNano() * int64(os.Getpid()))
// Initialise bot state
bot.Init()
// Connect to mongo
db.Init()
defer db.Close()
// Add drivers
calcdriver.Init()
decisiondriver.Init()
factdriver.Init()
karmadriver.Init()
markovdriver.Init()
netdriver.Init()
quotedriver.Init()
reminddriver.Init()
seendriver.Init()
statsdriver.Init()
urldriver.Init()
// Start up the HTTP server
go http.ListenAndServe(*httpPort, nil)
// Set up a signal handler to shut things down gracefully.
// NOTE: net/http doesn't provide for graceful shutdown :-/
go func() {
called := new(int32)
sigint := make(chan os.Signal, 1)
signal.Notify(sigint, syscall.SIGINT)
for _ = range sigint {
if atomic.AddInt32(called, 1) > 1 {
logging.Fatal("Recieved multiple interrupts, dying.")
}
bot.Shutdown()
}
}()
// Connect the bot to IRC and wait; reconnects are handled automatically.
// If we get true back from the bot, re-exec the (rebuilt) binary.
if <-bot.Connect() {
// Calling syscall.Exec probably means deferred functions won't get
// called, so disconnect from mongodb first for politeness' sake.
db.Close()
// If sp0rkle was run from PATH, we need to do that lookup manually.
fq, _ := exec.LookPath(os.Args[0])
logging.Warn("Re-executing sp0rkle with args '%v'.", os.Args)
err := syscall.Exec(fq, os.Args, os.Environ())
if err != nil {
// hmmmmmm
logging.Fatal("Couldn't re-exec sp0rkle: %v", err)
}
}
logging.Info("Shutting down cleanly.")
}
func bot_connected(line *base.Line) {
for _, c := range strings.Split(*channels, ",") {
logging.Info("Joining %s on startup.\n", c)
irc.Join(c)
}
}
func main() {
flag.Parse()
logging.InitFromFlags()
// Let's go find some mongo.
db.Init()
defer db.Close()
fc := factoids.Init()
// A communication channel of Factoids.
facts := make(chan *factoids.Factoid)
ptrs := make(chan []interface{})
rows := make(chan []interface{})
// Function to execute some queries on the SQLite db and shove the results
// into the ptrs and rows channels created above.
db_query := func(dbh *sqlite3.Database) {
_, err := dbh.Execute("SELECT * FROM Factoids WHERE Value LIKE '%*%';", feeder(ptrs))
close(ptrs)
if err != nil {
logging.Error("DB error: %s", err)
}
n, err := dbh.Execute("SELECT * FROM Factoids;", feeder(rows))
close(rows)
if err == nil {
logging.Info("Read %d rows from database.", n)
} else {
logging.Error("DB error: %s", err)
}
}
go func() {
sqlite3.Session(*file, db_query)
}()
// First, synchronously read all the stuff from the ptrs channel
// and build a set of all the factoid keys that are used as pointers
for row := range ptrs {
for _, val := range parseMultipleValues(toString(row[cValue])) {
if key, _, _ := util.FactPointer(val); key != "" {
ptrkeys[key] = true
}
}
}
// Now run another goroutine to munge the rows into factoids.
// This was originally done inside the SQLite callbacks, but
// cgo or sqlite3 obscures runtime panics and makes fail happen.
go func() {
for row := range rows {
parseFactoid(row, facts)
}
close(facts)
}()
// And finally...
count := 0
var err error
for fact := range facts {
// ... push each fact into mongo
err = fc.Insert(fact)
if err != nil {
logging.Error("Awww: %v\n", err)
} else {
if count%1000 == 0 {
fmt.Printf("%d...", count)
}
count++
}
}
fmt.Println("done.")
logging.Info("Inserted %d factoids.\n", count)
}
// Parses mode strings for a channel.
func (ch *Channel) ParseModes(modes string, modeargs ...string) {
var modeop bool // true => add mode, false => remove mode
var modestr string
for i := 0; i < len(modes); i++ {
switch m := modes[i]; m {
case '+':
modeop = true
modestr = string(m)
case '-':
modeop = false
modestr = string(m)
case 'i':
ch.Modes.InviteOnly = modeop
case 'm':
ch.Modes.Moderated = modeop
case 'n':
ch.Modes.NoExternalMsg = modeop
case 'p':
ch.Modes.Private = modeop
case 'r':
ch.Modes.Registered = modeop
case 's':
ch.Modes.Secret = modeop
case 't':
ch.Modes.ProtectedTopic = modeop
case 'z':
ch.Modes.SSLOnly = modeop
case 'Z':
ch.Modes.AllSSL = modeop
case 'O':
ch.Modes.OperOnly = modeop
case 'k':
if modeop && len(modeargs) != 0 {
ch.Modes.Key, modeargs = modeargs[0], modeargs[1:]
} else if !modeop {
ch.Modes.Key = ""
} else {
logging.Warn("Channel.ParseModes(): not enough arguments to "+
"process MODE %s %s%c", ch.Name, modestr, m)
}
case 'l':
if modeop && len(modeargs) != 0 {
ch.Modes.Limit, _ = strconv.Atoi(modeargs[0])
modeargs = modeargs[1:]
} else if !modeop {
ch.Modes.Limit = 0
} else {
logging.Warn("Channel.ParseModes(): not enough arguments to "+
"process MODE %s %s%c", ch.Name, modestr, m)
}
case 'q', 'a', 'o', 'h', 'v':
if len(modeargs) != 0 {
if nk, ok := ch.lookup[modeargs[0]]; ok {
cp := ch.nicks[nk]
switch m {
case 'q':
cp.Owner = modeop
case 'a':
cp.Admin = modeop
case 'o':
cp.Op = modeop
case 'h':
cp.HalfOp = modeop
case 'v':
cp.Voice = modeop
}
modeargs = modeargs[1:]
} else {
logging.Warn("Channel.ParseModes(): untracked nick %s "+
"received MODE on channel %s", modeargs[0], ch.Name)
}
} else {
logging.Warn("Channel.ParseModes(): not enough arguments to "+
"process MODE %s %s%c", ch.Name, modestr, m)
}
default:
logging.Info("Channel.ParseModes(): unknown mode char %c", m)
}
}
}
func main() {
flag.Parse()
logging.InitFromFlags()
// Let's go find some mongo.
db.Init()
defer db.Close()
uc := urls.Init()
work := make(chan *urls.Url)
quit := make(chan bool)
urls := make(chan *urls.Url)
rows := make(chan []interface{})
failed := 0
// If we're checking, spin up some workers
if *check {
for i := 1; i <= *workq; i++ {
go func(n int) {
count := 0
for u := range work {
count++
logging.Debug("w%02d r%04d: Fetching '%s'", n, count, u.Url)
res, err := http.Head(u.Url)
logging.Debug("w%02d r%04d: Response '%s'", n, count, res.Status)
if err == nil && res.StatusCode == 200 {
urls <- u
} else {
failed++
}
}
quit <- true
}(i)
}
}
// Function to feed rows into the rows channel.
row_feeder := func(sth *sqlite3.Statement, row ...interface{}) {
rows <- row
}
// Function to execute a query on the SQLite db.
db_query := func(dbh *sqlite3.Database) {
n, err := dbh.Execute("SELECT * FROM urls;", row_feeder)
if err == nil {
logging.Info("Read %d rows from database.\n", n)
} else {
logging.Error("DB error: %s\n", err)
}
}
// Open up the URL database in a goroutine and feed rows
// in on the input_rows channel.
go func() {
sqlite3.Session(*file, db_query)
// once we've done the query, close the channel to indicate this
close(rows)
}()
// Another goroutine to munge the rows into Urls and optionally send
// them to the pool of checker goroutines.
go func() {
for row := range rows {
u := parseUrl(row)
if *check {
work <- u
} else {
urls <- u
}
}
if *check {
// Close work channel and wait for all workers to quit.
close(work)
for i := 0; i < *workq; i++ {
<-quit
}
}
close(urls)
}()
// And finally...
count := 0
var err error
for u := range urls {
// ... push each url into mongo
err = uc.Insert(u)
if err != nil {
logging.Error("Awww: %v\n", err)
} else {
if count%1000 == 0 {
fmt.Printf("%d...", count)
}
count++
}
}
fmt.Println("done.")
if *check {
logging.Info("Dropped %d non-200 urls.", failed)
}
logging.Info("Inserted %d urls.", count)
}