func main() {
readConfig()
// load up available charsets
for _, name := range charset.Names() {
chartypeSet[name] = true
}
log.Printf("Starting with %d processes\n", numCPU)
runtime.GOMAXPROCS(numCPU)
conn := getRedisConn()
for {
obj := conn.BLPop(scraperConfig.QueueTimeout, "urljobs")
err := obj.Err()
if err != nil {
if err == redis.Nil {
log.Printf("No job seen for %d seconds. Exiting.\n", scraperConfig.QueueTimeout)
break
} else {
log.Fatalf("ERROR: %s\n", err)
}
} else {
url := obj.Val()[1]
workPool <- true
go fetchTitleJob(url)
}
}
}
func listCharsets(verbose bool, csname string) {
var buf bytes.Buffer
if !verbose {
if csname != "" {
cs := charset.Info(csname)
if cs == nil {
fatalf("no such charset %q", csname)
}
fmt.Fprintf(&buf, "%s %s\n", cs.Name, strings.Join(cs.Aliases, " "))
} else {
fmt.Fprintf(&buf, "%v\n", strings.Join(charset.Names(), " "))
}
} else {
var charsets []*charset.Charset
if csname != "" {
cs := charset.Info(csname)
if cs == nil {
fatalf("no such charset %q", csname)
}
charsets = []*charset.Charset{cs}
} else {
for _, name := range charset.Names() {
if cs := charset.Info(name); cs != nil {
charsets = append(charsets, cs)
}
}
}
for _, cs := range charsets {
fmt.Fprintf(&buf, "%s %s\n", cs.Name, strings.Join(cs.Aliases, " "))
if cs.Desc != "" {
fmt.Fprintf(&buf, "\t%s\n", cs.Desc)
}
}
}
os.Stdout.Write(buf.Bytes())
}
func TestNames(t *testing.T) {
charset.Names()
}