Пример #1
0
func main() {
	readConfig()

	// load up available charsets
	for _, name := range charset.Names() {
		chartypeSet[name] = true
	}

	log.Printf("Starting with %d processes\n", numCPU)
	runtime.GOMAXPROCS(numCPU)

	conn := getRedisConn()

	for {
		obj := conn.BLPop(scraperConfig.QueueTimeout, "urljobs")
		err := obj.Err()
		if err != nil {
			if err == redis.Nil {
				log.Printf("No job seen for %d seconds. Exiting.\n", scraperConfig.QueueTimeout)
				break
			} else {
				log.Fatalf("ERROR: %s\n", err)
			}
		} else {
			url := obj.Val()[1]
			workPool <- true
			go fetchTitleJob(url)
		}
	}
}
Пример #2
0
func listCharsets(verbose bool, csname string) {
	var buf bytes.Buffer
	if !verbose {
		if csname != "" {
			cs := charset.Info(csname)
			if cs == nil {
				fatalf("no such charset %q", csname)
			}
			fmt.Fprintf(&buf, "%s %s\n", cs.Name, strings.Join(cs.Aliases, " "))
		} else {
			fmt.Fprintf(&buf, "%v\n", strings.Join(charset.Names(), " "))
		}
	} else {
		var charsets []*charset.Charset
		if csname != "" {
			cs := charset.Info(csname)
			if cs == nil {
				fatalf("no such charset %q", csname)
			}
			charsets = []*charset.Charset{cs}
		} else {
			for _, name := range charset.Names() {
				if cs := charset.Info(name); cs != nil {
					charsets = append(charsets, cs)
				}
			}
		}
		for _, cs := range charsets {
			fmt.Fprintf(&buf, "%s %s\n", cs.Name, strings.Join(cs.Aliases, " "))
			if cs.Desc != "" {
				fmt.Fprintf(&buf, "\t%s\n", cs.Desc)
			}
		}
	}
	os.Stdout.Write(buf.Bytes())
}
Пример #3
0
func TestNames(t *testing.T) {
	charset.Names()
}