Esempio n. 1
0
func main() {
	filename := "pg100.txt"

	f, _ := os.Open(filename)
	reader := bufio.NewReader(f)
	i := 0
	N := 1000000
	t := time.Now().UnixNano()
	t0 := float64(t)
	counter := loglog.NewCounter(12)
	for {
		line, err := reader.ReadString('\n')
		fmt.Println(">" + line)
		if err != nil {
			break
		}
		for _, word := range strings.Fields(line) {
			i += 1
			counter.Digest(word)
			if i%N == 0 {
				d := time.Now().UnixNano() - t
				fmt.Printf("Scanned %d in %d ns.\n", N, d)
				t = time.Now().UnixNano()
			}
		}
	}
	fmt.Printf("Total fields: %d in %.2f s\n",
		i,
		(float64(time.Now().UnixNano())-t0)/1000000000.0)
	fmt.Printf("Estimated distinct: %f\n", counter.Estimate())
}
Esempio n. 2
0
func main() {
	var WORKERS int
	if len(os.Args) < 2 {
		fmt.Println("Usage default workers=8")
		WORKERS = 8
	} else {
		i, _ := strconv.ParseInt(os.Args[1], 10, 32)
		WORKERS = int(i)
	}

	c := loglog.NewCounter(MBITS)
	filename := "pg100.txt"
	words := streamWords(filename)
	hashvalues := scatter(words, WORKERS, c)

	t := time.Now().UnixNano()

	for {
		entry, ok := <-hashvalues
		if !ok {
			break
		}
		c.DigestEntry(entry)
	}

	fmt.Println(c.Estimate())
	d := float64(time.Now().UnixNano()-t) / 1000000000.0
	fmt.Printf("In %.2f seconds.\n", d)
}