func main() {
filename := "pg100.txt"
f, _ := os.Open(filename)
reader := bufio.NewReader(f)
i := 0
N := 1000000
t := time.Now().UnixNano()
t0 := float64(t)
counter := loglog.NewCounter(12)
for {
line, err := reader.ReadString('\n')
fmt.Println(">" + line)
if err != nil {
break
}
for _, word := range strings.Fields(line) {
i += 1
counter.Digest(word)
if i%N == 0 {
d := time.Now().UnixNano() - t
fmt.Printf("Scanned %d in %d ns.\n", N, d)
t = time.Now().UnixNano()
}
}
}
fmt.Printf("Total fields: %d in %.2f s\n",
i,
(float64(time.Now().UnixNano())-t0)/1000000000.0)
fmt.Printf("Estimated distinct: %f\n", counter.Estimate())
}
func main() {
var WORKERS int
if len(os.Args) < 2 {
fmt.Println("Usage default workers=8")
WORKERS = 8
} else {
i, _ := strconv.ParseInt(os.Args[1], 10, 32)
WORKERS = int(i)
}
c := loglog.NewCounter(MBITS)
filename := "pg100.txt"
words := streamWords(filename)
hashvalues := scatter(words, WORKERS, c)
t := time.Now().UnixNano()
for {
entry, ok := <-hashvalues
if !ok {
break
}
c.DigestEntry(entry)
}
fmt.Println(c.Estimate())
d := float64(time.Now().UnixNano()-t) / 1000000000.0
fmt.Printf("In %.2f seconds.\n", d)
}