func main() { parseFlags() fetcher, err := storage.NewFetcher(flags.StoragePrivkeyFile, flags.StorageLocation) if err != nil { log.Fatalf("error creating new fetcher: %v\n", err) return } pdf := NewPDF("hi") var pdfFilename string var handtekeningenIDList []uint64 if flags.Single != nil { pdfFilename = fmt.Sprintf("single-%d.pdf", *flags.Single) handtekeningenIDList = []uint64{*flags.Single} } else if flags.Partition != nil { pdfFilename = fmt.Sprintf("partition-%d.pdf", *flags.Partition) handtekeningenIDList, err = fetcher.ListPartition(*flags.Partition) if err != nil { log.Fatalf("error listing handtekeningen for partition %d: %v", *flags.Partition, err) } } for _, ID := range handtekeningenIDList { h, err := fetcher.Fetch(ID) if err != nil { log.Printf("error fetching: %v\n", err) continue } err = pdf.AddHandtekening(h) if err != nil { log.Printf("error rendering handtekening %d: %v", ID, err) continue } } err = pdf.Render(pdfFilename) if err != nil { log.Fatalf("error rendering final pdf: %v", err) } log.Printf("pdf `%s` successfully rendered\n", pdfFilename) if len(handtekeningenIDList) != 1000 { log.Printf("WARNING!! partitie had slechts %d handtekeningen, i.p.v. 1000. Gaat alles goed? Neem bij twijfel contact op met Geert-Johan.", len(handtekeningenIDList)) } }
func main() { parseFlags() runtime.GOMAXPROCS(runtime.NumCPU() - 1) fetcher, err := storage.NewFetcher(flags.StoragePrivkeyFile, flags.StorageLocation) if err != nil { log.Fatalf("error creating new fetcher: %v\n", err) return } chIDs := make(chan uint64) go func() { if flags.Single != nil { chIDs <- *flags.Single close(chIDs) } else if flags.Partition != nil { handtekeningenIDList, err := fetcher.ListPartition(*flags.Partition) if err != nil { log.Fatalf("error listing handtekeningen for partition %d: %v", *flags.Partition, err) } for _, id := range handtekeningenIDList { chIDs <- id } close(chIDs) } else if flags.All || flags.StartID != nil { var startID = uint64(1) if flags.StartID != nil { startID = *flags.StartID } startPartition := (startID-1)/1000 + 1 for i := startPartition; true; i++ { fmt.Printf("walking partition %04d\n", i) handtekeningenIDList, err := fetcher.ListPartition(i) if err != nil { log.Printf("error listing handtekeningen for partition %d: %v", i, err) break } for _, id := range handtekeningenIDList { if id < startID { continue } chIDs <- id } } close(chIDs) } }() var spreadsheetLock sync.Mutex spreadsheetFile, err := os.Create(flags.CSVFile) if err != nil { log.Fatalf("error opening csv file: %v", err) } spreadsheet := csv.NewWriter(spreadsheetFile) spreadsheet.Write([]string{"ID", "Voornaam", "Tussenvoegsel", "Achternaam", "Email"}) var numWorkers = (runtime.NumCPU() - 1) * 2 wgDone := sync.WaitGroup{} wgDone.Add(numWorkers) for n := 0; n < numWorkers; n++ { go func(n int) { for { id, ok := <-chIDs if !ok { break } h, err := fetcher.Fetch(id) if err != nil { log.Printf("error fetching %06d\n", id) continue } fields := []string{strconv.FormatUint(id, 10), h.Voornaam, h.Tussenvoegsel, h.Achternaam, h.Email} spreadsheetLock.Lock() err = spreadsheet.Write(fields) if err != nil { log.Printf("error writing fields for %06d: %v\n", id, err) } spreadsheetLock.Unlock() } fmt.Printf("worker %02d done\n", n) wgDone.Done() }(n) } wgDone.Wait() spreadsheet.Flush() err = spreadsheetFile.Close() if err != nil { log.Printf("error closing csv file: %v", err) } }
func main() { parseFlags() runtime.GOMAXPROCS(runtime.NumCPU() - 1) fetcher, err := storage.NewFetcher(flags.StoragePrivkeyFile, flags.StorageLocation) if err != nil { log.Fatalf("error creating new fetcher: %v\n", err) return } chIDs := make(chan uint64) go func() { for i := uint64(1); true; i++ { fmt.Printf("walking partition %04d\n", i) handtekeningenIDList, err := fetcher.ListPartition(i) if err != nil { log.Printf("error listing handtekeningen for partition %d: %v", i, err) break } for _, id := range handtekeningenIDList { chIDs <- id } } close(chIDs) }() type checklist struct { lock sync.Mutex list map[string]uint } var checks []*checklist for i := 0; i < 256; i++ { checks = append(checks, &checklist{ list: make(map[string]uint), }) } var numWorkers = (runtime.NumCPU() - 1) * 2 wgDone := sync.WaitGroup{} wgDone.Add(numWorkers) for n := 0; n < numWorkers; n++ { go func(n int) { for { id, ok := <-chIDs if !ok { break } h, err := fetcher.Fetch(id) if err != nil { log.Printf("error fetching %06d\n", id) continue } hash := hashHandtekening(h) cl := checks[int(hash[0])] cl.lock.Lock() cl.list[hash]++ cl.lock.Unlock() } fmt.Printf("worker %02d done\n", n) wgDone.Done() }(n) } wgDone.Wait() var total uint for i := 0; i < 256; i++ { cl := checks[i] for _, count := range cl.list { if count > 1 { total += count fmt.Printf("%d, ", count) } } } fmt.Println("") fmt.Println(total) }