func main() {
parseFlags()
fetcher, err := storage.NewFetcher(flags.StoragePrivkeyFile, flags.StorageLocation)
if err != nil {
log.Fatalf("error creating new fetcher: %v\n", err)
return
}
pdf := NewPDF("hi")
var pdfFilename string
var handtekeningenIDList []uint64
if flags.Single != nil {
pdfFilename = fmt.Sprintf("single-%d.pdf", *flags.Single)
handtekeningenIDList = []uint64{*flags.Single}
} else if flags.Partition != nil {
pdfFilename = fmt.Sprintf("partition-%d.pdf", *flags.Partition)
handtekeningenIDList, err = fetcher.ListPartition(*flags.Partition)
if err != nil {
log.Fatalf("error listing handtekeningen for partition %d: %v", *flags.Partition, err)
}
}
for _, ID := range handtekeningenIDList {
h, err := fetcher.Fetch(ID)
if err != nil {
log.Printf("error fetching: %v\n", err)
continue
}
err = pdf.AddHandtekening(h)
if err != nil {
log.Printf("error rendering handtekening %d: %v", ID, err)
continue
}
}
err = pdf.Render(pdfFilename)
if err != nil {
log.Fatalf("error rendering final pdf: %v", err)
}
log.Printf("pdf `%s` successfully rendered\n", pdfFilename)
if len(handtekeningenIDList) != 1000 {
log.Printf("WARNING!! partitie had slechts %d handtekeningen, i.p.v. 1000. Gaat alles goed? Neem bij twijfel contact op met Geert-Johan.", len(handtekeningenIDList))
}
}
func main() {
parseFlags()
runtime.GOMAXPROCS(runtime.NumCPU() - 1)
fetcher, err := storage.NewFetcher(flags.StoragePrivkeyFile, flags.StorageLocation)
if err != nil {
log.Fatalf("error creating new fetcher: %v\n", err)
return
}
chIDs := make(chan uint64)
go func() {
if flags.Single != nil {
chIDs <- *flags.Single
close(chIDs)
} else if flags.Partition != nil {
handtekeningenIDList, err := fetcher.ListPartition(*flags.Partition)
if err != nil {
log.Fatalf("error listing handtekeningen for partition %d: %v", *flags.Partition, err)
}
for _, id := range handtekeningenIDList {
chIDs <- id
}
close(chIDs)
} else if flags.All || flags.StartID != nil {
var startID = uint64(1)
if flags.StartID != nil {
startID = *flags.StartID
}
startPartition := (startID-1)/1000 + 1
for i := startPartition; true; i++ {
fmt.Printf("walking partition %04d\n", i)
handtekeningenIDList, err := fetcher.ListPartition(i)
if err != nil {
log.Printf("error listing handtekeningen for partition %d: %v", i, err)
break
}
for _, id := range handtekeningenIDList {
if id < startID {
continue
}
chIDs <- id
}
}
close(chIDs)
}
}()
var spreadsheetLock sync.Mutex
spreadsheetFile, err := os.Create(flags.CSVFile)
if err != nil {
log.Fatalf("error opening csv file: %v", err)
}
spreadsheet := csv.NewWriter(spreadsheetFile)
spreadsheet.Write([]string{"ID", "Voornaam", "Tussenvoegsel", "Achternaam", "Email"})
var numWorkers = (runtime.NumCPU() - 1) * 2
wgDone := sync.WaitGroup{}
wgDone.Add(numWorkers)
for n := 0; n < numWorkers; n++ {
go func(n int) {
for {
id, ok := <-chIDs
if !ok {
break
}
h, err := fetcher.Fetch(id)
if err != nil {
log.Printf("error fetching %06d\n", id)
continue
}
fields := []string{strconv.FormatUint(id, 10), h.Voornaam, h.Tussenvoegsel, h.Achternaam, h.Email}
spreadsheetLock.Lock()
err = spreadsheet.Write(fields)
if err != nil {
log.Printf("error writing fields for %06d: %v\n", id, err)
}
spreadsheetLock.Unlock()
}
fmt.Printf("worker %02d done\n", n)
wgDone.Done()
}(n)
}
wgDone.Wait()
spreadsheet.Flush()
err = spreadsheetFile.Close()
if err != nil {
log.Printf("error closing csv file: %v", err)
}
}
func main() {
parseFlags()
runtime.GOMAXPROCS(runtime.NumCPU() - 1)
fetcher, err := storage.NewFetcher(flags.StoragePrivkeyFile, flags.StorageLocation)
if err != nil {
log.Fatalf("error creating new fetcher: %v\n", err)
return
}
chIDs := make(chan uint64)
go func() {
for i := uint64(1); true; i++ {
fmt.Printf("walking partition %04d\n", i)
handtekeningenIDList, err := fetcher.ListPartition(i)
if err != nil {
log.Printf("error listing handtekeningen for partition %d: %v", i, err)
break
}
for _, id := range handtekeningenIDList {
chIDs <- id
}
}
close(chIDs)
}()
type checklist struct {
lock sync.Mutex
list map[string]uint
}
var checks []*checklist
for i := 0; i < 256; i++ {
checks = append(checks, &checklist{
list: make(map[string]uint),
})
}
var numWorkers = (runtime.NumCPU() - 1) * 2
wgDone := sync.WaitGroup{}
wgDone.Add(numWorkers)
for n := 0; n < numWorkers; n++ {
go func(n int) {
for {
id, ok := <-chIDs
if !ok {
break
}
h, err := fetcher.Fetch(id)
if err != nil {
log.Printf("error fetching %06d\n", id)
continue
}
hash := hashHandtekening(h)
cl := checks[int(hash[0])]
cl.lock.Lock()
cl.list[hash]++
cl.lock.Unlock()
}
fmt.Printf("worker %02d done\n", n)
wgDone.Done()
}(n)
}
wgDone.Wait()
var total uint
for i := 0; i < 256; i++ {
cl := checks[i]
for _, count := range cl.list {
if count > 1 {
total += count
fmt.Printf("%d, ", count)
}
}
}
fmt.Println("")
fmt.Println(total)
}