func processGenerator(params params.Values, client *mwclient.Client, flags flags, verbose *log.Logger, categoryMap map[string]string, allCategories map[string]bool, catRegex []catRegex, stats *stats) {
catCounts := make(map[string]int32)
warnings := make(warnings, 0, 200)
if flags.Gallery != "" {
// try to write gallery even if there's a panic while processing files.
defer checkWarnings(flags.Gallery, &warnings, client)
}
query := client.NewQuery(params)
for query.Next() {
json := query.Resp()
pages, err := json.GetObject("query", "pages")
if err != nil {
// result set may be empty due to "miser mode" in the
// the Mediawiki server.
continue
}
pagesMap := pages.Map()
if len(pagesMap) > 0 {
files := make([]fileData, len(pagesMap))
idx := 0
for id, page := range pagesMap {
if id == "-1" {
// Empty result set.
return
}
files[idx].pageObj, err = page.Object()
if err != nil {
panic(err)
}
idx++
stats.examined++
}
if idx > 0 {
processFiles(files, client, flags, verbose, categoryMap, allCategories, catRegex, catCounts, stats)
warnings.Append(files)
}
}
if flags.FileLimit > 0 && stats.examined >= flags.FileLimit {
break
}
if flags.WarningLimit > 0 && stats.warnings >= flags.WarningLimit {
break
}
}
if query.Err() != nil {
panic(query.Err())
}
}
