func processGenerator(params params.Values, client *mwclient.Client, flags flags, verbose *log.Logger, categoryMap map[string]string, allCategories map[string]bool, catRegex []catRegex, stats *stats) { catCounts := make(map[string]int32) warnings := make(warnings, 0, 200) if flags.Gallery != "" { // try to write gallery even if there's a panic while processing files. defer checkWarnings(flags.Gallery, &warnings, client) } query := client.NewQuery(params) for query.Next() { json := query.Resp() pages, err := json.GetObject("query", "pages") if err != nil { // result set may be empty due to "miser mode" in the // the Mediawiki server. continue } pagesMap := pages.Map() if len(pagesMap) > 0 { files := make([]fileData, len(pagesMap)) idx := 0 for id, page := range pagesMap { if id == "-1" { // Empty result set. return } files[idx].pageObj, err = page.Object() if err != nil { panic(err) } idx++ stats.examined++ } if idx > 0 { processFiles(files, client, flags, verbose, categoryMap, allCategories, catRegex, catCounts, stats) warnings.Append(files) } } if flags.FileLimit > 0 && stats.examined >= flags.FileLimit { break } if flags.WarningLimit > 0 && stats.warnings >= flags.WarningLimit { break } } if query.Err() != nil { panic(query.Err()) } }