func clearCookies(client *mwclient.Client, cookieFile string) {
cookies := mwlib.ReadCookies(cookieFile)
for idx, _ := range cookies {
cookies[idx].MaxAge = -1
}
client.LoadCookies(cookies)
}
func addCategory(page string, category string, client *mwclient.Client) {
// There's a small chance that saving a page may fail due to
// an edit conflict or other transient error. Try up to 3
// times before giving up.
var saveError error
for i := 0; i < 3; i++ {
text, timestamp, err := client.GetPageByName(page)
if err != nil {
panic(fmt.Sprintf("%v %v", page, err))
}
// Add the category at the end of the text, since categories
// are supposed to be at the end anyway. A previous version
// tried to add after the last existing category, but that
// can fail when the text contains comments.
last := len(text)
text = text[0:last] + "\n[[" + category + "]]"
editcfg := map[string]string{
"action": "edit",
"title": page,
"text": text,
"summary": "added [[" + category + "]]",
"minor": "",
"bot": "",
"basetimestamp": timestamp,
}
saveError = client.Edit(editcfg)
if saveError == nil {
break
}
}
if saveError != nil {
panic(fmt.Sprintf("Failed to save %v %v", page, saveError))
}
}
// Return true if the client is logged in.
func checkLogin(client *mwclient.Client) bool {
params := params.Values{
"action": "query",
"assert": "user",
"continue": "",
}
_, err := client.Get(params)
return err == nil
}
// Handler for processing to be done when bot is terminating.
func EndProc(client *mwclient.Client, stats *stats, cookieFile string) {
// Cookies can change while the bot is running, so save the latest values for the next run.
mwlib.WriteCookies(client.DumpCookies(), cookieFile)
if stats.examined > 1 {
fmt.Println()
stats.print()
}
}
func requestCategories(page string, client *mwclient.Client) *jason.Object {
params := params.Values{
"action": "query",
"titles": page,
"prop": "categories",
"cllimit": "max",
}
json, err := client.Get(params)
if err != nil {
panic(err)
}
return json
}
// Given an array of page titles, return a mapping from page title to the array
// of categories which the page is a member of.
// If the page doesn't exist, no entry is added to the map.
// If the page has no categories, it will map to nil.
func getPageCategories(pages []string, client *mwclient.Client) map[string][]string {
params := params.Values{
"action": "query",
"titles": mwlib.MakeTitleString(pages),
"prop": "categories",
"cllimit": "max",
"continue": "",
}
json, err := client.Post(params) // Get may fail on long queries.
if err != nil {
fmt.Println(params)
panic(err)
}
pageData, err := json.GetObject("query", "pages")
if err != nil {
panic(err)
}
pageMap := pageData.Map()
result := make(map[string][]string)
for pageId, page := range pageMap {
pageObj, err := page.Object()
if err != nil {
panic(err)
}
title, err := pageObj.GetString("title")
if err != nil {
panic(err)
}
if pageId[0] == '-' {
fmt.Println(title)
fmt.Println("File does not exist, possibly deleted.")
continue
}
categories, err := pageObj.GetObjectArray("categories")
if err != nil {
// Presumably the page has no categories.
result[title] = nil
continue
}
catArray := make([]string, len(categories))
for i := range categories {
catArray[i], err = categories[i].GetString("title")
if err != nil {
panic(err)
}
}
result[title] = catArray
}
return result
}
// Return a json object containing page title and imageinfo (Exif) data.
func GetImageinfo(page string, client *mwclient.Client) *jason.Object {
params := params.Values{
"action": "query",
"titles": page,
"prop": "imageinfo",
"iiprop": "commonmetadata",
"redirects": "", // follow redirects
"continue": "",
}
json, err := client.Get(params)
if err != nil {
panic(err)
}
return mwlib.GetJsonPage(json)
}
func processGenerator(params params.Values, client *mwclient.Client, flags flags, verbose *log.Logger, categoryMap map[string]string, allCategories map[string]bool, catRegex []catRegex, stats *stats) {
catCounts := make(map[string]int32)
warnings := make(warnings, 0, 200)
if flags.Gallery != "" {
// try to write gallery even if there's a panic while processing files.
defer checkWarnings(flags.Gallery, &warnings, client)
}
query := client.NewQuery(params)
for query.Next() {
json := query.Resp()
pages, err := json.GetObject("query", "pages")
if err != nil {
// result set may be empty due to "miser mode" in the
// the Mediawiki server.
continue
}
pagesMap := pages.Map()
if len(pagesMap) > 0 {
files := make([]fileData, len(pagesMap))
idx := 0
for id, page := range pagesMap {
if id == "-1" {
// Empty result set.
return
}
files[idx].pageObj, err = page.Object()
if err != nil {
panic(err)
}
idx++
stats.examined++
}
if idx > 0 {
processFiles(files, client, flags, verbose, categoryMap, allCategories, catRegex, catCounts, stats)
warnings.Append(files)
}
}
if flags.FileLimit > 0 && stats.examined >= flags.FileLimit {
break
}
if flags.WarningLimit > 0 && stats.warnings >= flags.WarningLimit {
break
}
}
if query.Err() != nil {
panic(query.Err())
}
}
// Examine the specified categories in the Wiki. For each one that actually
// exists, return the number of files that it contains. The result arrays
// give category names (in arbitrary order) and the corresponding count,
// and will have fewer entries than the input array if some categories
// were duplicated or didn't exist.
func catNumFiles(categories []string, client *mwclient.Client) ([]string, []int32) {
params := params.Values{
"action": "query",
"titles": mwlib.MakeTitleString(categories),
"prop": "categoryinfo",
}
json, err := client.Post(params) // Get may fail on long queries.
if err != nil {
panic(err)
}
pages, err := json.GetObject("query", "pages")
if err != nil {
panic(err)
}
pageMap := pages.Map()
var resultCats = make([]string, len(pageMap))
var resultCounts = make([]int32, len(pageMap))
var idx int32 = 0
for pageId, page := range pageMap {
pageObj, err := page.Object()
if err != nil {
panic(err)
}
if pageId[0] != '-' {
resultCats[idx], err = pageObj.GetString("title")
if err != nil {
panic(err)
}
info, err := pageObj.GetObject("categoryinfo")
// An error here means that the category is probably
// empty, so just leave count at 0.
if err == nil {
files, err := info.GetInt64("files")
if err != nil {
panic(err)
}
resultCounts[idx] = int32(files)
}
idx++
}
}
resultCats = resultCats[0:idx]
resultCounts = resultCounts[0:idx]
return resultCats, resultCounts
}
func login(client *mwclient.Client, flags flags) bool {
// Clear old session cookies, otherwise they remain in the cookiejar
// as duplicates and remain in use.
clearCookies(client, flags.CookieFile)
username := os.Getenv("takenwith_username")
if username == "" {
warn.Print("Username for login not set in environment.")
return false
}
password := os.Getenv("takenwith_password")
if password == "" {
warn.Print("Password for login not set in environment.")
return false
}
err := client.Login(username, password)
if err != nil {
log.Print(err)
return false
}
return true
}
// Create a gallery showing all the files with warnings. Page must already
// exist and will be replaced.
func (warnings warnings) createGallery(gallery string, client *mwclient.Client) {
var saveError error
sort.Sort(warnings)
for i := 0; i < 3; i++ {
_, timestamp, err := client.GetPageByName(gallery)
if err != nil {
panic(fmt.Sprintf("%v %v", gallery, err))
}
// Blank the page and create a fresh gallery
var buffer bytes.Buffer
buffer.WriteString("<gallery>\n")
for w := range warnings {
buffer.WriteString(warnings[w].title)
buffer.WriteByte('|')
if len(warnings[w].warning) > 200 {
buffer.WriteString(warnings[w].warning[0:200])
buffer.WriteString("...")
} else {
buffer.WriteString(warnings[w].warning)
}
buffer.WriteByte('\n')
}
buffer.WriteString("</gallery>")
editcfg := map[string]string{
"action": "edit",
"title": gallery,
"text": buffer.String(),
"bot": "",
"basetimestamp": timestamp,
}
saveError = client.Edit(editcfg)
if saveError == nil {
break
}
}
if saveError != nil {
panic(fmt.Sprintf("Failed to save %v %v", gallery, saveError))
}
}