Exemplo n.º 1
0
func clearCookies(client *mwclient.Client, cookieFile string) {
	cookies := mwlib.ReadCookies(cookieFile)
	for idx, _ := range cookies {
		cookies[idx].MaxAge = -1
	}
	client.LoadCookies(cookies)
}
Exemplo n.º 2
0
func addCategory(page string, category string, client *mwclient.Client) {
	// There's a small chance that saving a page may fail due to
	// an edit conflict or other transient error. Try up to 3
	// times before giving up.
	var saveError error
	for i := 0; i < 3; i++ {
		text, timestamp, err := client.GetPageByName(page)
		if err != nil {
			panic(fmt.Sprintf("%v %v", page, err))
		}
		// Add the category at the end of the text, since categories
		// are supposed to be at the end anyway. A previous version
		// tried to add after the last existing category, but that
		// can fail when the text contains comments.
		last := len(text)
		text = text[0:last] + "\n[[" + category + "]]"
		editcfg := map[string]string{
			"action":        "edit",
			"title":         page,
			"text":          text,
			"summary":       "added [[" + category + "]]",
			"minor":         "",
			"bot":           "",
			"basetimestamp": timestamp,
		}
		saveError = client.Edit(editcfg)
		if saveError == nil {
			break
		}
	}
	if saveError != nil {
		panic(fmt.Sprintf("Failed to save %v %v", page, saveError))
	}
}
Exemplo n.º 3
0
// Return true if the client is logged in.
func checkLogin(client *mwclient.Client) bool {
	params := params.Values{
		"action":   "query",
		"assert":   "user",
		"continue": "",
	}
	_, err := client.Get(params)
	return err == nil
}
Exemplo n.º 4
0
// Handler for processing to be done when bot is terminating.
func EndProc(client *mwclient.Client, stats *stats, cookieFile string) {
	// Cookies can change while the bot is running, so save the latest values for the next run.
	mwlib.WriteCookies(client.DumpCookies(), cookieFile)

	if stats.examined > 1 {
		fmt.Println()
		stats.print()
	}
}
Exemplo n.º 5
0
func requestCategories(page string, client *mwclient.Client) *jason.Object {
	params := params.Values{
		"action":  "query",
		"titles":  page,
		"prop":    "categories",
		"cllimit": "max",
	}
	json, err := client.Get(params)
	if err != nil {
		panic(err)
	}
	return json
}
Exemplo n.º 6
0
// Given an array of page titles, return a mapping from page title to the array
// of categories which the page is a member of.
// If the page doesn't exist, no entry is added to the map.
// If the page has no categories, it will map to nil.
func getPageCategories(pages []string, client *mwclient.Client) map[string][]string {
	params := params.Values{
		"action":   "query",
		"titles":   mwlib.MakeTitleString(pages),
		"prop":     "categories",
		"cllimit":  "max",
		"continue": "",
	}
	json, err := client.Post(params) // Get may fail on long queries.
	if err != nil {
		fmt.Println(params)
		panic(err)
	}
	pageData, err := json.GetObject("query", "pages")
	if err != nil {
		panic(err)
	}
	pageMap := pageData.Map()
	result := make(map[string][]string)
	for pageId, page := range pageMap {
		pageObj, err := page.Object()
		if err != nil {
			panic(err)
		}
		title, err := pageObj.GetString("title")
		if err != nil {
			panic(err)
		}
		if pageId[0] == '-' {
			fmt.Println(title)
			fmt.Println("File does not exist, possibly deleted.")
			continue
		}
		categories, err := pageObj.GetObjectArray("categories")
		if err != nil {
			// Presumably the page has no categories.
			result[title] = nil
			continue
		}
		catArray := make([]string, len(categories))
		for i := range categories {
			catArray[i], err = categories[i].GetString("title")
			if err != nil {
				panic(err)
			}
		}
		result[title] = catArray
	}
	return result
}
Exemplo n.º 7
0
// Return a json object containing page title and imageinfo (Exif) data.
func GetImageinfo(page string, client *mwclient.Client) *jason.Object {
	params := params.Values{
		"action":    "query",
		"titles":    page,
		"prop":      "imageinfo",
		"iiprop":    "commonmetadata",
		"redirects": "", // follow redirects
		"continue":  "",
	}
	json, err := client.Get(params)
	if err != nil {
		panic(err)
	}
	return mwlib.GetJsonPage(json)
}
Exemplo n.º 8
0
func processGenerator(params params.Values, client *mwclient.Client, flags flags, verbose *log.Logger, categoryMap map[string]string, allCategories map[string]bool, catRegex []catRegex, stats *stats) {
	catCounts := make(map[string]int32)
	warnings := make(warnings, 0, 200)
	if flags.Gallery != "" {
		// try to write gallery even if there's a panic while processing files.
		defer checkWarnings(flags.Gallery, &warnings, client)
	}
	query := client.NewQuery(params)
	for query.Next() {
		json := query.Resp()
		pages, err := json.GetObject("query", "pages")
		if err != nil {
			// result set may be empty due to "miser mode" in the
			// the Mediawiki server.
			continue
		}
		pagesMap := pages.Map()
		if len(pagesMap) > 0 {
			files := make([]fileData, len(pagesMap))
			idx := 0
			for id, page := range pagesMap {
				if id == "-1" {
					// Empty result set.
					return
				}
				files[idx].pageObj, err = page.Object()
				if err != nil {
					panic(err)
				}
				idx++
				stats.examined++
			}
			if idx > 0 {
				processFiles(files, client, flags, verbose, categoryMap, allCategories, catRegex, catCounts, stats)
				warnings.Append(files)
			}
		}
		if flags.FileLimit > 0 && stats.examined >= flags.FileLimit {
			break
		}
		if flags.WarningLimit > 0 && stats.warnings >= flags.WarningLimit {
			break
		}
	}
	if query.Err() != nil {
		panic(query.Err())
	}
}
Exemplo n.º 9
0
// Examine the specified categories in the Wiki. For each one that actually
// exists, return the number of files that it contains. The result arrays
// give category names (in arbitrary order) and the corresponding count,
// and will have fewer entries than the input array if some categories
// were duplicated or didn't exist.
func catNumFiles(categories []string, client *mwclient.Client) ([]string, []int32) {
	params := params.Values{
		"action": "query",
		"titles": mwlib.MakeTitleString(categories),
		"prop":   "categoryinfo",
	}
	json, err := client.Post(params) // Get may fail on long queries.
	if err != nil {
		panic(err)
	}
	pages, err := json.GetObject("query", "pages")
	if err != nil {
		panic(err)
	}
	pageMap := pages.Map()
	var resultCats = make([]string, len(pageMap))
	var resultCounts = make([]int32, len(pageMap))
	var idx int32 = 0
	for pageId, page := range pageMap {
		pageObj, err := page.Object()
		if err != nil {
			panic(err)
		}
		if pageId[0] != '-' {
			resultCats[idx], err = pageObj.GetString("title")
			if err != nil {
				panic(err)
			}
			info, err := pageObj.GetObject("categoryinfo")
			// An error here means that the category is probably
			// empty, so just leave count at 0.
			if err == nil {
				files, err := info.GetInt64("files")
				if err != nil {
					panic(err)
				}
				resultCounts[idx] = int32(files)
			}
			idx++
		}
	}
	resultCats = resultCats[0:idx]
	resultCounts = resultCounts[0:idx]
	return resultCats, resultCounts
}
Exemplo n.º 10
0
func login(client *mwclient.Client, flags flags) bool {
	// Clear old session cookies, otherwise they remain in the cookiejar
	// as duplicates and remain in use.
	clearCookies(client, flags.CookieFile)
	username := os.Getenv("takenwith_username")
	if username == "" {
		warn.Print("Username for login not set in environment.")
		return false
	}
	password := os.Getenv("takenwith_password")
	if password == "" {
		warn.Print("Password for login not set in environment.")
		return false
	}
	err := client.Login(username, password)
	if err != nil {
		log.Print(err)
		return false
	}
	return true
}
Exemplo n.º 11
0
// Create a gallery showing all the files with warnings. Page must already
// exist and will be replaced.
func (warnings warnings) createGallery(gallery string, client *mwclient.Client) {
	var saveError error
	sort.Sort(warnings)
	for i := 0; i < 3; i++ {
		_, timestamp, err := client.GetPageByName(gallery)
		if err != nil {
			panic(fmt.Sprintf("%v %v", gallery, err))
		}
		// Blank the page and create a fresh gallery
		var buffer bytes.Buffer
		buffer.WriteString("<gallery>\n")
		for w := range warnings {
			buffer.WriteString(warnings[w].title)
			buffer.WriteByte('|')
			if len(warnings[w].warning) > 200 {
				buffer.WriteString(warnings[w].warning[0:200])
				buffer.WriteString("...")
			} else {
				buffer.WriteString(warnings[w].warning)
			}
			buffer.WriteByte('\n')
		}
		buffer.WriteString("</gallery>")
		editcfg := map[string]string{
			"action":        "edit",
			"title":         gallery,
			"text":          buffer.String(),
			"bot":           "",
			"basetimestamp": timestamp,
		}
		saveError = client.Edit(editcfg)
		if saveError == nil {
			break
		}
	}
	if saveError != nil {
		panic(fmt.Sprintf("Failed to save %v %v", gallery, saveError))
	}
}