// Single takes a name and returns the potential images of that person.
func Single(name string) (u *url.URL, err error) {
urls, err := Lookup(name)
if err != nil {
return nil, errutil.Err(err)
}
if urls == nil {
return nil, errutil.NewNoPosf("no match found for: %s.", name)
}
if len(urls) > 1 {
log.Println(errutil.NewNoPosf("more than %d hits for: %s.", len(urls), name))
}
return urls[0], err
}
// Parse ini settings section to global setting.
func parseSettings(config ini.Section) (err error) {
for fieldName := range config {
if _, found := settingsFields[fieldName]; !found {
return errutil.NewNoPosf(errFieldNotExist, fieldName)
}
}
// Get time setting from INI.
// If interval setting wasn't found, default value is 1 minute
intervalStr := config.S(fieldInterval, settings.DefaultInterval.String())
// Parse string to duration.
settings.Global.Interval, err = time.ParseDuration(intervalStr)
if err != nil {
return errutil.Err(err)
}
// Set global file permissions.
settings.Global.FilePerms = os.FileMode(config.I(fieldFilePerms, int(settings.DefaultFilePerms)))
// Set port number.
settings.Global.PortNum = config.S(fieldPortNum, settings.DefaultPortNum)
// Set browser path.
settings.Global.Browser = config.S(fieldBrowser, "")
return nil
}
// urlEncode is used for xfingers ISO-8859-1 encoding of special characters.
func urlEncode(name string) (string, error) {
n, ok := mahonia.NewEncoder("ISO-8859-1").ConvertStringOK(name)
if !ok {
return "", errutil.NewNoPosf("name contains non illegal charset characters: %s.", n)
}
return url.QueryEscape(n), nil
}
// Download the page with or without user specified headers.
func (p *Page) download() (doc *html.Node, err error) {
// Construct the request.
req, err := http.NewRequest("GET", p.ReqUrl.String(), nil)
if err != nil {
return nil, errutil.Err(err)
}
// If special headers were specified, add them to the request.
if p.Settings.Header != nil {
for key, val := range p.Settings.Header {
req.Header.Add(key, val)
}
}
// Do request and read response.
resp, err := http.DefaultClient.Do(req)
if err != nil {
if serr, ok := err.(*url.Error); ok {
if serr.Err == io.EOF {
return nil, errutil.NewNoPosf("Update was empty: %s", p.ReqUrl)
}
}
return nil, errutil.Err(err)
}
defer resp.Body.Close()
// If response contained a client or server error, fail with that error.
if resp.StatusCode >= 400 {
return nil, errutil.Newf("%s: (%d) - %s", p.ReqUrl.String(), resp.StatusCode, resp.Status)
}
// Read the response body to []byte.
buf, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, errutil.Err(err)
}
// Fix charset problems with servers that doesn't use utf-8
charset := "utf-8"
content := string(buf)
types := strings.Split(resp.Header.Get("Content-Type"), ` `)
for _, typ := range types {
if strings.Contains(typ, "charset") {
keyval := strings.Split(typ, `=`)
if len(keyval) == 2 {
charset = keyval[1]
}
}
}
if charset != "utf-8" {
content = mahonia.NewDecoder(charset).ConvertString(content)
}
// Parse response into html.Node.
return html.Parse(strings.NewReader(content))
}
func parseRarity(rarity string) (int, error) {
switch rarity {
case "common":
return Common, nil
case "magical":
return Magical, nil
case "artifact":
return Artifact, nil
default:
return 0, errutil.NewNoPosf("invalid rarity: %s", rarity)
}
}
// download downloads non-placeholding Osquarulda images and saves them in the
// current folder.
func download(osq Osquarulda) (err error) {
buf, err := httputil.Get(osq.ImgUrl.String())
if err != nil {
return errutil.Err(err)
}
// Compare the image to the placeholder image.
if bytes.Equal(buf, defaultPic) {
return errutil.NewNoPosf("%s missing picture.", osq.Name)
}
err = ioutil.WriteFile(osq.Name+".png", buf, 0777)
if err != nil {
return errutil.Err(err)
}
return nil
}
// Parse ini mail section to global setting.
func parseMail(mail ini.Section) (err error) {
for fieldName := range mail {
if _, found := mailFields[fieldName]; !found {
return errutil.NewNoPosf(errFieldNotExist, fieldName)
}
}
// Set global sender mail.
settings.Global.SenderMail.Address = mail.S(fieldSendMail, "")
if settings.Global.SenderMail.Address == "" {
return errutil.NewNoPosf(errMailAddressNotFound)
} else if !strings.Contains(settings.Global.SenderMail.Address, "@") {
return errutil.NewNoPosf(errInvalidMailAddress, settings.Global.SenderMail.Address)
}
// Set global sender mail password.
settings.Global.SenderMail.Password = mail.S(fieldSendPass, "")
// Set global sender authorization server.
settings.Global.SenderMail.AuthServer = mail.S(fieldSendAuthServer, "")
if settings.Global.SenderMail.AuthServer == "" {
return errutil.NewNoPosf(errMailAuthServerNotFound)
}
// Set global sender mail outgoing server.
settings.Global.SenderMail.OutServer = mail.S(fieldSendOutServer, "")
if settings.Global.SenderMail.OutServer == "" {
return errutil.NewNoPosf(errMailOutServerNotFound)
}
// Set global receive mail.
settings.Global.RecvMail = mail.S(fieldRecvMail, "")
if settings.Global.RecvMail == "" {
return errutil.NewNoPosf(errMailAddressNotFound)
} else if !strings.Contains(settings.Global.RecvMail, "@") {
return errutil.NewNoPosf(errInvalidMailAddress, settings.Global.RecvMail)
}
return nil
}
// check is an non-exported function for better error handling.
func (p *Page) check() (err error) {
if settings.Verbose {
fmt.Println("[/] Downloading:", p.ReqUrl.String())
}
// Retrieve result from download or return timeout error.
var r struct {
*html.Node
error
}
select {
case r = <-errWrapDownload(p):
if r.error != nil {
return errutil.Err(r.error)
}
case <-time.After(settings.TimeoutDuration):
return errutil.NewNoPosf("timeout: %s", p.ReqUrl.String())
}
// Extract selection from downloaded source.
selection, err := p.makeSelection(r.Node)
if err != nil {
return errutil.Err(err)
}
// Filename is the URL encoded and the protocol is stripped.
linuxPath, err := filename.Encode(p.UrlAsFilename())
if err != nil {
return errutil.Err(err)
}
// Debug - no selection.
debug, err := htmlutil.RenderClean(r.Node)
if err != nil {
return errutil.Err(err)
}
// Update the debug comparison file.
debugCachePathName := settings.DebugCacheRoot + linuxPath + ".htm"
err = ioutil.WriteFile(debugCachePathName, []byte(debug), settings.Global.FilePerms)
if err != nil {
return errutil.Err(err)
}
// If the selection is empty, the CSS selection is probably wrong so we will
// alert the user about this problem.
if len(selection) == 0 {
return errutil.NewNoPosf("Update was empty. URL: %s", p.ReqUrl)
}
cachePathName := settings.CacheRoot + linuxPath + ".htm"
// Read in comparison.
buf, err := ioutil.ReadFile(cachePathName)
if err != nil {
if !os.IsNotExist(err) {
return errutil.Err(err)
}
// If the page hasn't been checked before, create a new comparison file.
err = ioutil.WriteFile(
cachePathName,
[]byte(selection),
settings.Global.FilePerms,
)
if err != nil {
return errutil.Err(err)
}
readPathName := settings.ReadRoot + linuxPath + ".htm"
// If the page hasn't been checked before, create a new comparison file.
err = ioutil.WriteFile(
readPathName,
[]byte(selection),
settings.Global.FilePerms,
)
if err != nil {
return errutil.Err(err)
}
debugReadPathName := settings.DebugReadRoot + linuxPath + ".htm"
// Update the debug prev file.
err = ioutil.WriteFile(debugReadPathName, []byte(debug), settings.Global.FilePerms)
if err != nil {
return errutil.Err(err)
}
if settings.Verbose {
fmt.Println("[+] New site added:", p.ReqUrl.String())
}
return nil
}
// The distance between to strings in percentage.
dist := distance.Approx(string(buf), selection)
// If the distance is within the threshold level, i.e if the check was a
// match.
if dist > p.Settings.Threshold {
u := p.ReqUrl.String()
settings.Updates[u] = true
if settings.Verbose {
fmt.Println("[!] Updated:", p.ReqUrl.String())
}
// If the page has a mail and all compulsory global mail settings are
// set, send a mail to notify the user about an update.
if p.Settings.RecvMail != "" &&
settings.Global.SenderMail.AuthServer != "" &&
settings.Global.SenderMail.OutServer != "" &&
settings.Global.SenderMail.Address != "" {
// Mail the selection without the stripping functions, since their
// only purpose is to remove false-positives. It will make the
// output look better.
mailPage := Page{p.ReqUrl, p.Settings}
mailPage.Settings.StripFuncs = nil
mailPage.Settings.Regexp = ""
sel, err := mailPage.makeSelection(r.Node)
if err != nil {
return errutil.Err(err)
}
err = mail.Send(p.ReqUrl, p.Settings.RecvMail, sel)
if err != nil {
return errutil.Err(err)
}
delete(settings.Updates, u)
}
// Save updates to file.
err = settings.SaveUpdates()
if err != nil {
return errutil.Err(err)
}
// Update the comparison file.
err = ioutil.WriteFile(cachePathName, []byte(selection), settings.Global.FilePerms)
if err != nil {
return errutil.Err(err)
}
} else {
if settings.Verbose {
fmt.Println("[-] No update:", p.ReqUrl.String())
}
}
return nil
}
// ReadPages reads pages file and returns a slice of pages.
func ReadPages(pagesPath string) (pages []*page.Page, err error) {
// Parse pages file.
file := ini.New()
err = file.Load(pagesPath)
if err != nil {
return nil, errutil.Err(err)
}
// Loop through the INI sections ([section]) and parse page settings.
for name, section := range file.Sections {
// Skip global scope INI values since they are empty.
if len(name) == 0 {
continue
}
if settings.Verbose {
fmt.Println("[o] Watching:", name)
}
for fieldName := range section {
if _, found := siteFields[fieldName]; !found {
return nil, errutil.NewNoPosf(errFieldNotExist, fieldName)
}
}
var p page.Page
var pageSettings settings.Page
// Make INI section ([http://example.org]) into url.URL.
p.ReqUrl, err = url.Parse(name)
if err != nil {
return nil, errutil.Err(err)
}
// Set CSS selector.
pageSettings.Selection = section.S(fieldSelection, "")
// Set regular expression string.
pageSettings.Regexp = section.S(fieldRegexp, "")
// Set "negexp" (negative regular expression) string which removes all
// that matches it.
pageSettings.Negexp = section.S(fieldNegexp, "")
// Set threshold value.
pageSettings.Threshold = section.F64(fieldThreshold, 0)
// Set interval time.
intervalStr := section.S(fieldInterval, settings.Global.Interval.String())
// Parse string to duration.
pageSettings.Interval, err = time.ParseDuration(intervalStr)
if err != nil {
return nil, errutil.Err(err)
}
// Set individual mail address.
pageSettings.RecvMail = section.S(fieldRecvMail, settings.Global.RecvMail)
if pageSettings.RecvMail != "" && !strings.Contains(pageSettings.RecvMail, "@") {
return nil, errutil.NewNoPosf(errInvalidMailAddress, pageSettings.RecvMail)
}
// Set individual header.
headers := section.List(fieldHeader)
m := make(map[string]string)
for _, header := range headers {
if strings.Contains(header, ":") {
keyVal := strings.SplitN(header, ":", 2)
m[strings.TrimSpace(keyVal[0])] = strings.TrimSpace(keyVal[1])
} else {
return nil, errutil.NewNoPosf(errInvalidHeader, header)
}
}
pageSettings.Header = m
// Set strip functions to use.
pageSettings.StripFuncs = section.List(fieldStrip)
if pageSettings.StripFuncs == nil {
if _, found := section[fieldStrip]; found {
return nil, errutil.NewNoPosf(errInvalidListDeclaration)
}
}
for _, stripFunc := range pageSettings.StripFuncs {
if _, found := stripFunctions[stripFunc]; !found {
return nil, errutil.NewNoPosf(errInvalidStripFunction, stripFunc)
}
}
p.Settings = pageSettings
pages = append(pages, &p)
}
if pages == nil {
return nil, errutil.NewNoPosf("no pages in %s", settings.PagesPath)
}
return pages, nil
}