// check is an non-exported function for better error handling.
func (p *Page) check() (err error) {
if settings.Verbose {
fmt.Println("[/] Downloading:", p.ReqUrl.String())
}
// Retrieve result from download or return timeout error.
var r struct {
*html.Node
error
}
select {
case r = <-errWrapDownload(p):
if r.error != nil {
return errutil.Err(r.error)
}
case <-time.After(settings.TimeoutDuration):
return errutil.NewNoPosf("timeout: %s", p.ReqUrl.String())
}
// Extract selection from downloaded source.
selection, err := p.makeSelection(r.Node)
if err != nil {
return errutil.Err(err)
}
// Filename is the URL encoded and the protocol is stripped.
linuxPath, err := filename.Encode(p.UrlAsFilename())
if err != nil {
return errutil.Err(err)
}
// Debug - no selection.
debug, err := htmlutil.RenderClean(r.Node)
if err != nil {
return errutil.Err(err)
}
// Update the debug comparison file.
debugCachePathName := settings.DebugCacheRoot + linuxPath + ".htm"
err = ioutil.WriteFile(debugCachePathName, []byte(debug), settings.Global.FilePerms)
if err != nil {
return errutil.Err(err)
}
// If the selection is empty, the CSS selection is probably wrong so we will
// alert the user about this problem.
if len(selection) == 0 {
return errutil.NewNoPosf("Update was empty. URL: %s", p.ReqUrl)
}
cachePathName := settings.CacheRoot + linuxPath + ".htm"
// Read in comparison.
buf, err := ioutil.ReadFile(cachePathName)
if err != nil {
if !os.IsNotExist(err) {
return errutil.Err(err)
}
// If the page hasn't been checked before, create a new comparison file.
err = ioutil.WriteFile(
cachePathName,
[]byte(selection),
settings.Global.FilePerms,
)
if err != nil {
return errutil.Err(err)
}
readPathName := settings.ReadRoot + linuxPath + ".htm"
// If the page hasn't been checked before, create a new comparison file.
err = ioutil.WriteFile(
readPathName,
[]byte(selection),
settings.Global.FilePerms,
)
if err != nil {
return errutil.Err(err)
}
debugReadPathName := settings.DebugReadRoot + linuxPath + ".htm"
// Update the debug prev file.
err = ioutil.WriteFile(debugReadPathName, []byte(debug), settings.Global.FilePerms)
if err != nil {
return errutil.Err(err)
}
if settings.Verbose {
fmt.Println("[+] New site added:", p.ReqUrl.String())
}
return nil
}
// The distance between to strings in percentage.
dist := distance.Approx(string(buf), selection)
// If the distance is within the threshold level, i.e if the check was a
// match.
if dist > p.Settings.Threshold {
u := p.ReqUrl.String()
settings.Updates[u] = true
if settings.Verbose {
fmt.Println("[!] Updated:", p.ReqUrl.String())
}
// If the page has a mail and all compulsory global mail settings are
// set, send a mail to notify the user about an update.
if p.Settings.RecvMail != "" &&
settings.Global.SenderMail.AuthServer != "" &&
settings.Global.SenderMail.OutServer != "" &&
settings.Global.SenderMail.Address != "" {
// Mail the selection without the stripping functions, since their
// only purpose is to remove false-positives. It will make the
// output look better.
mailPage := Page{p.ReqUrl, p.Settings}
mailPage.Settings.StripFuncs = nil
mailPage.Settings.Regexp = ""
sel, err := mailPage.makeSelection(r.Node)
if err != nil {
return errutil.Err(err)
}
err = mail.Send(p.ReqUrl, p.Settings.RecvMail, sel)
if err != nil {
return errutil.Err(err)
}
delete(settings.Updates, u)
}
// Save updates to file.
err = settings.SaveUpdates()
if err != nil {
return errutil.Err(err)
}
// Update the comparison file.
err = ioutil.WriteFile(cachePathName, []byte(selection), settings.Global.FilePerms)
if err != nil {
return errutil.Err(err)
}
} else {
if settings.Verbose {
fmt.Println("[-] No update:", p.ReqUrl.String())
}
}
return nil
}
// Select from the retrived page source the CSS selection defined in c4c.ini.
func (p *Page) makeSelection(htmlNode *html.Node) (selection string, err error) {
// --- [ CSS selection ] --------------------------------------------------/
// Write results into an array of nodes.
var result []*html.Node
// Append the whole page (htmlNode) to results if no selector where chosen.
if p.Settings.Selection == "" {
result = append(result, htmlNode)
} else {
// Make a selector from the user specified string.
s, err := cascadia.Compile(p.Settings.Selection)
if err != nil {
return "", errutil.Err(err)
}
// Find all nodes that matches selection s.
result = s.MatchAll(htmlNode)
}
// Loop through all the hits and render them to string.
for _, hit := range result {
s, err := htmlutil.RenderClean(hit)
if err != nil {
return "", errutil.Err(err)
}
selection += s
}
// --- [ /CSS selection ] -------------------------------------------------/
// --- [ Strip funcs ] ----------------------------------------------------/
for _, stripFunc := range p.Settings.StripFuncs {
doc, err := html.Parse(strings.NewReader(selection))
if err != nil {
return "", errutil.Err(err)
}
stripFunc = strings.ToLower(stripFunc)
switch stripFunc {
case "numbers":
strip.Numbers(doc)
case "attrs":
strip.Attrs(doc)
case "html":
strip.HTML(doc)
case "scripts":
strip.Scripts(doc)
}
selection, err = htmlutil.RenderClean(doc)
if err != nil {
return "", errutil.Err(err)
}
}
// --- [ /Strip funcs ] ---------------------------------------------------/
// --- [ Regexp ] ---------------------------------------------------------/
if p.Settings.Regexp != "" {
re, err := regexp.Compile(p.Settings.Regexp)
if err != nil {
return "", errutil.Err(err)
}
// -1 means to find all.
result := re.FindAllString(selection, -1)
selection = ""
for _, res := range result {
selection += res + settings.Newline
}
}
// --- [ /Regexp ] --------------------------------------------------------/
// --- [ Negexp ] ---------------------------------------------------------/
if p.Settings.Negexp != "" {
ne, err := regexp.Compile(p.Settings.Negexp)
if err != nil {
return "", errutil.Err(err)
}
// Remove all that matches the regular expression ne
selection = ne.ReplaceAllString(selection, "")
}
// --- [ /Negexp ] --------------------------------------------------------/
return selection, nil
}