// clean removes old cache files from cache root.
func clean() (err error) {
// Get a list of all pages.
pages, err := ini.ReadPages(settings.PagesPath)
if err != nil {
return errutil.Err(err)
}
// Get a list of all cached pages.
caches, err := ioutil.ReadDir(settings.CacheRoot)
if err != nil {
return errutil.Err(err)
}
for _, cache := range caches {
remove := true
for _, p := range pages {
pageName, err := filename.Encode(p.UrlAsFilename())
if err != nil {
return errutil.Err(err)
}
if cache.Name() == pageName+".htm" {
remove = false
break
}
}
if remove {
err = os.Remove(settings.CacheRoot + cache.Name())
if err != nil {
return err
}
}
}
return nil
}
// Removes all updates.
func clearAll(bw *bufioutil.Writer, conn net.Conn) (err error) {
ups, err := getUpdates(bw, conn)
if err != nil {
return errutil.Err(err)
}
for up := range ups {
u, err := url.Parse(up)
if err != nil {
return errutil.Err(err)
}
urlAsFilename := u.Host + u.Path + u.RawQuery
fname, err := filename.Encode(urlAsFilename)
if err != nil {
return errutil.Err(err)
}
cacheFile, err := os.Open(settings.CacheRoot + fname + ".htm")
if err != nil {
return errutil.Err(err)
}
defer cacheFile.Close()
readFile, err := os.Create(settings.ReadRoot + fname + ".htm")
if err != nil {
return errutil.Err(err)
}
defer readFile.Close()
_, err = io.Copy(readFile, cacheFile)
if err != nil {
return errutil.Err(err)
}
// Debug
debugCacheFile, err := os.Open(settings.DebugCacheRoot + fname + ".htm")
if err != nil {
return errutil.Err(err)
}
defer cacheFile.Close()
debugReadFile, err := os.Create(settings.DebugReadRoot + fname + ".htm")
if err != nil {
return errutil.Err(err)
}
defer readFile.Close()
_, err = io.Copy(debugReadFile, debugCacheFile)
if err != nil {
return errutil.Err(err)
}
}
// Send nyfikend a query to clear updates.
_, err = bw.WriteLine(settings.QueryClearAll)
if err != nil {
return errutil.Err(err)
}
fmt.Println("Updates list has been cleared!")
return nil
}
// check is an non-exported function for better error handling.
func (p *Page) check() (err error) {
if settings.Verbose {
fmt.Println("[/] Downloading:", p.ReqUrl.String())
}
// Retrieve result from download or return timeout error.
var r struct {
*html.Node
error
}
select {
case r = <-errWrapDownload(p):
if r.error != nil {
return errutil.Err(r.error)
}
case <-time.After(settings.TimeoutDuration):
return errutil.NewNoPosf("timeout: %s", p.ReqUrl.String())
}
// Extract selection from downloaded source.
selection, err := p.makeSelection(r.Node)
if err != nil {
return errutil.Err(err)
}
// Filename is the URL encoded and the protocol is stripped.
linuxPath, err := filename.Encode(p.UrlAsFilename())
if err != nil {
return errutil.Err(err)
}
// Debug - no selection.
debug, err := htmlutil.RenderClean(r.Node)
if err != nil {
return errutil.Err(err)
}
// Update the debug comparison file.
debugCachePathName := settings.DebugCacheRoot + linuxPath + ".htm"
err = ioutil.WriteFile(debugCachePathName, []byte(debug), settings.Global.FilePerms)
if err != nil {
return errutil.Err(err)
}
// If the selection is empty, the CSS selection is probably wrong so we will
// alert the user about this problem.
if len(selection) == 0 {
return errutil.NewNoPosf("Update was empty. URL: %s", p.ReqUrl)
}
cachePathName := settings.CacheRoot + linuxPath + ".htm"
// Read in comparison.
buf, err := ioutil.ReadFile(cachePathName)
if err != nil {
if !os.IsNotExist(err) {
return errutil.Err(err)
}
// If the page hasn't been checked before, create a new comparison file.
err = ioutil.WriteFile(
cachePathName,
[]byte(selection),
settings.Global.FilePerms,
)
if err != nil {
return errutil.Err(err)
}
readPathName := settings.ReadRoot + linuxPath + ".htm"
// If the page hasn't been checked before, create a new comparison file.
err = ioutil.WriteFile(
readPathName,
[]byte(selection),
settings.Global.FilePerms,
)
if err != nil {
return errutil.Err(err)
}
debugReadPathName := settings.DebugReadRoot + linuxPath + ".htm"
// Update the debug prev file.
err = ioutil.WriteFile(debugReadPathName, []byte(debug), settings.Global.FilePerms)
if err != nil {
return errutil.Err(err)
}
if settings.Verbose {
fmt.Println("[+] New site added:", p.ReqUrl.String())
}
return nil
}
// The distance between to strings in percentage.
dist := distance.Approx(string(buf), selection)
// If the distance is within the threshold level, i.e if the check was a
// match.
if dist > p.Settings.Threshold {
u := p.ReqUrl.String()
settings.Updates[u] = true
if settings.Verbose {
fmt.Println("[!] Updated:", p.ReqUrl.String())
}
// If the page has a mail and all compulsory global mail settings are
// set, send a mail to notify the user about an update.
if p.Settings.RecvMail != "" &&
settings.Global.SenderMail.AuthServer != "" &&
settings.Global.SenderMail.OutServer != "" &&
settings.Global.SenderMail.Address != "" {
// Mail the selection without the stripping functions, since their
// only purpose is to remove false-positives. It will make the
// output look better.
mailPage := Page{p.ReqUrl, p.Settings}
mailPage.Settings.StripFuncs = nil
mailPage.Settings.Regexp = ""
sel, err := mailPage.makeSelection(r.Node)
if err != nil {
return errutil.Err(err)
}
err = mail.Send(p.ReqUrl, p.Settings.RecvMail, sel)
if err != nil {
return errutil.Err(err)
}
delete(settings.Updates, u)
}
// Save updates to file.
err = settings.SaveUpdates()
if err != nil {
return errutil.Err(err)
}
// Update the comparison file.
err = ioutil.WriteFile(cachePathName, []byte(selection), settings.Global.FilePerms)
if err != nil {
return errutil.Err(err)
}
} else {
if settings.Verbose {
fmt.Println("[-] No update:", p.ReqUrl.String())
}
}
return nil
}