Beispiel #1
0
// clean removes old cache files from cache root.
func clean() (err error) {
	// Get a list of all pages.
	pages, err := ini.ReadPages(settings.PagesPath)
	if err != nil {
		return errutil.Err(err)
	}

	// Get a list of all cached pages.
	caches, err := ioutil.ReadDir(settings.CacheRoot)
	if err != nil {
		return errutil.Err(err)
	}

	for _, cache := range caches {
		remove := true
		for _, p := range pages {
			pageName, err := filename.Encode(p.UrlAsFilename())
			if err != nil {
				return errutil.Err(err)
			}
			if cache.Name() == pageName+".htm" {
				remove = false
				break
			}
		}
		if remove {
			err = os.Remove(settings.CacheRoot + cache.Name())
			if err != nil {
				return err
			}
		}
	}

	return nil
}
Beispiel #2
0
// Removes all updates.
func clearAll(bw *bufioutil.Writer, conn net.Conn) (err error) {
	ups, err := getUpdates(bw, conn)
	if err != nil {
		return errutil.Err(err)
	}

	for up := range ups {
		u, err := url.Parse(up)
		if err != nil {
			return errutil.Err(err)
		}

		urlAsFilename := u.Host + u.Path + u.RawQuery
		fname, err := filename.Encode(urlAsFilename)
		if err != nil {
			return errutil.Err(err)
		}

		cacheFile, err := os.Open(settings.CacheRoot + fname + ".htm")
		if err != nil {
			return errutil.Err(err)
		}
		defer cacheFile.Close()
		readFile, err := os.Create(settings.ReadRoot + fname + ".htm")
		if err != nil {
			return errutil.Err(err)
		}
		defer readFile.Close()

		_, err = io.Copy(readFile, cacheFile)
		if err != nil {
			return errutil.Err(err)
		}

		// Debug
		debugCacheFile, err := os.Open(settings.DebugCacheRoot + fname + ".htm")
		if err != nil {
			return errutil.Err(err)
		}
		defer cacheFile.Close()
		debugReadFile, err := os.Create(settings.DebugReadRoot + fname + ".htm")
		if err != nil {
			return errutil.Err(err)
		}
		defer readFile.Close()

		_, err = io.Copy(debugReadFile, debugCacheFile)
		if err != nil {
			return errutil.Err(err)
		}
	}

	// Send nyfikend a query to clear updates.
	_, err = bw.WriteLine(settings.QueryClearAll)
	if err != nil {
		return errutil.Err(err)
	}

	fmt.Println("Updates list has been cleared!")
	return nil
}
Beispiel #3
0
// check is an non-exported function for better error handling.
func (p *Page) check() (err error) {
	if settings.Verbose {
		fmt.Println("[/] Downloading:", p.ReqUrl.String())
	}

	// Retrieve result from download or return timeout error.
	var r struct {
		*html.Node
		error
	}
	select {
	case r = <-errWrapDownload(p):
		if r.error != nil {
			return errutil.Err(r.error)
		}
	case <-time.After(settings.TimeoutDuration):
		return errutil.NewNoPosf("timeout: %s", p.ReqUrl.String())
	}

	// Extract selection from downloaded source.
	selection, err := p.makeSelection(r.Node)
	if err != nil {
		return errutil.Err(err)
	}

	// Filename is the URL encoded and the protocol is stripped.
	linuxPath, err := filename.Encode(p.UrlAsFilename())
	if err != nil {
		return errutil.Err(err)
	}

	// Debug - no selection.
	debug, err := htmlutil.RenderClean(r.Node)
	if err != nil {
		return errutil.Err(err)
	}
	// Update the debug comparison file.
	debugCachePathName := settings.DebugCacheRoot + linuxPath + ".htm"
	err = ioutil.WriteFile(debugCachePathName, []byte(debug), settings.Global.FilePerms)
	if err != nil {
		return errutil.Err(err)
	}

	// If the selection is empty, the CSS selection is probably wrong so we will
	// alert the user about this problem.
	if len(selection) == 0 {
		return errutil.NewNoPosf("Update was empty. URL: %s", p.ReqUrl)
	}

	cachePathName := settings.CacheRoot + linuxPath + ".htm"

	// Read in comparison.
	buf, err := ioutil.ReadFile(cachePathName)
	if err != nil {
		if !os.IsNotExist(err) {
			return errutil.Err(err)
		}

		// If the page hasn't been checked before, create a new comparison file.
		err = ioutil.WriteFile(
			cachePathName,
			[]byte(selection),
			settings.Global.FilePerms,
		)
		if err != nil {
			return errutil.Err(err)
		}

		readPathName := settings.ReadRoot + linuxPath + ".htm"
		// If the page hasn't been checked before, create a new comparison file.
		err = ioutil.WriteFile(
			readPathName,
			[]byte(selection),
			settings.Global.FilePerms,
		)
		if err != nil {
			return errutil.Err(err)
		}

		debugReadPathName := settings.DebugReadRoot + linuxPath + ".htm"

		// Update the debug prev file.
		err = ioutil.WriteFile(debugReadPathName, []byte(debug), settings.Global.FilePerms)
		if err != nil {
			return errutil.Err(err)
		}

		if settings.Verbose {
			fmt.Println("[+] New site added:", p.ReqUrl.String())
		}

		return nil
	}

	// The distance between to strings in percentage.
	dist := distance.Approx(string(buf), selection)

	// If the distance is within the threshold level, i.e if the check was a
	// match.
	if dist > p.Settings.Threshold {
		u := p.ReqUrl.String()
		settings.Updates[u] = true

		if settings.Verbose {
			fmt.Println("[!] Updated:", p.ReqUrl.String())
		}

		// If the page has a mail and all compulsory global mail settings are
		// set, send a mail to notify the user about an update.
		if p.Settings.RecvMail != "" &&
			settings.Global.SenderMail.AuthServer != "" &&
			settings.Global.SenderMail.OutServer != "" &&
			settings.Global.SenderMail.Address != "" {

			// Mail the selection without the stripping functions, since their
			// only purpose is to remove false-positives. It will make the
			// output look better.
			mailPage := Page{p.ReqUrl, p.Settings}
			mailPage.Settings.StripFuncs = nil
			mailPage.Settings.Regexp = ""
			sel, err := mailPage.makeSelection(r.Node)
			if err != nil {
				return errutil.Err(err)
			}

			err = mail.Send(p.ReqUrl, p.Settings.RecvMail, sel)
			if err != nil {
				return errutil.Err(err)
			}
			delete(settings.Updates, u)
		}
		// Save updates to file.
		err = settings.SaveUpdates()
		if err != nil {
			return errutil.Err(err)
		}

		// Update the comparison file.
		err = ioutil.WriteFile(cachePathName, []byte(selection), settings.Global.FilePerms)
		if err != nil {
			return errutil.Err(err)
		}
	} else {
		if settings.Verbose {
			fmt.Println("[-] No update:", p.ReqUrl.String())
		}
	}
	return nil
}