func NewHawkHandler(handler http.Handler, secrets []string) *HawkHandler {
// the m value for the bloom filter is likely larger than
// we need. It figures 60,000 requests/minute * 50 = 3,000,000 bits
// or ~2.8MB. The code rotates between two of them so about 5.6MB
// of memory for nonce checking.
m := uint(1000 * 60 * 50)
return &HawkHandler{
handler: handler,
secrets: secrets,
bloomPrev: bloom.New(m, 5),
bloomNow: bloom.New(m, 5),
bloomHalflife: 30 * time.Second,
lastRotate: time.Now(),
}
}
func check(cmd *cobra.Command, args []string) error {
if err := InitializeConfig(checkCmd); err != nil {
return err
}
// Initialize bloom filter 1M max elements, 5 hash
bf := bloom.New(1000000, 5)
// Check if bloom filter file exists
fileName := viper.GetString("DatabasePath")
if _, err := os.Stat(fileName); err == nil {
bff, err2 := os.Open(fileName)
if err2 != nil {
panic(err2)
}
defer bff.Close()
if _, err3 := bf.ReadFrom(bff); err != nil {
panic(err3)
}
} else {
fmt.Println("You must build a filter first, use 'build' command.")
return err
}
if len(args) > 0 {
for _, domain := range args {
fmt.Printf("%s,%v\n", domain, bf.TestString(domain))
}
}
return nil
}
// Creates a new TCP Server with a bloom filter (m, k)
func NewServer(dsn string, m uint, k uint) (*Server, error) {
listener, err := net.Listen("tcp", dsn)
if listener == nil {
return nil, err
}
f := bloom.New(m, k)
return &Server{listener: listener, f: f}, nil
}
func build(cmd *cobra.Command, args []string) error {
if err := InitializeConfig(buildCmd); err != nil {
return err
}
// It is a good day to die !
now := time.Now().UTC()
// Initialize bloom filter 1M max elements, 5 hash
bf := bloom.New(1000000, 5)
// Check if bloom filter file exists
fileName := viper.GetString("DatabasePath")
if _, err := os.Stat(fileName); err == nil {
bff, err := os.Open(fileName)
if err != nil {
panic(err)
}
defer bff.Close()
if _, err := bf.ReadFrom(bff); err != nil {
panic(err)
}
}
// future days loop
for day := 0; day < afterDayCount; day++ {
dgaGenerate(now, bf)
// Add 1 day
now = now.AddDate(0, 0, 1)
}
// Reset to today
now = time.Now().UTC()
// past days loop
for day := 0; day < beforeDayCount; day++ {
dgaGenerate(now, bf)
// Substract 1 day
now = now.AddDate(0, 0, -1)
}
// Save bloom filter
bff, err := os.Create(fileName)
if err != nil {
panic(err)
}
defer bff.Close()
// Dump to file
_, err = bf.WriteTo(bff)
if err != nil {
panic(err)
}
return nil
}
func main() {
var n uint = 1000
filter := bloom.New(20*n, 5) // load of 20, 5 keys
filter.Add([]byte("Love"))
if filter.Test([]byte("Love")) {
println("It is in black list!")
}
}
// NewDeduplicator initializes a new deduplicator
func NewDeduplicator(computation StreamID, recordTracker ReceivedTracker) (Deduplicator, error) {
dd := &dedup{
computation: computation,
recordTracker: recordTracker,
filter: bloom.New(20000, 5),
}
recent, err := recordTracker.GetRecentReceived(computation)
if err != nil {
return nil, err
}
for _, recordID := range recent {
dd.filter.AddString(recordID)
}
return dd, nil
}
func main() {
m := flag.Uint("m", 0, "")
k := flag.Uint("k", 0, "")
flag.Parse()
filter := bloom.New(*m, *k)
scanner := bufio.NewScanner(os.Stdin)
n := 0
collisions := 0
for scanner.Scan() {
if filter.TestAndAdd(scanner.Bytes()) {
collisions++
}
n++
}
if err := scanner.Err(); err != nil {
fmt.Fprintf(os.Stderr, "error reading stdin: %s", err)
os.Exit(1)
}
fmt.Printf("collisions %d/%d (%f)\n", collisions, n, float64(collisions)/float64(n)*100)
}
// Creates a new walker object
func New(rawBaseUrl string) (w *Walker) {
// Parse the base url
baseUrl, err := url.Parse(rawBaseUrl)
// Fail on error
if err != nil {
log.Fatal("baseURL (", baseUrl, ") failed to parse: ", err)
}
rootNode := buildNode(nil, baseUrl)
// Create the bloom filter
filter := bloom.New(20000, 5)
// Create the object
w = &Walker{rootNode, filter}
// Return
return
}
func (s *Server) loadHostnameBloomFilter() error {
s.bloomFilter = bloom.New(256*1024*8, 2)
rows, err := s.dbpool.Query("SELECT hostname FROM hostname_watch")
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var hostname string
err := rows.Scan(&hostname)
if err != nil {
return err
}
s.bloomFilter.AddString(hostname)
}
return nil
}
func main() {
var args struct {
M uint `help:"num bits"`
K uint `help:"num hashing functions"`
}
tagflag.Parse(&args, tagflag.Description("adds lines from stdin to a bloom filter with the given configuration, and gives collision stats at EOF"))
filter := bloom.New(args.M, args.K)
scanner := bufio.NewScanner(os.Stdin)
n := 0
collisions := 0
for scanner.Scan() {
if filter.TestAndAdd(scanner.Bytes()) {
collisions++
}
n++
}
if err := scanner.Err(); err != nil {
fmt.Fprintf(os.Stderr, "error reading stdin: %s", err)
os.Exit(1)
}
fmt.Printf("collisions %d/%d (%f)\n", collisions, n, float64(collisions)/float64(n)*100)
}
"crypto/sha1"
"encoding/base64"
"errors"
"fmt"
"log"
"net/http"
"strings"
"time"
"code.google.com/p/go.crypto/bcrypt"
"github.com/mdlayher/goat/goat/data"
"github.com/willf/bloom"
)
// nonceFilter is a bloom filter containing nonce values we have seen previously
var nonceFilter = bloom.New(20000, 5)
// APIAuthenticator interface which defines methods required to implement an authentication method
type APIAuthenticator interface {
Auth(*http.Request) (error, error)
Session() (data.UserRecord, error)
}
// apiSignature generates a HMAC-SHA1 signature for use with the API
func apiSignature(userID int, nonce string, method string, resource string, secret string) (string, error) {
// Generate API signature string
signString := fmt.Sprintf("%d-%s-%s-%s", userID, nonce, method, resource)
// Calculate HMAC-SHA1 signature from string, using API secret
mac := hmac.New(sha1.New, []byte(secret))
if _, err := mac.Write([]byte(signString)); err != nil {
func main() {
fmt.Println(banner)
flag.StringVar(&starturl, "u", "", "Base URL")
flag.BoolVar(&debugMode, "debug", false, "log additional debug traces")
flag.BoolVar(&serverMode, "server", false, "launch testing server")
flag.Parse()
LogInit(debugMode)
if serverMode {
launchServer()
//initSignals()
os.Exit(1)
}
if starturl == "" {
flag.Usage()
os.Exit(1)
}
// Init internal resources and data structures
activity = make(chan struct{})
originTimeMutex = &sync.Mutex{}
exiting = make(chan struct{})
timeout := time.Duration(5 * time.Second)
jar, _ := cookiejar.New(nil)
//TODO: add redirect policy
client = &http.Client{
Timeout: timeout,
Jar: jar,
// CheckRedirect specifies the policy for handling redirects.
// If CheckRedirect is not nil, the client calls it before
// following an HTTP redirect. The arguments req and via are
// the upcoming request and the requests made already, oldest
// first. If CheckRedirect returns an error, the Client's Get
// method returns both the previous Response (with its Body
// closed) and CheckRedirect's error (wrapped in a url.Error)
// instead of issuing the Request req.
// As a special case, if CheckRedirect returns ErrUseLastResponse,
// then the most recent response is returned with its body
// unclosed, along with a nil error.
//
// If CheckRedirect is nil, the Client uses its default policy,
// which is to stop after 10 consecutive requests.
//CheckRedirect func(req *Request, via []*Request) error
}
originTime = make(map[string]int64)
bfilter = bloom.New(BLOOMSIZE, BLOOMHASHES)
debug("Bloom filter estimated FP Rate after 1M URLs:", strconv.FormatFloat(bfilter.EstimateFalsePositiveRate(1000000), 'f', -1, 64))
<-time.After(time.Second * 4)
initSignals()
initWatchdog()
// Launch Stages
// Request Filter Stage
// reqFilterInputQ--->[]--->[]-->. . . . . -->[]--->reqFilterOutputQ
reqFilterInputQ = make(chan *http.Request, bufferSize)
reqFilterOutputQ = make(chan *http.Request, bufferSize)
wg.Add(1)
go reqFilterPipeline(1)
// Download Stage
downloadOutputQ = make(chan *http.Response, bufferSize)
for id := 1; id <= workerCount; id++ {
wg.Add(1)
go httpClient(id)
}
// Response Processing Stage
for i := 1; i <= extractorCount; i++ {
wg.Add(1)
go linkExtractor()
}
//wg.Add(1)
//go responseProcessor(1)
// out queue
// launch response broker
//go responseBroker()
// out queue
//processQ = make(chan *http.Response, 1000)
// launch response processors
// seed start URL
req, _ := http.NewRequest("GET", starturl, nil)
tokens := strings.Split(req.URL.Host, ".")
scope = strings.Join(tokens[len(tokens)-2:], ".")
<-time.After(time.Second)
for i := 1; i <= workerCount; i++ {
req, _ := http.NewRequest("GET", starturl, nil)
reqFilterInputQ <- req
}
// sync workers
wg.Wait()
close(downloadOutputQ)
close(reqFilterInputQ)
close(reqFilterOutputQ)
// Wait for exiting
info("info test")
debug("debug test")
}
// NewBloomSketch ...
func NewBloomSketch(info *datamodel.Info) (*BloomSketch, error) {
// FIXME: We are converting from int64 to uint
sketch := bloom.New(uint(info.Properties.GetMaxUniqueItems()), 4)
d := BloomSketch{info, sketch}
return &d, nil
}