func NewHawkHandler(handler http.Handler, secrets []string) *HawkHandler {
	// the m value for the bloom filter is likely larger than
	// we need. It figures 60,000 requests/minute * 50 = 3,000,000 bits
	// or ~2.8MB. The code rotates between two of them so about 5.6MB
	// of memory for nonce checking.

	m := uint(1000 * 60 * 50)
	return &HawkHandler{
		handler:       handler,
		secrets:       secrets,
		bloomPrev:     bloom.New(m, 5),
		bloomNow:      bloom.New(m, 5),
		bloomHalflife: 30 * time.Second,
		lastRotate:    time.Now(),
	}
}
Beispiel #2
0
func check(cmd *cobra.Command, args []string) error {
	if err := InitializeConfig(checkCmd); err != nil {
		return err
	}

	// Initialize bloom filter 1M max elements, 5 hash
	bf := bloom.New(1000000, 5)

	// Check if bloom filter file exists
	fileName := viper.GetString("DatabasePath")
	if _, err := os.Stat(fileName); err == nil {
		bff, err2 := os.Open(fileName)
		if err2 != nil {
			panic(err2)
		}
		defer bff.Close()

		if _, err3 := bf.ReadFrom(bff); err != nil {
			panic(err3)
		}
	} else {
		fmt.Println("You must build a filter first, use 'build' command.")
		return err
	}

	if len(args) > 0 {
		for _, domain := range args {
			fmt.Printf("%s,%v\n", domain, bf.TestString(domain))
		}
	}

	return nil
}
Beispiel #3
0
// Creates a new TCP Server with a bloom filter (m, k)
func NewServer(dsn string, m uint, k uint) (*Server, error) {
	listener, err := net.Listen("tcp", dsn)
	if listener == nil {
		return nil, err
	}
	f := bloom.New(m, k)
	return &Server{listener: listener, f: f}, nil
}
Beispiel #4
0
func build(cmd *cobra.Command, args []string) error {
	if err := InitializeConfig(buildCmd); err != nil {
		return err
	}

	// It is a good day to die !
	now := time.Now().UTC()

	// Initialize bloom filter 1M max elements, 5 hash
	bf := bloom.New(1000000, 5)

	// Check if bloom filter file exists
	fileName := viper.GetString("DatabasePath")
	if _, err := os.Stat(fileName); err == nil {
		bff, err := os.Open(fileName)
		if err != nil {
			panic(err)
		}
		defer bff.Close()

		if _, err := bf.ReadFrom(bff); err != nil {
			panic(err)
		}
	}

	// future days loop
	for day := 0; day < afterDayCount; day++ {
		dgaGenerate(now, bf)
		// Add 1 day
		now = now.AddDate(0, 0, 1)
	}

	// Reset to today
	now = time.Now().UTC()

	// past days loop
	for day := 0; day < beforeDayCount; day++ {
		dgaGenerate(now, bf)
		// Substract 1 day
		now = now.AddDate(0, 0, -1)
	}

	// Save bloom filter
	bff, err := os.Create(fileName)
	if err != nil {
		panic(err)
	}
	defer bff.Close()

	// Dump to file
	_, err = bf.WriteTo(bff)
	if err != nil {
		panic(err)
	}

	return nil
}
Beispiel #5
0
func main() {
	var n uint = 1000
	filter := bloom.New(20*n, 5) // load of 20, 5 keys
	filter.Add([]byte("Love"))

	if filter.Test([]byte("Love")) {
		println("It is in black list!")
	}
}
Beispiel #6
0
// NewDeduplicator initializes a new deduplicator
func NewDeduplicator(computation StreamID, recordTracker ReceivedTracker) (Deduplicator, error) {
	dd := &dedup{
		computation:   computation,
		recordTracker: recordTracker,
		filter:        bloom.New(20000, 5),
	}
	recent, err := recordTracker.GetRecentReceived(computation)
	if err != nil {
		return nil, err
	}

	for _, recordID := range recent {
		dd.filter.AddString(recordID)
	}
	return dd, nil
}
Beispiel #7
0
func main() {
	m := flag.Uint("m", 0, "")
	k := flag.Uint("k", 0, "")
	flag.Parse()
	filter := bloom.New(*m, *k)
	scanner := bufio.NewScanner(os.Stdin)
	n := 0
	collisions := 0
	for scanner.Scan() {
		if filter.TestAndAdd(scanner.Bytes()) {
			collisions++
		}
		n++
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintf(os.Stderr, "error reading stdin: %s", err)
		os.Exit(1)
	}
	fmt.Printf("collisions %d/%d (%f)\n", collisions, n, float64(collisions)/float64(n)*100)
}
Beispiel #8
0
// Creates a new walker object
func New(rawBaseUrl string) (w *Walker) {
	// Parse the base url
	baseUrl, err := url.Parse(rawBaseUrl)

	// Fail on error
	if err != nil {
		log.Fatal("baseURL (", baseUrl, ") failed to parse: ", err)
	}

	rootNode := buildNode(nil, baseUrl)

	// Create the bloom filter
	filter := bloom.New(20000, 5)

	// Create the object
	w = &Walker{rootNode, filter}

	// Return
	return
}
Beispiel #9
0
func (s *Server) loadHostnameBloomFilter() error {
	s.bloomFilter = bloom.New(256*1024*8, 2)

	rows, err := s.dbpool.Query("SELECT hostname FROM hostname_watch")
	if err != nil {
		return err
	}

	defer rows.Close()
	for rows.Next() {
		var hostname string
		err := rows.Scan(&hostname)
		if err != nil {
			return err
		}

		s.bloomFilter.AddString(hostname)
	}

	return nil
}
Beispiel #10
0
func main() {
	var args struct {
		M uint `help:"num bits"`
		K uint `help:"num hashing functions"`
	}
	tagflag.Parse(&args, tagflag.Description("adds lines from stdin to a bloom filter with the given configuration, and gives collision stats at EOF"))
	filter := bloom.New(args.M, args.K)
	scanner := bufio.NewScanner(os.Stdin)
	n := 0
	collisions := 0
	for scanner.Scan() {
		if filter.TestAndAdd(scanner.Bytes()) {
			collisions++
		}
		n++
	}
	if err := scanner.Err(); err != nil {
		fmt.Fprintf(os.Stderr, "error reading stdin: %s", err)
		os.Exit(1)
	}
	fmt.Printf("collisions %d/%d (%f)\n", collisions, n, float64(collisions)/float64(n)*100)
}
Beispiel #11
0
	"crypto/sha1"
	"encoding/base64"
	"errors"
	"fmt"
	"log"
	"net/http"
	"strings"
	"time"

	"code.google.com/p/go.crypto/bcrypt"
	"github.com/mdlayher/goat/goat/data"
	"github.com/willf/bloom"
)

// nonceFilter is a bloom filter containing nonce values we have seen previously
var nonceFilter = bloom.New(20000, 5)

// APIAuthenticator interface which defines methods required to implement an authentication method
type APIAuthenticator interface {
	Auth(*http.Request) (error, error)
	Session() (data.UserRecord, error)
}

// apiSignature generates a HMAC-SHA1 signature for use with the API
func apiSignature(userID int, nonce string, method string, resource string, secret string) (string, error) {
	// Generate API signature string
	signString := fmt.Sprintf("%d-%s-%s-%s", userID, nonce, method, resource)

	// Calculate HMAC-SHA1 signature from string, using API secret
	mac := hmac.New(sha1.New, []byte(secret))
	if _, err := mac.Write([]byte(signString)); err != nil {
Beispiel #12
0
func main() {
	fmt.Println(banner)
	flag.StringVar(&starturl, "u", "", "Base URL")
	flag.BoolVar(&debugMode, "debug", false, "log additional debug traces")
	flag.BoolVar(&serverMode, "server", false, "launch testing server")

	flag.Parse()

	LogInit(debugMode)

	if serverMode {
		launchServer()
		//initSignals()
		os.Exit(1)
	}

	if starturl == "" {
		flag.Usage()
		os.Exit(1)
	}

	// Init internal resources and data structures

	activity = make(chan struct{})
	originTimeMutex = &sync.Mutex{}
	exiting = make(chan struct{})
	timeout := time.Duration(5 * time.Second)
	jar, _ := cookiejar.New(nil)

	//TODO: add redirect policy
	client = &http.Client{
		Timeout: timeout,
		Jar:     jar,
		// CheckRedirect specifies the policy for handling redirects.
		// If CheckRedirect is not nil, the client calls it before
		// following an HTTP redirect. The arguments req and via are
		// the upcoming request and the requests made already, oldest
		// first. If CheckRedirect returns an error, the Client's Get
		// method returns both the previous Response (with its Body
		// closed) and CheckRedirect's error (wrapped in a url.Error)
		// instead of issuing the Request req.
		// As a special case, if CheckRedirect returns ErrUseLastResponse,
		// then the most recent response is returned with its body
		// unclosed, along with a nil error.
		//
		// If CheckRedirect is nil, the Client uses its default policy,
		// which is to stop after 10 consecutive requests.
		//CheckRedirect func(req *Request, via []*Request) error
	}
	originTime = make(map[string]int64)
	bfilter = bloom.New(BLOOMSIZE, BLOOMHASHES)
	debug("Bloom filter estimated FP Rate after 1M URLs:", strconv.FormatFloat(bfilter.EstimateFalsePositiveRate(1000000), 'f', -1, 64))
	<-time.After(time.Second * 4)

	initSignals()
	initWatchdog()

	// Launch Stages

	// Request Filter Stage
	// reqFilterInputQ--->[]--->[]-->. . . . . -->[]--->reqFilterOutputQ

	reqFilterInputQ = make(chan *http.Request, bufferSize)
	reqFilterOutputQ = make(chan *http.Request, bufferSize)
	wg.Add(1)
	go reqFilterPipeline(1)

	// Download Stage
	downloadOutputQ = make(chan *http.Response, bufferSize)
	for id := 1; id <= workerCount; id++ {
		wg.Add(1)
		go httpClient(id)
	}
	// Response Processing Stage
	for i := 1; i <= extractorCount; i++ {
		wg.Add(1)
		go linkExtractor()
	}
	//wg.Add(1)
	//go responseProcessor(1)

	// out queue

	// launch response broker
	//go responseBroker()
	// out queue
	//processQ = make(chan *http.Response, 1000)

	// launch response processors

	// seed start URL
	req, _ := http.NewRequest("GET", starturl, nil)
	tokens := strings.Split(req.URL.Host, ".")
	scope = strings.Join(tokens[len(tokens)-2:], ".")
	<-time.After(time.Second)
	for i := 1; i <= workerCount; i++ {
		req, _ := http.NewRequest("GET", starturl, nil)
		reqFilterInputQ <- req
	}

	// sync workers
	wg.Wait()
	close(downloadOutputQ)
	close(reqFilterInputQ)
	close(reqFilterOutputQ)

	// Wait for exiting

	info("info test")
	debug("debug test")

}
Beispiel #13
0
// NewBloomSketch ...
func NewBloomSketch(info *datamodel.Info) (*BloomSketch, error) {
	// FIXME: We are converting from int64 to uint
	sketch := bloom.New(uint(info.Properties.GetMaxUniqueItems()), 4)
	d := BloomSketch{info, sketch}
	return &d, nil
}