示例#1
0
func main() {

	enableTls, port, certFile, keyFile, chainHandler := parseFlags()

	if connections.RedisPool != nil {
		defer connections.RedisPool.Close()
	}

	if *enableTls {

		// serve over https
		// if you want to test this locally, generate a cert and key file using
		// go by running a command similar to :
		// "go run /usr/local/go/src/crypto/tls/generate_cert.go --host localhost"
		// and change the location to wherever go is installed on your system
		err := http.ListenAndServeTLS(*port, *certFile, *keyFile, chainHandler)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		// serve over http (non-secure)
		glog.Infof("WARNING: Running over plaintext http (insecure). " +
			"To enable https, use '-enable-tls'")
		http.ListenAndServe(*port, chainHandler)
	}

}
func storeRequest(r *http.Request) {
	ip := connections.FindIp(r)
	resource := r.URL.Path

	key := lastSerialized.String() + DELIMETER + ip + DELIMETER + resource

	mutex.Lock()
	// increment count of times this ip/resource pair has been seen
	userResourceCounts[key] = userResourceCounts[key] + 1
	mutex.Unlock()
	runtime.Gosched()

	// if we hit the serialization time ..
	if time.Now().After(lastSerialized.Add(serializationDuration)) {
		glog.Infof("Purging now - comparing %s + ms (%s) <--> %s",
			lastSerialized.String(), serializationDuration.String(),
			lastSerialized.Add(serializationDuration).String(), time.Now())

		serializedTimestampString := lastSerialized.String()

		// update last serialized for next batch
		lastSerialized = time.Now()

		flushToRedis(serializedTimestampString)
	}
}
func RefreshBlocks(blockRefreshUrl *string) {
	//"http://localhost:8090/api/v1.0/blocks"
	_, body, _ := gorequest.New().Get(*blockRefreshUrl).End()

	var blocks Blocks

	if err := json.Unmarshal([]byte(body), &blocks); err != nil {
		panic(err)
	}

	for _, element := range blocks {

		// temporary hack for dealing w/ java block store
		// it keeps timestamp in microseconds for some reason
		element.Endtime = element.Endtime / 1000

		t, _ := unixToTime(element.Endtime)

		// only add if this is still a valid time block
		if time.Now().Before(t) {
			//back to json so it's hashable
			jsonStr, _ := json.Marshal(element)
			StoredBlocks.Add(string(jsonStr))
		}

	}

	//	glog.Info(StoredBlocks.Len())
	glog.Infof("Retrieved %d blocks, total stored: %d", len(blocks), StoredBlocks.Len())
	//	glog.Info("STORED blocks: ", StoredBlocks.Flatten())
	//	glog.Info("Printing response: ", resp)
	//	glog.Info("Printing body: ", body)
}
func ExpireBlocks() {
	glog.Info("Evaluating blocks for expiration.")

	var removableBlocks []string

	for _, element := range StoredBlocks.Flatten() {

		var block Block

		//glog.Info(element)
		if err := json.Unmarshal([]byte(element.(string)), &block); err != nil {
			panic(err)
		}

		t, err := unixToTime(block.Endtime)
		if err != nil {
			glog.Fatal(err)
		}

		if time.Now().After(t) {
			glog.Info("Expiring block: ", block)
			removableBlocks = append(removableBlocks, element.(string))
		} else {
			glog.Infof("Not expiring block:")
			glog.Infof("\tnow: ", time.Now())
			glog.Infof("\tt: ", t)
			glog.Infof("\tnow unix: ", time.Now().Unix())
			glog.Infof("\tt unix: ", t.Unix())
		}

	}

	if len(removableBlocks) > 0 {
		for _, removableBlock := range removableBlocks {
			StoredBlocks.Remove(removableBlock)
		}
	}

}
func ConnectRedis(redisAddress *string, maxRedisConnections *int) {

	glog.Infof("Attempting redis connection at %s with %d connections",
		*redisAddress, *maxRedisConnections)

	RedisPool = redis.NewPool(func() (redis.Conn, error) {
		c, err := redis.Dial("dtcp", *redisAddress)

		if err != nil {
			glog.Fatal("redis connection could not be made")
			return nil, err
		}

		return c, err
	}, *maxRedisConnections)
}
// from https://justinas.org/writing-http-middleware-in-go/
func LogExecutionTime(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

		start := time.Now()

		rec := httptest.NewRecorder()
		// passing a ResponseRecorder instead of the original RW
		next.ServeHTTP(rec, r)

		// we copy the original headers first
		for k, v := range rec.Header() {
			w.Header()[k] = v
		}

		w.WriteHeader(rec.Code)

		// then write out the original body
		w.Write(rec.Body.Bytes())

		glog.Infof("%v %s on %s %s in %v", rec.Code, http.StatusText(rec.Code), r.Method, r.URL.Path, time.Since(start))

	})
}
示例#7
0
func parseFlags() (*bool, *string, *string, *string, http.Handler) {

	const (
		defaultPort                     = ":8080"
		defaultPortUsage                = "default server port, ':80', ':8080'..."
		defaultTarget                   = "http://127.0.0.1:8090"
		defaultTargetUsage              = "default redirect url, 'http://127.0.0.1:8080'"
		defaultBooleanUsage             = "true or false"
		defaultRefreshUsage             = "number of seconds between block refreshes"
		defaultExpireUsage              = "number of seconds between expiration sweeps"
		defaultBlocksRefreshUrl         = "block refresh url, 'http://localhost:8090/api/v1.0/blocks'"
		defaultCertFile                 = "cert file (e.g. cert.pem)"
		defaultKeyFile                  = "key file (e.g. key.pem)"
		defaultResourceVerbsMappingFile = "resource verbs mapping file (e.g. resource-verbs-mapping.yml)"
		defaultResourcesFile            = "resources file (e.g. resources.yml)"
	)

	// flags
	port := flag.String("port", defaultPort, defaultPortUsage)
	url := flag.String("url", defaultTarget, defaultTargetUsage)

	// for running in TLS mode
	enableTls := flag.Bool("enable-tls", false, defaultBooleanUsage)
	certFile := flag.String("cert-file", "cert.pem", defaultCertFile)
	keyFile := flag.String("key-file", "key.pem", defaultKeyFile)

	// related to trend detection
	enableTrends := flag.Bool("enable-trend-tracking", false, defaultBooleanUsage)

	// related to blocking
	enableBlocking := flag.Bool("enable-blocking", false, defaultBooleanUsage)
	refreshSeconds := flag.Int("blocking-refresh-rate-seconds", 30, defaultRefreshUsage)
	expireSeconds := flag.Int("blocking-expire-rate-seconds", 30, defaultExpireUsage)
	blocksRefreshUrl := flag.String("blocking-blocks-refresh-url", "http://localhost", defaultBlocksRefreshUrl)

	// for redis, if used
	redisAddress := flag.String("redis-address", ":6379", "Address to the Redis server")
	maxRedisConnections := flag.Int("max-redis-connections", 10, "Max connections to Redis")

	// related to invalid http verbs
	// (instead of GET, POST, HEAD ... you receive DOG, SHOE, FOREST, etc.)
	enableInvalidVerbs := flag.Bool("enable-invalid-verb-checking", false, defaultBooleanUsage)

	// related to unexpected http verbs
	// requires config file
	// config file states this file reachable over GET or HEAD, and you get a POST
	enableUnexpectedVerbs := flag.Bool("enable-unexpected-verb-checking", false, defaultBooleanUsage)
	resourceVerbsMappingFile := flag.String("resource-verbs-mapping-file", "resource-verbs-mapping.yml", defaultResourceVerbsMappingFile)

	// related to unexpected resources
	// requires config file
	// config file states all resources expected in the app (think site-map)
	// this analysis should help detect scanners probing for non-existent resources
	enableUnexpectedResources := flag.Bool("enable-unexpected-resource-checking", false, defaultBooleanUsage)
	resourcesFile := flag.String("resources-file", "resources.yml", defaultResourcesFile)

	flag.Parse()

	glog.Info("------------------------------------------------")
	glog.Infof("Settings:")
	glog.Infof("\tServer port %s", *port)
	glog.Infof("\tProxy url: %s", *url)
	glog.Infof("\tEnable TLS: %t", *enableTls)
	glog.Infof("\t\tCert File: %s", *certFile)
	glog.Infof("\t\tKey File: %s", *keyFile)
	glog.Infof("\tEnable trend tracking: %t", *enableTrends)
	glog.Infof("\tEnable blocking: %t", *enableBlocking)
	glog.Infof("\t\tRefresh rate (seconds): %d", *refreshSeconds)
	glog.Infof("\t\tExpire rate (seconds): %d", *expireSeconds)
	glog.Infof("\t\tBlock refresh url: %s", *blocksRefreshUrl)
	glog.Infof("\tEnable invalid verbs: %t", *enableInvalidVerbs)
	glog.Infof("\tEnable unexpected verbs: %t", *enableUnexpectedVerbs)
	glog.Infof("\t\tResource verbs mapping file: %s", *resourceVerbsMappingFile)
	glog.Infof("\tEnable unexpected resources: %t", *enableUnexpectedResources)
	glog.Infof("\t\tResources file: %s", *resourcesFile)
	glog.Info("------------------------------------------------")

	// proxy
	proxy := New(*url)

	proxyHandler := http.HandlerFunc(proxy.handle)

	// chain default handlers (clear context, recovery, and log execution time)
	chain := alice.New(
		context.ClearHandler,
		middleware.Recovery,
		middleware.LogExecutionTime)

	// if blocking is enabled, add it to alice chain
	if *enableBlocking {
		chain = chain.Append(middleware.Block)
		go doEvery((time.Duration(*refreshSeconds) * time.Second), func() { blocks.RefreshBlocks(blocksRefreshUrl) })
		go doEvery((time.Duration(*expireSeconds) * time.Second), blocks.ExpireBlocks)
	}

	// if trending is enabled, add it to alice chain
	if *enableTrends {
		chain = chain.Append(middleware.Trend)
	}

	// if invalid verb checking is enabled, add it to alice chain
	if *enableInvalidVerbs {
		chain = chain.Append(middleware.InvalidVerbs)
	}

	// if unexpected verb checking is enabled, add it to alice chain
	if *enableUnexpectedVerbs {
		middleware.PopulateExpectedVerbs(resourceVerbsMappingFile)
		chain = chain.Append(middleware.UnexpectedVerbs)
	}

	// if unexpected resource checking is enabled, add it to alice chain
	if *enableUnexpectedResources {
		middleware.PopulateExpectedResources(resourcesFile)
		chain = chain.Append(middleware.UnexpectedResources)
	}

	// if any of these settings are enabled, connect to redis
	if *enableTrends {
		connections.ConnectRedis(redisAddress, maxRedisConnections)
	}

	// if any of these settings are enabled, setup rest client to appsensor backend
	if *enableInvalidVerbs || *enableTrends ||
		*enableUnexpectedVerbs || *enableUnexpectedResources {
		ids.InitializeAppSensorRestClient()
	}

	// always use reverse proxy as chain target (final)
	chainHandler := chain.Then(proxyHandler)

	return enableTls, port, certFile, keyFile, chainHandler
}