func main() {
enableTls, port, certFile, keyFile, chainHandler := parseFlags()
if connections.RedisPool != nil {
defer connections.RedisPool.Close()
}
if *enableTls {
// serve over https
// if you want to test this locally, generate a cert and key file using
// go by running a command similar to :
// "go run /usr/local/go/src/crypto/tls/generate_cert.go --host localhost"
// and change the location to wherever go is installed on your system
err := http.ListenAndServeTLS(*port, *certFile, *keyFile, chainHandler)
if err != nil {
log.Fatal(err)
}
} else {
// serve over http (non-secure)
glog.Infof("WARNING: Running over plaintext http (insecure). " +
"To enable https, use '-enable-tls'")
http.ListenAndServe(*port, chainHandler)
}
}
func storeRequest(r *http.Request) {
ip := connections.FindIp(r)
resource := r.URL.Path
key := lastSerialized.String() + DELIMETER + ip + DELIMETER + resource
mutex.Lock()
// increment count of times this ip/resource pair has been seen
userResourceCounts[key] = userResourceCounts[key] + 1
mutex.Unlock()
runtime.Gosched()
// if we hit the serialization time ..
if time.Now().After(lastSerialized.Add(serializationDuration)) {
glog.Infof("Purging now - comparing %s + ms (%s) <--> %s",
lastSerialized.String(), serializationDuration.String(),
lastSerialized.Add(serializationDuration).String(), time.Now())
serializedTimestampString := lastSerialized.String()
// update last serialized for next batch
lastSerialized = time.Now()
flushToRedis(serializedTimestampString)
}
}
func RefreshBlocks(blockRefreshUrl *string) {
//"http://localhost:8090/api/v1.0/blocks"
_, body, _ := gorequest.New().Get(*blockRefreshUrl).End()
var blocks Blocks
if err := json.Unmarshal([]byte(body), &blocks); err != nil {
panic(err)
}
for _, element := range blocks {
// temporary hack for dealing w/ java block store
// it keeps timestamp in microseconds for some reason
element.Endtime = element.Endtime / 1000
t, _ := unixToTime(element.Endtime)
// only add if this is still a valid time block
if time.Now().Before(t) {
//back to json so it's hashable
jsonStr, _ := json.Marshal(element)
StoredBlocks.Add(string(jsonStr))
}
}
// glog.Info(StoredBlocks.Len())
glog.Infof("Retrieved %d blocks, total stored: %d", len(blocks), StoredBlocks.Len())
// glog.Info("STORED blocks: ", StoredBlocks.Flatten())
// glog.Info("Printing response: ", resp)
// glog.Info("Printing body: ", body)
}
func ExpireBlocks() {
glog.Info("Evaluating blocks for expiration.")
var removableBlocks []string
for _, element := range StoredBlocks.Flatten() {
var block Block
//glog.Info(element)
if err := json.Unmarshal([]byte(element.(string)), &block); err != nil {
panic(err)
}
t, err := unixToTime(block.Endtime)
if err != nil {
glog.Fatal(err)
}
if time.Now().After(t) {
glog.Info("Expiring block: ", block)
removableBlocks = append(removableBlocks, element.(string))
} else {
glog.Infof("Not expiring block:")
glog.Infof("\tnow: ", time.Now())
glog.Infof("\tt: ", t)
glog.Infof("\tnow unix: ", time.Now().Unix())
glog.Infof("\tt unix: ", t.Unix())
}
}
if len(removableBlocks) > 0 {
for _, removableBlock := range removableBlocks {
StoredBlocks.Remove(removableBlock)
}
}
}
func ConnectRedis(redisAddress *string, maxRedisConnections *int) {
glog.Infof("Attempting redis connection at %s with %d connections",
*redisAddress, *maxRedisConnections)
RedisPool = redis.NewPool(func() (redis.Conn, error) {
c, err := redis.Dial("dtcp", *redisAddress)
if err != nil {
glog.Fatal("redis connection could not be made")
return nil, err
}
return c, err
}, *maxRedisConnections)
}
// from https://justinas.org/writing-http-middleware-in-go/
func LogExecutionTime(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
rec := httptest.NewRecorder()
// passing a ResponseRecorder instead of the original RW
next.ServeHTTP(rec, r)
// we copy the original headers first
for k, v := range rec.Header() {
w.Header()[k] = v
}
w.WriteHeader(rec.Code)
// then write out the original body
w.Write(rec.Body.Bytes())
glog.Infof("%v %s on %s %s in %v", rec.Code, http.StatusText(rec.Code), r.Method, r.URL.Path, time.Since(start))
})
}
func parseFlags() (*bool, *string, *string, *string, http.Handler) {
const (
defaultPort = ":8080"
defaultPortUsage = "default server port, ':80', ':8080'..."
defaultTarget = "http://127.0.0.1:8090"
defaultTargetUsage = "default redirect url, 'http://127.0.0.1:8080'"
defaultBooleanUsage = "true or false"
defaultRefreshUsage = "number of seconds between block refreshes"
defaultExpireUsage = "number of seconds between expiration sweeps"
defaultBlocksRefreshUrl = "block refresh url, 'http://localhost:8090/api/v1.0/blocks'"
defaultCertFile = "cert file (e.g. cert.pem)"
defaultKeyFile = "key file (e.g. key.pem)"
defaultResourceVerbsMappingFile = "resource verbs mapping file (e.g. resource-verbs-mapping.yml)"
defaultResourcesFile = "resources file (e.g. resources.yml)"
)
// flags
port := flag.String("port", defaultPort, defaultPortUsage)
url := flag.String("url", defaultTarget, defaultTargetUsage)
// for running in TLS mode
enableTls := flag.Bool("enable-tls", false, defaultBooleanUsage)
certFile := flag.String("cert-file", "cert.pem", defaultCertFile)
keyFile := flag.String("key-file", "key.pem", defaultKeyFile)
// related to trend detection
enableTrends := flag.Bool("enable-trend-tracking", false, defaultBooleanUsage)
// related to blocking
enableBlocking := flag.Bool("enable-blocking", false, defaultBooleanUsage)
refreshSeconds := flag.Int("blocking-refresh-rate-seconds", 30, defaultRefreshUsage)
expireSeconds := flag.Int("blocking-expire-rate-seconds", 30, defaultExpireUsage)
blocksRefreshUrl := flag.String("blocking-blocks-refresh-url", "http://localhost", defaultBlocksRefreshUrl)
// for redis, if used
redisAddress := flag.String("redis-address", ":6379", "Address to the Redis server")
maxRedisConnections := flag.Int("max-redis-connections", 10, "Max connections to Redis")
// related to invalid http verbs
// (instead of GET, POST, HEAD ... you receive DOG, SHOE, FOREST, etc.)
enableInvalidVerbs := flag.Bool("enable-invalid-verb-checking", false, defaultBooleanUsage)
// related to unexpected http verbs
// requires config file
// config file states this file reachable over GET or HEAD, and you get a POST
enableUnexpectedVerbs := flag.Bool("enable-unexpected-verb-checking", false, defaultBooleanUsage)
resourceVerbsMappingFile := flag.String("resource-verbs-mapping-file", "resource-verbs-mapping.yml", defaultResourceVerbsMappingFile)
// related to unexpected resources
// requires config file
// config file states all resources expected in the app (think site-map)
// this analysis should help detect scanners probing for non-existent resources
enableUnexpectedResources := flag.Bool("enable-unexpected-resource-checking", false, defaultBooleanUsage)
resourcesFile := flag.String("resources-file", "resources.yml", defaultResourcesFile)
flag.Parse()
glog.Info("------------------------------------------------")
glog.Infof("Settings:")
glog.Infof("\tServer port %s", *port)
glog.Infof("\tProxy url: %s", *url)
glog.Infof("\tEnable TLS: %t", *enableTls)
glog.Infof("\t\tCert File: %s", *certFile)
glog.Infof("\t\tKey File: %s", *keyFile)
glog.Infof("\tEnable trend tracking: %t", *enableTrends)
glog.Infof("\tEnable blocking: %t", *enableBlocking)
glog.Infof("\t\tRefresh rate (seconds): %d", *refreshSeconds)
glog.Infof("\t\tExpire rate (seconds): %d", *expireSeconds)
glog.Infof("\t\tBlock refresh url: %s", *blocksRefreshUrl)
glog.Infof("\tEnable invalid verbs: %t", *enableInvalidVerbs)
glog.Infof("\tEnable unexpected verbs: %t", *enableUnexpectedVerbs)
glog.Infof("\t\tResource verbs mapping file: %s", *resourceVerbsMappingFile)
glog.Infof("\tEnable unexpected resources: %t", *enableUnexpectedResources)
glog.Infof("\t\tResources file: %s", *resourcesFile)
glog.Info("------------------------------------------------")
// proxy
proxy := New(*url)
proxyHandler := http.HandlerFunc(proxy.handle)
// chain default handlers (clear context, recovery, and log execution time)
chain := alice.New(
context.ClearHandler,
middleware.Recovery,
middleware.LogExecutionTime)
// if blocking is enabled, add it to alice chain
if *enableBlocking {
chain = chain.Append(middleware.Block)
go doEvery((time.Duration(*refreshSeconds) * time.Second), func() { blocks.RefreshBlocks(blocksRefreshUrl) })
go doEvery((time.Duration(*expireSeconds) * time.Second), blocks.ExpireBlocks)
}
// if trending is enabled, add it to alice chain
if *enableTrends {
chain = chain.Append(middleware.Trend)
}
// if invalid verb checking is enabled, add it to alice chain
if *enableInvalidVerbs {
chain = chain.Append(middleware.InvalidVerbs)
}
// if unexpected verb checking is enabled, add it to alice chain
if *enableUnexpectedVerbs {
middleware.PopulateExpectedVerbs(resourceVerbsMappingFile)
chain = chain.Append(middleware.UnexpectedVerbs)
}
// if unexpected resource checking is enabled, add it to alice chain
if *enableUnexpectedResources {
middleware.PopulateExpectedResources(resourcesFile)
chain = chain.Append(middleware.UnexpectedResources)
}
// if any of these settings are enabled, connect to redis
if *enableTrends {
connections.ConnectRedis(redisAddress, maxRedisConnections)
}
// if any of these settings are enabled, setup rest client to appsensor backend
if *enableInvalidVerbs || *enableTrends ||
*enableUnexpectedVerbs || *enableUnexpectedResources {
ids.InitializeAppSensorRestClient()
}
// always use reverse proxy as chain target (final)
chainHandler := chain.Then(proxyHandler)
return enableTls, port, certFile, keyFile, chainHandler
}