func GetBucket(bucketPrefix string) (*s3.Bucket, error) {
auth, err := aws.GetAuth("", "", "", time.Time{})
if err != nil {
return nil, err
}
s := s3.New(auth, aws.USWest2)
s.ConnectTimeout = time.Second * 30
s.ReadTimeout = time.Second * 30
bucketName := strings.TrimPrefix(bucketPrefix, "s3://") + "-" + environment.GetCloudEnv()
return s.Bucket(bucketName), nil
}
"strings"
"syscall"
"time"
"github.com/cactus/go-statsd-client/statsd"
"github.com/crowdmob/goamz/aws"
"github.com/crowdmob/goamz/s3"
)
const (
auditBucketName = "spade-audits"
eventBucketName = "spade-edge"
)
var (
CLOUD_ENV = environment.GetCloudEnv()
stats_prefix = flag.String("stat_prefix", "edge", "statsd prefix")
logging_dir = flag.String("log_dir", ".", "The directory to log these files to")
listen_port = flag.String(
"port",
":8888",
"Which port are we listenting on form: ':<port>' e.g. :8888",
)
brokers = flag.String("kafka_brokers", "",
`<host>:<port>,<host>:<port>,... of kafka brokers.
Can leave empty if not using`)
clientId = flag.String("client_id", "", "id of this client.")
VERSION, _ = strconv.Atoi(os.Getenv("EDGE_VERSION"))
maxLogLines = int(getInt64FromEnv("MAX_LOG_LINES", 1000000)) // default 1 million
maxLogAge = time.Duration(getInt64FromEnv("MAX_LOG_AGE_SECS", 10*60)) * time.Second // default 10 mins
"os/signal"
"strings"
"syscall"
"time"
"github.com/crowdmob/goamz/aws"
"github.com/crowdmob/goamz/sqs"
"github.com/twitchscience/aws_utils/environment"
"github.com/twitchscience/aws_utils/listener"
"github.com/twitchscience/rs_ingester/lib"
"github.com/twitchscience/rs_ingester/metadata"
"github.com/twitchscience/scoop_protocol/scoop_protocol"
)
var (
env = environment.GetCloudEnv()
pgConfig metadata.PGConfig
loadAgeSeconds int
statsPrefix string
)
type RDSPipeHandler struct {
MetadataStorer metadata.MetadataStorer
Signer scoop_protocol.ScoopSigner
Statter lib.Stats
}
func init() {
flag.StringVar(&pgConfig.DatabaseURL, "databaseURL", "", "Postgres-scheme url for the RDS instance")
flag.StringVar(&statsPrefix, "statsPrefix", "dbstorer", "the prefix to statsd")
flag.IntVar(&pgConfig.MaxConnections, "maxDBConnections", 5, "Max number of database connections to open")
func GetTopic() string {
return "spade-edge-" + env.GetCloudEnv()
}