Example #1
0
func GetBucket(bucketPrefix string) (*s3.Bucket, error) {
	auth, err := aws.GetAuth("", "", "", time.Time{})
	if err != nil {
		return nil, err
	}

	s := s3.New(auth, aws.USWest2)
	s.ConnectTimeout = time.Second * 30
	s.ReadTimeout = time.Second * 30

	bucketName := strings.TrimPrefix(bucketPrefix, "s3://") + "-" + environment.GetCloudEnv()
	return s.Bucket(bucketName), nil
}
Example #2
0
	"strings"
	"syscall"
	"time"

	"github.com/cactus/go-statsd-client/statsd"
	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/s3"
)

const (
	auditBucketName = "spade-audits"
	eventBucketName = "spade-edge"
)

var (
	CLOUD_ENV    = environment.GetCloudEnv()
	stats_prefix = flag.String("stat_prefix", "edge", "statsd prefix")
	logging_dir  = flag.String("log_dir", ".", "The directory to log these files to")
	listen_port  = flag.String(
		"port",
		":8888",
		"Which port are we listenting on form: ':<port>' e.g. :8888",
	)
	brokers = flag.String("kafka_brokers", "",
		`<host>:<port>,<host>:<port>,... of kafka brokers.
		Can leave empty if not using`)
	clientId   = flag.String("client_id", "", "id of this client.")
	VERSION, _ = strconv.Atoi(os.Getenv("EDGE_VERSION"))

	maxLogLines = int(getInt64FromEnv("MAX_LOG_LINES", 1000000))                          // default 1 million
	maxLogAge   = time.Duration(getInt64FromEnv("MAX_LOG_AGE_SECS", 10*60)) * time.Second // default 10 mins
Example #3
0
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/sqs"
	"github.com/twitchscience/aws_utils/environment"
	"github.com/twitchscience/aws_utils/listener"
	"github.com/twitchscience/rs_ingester/lib"
	"github.com/twitchscience/rs_ingester/metadata"
	"github.com/twitchscience/scoop_protocol/scoop_protocol"
)

var (
	env            = environment.GetCloudEnv()
	pgConfig       metadata.PGConfig
	loadAgeSeconds int
	statsPrefix    string
)

type RDSPipeHandler struct {
	MetadataStorer metadata.MetadataStorer
	Signer         scoop_protocol.ScoopSigner
	Statter        lib.Stats
}

func init() {
	flag.StringVar(&pgConfig.DatabaseURL, "databaseURL", "", "Postgres-scheme url for the RDS instance")
	flag.StringVar(&statsPrefix, "statsPrefix", "dbstorer", "the prefix to statsd")
	flag.IntVar(&pgConfig.MaxConnections, "maxDBConnections", 5, "Max number of database connections to open")
Example #4
0
func GetTopic() string {
	return "spade-edge-" + env.GetCloudEnv()
}