func TestReadFromSQS(t *testing.T) {
auth, err := aws.EnvAuth()
if err != nil {
panic(err)
}
sqs := New(auth, aws.USWest2)
queue, err := sqs.GetQueue("tracker-alpha-matt")
if err != nil {
panic(err)
}
results, err := queue.ReceiveMessage([]string{"All"}, 10, 600)
count := len(results.Messages)
deleteMessageBatch := make([]DeleteMessageBatch, count)
for index, message := range results.Messages {
fmt.Println(message.Body)
deleteMessageBatch[index] = DeleteMessageBatch{
Id: strconv.Itoa(index),
ReceiptHandle: message.ReceiptHandle,
}
}
fmt.Println(fmt.Sprintf("%#v", deleteMessageBatch))
response, err := queue.DeleteMessageBatch(deleteMessageBatch)
if err != nil {
panic(err)
}
fmt.Println(fmt.Sprintf("%#v", response))
}
func Sync(localPath, bucketName, awsRegion, acl string) {
if localPath == "" || bucketName == "" {
flag.PrintDefaults()
return
}
auth, err := aws.EnvAuth()
if err != nil {
panic(err.Error())
}
s3Conn := s3.New(auth, aws.Regions[awsRegion])
bucket := s3Conn.Bucket(bucketName)
pathChan := make(chan string)
doneChan := make(chan int)
go doTheWalk(localPath, pathChan)
count := 0
for path := range pathChan {
if path == "" {
break
}
go compareAndSync(acl, localPath, path, bucket, doneChan)
count++
}
for i := 0; i < count; i++ {
<-doneChan
}
}
func main() {
flag.Parse()
cmds := flag.Args()
if len(cmds) <= 0 {
fmt.Println("Command required")
os.Exit(1)
}
cmd := Cmd{}
cmd.name = cmds[0]
cmd.modifiers = cmds[1:]
auth, err := aws.EnvAuth()
if err != nil {
panic(err)
}
cmd.conn = s3.New(auth, getRegion(region))
s3url, err := url.Parse(cmd.modifiers[0])
cmd.url = s3url
cmd.bucket = s3.Bucket{cmd.conn, cmd.url.Host}
RunCommand(cmd)
}
func main() {
flag.Parse()
if *toHash == "" || *pkgDirs == "" || *cmd == "" || *bucketName == "" {
log.Print("You are missing one or more mandatory command line arguments.")
flag.Usage()
os.Exit(1)
}
cred, err := aws.EnvAuth()
if err != nil {
log.Fatalf("Couldn't auth into s3. Did you set up ENV? Error: %s", err)
}
s3client := s3.New(cred, aws.USWest)
bucket := s3client.Bucket(*bucketName)
// extract.
checksum := fmt.Sprintf("%x", hashFiles(*toHash).Sum(nil))
filename := *outfile
// consider bucket.GetReader to pipe directly into gunzip/untar
file, err := bucket.Get(fmt.Sprintf("%s/%s", checksum, filename))
if err != nil {
fmt.Printf("%s\n", err)
build(*cmd)
archive(*pkgDirs, filename)
upload(bucket, checksum, filename)
} else {
extract(file)
}
}
func PostSignup(w *rest.ResponseWriter, req *rest.Request) {
signup := Signup{}
err := req.DecodeJsonPayload(&signup)
if err != nil {
rest.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if signup.Email == "" {
rest.Error(w, "email is required", 400)
return
}
auth, err := aws.EnvAuth()
if err != nil {
panic(err.Error())
}
s := s3.New(auth, aws.EUWest)
bucket := s.Bucket("seq-signup")
data := []byte(signup.Email)
err = bucket.Put(signup.Email, data, "text/plain", s3.BucketOwnerFull)
if err != nil {
panic(err.Error())
}
fmt.Println(signup)
w.Header().Set("Access-Control-Allow-Origin", "http://sequenceiq.com")
w.WriteJson(&signup)
}
// Goroutine for saving the state.
func saver(ticker *time.Ticker) {
for {
select {
case <-ticker.C:
println("Writing state")
lock.Lock()
if dirty {
dirty = false
state := State{folders}
bytes, err := json.Marshal(state)
if err != nil {
panic(err)
}
// write to S3
auth, err := aws.EnvAuth()
if err != nil {
panic(err)
}
s := s3.New(auth, aws.USEast)
bucket := s.Bucket(s3_bucket)
err = bucket.Put("rss.json", bytes, "application/json", s3.ACL("private"))
if err != nil {
panic(err)
}
}
lock.Unlock()
}
}
}
// Load a brain
func (brain *Brain) Load() {
// The AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables are used.
auth, err := aws.EnvAuth()
if err != nil {
panic(err.Error())
}
// Open Bucket
s := s3.New(auth, aws.USEast)
// Load the database from an S3 bucket
bucket := s.Bucket(bucketName)
// Create a bytes.Buffer
n, err := bucket.Get(filename)
if err != nil {
panic(err)
}
p := bytes.NewBuffer(n)
dec := gob.NewDecoder(p)
err = dec.Decode(&brain)
if err != nil {
log.Print("There was an error loading the brain. Using a blank one.")
}
}
// Persist a brain
func (brain *Brain) Save() {
// Persist the database to file
var data bytes.Buffer
contents := gob.NewEncoder(&data)
err := contents.Encode(brain)
if err != nil {
panic(err)
}
// The AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables are used.
auth, err := aws.EnvAuth()
if err != nil {
panic(err.Error())
}
// Open Bucket
s := s3.New(auth, aws.USEast)
// Load the database from an S3 bucket
bucket := s.Bucket(bucketName)
err = bucket.Put(filename, data.Bytes(), "text/plain", s3.BucketOwnerFull)
if err != nil {
panic(err.Error())
}
}
func main() {
flag.Parse()
if flag.Arg(0) == "" {
errorf("no command")
os.Exit(2)
}
auth, err := aws.EnvAuth()
if err != nil {
fatalf("envauth: %v", err)
}
conn := ec2.New(auth, aws.USEast)
if flag.Arg(0) == "help" {
for _, c := range cmds {
c.printUsage()
}
return
}
for _, c := range cmds {
if flag.Arg(0) == c.name {
c.run(conn, flag.Args()[1:])
return
}
}
errorf("unknown command %q", flag.Arg(0))
os.Exit(2)
}
func (s *AmazonServer) SetUp(c *C) {
auth, err := aws.EnvAuth()
if err != nil {
c.Fatal(err)
}
s.auth = auth
}
// Read the state from S3.
func readState() {
state := State{}
wd, err := os.Getwd()
fmt.Printf("wd: %s\n", wd)
// read from S3
auth, err := aws.EnvAuth()
if err != nil {
panic(err.Error())
}
s := s3.New(auth, aws.USEast)
bucket := s.Bucket(s3_bucket)
data, err := bucket.Get("rss.json")
if err != nil {
panic(err)
}
fmt.Printf("Data read from S3\n")
// old way
/*
file, err := os.Open("state.json")
if err != nil {
panic(err)
}
data, err := ioutil.ReadAll(file)
if err != nil {
panic(err)
}
*/
err = json.Unmarshal(data, &state)
if err != nil {
panic(err)
}
folders = state.Folders
}
func TestRealtimeTrace(t *testing.T) {
auth, err := aws.EnvAuth()
if err != nil {
panic(err)
}
s := ec2.New(auth, aws.EUWest)
m, err := NewEC2Monitor(s, "m1.medium",
"Linux/UNIX",
"eu-west-1b", nil)
to := time.Now()
from := to.AddDate(0, -3, 0)
trace, err := m.Trace(from, to)
if err != nil {
t.Fatal(err)
}
if len(trace) == 0 {
t.Fatal("trace did not produce any history")
}
if len(trace) < 5 {
for i := 0; i < len(trace); i++ {
pp := trace[i]
fmt.Printf("\t%.4f - %v\n", pp.Price(), pp.Time())
}
} else {
fmt.Println("last 5 traces:")
for i := 0; i < 5; i++ {
pp := trace[i]
fmt.Printf("\t%.4f - %v\n", pp.Price(), pp.Time())
}
}
pChan, errChan := m.Start(time.Second * 5)
ticker := time.NewTicker(time.Minute * 5)
fmt.Println("Starting EC2 Spot Price Monitor.\n\tTimeout: 5 minutes")
for {
select {
case p := <-pChan:
fmt.Printf("PriceChange: %v On: %v\n", p.Price(), p.Time())
case err := <-errChan:
fmt.Println("error: ", err.Error())
return
case <-ticker.C:
fmt.Println("test shut down after 5 minutes")
return
}
}
fmt.Println("Stopping ticker, and exiting test..")
ticker.Stop()
fmt.Println("Done")
}
func auth() aws.Auth {
creds, err := aws.EnvAuth()
if err != nil {
log.Println("Error with aws credentials: %v", err)
os.Exit(1)
}
return creds
}
func (s *S) TestEnvAuth(c *C) {
os.Clearenv()
os.Setenv("AWS_SECRET_ACCESS_KEY", "secret")
os.Setenv("AWS_ACCESS_KEY_ID", "access")
auth, err := aws.EnvAuth()
c.Assert(err, IsNil)
c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"})
}
func awsInit(config *ec2Config) (ec2Conn *ec2.EC2, err error) {
auth, err := aws.EnvAuth()
if err != nil {
return nil, err
}
return ec2.New(auth, config.region), nil
}
func (s *SuiteI) SetUpSuite(c *C) {
if !*integration {
c.Skip("Integration tests not enabled (-i flag)")
}
auth, err := aws.EnvAuth()
if err != nil {
c.Fatal(err.String())
}
s.auth = auth
}
func TestWrapS3(t *testing.T) {
auth, err := aws.EnvAuth()
if err != nil {
t.Fatalf("Need AWS auth for testing: %v", err)
}
// TODO: CL switch or similar to change the AWS zone
backend := WrapS3(goamzs3.New(auth, aws.EUWest))
testS3backend(backend, t)
return
}
func (s *LiveSuite) SetUpSuite(c *C) {
if !Amazon {
c.Skip("amazon tests not enabled (-amazon flag)")
}
auth, err := aws.EnvAuth()
if err != nil {
c.Fatal(err.Error())
}
s.auth = auth
}
func authS3(auth aws.Auth) *s3.S3 {
var err error
if auth.AccessKey == "" && auth.SecretKey == "" {
auth, err = aws.EnvAuth()
if err != nil {
log.Println(err.Error())
panic(err.Error())
}
}
return s3.New(auth, aws.USEast)
}
func connectS3() {
s3auth, err := aws.EnvAuth()
if err != nil {
panic(err)
}
s3conn := s3.New(s3auth, aws.Regions["eu-west-1"])
bucket := s3conn.Bucket("gobuild.luzifer.io")
s3Bucket = bucket
}
func Main() {
// Read in any environment variables we care about
readEnv()
var err error
AWS_AUTH, err = aws.EnvAuth()
if err != nil {
panic(err.Error())
}
// Open database connection
db, err := models.OpenDB("postgres", IRLMOJI_DBURI)
if err != nil {
log.Fatalf("Error opening database connection: %v", err.Error())
return
}
if err = createAllTables(db); err != nil {
log.Fatalf("Error creating database table: %v", err.Error())
}
// Start setting up martini
m := martini.Classic()
// Set up the middleware
m.Use(gzip.All())
m.Use(render.Renderer())
m.Use(BackchannelAuth(IRLMOJI_API_BASIC_USER))
// Inject the database
m.Map(db)
// Map the URL routes
m.Get("/", HandleIndex)
// User routes (see handlers_user.go)
m.Get("/api/v1/users/current.json", HandleGetCurrentUser)
m.Post("/api/v1/users/twitter.json", binding.Json(models.UserForm{}), HandleCreateUserByTwitter)
// IRLMoji routes (see handlers_irlmoji.go)
m.Get("/api/v1/timelines/home.json", binding.Form(Limit{}), HandleGetHomeTimeline)
m.Get("/api/v1/timelines/user/username/:username.json", binding.Form(Limit{}), HandleGetUserTimeline)
m.Get("/api/v1/timelines/emoji/:emoji.json", binding.Form(Limit{}), HandleGetEmojiTimeline)
m.Get("/api/v1/irlmoji/id/:irlmojiId.json", binding.Form(Limit{}), HandleGetIRLMoji)
m.Delete("/api/v1/irlmoji/id/:irlmojiId.json", HandleDeleteIRLMoji)
m.Post("/api/v1/irlmoji.json", binding.Json(models.IRLMoji{}), HandleCreateIRLMoji)
m.Post("/api/v1/irlmoji/id/:irlmojiId/heart.json", binding.Json(models.Heart{}), HandleToggleHeart)
m.Post("/upload", HandleUpload)
m.NotFound(HandleNotFound)
m.Run()
}
func main() {
fmt.Printf("%v\n", "foo bar")
_, err := aws.EnvAuth()
if err != nil {
panic(err.Error())
}
}
func NewDomain(db_prefix string) (*Domain, error) {
auth, err := aws.EnvAuth()
if err != nil {
log.Printf("AWS auth error: %s", err.Error())
return nil, err
}
var d Domain
d.db_prefix = db_prefix
d.auth = auth
return &d, nil
}
func AWSInit(access_key string, secret_key string) {
var auth aws.Auth
var err error
if access_key != "" && secret_key != "" {
auth = aws.Auth{access_key, secret_key}
} else {
auth, err = aws.EnvAuth()
if err != nil {
log.Fatal(err)
}
}
s := s3.New(auth, aws.USEast)
PicsBucket = s.Bucket(BUCKET_NAME)
}
func init() {
var err error
log.Out = os.Stderr
conf = config.Load()
// Add Papertrail connection for logging
if conf.Papertrail.Port != 0 {
hook, err := logrus_papertrail.NewPapertrailHook(conf.Papertrail.Host, conf.Papertrail.Port, "GoBuilder Starter")
if err != nil {
log.WithFields(logrus.Fields{
"host": hostname,
}).Panic("Unable to create papertrail connection")
os.Exit(1)
}
log.Hooks.Add(hook)
} else {
log.WithFields(logrus.Fields{
"host": hostname,
}).Info("Failed to read papertrail_port, using only STDERR")
}
connectRedis()
awsAuth, err := aws.EnvAuth()
if err != nil {
log.WithFields(logrus.Fields{
"host": hostname,
"err": err,
}).Panic("Unable to read AWS credentials")
os.Exit(1)
}
s3Bucket = s3.New(awsAuth, aws.EUWest).Bucket("gobuild.luzifer.io")
dockerClient, err = docker.NewClient("unix:///var/run/docker.sock")
if err != nil {
log.WithFields(logrus.Fields{
"host": hostname,
"err": err,
}).Panic("Unable to connect to docker daemon")
os.Exit(1)
}
currentJobs = make(chan bool, maxConcurrentBuilds)
hostname, err = os.Hostname()
}
func (p environProvider) Validate(cfg, old *config.Config) (valid *config.Config, err error) {
v, err := configChecker.Coerce(cfg.UnknownAttrs(), nil)
if err != nil {
return nil, err
}
ecfg := &environConfig{cfg, v.(map[string]interface{})}
if ecfg.accessKey() == "" || ecfg.secretKey() == "" {
auth, err := aws.EnvAuth()
if err != nil || ecfg.accessKey() != "" || ecfg.secretKey() != "" {
return nil, fmt.Errorf("environment has no access-key or secret-key")
}
ecfg.attrs["access-key"] = auth.AccessKey
ecfg.attrs["secret-key"] = auth.SecretKey
}
if _, ok := aws.Regions[ecfg.region()]; !ok {
return nil, fmt.Errorf("invalid region name %q", ecfg.region())
}
if _, ok := aws.Regions[ecfg.publicBucketRegion()]; !ok {
return nil, fmt.Errorf("invalid public-bucket-region name %q", ecfg.publicBucketRegion())
}
if old != nil {
attrs := old.UnknownAttrs()
if region, _ := attrs["region"].(string); ecfg.region() != region {
return nil, fmt.Errorf("cannot change region from %q to %q", region, ecfg.region())
}
if bucket, _ := attrs["control-bucket"].(string); ecfg.controlBucket() != bucket {
return nil, fmt.Errorf("cannot change control-bucket from %q to %q", bucket, ecfg.controlBucket())
}
}
switch cfg.FirewallMode() {
case config.FwDefault:
ecfg.attrs["firewall-mode"] = config.FwInstance
case config.FwInstance, config.FwGlobal:
default:
return nil, fmt.Errorf("unsupported firewall mode: %q", cfg.FirewallMode())
}
// ssl-hostname-verification cannot be disabled
if !ecfg.SSLHostnameVerification() {
return nil, fmt.Errorf("disabling ssh-hostname-verification is not supported")
}
return cfg.Apply(ecfg.attrs)
}
// Parse and validate our options
func setupOptions() (roadcrewOptions, error) {
opts := roadcrewOptions{}
// Handle command line options
arguments, err := docopt.Parse(usage, nil, true, "roadcrew 0.2", false)
if err != nil {
return opts, err
}
opts.interval = arguments["<interval>"].(string)
opts.timeout, err = time.ParseDuration(opts.interval + "s")
if err != nil {
log.Fatal(err)
}
opts.bucketName = arguments["<bucket_name>"].(string)
opts.tmpDir = "/tmp"
if arguments["--tmp_dir"] != nil {
opts.tmpDir = arguments["--tmp_dir"].(string)
}
// Check if valid AWS region
if _, present := aws.Regions[arguments["-r"].(string)]; present == false {
log.Fatalf("Invalid AWS region: %s", arguments["-r"])
}
opts.region = aws.Regions[arguments["-r"].(string)]
// Read AWS credentials from the environment
opts.auth, err = aws.EnvAuth()
if err != nil {
// Read AWS credentials from the CLI if not in env
if arguments["-K"] == nil || arguments["-S"] == nil {
return opts, errors.New("AWS credentials not found: must use -K and -S flags, or set these env vars:\n\texport AWS_ACCESS_KEY_ID=\"AAA...\"\n\texport AWS_SECRET_ACCESS_KEY=\"BBBB...\"\n")
}
opts.auth = aws.Auth{
AccessKey: arguments["-K"].(string),
SecretKey: arguments["-S"].(string),
}
}
if err := checkDependencies(opts); err != nil {
return opts, err
}
return opts, nil
}
// NewUploader returns a new uploader for the given S3 bucket
//
// AWS credentials must be set in the environment variables
// AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
func NewUploader(bucket string, workerCount int) (*Uploader, error) {
u := new(Uploader)
auth, err := aws.EnvAuth()
if err != nil {
return nil, err
}
s := s3.New(auth, aws.USEast)
u.bucket = s.Bucket(bucket)
u.uploads = make(chan *Upload)
for i := 1; i <= workerCount; i++ {
go u.worker(i)
}
return u, nil
}
func (p environProvider) Validate(cfg, old *config.Config) (valid *config.Config, err error) {
// Check for valid changes for the base config values.
if err := config.Validate(cfg, old); err != nil {
return nil, err
}
validated, err := cfg.ValidateUnknownAttrs(configFields, configDefaults)
if err != nil {
return nil, err
}
ecfg := &environConfig{cfg, validated}
if ecfg.accessKey() == "" || ecfg.secretKey() == "" {
auth, err := aws.EnvAuth()
if err != nil || ecfg.accessKey() != "" || ecfg.secretKey() != "" {
return nil, fmt.Errorf("environment has no access-key or secret-key")
}
ecfg.attrs["access-key"] = auth.AccessKey
ecfg.attrs["secret-key"] = auth.SecretKey
}
if _, ok := aws.Regions[ecfg.region()]; !ok {
return nil, fmt.Errorf("invalid region name %q", ecfg.region())
}
if _, ok := aws.Regions[ecfg.publicBucketRegion()]; !ok {
return nil, fmt.Errorf("invalid public-bucket-region name %q", ecfg.publicBucketRegion())
}
if old != nil {
attrs := old.UnknownAttrs()
if region, _ := attrs["region"].(string); ecfg.region() != region {
return nil, fmt.Errorf("cannot change region from %q to %q", region, ecfg.region())
}
if bucket, _ := attrs["control-bucket"].(string); ecfg.controlBucket() != bucket {
return nil, fmt.Errorf("cannot change control-bucket from %q to %q", bucket, ecfg.controlBucket())
}
}
// ssl-hostname-verification cannot be disabled
if !ecfg.SSLHostnameVerification() {
return nil, fmt.Errorf("disabling ssh-hostname-verification is not supported")
}
// Apply the coerced unknown values back into the config.
return cfg.Apply(ecfg.attrs)
}
func ReadConfig(filename string) (*Config, error) {
b, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
c := new(Config)
if err := json.Unmarshal(b, c); err != nil {
return nil, err
}
auth, err := aws.EnvAuth()
if err != nil {
return nil, err
}
c.auth = auth
return c, nil
}