Esempio n. 1
0
func TestReadFromSQS(t *testing.T) {
	auth, err := aws.EnvAuth()
	if err != nil {
		panic(err)
	}

	sqs := New(auth, aws.USWest2)
	queue, err := sqs.GetQueue("tracker-alpha-matt")
	if err != nil {
		panic(err)
	}

	results, err := queue.ReceiveMessage([]string{"All"}, 10, 600)
	count := len(results.Messages)
	deleteMessageBatch := make([]DeleteMessageBatch, count)
	for index, message := range results.Messages {
		fmt.Println(message.Body)
		deleteMessageBatch[index] = DeleteMessageBatch{
			Id:            strconv.Itoa(index),
			ReceiptHandle: message.ReceiptHandle,
		}
	}

	fmt.Println(fmt.Sprintf("%#v", deleteMessageBatch))
	response, err := queue.DeleteMessageBatch(deleteMessageBatch)
	if err != nil {
		panic(err)
	}
	fmt.Println(fmt.Sprintf("%#v", response))
}
Esempio n. 2
0
func Sync(localPath, bucketName, awsRegion, acl string) {
	if localPath == "" || bucketName == "" {
		flag.PrintDefaults()
		return
	}
	auth, err := aws.EnvAuth()
	if err != nil {
		panic(err.Error())
	}
	s3Conn := s3.New(auth, aws.Regions[awsRegion])
	bucket := s3Conn.Bucket(bucketName)
	pathChan := make(chan string)
	doneChan := make(chan int)
	go doTheWalk(localPath, pathChan)
	count := 0
	for path := range pathChan {
		if path == "" {
			break
		}
		go compareAndSync(acl, localPath, path, bucket, doneChan)
		count++
	}
	for i := 0; i < count; i++ {
		<-doneChan
	}
}
Esempio n. 3
0
func main() {
	flag.Parse()

	cmds := flag.Args()

	if len(cmds) <= 0 {
		fmt.Println("Command required")
		os.Exit(1)
	}

	cmd := Cmd{}
	cmd.name = cmds[0]
	cmd.modifiers = cmds[1:]

	auth, err := aws.EnvAuth()
	if err != nil {
		panic(err)
	}

	cmd.conn = s3.New(auth, getRegion(region))
	s3url, err := url.Parse(cmd.modifiers[0])
	cmd.url = s3url
	cmd.bucket = s3.Bucket{cmd.conn, cmd.url.Host}
	RunCommand(cmd)

}
Esempio n. 4
0
func main() {
	flag.Parse()

	if *toHash == "" || *pkgDirs == "" || *cmd == "" || *bucketName == "" {
		log.Print("You are missing one or more mandatory command line arguments.")
		flag.Usage()
		os.Exit(1)
	}

	cred, err := aws.EnvAuth()
	if err != nil {
		log.Fatalf("Couldn't auth into s3. Did you set up ENV? Error: %s", err)
	}
	s3client := s3.New(cred, aws.USWest)
	bucket := s3client.Bucket(*bucketName)

	// extract.
	checksum := fmt.Sprintf("%x", hashFiles(*toHash).Sum(nil))

	filename := *outfile
	// consider bucket.GetReader to pipe directly into gunzip/untar
	file, err := bucket.Get(fmt.Sprintf("%s/%s", checksum, filename))
	if err != nil {
		fmt.Printf("%s\n", err)
		build(*cmd)
		archive(*pkgDirs, filename)
		upload(bucket, checksum, filename)
	} else {
		extract(file)
	}
}
Esempio n. 5
0
func PostSignup(w *rest.ResponseWriter, req *rest.Request) {
	signup := Signup{}
	err := req.DecodeJsonPayload(&signup)
	if err != nil {
		rest.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if signup.Email == "" {
		rest.Error(w, "email is required", 400)
		return
	}

	auth, err := aws.EnvAuth()
	if err != nil {
		panic(err.Error())
	}

	s := s3.New(auth, aws.EUWest)
	bucket := s.Bucket("seq-signup")
	data := []byte(signup.Email)
	err = bucket.Put(signup.Email, data, "text/plain", s3.BucketOwnerFull)
	if err != nil {
		panic(err.Error())
	}

	fmt.Println(signup)
	w.Header().Set("Access-Control-Allow-Origin", "http://sequenceiq.com")
	w.WriteJson(&signup)
}
Esempio n. 6
0
// Goroutine for saving the state.
func saver(ticker *time.Ticker) {
	for {
		select {
		case <-ticker.C:
			println("Writing state")
			lock.Lock()
			if dirty {
				dirty = false
				state := State{folders}

				bytes, err := json.Marshal(state)
				if err != nil {
					panic(err)
				}

				// write to S3
				auth, err := aws.EnvAuth()
				if err != nil {
					panic(err)
				}
				s := s3.New(auth, aws.USEast)
				bucket := s.Bucket(s3_bucket)
				err = bucket.Put("rss.json", bytes, "application/json", s3.ACL("private"))
				if err != nil {
					panic(err)
				}
			}
			lock.Unlock()
		}
	}
}
Esempio n. 7
0
// Load a brain
func (brain *Brain) Load() {
	// The AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables are used.
	auth, err := aws.EnvAuth()
	if err != nil {
		panic(err.Error())
	}

	// Open Bucket
	s := s3.New(auth, aws.USEast)

	// Load the database from an S3 bucket
	bucket := s.Bucket(bucketName)

	// Create a bytes.Buffer
	n, err := bucket.Get(filename)
	if err != nil {
		panic(err)
	}

	p := bytes.NewBuffer(n)
	dec := gob.NewDecoder(p)

	err = dec.Decode(&brain)

	if err != nil {
		log.Print("There was an error loading the brain. Using a blank one.")
	}
}
Esempio n. 8
0
// Persist a brain
func (brain *Brain) Save() {
	// Persist the database to file
	var data bytes.Buffer
	contents := gob.NewEncoder(&data)
	err := contents.Encode(brain)
	if err != nil {
		panic(err)
	}

	// The AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables are used.
	auth, err := aws.EnvAuth()
	if err != nil {
		panic(err.Error())
	}

	// Open Bucket
	s := s3.New(auth, aws.USEast)

	// Load the database from an S3 bucket
	bucket := s.Bucket(bucketName)

	err = bucket.Put(filename, data.Bytes(), "text/plain", s3.BucketOwnerFull)
	if err != nil {
		panic(err.Error())
	}
}
Esempio n. 9
0
func main() {
	flag.Parse()
	if flag.Arg(0) == "" {
		errorf("no command")
		os.Exit(2)
	}
	auth, err := aws.EnvAuth()
	if err != nil {
		fatalf("envauth: %v", err)
	}
	conn := ec2.New(auth, aws.USEast)

	if flag.Arg(0) == "help" {
		for _, c := range cmds {
			c.printUsage()
		}
		return
	}

	for _, c := range cmds {
		if flag.Arg(0) == c.name {
			c.run(conn, flag.Args()[1:])
			return
		}
	}
	errorf("unknown command %q", flag.Arg(0))
	os.Exit(2)
}
Esempio n. 10
0
func (s *AmazonServer) SetUp(c *C) {
	auth, err := aws.EnvAuth()
	if err != nil {
		c.Fatal(err)
	}
	s.auth = auth
}
Esempio n. 11
0
// Read the state from S3.
func readState() {
	state := State{}
	wd, err := os.Getwd()
	fmt.Printf("wd: %s\n", wd)

	// read from S3
	auth, err := aws.EnvAuth()
	if err != nil {
		panic(err.Error())
	}
	s := s3.New(auth, aws.USEast)
	bucket := s.Bucket(s3_bucket)
	data, err := bucket.Get("rss.json")
	if err != nil {
		panic(err)
	}
	fmt.Printf("Data read from S3\n")

	// old way
	/*
		file, err := os.Open("state.json")
		if err != nil {
			panic(err)
		}
		data, err := ioutil.ReadAll(file)
		if err != nil {
			panic(err)
		}
	*/
	err = json.Unmarshal(data, &state)
	if err != nil {
		panic(err)
	}
	folders = state.Folders
}
Esempio n. 12
0
func TestRealtimeTrace(t *testing.T) {

	auth, err := aws.EnvAuth()
	if err != nil {
		panic(err)
	}
	s := ec2.New(auth, aws.EUWest)

	m, err := NewEC2Monitor(s, "m1.medium",
		"Linux/UNIX",
		"eu-west-1b", nil)

	to := time.Now()
	from := to.AddDate(0, -3, 0)

	trace, err := m.Trace(from, to)
	if err != nil {
		t.Fatal(err)
	}

	if len(trace) == 0 {
		t.Fatal("trace did not produce any history")
	}

	if len(trace) < 5 {
		for i := 0; i < len(trace); i++ {
			pp := trace[i]
			fmt.Printf("\t%.4f - %v\n", pp.Price(), pp.Time())
		}
	} else {
		fmt.Println("last 5 traces:")
		for i := 0; i < 5; i++ {
			pp := trace[i]
			fmt.Printf("\t%.4f - %v\n", pp.Price(), pp.Time())
		}
	}

	pChan, errChan := m.Start(time.Second * 5)

	ticker := time.NewTicker(time.Minute * 5)

	fmt.Println("Starting EC2 Spot Price Monitor.\n\tTimeout: 5 minutes")

	for {
		select {
		case p := <-pChan:
			fmt.Printf("PriceChange: %v On: %v\n", p.Price(), p.Time())
		case err := <-errChan:
			fmt.Println("error: ", err.Error())
			return
		case <-ticker.C:
			fmt.Println("test shut down after 5 minutes")
			return
		}
	}
	fmt.Println("Stopping ticker, and exiting test..")
	ticker.Stop()
	fmt.Println("Done")
}
Esempio n. 13
0
func auth() aws.Auth {
	creds, err := aws.EnvAuth()
	if err != nil {
		log.Println("Error with aws credentials: %v", err)
		os.Exit(1)
	}
	return creds
}
Esempio n. 14
0
func (s *S) TestEnvAuth(c *C) {
	os.Clearenv()
	os.Setenv("AWS_SECRET_ACCESS_KEY", "secret")
	os.Setenv("AWS_ACCESS_KEY_ID", "access")
	auth, err := aws.EnvAuth()
	c.Assert(err, IsNil)
	c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"})
}
Esempio n. 15
0
func awsInit(config *ec2Config) (ec2Conn *ec2.EC2, err error) {
	auth, err := aws.EnvAuth()

	if err != nil {
		return nil, err
	}

	return ec2.New(auth, config.region), nil
}
Esempio n. 16
0
func (s *SuiteI) SetUpSuite(c *C) {
	if !*integration {
		c.Skip("Integration tests not enabled (-i flag)")
	}
	auth, err := aws.EnvAuth()
	if err != nil {
		c.Fatal(err.String())
	}
	s.auth = auth
}
Esempio n. 17
0
func TestWrapS3(t *testing.T) {
	auth, err := aws.EnvAuth()
	if err != nil {
		t.Fatalf("Need AWS auth for testing: %v", err)
	}
	// TODO: CL switch or similar to change the AWS zone
	backend := WrapS3(goamzs3.New(auth, aws.EUWest))
	testS3backend(backend, t)
	return
}
Esempio n. 18
0
func (s *LiveSuite) SetUpSuite(c *C) {
	if !Amazon {
		c.Skip("amazon tests not enabled (-amazon flag)")
	}
	auth, err := aws.EnvAuth()
	if err != nil {
		c.Fatal(err.Error())
	}
	s.auth = auth
}
Esempio n. 19
0
func authS3(auth aws.Auth) *s3.S3 {
	var err error
	if auth.AccessKey == "" && auth.SecretKey == "" {
		auth, err = aws.EnvAuth()
		if err != nil {
			log.Println(err.Error())
			panic(err.Error())
		}
	}
	return s3.New(auth, aws.USEast)
}
Esempio n. 20
0
func connectS3() {
	s3auth, err := aws.EnvAuth()
	if err != nil {
		panic(err)
	}

	s3conn := s3.New(s3auth, aws.Regions["eu-west-1"])
	bucket := s3conn.Bucket("gobuild.luzifer.io")

	s3Bucket = bucket
}
Esempio n. 21
0
func Main() {
	// Read in any environment variables we care about
	readEnv()

	var err error

	AWS_AUTH, err = aws.EnvAuth()
	if err != nil {
		panic(err.Error())
	}

	// Open database connection
	db, err := models.OpenDB("postgres", IRLMOJI_DBURI)
	if err != nil {
		log.Fatalf("Error opening database connection: %v", err.Error())
		return
	}
	if err = createAllTables(db); err != nil {
		log.Fatalf("Error creating database table: %v", err.Error())
	}

	// Start setting up martini
	m := martini.Classic()

	// Set up the middleware
	m.Use(gzip.All())
	m.Use(render.Renderer())
	m.Use(BackchannelAuth(IRLMOJI_API_BASIC_USER))

	// Inject the database
	m.Map(db)

	// Map the URL routes
	m.Get("/", HandleIndex)

	// User routes (see handlers_user.go)
	m.Get("/api/v1/users/current.json", HandleGetCurrentUser)
	m.Post("/api/v1/users/twitter.json", binding.Json(models.UserForm{}), HandleCreateUserByTwitter)

	// IRLMoji routes (see handlers_irlmoji.go)
	m.Get("/api/v1/timelines/home.json", binding.Form(Limit{}), HandleGetHomeTimeline)
	m.Get("/api/v1/timelines/user/username/:username.json", binding.Form(Limit{}), HandleGetUserTimeline)
	m.Get("/api/v1/timelines/emoji/:emoji.json", binding.Form(Limit{}), HandleGetEmojiTimeline)
	m.Get("/api/v1/irlmoji/id/:irlmojiId.json", binding.Form(Limit{}), HandleGetIRLMoji)
	m.Delete("/api/v1/irlmoji/id/:irlmojiId.json", HandleDeleteIRLMoji)
	m.Post("/api/v1/irlmoji.json", binding.Json(models.IRLMoji{}), HandleCreateIRLMoji)
	m.Post("/api/v1/irlmoji/id/:irlmojiId/heart.json", binding.Json(models.Heart{}), HandleToggleHeart)

	m.Post("/upload", HandleUpload)

	m.NotFound(HandleNotFound)

	m.Run()
}
Esempio n. 22
0
func main() {

	fmt.Printf("%v\n", "foo bar")

	_, err := aws.EnvAuth()

	if err != nil {
		panic(err.Error())
	}

}
Esempio n. 23
0
func NewDomain(db_prefix string) (*Domain, error) {
	auth, err := aws.EnvAuth()
	if err != nil {
		log.Printf("AWS auth error: %s", err.Error())
		return nil, err
	}

	var d Domain
	d.db_prefix = db_prefix
	d.auth = auth
	return &d, nil
}
Esempio n. 24
0
func AWSInit(access_key string, secret_key string) {
	var auth aws.Auth
	var err error
	if access_key != "" && secret_key != "" {
		auth = aws.Auth{access_key, secret_key}
	} else {
		auth, err = aws.EnvAuth()
		if err != nil {
			log.Fatal(err)
		}
	}

	s := s3.New(auth, aws.USEast)
	PicsBucket = s.Bucket(BUCKET_NAME)
}
Esempio n. 25
0
func init() {
	var err error
	log.Out = os.Stderr

	conf = config.Load()

	// Add Papertrail connection for logging
	if conf.Papertrail.Port != 0 {
		hook, err := logrus_papertrail.NewPapertrailHook(conf.Papertrail.Host, conf.Papertrail.Port, "GoBuilder Starter")
		if err != nil {
			log.WithFields(logrus.Fields{
				"host": hostname,
			}).Panic("Unable to create papertrail connection")
			os.Exit(1)
		}

		log.Hooks.Add(hook)
	} else {
		log.WithFields(logrus.Fields{
			"host": hostname,
		}).Info("Failed to read papertrail_port, using only STDERR")
	}

	connectRedis()

	awsAuth, err := aws.EnvAuth()
	if err != nil {
		log.WithFields(logrus.Fields{
			"host": hostname,
			"err":  err,
		}).Panic("Unable to read AWS credentials")
		os.Exit(1)
	}
	s3Bucket = s3.New(awsAuth, aws.EUWest).Bucket("gobuild.luzifer.io")

	dockerClient, err = docker.NewClient("unix:///var/run/docker.sock")
	if err != nil {
		log.WithFields(logrus.Fields{
			"host": hostname,
			"err":  err,
		}).Panic("Unable to connect to docker daemon")
		os.Exit(1)
	}

	currentJobs = make(chan bool, maxConcurrentBuilds)

	hostname, err = os.Hostname()
}
Esempio n. 26
0
func (p environProvider) Validate(cfg, old *config.Config) (valid *config.Config, err error) {
	v, err := configChecker.Coerce(cfg.UnknownAttrs(), nil)
	if err != nil {
		return nil, err
	}
	ecfg := &environConfig{cfg, v.(map[string]interface{})}
	if ecfg.accessKey() == "" || ecfg.secretKey() == "" {
		auth, err := aws.EnvAuth()
		if err != nil || ecfg.accessKey() != "" || ecfg.secretKey() != "" {
			return nil, fmt.Errorf("environment has no access-key or secret-key")
		}
		ecfg.attrs["access-key"] = auth.AccessKey
		ecfg.attrs["secret-key"] = auth.SecretKey
	}
	if _, ok := aws.Regions[ecfg.region()]; !ok {
		return nil, fmt.Errorf("invalid region name %q", ecfg.region())
	}
	if _, ok := aws.Regions[ecfg.publicBucketRegion()]; !ok {
		return nil, fmt.Errorf("invalid public-bucket-region name %q", ecfg.publicBucketRegion())
	}

	if old != nil {
		attrs := old.UnknownAttrs()
		if region, _ := attrs["region"].(string); ecfg.region() != region {
			return nil, fmt.Errorf("cannot change region from %q to %q", region, ecfg.region())
		}
		if bucket, _ := attrs["control-bucket"].(string); ecfg.controlBucket() != bucket {
			return nil, fmt.Errorf("cannot change control-bucket from %q to %q", bucket, ecfg.controlBucket())
		}
	}

	switch cfg.FirewallMode() {
	case config.FwDefault:
		ecfg.attrs["firewall-mode"] = config.FwInstance
	case config.FwInstance, config.FwGlobal:
	default:
		return nil, fmt.Errorf("unsupported firewall mode: %q", cfg.FirewallMode())
	}

	// ssl-hostname-verification cannot be disabled
	if !ecfg.SSLHostnameVerification() {
		return nil, fmt.Errorf("disabling ssh-hostname-verification is not supported")
	}

	return cfg.Apply(ecfg.attrs)
}
Esempio n. 27
0
// Parse and validate our options
func setupOptions() (roadcrewOptions, error) {
	opts := roadcrewOptions{}

	// Handle command line options
	arguments, err := docopt.Parse(usage, nil, true, "roadcrew 0.2", false)
	if err != nil {
		return opts, err
	}

	opts.interval = arguments["<interval>"].(string)
	opts.timeout, err = time.ParseDuration(opts.interval + "s")
	if err != nil {
		log.Fatal(err)
	}

	opts.bucketName = arguments["<bucket_name>"].(string)
	opts.tmpDir = "/tmp"
	if arguments["--tmp_dir"] != nil {
		opts.tmpDir = arguments["--tmp_dir"].(string)
	}

	// Check if valid AWS region
	if _, present := aws.Regions[arguments["-r"].(string)]; present == false {
		log.Fatalf("Invalid AWS region: %s", arguments["-r"])
	}
	opts.region = aws.Regions[arguments["-r"].(string)]

	// Read AWS credentials from the environment
	opts.auth, err = aws.EnvAuth()
	if err != nil {
		// Read AWS credentials from the CLI if not in env
		if arguments["-K"] == nil || arguments["-S"] == nil {
			return opts, errors.New("AWS credentials not found: must use -K and -S flags, or set these env vars:\n\texport AWS_ACCESS_KEY_ID=\"AAA...\"\n\texport AWS_SECRET_ACCESS_KEY=\"BBBB...\"\n")
		}
		opts.auth = aws.Auth{
			AccessKey: arguments["-K"].(string),
			SecretKey: arguments["-S"].(string),
		}
	}

	if err := checkDependencies(opts); err != nil {
		return opts, err
	}

	return opts, nil
}
Esempio n. 28
0
// NewUploader returns a new uploader for the given S3 bucket
//
// AWS credentials must be set in the environment variables
// AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
func NewUploader(bucket string, workerCount int) (*Uploader, error) {
	u := new(Uploader)

	auth, err := aws.EnvAuth()
	if err != nil {
		return nil, err
	}

	s := s3.New(auth, aws.USEast)
	u.bucket = s.Bucket(bucket)
	u.uploads = make(chan *Upload)

	for i := 1; i <= workerCount; i++ {
		go u.worker(i)
	}

	return u, nil
}
Esempio n. 29
0
func (p environProvider) Validate(cfg, old *config.Config) (valid *config.Config, err error) {
	// Check for valid changes for the base config values.
	if err := config.Validate(cfg, old); err != nil {
		return nil, err
	}
	validated, err := cfg.ValidateUnknownAttrs(configFields, configDefaults)
	if err != nil {
		return nil, err
	}
	ecfg := &environConfig{cfg, validated}
	if ecfg.accessKey() == "" || ecfg.secretKey() == "" {
		auth, err := aws.EnvAuth()
		if err != nil || ecfg.accessKey() != "" || ecfg.secretKey() != "" {
			return nil, fmt.Errorf("environment has no access-key or secret-key")
		}
		ecfg.attrs["access-key"] = auth.AccessKey
		ecfg.attrs["secret-key"] = auth.SecretKey
	}
	if _, ok := aws.Regions[ecfg.region()]; !ok {
		return nil, fmt.Errorf("invalid region name %q", ecfg.region())
	}
	if _, ok := aws.Regions[ecfg.publicBucketRegion()]; !ok {
		return nil, fmt.Errorf("invalid public-bucket-region name %q", ecfg.publicBucketRegion())
	}

	if old != nil {
		attrs := old.UnknownAttrs()
		if region, _ := attrs["region"].(string); ecfg.region() != region {
			return nil, fmt.Errorf("cannot change region from %q to %q", region, ecfg.region())
		}
		if bucket, _ := attrs["control-bucket"].(string); ecfg.controlBucket() != bucket {
			return nil, fmt.Errorf("cannot change control-bucket from %q to %q", bucket, ecfg.controlBucket())
		}
	}

	// ssl-hostname-verification cannot be disabled
	if !ecfg.SSLHostnameVerification() {
		return nil, fmt.Errorf("disabling ssh-hostname-verification is not supported")
	}

	// Apply the coerced unknown values back into the config.
	return cfg.Apply(ecfg.attrs)
}
Esempio n. 30
0
func ReadConfig(filename string) (*Config, error) {
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	c := new(Config)
	if err := json.Unmarshal(b, c); err != nil {
		return nil, err
	}

	auth, err := aws.EnvAuth()
	if err != nil {
		return nil, err
	}
	c.auth = auth

	return c, nil
}