// InitializeAWS sets up access to the AWS Simple Token Service
func InitializeAWS() error {
if *awsRegion == "" {
*awsRegion = aws.InstanceRegion()
if *awsRegion == "unknown" {
*awsRegion = "us-east-1"
}
}
if *awsAccessKey == "" || *awsSecretKey == "" {
return fmt.Errorf("you must specify aws-access-key-id and " +
"aws-secret-access-key in the config file or " +
"AWSAUTHD_AWS_ACCESS_KEY_ID and AWSAUTHD_AWS_SECRET_ACCESS_KEY in " +
"the environment. These must be regular permanent credentials, not " +
"temporary or instance credentials.")
}
maybeAWSAuth := aws.Auth{
AccessKey: *awsAccessKey,
SecretKey: *awsSecretKey,
}
stsConnection := sts.New(maybeAWSAuth, aws.GetRegion(*awsRegion))
_, err := stsConnection.GetFederationToken("snakeoil", "", 900)
if err != nil {
return fmt.Errorf("Your credentials don't work to call "+
"GetFederationToken(). You must specify aws-access-key-id and "+
"aws-secret-access-key in the config file or "+
"AWSAUTHD_AWS_ACCESS_KEY_ID and AWSAUTHD_AWS_SECRET_ACCESS_KEY in "+
"the environment. These must be regular permanent credentials, not "+
"temporary or instance credentials. (err=%s)", err)
}
// If GetFederationToken worked then we are good to go.
awsAuth = maybeAWSAuth
return nil
}
func main() {
var inst_id string
if instanceID == "" {
inst_id := aws.InstanceId()
if inst_id == "unknown" {
log.Fatalln("Unable to get instance id")
}
} else {
inst_id = instanceID
}
auth, err := aws.GetAuth(accesskey, secretkey, "", time.Time{})
if err != nil {
log.Fatalln("Unable to get AWS auth", err)
}
awsec2 = ec2.New(auth, aws.GetRegion(region))
groupMap := getSecurityGroupIds(inst_id)
for _, id := range securityGroupIDs {
groupMap[id] = true
}
groupIds := make([]string, 0, len(groupMap))
for id := range groupMap {
groupIds = append(groupIds, id)
}
opts := &ec2.ModifyInstanceAttributeOptions{SecurityGroups: ec2.SecurityGroupIds(groupIds...)}
resp, err := awsec2.ModifyInstanceAttribute(inst_id, opts)
if err != nil || !resp.Return {
log.Fatalln("Error adding security groups to instance", err)
}
log.Printf("Added security groups %s to instance %s\n", securityGroupIDs.String(), inst_id)
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL)
// this waits until we get a kill signal
<-c
groupMap = getSecurityGroupIds(inst_id)
for _, id := range securityGroupIDs {
delete(groupMap, id)
}
groupIds = make([]string, 0, len(groupMap))
for id := range groupMap {
groupIds = append(groupIds, id)
}
opts = &ec2.ModifyInstanceAttributeOptions{SecurityGroups: ec2.SecurityGroupIds(groupIds...)}
resp, err = awsec2.ModifyInstanceAttribute(inst_id, opts)
if err != nil || !resp.Return {
log.Fatalln("Error removing security groups from instance", err)
}
log.Printf("Removed security groups %s from instance %s\n", securityGroupIDs.String(), inst_id)
}
// GetCredentials fetches credentials for the specified user and policy.
func GetCredentials(user string, policyString string,
tokenLifetime time.Duration) (*sts.Credentials, error) {
stsConnection := sts.New(awsAuth, aws.GetRegion(*awsRegion))
getTokenResult, err := stsConnection.GetFederationToken(user, policyString,
int(tokenLifetime.Seconds()))
if err != nil {
return nil, fmt.Errorf("GetFederationToken: %s", err)
}
return &getTokenResult.Credentials, nil
}
func LoadAWSConfigFile(fileName string, profileName string) error {
if profileName == "" {
profileName = "default"
}
f, err := os.Open(fileName)
if err != nil {
return fmt.Errorf("Cannot open %s %s", fileName, err)
}
defer f.Close()
body, err := ioutil.ReadAll(f)
if err != nil {
return fmt.Errorf("Cannot read %s %s", fileName, err)
}
profiles := map[string]map[string]string{}
currentProfile := ""
for _, line := range strings.Split(string(body), "\n") {
if strings.Index(line, "[profile ") == 0 {
p := strings.Split(line, " ")
if len(p) < 2 {
continue
}
currentProfile = strings.Replace(p[1], "]", "", 1)
} else if strings.Index(line, "[default]") == 0 {
currentProfile = "default"
} else if strings.Index(line, "=") != -1 {
pair := strings.Split(line, "=")
key := strings.Trim(pair[0], " ")
val := strings.Trim(pair[1], " ")
if _, ok := profiles[currentProfile]; !ok {
profiles[currentProfile] = map[string]string{}
}
profiles[currentProfile][key] = val
}
}
profile, ok := profiles[profileName]
if !ok {
return fmt.Errorf("profile [%s] not found in %s", profileName, fileName)
}
AWSAuth = aws.Auth{
AccessKey: profile["aws_access_key_id"],
SecretKey: profile["aws_secret_access_key"],
}
if profile["region"] == "" {
profile["region"] = AWSDefaultRegionName
}
AWSRegion = aws.GetRegion(profile["region"])
log.Printf("aws_access_key_id=%s", AWSAuth.AccessKey)
log.Printf("region=%s", AWSRegion.Name)
return nil
}
func main() {
var (
auth aws.Auth
targetURL url.URL
listenAddress string
serviceName string
region aws.Region
)
const flagEnvPrefix = "AWS_AUTH_PROXY"
fs := flag.NewFlagSet("aws-auth-proxy", flag.ExitOnError)
fs.StringVar(&auth.AccessKey, "access-key", "", "aws access key id")
fs.StringVar(&auth.SecretKey, "secret-key", "", "aws secret access key")
fs.StringVar(&serviceName, "service-name", "", "aws service name")
var regionName string
fs.StringVar(®ionName, "region-name", "", "aws region name")
fs.StringVar(&targetURL.Host, "upstream-host", "", "host or host:port for upstream endpoint")
fs.StringVar(&targetURL.Scheme, "upstream-scheme", "https", "scheme for upstream endpoint")
fs.StringVar(&listenAddress, "listen-address", ":8080", "address for proxy to listen on")
if err := flagutil.SetFlagsFromEnv(fs, flagEnvPrefix); err != nil {
log.Fatal(err)
}
if len(os.Args) >= 2 && os.Args[1] == "--help" {
fs.PrintDefaults()
fmt.Printf("\nflagutil prefix is '%s'\n", flagEnvPrefix)
fmt.Printf("example:\n\t-access-key=xxx OR export %s_ACCESS_KEY=xxx\n", flagEnvPrefix)
os.Exit(0)
}
fs.Parse(os.Args[1:])
region = aws.GetRegion(regionName)
signer := aws.NewV4Signer(auth, serviceName, region)
proxyHandler := &AWSProxy{
TargetURL: &targetURL,
Signer: signer,
}
fmt.Printf("Listening on %s\n", listenAddress)
log.Fatal(http.ListenAndServe(listenAddress, proxyHandler))
}
func NewDynamoStore(accessKey string, secretKey string, tableName string, region string, keyPairs ...[]byte) (*DynamoStore, error) {
regionObj := aws.GetRegion(region)
return NewDynamoStoreWithRegionObj(accessKey, secretKey, tableName, regionObj, keyPairs...)
}
func pushToS3(bundlesPath string) error {
if _, err := os.Stat(bundlesPath); os.IsNotExist(err) {
return fmt.Errorf("This is awkward, the bundles path DNE: %s", bundlesPath)
}
// use env variables to connect to s3
auth, err := aws.EnvAuth()
if err != nil {
return fmt.Errorf("AWS Auth failed: %v", err)
}
// connect to s3 bucket
s := s3.New(auth, aws.GetRegion(region))
bucketname, bucketpath := bucketParts(bucket)
bucket := s.Bucket(bucketname)
//walk the bundles directory
var html string
walkFn := func(fpath string, info os.FileInfo, err error) error {
stat, err := os.Stat(fpath)
if err != nil {
return err
}
relFilePath, err := filepath.Rel(bundlesPath, fpath)
if err != nil || (fpath == bundlesPath && stat.IsDir()) {
// Error getting relative path OR we are looking
// at the root path. Skip in both situations.
return nil
}
if stat.IsDir() {
return nil
}
if err = uploadFileToS3(bucket, fpath, path.Join(bucketpath, relFilePath)); err != nil {
log.Warnf("Uploading %s to s3 failed: %v", fpath, err)
return err
}
// add to html
image := "default"
if strings.HasSuffix(relFilePath, ".sha256") || strings.HasSuffix(relFilePath, ".md5") {
image = "text"
}
html += fmt.Sprintf(`<tr>
<td valign="top"><a href="%s"><img src="/static/%s.png" alt="[ICO]"/></a></td>
<td><a href="%s">%s</a></td>
<td>%s</td>
<td>%s</td>
</tr>`, relFilePath, image, relFilePath, relFilePath, humanSize(stat.Size()), stat.ModTime().Format(time.RFC3339))
return nil
}
// walk the filepath
if err := filepath.Walk(bundlesPath, walkFn); err != nil {
return err
}
// add html to template
if err := createIndexFile(bucket, bucketpath, html); err != nil {
return err
}
return nil
}
func main() {
var inst_id string
if instanceID == "" {
inst_id := aws.InstanceId()
if inst_id == "unknown" {
log.Fatalln("Unable to get instance id")
}
} else {
inst_id = instanceID
}
auth, err := aws.GetAuth(accesskey, secretkey, "", time.Time{})
if err != nil {
log.Fatalln("Unable to get AWS auth", err)
}
if securityGroupID != "" {
awsec2 = ec2.New(auth, aws.GetRegion(region))
groupMap := getSecurityGroupIds(inst_id)
groupMap[securityGroupID] = true
groupIds := make([]string, 0, len(groupMap))
for id := range groupMap {
groupIds = append(groupIds, id)
}
opts := &ec2.ModifyInstanceAttributeOptions{SecurityGroups: ec2.SecurityGroupIds(groupIds...)}
resp, err := awsec2.ModifyInstanceAttribute(inst_id, opts)
if err != nil || !resp.Return {
log.Fatalln("Error adding security group to instance", err)
}
log.Printf("Added security group %s to instance %s\n", securityGroupID, inst_id)
}
awselb := elb.New(auth, aws.GetRegion(region))
for _, lbname := range lbnames {
_, err = awselb.RegisterInstancesWithLoadBalancer([]string{inst_id}, lbname)
if err != nil {
log.Fatalln("Error registering instance", err)
}
log.Printf("Registered instance %s with elb %s\n", inst_id, lbname)
}
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL)
// this waits until we get a kill signal
<-c
for _, lbname := range lbnames {
_, err = awselb.DeregisterInstancesFromLoadBalancer([]string{inst_id}, lbname)
if err != nil {
log.Fatalln("Error deregistering instance", err)
}
log.Printf("Deregistered instance %s with elb %s\n", inst_id, lbname)
}
if securityGroupID != "" {
groupMap := getSecurityGroupIds(inst_id)
delete(groupMap, securityGroupID)
groupIds := make([]string, 0, len(groupMap))
for id := range groupMap {
groupIds = append(groupIds, id)
}
opts := &ec2.ModifyInstanceAttributeOptions{SecurityGroups: ec2.SecurityGroupIds(groupIds...)}
resp, err := awsec2.ModifyInstanceAttribute(inst_id, opts)
if err != nil || !resp.Return {
log.Fatalln("Error removing security group from instance", err)
}
log.Printf("Removed security group %s from instance %s\n", securityGroupID, inst_id)
}
}
// HandleMessage reads the nsq message body and parses it as a github webhook,
// checks out the source for the repository & builds/uploads the binaries.
func (h *Handler) HandleMessage(m *nsq.Message) error {
hook, err := github.ParseHook(m.Body)
if err != nil {
// Errors will most likely occur because not all GH
// hooks are the same format
// we care about those that are pushes to master
logrus.Debugf("Error parsing hook: %v", err)
return nil
}
shortSha := hook.After[0:7]
// checkout the code in a temp dir
temp, err := ioutil.TempDir("", fmt.Sprintf("commit-%s", shortSha))
if err != nil {
return err
}
defer os.RemoveAll(temp)
if err := checkout(temp, hook.Repo.Url, hook.After); err != nil {
logrus.Warn(err)
return err
}
logrus.Debugf("Checked out %s for %s", hook.After, hook.Repo.Url)
var (
image = fmt.Sprintf("docker:commit-%s", shortSha)
container = fmt.Sprintf("build-%s", shortSha)
)
logrus.Infof("image=%s container=%s\n", image, container)
// build the image
if err := build(temp, image); err != nil {
logrus.Warn(err)
return err
}
logrus.Debugf("Successfully built image %s", image)
// make the binary
defer removeContainer(container)
if err = makeBinary(temp, image, container, 20*time.Minute); err != nil {
logrus.Warn(err)
return err
}
logrus.Debugf("Successfully built binaries for %s", hook.After)
// read the version
version, err := getBinaryVersion(temp)
if err != nil {
logrus.Warnf("Getting binary version failed: %v", err)
return err
}
bundlesPath := path.Join(temp, "bundles", version, "cross")
// create commit file
if err := ioutil.WriteFile(path.Join(bundlesPath, "commit"), []byte(hook.After), 0755); err != nil {
return err
}
// create version file
if err := ioutil.WriteFile(path.Join(bundlesPath, "version"), []byte(version), 0755); err != nil {
return err
}
// use env variables to connect to s3
auth, err := aws.EnvAuth()
if err != nil {
return fmt.Errorf("AWS Auth failed: %v", err)
}
// connect to s3 bucket
s := s3.New(auth, aws.GetRegion(region))
bucketname, bucketpath := bucketParts(bucket)
bucket := s.Bucket(bucketname)
// push to s3
if err = pushToS3(bucket, bucketpath, bundlesPath); err != nil {
logrus.Warn(err)
return err
}
// add html to template
if err := createIndexFile(bucket, bucketpath); err != nil {
logrus.Warn(err)
return err
}
return nil
}
func main() {
flag.Parse()
dockerClient, err := docker.NewClient(*dockerEndpoint)
if err != nil {
fmt.Printf("cannot connect to docker: %s\n", err)
os.Exit(1)
}
// Wait for the container to start
// TODO(ross): this could be done with the events interface perhaps?
for {
_, err := dockerClient.InspectContainer(*containerID)
if err == nil {
break
}
if _, ok := err.(*docker.NoSuchContainer); ok {
time.Sleep(time.Second)
continue
}
fmt.Printf("docker: %s: %s\n", *containerID, err)
os.Exit(1)
}
instanceID := aws.InstanceId()
if instanceID == "unknown" {
fmt.Printf("cannot determine AWS instance ID. not running in EC2?\n")
os.Exit(1)
}
awsAuth, err := aws.GetAuth("", "", "", time.Time{})
if err != nil {
fmt.Printf("cannot get AWS auth: %s\n", err)
os.Exit(1)
}
elbConn := elb.New(awsAuth, aws.GetRegion(aws.InstanceRegion()))
_, err = elbConn.RegisterInstancesWithLoadBalancer(
[]string{instanceID},
*loadBalancerName)
if err != nil {
fmt.Printf("cannot register instance: %s\n", err)
os.Exit(1)
}
fmt.Printf("registered instance %s from ELB %s\n", instanceID,
*loadBalancerName)
// Wait for the container to exit
_, err = dockerClient.WaitContainer(*containerID)
if err != nil {
fmt.Printf("docker: %s: waiting: %s\n", *containerID, err)
} else {
fmt.Printf("docker: %s exited\n", *containerID)
}
_, err = elbConn.DeregisterInstancesFromLoadBalancer(
[]string{instanceID}, *loadBalancerName)
if err != nil {
fmt.Printf("cannot unregister instance: %s\n", err)
os.Exit(1)
}
fmt.Printf("unregisterd instance %s from ELB %s\n",
instanceID, *loadBalancerName)
}