func main() { flag.Parse() var err error rs, err = rackspace.NewClient() if err != nil { log.Fatal("Could not create new rackspace client: %v\n", err) } serverId := *dcs0ServerId if serverId == "" { serverId = rs.ServerFromSnapshot("dcs-20g-systemd+dcs+psql", rackspace.CreateServerRequest{ Name: "NEW-dcs-0", // This is a 4 GB standard instance FlavorRef: "5", }) } log.Printf("-dcs0_server_id=%s\n", serverId) server, err := rs.GetServer(serverId) if err != nil { log.Fatal(err) } server = server.BecomeActiveOrDie(2 * time.Hour) // Attach an SSD block storage volume (unless one was specified) volumeId := *dcs0VolumeId if volumeId == "" { volumeId, err = rs.CreateBlockStorage( rackspace.CreateBlockStorageRequest{ DisplayName: "NEW-dcs-src-0", Size: 220, VolumeType: "SSD", }) if err != nil { log.Fatal(err) } } // Attach an SSD block storage volume (unless one was specified) volumeMirrorId := *dcs0VolumeMirrorId if volumeMirrorId == "" { volumeMirrorId, err = rs.CreateBlockStorage( rackspace.CreateBlockStorageRequest{ DisplayName: "NEW-dcs-mirror-0", // 100 GB is the minimum size. Last time we used 43 GB. Size: 100, VolumeType: "SSD", }) if err != nil { log.Fatal(err) } } log.Printf("-dcs0_volume_id_dcs=%s\n", volumeId) log.Printf("-dcs0_volume_id_mirror=%s\n", volumeMirrorId) // We chose xvd[gh] so that it does not clash with xvda, xvdb and // whichever new devices a Rackspace base image might use in the // future :). We use a predictable path because it is easier than // figuring out the path in the subsequent automation. attachBlockStorageVolume(serverId, volumeId, "/dev/xvdg") attachBlockStorageVolume(serverId, volumeMirrorId, "/dev/xvdh") client, err := sshutil.Connect(server.AccessIPv6) if err != nil { log.Fatal(err) } // The /dcs/NEW/index.*.idx files are the artifact of the first stage, so // if they are present, skip the first stage entirely. if !client.Successful(`[ -s /dcs/NEW/index.0.idx ]`) { log.Printf("/dcs/NEW/index.0.idx not present, creating new index.\n") // Partition the remaining capacity on the server (≈ 160G) as on-disk // temporary space, since the temporary files exceed the 18G available in // /tmp on our base image. client = mountBlockStorage(client, "/dev/xvda2", "/tmp-disk") // Attach the SSD Block Storage volumes. client = mountBlockStorage(client, "/dev/xvdg1", "/dcs") client = mountBlockStorage(client, "/dev/xvdh1", "/dcs/source-mirror") client.RunOrDie("chown -R dcs.dcs /dcs/source-mirror") client.RunOrDie("chown -R dcs.dcs /dcs/") client.RunOrDie(`chmod 1777 /tmp-disk`) // TODO: timestamps after each step in update-index.sh would be nice client.WriteToFileOrDie("/dcs/update-index.sh", []byte(`#!/bin/sh # Updates the source mirror, generates a new index, verifies it is serving and # then swaps the old index with the new one. # # In case anything goes wrong, you can manually swap back the old index, see # swap-index.sh set -e /bin/rm -rf /dcs/NEW /dcs/OLD /dcs/unpacked-new /bin/mkdir /dcs/NEW /dcs/OLD [ -d ~/.gnupg ] || mkdir ~/.gnupg [ -e ~/.gnupg/trustedkeys.gpg ] || cp /usr/share/keyrings/debian-archive-keyring.gpg ~/.gnupg/trustedkeys.gpg GOMAXPROCS=2 /usr/bin/dcs-debmirror -tcp_conns=20 >/tmp/fdm.log 2>&1 /usr/bin/debmirror --diff=none --method=http --rsync-extra=none -a none --source -s main -h http.debian.net -r /debian /dcs/source-mirror >/dev/null /usr/bin/debmirror --diff=none --method=http --rsync-extra=none --exclude-deb-section=.* --include golang-mode --nocleanup -a none --arch amd64 -s main -h http.debian.net -r /debian /dcs/source-mirror >/dev/null POPCONDUMP=$(mktemp) if ! wget -q http://udd.debian.org/udd-popcon.sql.xz -O $POPCONDUMP then wget -q http://public-udd-mirror.xvm.mit.edu/snapshots/udd-popcon.sql.xz -O $POPCONDUMP fi echo 'DROP TABLE popcon; DROP TABLE popcon_src;' | psql udd xz -d -c $POPCONDUMP | psql udd rm $POPCONDUMP /usr/bin/compute-ranking \ -mirror_path=/dcs/source-mirror /usr/bin/dcs-unpack \ -mirror_path=/dcs/source-mirror \ -new_unpacked_path=/dcs/unpacked-new \ -old_unpacked_path=/dcs/unpacked >/dev/null /usr/bin/dcs-index \ -index_shard_path=/dcs/NEW/ \ -unpacked_path=/dcs/unpacked-new/ \ -shards 6 >/dev/null [ -d /dcs/unpacked ] && mv /dcs/unpacked /dcs/unpacked-old || true mv /dcs/unpacked-new /dcs/unpacked `)) client.RunOrDie(`chmod +x /dcs/update-index.sh`) client.RunOrDie(`TMPDIR=/tmp-disk nohup su dcs -c "/bin/sh -c 'sh -x /dcs/update-index.sh >/tmp/update.log 2>&1 &'"`) // TODO: i also think we need some sort of lock here. perhaps let systemd run the updater? } log.Printf("Now waiting until /dcs/NEW/index.*.idx appear and are > 0 bytes\n") start := time.Now() errors := 0 for time.Since(start) < 24*time.Hour { pollclient, err := sshutil.Connect(server.AccessIPv6) if err != nil { log.Printf("Non-fatal polling connection error: %v\n", err) errors++ if errors > 30 { log.Fatal("More than 30 connection errors connecting to %s, giving up.\n", server.AccessIPv6) } continue } // TODO: flag for the number of shards shardsFound := `[ $(find /dcs/NEW/ -iname "index.*.idx" -size +0 -mmin +15 | wc -l) -eq 6 ]` if pollclient.Successful(shardsFound) { log.Printf("All shards present.\n") break } time.Sleep(15 * time.Minute) } var indexServerIds []string indexServerIds = strings.Split(*dcsIndexServerIds, ",") if *dcsIndexServerIds == "" { // TODO: flag for the number of shards/servers indexServerIds = make([]string, 6) for i := 0; i < len(indexServerIds); i++ { indexServerIds[i] = rs.ServerFromSnapshot("dcs-20g-systemd+dcs", rackspace.CreateServerRequest{ Name: fmt.Sprintf("NEW-dcs-index-%d", i), // This is a 2 GB standard instance FlavorRef: "4", }) } } log.Printf("-index_server_ids=%s\n", strings.Join(indexServerIds, ",")) done := make(chan bool) indexServers := make([]rackspace.Server, len(indexServerIds)) for i, _ := range indexServers { go func(i int) { server, err := rs.GetServer(indexServerIds[i]) if err != nil { log.Fatal(err) } indexServers[i] = server.BecomeActiveOrDie(2 * time.Hour) done <- true }(i) } log.Printf("Waiting for all %d index servers to be available…\n", len(indexServerIds)) for _ = range indexServers { <-done } log.Printf("Index servers available. Copying index…") pubkey, err := ioutil.ReadFile("/home/michael/.ssh/dcs-auto-rs") if err != nil { log.Fatal(err) } client.WriteToFileOrDie("~/.ssh/dcs-auto-rs", pubkey) client.RunOrDie("chmod 600 ~/.ssh/dcs-auto-rs") for i, server := range indexServers { go func(i int, server rackspace.Server) { // Create /dcs/ indexclient, err := sshutil.Connect(server.AccessIPv6) if err != nil { log.Fatal("Failed to dial: " + err.Error()) } indexclient.RunOrDie("mkdir -p /dcs/") if indexclient.Successful(fmt.Sprintf("[ -e /dcs/index.%d.idx ]", i)) { log.Printf("Index already present, skipping.\n") done <- true return } // “|| true” instead of “rm -f” because globbing fails when there are // no matching files. indexclient.RunOrDie("rm /dcs/index.*.idx || true") client.RunOrDie( fmt.Sprintf("scp -o StrictHostKeyChecking=no -i ~/.ssh/dcs-auto-rs /dcs/NEW/index.%d.idx root@%s:/dcs/", i, server.PrivateIPv4())) indexclient.RunOrDie(fmt.Sprintf("systemctl restart dcs-index-backend@%d.service", i)) indexclient.RunOrDie(fmt.Sprintf("systemctl enable dcs-index-backend@%d.service", i)) done <- true }(i, server) } log.Printf("Waiting for the index to be copied to all %d index servers…\n", len(indexServerIds)) for _ = range indexServers { <-done } log.Printf("index copied!\n") backends := []string{} for i, server := range indexServers { backends = append(backends, fmt.Sprintf("%s:%d", server.PrivateIPv4(), 29080+i)) } // TODO(longterm): configure firewall? client.RunOrDie("mkdir -p /etc/systemd/system/dcs-web.service.d/") client.WriteToFileOrDie( "/etc/systemd/system/dcs-web.service.d/backends.conf", []byte(`[Service] Environment=GOMAXPROCS=2 ExecStart= ExecStart=/usr/bin/dcs-web \ -template_pattern=/usr/share/dcs/templates/* \ -listen_address=`+server.PrivateIPv4()+`:28080 \ -use_sources_debian_net=true \ -index_backends=`+strings.Join(backends, ","))) client.RunOrDie("systemctl daemon-reload") client.RunOrDie("systemctl enable dcs-web.service") client.RunOrDie("systemctl enable dcs-source-backend.service") client.RunOrDie("systemctl restart dcs-source-backend.service") client.RunOrDie("systemctl restart dcs-web.service") // Install and configure nginx. client.RunOrDie("DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true LC_ALL=C LANGUAGE=C LANG=C apt-get update") client.RunOrDie("DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true LC_ALL=C LANGUAGE=C LANG=C apt-get --force-yes -y install nginx") client.RunOrDie("rm /etc/nginx/sites-enabled/*") client.RunOrDie("mkdir -p /var/cache/nginx/cache") client.RunOrDie("mkdir -p /var/cache/nginx/tmp") client.RunOrDie("chown -R www-data.www-data /var/cache/nginx/") nginxHost, err := ioutil.ReadFile("/home/michael/gocode/src/github.com/Debian/dcs/nginx.example") if err != nil { log.Fatal(err) } // dcs-web is listening on the Rackspace ServiceNet (private) IP address. nginxReplaced := strings.Replace(string(nginxHost), "localhost:28080", server.PrivateIPv4()+":28080", -1) client.WriteToFileOrDie("/etc/nginx/sites-available/codesearch", []byte(nginxReplaced)) client.RunOrDie("ln -s /etc/nginx/sites-available/codesearch /etc/nginx/sites-enabled/codesearch") client.RunOrDie("systemctl restart nginx.service") // Update DNS domainId, err := rs.GetDomainId("rackspace.zekjur.net") if err != nil { log.Fatal(err) } records, err := rs.GetDomainRecords(domainId) if err != nil { log.Fatal(err) } var updates []rackspace.Record for _, record := range records { if record.Name == "codesearch.rackspace.zekjur.net" { log.Printf("record %v\n", record) newIp := server.AccessIPv4 if record.Type == "AAAA" { newIp = server.AccessIPv6 } updates = append(updates, rackspace.Record{ Id: record.Id, Name: record.Name, Data: newIp, }) } else if record.Name == "int-dcs-web.rackspace.zekjur.net" { // This record points to the private IPv4 address, used by our // monitoring. log.Printf("record %v\n", record) newIp := server.PrivateIPv4() updates = append(updates, rackspace.Record{ Id: record.Id, Name: record.Name, Data: newIp, }) } else if record.Name == "int-dcs-source-backend.rackspace.zekjur.net" { // This record points to the private IPv4 address, used by our // monitoring. log.Printf("record %v\n", record) newIp := server.PrivateIPv4() updates = append(updates, rackspace.Record{ Id: record.Id, Name: record.Name, Data: newIp, }) } } if err := rs.UpdateRecords(domainId, updates); err != nil { log.Fatal(err) } // TODO: reverse dns for the server log.Printf(` codesearch was deployed to: http://codesearch.rackspace.zekjur.net/ http://[%s]/ http://%s/ `, server.AccessIPv6, server.AccessIPv4) }
func main() { flag.Parse() var err error rs, err = rackspace.NewClient() if err != nil { log.Fatal("Could not create new rackspace client: %v\n", err) } servers, err := rs.GetServers() if err != nil { log.Fatal(err) } // We look for servers called NEW-dcs-0 to figure out the newest. var ( newestId string newestCreation time.Time ) for _, server := range servers { if server.Name == "NEW-dcs-0" && server.Created().After(newestCreation) { newestId = server.Id newestCreation = server.Created() } } log.Printf("Newest NEW-dcs-0 server is %s (created at %v)\n", newestId, newestCreation) for _, server := range servers { if strings.HasPrefix(server.Name, "NEW-dcs-") && server.Created().Before(newestCreation) { log.Printf("Deleting server %s (created %v, IPv4 %s, ID %s)\n", server.Name, server.Created(), server.AccessIPv4, server.Id) if !*dryrun { if err := rs.DeleteServer(server.Id); err != nil { log.Fatal(err) } } } } // TODO: wait until all servers have status == deleted // cleanup unused volumes volumes, err := rs.GetVolumes() if err != nil { log.Fatal(err) } for _, volume := range volumes { if strings.HasPrefix(volume.DisplayName, "NEW-dcs-") && volume.Status() == "AVAILABLE" { log.Printf("Deleting unused codesearch volume %s\n", volume.Id) if !*dryrun { if err := rs.DeleteVolume(volume.Id); err != nil { log.Fatal(err) } } } } }