func killSession(session *gexec.Session) {
if runtime.GOOS == "windows" {
session.Kill().Wait()
} else {
session.Terminate().Wait()
}
}
func TestLats(t *testing.T) {
RegisterFailHandler(Fail)
var metronSession *gexec.Session
config = latsConfig.Load()
BeforeSuite(func() {
config.SaveMetronConfig()
helpers.Initialize(config)
metronSession = setupMetron()
})
AfterSuite(func() {
metronSession.Kill().Wait()
})
RunSpecs(t, "Lats Suite")
}
)
var _ = Describe("CfExampleBroker", func() {
var (
session *gexec.Session
)
BeforeEach(func() {
args := []string{}
session = execBin(args...)
time.Sleep(1 * time.Second)
})
AfterEach(func() {
session.Kill()
})
provisionRequest := func(provisionDetails brokerapi.ProvisionDetails) (*http.Response, error) {
client := &http.Client{}
path := fmt.Sprintf("http://localhost:3000/v2/service_instances/%s", provisionDetails.ID)
buffer := &bytes.Buffer{}
json.NewEncoder(buffer).Encode(provisionDetails)
request, err := http.NewRequest("PUT", path, buffer)
Expect(err).NotTo(HaveOccurred())
request.Header.Add("Content-Type", "application/json")
request.SetBasicAuth("username", "password")
}
})
Context("when connecting over TCP", func() {
var (
syslogDrainAddress = fmt.Sprintf("%s:%d", localIPAddress, syslogPort)
drainSession *gexec.Session
)
Context("when forwarding to an unencrypted syslog:// endpoint", func() {
BeforeEach(func() {
drainSession = StartUnencryptedTCPServer(pathToTCPEchoServer, syslogDrainAddress)
})
AfterEach(func() {
drainSession.Kill().Wait()
})
It("forwards log messages to a syslog", func() {
syslogDrainURL := "syslog://" + syslogDrainAddress
key := DrainKey(appID, syslogDrainURL)
AddETCDNode(etcdAdapter, key, syslogDrainURL)
Eventually(func() *gbytes.Buffer {
SendAppLog(appID, "syslog-message", inputConnection)
return drainSession.Out
}, 20, 1).Should(gbytes.Say("syslog-message"))
})
It("reconnects to a reappearing syslog server after an unexpected close", func() {
etcdPort = 5800 + (config.GinkgoConfig.ParallelNode-1)*10
etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, nil)
etcdRunner.Start()
})
var _ = BeforeEach(func() {
etcdRunner.Reset()
var err error
command := exec.Command(pathToMetronExecutable, "--config=fixtures/metron.json", "--debug")
metronSession, err = gexec.Start(command, gexec.NewPrefixedWriter("[o][metron]", GinkgoWriter), gexec.NewPrefixedWriter("[e][metron]", GinkgoWriter))
Expect(err).ShouldNot(HaveOccurred())
})
var _ = AfterEach(func() {
metronSession.Kill().Wait()
})
var _ = AfterSuite(func() {
gexec.CleanupBuildArtifacts()
etcdRunner.Adapter(nil).Disconnect()
etcdRunner.Stop()
})
func basicValueMessage() []byte {
message, _ := proto.Marshal(basicValueMessageEnvelope())
return message
}
func basicValueMessageEnvelope() *events.Envelope {
func startComponent(path string, shortName string, colorCode uint64, arg ...string) *gexec.Session {
var session *gexec.Session
var err error
startCommand := exec.Command(path, arg...)
session, err = gexec.Start(
startCommand,
gexec.NewPrefixedWriter(fmt.Sprintf("\x1b[32m[o]\x1b[%dm[%s]\x1b[0m ", colorCode, shortName), GinkgoWriter),
gexec.NewPrefixedWriter(fmt.Sprintf("\x1b[91m[e]\x1b[%dm[%s]\x1b[0m ", colorCode, shortName), GinkgoWriter))
Expect(err).ShouldNot(HaveOccurred())
return session
}
func waitOnURL(url string) {
Eventually(func() error {
_, err := http.Get(url)
return err
}, 3).ShouldNot(HaveOccurred())
}
var _ = AfterEach(func() {
metronSession.Kill().Wait()
dopplerSession.Kill().Wait()
tcSession.Kill().Wait()
})
var _ = AfterSuite(func() {
etcdAdapter.Disconnect()
etcdRunner.Stop()
gexec.CleanupBuildArtifacts()
})
)
BeforeEach(func() {
command := exec.Command(pathToDopplerExec, "--config=fixtures/doppler_without_tls.json")
dopplerSessionWithoutTLS, err = gexec.Start(command, GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())
Eventually(dopplerSessionWithoutTLS.Out, 3).Should(gbytes.Say("Startup: doppler server started"))
Eventually(func() error {
_, err := etcdAdapter.Get("/doppler/meta/z1/doppler_z1/1")
return err
}, time.Second+config.HeartbeatInterval).ShouldNot(HaveOccurred())
})
AfterEach(func() {
dopplerSessionWithoutTLS.Kill().Wait()
})
It("advertises udp, tcp, and ws endpoints", func() {
node, err := etcdAdapter.Get("doppler/meta/z1/doppler_z1/1")
Expect(err).ToNot(HaveOccurred())
expectedJSON := fmt.Sprintf(
`{"version": 1, "endpoints":["udp://%[1]s:47654", "tcp://%[1]s:43210", "ws://%[1]s:45678"]}`,
localIPAddress)
Expect(node.Value).To(MatchJSON(expectedJSON))
})
})
})
handleError(err)
}()
time.Sleep(500 * time.Millisecond)
reader := bufio.NewReader(remoteClient)
defer remoteClient.Close()
natsClient.Publish("testSubject", []byte("test message"))
time.Sleep(1 * time.Second)
logLine, _, err := reader.ReadLine()
handleError(err)
Expect(string(logLine)).To(MatchRegexp(`^<6>.*nats-to-syslog.*{"Data":"test message","Reply":"","Subject":"testSubject"}`))
})
AfterEach(func() {
gnatsd.Kill()
syslogServer.Close()
})
})
func startSyslogServer() *net.TCPListener {
listener, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP("0.0.0.0"), Port: 6789})
handleError(err)
return listener
}
func startGNATSd() *gexec.Session {
cmd := exec.Command("gnatsd", "--port", "4567", "--user", "nats", "--pass", "c1oudc0w", "-D", "-V")
deaAgentSession, err = gexec.Start(deaagentCommand, GinkgoWriter, GinkgoWriter)
Expect(err).ShouldNot(HaveOccurred())
newFile, err = os.OpenFile("fixtures/instances.json", os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
println(err.Error())
}
etcdPort = 5800 + (config.GinkgoConfig.ParallelNode-1)*10
etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1)
etcdRunner.Start()
})
var _ = AfterSuite(func() {
task1InputListener.Close()
task1StderrListener.Close()
deaAgentSession.Kill().Wait(5)
gexec.CleanupBuildArtifacts()
etcdRunner.Adapter().Disconnect()
etcdRunner.Stop()
os.Remove(instancesJsonPath)
os.RemoveAll("fixtures/jobs")
})
var _ = Describe("DeaLoggingAgent integration tests", func() {
Context("It sends messages from the input file out to metron", func() {
It("gets sent messages and internally emitted metrics", func(done Done) {
m := &messageHolder{}
conn := makeDeaLoggingAgentOutputConn()
defer conn.Close()
goRoutineSpawned.Add(1)
err error
)
BeforeEach(func() {
clientCommand := exec.Command(pathToGoStatsdClient, "51162")
clientInput, err = clientCommand.StdinPipe()
Expect(err).NotTo(HaveOccurred())
clientSession, err = gexec.Start(clientCommand, GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())
})
AfterEach(func() {
clientInput.Close()
clientSession.Kill().Wait()
})
It("forwards gauges as signed value metric messages", func(done Done) {
defer close(done)
clientInput.Write([]byte("gauge test.gauge 23\n"))
Eventually(statsdInjectorSession).Should(gbytes.Say("StatsdListener: Read "))
expected := basicValueMetric("test.gauge", 23, "gauge")
Eventually(getValueMetric).Should(Equal(expected))
clientInput.Write([]byte("gaugedelta test.gauge 7\n"))
Eventually(statsdInjectorSession).Should(gbytes.Say("StatsdListener: Read "))
expected = basicValueMetric("test.gauge", 30, "gauge")
Eventually(getValueMetric).Should(Equal(expected))
pathToDopplerExec, err = gexec.Build("doppler")
Expect(err).NotTo(HaveOccurred())
})
var _ = BeforeEach(func() {
var err error
etcdRunner.Reset()
command := exec.Command(pathToDopplerExec, "--config=fixtures/doppler.json")
dopplerSession, err = gexec.Start(command, GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())
Eventually(dopplerSession.Out, 3).Should(gbytes.Say("Startup: doppler server started"))
localIPAddress, _ = localip.LocalIP()
Eventually(func() error {
_, err := etcdAdapter.Get("healthstatus/doppler/z1/doppler_z1/0")
return err
}, time.Second+config.HeartbeatInterval).ShouldNot(HaveOccurred())
})
var _ = AfterEach(func() {
dopplerSession.Kill().Wait()
})
var _ = AfterSuite(func() {
etcdAdapter.Disconnect()
etcdRunner.Stop()
gexec.CleanupBuildArtifacts()
})
trafficControllerSession, err = gexec.Start(trafficControllerCommand, GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())
// wait for TC
trafficControllerDropsondeEndpoint := fmt.Sprintf("http://%s:%d", localIPAddress, 4566)
Eventually(func() error {
resp, err := http.Get(trafficControllerDropsondeEndpoint)
if err == nil {
resp.Body.Close()
}
return err
}).Should(Succeed())
})
var _ = AfterEach(func() {
trafficControllerSession.Kill().Wait()
})
var _ = AfterSuite(func() {
gnatsdSession.Kill().Wait()
gexec.CleanupBuildArtifacts()
etcdRunner.Stop()
})
func setupEtcdAdapter() {
etcdPort = 4001
etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, nil)
etcdRunner.Start()
}
func setupDopplerInEtcd() {
var _ = BeforeSuite(func() {
logging.SetLevel(logging.CRITICAL)
consulPath, err := gexec.Build("github.com/hashicorp/consul")
Expect(err).ToNot(HaveOccurred())
tmpDir, err = ioutil.TempDir("", "consul")
Expect(err).ToNot(HaveOccurred())
consulCmd := exec.Command(consulPath, "agent", "-server", "-bootstrap-expect", "1", "-data-dir", tmpDir, "-bind", "127.0.0.1")
consulSession, err = gexec.Start(consulCmd, nil, nil)
Expect(err).ToNot(HaveOccurred())
Consistently(consulSession).ShouldNot(gexec.Exit())
consulClient, err = api.NewClient(api.DefaultConfig())
Expect(err).ToNot(HaveOccurred())
f := func() error {
_, _, err := consulClient.Catalog().Nodes(nil)
return err
}
Eventually(f, 10).Should(BeNil())
})
var _ = AfterSuite(func() {
consulSession.Kill()
consulSession.Wait("60s", "200ms")
Expect(os.RemoveAll(tmpDir)).To(Succeed())
gexec.CleanupBuildArtifacts()
})
fakeUAA = &http.Server{
Addr: ":8084",
Handler: http.HandlerFunc(fakeUAAHandler),
}
fakeFirehose = &http.Server{
Addr: ":8086",
Handler: http.HandlerFunc(fakeFirehoseHandler),
}
go fakeUAA.ListenAndServe()
go fakeFirehose.ListenAndServe()
})
AfterEach(func() {
nozzleSession.Kill().Wait()
})
Context("with an HTTP OpenTSDB endpoint", func() {
BeforeEach(func() {
fakeOpentsdb = &http.Server{
Addr: ":8087",
Handler: http.HandlerFunc(fakeOpentsdbHandler),
}
go fakeOpentsdb.ListenAndServe()
var err error
nozzleCommand := exec.Command(pathToNozzleExecutable, "-config", "fixtures/http-test-config.json")
nozzleSession, err = gexec.Start(
nozzleCommand,
gexec.NewPrefixedWriter("[o][nozzle] ", GinkgoWriter),
func itWorksWithBranch(branchName string) {
Context("when the branch name is "+branchName, func() {
var gitRepo string
var bareGitRepo string
var sourceDir string
var outResponse out.OutResponse
var outRequest out.OutRequest
BeforeEach(func() {
var err error
gitRepo, err = ioutil.TempDir("", "git-repo")
Ω(err).ShouldNot(HaveOccurred())
bareGitRepo, err = ioutil.TempDir("", "bare-git-repo")
Ω(err).ShouldNot(HaveOccurred())
sourceDir, err = ioutil.TempDir("", "source-dir")
Ω(err).ShouldNot(HaveOccurred())
setupGitRepo(gitRepo)
bareGitSetup := exec.Command("bash", "-e", "-c", fmt.Sprintf(`
git clone %s --bare .
`, gitRepo))
bareGitSetup.Dir = bareGitRepo
err = bareGitSetup.Run()
Ω(err).ShouldNot(HaveOccurred())
})
AfterEach(func() {
err := os.RemoveAll(bareGitRepo)
Ω(err).ShouldNot(HaveOccurred())
err = os.RemoveAll(gitRepo)
Ω(err).ShouldNot(HaveOccurred())
err = os.RemoveAll(sourceDir)
Ω(err).ShouldNot(HaveOccurred())
})
Context("when the config is incomplete", func() {
var session *gexec.Session
BeforeEach(func() {
outRequest = out.OutRequest{
Source: out.Source{
URI: bareGitRepo,
Branch: branchName,
Pool: "lock-pool",
RetryDelay: 100 * time.Millisecond,
},
Params: out.OutParams{
Acquire: true,
},
}
})
JustBeforeEach(func() {
session = runOut(outRequest, sourceDir)
Eventually(session).Should(gexec.Exit(1))
})
Context("when the uri isn't set", func() {
BeforeEach(func() {
outRequest.Source.URI = ""
})
It("complains about it", func() {
errorMessages := string(session.Err.Contents())
Ω(errorMessages).Should(ContainSubstring("invalid payload (missing uri)"))
})
})
Context("when the pool isn't set", func() {
BeforeEach(func() {
outRequest.Source.Pool = ""
})
It("complains about it", func() {
errorMessages := string(session.Err.Contents())
Ω(errorMessages).Should(ContainSubstring("invalid payload (missing pool)"))
})
})
Context("when the branch isn't set", func() {
BeforeEach(func() {
outRequest.Source.Branch = ""
})
It("complains about it", func() {
errorMessages := string(session.Err.Contents())
Ω(errorMessages).Should(ContainSubstring("invalid payload (missing branch)"))
})
})
Context("when the params aren't set", func() {
BeforeEach(func() {
outRequest.Params = out.OutParams{}
})
It("complains about it", func() {
errorMessages := string(session.Err.Contents())
Ω(errorMessages).Should(ContainSubstring("invalid payload (missing acquire, release, remove, claim, add, or add_claimed)"))
})
})
})
Context("when acquiring a lock", func() {
BeforeEach(func() {
outRequest = out.OutRequest{
Source: out.Source{
URI: bareGitRepo,
Branch: branchName,
Pool: "lock-pool",
RetryDelay: 100 * time.Millisecond,
},
Params: out.OutParams{
Acquire: true,
},
}
session := runOut(outRequest, sourceDir)
Eventually(session).Should(gexec.Exit(0))
err := json.Unmarshal(session.Out.Contents(), &outResponse)
Ω(err).ShouldNot(HaveOccurred())
})
It("moves a lock to claimed", func() {
version := getVersion(bareGitRepo, "origin/"+branchName)
reCloneRepo, err := ioutil.TempDir("", "git-version-repo")
Ω(err).ShouldNot(HaveOccurred())
defer os.RemoveAll(reCloneRepo)
reClone := exec.Command("git", "clone", "--branch", branchName, bareGitRepo, ".")
reClone.Dir = reCloneRepo
err = reClone.Run()
Ω(err).ShouldNot(HaveOccurred())
claimedFiles, err := ioutil.ReadDir(filepath.Join(reCloneRepo, "lock-pool", "claimed"))
Ω(err).ShouldNot(HaveOccurred())
Ω(len(claimedFiles)).Should(Equal(2))
var lockFile string
for _, file := range claimedFiles {
filename := filepath.Base(file.Name())
if filename != ".gitkeep" {
lockFile = filename
}
}
Ω(outResponse).Should(Equal(out.OutResponse{
Version: version,
Metadata: []out.MetadataPair{
{Name: "lock_name", Value: lockFile},
{Name: "pool_name", Value: "lock-pool"},
},
}))
})
It("commits with a descriptive message", func() {
log := exec.Command("git", "log", "--oneline", "-1", outResponse.Version.Ref)
log.Dir = bareGitRepo
session, err := gexec.Start(log, GinkgoWriter, GinkgoWriter)
Ω(err).ShouldNot(HaveOccurred())
<-session.Exited
Ω(session).Should(gbytes.Say("pipeline-name/job-name #42 claiming: " + outResponse.Metadata[0].Value))
})
})
Context("when there are no locks to be claimed", func() {
var session *gexec.Session
var claimAllLocksDir string
BeforeEach(func() {
var err error
outRequest = out.OutRequest{
Source: out.Source{
URI: bareGitRepo,
Branch: branchName,
Pool: "lock-pool",
RetryDelay: 1 * time.Second,
},
Params: out.OutParams{
Acquire: true,
},
}
claimAllLocksDir, err = ioutil.TempDir("", "claiming-locks")
Ω(err).ShouldNot(HaveOccurred())
claimAllLocks := exec.Command("bash", "-e", "-c", fmt.Sprintf(`
git clone --branch %s %s .
git config user.email "ginkgo@localhost"
git config user.name "Ginkgo Local"
git mv lock-pool/unclaimed/* lock-pool/claimed/
git commit -am "claiming all locks"
git push
`, branchName, bareGitRepo))
claimAllLocks.Stdout = GinkgoWriter
claimAllLocks.Stderr = GinkgoWriter
claimAllLocks.Dir = claimAllLocksDir
err = claimAllLocks.Run()
Ω(err).ShouldNot(HaveOccurred())
session = runOut(outRequest, sourceDir)
})
AfterEach(func() {
err := os.RemoveAll(claimAllLocksDir)
Ω(err).ShouldNot(HaveOccurred())
})
It("retries until a lock can be claimed", func() {
Consistently(session, 2*time.Second).ShouldNot(gexec.Exit(0))
releaseLock := exec.Command("bash", "-e", "-c", fmt.Sprint(`
git mv lock-pool/claimed/some-lock lock-pool/unclaimed/some-lock
git commit -am "unclaiming some-lock"
git push
`))
releaseLock.Dir = claimAllLocksDir
err := releaseLock.Run()
Ω(err).ShouldNot(HaveOccurred())
Eventually(session, 2*time.Second).Should(gexec.Exit(0))
err = json.Unmarshal(session.Out.Contents(), &outResponse)
Ω(err).ShouldNot(HaveOccurred())
Ω(outResponse).Should(Equal(out.OutResponse{
Version: out.Version{
Ref: outResponse.Version.Ref,
},
Metadata: []out.MetadataPair{
{Name: "lock_name", Value: "some-lock"},
{Name: "pool_name", Value: "lock-pool"},
},
}))
})
})
Context("when claiming a specific lock", func() {
BeforeEach(func() {
outRequest = out.OutRequest{
Source: out.Source{
URI: bareGitRepo,
Branch: branchName,
Pool: "lock-pool",
RetryDelay: 100 * time.Millisecond,
},
Params: out.OutParams{
Claim: "some-lock",
},
}
session := runOut(outRequest, sourceDir)
Eventually(session).Should(gexec.Exit(0))
err := json.Unmarshal(session.Out.Contents(), &outResponse)
Ω(err).ShouldNot(HaveOccurred())
})
It("moves the specific lock to claimed", func() {
version := getVersion(bareGitRepo, "origin/"+branchName)
reCloneRepo, err := ioutil.TempDir("", "git-version-repo")
Ω(err).ShouldNot(HaveOccurred())
defer os.RemoveAll(reCloneRepo)
reClone := exec.Command("git", "clone", "--branch", branchName, bareGitRepo, ".")
reClone.Dir = reCloneRepo
err = reClone.Run()
Ω(err).ShouldNot(HaveOccurred())
_, err = ioutil.ReadFile(filepath.Join(reCloneRepo, "lock-pool", "claimed", "some-lock"))
Ω(err).ShouldNot(HaveOccurred())
Ω(outResponse).Should(Equal(out.OutResponse{
Version: version,
Metadata: []out.MetadataPair{
{Name: "lock_name", Value: "some-lock"},
{Name: "pool_name", Value: "lock-pool"},
},
}))
})
It("commits with a descriptive message", func() {
log := exec.Command("git", "log", "--oneline", "-1", outResponse.Version.Ref)
log.Dir = bareGitRepo
session, err := gexec.Start(log, GinkgoWriter, GinkgoWriter)
Ω(err).ShouldNot(HaveOccurred())
<-session.Exited
Ω(session).Should(gbytes.Say("pipeline-name/job-name #42 claiming: some-lock"))
})
Context("when the specific lock has already been claimed", func() {
var unclaimLockDir string
BeforeEach(func() {
var err error
unclaimLockDir, err = ioutil.TempDir("", "claiming-locks")
Ω(err).ShouldNot(HaveOccurred())
})
AfterEach(func() {
err := os.RemoveAll(unclaimLockDir)
Ω(err).ShouldNot(HaveOccurred())
})
It("continues to acquire the same lock", func() {
claimSession := runOut(outRequest, sourceDir)
Consistently(claimSession).ShouldNot(gexec.Exit(0))
unclaimLock := exec.Command("bash", "-e", "-c", fmt.Sprintf(`
git clone --branch %s %s .
git config user.email "ginkgo@localhost"
git config user.name "Ginkgo Local"
git mv lock-pool/claimed/some-lock lock-pool/unclaimed/
git commit -am "unclaim some-lock"
git push
`, branchName, bareGitRepo))
unclaimLock.Stdout = GinkgoWriter
unclaimLock.Stderr = GinkgoWriter
unclaimLock.Dir = unclaimLockDir
err := unclaimLock.Run()
Ω(err).ShouldNot(HaveOccurred())
Eventually(claimSession).Should(gexec.Exit(0))
})
})
})
Context("when removing a lock", func() {
var myLocksGetDir string
var outRemoveRequest out.OutRequest
var outRemoveResponse out.OutResponse
var session *gexec.Session
BeforeEach(func() {
outRequest = out.OutRequest{
Source: out.Source{
URI: bareGitRepo,
Branch: branchName,
Pool: "lock-pool",
},
Params: out.OutParams{
Acquire: true,
},
}
session = runOut(outRequest, sourceDir)
Eventually(session).Should(gexec.Exit(0))
err := json.Unmarshal(session.Out.Contents(), &outResponse)
Ω(err).ShouldNot(HaveOccurred())
})
AfterEach(func() {
session.Kill()
})
JustBeforeEach(func() {
var err error
myLocksGetDir, err = ioutil.TempDir("", "my-locks")
Ω(err).ShouldNot(HaveOccurred())
jsonIn := fmt.Sprintf(`
{
"source": {
"uri": "%s",
"branch": "%s",
"pool": "lock-pool"
},
"version": {
"ref": "%s"
}
}`, bareGitRepo, branchName, string(outResponse.Version.Ref))
runIn(jsonIn, filepath.Join(myLocksGetDir, "lock-step-name"), 0)
outRemoveRequest = out.OutRequest{
Source: out.Source{
URI: bareGitRepo,
Branch: branchName,
Pool: "lock-pool",
},
Params: out.OutParams{
Remove: "lock-step-name",
},
}
session := runOut(outRemoveRequest, myLocksGetDir)
Eventually(session).Should(gexec.Exit(0))
err = json.Unmarshal(session.Out.Contents(), &outRemoveResponse)
Ω(err).ShouldNot(HaveOccurred())
})
AfterEach(func() {
err := os.RemoveAll(myLocksGetDir)
Ω(err).ShouldNot(HaveOccurred())
})
It("removes the lock from the pool", func() {
version := getVersion(bareGitRepo, "origin/"+branchName)
reCloneRepo, err := ioutil.TempDir("", "git-version-repo")
Ω(err).ShouldNot(HaveOccurred())
defer os.RemoveAll(reCloneRepo)
reClone := exec.Command("git", "clone", "--branch", branchName, bareGitRepo, ".")
reClone.Dir = reCloneRepo
err = reClone.Run()
Ω(err).ShouldNot(HaveOccurred())
claimedFiles, err := ioutil.ReadDir(filepath.Join(reCloneRepo, "lock-pool", "claimed"))
Ω(err).ShouldNot(HaveOccurred())
Ω(len(claimedFiles)).Should(Equal(1))
unclaimedFiles, err := ioutil.ReadDir(filepath.Join(reCloneRepo, "lock-pool", "unclaimed"))
Ω(err).ShouldNot(HaveOccurred())
Ω(len(unclaimedFiles)).Should(Equal(2))
var removedLockName string
for _, metaDataPair := range outResponse.Metadata {
if metaDataPair.Name == "lock_name" {
removedLockName = metaDataPair.Value
}
}
Ω(outRemoveResponse).Should(Equal(out.OutResponse{
Version: version,
Metadata: []out.MetadataPair{
{Name: "lock_name", Value: removedLockName},
{Name: "pool_name", Value: "lock-pool"},
},
}))
})
It("commits with a descriptive message", func() {
log := exec.Command("git", "log", "--oneline", "-1", outRemoveResponse.Version.Ref)
log.Dir = bareGitRepo
session, err := gexec.Start(log, GinkgoWriter, GinkgoWriter)
Ω(err).ShouldNot(HaveOccurred())
<-session.Exited
Ω(session).Should(gbytes.Say("pipeline-name/job-name #42 removing: " + outRemoveResponse.Metadata[0].Value))
})
})
Context("when releasing a lock", func() {
var myLocksGetDir string
var outReleaseRequest out.OutRequest
var outReleaseResponse out.OutResponse
BeforeEach(func() {
outRequest = out.OutRequest{
Source: out.Source{
URI: bareGitRepo,
Branch: branchName,
Pool: "lock-pool",
},
Params: out.OutParams{
Acquire: true,
},
}
session := runOut(outRequest, sourceDir)
Eventually(session).Should(gexec.Exit(0))
err := json.Unmarshal(session.Out.Contents(), &outResponse)
Ω(err).ShouldNot(HaveOccurred())
})
JustBeforeEach(func() {
var err error
myLocksGetDir, err = ioutil.TempDir("", "my-locks")
Ω(err).ShouldNot(HaveOccurred())
jsonIn := fmt.Sprintf(`
{
"source": {
"uri": "%s",
"branch": "%s",
"pool": "lock-pool"
},
"version": {
"ref": "%s"
}
}`, bareGitRepo, branchName, string(outResponse.Version.Ref))
runIn(jsonIn, filepath.Join(myLocksGetDir, "lock-step-name"), 0)
outReleaseRequest = out.OutRequest{
Source: out.Source{
URI: bareGitRepo,
Branch: branchName,
Pool: "lock-pool",
},
Params: out.OutParams{
Release: "lock-step-name",
},
}
session := runOut(outReleaseRequest, myLocksGetDir)
Eventually(session).Should(gexec.Exit(0))
err = json.Unmarshal(session.Out.Contents(), &outReleaseResponse)
Ω(err).ShouldNot(HaveOccurred())
})
AfterEach(func() {
err := os.RemoveAll(myLocksGetDir)
Ω(err).ShouldNot(HaveOccurred())
})
It("moves the lock to unclaimed", func() {
version := getVersion(bareGitRepo, "origin/"+branchName)
reCloneRepo, err := ioutil.TempDir("", "git-version-repo")
Ω(err).ShouldNot(HaveOccurred())
defer os.RemoveAll(reCloneRepo)
reClone := exec.Command("git", "clone", "--branch", branchName, bareGitRepo, ".")
reClone.Dir = reCloneRepo
err = reClone.Run()
Ω(err).ShouldNot(HaveOccurred())
claimedFiles, err := ioutil.ReadDir(filepath.Join(reCloneRepo, "lock-pool", "claimed"))
Ω(err).ShouldNot(HaveOccurred())
Ω(len(claimedFiles)).Should(Equal(1))
unclaimedFiles, err := ioutil.ReadDir(filepath.Join(reCloneRepo, "lock-pool", "unclaimed"))
Ω(err).ShouldNot(HaveOccurred())
Ω(len(unclaimedFiles)).Should(Equal(3))
var releasedLockName string
for _, metaDataPair := range outResponse.Metadata {
if metaDataPair.Name == "lock_name" {
releasedLockName = metaDataPair.Value
}
}
Ω(outReleaseResponse).Should(Equal(out.OutResponse{
Version: version,
Metadata: []out.MetadataPair{
{Name: "lock_name", Value: releasedLockName},
{Name: "pool_name", Value: "lock-pool"},
},
}))
})
It("commits with a descriptive message", func() {
log := exec.Command("git", "log", "--oneline", "-1", outReleaseResponse.Version.Ref)
log.Dir = bareGitRepo
session, err := gexec.Start(log, GinkgoWriter, GinkgoWriter)
Ω(err).ShouldNot(HaveOccurred())
<-session.Exited
Ω(session).Should(gbytes.Say("pipeline-name/job-name #42 unclaiming: " + outReleaseResponse.Metadata[0].Value))
})
})
Context("when adding an initially unclaimed lock to the pool", func() {
var lockToAddDir string
var cloneDir string
BeforeEach(func() {
lockToAddDir, err := ioutil.TempDir("", "lock-to-add")
Ω(err).ShouldNot(HaveOccurred())
cloneDir, err = ioutil.TempDir("", "clone")
Ω(err).ShouldNot(HaveOccurred())
taskDir := filepath.Join(lockToAddDir, "task-name")
err = os.Mkdir(taskDir, 0755)
err = ioutil.WriteFile(filepath.Join(taskDir, "metadata"), []byte("hello"), 0555)
Ω(err).ShouldNot(HaveOccurred())
err = ioutil.WriteFile(filepath.Join(taskDir, "name"), []byte("added-lock-name"), 0555)
Ω(err).ShouldNot(HaveOccurred())
outRequest = out.OutRequest{
Source: out.Source{
URI: bareGitRepo,
Branch: branchName,
Pool: "lock-pool",
RetryDelay: 100 * time.Millisecond,
},
Params: out.OutParams{
Add: "task-name",
},
}
session := runOut(outRequest, lockToAddDir)
Eventually(session).Should(gexec.Exit(0))
err = json.Unmarshal(session.Out.Contents(), &outResponse)
Ω(err).ShouldNot(HaveOccurred())
})
AfterEach(func() {
err := os.RemoveAll(lockToAddDir)
Ω(err).ShouldNot(HaveOccurred())
err = os.RemoveAll(cloneDir)
Ω(err).ShouldNot(HaveOccurred())
})
It("adds the new lock in an unclaimed state", func() {
clone := exec.Command("git", "clone", "--branch", branchName, bareGitRepo, ".")
clone.Dir = cloneDir
err := clone.Run()
Ω(err).ShouldNot(HaveOccurred())
lockPath := filepath.Join(cloneDir, "lock-pool", "unclaimed", "added-lock-name")
Ω(lockPath).Should(BeARegularFile())
contents, err := ioutil.ReadFile(lockPath)
Ω(err).ShouldNot(HaveOccurred())
Ω(string(contents)).Should(Equal("hello"))
})
It("commits with a descriptive message", func() {
log := exec.Command("git", "log", "--oneline", "-1", outResponse.Version.Ref)
log.Dir = bareGitRepo
session, err := gexec.Start(log, GinkgoWriter, GinkgoWriter)
Ω(err).ShouldNot(HaveOccurred())
<-session.Exited
Ω(session).Should(gbytes.Say("pipeline-name/job-name #42 adding unclaimed: " + outResponse.Metadata[0].Value))
})
})
Context("when adding an initially claimed lock to the pool", func() {
var lockToAddDir string
var cloneDir string
BeforeEach(func() {
lockToAddDir, err := ioutil.TempDir("", "lock-to-add")
Ω(err).ShouldNot(HaveOccurred())
cloneDir, err = ioutil.TempDir("", "clone")
Ω(err).ShouldNot(HaveOccurred())
taskDir := filepath.Join(lockToAddDir, "task-name")
err = os.Mkdir(taskDir, 0755)
err = ioutil.WriteFile(filepath.Join(taskDir, "metadata"), []byte("hello"), 0555)
Ω(err).ShouldNot(HaveOccurred())
err = ioutil.WriteFile(filepath.Join(taskDir, "name"), []byte("claimed-lock-name"), 0555)
Ω(err).ShouldNot(HaveOccurred())
outRequest = out.OutRequest{
Source: out.Source{
URI: bareGitRepo,
Branch: branchName,
Pool: "lock-pool",
RetryDelay: 100 * time.Millisecond,
},
Params: out.OutParams{
AddClaimed: "task-name",
},
}
session := runOut(outRequest, lockToAddDir)
Eventually(session).Should(gexec.Exit(0))
err = json.Unmarshal(session.Out.Contents(), &outResponse)
Ω(err).ShouldNot(HaveOccurred())
})
AfterEach(func() {
err := os.RemoveAll(lockToAddDir)
Ω(err).ShouldNot(HaveOccurred())
err = os.RemoveAll(cloneDir)
Ω(err).ShouldNot(HaveOccurred())
})
It("adds the new lock in a claimed state", func() {
clone := exec.Command("git", "clone", "--branch", branchName, bareGitRepo, ".")
clone.Dir = cloneDir
err := clone.Run()
Ω(err).ShouldNot(HaveOccurred())
lockPath := filepath.Join(cloneDir, "lock-pool", "claimed", "claimed-lock-name")
Ω(lockPath).Should(BeARegularFile())
contents, err := ioutil.ReadFile(lockPath)
Ω(err).ShouldNot(HaveOccurred())
Ω(string(contents)).Should(Equal("hello"))
})
It("commits with a descriptive message", func() {
log := exec.Command("git", "log", "--oneline", "-1", outResponse.Version.Ref)
log.Dir = bareGitRepo
session, err := gexec.Start(log, GinkgoWriter, GinkgoWriter)
Ω(err).ShouldNot(HaveOccurred())
<-session.Exited
Ω(session).Should(gbytes.Say("pipeline-name/job-name #42 adding claimed: " + outResponse.Metadata[0].Value))
})
})
Context("when 2 processes are acquiring a lock at the same time", func() {
var sessionOne *gexec.Session
var sessionTwo *gexec.Session
var gitServerSession *gexec.Session
var sessionOneDir string
var sessionTwoDir string
var claimLockDir string
var exitedCounter uint64
BeforeEach(func() {
var err error
sessionOneDir, err = ioutil.TempDir("", "session-one")
Ω(err).ShouldNot(HaveOccurred())
sessionTwoDir, err = ioutil.TempDir("", "session-two")
Ω(err).ShouldNot(HaveOccurred())
claimLockDir, err = ioutil.TempDir("", "claiming-locks")
Ω(err).ShouldNot(HaveOccurred())
gitPort := GinkgoParallelNode() + 9418
gitURI := fmt.Sprintf("git://*****:*****@localhost"
git config user.name "Ginkgo Local"
git mv lock-pool/unclaimed/some-lock lock-pool/claimed/
git commit -am "claiming a lock"
git push
`, branchName, bareGitRepo))
claimOneLock.Stdout = GinkgoWriter
claimOneLock.Stderr = GinkgoWriter
claimOneLock.Dir = claimLockDir
err = claimOneLock.Run()
Ω(err).ShouldNot(HaveOccurred())
})
It("does not acquire the same lock", func() {
Eventually(func() uint64 {
return atomic.LoadUint64(&exitedCounter)
}, 5*time.Second).Should(Equal(uint64(1)))
Consistently(func() uint64 {
return atomic.LoadUint64(&exitedCounter)
}, 2*time.Second).Should(Equal(uint64(1)))
sessionOne.Terminate().Wait()
sessionTwo.Terminate().Wait()
Ω(sessionOne.Err).ShouldNot(gbytes.Say("err"))
Ω(sessionTwo.Err).ShouldNot(gbytes.Say("err"))
})
})
})
})
}
var _ = AfterSuite(func() {
Expect(agoutiDriver.Stop()).To(Succeed())
gexec.CleanupBuildArtifacts()
})
var _ = BeforeEach(func() {
cwd, err := os.Getwd()
Expect(err).NotTo(HaveOccurred())
storeDir := filepath.Join(cwd, "..", "db")
os.Remove(filepath.Join(storeDir, "sqlite", "store.db"))
buildsDir, err = ioutil.TempDir("", "integration-builds")
Expect(err).NotTo(HaveOccurred())
runningExecutable, err = gexec.Start(exec.Command(
executablePath, "-port=3001",
"-templateDir", filepath.Join(cwd, "..", "web", "templates"),
"-storeDir", storeDir,
"-gooseCmd=goose",
"-buildsDir", buildsDir,
"-assetsDir", filepath.Join(cwd, "..", "web", "assets"),
), GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())
})
var _ = AfterEach(func() {
Eventually(runningExecutable.Kill()).Should(gexec.Exit())
Expect(os.RemoveAll(buildsDir)).To(Succeed())
})
port := 9800 + config.GinkgoConfig.ParallelNode
address := net.JoinHostPort("127.0.0.1", strconv.Itoa(port))
outputURL := &url.URL{Scheme: "syslog", Host: address}
syslogServerSession = startSyslogServer(address)
sysLogWriter, _ = syslogwriter.NewSyslogWriter(outputURL, "appId", dialer, 0)
Eventually(func() error {
err := sysLogWriter.Connect()
return err
}, 5, 1).ShouldNot(HaveOccurred())
}, 10)
AfterEach(func() {
sysLogWriter.Close()
syslogServerSession.Kill().Wait()
})
Context("Message Format", func() {
It("sends messages in the proper format", func() {
sysLogWriter.Write(standardOutPriority, []byte("just a test"), "App", "2", time.Now().UnixNano())
Eventually(syslogServerSession, 5).Should(gbytes.Say(`\d <\d+>1 \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{1,6}([-+]\d{2}:\d{2}) loggregator appId \[App/2\] - - just a test\n`))
}, 10)
It("strips null termination char from message", func() {
sysLogWriter.Write(standardOutPriority, []byte(string(0)+" hi"), "appId", "", time.Now().UnixNano())
Expect(syslogServerSession).ToNot(gbytes.Say("\000"))
})
})
Expect(err).NotTo(HaveOccurred())
Eventually(session).Should(gbytes.Say("file-server.ready"))
return session
}
BeforeEach(func() {
servedDirectory, err = ioutil.TempDir("", "file_server-test")
Expect(err).NotTo(HaveOccurred())
port = 8182 + config.GinkgoConfig.ParallelNode
address = fmt.Sprintf("http://localhost:%d", port)
})
AfterEach(func() {
session.Kill().Wait()
os.RemoveAll(servedDirectory)
})
Context("when started without any arguments", func() {
It("should fail", func() {
session, err = gexec.Start(exec.Command(fileServerBinary), GinkgoWriter, GinkgoWriter)
Expect(err).NotTo(HaveOccurred())
Eventually(session).Should(gexec.Exit(2))
})
})
Context("when started correctly", func() {
BeforeEach(func() {
session = start()
ioutil.WriteFile(filepath.Join(servedDirectory, "test"), []byte("hello"), os.ModePerm)
command := exec.Command(pathToMetronExecutable, "--config=fixtures/metron.json", "--debug")
metronSession, err = gexec.Start(command, gexec.NewPrefixedWriter("[o][metron]", GinkgoWriter), gexec.NewPrefixedWriter("[e][metron]", GinkgoWriter))
Expect(err).ShouldNot(HaveOccurred())
pathToStatsdInjectorExecutable, err := gexec.Build("statsd-injector")
Expect(err).NotTo(HaveOccurred())
command = exec.Command(pathToStatsdInjectorExecutable, "-statsdPort=51162", "-metronPort=51161", "-logLevel=debug")
statsdInjectorSession, err = gexec.Start(command, gexec.NewPrefixedWriter("[o][statsdInjector]", GinkgoWriter), gexec.NewPrefixedWriter("[e][statsdInjector]", GinkgoWriter))
Expect(err).ShouldNot(HaveOccurred())
localIPAddress, _ = localip.LocalIP()
etcdPort = 5800 + (config.GinkgoConfig.ParallelNode-1)*10
etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, nil)
etcdRunner.Start()
Eventually(metronSession, 1).Should(gbytes.Say("metron started"))
Eventually(statsdInjectorSession).Should(gbytes.Say("Listening for statsd on host :51162"))
})
var _ = AfterSuite(func() {
metronSession.Kill().Wait()
statsdInjectorSession.Kill().Wait()
gexec.CleanupBuildArtifacts()
etcdRunner.Adapter(nil).Disconnect()
etcdRunner.Stop()
})
handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
close(rxCh)
})
)
BeforeEach(func() {
rxCh = make(chan struct{})
logFinServer = httptest.NewTLSServer(handler)
startLogEmitter(map[string]string{
"LOGFIN_URL": logFinServer.URL,
"PORT": "12345",
"TIME": "1ms",
"SKIP_CERT_VERIFY": "true",
})
})
AfterEach(func() {
session.Kill().Wait(5 * time.Second)
logFinServer.Close()
gexec.CleanupBuildArtifacts()
})
It("utilizes the SKIP_CERT_VERIFY env variable", func() {
Eventually(rxCh, 5).Should(BeClosed())
})
})
})
})