func TestAddToArchiveWithClosedWriter(t *testing.T) {
dir, err := ioutil.TempDir("", "tarwriter_test")
if err != nil {
assert.FailNow(t, "Cannot create temp dir", err.Error())
}
tempFilePath := filepath.Join(dir, "test_file.tar")
defer os.RemoveAll(dir)
w := tarfile.NewWriter(tempFilePath)
// Note that we have not opened the writer
err = w.AddToArchive(pathToTestFile("cleanup_result.json"), "file1.json")
if err == nil {
assert.FailNow(t, "Should have gotten a tar write error")
}
assert.True(t, strings.HasPrefix(err.Error(), "Underlying TarWriter is nil"))
// Open and close the writer, so the file exists.
w.Open()
w.Close()
if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
}
err = w.AddToArchive(pathToTestFile("cleanup_result.json"), "file1.json")
if err == nil {
assert.FailNow(t, "Should have gotten a tar write error")
}
assert.True(t, strings.HasPrefix(err.Error(), "archive/tar: write after close"))
}
func TestS3ObjectDelete(t *testing.T) {
if !canTestS3() {
return
}
// Hmmm... don't like having to upload objects first.
// But how else to test delete?
err := upload(t, "test_obj_1.tar")
if err != nil {
assert.FailNow(t, "Could not upload file 1 for delete test")
}
err = upload(t, "test_obj_2.tar")
if err != nil {
assert.FailNow(t, "Could not upload file 2 for delete test")
}
// Now delete those objects
s3ObjectDelete := network.NewS3ObjectDelete(
constants.AWSVirginia,
testBucket,
[]string{"test_obj_1.tar", "test_obj_2.tar"},
)
s3ObjectDelete.DeleteList()
assert.Equal(t, "", s3ObjectDelete.ErrorMessage)
assert.Equal(t, 2, len(s3ObjectDelete.Response.Deleted))
assert.Empty(t, s3ObjectDelete.Response.Errors)
}
func TestFetchWithoutChecksums(t *testing.T) {
if !canTestS3() {
return
}
download := getS3DownloadObject(t)
if download == nil {
return
}
download.CalculateMd5 = false
download.CalculateSha256 = false
download.Fetch()
defer os.Remove(download.LocalPath)
assert.Empty(t, download.ErrorMessage)
if download.Response == nil {
assert.FailNow(t, "Response object is nil")
}
assert.Equal(t, testFileETag, *download.Response.ETag)
assert.Equal(t, int64(testFileSize), *download.Response.ContentLength)
assert.Equal(t, int64(testFileSize), download.BytesCopied)
fileStat, err := os.Stat(download.LocalPath)
if err != nil {
assert.FailNow(t, "Download file '%s' does not exist", download.LocalPath)
}
assert.Equal(t, int64(testFileSize), fileStat.Size())
assert.Empty(t, download.Md5Digest)
assert.Empty(t, download.Sha256Digest)
}
func expectTaskUpdate(t *testing.T, watch chan events.Event) {
for {
select {
case event := <-watch:
if _, ok := event.(state.EventUpdateTask); !ok {
assert.FailNow(t, "expected task update event, got", fmt.Sprint(event))
}
return
case <-time.After(time.Second):
assert.FailNow(t, "no task update event")
}
}
}
func expectDeleteService(t *testing.T, watch chan events.Event) {
for {
select {
case event := <-watch:
if _, ok := event.(state.EventDeleteService); !ok {
assert.FailNow(t, "expected service delete event, got", fmt.Sprint(event))
}
return
case <-time.After(time.Second):
assert.FailNow(t, "no service delete event")
}
}
}
func expectCommit(t *testing.T, watch chan events.Event) {
for {
select {
case event := <-watch:
if _, ok := event.(state.EventCommit); !ok {
assert.FailNow(t, "expected commit event, got", fmt.Sprint(event))
}
return
case <-time.After(time.Second):
assert.FailNow(t, "no commit event")
}
}
}
func TestNewReader(t *testing.T) {
r := getReader("virginia.edu.uva-lib_2278801.tar")
if r.Manifest == nil {
assert.FailNow(t, "Reader Manifest should not be nil")
}
if r.Manifest.Object == nil {
assert.FailNow(t, "Reader Manifest Object should not be nil")
}
assert.Equal(t, "virginia.edu.uva-lib_2278801", r.Manifest.Object.Identifier)
assert.Equal(t, "virginia.edu", r.Manifest.Object.Institution)
assert.Equal(t, "uva-lib_2278801", r.Manifest.Object.BagName)
assert.True(t, strings.HasPrefix(r.Manifest.Object.IngestTarFilePath, "/"))
assert.True(t, strings.HasSuffix(r.Manifest.Object.IngestTarFilePath, "testdata/unit_test_bags/virginia.edu.uva-lib_2278801.tar"))
}
// Expect fails the test if the next event is not one of the specified events.
func Expect(t *testing.T, watch chan events.Event, specifiers ...state.Event) {
matcher := state.Matcher(specifiers...)
for {
select {
case event := <-watch:
if !matcher.Match(event) {
assert.FailNow(t, fmt.Sprintf("unexpected event: %T", event))
}
return
case <-time.After(time.Second):
assert.FailNow(t, "no commit event")
}
}
}
func watchShutdownTask(t *testing.T, watch chan events.Event) *api.Task {
for {
select {
case event := <-watch:
if task, ok := event.(state.EventUpdateTask); ok && task.Task.DesiredState == api.TaskStateShutdown {
return task.Task
}
if _, ok := event.(state.EventCreateTask); ok {
assert.FailNow(t, "got EventCreateTask when expecting EventUpdateTask", fmt.Sprint(event))
}
case <-time.After(time.Second):
assert.FailNow(t, "no task deletion")
}
}
}
func watchTaskUpdate(t *testing.T, watch chan events.Event) *api.Task {
for {
select {
case event := <-watch:
if task, ok := event.(state.EventUpdateTask); ok {
return task.Task
}
if _, ok := event.(state.EventCreateTask); ok {
assert.FailNow(t, "got EventCreateTask when expecting EventUpdateTask", fmt.Sprint(event))
}
case <-time.After(time.Second):
assert.FailNow(t, "no task update")
}
}
}
func (suite *LoggerTester) Test01RawGet() {
t := suite.T()
assert := assert.New(t)
suite.setupFixture()
defer suite.teardownFixture()
resp, err := http.Get(suite.url + "/message?perPage=13&page=&0")
assert.NoError(err)
assert.True(resp.ContentLength >= 0)
if resp.ContentLength == -1 {
assert.FailNow("bonk")
}
assert.True(resp.ContentLength > 0)
raw := make([]byte, resp.ContentLength)
_, err = io.ReadFull(resp.Body, raw)
defer func() {
err = resp.Body.Close()
assert.NoError(err)
}()
if err != nil && err != io.EOF {
assert.NoError(err)
}
assert.Equal(200, resp.StatusCode)
}
func TestRangeNetwork(t *testing.T) {
var tests = []struct {
r *Range
n *net.IPNet
}{
{ParseRange("10.10.10.10/24"), &net.IPNet{IP: net.ParseIP("10.10.10.0"), Mask: net.CIDRMask(24, 32)}},
{ParseRange("10.10.10.10-10.10.14.11"), nil},
{ParseRange("10.10.10.10-10.10.10.11"), &net.IPNet{IP: net.ParseIP("10.10.10.10"), Mask: net.CIDRMask(31, 32)}},
}
for _, te := range tests {
n := te.r.Network()
if te.n != nil {
assert.NotNil(t, n)
} else {
assert.Nil(t, n)
continue
}
if !n.IP.Equal(te.n.IP) {
assert.FailNow(t, fmt.Sprintf("got %s, want %s", n, te.n))
}
assert.EqualValues(t, n.Mask, te.n.Mask)
}
}
func TestNT(t *testing.T) {
assert := assert.New(t)
var genericServer *GenericServer
var server *ThingServer
var sys *SystemConfig
{
var err error
required := []ServiceName{}
sys, err = NewSystemConfig(PzGoCommonTest, required)
if err != nil {
assert.FailNow(err.Error())
}
genericServer = &GenericServer{Sys: sys}
server = &ThingServer{}
service := &ThingService{
assert: assert,
IDCount: 0,
Data: make(map[string]string),
}
server.Init(service)
}
{
var err error
err = genericServer.Configure(server.routes)
if err != nil {
assert.FailNow("server failed to configure: " + err.Error())
}
_, err = genericServer.Start()
if err != nil {
assert.FailNow("server failed to start: " + err.Error())
}
}
GetValueFromHeader(http.Header{}, "Content-Type")
_, _, _, err := HTTP(GET, "localhost:"+LocalPortNumbers[PzGoCommonTest], NewHeaderBuilder().AddJsonContentType().AddBasicAuth("foo", "bar").GetHeader(), nil)
if err != nil {
assert.FailNow(err.Error())
}
err = genericServer.Stop()
assert.NoError(err)
}
func TestNewWriter(t *testing.T) {
dir, err := ioutil.TempDir("", "tarwriter_test")
if err != nil {
assert.FailNow(t, "Cannot create temp dir", err.Error())
}
tempFilePath := filepath.Join(dir, "test_file.tar")
defer os.RemoveAll(dir)
w := tarfile.NewWriter(tempFilePath)
assert.NotNil(t, w)
assert.Equal(t, tempFilePath, w.PathToTarFile)
}
func (params *benchParams) createClients() (clients []client.Client) {
wsURL := "ws://" + params.service.WebServer().GetAddr() + "/stream/user/"
for clientID := 0; clientID < params.clients; clientID++ {
location := wsURL + strconv.Itoa(clientID)
c, err := client.Open(location, "http://localhost/", 1000, true)
if err != nil {
assert.FailNow(params, "guble client could not connect to server")
}
clients = append(clients, c)
}
return
}
func watchTaskDelete(t *testing.T, watch chan events.Event) *api.Task {
for {
select {
case event := <-watch:
if task, ok := event.(state.EventDeleteTask); ok {
return task.Task
}
case <-time.After(time.Second):
assert.FailNow(t, "no task deletion")
}
}
}
func TestAddToArchiveWithBadFilePath(t *testing.T) {
dir, err := ioutil.TempDir("", "tarwriter_test")
if err != nil {
assert.FailNow(t, "Cannot create temp dir", err.Error())
}
tempFilePath := filepath.Join(dir, "test_file.tar")
defer os.RemoveAll(dir)
w := tarfile.NewWriter(tempFilePath)
defer w.Close()
err = w.Open()
assert.Nil(t, err)
if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
}
// This file doesn't exist. Make sure we get the right error.
err = w.AddToArchive(pathToTestFile("this_file_does_not_exist"), "file1.json")
if err == nil {
assert.FailNow(t, "Should have gotten a tar write error")
}
assert.True(t, strings.Contains(err.Error(), "no such file or directory"))
}
func TestAddToArchive(t *testing.T) {
dir, err := ioutil.TempDir("", "tarwriter_test")
if err != nil {
assert.FailNow(t, "Cannot create temp dir", err.Error())
}
tempFilePath := filepath.Join(dir, "test_file.tar")
defer os.RemoveAll(dir)
w := tarfile.NewWriter(tempFilePath)
defer w.Close()
err = w.Open()
assert.Nil(t, err)
if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
}
err = w.AddToArchive(pathToTestFile("cleanup_result.json"), "file1.json")
assert.Nil(t, err)
err = w.AddToArchive(pathToTestFile("ingest_result.json"), "data/subdir/file2.json")
assert.Nil(t, err)
w.Close()
file, err := os.Open(w.PathToTarFile)
if file != nil {
defer file.Close()
}
if err != nil {
assert.FailNow(t, "Could not open tar file", err.Error())
}
filesInArchive := make([]string, 0)
reader := tar.NewReader(file)
for {
header, err := reader.Next()
if err != nil {
break
}
filesInArchive = append(filesInArchive, header.Name)
}
assert.Equal(t, "file1.json", filesInArchive[0])
assert.Equal(t, "data/subdir/file2.json", filesInArchive[1])
}
func parse(t *testing.T, remote string) *Reference {
reference, err := Parse(remote)
if err != nil {
assert.FailNow(t, "An error has occurred with the parser")
return nil
}
assert.NotNil(t, reference)
assert.NotEmpty(t, reference)
return reference
}
func assertDeepValues(t *testing.T, expected, actual []inference.Field) {
for _, v := range expected {
dataVal, err := whereKeyEquals(actual, v.Name)
if err != nil {
assert.FailNow(t, err.Error())
}
if dataVal.Type == "array" || dataVal.Type == "object" {
assertDeepValues(t, v.SubObject, dataVal.SubObject)
} else {
assert.EqualValues(t, v, *dataVal)
}
}
}
func TestAndCloseOpen(t *testing.T) {
dir, err := ioutil.TempDir("", "tarwriter_test")
if err != nil {
assert.FailNow(t, "Cannot create temp dir", err.Error())
}
tempFilePath := filepath.Join(dir, "test_file.tar")
defer os.RemoveAll(dir)
w := tarfile.NewWriter(tempFilePath)
defer w.Close()
err = w.Open()
assert.Nil(t, err)
if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
}
err = w.Close()
assert.Nil(t, err)
}
func TestSyncPaths(t *testing.T) {
m, err := manifestFixture("sync-path")
if err != nil {
assert.FailNow(t, fmt.Sprintf("failed to read fixture: %s", err.Error()))
}
expectedMap := map[string]string{
".": "/app",
"Gemfile": "/app/Gemfile",
"Gemfile.lock": "/app/Gemfile.lock",
"Rakefile": "/app/Rakefile",
"config": "/app/config/bar",
"public": "/app/public/$FAKE",
"app/assets": "/app/app/assets",
}
for _, s := range m.Services {
sp, err := s.SyncPaths()
if assert.Nil(t, err) {
assert.EqualValues(t, expectedMap, sp)
}
}
}
// FailNow fails test
func FailNow(t TestingT, failureMessage string, msgAndArgs ...interface{}) {
if !assert.FailNow(t, failureMessage, msgAndArgs...) {
t.FailNow()
}
}
func TestUpdateNode(t *testing.T) {
tc := cautils.NewTestCA(nil)
defer tc.Stop()
ts := newTestServer(t)
defer ts.Stop()
nodes := make(map[uint64]*raftutils.TestNode)
nodes[1], _ = raftutils.NewInitNode(t, tc, nil)
defer raftutils.TeardownCluster(t, nodes)
nodeID := nodes[1].SecurityConfig.ClientTLSCreds.NodeID()
// Assign one of the raft node to the test server
ts.Server.raft = nodes[1].Node
ts.Server.store = nodes[1].MemoryStore()
_, err := ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{
NodeID: nodeID,
Spec: &api.NodeSpec{
Availability: api.NodeAvailabilityDrain,
},
NodeVersion: &api.Version{},
})
assert.Error(t, err)
assert.Equal(t, codes.NotFound, grpc.Code(err))
// Create a node object for the manager
assert.NoError(t, nodes[1].MemoryStore().Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateNode(tx, &api.Node{
ID: nodes[1].SecurityConfig.ClientTLSCreds.NodeID(),
Spec: api.NodeSpec{
Role: api.NodeRoleManager,
Membership: api.NodeMembershipAccepted,
},
}))
return nil
}))
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{})
assert.Error(t, err)
assert.Equal(t, codes.InvalidArgument, grpc.Code(err))
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{NodeID: "invalid", Spec: &api.NodeSpec{}, NodeVersion: &api.Version{}})
assert.Error(t, err)
assert.Equal(t, codes.NotFound, grpc.Code(err))
r, err := ts.Client.GetNode(context.Background(), &api.GetNodeRequest{NodeID: nodeID})
assert.NoError(t, err)
if !assert.NotNil(t, r) {
assert.FailNow(t, "got unexpected nil response from GetNode")
}
assert.NotNil(t, r.Node)
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{NodeID: nodeID})
assert.Error(t, err)
assert.Equal(t, codes.InvalidArgument, grpc.Code(err))
spec := r.Node.Spec.Copy()
spec.Availability = api.NodeAvailabilityDrain
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{
NodeID: nodeID,
Spec: spec,
})
assert.Error(t, err)
assert.Equal(t, codes.InvalidArgument, grpc.Code(err))
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{
NodeID: nodeID,
Spec: spec,
NodeVersion: &r.Node.Meta.Version,
})
assert.NoError(t, err)
r, err = ts.Client.GetNode(context.Background(), &api.GetNodeRequest{NodeID: nodeID})
assert.NoError(t, err)
if !assert.NotNil(t, r) {
assert.FailNow(t, "got unexpected nil response from GetNode")
}
assert.NotNil(t, r.Node)
assert.NotNil(t, r.Node.Spec)
assert.Equal(t, api.NodeAvailabilityDrain, r.Node.Spec.Availability)
version := &r.Node.Meta.Version
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{NodeID: nodeID, Spec: &r.Node.Spec, NodeVersion: version})
assert.NoError(t, err)
// Perform an update with the "old" version.
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{NodeID: nodeID, Spec: &r.Node.Spec, NodeVersion: version})
assert.Error(t, err)
}
func Test07Server(t *testing.T) {
assert := assert.New(t)
JsonResponseDataTypes["string"] = "string"
JsonResponseDataTypes["piazza.Thing"] = "thing"
JsonResponseDataTypes["*piazza.Thing"] = "thing"
var genericServer *GenericServer
var server *ThingServer
var sys *SystemConfig
{
var err error
required := []ServiceName{}
sys, err = NewSystemConfig(PzGoCommon, required)
assert.NoError(err)
genericServer = &GenericServer{Sys: sys}
server = &ThingServer{}
service := &ThingService{
assert: assert,
IDCount: 0,
Data: make(map[string]string),
}
server.Init(service)
}
h := &Http{}
{
var err error
err = genericServer.Configure(server.routes)
if err != nil {
assert.FailNow("server failed to configure: " + err.Error())
}
_, err = genericServer.Start()
if err != nil {
assert.FailNow("server failed to start: " + err.Error())
}
h.BaseUrl = "http://" + sys.BindTo
}
var input *Thing
var output Thing
var jresp *JsonResponse = &JsonResponse{}
{
var err error
// GET /
jresp = h.PzGet("/")
assert.Equal(200, jresp.StatusCode)
assert.EqualValues("string", jresp.Type)
// GET bad
jresp = h.PzGet("/mpg")
assert.Equal(404, jresp.StatusCode)
// POST 1
input = &Thing{Value: "17"}
jresp = h.PzPost("/", input)
assert.Equal(201, jresp.StatusCode)
assert.EqualValues("thing", jresp.Type)
err = jresp.ExtractData(&output)
assert.EqualValues("1", output.ID)
assert.EqualValues("17", output.Value)
// POST bad
input = &Thing{Value: "NULL"}
jresp = h.PzPost("/", input)
assert.Equal(400, jresp.StatusCode)
// POST 2
input = &Thing{Value: "18"}
jresp = h.PzPost("/", input)
assert.Equal(201, jresp.StatusCode)
assert.EqualValues("thing", jresp.Type)
err = jresp.ExtractData(&output)
assert.EqualValues("2", output.ID)
assert.EqualValues("18", output.Value)
// GET 2
jresp = h.PzGet("/2")
assert.Equal(200, jresp.StatusCode)
assert.EqualValues("thing", jresp.Type)
err = jresp.ExtractData(&output)
assert.NoError(err)
assert.EqualValues("2", output.ID)
assert.EqualValues("18", output.Value)
// PUT 1
input = &Thing{Value: "71"}
jresp = h.PzPut("/1", input)
assert.Equal(200, jresp.StatusCode)
assert.EqualValues("thing", jresp.Type)
err = jresp.ExtractData(&output)
assert.NoError(err)
assert.EqualValues("71", output.Value)
// GET 1
jresp = h.PzGet("/1")
assert.Equal(200, jresp.StatusCode)
assert.EqualValues("thing", jresp.Type)
err = jresp.ExtractData(&output)
assert.NoError(err)
assert.EqualValues("1", output.ID)
assert.EqualValues("71", output.Value)
// DELETE 3
jresp = h.PzDelete("/3")
assert.Equal(404, jresp.StatusCode)
// DELETE 1
jresp = h.PzDelete("/1")
assert.Equal(200, jresp.StatusCode)
// GET 1
jresp = h.PzGet("/1")
assert.Equal(404, jresp.StatusCode)
}
// raw PUT and DELETE
{
// PUT
input = &Thing{Value: "72"}
body, err := h.convertObjectToReader(input)
assert.NoError(err)
resp, err := HTTPPut(h.BaseUrl+"/2", ContentTypeJSON, body)
assert.NoError(err)
assert.Equal(200, resp.StatusCode)
// check return
err = h.convertResponseBodyToObject(resp, jresp)
assert.NoError(err)
m := jresp.Data.(map[string]interface{})
assert.Equal("72", m["value"])
// DELETE
resp, err = HTTPDelete(h.BaseUrl + "/2")
assert.NoError(err)
assert.Equal(200, resp.StatusCode)
// check return
// GET 2
jresp = h.PzGet("/2")
assert.Equal(404, jresp.StatusCode)
}
{
err := genericServer.Stop()
//assert.NoError(err)
_, err = http.Get(h.BaseUrl)
assert.Error(err)
}
}
func TestNewModules(t *testing.T) {
const (
name = "HostParser"
host = "example.com"
uri = "http://" + host
)
r := newTestRegistry(t)
factory := func(base BaseMetricSet) (MetricSet, error) {
return &testMetricSet{base}, nil
}
hostParser := func(m Module, rawHost string) (HostData, error) {
return HostData{URI: uri, Host: host}, nil
}
if err := r.AddMetricSet(moduleName, name, factory, hostParser); err != nil {
t.Fatal(err)
}
t.Run("MetricSet without HostParser", func(t *testing.T) {
c := newConfig(t, map[string]interface{}{
"module": moduleName,
"metricsets": []string{metricSetName},
"hosts": []string{uri},
})
modules, err := NewModules(c, r)
if err != nil {
t.Fatal(err)
}
for _, metricSets := range modules {
metricSet := metricSets[0]
// The URI is passed through in the Host() and HostData().URI.
assert.Equal(t, uri, metricSet.Host())
assert.Equal(t, HostData{URI: uri}, metricSet.HostData())
return
}
assert.FailNow(t, "no modules found")
})
t.Run("MetricSet with HostParser", func(t *testing.T) {
c := newConfig(t, map[string]interface{}{
"module": moduleName,
"metricsets": []string{name},
"hosts": []string{uri},
})
modules, err := NewModules(c, r)
if err != nil {
t.Fatal(err)
}
for _, metricSets := range modules {
metricSet := metricSets[0]
// The URI is passed through in the Host() and HostData().URI.
assert.Equal(t, host, metricSet.Host())
assert.Equal(t, HostData{URI: uri, Host: host}, metricSet.HostData())
return
}
assert.FailNow(t, "no modules found")
})
}