func TestFsChildRemoveDeleteNetworkTimeout(t *testing.T) {
SetupCluster()
log.Info("Testing TestFsChildRemoveDeleteNetworkTimeout...")
resp3, other3 := GetProcessForPath("/test/timeout/delete1", "/test/timeout")
resp2, other2 := GetProcessForPath("/test/timeout/delete2", "/test/timeout", "/test/timeout/delete2")
resp1, other1 := GetProcessForPath("/test/timeout")
buf := buffer.NewFromString("write1")
resp3.Fss.Write(fs.NewPath("/test/timeout/delete1"), buf.Size, "", buf, nil)
buf = buffer.NewFromString("write1")
resp3.Fss.Write(fs.NewPath("/test/timeout/delete2"), buf.Size, "", buf, nil)
initadr := resp2.Cluster.Nodes.Get(resp1.Cluster.MyNode.Id).Address
resp2.Cluster.Nodes.Get(resp1.Cluster.MyNode.Id).Address = net.ParseIP("224.0.0.2")
resp3.Cluster.Nodes.Get(resp1.Cluster.MyNode.Id).Address = net.ParseIP("224.0.0.2")
time.Sleep(900000000)
// Delete child, it should never get delete from the parent
err := other2.Fss.Delete(fs.NewPath("/test/timeout/delete1"), false, nil)
if err != nil {
t.Error("1) Received an error: %s", err)
}
time.Sleep(900000000)
children, _ := other1.Fss.Children(fs.NewPath("/test/timeout"), nil)
found := false
for _, child := range children {
if child.Name == "delete1" {
found = true
}
}
if !found {
t.Error("2) Child should not have been deleted")
}
// Test timeout of the first and write of a new one
err = other3.Fss.Delete(fs.NewPath("/test/timeout/delete2"), false, nil)
if err != nil {
t.Error("3) Received an error: %s", err)
}
// Set back the address
resp2.Cluster.Nodes.Get(resp1.Cluster.MyNode.Id).Address = initadr
resp3.Cluster.Nodes.Get(resp1.Cluster.MyNode.Id).Address = initadr
time.Sleep(6000000000)
children, _ = other1.Fss.Children(fs.NewPath("/test/timeout"), nil)
found1, found2 := false, false
for _, child := range children {
if child.Name == "delete2" {
found2 = true
} else if child.Name == "delete1" {
found1 = true
}
}
if found1 || found2 {
t.Error("4) At least one of the children is still there", found1, found2)
}
}
func TestFsChildAddWriteNeedNetworkTimeout(t *testing.T) {
SetupCluster()
log.Info("Testing TestFsChildAddWriteNeedNetworkTimeout...")
resp2, other2 := GetProcessForPath("/tests/timeout2/write")
resp1, other1 := GetProcessForPath("/tests/timeout2")
initadr := resp2.Cluster.Nodes.Get(resp1.Cluster.MyNode.Id).Address
resp2.Cluster.Nodes.Get(resp1.Cluster.MyNode.Id).Address = net.ParseIP("224.0.0.2")
// Create the child, it should never get added to the parent
buf := buffer.NewFromString("write1")
err := other2.Fss.Write(fs.NewPath("/tests/timeout2/write"), buf.Size, "", buf, nil)
if err != nil && err != comm.ErrorTimeout {
t.Error("1) Received an error: %s", err)
}
time.Sleep(900000000)
children, _ := other1.Fss.Children(fs.NewPath("/tests/timeout2"), nil)
found := false
for _, child := range children {
if child.Name == "write" {
found = true
}
}
if found {
t.Error("2) Child should not have been added")
}
// Test one timeout + 1 retry, no error
buf = buffer.NewFromString("write1")
go func() {
time.Sleep(1500 * 1000 * 1000)
// set the address back
resp2.Cluster.Nodes.Get(resp1.Cluster.MyNode.Id).Address = initadr
}()
err = other2.Fss.Write(fs.NewPath("/tests/timeout2/write3"), buf.Size, "", buf, nil)
if err != nil {
t.Error("3) Received an error: %s", err)
}
children, err = other1.Fss.Children(fs.NewPath("/tests/timeout2"), nil)
found = false
for _, child := range children {
if child.Name == "write3" {
found = true
}
}
if !found {
t.Error("4) Child should have been added")
}
}
func TestFsDeleteNetworkTimeout(t *testing.T) {
SetupCluster()
log.Info("Testing TestFsDeleteNetworkTimeout...")
resp, other := GetProcessForPath("/tests/timeouts/delete1")
buf := buffer.NewFromString("write1")
other.Fss.Write(fs.NewPath("/tests/timeouts/delete1"), buf.Size, "", buf, nil)
initadr := other.Cluster.Nodes.Get(resp.Cluster.MyNode.Id).Address
other.Cluster.Nodes.Get(resp.Cluster.MyNode.Id).Address = net.ParseIP("224.0.0.2")
// Test full timeout and return of an error
timeoutTest(1500*4, func() {
buf = buffer.NewFromString("write1")
err := other.Fss.Delete(fs.NewPath("/tests/timeouts/delete1"), false, nil)
if err != comm.ErrorTimeout {
t.Error("1) Didn't receive timeout error: %s", err)
}
},
func(returned bool) {
if !returned {
t.Error("2) Write network timeout is endlessly sleeping")
}
})
resp, other = GetProcessForPath("/tests/timeouts/delete2")
buf = buffer.NewFromString("write1")
other.Fss.Write(fs.NewPath("/tests/timeouts/delete2"), buf.Size, "", buf, nil)
// Test one timeout + 1 retry, no error
timeoutTest(1500*4, func() {
// wait 50ms and change addr of the remote node
go func() {
time.Sleep(500000000)
other.Cluster.Nodes.Get(resp.Cluster.MyNode.Id).Address = initadr
}()
buf = buffer.NewFromString("write1")
err := other.Fss.Write(fs.NewPath("/tests/timeouts/delete2"), buf.Size, "", buf, nil)
if err == comm.ErrorTimeout {
t.Error("3) Received timeout error: %s", err)
}
},
func(returned bool) {
if !returned {
t.Error("4) Write network timeout is endlessly sleeping")
}
})
}
func TestString(t *testing.T) {
buff := buffer.NewFromString("test")
str, err := buff.ReadString()
if str != "test" {
t.Errorf("1) Read string wasn't 'test', got %s", str)
}
if err != nil {
t.Errorf("1) Got an error: %s", err)
}
buff.WriteString("toto")
buff.Seek(0, 0)
str, err = buff.ReadString()
if str != "test" {
t.Errorf("2) Read string wasn't 'test', got %s", str)
}
if err != nil {
t.Errorf("2) Got an error: %s", err)
}
str, err = buff.ReadString()
if str != "toto" {
t.Errorf("3) Read string wasn't 'toto', got %s", str)
}
if err != nil {
t.Errorf("3) Got an error: %s", err)
}
}
func TestChildAddReplication(t *testing.T) {
SetupCluster()
log.Info("Testing TestChildAddReplication...")
_, other := GetProcessForPath("/tests/replication/write/childadd")
resolv := tc.nodes[0].Cluster.Rings.GetGlobalRing().Resolve("/tests/replication/write")
_, otherparent := GetProcessForPath("/tests/replication/write")
buf := buffer.NewFromString("write1")
err := other.Fss.Write(fs.NewPath("/tests/replication/write/childadd"), buf.Size, "application/mytest", buf, nil)
if err != nil {
t.Errorf("1) Got an error while write: %s", err)
}
// check if its not on another node
context := tc.nodes[0].Fss.NewContext()
context.ForceLocal = true
children, err := otherparent.Fss.Children(fs.NewPath("/tests/replication/write"), context)
if err == nil && len(children) != 0 {
t.Errorf("2) Children should not exists on another node")
}
// check on first secondary
secondaryid := resolv.GetOnline(1).Id
context = tc.nodes[secondaryid].Fss.NewContext()
context.ForceLocal = true
children, err = tc.nodes[secondaryid].Fss.Children(fs.NewPath("/tests/replication/write"), context)
if err != nil {
t.Errorf("3) Got an error while exists: %s", err)
}
found := false
for _, child := range children {
if child.Name == "childadd" {
found = true
}
}
if !found {
t.Errorf("4) Couldn't find child 'childadd'")
}
// check on second secondary
secondaryid = resolv.GetOnline(1).Id
context = tc.nodes[secondaryid].Fss.NewContext()
context.ForceLocal = true
children, err = tc.nodes[secondaryid].Fss.Children(fs.NewPath("/tests/replication/write"), context)
if err != nil {
t.Errorf("5) Got an error while exists: %s", err)
}
found = false
for _, child := range children {
if child.Name == "childadd" {
found = true
}
}
if !found {
t.Errorf("6) Couldn't find child 'childadd'")
}
}
func TestWriteReadReplication(t *testing.T) {
SetupCluster()
log.Info("Testing TestWriteReadReplication...")
_, other := GetProcessForPath("/tests/replication/write/read")
resolv := tc.nodes[0].Cluster.Rings.GetGlobalRing().Resolve("/tests/replication/write/read")
buf := buffer.NewFromString("write1")
err := other.Fss.Write(fs.NewPath("/tests/replication/write/read"), buf.Size, "application/mytest", buf, nil)
if err != nil {
t.Errorf("1) Got an error while write: %s", err)
}
// test force local on exists
context := tc.nodes[0].Fss.NewContext()
context.ForceLocal = true
bufwriter := bytes.NewBuffer(make([]byte, 0))
buffer := io.Writer(bufwriter)
n, err := other.Fss.Read(fs.NewPath("/tests/replication/write/read"), 0, -1, 0, buffer, context)
if err != fs.ErrorFileNotFound {
t.Errorf("2) File shouldn't exists on another node: %s", err)
}
// force local replication on each secondary
tc.nodes[resolv.GetOnline(1).Id].Fss.Flush()
tc.nodes[resolv.GetOnline(2).Id].Fss.Flush()
// check on first secondary
secondaryid := resolv.GetOnline(1).Id
context = tc.nodes[secondaryid].Fss.NewContext()
context.ForceLocal = true
bufwriter = bytes.NewBuffer(make([]byte, 0))
buffer = io.Writer(bufwriter)
n, err = tc.nodes[secondaryid].Fss.Read(fs.NewPath("/tests/replication/write/read"), 0, -1, 0, buffer, context)
if err != nil {
t.Errorf("3) Got an error from read: %s", err)
}
if n != buf.Size || bytes.Compare(bufwriter.Bytes(), buf.Bytes()) != 0 {
t.Errorf("4) Didn't read what was written: %s!=%s", buf.Bytes(), bufwriter)
}
// check on second secondary
secondaryid = resolv.GetOnline(2).Id
context = tc.nodes[secondaryid].Fss.NewContext()
context.ForceLocal = true
bufwriter = bytes.NewBuffer(make([]byte, 0))
buffer = io.Writer(bufwriter)
n, err = tc.nodes[secondaryid].Fss.Read(fs.NewPath("/tests/replication/write/read"), 0, -1, 0, buffer, context)
if err != nil {
t.Errorf("3) Got an error from read: %s", err)
}
if n != buf.Size || bytes.Compare(bufwriter.Bytes(), buf.Bytes()) != 0 {
t.Errorf("4) Didn't read what was written: %s!=%s", buf.Bytes(), bufwriter)
}
}
func TestWriteExistsReplication(t *testing.T) {
SetupCluster()
log.Info("Testing TestWriteExistsReplication...")
_, other := GetProcessForPath("/tests/replication/write/exists")
resolv := tc.nodes[0].Cluster.Rings.GetGlobalRing().Resolve("/tests/replication/write/exists")
buf := buffer.NewFromString("write1")
err := other.Fss.Write(fs.NewPath("/tests/replication/write/exists"), buf.Size, "application/mytest", buf, nil)
if err != nil {
t.Errorf("1) Got an error while write: %s", err)
}
// test force local on exists
context := tc.nodes[0].Fss.NewContext()
context.ForceLocal = true
exists, err := other.Fss.Exists(fs.NewPath("/tests/replication/write/exists"), context)
if err != nil {
t.Errorf("2) Got an error while exists: %s", err)
}
if exists {
t.Errorf("3) Shouldn't exists on another node when forced local")
}
// check on first secondary
secondaryid := resolv.GetOnline(1).Id
context = tc.nodes[secondaryid].Fss.NewContext()
context.ForceLocal = true
exists, err = tc.nodes[secondaryid].Fss.Exists(fs.NewPath("/tests/replication/write/exists"), context)
if err != nil {
t.Errorf("4) Got an error while exists: %s", err)
}
if !exists {
t.Errorf("5) Received exists = false, should be true")
}
// check on second secondary
secondaryid = resolv.GetOnline(2).Id
exists, err = tc.nodes[secondaryid].Fss.Exists(fs.NewPath("/tests/replication/write/exists"), context)
if err != nil {
t.Errorf("6) Got an error while exists: %s", err)
}
if !exists {
t.Errorf("7) Received exists = false, should be true")
}
}
func TestWriteHeaderReplication(t *testing.T) {
SetupCluster()
log.Info("Testing TestWriteHeaderReplication...")
resp, other := GetProcessForPath("/tests/replication/write/header")
resolv := tc.nodes[0].Cluster.Rings.GetGlobalRing().Resolve("/tests/replication/write/header")
buf := buffer.NewFromString("write1")
err := other.Fss.Write(fs.NewPath("/tests/replication/write/header"), buf.Size, "application/mytest", buf, nil)
if err != nil {
t.Error("1) Got an error while write: %s", err)
}
// check header on master
header, err := resp.Fss.Header(fs.NewPath("/tests/replication/write/header"), nil)
version := header.Version
// check on first secondary
secondaryid := resolv.GetOnline(1).Id
header, err = tc.nodes[secondaryid].Fss.Header(fs.NewPath("/tests/replication/write/header"), nil)
if header.Version != version {
t.Errorf("2) Version on secondary node wasn't the same as on master: %d != %d", version, header.Version)
}
if header.MimeType != "application/mytest" {
t.Errorf("3) Mimetype on secondary node wasn't the same as on master: %s != %s", "application/mytest", header.MimeType)
}
if header.Size != buf.Size {
t.Errorf("4) Size on secondary node wasn't the same as on master: %d != %d", 6, header.Size)
}
// check on second secondary
secondaryid = resolv.GetOnline(2).Id
header, err = tc.nodes[secondaryid].Fss.Header(fs.NewPath("/tests/replication/write/header"), nil)
if header.Version != version {
t.Errorf("5) Version on secondary node wasn't the same as on master: %d != %d", version, header.Version)
}
if header.MimeType != "application/mytest" {
t.Errorf("6) Mimetype on secondary node wasn't the same as on master: %s != %s", "application/mytest", header.MimeType)
}
if header.Size != buf.Size {
t.Errorf("7) Size on secondary node wasn't the same as on master: %d != %d", 6, header.Size)
}
}
func TestUnicode(t *testing.T) {
SetupCluster()
log.Info("Testing TestUnicode...")
// Read write test
path := fs.NewPath("/tests/io/writeunicode")
byts := []byte("Some é unicode à data 世界")
buf := bytes.NewBuffer(byts)
err := tc.nodes[0].Fss.Write(path, int64(len(byts)), "", buf, nil)
if err != nil {
t.Errorf("1) Write returned an an error for: %s\n", err)
}
// Check if read data is the same as written
bufwriter := bytes.NewBuffer(make([]byte, 0))
n, err := tc.nodes[2].Fss.Read(path, 0, -1, 0, io.Writer(bufwriter), nil)
if n != int64(len(byts)) || bytes.Compare(bufwriter.Bytes(), byts) != 0 {
t.Errorf("2) Didn't read written data correctly: %s!=%s\n", byts, bufwriter)
}
// Test unicode path
path = fs.NewPath("/école èàaû/世界.txt")
wrtnData := buffer.NewFromString("some data")
err = tc.nodes[0].Fss.Write(path, wrtnData.Size, "", wrtnData, nil)
if err != nil {
t.Errorf("3) Error while writing: %s\n", err)
}
// Reading written data
rdData := buffer.NewWithSize(0, false)
n, err = tc.nodes[1].Fss.Read(path, 0, -1, 0, rdData, nil)
if n != wrtnData.Size || bytes.Compare(rdData.Bytes(), wrtnData.Bytes()) != 0 {
t.Errorf("4) Didn't read what had ben written: %s!=%s", rdData.Bytes(), wrtnData.Bytes())
}
}
func TestDeleteReplication(t *testing.T) {
SetupCluster()
log.Info("Testing TestDeleteReplication...")
resp, other := GetProcessForPath("/tests/replication/write/childdelete")
resolv := tc.nodes[0].Cluster.Rings.GetGlobalRing().Resolve("/tests/replication/write/childdelete")
respparent, _ := GetProcessForPath("/tests/replication/write")
resolvparent := tc.nodes[0].Cluster.Rings.GetGlobalRing().Resolve("/tests/replication/write")
buf := buffer.NewFromString("write1")
err := other.Fss.Write(fs.NewPath("/tests/replication/write/childdelete"), buf.Size, "application/mytest", buf, nil)
if err != nil {
t.Errorf("1) Got an error while write: %s", err)
}
// check if its not on another node
err = other.Fss.Delete(fs.NewPath("/tests/replication/write/childdelete"), true, nil)
if err != nil {
t.Errorf("2) Got an error while deleting: %s", err)
}
// check on master
context := tc.nodes[0].Fss.NewContext()
context.ForceLocal = true
exists, err := resp.Fss.Exists(fs.NewPath("/tests/replication/write/childdelete"), context)
if err != nil {
t.Errorf("3) Got an error while exists: %s", err)
}
if exists {
t.Errorf("4) Shouldn't exists on master")
}
// check on first secondary
secondaryid := resolv.GetOnline(1).Id
context = tc.nodes[secondaryid].Fss.NewContext()
context.ForceLocal = true
exists, err = tc.nodes[secondaryid].Fss.Exists(fs.NewPath("/tests/replication/write/childdelete"), context)
if err != nil {
t.Errorf("5) Got an error while exists: %s", err)
}
if exists {
t.Errorf("6) Shouldn't exists on master")
}
// check on second secondary
secondaryid = resolv.GetOnline(2).Id
context = tc.nodes[secondaryid].Fss.NewContext()
context.ForceLocal = true
exists, err = tc.nodes[secondaryid].Fss.Exists(fs.NewPath("/tests/replication/write/childdelete"), context)
if err != nil {
t.Errorf("7) Got an error while exists: %s", err)
}
if exists {
t.Errorf("8) Shouldn't exists on master")
}
time.Sleep(500000000)
// check master parent
secondaryid = resolvparent.GetOnline(2).Id
context = tc.nodes[secondaryid].Fss.NewContext()
context.ForceLocal = true
children, err := respparent.Fss.Children(fs.NewPath("/tests/replication/write"), context)
if err != nil {
t.Errorf("9) Got an error while exists: %s", err)
}
found := false
for _, child := range children {
if child.Name == "childdelete" {
found = true
}
}
if found {
t.Errorf("10) Child 'childdelete' should exists on master anymore")
}
// check first secondary
secondaryid = resolvparent.GetOnline(1).Id
context = tc.nodes[secondaryid].Fss.NewContext()
context.ForceLocal = true
children, err = tc.nodes[secondaryid].Fss.Children(fs.NewPath("/tests/replication/write"), context)
if err != nil {
t.Errorf("11) Got an error while exists: %s", err)
}
found = false
for _, child := range children {
if child.Name == "childdelete" {
found = true
}
}
if found {
t.Errorf("12) Child 'childdelete' should exists on secondary anymore")
}
// check second secondary
secondaryid = resolvparent.GetOnline(2).Id
context = tc.nodes[secondaryid].Fss.NewContext()
context.ForceLocal = true
children, err = tc.nodes[secondaryid].Fss.Children(fs.NewPath("/tests/replication/write"), context)
if err != nil {
t.Errorf("13) Got an error while exists: %s", err)
}
found = false
for _, child := range children {
if child.Name == "childdelete" {
found = true
}
}
if found {
t.Errorf("14) Child 'childdelete' should exists on secondary anymore")
}
}