func (s *XLSuite) TestChunkList(c *C) {
if VERBOSITY > 0 {
fmt.Println("TEST_CHUNK_LIST")
}
rng := xr.MakeSimpleRNG()
dataLen := 1 + rng.Intn(3*MAX_DATA_BYTES)
data := make([]byte, dataLen)
rng.NextBytes(data)
reader := bytes.NewReader(data)
d := sha1.New()
d.Write(data)
datum := d.Sum(nil)
nodeID, err := xi.NewNodeID(datum)
c.Assert(err, IsNil)
skPriv, err := rsa.GenerateKey(rand.Reader, 1024) // cheap key
sk := &skPriv.PublicKey
title := rng.NextFileName(8)
timestamp := xu.Timestamp(rng.Int63())
cl, err := NewChunkList(sk, title, timestamp, reader, int64(dataLen), datum, nil)
c.Assert(err, IsNil)
c.Assert(cl, NotNil)
chunkCount := uint((dataLen + MAX_DATA_BYTES - 1) / MAX_DATA_BYTES)
c.Assert(cl.Size(), Equals, chunkCount)
for i := uint(0); i < chunkCount; i++ {
actual, err := cl.HashItem(i)
c.Assert(err, IsNil)
expected := s.calculateChunkHash(c, i, datum, data)
// DEBUG
fmt.Printf("chunk %d hash\n actual %x\n expected %x\n",
i, actual, expected)
// END
//c.Assert(actual, DeepEquals, expected)
// compare with result of calculation in NewChunk -----------
var chunk *Chunk
var slice []byte
if i == chunkCount-1 {
slice = data[i*MAX_DATA_BYTES:]
} else {
slice = data[i*MAX_DATA_BYTES : (i+1)*MAX_DATA_BYTES]
}
chunk, err = NewChunk(nodeID, uint32(i), slice)
c.Assert(err, Equals, nil)
c.Assert(chunk.GetChunkHash(), DeepEquals, expected)
}
}
func makeNodeID(rng *xr.PRNG) (*xi.NodeID, error) {
var buffer []byte
// quasi-random choice, whether to use an SHA1 or SHA3 nodeID
if rng.NextBoolean() {
buffer = make([]byte, xu.SHA1_BIN_LEN)
} else {
buffer = make([]byte, xu.SHA3_BIN_LEN)
}
rng.NextBytes(buffer)
return xi.NewNodeID(buffer)
}
func (s *XLSuite) TestSomeInserts(c *C) {
mc, err := NewMemCache(uint64(1024*1024), uint(1024))
c.Assert(err, IsNil)
c.Assert(mc, NotNil)
c.Assert(mc.ItemCount(), Equals, uint(0))
c.Assert(mc.ByteCount(), Equals, uint64(0))
rng := xr.MakeSimpleRNG()
count := uint(16 + rng.Intn(17))
values := make([][]byte, count)
hashes := make([][]byte, count)
ids := make([]*xi.NodeID, count)
for i := uint(0); i < count; i++ {
size := 256 + rng.Intn(257)
values[i] = make([]byte, size)
rng.NextBytes(values[i])
d := sha1.New()
d.Write(values[i])
hashes[i] = d.Sum(nil)
id, err := xi.NewNodeID(hashes[i])
c.Assert(err, IsNil)
ids[i] = id
}
var totalBytes uint64
var totalItems uint
for i := uint(0); i < count; i++ {
err = mc.Add(ids[i], values[i])
c.Assert(err, IsNil)
totalItems++
totalBytes += uint64(len(values[i]))
c.Assert(mc.ItemCount(), Equals, totalItems)
c.Assert(mc.ByteCount(), Equals, totalBytes)
}
// test idempotence
for i := uint(0); i < count; i++ {
err = mc.Add(ids[i], values[i])
c.Assert(err, IsNil)
c.Assert(mc.ItemCount(), Equals, totalItems)
c.Assert(mc.ByteCount(), Equals, totalBytes)
}
}
func (s *XLSuite) TestMakeHelloMsg(c *C) {
if VERBOSITY > 0 {
fmt.Println("TEST_MAKE_HELLO_MSG")
}
rng := xr.MakeSimpleRNG()
id := make([]byte, xu.SHA1_BIN_LEN)
rng.NextBytes(id)
nodeID, err := xi.NewNodeID(id)
c.Assert(err, IsNil)
name := rng.NextFileName(8)
lfs := "tmp/" + hex.EncodeToString(id)
mrX, err := xn.NewNew(name, nodeID, lfs)
c.Assert(err, IsNil)
cPubKey := mrX.GetCommsPublicKey()
c.Assert(cPubKey, Not(IsNil))
sPubKey := mrX.GetSigPublicKey()
c.Assert(sPubKey, Not(IsNil))
// convert MrX's keys to wire form as byte slices
wcPubKey, err := xc.RSAPubKeyToWire(cPubKey)
c.Assert(err, IsNil)
c.Assert(len(wcPubKey) > 0, Equals, true)
wsPubKey, err := xc.RSAPubKeyToWire(sPubKey)
c.Assert(err, IsNil)
c.Assert(len(wsPubKey) > 0, Equals, true)
c.Assert(wsPubKey, Not(IsNil))
hello, err := MakeHelloMsg(mrX)
c.Assert(err, IsNil)
c.Assert(hello, Not(IsNil))
// check NodeID
idInMsg := hello.GetID() // a byte slice, not a NodeID
c.Assert(id, DeepEquals, idInMsg)
// these are byte slices
mcPubKey := hello.GetCommsKey()
msPubKey := hello.GetSigKey()
c.Assert(len(mcPubKey), Equals, len(wcPubKey))
c.Assert(len(msPubKey), Equals, len(wsPubKey)) // FAILS 0, 294
c.Assert(wcPubKey, DeepEquals, mcPubKey)
c.Assert(wsPubKey, DeepEquals, msPubKey)
}
func (s *XLSuite) makeANode(c *C) (badGuy *xn.Node, acc xt.AcceptorI) {
rng := xr.MakeSimpleRNG()
id := make([]byte, xu.SHA1_BIN_LEN)
rng.NextBytes(id)
nodeID, err := xi.NewNodeID(id)
c.Assert(err, IsNil)
name := rng.NextFileName(8)
lfs := "tmp/" + hex.EncodeToString(id)
badGuy, err = xn.NewNew(name, nodeID, lfs)
c.Assert(err, IsNil)
accCount := badGuy.SizeAcceptors()
c.Assert(accCount, Equals, 0)
ep, err := xt.NewTcpEndPoint("127.0.0.1:0")
c.Assert(err, IsNil)
ndx, err := badGuy.AddEndPoint(ep)
c.Assert(err, IsNil)
c.Assert(ndx, Equals, 0)
acc = badGuy.GetAcceptor(0)
return
}
func (s *XLSuite) TestChunkListAssyDisassy(c *C) {
if VERBOSITY > 0 {
fmt.Println("TEST_CHUNK_LIST_ASSY_DISASSY")
}
rng := xr.MakeSimpleRNG()
// make a slice 3 to 7 chunks long, fill with random-ish data ---
chunkCount := 3 + rng.Intn(5) // so 3 to 7, inclusive
lastChunkLen := 1 + rng.Intn(MAX_DATA_BYTES-1)
dataLen := (chunkCount-1)*MAX_DATA_BYTES + lastChunkLen
data := make([]byte, dataLen)
rng.NextBytes(data)
// calculate datum, the SHA hash of the data --------------------
//d := sha3.NewKeccak256()
d := sha1.New()
d.Write(data)
hash := d.Sum(nil)
datum, err := xi.NewNodeID(hash)
c.Assert(err, IsNil)
// create tmp if it doesn't exist -------------------------------
found, err := xf.PathExists("tmp")
c.Assert(err, IsNil)
if !found {
err = os.MkdirAll("tmp", 0755)
c.Assert(err, IsNil)
}
// create scratch subdir with unique name -----------------------
var pathToU string
for {
dirName := rng.NextFileName(8)
pathToU = path.Join("tmp", dirName)
found, err = xf.PathExists(pathToU)
c.Assert(err, IsNil)
if !found {
break
}
}
// create a FLAT uDir at that point -----------------------------
myU, err := u.New(pathToU, u.DIR_FLAT, 0) // 0 means default perm
c.Assert(err, IsNil)
// write the test data into uDir --------------------------------
bytesWritten, key, err := myU.PutData(data, datum.Value())
c.Assert(err, IsNil)
c.Assert(bytes.Equal(datum.Value(), key), Equals, true)
c.Assert(bytesWritten, Equals, int64(dataLen))
skPriv, err := rsa.GenerateKey(rand.Reader, 1024) // cheap key
sk := &skPriv.PublicKey
c.Assert(err, IsNil)
c.Assert(skPriv, NotNil)
// Verify the file is present in uDir ---------------------------
// (yes this is a test of uDir logic but these are early days ---
// XXX uDir.Exist(arg) - arg should be []byte, no string
keyStr := hex.EncodeToString(key)
found, err = myU.HexKeyExists(keyStr)
c.Assert(err, IsNil)
c.Assert(found, Equals, true)
// use the data file to build a chunkList, writing the chunks ---
title := rng.NextFileName(8)
now := xu.Timestamp(time.Now().UnixNano())
// make a reader --------------------------------------
pathToData, err := myU.GetPathForHexKey(keyStr)
c.Assert(err, IsNil)
reader, err := os.Open(pathToData) // open for read only
c.Assert(err, IsNil)
defer reader.Close()
chunkList, err := NewChunkList(sk, title, now, reader, int64(dataLen), key, myU)
c.Assert(err, IsNil)
err = chunkList.Sign(skPriv)
c.Assert(err, IsNil)
digSig, err := chunkList.GetDigSig()
c.Assert(err, IsNil)
c.Assert(bytes.Equal(digSig, chunkList.digSig), Equals, true)
err = chunkList.Verify()
c.Assert(err, IsNil)
// REBUILD AND CHECK --------------------------------------------
// rebuild the complete file from the chunkList and files present
// in myU
u2, err := u.New(pathToU, u.DIR_FLAT, 0) // 0 means default perm
c.Assert(err, IsNil)
var data2 []byte // should become copy of original
count := chunkList.Size()
for i := uint(0); i < count; i++ {
chunkHash, err := chunkList.HashItem(i)
c.Assert(err, IsNil)
c.Assert(chunkHash, NotNil)
var raw []byte
// THIS IS A CHUNK, and so has a header, possibly some
// padding, and then its own hash :-). Need to verify
// and discard the last, then drop the padding.
// CORRECTION: hash should NOT have been written to disk
raw, err = u2.GetData(chunkHash)
c.Assert(err, IsNil)
chunk := &Chunk{packet: raw}
ndx := chunk.GetIndex()
c.Assert(ndx, Equals, uint32(i))
rawLen := uint32(len(raw))
dataLen := chunk.GetDataLen()
//fmt.Printf("chunk %2d: index is %8d\n", i, ndx)
//fmt.Printf(" len raw is %6d (%4x)\n", rawLen, rawLen)
//fmt.Printf(" dataLen is %6d (%4x)\n", dataLen, dataLen)
// if this isn't true, we get a panic
c.Assert(dataLen < rawLen, Equals, true)
payload := chunk.GetData()
data2 = append(data2, payload...)
}
// verify that the content key of the rebuilt file is identical to
// that of the original
//d2D := sha3.NewKeccak256()
d2D := sha1.New()
d2D.Write(data2)
hash2 := d2D.Sum(nil)
datum2, err := xi.NewNodeID(hash2)
c.Assert(err, IsNil)
// DEBUG
//fmt.Printf("datum: %x\ndatum2: %x\n", datum.Value(), datum2.Value())
//fmt.Printf("data: %x\ndata2: %x\n", data, data2)
// END
c.Assert(bytes.Equal(datum.Value(), datum2.Value()), Equals, true)
// presumably pure pedantry: verify that the file contents are
// also equal
c.Assert(bytes.Equal(data, data2), Equals, true)
}
func MockLocalHostCluster(K int) (nodes []*Node, accs []*xt.TcpAcceptor) {
rng := xr.MakeSimpleRNG()
// Create K nodes, each with a NodeID, two RSA private keys (sig and
// comms), and two RSA public keys. Each node creates a TcpAcceptor
// running on 127.0.0.1 and a random (= system-supplied) port.
names := make([]string, K)
nodeIDs := make([]*xi.NodeID, K)
for i := 0; i < K; i++ {
// TODO: MAKE NAMES UNIQUE
names[i] = rng.NextFileName(4)
val := make([]byte, xu.SHA1_BIN_LEN)
rng.NextBytes(val)
nodeIDs[i], _ = xi.NewNodeID(val)
}
nodes = make([]*Node, K)
accs = make([]*xt.TcpAcceptor, K)
accEndPoints := make([]*xt.TcpEndPoint, K)
for i := 0; i < K; i++ {
lfs := "tmp/" + hex.EncodeToString(nodeIDs[i].Value())
nodes[i], _ = NewNew(names[i], nodeIDs[i], lfs)
}
// XXX We need this functionality in using code
// defer func() {
// for i := 0; i < K; i++ {
// if accs[i] != nil {
// accs[i].CloseAcc()
// }
// }
// }()
// Collect the nodeID, public keys, and listening address from each
// node.
// all nodes on the same overlay
ar, _ := xo.NewCIDRAddrRange("127.0.0.0/8")
overlay, _ := xo.NewIPOverlay("XO", ar, "tcp", 1.0)
// add an endpoint to each node
for i := 0; i < K; i++ {
ep, _ := xt.NewTcpEndPoint("127.0.0.1:0")
nodes[i].AddEndPoint(ep)
nodes[i].OpenAcc() // XXX POSSIBLE ERRORS IGNORED
accs[i] = nodes[i].GetAcceptor(0).(*xt.TcpAcceptor)
accEndPoints[i] = accs[i].GetEndPoint().(*xt.TcpEndPoint)
}
ckPrivs := make([]*rsa.PublicKey, K)
skPrivs := make([]*rsa.PublicKey, K)
ctors := make([]*xt.TcpConnector, K)
for i := 0; i < K; i++ {
// we already have nodeIDs
ckPrivs[i] = nodes[i].GetCommsPublicKey()
skPrivs[i] = nodes[i].GetSigPublicKey()
ctors[i], _ = xt.NewTcpConnector(accEndPoints[i])
}
overlaySlice := []xo.OverlayI{overlay}
peers := make([]*Peer, K)
for i := 0; i < K; i++ {
ctorSlice := []xt.ConnectorI{ctors[i]}
_ = ctorSlice
peers[i], _ = NewPeer(names[i], nodeIDs[i], ckPrivs[i], skPrivs[i],
overlaySlice, ctorSlice)
}
// Use the information collected to configure each node.
for i := 0; i < K; i++ {
for j := 0; j < K; j++ {
if i != j {
nodes[i].AddPeer(peers[j])
}
}
}
return
}
// Version of the above which consumes a slice of strings. XXX Copies the
// slice unnecessarily.
func ParseBNFromStrings(ss []string, whichType string) (bn *BaseNode, rest []string, err error) {
var (
name string
nodeID *xi.NodeID
commsPubKey *rsa.PublicKey
sigPubKey *rsa.PublicKey
overlays []xo.OverlayI
)
s, err := xc.NextNBLine(&ss)
if err == nil {
opener := fmt.Sprintf("%s {", whichType) // "peer {" or "node {"
if s != opener {
err = NotExpectedOpener
}
}
if err == nil {
s, err := xc.NextNBLine(&ss)
if err == nil {
if strings.HasPrefix(s, "name: ") {
name = s[6:]
} else {
err = NotABaseNode
}
}
}
if err == nil {
s, err = xc.NextNBLine(&ss)
if err == nil {
if strings.HasPrefix(s, "nodeID: ") {
var val []byte
val, err = hex.DecodeString(s[8:])
if err == nil {
nodeID, err = xi.NewNodeID(val)
}
} else {
err = NotABaseNode
}
}
}
if err == nil {
s, err = xc.NextNBLine(&ss)
if err == nil {
if strings.HasPrefix(s, "commsPubKey: ") {
var ckPEM []byte
ckPEM, err = xc.CollectPEMRSAPublicKey(s[13:], &ss)
if err == nil {
commsPubKey, err = xc.RSAPubKeyFromPEM(ckPEM)
}
} else {
// DEBUG
fmt.Printf("\ndon't see commsPubKey\n")
// END
err = NotABaseNode
}
}
}
if err == nil {
s, err = xc.NextNBLine(&ss)
if err == nil {
if strings.HasPrefix(s, "sigPubKey: ") {
var skPEM []byte
skPEM, err = xc.CollectPEMRSAPublicKey(s[11:], &ss)
if err == nil {
sigPubKey, err = xc.RSAPubKeyFromPEM(skPEM)
}
} else {
// DEBUG
fmt.Printf("\ndon't see sigPubKey\n")
// END
err = NotABaseNode
}
}
}
if err == nil {
s, err = xc.NextNBLine(&ss)
if err == nil {
if s == "overlays {" {
for {
s, err = xc.NextNBLine(&ss)
if err == nil {
if s == "" { // end of strings
err = NotABaseNode
break
} else if s == "}" {
prepend := []string{s}
ss = append(prepend, ss...)
break
}
}
var o xo.OverlayI
o, err = xo.Parse(s)
if err == nil {
overlays = append(overlays, o)
}
}
} else {
err = NotABaseNode
}
}
}
if err == nil {
s, err = xc.NextNBLine(&ss)
if err == nil {
if s != "}" {
err = NotABaseNode
}
}
}
if err == nil {
var bn = BaseNode{name, nodeID, commsPubKey, sigPubKey, overlays}
return &bn, ss, nil
} else {
return nil, nil, err
}
}