Example #1
0
func (s *XLSuite) TestChunkList(c *C) {
	if VERBOSITY > 0 {
		fmt.Println("TEST_CHUNK_LIST")
	}
	rng := xr.MakeSimpleRNG()

	dataLen := 1 + rng.Intn(3*MAX_DATA_BYTES)
	data := make([]byte, dataLen)
	rng.NextBytes(data)

	reader := bytes.NewReader(data)
	d := sha1.New()
	d.Write(data)
	datum := d.Sum(nil)
	nodeID, err := xi.NewNodeID(datum)
	c.Assert(err, IsNil)

	skPriv, err := rsa.GenerateKey(rand.Reader, 1024) // cheap key

	sk := &skPriv.PublicKey
	title := rng.NextFileName(8)
	timestamp := xu.Timestamp(rng.Int63())

	cl, err := NewChunkList(sk, title, timestamp, reader, int64(dataLen), datum, nil)
	c.Assert(err, IsNil)
	c.Assert(cl, NotNil)

	chunkCount := uint((dataLen + MAX_DATA_BYTES - 1) / MAX_DATA_BYTES)
	c.Assert(cl.Size(), Equals, chunkCount)

	for i := uint(0); i < chunkCount; i++ {
		actual, err := cl.HashItem(i)
		c.Assert(err, IsNil)
		expected := s.calculateChunkHash(c, i, datum, data)
		// DEBUG
		fmt.Printf("chunk %d hash\n    actual   %x\n    expected %x\n",
			i, actual, expected)
		// END
		//c.Assert(actual, DeepEquals, expected)

		// compare with result of calculation in NewChunk -----------
		var chunk *Chunk
		var slice []byte
		if i == chunkCount-1 {
			slice = data[i*MAX_DATA_BYTES:]
		} else {
			slice = data[i*MAX_DATA_BYTES : (i+1)*MAX_DATA_BYTES]
		}

		chunk, err = NewChunk(nodeID, uint32(i), slice)
		c.Assert(err, Equals, nil)
		c.Assert(chunk.GetChunkHash(), DeepEquals, expected)
	}

}
Example #2
0
func makeNodeID(rng *xr.PRNG) (*xi.NodeID, error) {
	var buffer []byte
	// quasi-random choice, whether to use an SHA1 or SHA3 nodeID
	if rng.NextBoolean() {
		buffer = make([]byte, xu.SHA1_BIN_LEN)
	} else {
		buffer = make([]byte, xu.SHA3_BIN_LEN)
	}
	rng.NextBytes(buffer)
	return xi.NewNodeID(buffer)
}
Example #3
0
func (s *XLSuite) TestSomeInserts(c *C) {

	mc, err := NewMemCache(uint64(1024*1024), uint(1024))
	c.Assert(err, IsNil)
	c.Assert(mc, NotNil)
	c.Assert(mc.ItemCount(), Equals, uint(0))
	c.Assert(mc.ByteCount(), Equals, uint64(0))

	rng := xr.MakeSimpleRNG()
	count := uint(16 + rng.Intn(17))
	values := make([][]byte, count)
	hashes := make([][]byte, count)
	ids := make([]*xi.NodeID, count)

	for i := uint(0); i < count; i++ {
		size := 256 + rng.Intn(257)
		values[i] = make([]byte, size)
		rng.NextBytes(values[i])
		d := sha1.New()
		d.Write(values[i])
		hashes[i] = d.Sum(nil)
		id, err := xi.NewNodeID(hashes[i])
		c.Assert(err, IsNil)
		ids[i] = id
	}

	var totalBytes uint64
	var totalItems uint
	for i := uint(0); i < count; i++ {
		err = mc.Add(ids[i], values[i])
		c.Assert(err, IsNil)
		totalItems++
		totalBytes += uint64(len(values[i]))

		c.Assert(mc.ItemCount(), Equals, totalItems)
		c.Assert(mc.ByteCount(), Equals, totalBytes)
	}
	// test idempotence
	for i := uint(0); i < count; i++ {
		err = mc.Add(ids[i], values[i])
		c.Assert(err, IsNil)

		c.Assert(mc.ItemCount(), Equals, totalItems)
		c.Assert(mc.ByteCount(), Equals, totalBytes)
	}

}
Example #4
0
func (s *XLSuite) TestMakeHelloMsg(c *C) {
	if VERBOSITY > 0 {
		fmt.Println("TEST_MAKE_HELLO_MSG")
	}
	rng := xr.MakeSimpleRNG()
	id := make([]byte, xu.SHA1_BIN_LEN)
	rng.NextBytes(id)
	nodeID, err := xi.NewNodeID(id)
	c.Assert(err, IsNil)

	name := rng.NextFileName(8)
	lfs := "tmp/" + hex.EncodeToString(id)
	mrX, err := xn.NewNew(name, nodeID, lfs)
	c.Assert(err, IsNil)
	cPubKey := mrX.GetCommsPublicKey()
	c.Assert(cPubKey, Not(IsNil))
	sPubKey := mrX.GetSigPublicKey()
	c.Assert(sPubKey, Not(IsNil))

	// convert MrX's keys to wire form as byte slices
	wcPubKey, err := xc.RSAPubKeyToWire(cPubKey)
	c.Assert(err, IsNil)
	c.Assert(len(wcPubKey) > 0, Equals, true)
	wsPubKey, err := xc.RSAPubKeyToWire(sPubKey)
	c.Assert(err, IsNil)
	c.Assert(len(wsPubKey) > 0, Equals, true)
	c.Assert(wsPubKey, Not(IsNil))

	hello, err := MakeHelloMsg(mrX)
	c.Assert(err, IsNil)
	c.Assert(hello, Not(IsNil))

	// check NodeID
	idInMsg := hello.GetID() // a byte slice, not a NodeID
	c.Assert(id, DeepEquals, idInMsg)

	// these are byte slices
	mcPubKey := hello.GetCommsKey()
	msPubKey := hello.GetSigKey()

	c.Assert(len(mcPubKey), Equals, len(wcPubKey))
	c.Assert(len(msPubKey), Equals, len(wsPubKey)) // FAILS 0, 294

	c.Assert(wcPubKey, DeepEquals, mcPubKey)
	c.Assert(wsPubKey, DeepEquals, msPubKey)
}
Example #5
0
func (s *XLSuite) makeANode(c *C) (badGuy *xn.Node, acc xt.AcceptorI) {
	rng := xr.MakeSimpleRNG()
	id := make([]byte, xu.SHA1_BIN_LEN)
	rng.NextBytes(id)
	nodeID, err := xi.NewNodeID(id)
	c.Assert(err, IsNil)
	name := rng.NextFileName(8)
	lfs := "tmp/" + hex.EncodeToString(id)
	badGuy, err = xn.NewNew(name, nodeID, lfs)
	c.Assert(err, IsNil)
	accCount := badGuy.SizeAcceptors()
	c.Assert(accCount, Equals, 0)
	ep, err := xt.NewTcpEndPoint("127.0.0.1:0")
	c.Assert(err, IsNil)
	ndx, err := badGuy.AddEndPoint(ep)
	c.Assert(err, IsNil)
	c.Assert(ndx, Equals, 0)
	acc = badGuy.GetAcceptor(0)
	return
}
func (s *XLSuite) TestChunkListAssyDisassy(c *C) {
	if VERBOSITY > 0 {
		fmt.Println("TEST_CHUNK_LIST_ASSY_DISASSY")
	}
	rng := xr.MakeSimpleRNG()

	// make a slice 3 to 7 chunks long, fill with random-ish data ---
	chunkCount := 3 + rng.Intn(5) // so 3 to 7, inclusive
	lastChunkLen := 1 + rng.Intn(MAX_DATA_BYTES-1)
	dataLen := (chunkCount-1)*MAX_DATA_BYTES + lastChunkLen
	data := make([]byte, dataLen)
	rng.NextBytes(data)

	// calculate datum, the SHA hash of the data --------------------
	//d := sha3.NewKeccak256()
	d := sha1.New()
	d.Write(data)
	hash := d.Sum(nil)
	datum, err := xi.NewNodeID(hash)
	c.Assert(err, IsNil)

	// create tmp if it doesn't exist -------------------------------
	found, err := xf.PathExists("tmp")
	c.Assert(err, IsNil)
	if !found {
		err = os.MkdirAll("tmp", 0755)
		c.Assert(err, IsNil)
	}

	// create scratch subdir with unique name -----------------------
	var pathToU string
	for {
		dirName := rng.NextFileName(8)
		pathToU = path.Join("tmp", dirName)
		found, err = xf.PathExists(pathToU)
		c.Assert(err, IsNil)
		if !found {
			break
		}
	}

	// create a FLAT uDir at that point -----------------------------
	myU, err := u.New(pathToU, u.DIR_FLAT, 0) // 0 means default perm
	c.Assert(err, IsNil)

	// write the test data into uDir --------------------------------
	bytesWritten, key, err := myU.PutData(data, datum.Value())
	c.Assert(err, IsNil)
	c.Assert(bytes.Equal(datum.Value(), key), Equals, true)
	c.Assert(bytesWritten, Equals, int64(dataLen))

	skPriv, err := rsa.GenerateKey(rand.Reader, 1024) // cheap key
	sk := &skPriv.PublicKey
	c.Assert(err, IsNil)
	c.Assert(skPriv, NotNil)

	// Verify the file is present in uDir ---------------------------
	// (yes this is a test of uDir logic but these are early days ---
	// XXX uDir.Exist(arg) - arg should be []byte, no string
	keyStr := hex.EncodeToString(key)
	found, err = myU.HexKeyExists(keyStr)
	c.Assert(err, IsNil)
	c.Assert(found, Equals, true)

	// use the data file to build a chunkList, writing the chunks ---
	title := rng.NextFileName(8)
	now := xu.Timestamp(time.Now().UnixNano())

	// make a reader --------------------------------------
	pathToData, err := myU.GetPathForHexKey(keyStr)
	c.Assert(err, IsNil)
	reader, err := os.Open(pathToData) // open for read only
	c.Assert(err, IsNil)
	defer reader.Close()

	chunkList, err := NewChunkList(sk, title, now, reader, int64(dataLen), key, myU)
	c.Assert(err, IsNil)
	err = chunkList.Sign(skPriv)
	c.Assert(err, IsNil)

	digSig, err := chunkList.GetDigSig()
	c.Assert(err, IsNil)
	c.Assert(bytes.Equal(digSig, chunkList.digSig), Equals, true)

	err = chunkList.Verify()
	c.Assert(err, IsNil)

	// REBUILD AND CHECK --------------------------------------------
	// rebuild the complete file from the chunkList and files present
	// in myU

	u2, err := u.New(pathToU, u.DIR_FLAT, 0) // 0 means default perm
	c.Assert(err, IsNil)

	var data2 []byte // should become copy of original
	count := chunkList.Size()
	for i := uint(0); i < count; i++ {
		chunkHash, err := chunkList.HashItem(i)
		c.Assert(err, IsNil)
		c.Assert(chunkHash, NotNil)
		var raw []byte
		// THIS IS A CHUNK, and so has a header, possibly some
		// padding, and then its own hash :-).  Need to verify
		// and discard the last, then drop the padding.
		// CORRECTION: hash should NOT have been written to disk
		raw, err = u2.GetData(chunkHash)
		c.Assert(err, IsNil)

		chunk := &Chunk{packet: raw}
		ndx := chunk.GetIndex()
		c.Assert(ndx, Equals, uint32(i))
		rawLen := uint32(len(raw))
		dataLen := chunk.GetDataLen()

		//fmt.Printf("chunk %2d: index is %8d\n", i, ndx)
		//fmt.Printf("          len raw is %6d (%4x)\n", rawLen, rawLen)
		//fmt.Printf("          dataLen is %6d (%4x)\n", dataLen, dataLen)

		// if this isn't true, we get a panic
		c.Assert(dataLen < rawLen, Equals, true)
		payload := chunk.GetData()

		data2 = append(data2, payload...)
	}
	// verify that the content key of the rebuilt file is identical to
	// that of the original

	//d2D := sha3.NewKeccak256()
	d2D := sha1.New()
	d2D.Write(data2)
	hash2 := d2D.Sum(nil)
	datum2, err := xi.NewNodeID(hash2)
	c.Assert(err, IsNil)
	// DEBUG
	//fmt.Printf("datum:  %x\ndatum2: %x\n", datum.Value(), datum2.Value())
	//fmt.Printf("data: %x\ndata2: %x\n", data, data2)
	// END
	c.Assert(bytes.Equal(datum.Value(), datum2.Value()), Equals, true)

	// presumably pure pedantry: verify that the file contents are
	// also equal

	c.Assert(bytes.Equal(data, data2), Equals, true)
}
func MockLocalHostCluster(K int) (nodes []*Node, accs []*xt.TcpAcceptor) {

	rng := xr.MakeSimpleRNG()

	// Create K nodes, each with a NodeID, two RSA private keys (sig and
	// comms), and two RSA public keys.  Each node creates a TcpAcceptor
	// running on 127.0.0.1 and a random (= system-supplied) port.
	names := make([]string, K)
	nodeIDs := make([]*xi.NodeID, K)
	for i := 0; i < K; i++ {
		// TODO: MAKE NAMES UNIQUE
		names[i] = rng.NextFileName(4)
		val := make([]byte, xu.SHA1_BIN_LEN)
		rng.NextBytes(val)
		nodeIDs[i], _ = xi.NewNodeID(val)
	}
	nodes = make([]*Node, K)
	accs = make([]*xt.TcpAcceptor, K)
	accEndPoints := make([]*xt.TcpEndPoint, K)
	for i := 0; i < K; i++ {
		lfs := "tmp/" + hex.EncodeToString(nodeIDs[i].Value())
		nodes[i], _ = NewNew(names[i], nodeIDs[i], lfs)
	}
	// XXX We need this functionality in using code
	//	defer func() {
	//		for i := 0; i < K; i++ {
	//			if accs[i] != nil {
	//				accs[i].CloseAcc()
	//			}
	//		}
	//	}()

	// Collect the nodeID, public keys, and listening address from each
	// node.

	// all nodes on the same overlay
	ar, _ := xo.NewCIDRAddrRange("127.0.0.0/8")
	overlay, _ := xo.NewIPOverlay("XO", ar, "tcp", 1.0)

	// add an endpoint to each node
	for i := 0; i < K; i++ {
		ep, _ := xt.NewTcpEndPoint("127.0.0.1:0")
		nodes[i].AddEndPoint(ep)
		nodes[i].OpenAcc() // XXX POSSIBLE ERRORS IGNORED
		accs[i] = nodes[i].GetAcceptor(0).(*xt.TcpAcceptor)
		accEndPoints[i] = accs[i].GetEndPoint().(*xt.TcpEndPoint)
	}

	ckPrivs := make([]*rsa.PublicKey, K)
	skPrivs := make([]*rsa.PublicKey, K)
	ctors := make([]*xt.TcpConnector, K)

	for i := 0; i < K; i++ {
		// we already have nodeIDs
		ckPrivs[i] = nodes[i].GetCommsPublicKey()
		skPrivs[i] = nodes[i].GetSigPublicKey()
		ctors[i], _ = xt.NewTcpConnector(accEndPoints[i])
	}

	overlaySlice := []xo.OverlayI{overlay}
	peers := make([]*Peer, K)
	for i := 0; i < K; i++ {
		ctorSlice := []xt.ConnectorI{ctors[i]}
		_ = ctorSlice
		peers[i], _ = NewPeer(names[i], nodeIDs[i], ckPrivs[i], skPrivs[i],
			overlaySlice, ctorSlice)
	}

	// Use the information collected to configure each node.
	for i := 0; i < K; i++ {
		for j := 0; j < K; j++ {
			if i != j {
				nodes[i].AddPeer(peers[j])
			}
		}
	}
	return
}
Example #8
0
// Version of the above which consumes a slice of strings.  XXX Copies the
// slice unnecessarily.
func ParseBNFromStrings(ss []string, whichType string) (bn *BaseNode, rest []string, err error) {
	var (
		name        string
		nodeID      *xi.NodeID
		commsPubKey *rsa.PublicKey
		sigPubKey   *rsa.PublicKey
		overlays    []xo.OverlayI
	)
	s, err := xc.NextNBLine(&ss)
	if err == nil {
		opener := fmt.Sprintf("%s {", whichType) // "peer {" or "node {"
		if s != opener {
			err = NotExpectedOpener
		}
	}
	if err == nil {
		s, err := xc.NextNBLine(&ss)
		if err == nil {
			if strings.HasPrefix(s, "name: ") {
				name = s[6:]
			} else {
				err = NotABaseNode
			}
		}
	}
	if err == nil {
		s, err = xc.NextNBLine(&ss)
		if err == nil {
			if strings.HasPrefix(s, "nodeID: ") {
				var val []byte
				val, err = hex.DecodeString(s[8:])
				if err == nil {
					nodeID, err = xi.NewNodeID(val)
				}
			} else {
				err = NotABaseNode
			}
		}
	}
	if err == nil {
		s, err = xc.NextNBLine(&ss)
		if err == nil {
			if strings.HasPrefix(s, "commsPubKey: ") {
				var ckPEM []byte
				ckPEM, err = xc.CollectPEMRSAPublicKey(s[13:], &ss)
				if err == nil {
					commsPubKey, err = xc.RSAPubKeyFromPEM(ckPEM)
				}
			} else {
				// DEBUG
				fmt.Printf("\ndon't see commsPubKey\n")
				// END
				err = NotABaseNode
			}
		}
	}
	if err == nil {
		s, err = xc.NextNBLine(&ss)
		if err == nil {
			if strings.HasPrefix(s, "sigPubKey: ") {
				var skPEM []byte
				skPEM, err = xc.CollectPEMRSAPublicKey(s[11:], &ss)
				if err == nil {
					sigPubKey, err = xc.RSAPubKeyFromPEM(skPEM)
				}
			} else {
				// DEBUG
				fmt.Printf("\ndon't see sigPubKey\n")
				// END
				err = NotABaseNode
			}
		}
	}
	if err == nil {
		s, err = xc.NextNBLine(&ss)
		if err == nil {
			if s == "overlays {" {
				for {
					s, err = xc.NextNBLine(&ss)
					if err == nil {
						if s == "" { // end of strings
							err = NotABaseNode
							break
						} else if s == "}" {
							prepend := []string{s}
							ss = append(prepend, ss...)
							break
						}
					}
					var o xo.OverlayI
					o, err = xo.Parse(s)
					if err == nil {
						overlays = append(overlays, o)
					}
				}
			} else {
				err = NotABaseNode
			}
		}
	}
	if err == nil {
		s, err = xc.NextNBLine(&ss)
		if err == nil {
			if s != "}" {
				err = NotABaseNode
			}
		}
	}
	if err == nil {
		var bn = BaseNode{name, nodeID, commsPubKey, sigPubKey, overlays}
		return &bn, ss, nil
	} else {
		return nil, nil, err
	}
}