func TestToAndFromNetMessage(t *testing.T) {
original := New(true)
original.AddBlock(blocks.NewBlock([]byte("W")))
original.AddBlock(blocks.NewBlock([]byte("E")))
original.AddBlock(blocks.NewBlock([]byte("F")))
original.AddBlock(blocks.NewBlock([]byte("M")))
buf := new(bytes.Buffer)
if err := original.ToNet(buf); err != nil {
t.Fatal(err)
}
m2, err := FromNet(buf)
if err != nil {
t.Fatal(err)
}
keys := make(map[key.Key]bool)
for _, b := range m2.Blocks() {
keys[b.Key()] = true
}
for _, b := range original.Blocks() {
if _, ok := keys[b.Key()]; !ok {
t.Fail()
}
}
}
func TestPutManyAddsToBloom(t *testing.T) {
bs := NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
ctx, _ := context.WithTimeout(context.Background(), 1*time.Second)
cachedbs, err := testBloomCached(bs, ctx)
select {
case <-cachedbs.rebuildChan:
case <-ctx.Done():
t.Fatalf("Timeout wating for rebuild: %d", cachedbs.bloom.ElementsAdded())
}
block1 := blocks.NewBlock([]byte("foo"))
block2 := blocks.NewBlock([]byte("bar"))
cachedbs.PutMany([]blocks.Block{block1})
has, err := cachedbs.Has(block1.Cid())
if err != nil {
t.Fatal(err)
}
if has == false {
t.Fatal("added block is reported missing")
}
has, err = cachedbs.Has(block2.Cid())
if err != nil {
t.Fatal(err)
}
if has == true {
t.Fatal("not added block is reported to be in blockstore")
}
}
func TestHasIsBloomCached(t *testing.T) {
cd := &callbackDatastore{f: func() {}, ds: ds.NewMapDatastore()}
bs := NewBlockstore(syncds.MutexWrap(cd))
for i := 0; i < 1000; i++ {
bs.Put(blocks.NewBlock([]byte(fmt.Sprintf("data: %d", i))))
}
ctx, _ := context.WithTimeout(context.Background(), 1*time.Second)
cachedbs, err := testBloomCached(bs, ctx)
if err != nil {
t.Fatal(err)
}
select {
case <-cachedbs.rebuildChan:
case <-ctx.Done():
t.Fatalf("Timeout wating for rebuild: %d", cachedbs.bloom.ElementsAdded())
}
cacheFails := 0
cd.SetFunc(func() {
cacheFails++
})
for i := 0; i < 1000; i++ {
cachedbs.Has(blocks.NewBlock([]byte(fmt.Sprintf("data: %d", i+2000))).Key())
}
if float64(cacheFails)/float64(1000) > float64(0.05) {
t.Fatal("Bloom filter has cache miss rate of more than 5%")
}
}
func TestHasIsBloomCached(t *testing.T) {
cd := &callbackDatastore{f: func() {}, ds: ds.NewMapDatastore()}
bs := NewBlockstore(syncds.MutexWrap(cd))
for i := 0; i < 1000; i++ {
bs.Put(blocks.NewBlock([]byte(fmt.Sprintf("data: %d", i))))
}
ctx, _ := context.WithTimeout(context.Background(), 1*time.Second)
cachedbs, err := testBloomCached(bs, ctx)
if err != nil {
t.Fatal(err)
}
select {
case <-cachedbs.rebuildChan:
case <-ctx.Done():
t.Fatalf("Timeout wating for rebuild: %d", cachedbs.bloom.ElementsAdded())
}
cacheFails := 0
cd.SetFunc(func() {
cacheFails++
})
for i := 0; i < 1000; i++ {
cachedbs.Has(blocks.NewBlock([]byte(fmt.Sprintf("data: %d", i+2000))).Cid())
}
if float64(cacheFails)/float64(1000) > float64(0.05) {
t.Fatal("Bloom filter has cache miss rate of more than 5%")
}
cacheFails = 0
block := blocks.NewBlock([]byte("newBlock"))
cachedbs.PutMany([]blocks.Block{block})
if cacheFails != 2 {
t.Fatalf("expected two datastore hits: %d", cacheFails)
}
cachedbs.Put(block)
if cacheFails != 3 {
t.Fatalf("expected datastore hit: %d", cacheFails)
}
if has, err := cachedbs.Has(block.Cid()); !has || err != nil {
t.Fatal("has gave wrong response")
}
bl, err := cachedbs.Get(block.Cid())
if bl.String() != block.String() {
t.Fatal("block data doesn't match")
}
if err != nil {
t.Fatal("there should't be an error")
}
}
func TestSendMessageAsyncButWaitForResponse(t *testing.T) {
net := VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0))
responderPeer := testutil.RandIdentityOrFatal(t)
waiter := net.Adapter(testutil.RandIdentityOrFatal(t))
responder := net.Adapter(responderPeer)
var wg sync.WaitGroup
wg.Add(1)
expectedStr := "received async"
responder.SetDelegate(lambda(func(
ctx context.Context,
fromWaiter peer.ID,
msgFromWaiter bsmsg.BitSwapMessage) {
msgToWaiter := bsmsg.New(true)
msgToWaiter.AddBlock(blocks.NewBlock([]byte(expectedStr)))
waiter.SendMessage(ctx, fromWaiter, msgToWaiter)
}))
waiter.SetDelegate(lambda(func(
ctx context.Context,
fromResponder peer.ID,
msgFromResponder bsmsg.BitSwapMessage) {
// TODO assert that this came from the correct peer and that the message contents are as expected
ok := false
for _, b := range msgFromResponder.Blocks() {
if string(b.Data) == expectedStr {
wg.Done()
ok = true
}
}
if !ok {
t.Fatal("Message not received from the responder")
}
}))
messageSentAsync := bsmsg.New(true)
messageSentAsync.AddBlock(blocks.NewBlock([]byte("data")))
errSending := waiter.SendMessage(
context.Background(), responderPeer.ID(), messageSentAsync)
if errSending != nil {
t.Fatal(errSending)
}
wg.Wait() // until waiter delegate function is executed
}
func BenchmarkHandle10KBlocks(b *testing.B) {
bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
var testdata []*blocks.Block
for i := 0; i < 10000; i++ {
testdata = append(testdata, blocks.NewBlock([]byte(string(i))))
}
b.ResetTimer()
b.SetBytes(10000)
for i := 0; i < b.N; i++ {
b.StopTimer()
w := NewWorker(offline.Exchange(bstore), Config{
NumWorkers: 1,
ClientBufferSize: 0,
WorkerBufferSize: 0,
})
b.StartTimer()
for _, block := range testdata {
if err := w.HasBlock(block); err != nil {
b.Fatal(err)
}
}
b.StopTimer()
w.Close()
b.StartTimer()
}
}
func (bs *blockstore) Get(k key.Key) (blocks.Block, error) {
if k == "" {
return nil, ErrNotFound
}
maybeData, err := bs.datastore.Get(k.DsKey())
if err == ds.ErrNotFound {
return nil, ErrNotFound
}
if err != nil {
return nil, err
}
bdata, ok := maybeData.([]byte)
if !ok {
return nil, ValueTypeMismatch
}
if bs.rehash {
rb := blocks.NewBlock(bdata)
if rb.Key() != k {
return nil, ErrHashMismatch
} else {
return rb, nil
}
} else {
return blocks.NewBlockWithHash(bdata, mh.Multihash(k))
}
}
func BenchmarkRandomBlockWrites(n *core.IpfsNode, cfg *BenchCfg) error {
buf := make([]byte, cfg.Blocksize)
read := randbo.New()
var keys []key.Key
f := func(b *testing.B) {
b.SetBytes(cfg.Blocksize)
for i := 0; i < b.N; i++ {
read.Read(buf)
blk := blocks.NewBlock(buf)
k, err := n.Blocks.AddBlock(blk)
if err != nil {
b.Fatal(err)
}
keys = append(keys, k)
}
}
br := testing.Benchmark(f)
fmt.Printf("RandomBlockWrites:\t%s\n", br)
// clean up
for _, k := range keys {
err := n.Blocks.DeleteBlock(k)
if err != nil {
return err
}
}
return nil
}
func BenchmarkBlockRewrites(n *core.IpfsNode, cfg *BenchCfg) error {
buf := make([]byte, cfg.Blocksize)
randbo.New().Read(buf)
blk := blocks.NewBlock(buf)
// write the block first, before starting the benchmark.
// we're just looking at the time it takes to write a block thats already
// been written
k, err := n.Blocks.AddBlock(blk)
if err != nil {
return err
}
f := func(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := n.Blocks.AddBlock(blk)
if err != nil {
b.Fatal(err)
}
}
}
br := testing.Benchmark(f)
fmt.Printf("BlockRewrites:\t\t%s\n", br)
// clean up
err = n.Blocks.DeleteBlock(k)
if err != nil {
return err
}
return nil
}
func TestConsistentAccounting(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sender := newEngine(ctx, "Ernie")
receiver := newEngine(ctx, "Bert")
// Send messages from Ernie to Bert
for i := 0; i < 1000; i++ {
m := message.New(false)
content := []string{"this", "is", "message", "i"}
m.AddBlock(blocks.NewBlock([]byte(strings.Join(content, " "))))
sender.Engine.MessageSent(receiver.Peer, m)
receiver.Engine.MessageReceived(sender.Peer, m)
}
// Ensure sender records the change
if sender.Engine.numBytesSentTo(receiver.Peer) == 0 {
t.Fatal("Sent bytes were not recorded")
}
// Ensure sender and receiver have the same values
if sender.Engine.numBytesSentTo(receiver.Peer) != receiver.Engine.numBytesReceivedFrom(sender.Peer) {
t.Fatal("Inconsistent book-keeping. Strategies don't agree")
}
// Ensure sender didn't record receving anything. And that the receiver
// didn't record sending anything
if receiver.Engine.numBytesSentTo(sender.Peer) != 0 || sender.Engine.numBytesReceivedFrom(receiver.Peer) != 0 {
t.Fatal("Bert didn't send bytes to Ernie")
}
}
func TestGetBlockFromPeerAfterPeerAnnounces(t *testing.T) {
net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay))
block := blocks.NewBlock([]byte("block"))
g := NewTestSessionGenerator(net)
defer g.Close()
peers := g.Instances(2)
hasBlock := peers[0]
defer hasBlock.Exchange.Close()
if err := hasBlock.Exchange.HasBlock(block); err != nil {
t.Fatal(err)
}
wantsBlock := peers[1]
defer wantsBlock.Exchange.Close()
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
received, err := wantsBlock.Exchange.GetBlock(ctx, block.Key())
if err != nil {
t.Log(err)
t.Fatal("Expected to succeed")
}
if !bytes.Equal(block.Data, received.Data) {
t.Fatal("Data doesn't match")
}
}
func execute(cmdString string) error {
// Check for comment
if cmdString[0] == '#' {
return nil
}
if strings.Contains(cmdString, "->") {
return connectCmd(cmdString)
}
// Check for dummy file command, could move some of this into a method in dummyfiles.go
split := strings.Split(cmdString, " ")
if split[0] == "create_dummy_files" {
numfiles, err := strconv.Atoi(split[1])
if err != nil {
log.Fatalf("Line %d: Invalid argument for create_dummy_files.", currLine)
}
filesize, err := strconv.Atoi(split[2])
if err != nil {
log.Fatalf("Line %d: Invalid argument for create_dummy_files.", currLine)
}
createDummyFiles(numfiles, filesize)
return nil
}
command := split[1]
arg := split[2]
// Command in form "node# get/put/leave arg"
nodes := getRange(split[0])
switch command {
case "putb":
return putCmd(nodes, blocks.NewBlock([]byte(arg)))
case "put":
return putFileCmd(nodes, arg)
case "getb":
return getCmd(nodes, blocks.NewBlock([]byte(arg)))
case "get":
return getFileCmd(nodes, arg)
case "leave":
return leaveCmd(nodes, arg)
default:
return fmt.Errorf("Error on line %d: expected get/put/leave, found %s.", currLine, command)
}
}
func partnerWants(e *Engine, keys []string, partner peer.ID) {
add := message.New(false)
for i, letter := range keys {
block := blocks.NewBlock([]byte(letter))
add.AddEntry(block.Cid(), math.MaxInt32-i)
}
e.MessageReceived(partner, add)
}
func partnerCancels(e *Engine, keys []string, partner peer.ID) {
cancels := message.New(false)
for _, k := range keys {
block := blocks.NewBlock([]byte(k))
cancels.Cancel(block.Cid())
}
e.MessageReceived(partner, cancels)
}
func newMessageFromProto(pbm pb.Message) BitSwapMessage {
m := newMsg(pbm.GetWantlist().GetFull())
for _, e := range pbm.GetWantlist().GetEntries() {
m.addEntry(key.Key(e.GetBlock()), int(e.GetPriority()), e.GetCancel())
}
for _, d := range pbm.GetBlocks() {
b := blocks.NewBlock(d)
m.AddBlock(b)
}
return m
}
func TestCarryOnWhenDeadlineExpires(t *testing.T) {
impossibleDeadline := time.Nanosecond
fastExpiringCtx, _ := context.WithTimeout(context.Background(), impossibleDeadline)
n := New()
defer n.Shutdown()
block := blocks.NewBlock([]byte("A Missed Connection"))
blockChannel := n.Subscribe(fastExpiringCtx, block.Key())
assertBlockChannelNil(t, blockChannel)
}
func checkHandledInOrder(t *testing.T, e *Engine, keys []string) error {
for _, k := range keys {
next := <-e.Outbox()
envelope := <-next
received := envelope.Block
expected := blocks.NewBlock([]byte(k))
if !received.Cid().Equals(expected.Cid()) {
return errors.New(fmt.Sprintln("received", string(received.RawData()), "expected", string(expected.RawData())))
}
}
return nil
}
func TestPutBlockCmd(t *testing.T) {
configure("node_count:1", nil)
net, peers = createTestNetwork()
testBlock := blocks.NewBlock([]byte("testblock"))
err := putCmd([]int{0}, testBlock)
check(err)
if !checkHasBlock([]int{0}, testBlock.Key()) {
t.Error("Peer 0 never add block to blockstore.")
}
}
func TestSubscribeMany(t *testing.T) {
e1 := blocks.NewBlock([]byte("1"))
e2 := blocks.NewBlock([]byte("2"))
n := New()
defer n.Shutdown()
ch := n.Subscribe(context.Background(), e1.Key(), e2.Key())
n.Publish(e1)
r1, ok := <-ch
if !ok {
t.Fatal("didn't receive first expected block")
}
assertBlocksEqual(t, e1, r1)
n.Publish(e2)
r2, ok := <-ch
if !ok {
t.Fatal("didn't receive second expected block")
}
assertBlocksEqual(t, e2, r2)
}
func TestValueTypeMismatch(t *testing.T) {
block := blocks.NewBlock([]byte("some data"))
datastore := ds.NewMapDatastore()
k := BlockPrefix.Child(block.Key().DsKey())
datastore.Put(k, "data that isn't a block!")
blockstore := NewBlockstore(ds_sync.MutexWrap(datastore))
_, err := blockstore.Get(block.Key())
if err != ValueTypeMismatch {
t.Fatal(err)
}
}
func TestHasBlockReturnsNil(t *testing.T) {
store := bstore()
ex := Exchange(store)
block := blocks.NewBlock([]byte("data"))
err := ex.HasBlock(context.Background(), block)
if err != nil {
t.Fail()
}
if _, err := store.Get(block.Key()); err != nil {
t.Fatal(err)
}
}
func TestRuntimeHashing(t *testing.T) {
bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
bl := blocks.NewBlock([]byte("some data"))
blBad, err := blocks.NewBlockWithHash([]byte("some other data"), bl.Key().ToMultihash())
if err != nil {
t.Fatal("Debug is enabled")
}
bs.Put(blBad)
bs.RuntimeHashing(true)
if _, err := bs.Get(bl.Key()); err != ErrHashMismatch {
t.Fatalf("Expected '%v' got '%v'\n", ErrHashMismatch, err)
}
}
func TestDuplicates(t *testing.T) {
b1 := blocks.NewBlock([]byte("1"))
b2 := blocks.NewBlock([]byte("2"))
n := New()
defer n.Shutdown()
ch := n.Subscribe(context.Background(), b1.Key(), b2.Key())
n.Publish(b1)
blockRecvd, ok := <-ch
if !ok {
t.Fail()
}
assertBlocksEqual(t, b1, blockRecvd)
n.Publish(b1) // ignored duplicate
n.Publish(b2)
blockRecvd, ok = <-ch
if !ok {
t.Fail()
}
assertBlocksEqual(t, b2, blockRecvd)
}
func TestPublishSubscribe(t *testing.T) {
blockSent := blocks.NewBlock([]byte("Greetings from The Interval"))
n := New()
defer n.Shutdown()
ch := n.Subscribe(context.Background(), blockSent.Key())
n.Publish(blockSent)
blockRecvd, ok := <-ch
if !ok {
t.Fail()
}
assertBlocksEqual(t, blockRecvd, blockSent)
}
func TestDuplicates(t *testing.T) {
b := blocks.NewBlock([]byte("foo"))
msg := New(true)
msg.AddEntry(b.Key(), 1)
msg.AddEntry(b.Key(), 1)
if len(msg.Wantlist()) != 1 {
t.Fatal("Duplicate in BitSwapMessage")
}
msg.AddBlock(b)
msg.AddBlock(b)
if len(msg.Blocks()) != 1 {
t.Fatal("Duplicate in BitSwapMessage")
}
}
func TestElideDuplicateWrite(t *testing.T) {
cd := &callbackDatastore{f: func() {}, ds: ds.NewMapDatastore()}
bs := NewBlockstore(syncds.MutexWrap(cd))
cachedbs, err := WriteCached(bs, 1)
if err != nil {
t.Fatal(err)
}
b1 := blocks.NewBlock([]byte("foo"))
cachedbs.Put(b1)
cd.SetFunc(func() {
t.Fatal("write hit the datastore")
})
cachedbs.Put(b1)
}
func newBlockStoreWithKeys(t *testing.T, d ds.Datastore, N int) (Blockstore, []key.Key) {
if d == nil {
d = ds.NewMapDatastore()
}
bs := NewBlockstore(ds_sync.MutexWrap(d))
keys := make([]key.Key, N)
for i := 0; i < N; i++ {
block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
err := bs.Put(block)
if err != nil {
t.Fatal(err)
}
keys[i] = block.Key()
}
return bs, keys
}
func TestPutThenGetBlock(t *testing.T) {
bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
block := blocks.NewBlock([]byte("some data"))
err := bs.Put(block)
if err != nil {
t.Fatal(err)
}
blockFromBlockstore, err := bs.Get(block.Key())
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(block.Data(), blockFromBlockstore.Data()) {
t.Fail()
}
}
func TestBlocks(t *testing.T) {
bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
bs := New(bstore, offline.Exchange(bstore))
defer bs.Close()
_, err := bs.GetBlock(context.Background(), key.Key(""))
if err != ErrNotFound {
t.Error("Empty String Key should error", err)
}
b := blocks.NewBlock([]byte("beep boop"))
h := u.Hash([]byte("beep boop"))
if !bytes.Equal(b.Multihash(), h) {
t.Error("Block Multihash and data multihash not equal")
}
if b.Key() != key.Key(h) {
t.Error("Block key and data multihash key not equal")
}
k, err := bs.AddBlock(b)
if err != nil {
t.Error("failed to add block to BlockService", err)
return
}
if k != b.Key() {
t.Error("returned key is not equal to block key", err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
b2, err := bs.GetBlock(ctx, b.Key())
if err != nil {
t.Error("failed to retrieve block from BlockService", err)
return
}
if b.Key() != b2.Key() {
t.Error("Block keys not equal.")
}
if !bytes.Equal(b.Data(), b2.Data()) {
t.Error("Block data is not equal.")
}
}
func TestBlocks(t *testing.T) {
bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
bs, err := New(bstore, offline.Exchange(bstore))
if err != nil {
t.Error("failed to construct block service", err)
return
}
defer bs.Close()
b := blocks.NewBlock([]byte("beep boop"))
h := u.Hash([]byte("beep boop"))
if !bytes.Equal(b.Multihash, h) {
t.Error("Block Multihash and data multihash not equal")
}
if b.Key() != key.Key(h) {
t.Error("Block key and data multihash key not equal")
}
k, err := bs.AddBlock(b)
if err != nil {
t.Error("failed to add block to BlockService", err)
return
}
if k != b.Key() {
t.Error("returned key is not equal to block key", err)
}
ctx, _ := context.WithTimeout(context.TODO(), time.Second*5)
b2, err := bs.GetBlock(ctx, b.Key())
if err != nil {
t.Error("failed to retrieve block from BlockService", err)
return
}
if b.Key() != b2.Key() {
t.Error("Block keys not equal.")
}
if !bytes.Equal(b.Data, b2.Data) {
t.Error("Block data is not equal.")
}
}
