// Check that torrent Info is obtained from the metainfo file cache.
func TestAddTorrentMetainfoInCache(t *testing.T) {
cfg := TestingConfig
cfg.DisableMetainfoCache = false
cfg.ConfigDir, _ = ioutil.TempDir(os.TempDir(), "")
defer os.RemoveAll(cfg.ConfigDir)
cl, err := NewClient(&cfg)
require.NoError(t, err)
defer cl.Close()
dir, mi := testutil.GreetingTestTorrent()
defer os.RemoveAll(dir)
tt, new, err := cl.AddTorrentSpec(TorrentSpecFromMetaInfo(mi))
require.NoError(t, err)
require.True(t, new)
require.NotNil(t, tt.Info())
_, err = os.Stat(filepath.Join(cfg.ConfigDir, "torrents", fmt.Sprintf("%x.torrent", mi.Info.Hash)))
require.NoError(t, err)
// Contains only the infohash.
var ts TorrentSpec
missinggo.CopyExact(&ts.InfoHash, mi.Info.Hash)
_, ok := cl.Torrent(ts.InfoHash)
require.True(t, ok)
tt.Drop()
_, ok = cl.Torrent(ts.InfoHash)
require.False(t, ok)
tt, new, err = cl.AddTorrentSpec(&ts)
require.NoError(t, err)
require.True(t, new)
// Obtained from the metainfo cache.
require.NotNil(t, tt.Info())
}
// Called when metadata for a torrent becomes available.
func (t *torrent) setMetadata(md *metainfo.Info, infoBytes []byte, eventLocker sync.Locker) (err error) {
err = validateInfo(md)
if err != nil {
err = fmt.Errorf("bad info: %s", err)
return
}
t.Info = md
t.length = 0
for _, f := range t.Info.UpvertedFiles() {
t.length += f.Length
}
t.MetaData = infoBytes
t.metadataHave = nil
hashes := infoPieceHashes(md)
t.Pieces = make([]piece, len(hashes))
for i, hash := range hashes {
piece := &t.Pieces[i]
piece.Event.L = eventLocker
piece.noPendingWrites.L = &piece.pendingWritesMutex
missinggo.CopyExact(piece.Hash[:], hash)
}
for _, conn := range t.Conns {
t.initRequestOrdering(conn)
if err := conn.setNumPieces(t.numPieces()); err != nil {
log.Printf("closing connection: %s", err)
conn.Close()
}
}
return
}
func torrentFileInfoHash(fileName string) (ih torrent.InfoHash, ok bool) {
mi, _ := metainfo.LoadFromFile(fileName)
if mi == nil {
return
}
missinggo.CopyExact(ih[:], mi.Info.Hash)
ok = true
return
}
// Called when metadata for a torrent becomes available.
func (t *Torrent) setInfoBytes(b []byte) error {
if t.haveInfo() {
return nil
}
var ie *metainfo.InfoEx
err := bencode.Unmarshal(b, &ie)
if err != nil {
return fmt.Errorf("error unmarshalling info bytes: %s", err)
}
if ie.Hash() != t.infoHash {
return errors.New("info bytes have wrong hash")
}
err = validateInfo(&ie.Info)
if err != nil {
return fmt.Errorf("bad info: %s", err)
}
defer t.updateWantPeersEvent()
t.info = ie
t.cl.event.Broadcast()
t.gotMetainfo.Set()
t.storage, err = t.storageOpener.OpenTorrent(t.info)
if err != nil {
return fmt.Errorf("error opening torrent storage: %s", err)
}
t.length = 0
for _, f := range t.info.UpvertedFiles() {
t.length += f.Length
}
t.metadataBytes = b
t.metadataCompletedChunks = nil
hashes := infoPieceHashes(&t.info.Info)
t.pieces = make([]piece, len(hashes))
for i, hash := range hashes {
piece := &t.pieces[i]
piece.t = t
piece.index = i
piece.noPendingWrites.L = &piece.pendingWritesMutex
missinggo.CopyExact(piece.Hash[:], hash)
}
for _, conn := range t.conns {
if err := conn.setNumPieces(t.numPieces()); err != nil {
log.Printf("closing connection: %s", err)
conn.Close()
}
}
for i := range t.pieces {
t.updatePieceCompletion(i)
t.pieces[i].QueuedForHash = true
}
go func() {
for i := range t.pieces {
t.verifyPiece(i)
}
}()
return nil
}
func scanDir(dirName string) (ee map[torrent.InfoHash]entity) {
d, err := os.Open(dirName)
if err != nil {
log.Print(err)
return
}
defer d.Close()
names, err := d.Readdirnames(-1)
if err != nil {
log.Print(err)
return
}
ee = make(map[torrent.InfoHash]entity, len(names))
addEntity := func(e entity) {
e0, ok := ee[e.InfoHash]
if ok {
if e0.MagnetURI == "" || len(e.MagnetURI) < len(e0.MagnetURI) {
return
}
}
ee[e.InfoHash] = e
}
for _, n := range names {
fullName := filepath.Join(dirName, n)
switch filepath.Ext(n) {
case ".torrent":
ih, ok := torrentFileInfoHash(fullName)
if !ok {
break
}
e := entity{
TorrentFilePath: fullName,
}
missinggo.CopyExact(&e.InfoHash, ih)
addEntity(e)
case ".magnet":
uris, err := magnetFileURIs(fullName)
if err != nil {
log.Print(err)
break
}
for _, uri := range uris {
m, err := torrent.ParseMagnetURI(uri)
if err != nil {
log.Printf("error parsing %q in file %q: %s", uri, fullName, err)
continue
}
addEntity(entity{
InfoHash: m.InfoHash,
MagnetURI: uri,
})
}
}
}
return
}
func (this *InfoEx) UnmarshalBencode(data []byte) error {
this.Bytes = append(make([]byte, 0, len(data)), data...)
h := sha1.New()
_, err := h.Write(this.Bytes)
if err != nil {
panic(err)
}
this.Hash = new(Hash)
missinggo.CopyExact(this.Hash, h.Sum(nil))
return bencode.Unmarshal(data, &this.Info)
}
func (t *Torrent) makePieces() {
hashes := infoPieceHashes(&t.info.Info)
t.pieces = make([]piece, len(hashes))
for i, hash := range hashes {
piece := &t.pieces[i]
piece.t = t
piece.index = i
piece.noPendingWrites.L = &piece.pendingWritesMutex
missinggo.CopyExact(piece.Hash[:], hash)
}
}
func (cni *NodeInfo) UnmarshalCompactIPv4(b []byte) error {
if len(b) != CompactIPv4NodeInfoLen {
return errors.New("expected 26 bytes")
}
missinggo.CopyExact(cni.ID[:], b[:20])
cni.Addr = newDHTAddr(&net.UDPAddr{
IP: append(make([]byte, 0, 4), b[20:24]...),
Port: int(binary.BigEndian.Uint16(b[24:26])),
})
return nil
}
func (cni *NodeInfo) UnmarshalCompactIPv4(b []byte) error {
if len(b) != 26 {
return errors.New("expected 26 bytes")
}
missinggo.CopyExact(cni.ID[:], b[:20])
cni.Addr = newDHTAddr(&net.UDPAddr{
IP: net.IPv4(b[20], b[21], b[22], b[23]),
Port: int(binary.BigEndian.Uint16(b[24:26])),
})
return nil
}
func (t *torrent) hashPiece(piece pp.Integer) (ps pieceSum) {
hash := pieceHash.New()
p := t.Pieces[piece]
p.pendingWritesMutex.Lock()
for p.pendingWrites != 0 {
p.noPendingWrites.Wait()
}
p.pendingWritesMutex.Unlock()
t.data.WriteSectionTo(hash, int64(piece)*t.Info.PieceLength, t.Info.PieceLength)
missinggo.CopyExact(ps[:], hash.Sum(nil))
return
}
func (t *Torrent) hashPiece(piece int) (ret metainfo.Hash) {
hash := pieceHash.New()
p := &t.pieces[piece]
p.waitNoPendingWrites()
ip := t.info.Piece(piece)
pl := ip.Length()
n, err := io.Copy(hash, io.NewSectionReader(t.pieces[piece].Storage(), 0, pl))
if n == pl {
missinggo.CopyExact(&ret, hash.Sum(nil))
return
}
if err != io.ErrUnexpectedEOF {
log.Printf("unexpected error hashing piece with %T: %s", t.storage, err)
}
return
}
func TestTorrentDroppedBeforeGotInfo(t *testing.T) {
dir, mi := testutil.GreetingTestTorrent()
os.RemoveAll(dir)
cl, _ := NewClient(&TestingConfig)
defer cl.Close()
var ts TorrentSpec
missinggo.CopyExact(&ts.InfoHash, mi.Info.Hash)
tt, _, _ := cl.AddTorrentSpec(&ts)
tt.Drop()
assert.EqualValues(t, 0, len(cl.Torrents()))
select {
case <-tt.GotInfo():
t.FailNow()
default:
}
}
// Called when metadata for a torrent becomes available.
func (t *Torrent) setMetadata(md *metainfo.Info, infoBytes []byte) (err error) {
err = validateInfo(md)
if err != nil {
err = fmt.Errorf("bad info: %s", err)
return
}
t.info = &metainfo.InfoEx{
Info: *md,
Bytes: infoBytes,
Hash: &t.infoHash,
}
t.storage, err = t.storageOpener.OpenTorrent(t.info)
if err != nil {
return
}
t.length = 0
for _, f := range t.info.UpvertedFiles() {
t.length += f.Length
}
t.metadataBytes = infoBytes
t.metadataCompletedChunks = nil
hashes := infoPieceHashes(md)
t.pieces = make([]piece, len(hashes))
for i, hash := range hashes {
piece := &t.pieces[i]
piece.t = t
piece.index = i
piece.noPendingWrites.L = &piece.pendingWritesMutex
missinggo.CopyExact(piece.Hash[:], hash)
}
for _, conn := range t.conns {
if err := conn.setNumPieces(t.numPieces()); err != nil {
log.Printf("closing connection: %s", err)
conn.Close()
}
}
for i := range t.pieces {
t.updatePieceCompletion(i)
t.pieces[i].QueuedForHash = true
}
go func() {
for i := range t.pieces {
t.verifyPiece(i)
}
}()
return
}
// Check that stuff is merged in subsequent AddTorrentSpec for the same
// infohash.
func TestAddTorrentSpecMerging(t *testing.T) {
cl, err := NewClient(&TestingConfig)
require.NoError(t, err)
defer cl.Close()
dir, mi := testutil.GreetingTestTorrent()
defer os.RemoveAll(dir)
var ts TorrentSpec
missinggo.CopyExact(&ts.InfoHash, mi.Info.Hash)
tt, new, err := cl.AddTorrentSpec(&ts)
require.NoError(t, err)
require.True(t, new)
require.Nil(t, tt.Info())
_, new, err = cl.AddTorrentSpec(TorrentSpecFromMetaInfo(mi))
require.NoError(t, err)
require.False(t, new)
require.NotNil(t, tt.Info())
}
func (t *torrent) hashPiece(piece int) (ps pieceSum) {
hash := pieceHash.New()
p := &t.Pieces[piece]
p.waitNoPendingWrites()
pl := t.Info.Piece(int(piece)).Length()
n, err := t.data.WriteSectionTo(hash, int64(piece)*t.Info.PieceLength, pl)
if err != nil {
if err != io.ErrUnexpectedEOF {
log.Printf("error hashing piece with %T: %s", t.data, err)
}
return
}
if n != pl {
panic(fmt.Sprintf("%T: %d != %d", t.data, n, pl))
}
missinggo.CopyExact(ps[:], hash.Sum(nil))
return
}
func TestTorrentInitialState(t *testing.T) {
dir, mi := testutil.GreetingTestTorrent()
defer os.RemoveAll(dir)
tor := newTorrent(func() (ih metainfo.Hash) {
missinggo.CopyExact(ih[:], mi.Info.Hash)
return
}())
tor.chunkSize = 2
tor.storageOpener = storage.NewFile(dir)
// Needed to lock for asynchronous piece verification.
tor.cl = new(Client)
err := tor.setMetadata(&mi.Info.Info, mi.Info.Bytes)
require.NoError(t, err)
require.Len(t, tor.pieces, 3)
tor.pendAllChunkSpecs(0)
assert.EqualValues(t, 3, tor.pieceNumPendingChunks(0))
assert.EqualValues(t, chunkSpec{4, 1}, chunkIndexSpec(2, tor.pieceLength(0), tor.chunkSize))
}
func TestTorrentInitialState(t *testing.T) {
dir, mi := testutil.GreetingTestTorrent()
defer os.RemoveAll(dir)
tor := newTorrent(func() (ih InfoHash) {
missinggo.CopyExact(ih[:], mi.Info.Hash)
return
}())
tor.chunkSize = 2
err := tor.setMetadata(&mi.Info.Info, mi.Info.Bytes)
if err != nil {
t.Fatal(err)
}
if len(tor.Pieces) != 3 {
t.Fatal("wrong number of pieces")
}
tor.pendAllChunkSpecs(0)
assert.EqualValues(t, 3, tor.pieceNumPendingChunks(0))
assert.EqualValues(t, chunkSpec{4, 1}, chunkIndexSpec(2, tor.pieceLength(0), tor.chunkSize))
}
func (t *torrent) hashPiece(piece pp.Integer) (ps pieceSum) {
hash := pieceHash.New()
p := t.Pieces[piece]
p.pendingWritesMutex.Lock()
for p.pendingWrites != 0 {
p.noPendingWrites.Wait()
}
p.pendingWritesMutex.Unlock()
pl := t.Info.Piece(int(piece)).Length()
n, err := t.data.WriteSectionTo(hash, int64(piece)*t.Info.PieceLength, pl)
if err != nil {
if err != io.ErrUnexpectedEOF {
log.Printf("error hashing piece with %T: %s", t.data, err)
}
return
}
if n != pl {
panic("lame")
}
missinggo.CopyExact(ps[:], hash.Sum(nil))
return
}
// ih is nil if we expect the peer to declare the InfoHash, such as when the
// peer initiated the connection. Returns ok if the handshake was successful,
// and err if there was an unexpected condition other than the peer simply
// abandoning the handshake.
func handshake(sock io.ReadWriter, ih *metainfo.Hash, peerID [20]byte, extensions peerExtensionBytes) (res handshakeResult, ok bool, err error) {
// Bytes to be sent to the peer. Should never block the sender.
postCh := make(chan []byte, 4)
// A single error value sent when the writer completes.
writeDone := make(chan error, 1)
// Performs writes to the socket and ensures posts don't block.
go handshakeWriter(sock, postCh, writeDone)
defer func() {
close(postCh) // Done writing.
if !ok {
return
}
if err != nil {
panic(err)
}
// Wait until writes complete before returning from handshake.
err = <-writeDone
if err != nil {
err = fmt.Errorf("error writing: %s", err)
}
}()
post := func(bb []byte) {
select {
case postCh <- bb:
default:
panic("mustn't block while posting")
}
}
post([]byte(pp.Protocol))
post(extensions[:])
if ih != nil { // We already know what we want.
post(ih[:])
post(peerID[:])
}
var b [68]byte
_, err = io.ReadFull(sock, b[:68])
if err != nil {
err = nil
return
}
if string(b[:20]) != pp.Protocol {
return
}
missinggo.CopyExact(&res.peerExtensionBytes, b[20:28])
missinggo.CopyExact(&res.Hash, b[28:48])
missinggo.CopyExact(&res.peerID, b[48:68])
peerExtensions.Add(hex.EncodeToString(res.peerExtensionBytes[:]), 1)
// TODO: Maybe we can just drop peers here if we're not interested. This
// could prevent them trying to reconnect, falsely believing there was
// just a problem.
if ih == nil { // We were waiting for the peer to tell us what they wanted.
post(res.Hash[:])
post(peerID[:])
}
ok = true
return
}
// Creates a new client.
func NewClient(cfg *Config) (cl *Client, err error) {
if cfg == nil {
cfg = &Config{}
}
defer func() {
if err != nil {
cl = nil
}
}()
cl = &Client{
halfOpenLimit: socketsPerTorrent,
config: *cfg,
defaultStorage: cfg.DefaultStorage,
dopplegangerAddrs: make(map[string]struct{}),
torrents: make(map[metainfo.Hash]*Torrent),
}
missinggo.CopyExact(&cl.extensionBytes, defaultExtensionBytes)
cl.event.L = &cl.mu
if cl.defaultStorage == nil {
cl.defaultStorage = storage.NewFile(cfg.DataDir)
}
if cfg.IPBlocklist != nil {
cl.ipBlockList = cfg.IPBlocklist
}
if cfg.PeerID != "" {
missinggo.CopyExact(&cl.peerID, cfg.PeerID)
} else {
o := copy(cl.peerID[:], bep20)
_, err = rand.Read(cl.peerID[o:])
if err != nil {
panic("error generating peer id")
}
}
cl.tcpListener, cl.utpSock, cl.listenAddr, err = listen(
!cl.config.DisableTCP,
!cl.config.DisableUTP,
func() string {
if cl.config.DisableIPv6 {
return "4"
} else {
return ""
}
}(),
cl.config.ListenAddr)
if err != nil {
return
}
if cl.tcpListener != nil {
go cl.acceptConnections(cl.tcpListener, false)
}
if cl.utpSock != nil {
go cl.acceptConnections(cl.utpSock, true)
}
if !cfg.NoDHT {
dhtCfg := cfg.DHTConfig
if dhtCfg.IPBlocklist == nil {
dhtCfg.IPBlocklist = cl.ipBlockList
}
dhtCfg.Addr = firstNonEmptyString(dhtCfg.Addr, cl.listenAddr, cl.config.ListenAddr)
if dhtCfg.Conn == nil && cl.utpSock != nil {
dhtCfg.Conn = cl.utpSock
}
cl.dHT, err = dht.NewServer(&dhtCfg)
if err != nil {
return
}
}
return
}
// Processes incoming bittorrent messages. The client lock is held upon entry
// and exit. Returning will end the connection.
func (cl *Client) connectionLoop(t *Torrent, c *connection) error {
decoder := pp.Decoder{
R: bufio.NewReader(c.rw),
MaxLength: 256 * 1024,
}
for {
cl.mu.Unlock()
var msg pp.Message
err := decoder.Decode(&msg)
cl.mu.Lock()
if cl.closed.IsSet() || c.closed.IsSet() || err == io.EOF {
return nil
}
if err != nil {
return err
}
c.lastMessageReceived = time.Now()
if msg.Keepalive {
receivedKeepalives.Add(1)
continue
}
receivedMessageTypes.Add(strconv.FormatInt(int64(msg.Type), 10), 1)
switch msg.Type {
case pp.Choke:
c.PeerChoked = true
c.Requests = nil
// We can then reset our interest.
c.updateRequests()
case pp.Reject:
cl.connDeleteRequest(t, c, newRequest(msg.Index, msg.Begin, msg.Length))
c.updateRequests()
case pp.Unchoke:
c.PeerChoked = false
cl.peerUnchoked(t, c)
case pp.Interested:
c.PeerInterested = true
cl.upload(t, c)
case pp.NotInterested:
c.PeerInterested = false
c.Choke()
case pp.Have:
err = c.peerSentHave(int(msg.Index))
case pp.Request:
if c.Choked {
break
}
if !c.PeerInterested {
err = errors.New("peer sent request but isn't interested")
break
}
if !t.havePiece(msg.Index.Int()) {
// This isn't necessarily them screwing up. We can drop pieces
// from our storage, and can't communicate this to peers
// except by reconnecting.
requestsReceivedForMissingPieces.Add(1)
err = errors.New("peer requested piece we don't have")
break
}
if c.PeerRequests == nil {
c.PeerRequests = make(map[request]struct{}, maxRequests)
}
c.PeerRequests[newRequest(msg.Index, msg.Begin, msg.Length)] = struct{}{}
cl.upload(t, c)
case pp.Cancel:
req := newRequest(msg.Index, msg.Begin, msg.Length)
if !c.PeerCancel(req) {
unexpectedCancels.Add(1)
}
case pp.Bitfield:
err = c.peerSentBitfield(msg.Bitfield)
case pp.HaveAll:
err = c.peerSentHaveAll()
case pp.HaveNone:
err = c.peerSentHaveNone()
case pp.Piece:
cl.downloadedChunk(t, c, &msg)
case pp.Extended:
switch msg.ExtendedID {
case pp.HandshakeExtendedID:
// TODO: Create a bencode struct for this.
var d map[string]interface{}
err = bencode.Unmarshal(msg.ExtendedPayload, &d)
if err != nil {
err = fmt.Errorf("error decoding extended message payload: %s", err)
break
}
// log.Printf("got handshake from %q: %#v", c.Socket.RemoteAddr().String(), d)
if reqq, ok := d["reqq"]; ok {
if i, ok := reqq.(int64); ok {
c.PeerMaxRequests = int(i)
}
}
if v, ok := d["v"]; ok {
c.PeerClientName = v.(string)
}
m, ok := d["m"]
if !ok {
err = errors.New("handshake missing m item")
break
}
mTyped, ok := m.(map[string]interface{})
if !ok {
err = errors.New("handshake m value is not dict")
break
}
if c.PeerExtensionIDs == nil {
c.PeerExtensionIDs = make(map[string]byte, len(mTyped))
}
for name, v := range mTyped {
id, ok := v.(int64)
if !ok {
log.Printf("bad handshake m item extension ID type: %T", v)
continue
}
if id == 0 {
delete(c.PeerExtensionIDs, name)
} else {
if c.PeerExtensionIDs[name] == 0 {
supportedExtensionMessages.Add(name, 1)
}
c.PeerExtensionIDs[name] = byte(id)
}
}
metadata_sizeUntyped, ok := d["metadata_size"]
if ok {
metadata_size, ok := metadata_sizeUntyped.(int64)
if !ok {
log.Printf("bad metadata_size type: %T", metadata_sizeUntyped)
} else {
err = t.setMetadataSize(metadata_size)
if err != nil {
err = fmt.Errorf("error setting metadata size to %d", metadata_size)
break
}
}
}
if _, ok := c.PeerExtensionIDs["ut_metadata"]; ok {
c.requestPendingMetadata()
}
case metadataExtendedId:
err = cl.gotMetadataExtensionMsg(msg.ExtendedPayload, t, c)
if err != nil {
err = fmt.Errorf("error handling metadata extension message: %s", err)
}
case pexExtendedId:
if cl.config.DisablePEX {
break
}
var pexMsg peerExchangeMessage
err = bencode.Unmarshal(msg.ExtendedPayload, &pexMsg)
if err != nil {
err = fmt.Errorf("error unmarshalling PEX message: %s", err)
break
}
go func() {
cl.mu.Lock()
t.addPeers(func() (ret []Peer) {
for i, cp := range pexMsg.Added {
p := Peer{
IP: make([]byte, 4),
Port: cp.Port,
Source: peerSourcePEX,
}
if i < len(pexMsg.AddedFlags) && pexMsg.AddedFlags[i]&0x01 != 0 {
p.SupportsEncryption = true
}
missinggo.CopyExact(p.IP, cp.IP[:])
ret = append(ret, p)
}
return
}())
cl.mu.Unlock()
}()
default:
err = fmt.Errorf("unexpected extended message ID: %v", msg.ExtendedID)
}
if err != nil {
// That client uses its own extension IDs for outgoing message
// types, which is incorrect.
if bytes.HasPrefix(c.PeerID[:], []byte("-SD0100-")) ||
strings.HasPrefix(string(c.PeerID[:]), "-XL0012-") {
return nil
}
}
case pp.Port:
if cl.dHT == nil {
break
}
pingAddr, err := net.ResolveUDPAddr("", c.remoteAddr().String())
if err != nil {
panic(err)
}
if msg.Port != 0 {
pingAddr.Port = int(msg.Port)
}
cl.dHT.Ping(pingAddr)
default:
err = fmt.Errorf("received unknown message type: %#v", msg.Type)
}
if err != nil {
return err
}
}
}
func (p Piece) Hash() (ret Hash) {
missinggo.CopyExact(&ret, p.Info.Pieces[p.i*20:(p.i+1)*20])
return
}
func (me Piece) Hash() (ret Hash) {
missinggo.CopyExact(&ret, me.Info.Pieces[me.i*20:(me.i+1)*20])
return
}
