Cieric/Naut-Peer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Initial commit: multi-peer torrent download engine

Browse source 
Reactor/loop-pool engine with TCP/µTP/MSE transports, per-connection
pipelining, priority-driven piece selection with endgame, and the Python
FFI test harness.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
ookami125 2026-06-21 23:12:32 -04:00
commit d8208685a2
55 changed files with 9989 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

202
harness/engine_ffi.py Normal file
View file

@ -0,0 +1,202 @@
"""
ctypes bindings for the multi-peer engine ABI (include/engine.h).
The engine owns a pool of event-loop threads; torrents are pinned to a loop and
every connection of a torrent lives there. Each loop has its own arena, so a
delivered block names both the loop and the slot. We wrap each loop's arena once
as a zero-copy ``memoryview`` and slice per block; returning the slot via
``release()`` is what lets the engine issue new requests (credit-based flow
control).
"""
from __future__ import annotations
import ctypes as C
import os
# peer_state / peer_error mirrors of include/engine.h
STATE_IDLE, STATE_CONNECTING, STATE_HANDSHAKE, STATE_CHOKED, \
STATE_RUNNING, STATE_STOPPED, STATE_ERROR = range(7)
STATE_NAMES = ["IDLE", "CONNECTING", "HANDSHAKE", "CHOKED",
"RUNNING", "STOPPED", "ERROR"]
ERROR_NAMES = ["OK", "CONNECT", "HANDSHAKE", "CLOSED", "PROTOCOL", "IO", "NOMEM"]
BLOCK_SIZE = 16384
class EngineConfig(C.Structure):
_fields_ = [
("loop_count", C.c_uint32),
("slots_per_loop", C.c_uint32),
("max_pipeline", C.c_uint32),
("request_timeout_ms", C.c_uint32),
("recv_buffer_bytes", C.c_uint32),
("encryption", C.c_uint32),
("utp", C.c_uint32),
("connect_timeout_ms", C.c_uint32),
("fallback", C.c_uint32),
]
class EngineBlock(C.Structure):
_fields_ = [
("torrent", C.c_uint32),
("piece", C.c_uint32),
("begin", C.c_uint32),
("len", C.c_uint32),
("loop", C.c_uint32),
("slot", C.c_uint32),
]
class TorrentStatus(C.Structure):
_fields_ = [
("state", C.c_int32),
("error", C.c_int32),
("bytes_received", C.c_uint64),
("blocks_received", C.c_uint64),
("peers", C.c_uint32),
("peers_connected", C.c_uint32),
("peers_failed", C.c_uint32),
("outstanding", C.c_uint32),
("free_slots", C.c_uint32),
("pipeline_target", C.c_uint32),
("rate_bps", C.c_double),
("rtt_min_ms", C.c_double),
]
def _default_lib_path() -> str:
here = os.path.dirname(os.path.abspath(__file__))
cand = [
os.path.join(here, "..", "build", "libtorrentpeer.so"),
os.path.join(here, "..", "build", "lib", "libtorrentpeer.so"),
]
for p in cand:
if os.path.exists(p):
return os.path.abspath(p)
return os.path.abspath(cand[0])
def _load(lib_path: str | None) -> C.CDLL:
lib = C.CDLL(lib_path or _default_lib_path())
lib.engine_create.restype = C.c_void_p
lib.engine_create.argtypes = [C.POINTER(EngineConfig)]
lib.engine_destroy.restype = None
lib.engine_destroy.argtypes = [C.c_void_p]
lib.engine_add_torrent.restype = C.c_int32
lib.engine_add_torrent.argtypes = [
C.c_void_p, C.POINTER(C.c_uint8), C.POINTER(C.c_uint8),
C.c_uint64, C.c_uint64, C.c_uint32,
]
lib.engine_add_peer.restype = C.c_int
lib.engine_add_peer.argtypes = [C.c_void_p, C.c_uint32, C.c_char_p, C.c_uint16]
lib.engine_set_priorities.restype = C.c_int
lib.engine_set_priorities.argtypes = [
C.c_void_p, C.c_uint32, C.POINTER(C.c_uint8), C.c_uint32]
lib.engine_set_priority.restype = C.c_int
lib.engine_set_priority.argtypes = [C.c_void_p, C.c_uint32, C.c_uint32, C.c_uint8]
lib.engine_request_piece.restype = C.c_int
lib.engine_request_piece.argtypes = [C.c_void_p, C.c_uint32, C.c_uint32]
lib.engine_poll_ready.restype = C.c_uint32
lib.engine_poll_ready.argtypes = [C.c_void_p, C.POINTER(EngineBlock), C.c_uint32]
lib.engine_release_slot.restype = None
lib.engine_release_slot.argtypes = [C.c_void_p, C.c_uint32, C.c_uint32]
lib.engine_wait.restype = C.c_int
lib.engine_wait.argtypes = [C.c_void_p, C.c_int]
lib.engine_arena_base.restype = C.c_void_p
lib.engine_arena_base.argtypes = [C.c_void_p, C.c_uint32]
lib.engine_arena_bytes.restype = C.c_uint64
lib.engine_arena_bytes.argtypes = [C.c_void_p, C.c_uint32]
lib.engine_loop_count.restype = C.c_uint32
lib.engine_loop_count.argtypes = [C.c_void_p]
lib.engine_torrent_status.restype = None
lib.engine_torrent_status.argtypes = [C.c_void_p, C.c_uint32, C.POINTER(TorrentStatus)]
return lib
class Engine:
"""Pythonic wrapper around one engine instance (a pool of loops)."""
def __init__(self, cfg: EngineConfig | None = None,
lib_path: str | None = None, poll_batch: int = 1024):
self._lib = _load(lib_path)
self._e = self._lib.engine_create(C.byref(cfg) if cfg else None)
if not self._e:
raise RuntimeError("engine_create failed")
# One zero-copy memoryview per loop arena.
self.nloops = self._lib.engine_loop_count(self._e)
self._arenas = []
self.arenas = []
for i in range(self.nloops):
base = self._lib.engine_arena_base(self._e, i)
nbytes = self._lib.engine_arena_bytes(self._e, i)
buf = (C.c_char * nbytes).from_address(base)
self._arenas.append(buf)
self.arenas.append(memoryview(buf).cast("B"))
self._batch = poll_batch
self._blocks = (EngineBlock * poll_batch)()
def add_torrent(self, info_hash: bytes, peer_id: bytes, piece_length: int,
total_size: int, num_pieces: int) -> int:
ih = (C.c_uint8 * 20).from_buffer_copy(info_hash)
pid = (C.c_uint8 * 20).from_buffer_copy(peer_id)
tid = self._lib.engine_add_torrent(self._e, ih, pid, piece_length,
total_size, num_pieces)
if tid < 0:
raise RuntimeError("engine_add_torrent failed")
return tid
def add_peer(self, torrent_id: int, ip: str, port: int) -> None:
if self._lib.engine_add_peer(self._e, torrent_id, ip.encode(), port) != 0:
raise RuntimeError("engine_add_peer failed")
def set_priorities(self, torrent_id: int, priorities) -> None:
buf = bytes(priorities)
arr = (C.c_uint8 * len(buf)).from_buffer_copy(buf)
if self._lib.engine_set_priorities(self._e, torrent_id, arr, len(buf)) != 0:
raise ValueError("set_priorities: length must equal num_pieces")
def set_priority(self, torrent_id: int, piece_index: int, priority: int) -> None:
if self._lib.engine_set_priority(self._e, torrent_id, piece_index, priority) != 0:
raise ValueError(f"set_priority({piece_index}) out of range")
def request_piece(self, torrent_id: int, piece_index: int) -> None:
if self._lib.engine_request_piece(self._e, torrent_id, piece_index) != 0:
raise ValueError(f"request_piece({piece_index}) out of range")
def poll_ready(self):
"""Return a list of EngineBlock for completed blocks (may be empty)."""
n = self._lib.engine_poll_ready(self._e, self._blocks, self._batch)
return [self._blocks[i] for i in range(n)]
def block_data(self, loop: int, slot: int, length: int) -> memoryview:
off = slot * BLOCK_SIZE
return self.arenas[loop][off:off + length]
def release(self, loop: int, slot: int) -> None:
self._lib.engine_release_slot(self._e, loop, slot)
def wait(self, timeout_ms: int) -> int:
return self._lib.engine_wait(self._e, timeout_ms)
def status(self, torrent_id: int) -> TorrentStatus:
st = TorrentStatus()
self._lib.engine_torrent_status(self._e, torrent_id, C.byref(st))
return st
def close(self) -> None:
if self._e:
for mv in self.arenas:
mv.release()
self.arenas = []
self._arenas = []
self._lib.engine_destroy(self._e)
self._e = None
def __enter__(self):
return self
def __exit__(self, *exc):
self.close()

202
harness/harness.py Normal file
View file

@ -0,0 +1,202 @@
"""
Test/driver harness for the C peer.
The Python harness parses .torrent metadata and can discover peers through the
sibling torrent-tracker C library's DHT and HTTP/UDP tracker helpers. The C peer
does the fast part: pull the requested blocks from one peer. This harness owns
what to download, reassembles pieces, and verifies SHA-1 hashes -- the peer
never hashes or persists anything.
"""
from __future__ import annotations
import argparse
import hashlib
import os
import secrets
import time
from peer_ffi import Peer, PeerConfig, STATE_ERROR, STATE_NAMES, ERROR_NAMES
from torrent_meta import Metadata, load_metadata, load_torrent
from tracker_ffi import DHTClient, TrackerClient
def make_peer_id() -> bytes:
return b"-PC0001-" + secrets.token_bytes(12)
def discover_peers(torrent_path: str, max_wait: float = 20.0) -> list[tuple[str, int]]:
"""Discover peer endpoints through DHT first, then torrent trackers."""
tf = load_torrent(torrent_path)
peer_id = make_peer_id()
key = secrets.randbits(32)
deadline = time.time() + max_wait
seen: set[tuple[str, int]] = set()
dht_budget = max(0.5, min(6.0, max_wait / 2.0))
dht_result = DHTClient().lookup(tf.metadata.info_hash, timeout=dht_budget)
seen.update(dht_result.peers)
if seen:
return sorted(seen)
client = TrackerClient()
for url in tf.trackers:
if time.time() >= deadline:
break
result = client.announce(
url, tf.metadata, peer_id, port=6881, key=key, numwant=50,
event="started", timeout=max(0.5, deadline - time.time()))
if result.ok:
seen.update(result.peers)
if seen:
break
return sorted(seen)
class Downloader:
def __init__(self, meta: Metadata, *, peer_id: bytes | None = None,
num_slots: int = 0, max_pipeline: int = 0,
request_timeout_ms: int = 0, recv_buffer_bytes: int = 0,
lib_path: str | None = None):
self.meta = meta
cfg = PeerConfig()
cfg.info_hash[:] = meta.info_hash
cfg.peer_id[:] = peer_id or make_peer_id()
cfg.piece_length = meta.piece_length
cfg.total_size = meta.total_size
cfg.num_pieces = meta.num_pieces
cfg.num_slots = num_slots
cfg.max_pipeline = max_pipeline
cfg.request_timeout_ms = request_timeout_ms
cfg.recv_buffer_bytes = recv_buffer_bytes
self.peer = Peer(cfg, lib_path)
self.buffers: list[bytearray | None] = [None] * meta.num_pieces
self.received = [0] * meta.num_pieces
self.done = bytearray(meta.num_pieces)
self.done_count = 0
def download(self, ip: str, port: int, pieces=None, priorities=None,
timeout: float = 60.0, progress_every: float = 1.0,
output: str | None = None) -> bytes | None:
meta = self.meta
pieces = list(pieces) if pieces is not None else list(range(meta.num_pieces))
out_fh = open(output, "wb") if output else None
if out_fh:
out_fh.truncate(meta.total_size)
# Build the priority vector: caller-supplied scheme, or uniform "1" over
# the wanted pieces (0 = not wanted). The peer masks this with what the
# remote actually has, so a peer missing a piece is simply skipped.
prio = bytearray(priorities) if priorities is not None \
else bytearray(meta.num_pieces)
for i in pieces:
self.buffers[i] = bytearray(meta.piece_len(i))
if priorities is None:
prio[i] = 1
self.peer.start(ip, port)
self.peer.set_priorities(prio)
want = len(pieces)
deadline = time.time() + timeout
last_print = 0.0
last_progress_bytes = 0
last_progress_time = time.time()
while self.done_count < want:
st = self.peer.status()
if st.state == STATE_ERROR:
raise RuntimeError(f"peer error: {ERROR_NAMES[st.error]}")
descs = self.peer.poll_ready()
if not descs:
self.peer.wait(100)
now = time.time()
if st.bytes_received != last_progress_bytes:
last_progress_bytes = st.bytes_received
last_progress_time = now
if now > deadline and now - last_progress_time > timeout:
raise TimeoutError(
f"stalled: {self.done_count}/{want} pieces, "
f"state={STATE_NAMES[st.state]}")
if now - last_print >= progress_every:
last_print = now
print(f" {self.done_count}/{want} pieces "
f"{st.rate_bps/1e6:.1f} MB/s "
f"outstanding={st.outstanding} free={st.free_slots}")
continue
for d in descs:
buf = self.buffers[d.piece]
buf[d.begin:d.begin + d.len] = self.peer.block_data(d.slot, d.len)
self.peer.release(d.slot)
self.received[d.piece] += d.len
if (not self.done[d.piece]
and self.received[d.piece] >= meta.piece_len(d.piece)):
digest = hashlib.sha1(bytes(buf)).digest()
if digest != meta.piece_hashes[d.piece]:
raise ValueError(f"piece {d.piece} hash mismatch")
self.done[d.piece] = 1
self.done_count += 1
# Drop priority so a verified piece is no longer a
# selection candidate (the peer also won't re-request it).
self.peer.set_priority(d.piece, 0)
if out_fh:
out_fh.seek(d.piece * meta.piece_length)
out_fh.write(buf)
if not output:
pass # keep in memory for return
else:
self.buffers[d.piece] = None # free once flushed
if out_fh:
out_fh.close()
return None
return b"".join(bytes(self.buffers[i]) for i in pieces)
def close(self):
self.peer.stop()
self.peer.close()
def main() -> int:
ap = argparse.ArgumentParser(description="Drive the C peer to download a torrent.")
ap.add_argument("torrent", help="path to .torrent file")
ap.add_argument("--peer", help="explicit peer ip:port (skip tracker)")
ap.add_argument("--output", "-o", help="write downloaded data here")
ap.add_argument("--slots", type=int, default=0, help="arena slots (16 KiB each)")
ap.add_argument("--pipeline", type=int, default=0, help="max outstanding requests")
ap.add_argument("--timeout", type=float, default=120.0)
args = ap.parse_args()
meta = load_metadata(args.torrent)
print(f"torrent: {meta.name} {meta.total_size} bytes "
f"{meta.num_pieces} pieces x {meta.piece_length}")
if args.peer:
host, port = args.peer.rsplit(":", 1)
endpoints = [(host, int(port))]
else:
print("discovering peers via tracker/DHT...")
endpoints = discover_peers(args.torrent)
if not endpoints:
print("no peers found")
return 1
print(f"found {len(endpoints)} peer(s); using {endpoints[0]}")
dl = Downloader(meta, num_slots=args.slots, max_pipeline=args.pipeline)
try:
ip, port = endpoints[0]
t0 = time.time()
dl.download(ip, port, timeout=args.timeout, output=args.output)
dt = time.time() - t0
mb = meta.total_size / 1e6
print(f"done: {mb:.1f} MB in {dt:.2f}s = {mb/dt:.1f} MB/s")
finally:
dl.close()
return 0
if __name__ == "__main__":
raise SystemExit(main())

181
harness/peer_ffi.py Normal file
View file

@ -0,0 +1,181 @@
"""
ctypes bindings for libtorrentpeer.so.
The arena is wrapped once as a zero-copy ``memoryview``; ``block_data()`` returns
a slice into it, so the harness never copies a block until it chooses to (e.g.
into a per-piece buffer for hashing). Returning the slot via ``release()`` is what
lets the peer issue new requests (credit-based flow control).
"""
from __future__ import annotations
import ctypes as C
import os
# peer_state / peer_error mirrors of include/peer.h
STATE_IDLE, STATE_CONNECTING, STATE_HANDSHAKE, STATE_CHOKED, \
STATE_RUNNING, STATE_STOPPED, STATE_ERROR = range(7)
STATE_NAMES = ["IDLE", "CONNECTING", "HANDSHAKE", "CHOKED",
"RUNNING", "STOPPED", "ERROR"]
ERROR_NAMES = ["OK", "CONNECT", "HANDSHAKE", "CLOSED", "PROTOCOL", "IO", "NOMEM"]
BLOCK_SIZE = 16384
class PeerConfig(C.Structure):
_fields_ = [
("info_hash", C.c_uint8 * 20),
("peer_id", C.c_uint8 * 20),
("piece_length", C.c_uint64),
("total_size", C.c_uint64),
("num_pieces", C.c_uint32),
("num_slots", C.c_uint32),
("max_pipeline", C.c_uint32),
("request_timeout_ms", C.c_uint32),
("recv_buffer_bytes", C.c_uint32),
]
class BlockDesc(C.Structure):
_fields_ = [
("piece", C.c_uint32),
("begin", C.c_uint32),
("len", C.c_uint32),
("slot", C.c_uint32),
]
class PeerStatus(C.Structure):
_fields_ = [
("state", C.c_int32),
("error", C.c_int32),
("bytes_received", C.c_uint64),
("blocks_received", C.c_uint64),
("outstanding", C.c_uint32),
("free_slots", C.c_uint32),
("pipeline_target", C.c_uint32),
("rate_bps", C.c_double),
("rtt_min_ms", C.c_double),
]
def _default_lib_path() -> str:
here = os.path.dirname(os.path.abspath(__file__))
cand = [
os.path.join(here, "..", "build", "libtorrentpeer.so"),
os.path.join(here, "..", "build", "lib", "libtorrentpeer.so"),
]
for p in cand:
if os.path.exists(p):
return os.path.abspath(p)
return os.path.abspath(cand[0])
def _load(lib_path: str | None) -> C.CDLL:
lib = C.CDLL(lib_path or _default_lib_path())
lib.peer_create.restype = C.c_void_p
lib.peer_create.argtypes = [C.POINTER(PeerConfig)]
lib.peer_start.restype = C.c_int
lib.peer_start.argtypes = [C.c_void_p, C.c_char_p, C.c_uint16]
lib.peer_set_priorities.restype = C.c_int
lib.peer_set_priorities.argtypes = [C.c_void_p, C.POINTER(C.c_uint8), C.c_uint32]
lib.peer_set_priority.restype = C.c_int
lib.peer_set_priority.argtypes = [C.c_void_p, C.c_uint32, C.c_uint8]
lib.peer_request_piece.restype = C.c_int
lib.peer_request_piece.argtypes = [C.c_void_p, C.c_uint32]
lib.peer_stop.restype = None
lib.peer_stop.argtypes = [C.c_void_p]
lib.peer_destroy.restype = None
lib.peer_destroy.argtypes = [C.c_void_p]
lib.peer_arena_base.restype = C.c_void_p
lib.peer_arena_base.argtypes = [C.c_void_p]
lib.peer_arena_bytes.restype = C.c_uint64
lib.peer_arena_bytes.argtypes = [C.c_void_p]
lib.peer_poll_ready.restype = C.c_uint32
lib.peer_poll_ready.argtypes = [C.c_void_p, C.POINTER(BlockDesc), C.c_uint32]
lib.peer_release_slot.restype = None
lib.peer_release_slot.argtypes = [C.c_void_p, C.c_uint32]
lib.peer_wait.restype = C.c_int
lib.peer_wait.argtypes = [C.c_void_p, C.c_int]
lib.peer_get_status.restype = None
lib.peer_get_status.argtypes = [C.c_void_p, C.POINTER(PeerStatus)]
return lib
class Peer:
"""Pythonic wrapper around one peer_handle."""
def __init__(self, cfg: PeerConfig, lib_path: str | None = None,
poll_batch: int = 1024):
self._lib = _load(lib_path)
self._h = self._lib.peer_create(C.byref(cfg))
if not self._h:
raise RuntimeError("peer_create failed (bad config or OOM)")
base = self._lib.peer_arena_base(self._h)
nbytes = self._lib.peer_arena_bytes(self._h)
arena_t = (C.c_char * nbytes)
self._arena = arena_t.from_address(base)
# zero-copy view over the slab, as unsigned bytes for clean slicing
self.arena = memoryview(self._arena).cast("B")
self._batch = poll_batch
self._descs = (BlockDesc * poll_batch)()
def start(self, ip: str, port: int) -> None:
rc = self._lib.peer_start(self._h, ip.encode(), port)
if rc != 0:
raise RuntimeError("peer_start failed")
def set_priorities(self, priorities) -> None:
"""Set the whole per-piece priority vector (len must == num_pieces)."""
buf = bytes(priorities)
arr = (C.c_uint8 * len(buf)).from_buffer_copy(buf)
if self._lib.peer_set_priorities(self._h, arr, len(buf)) != 0:
raise ValueError("set_priorities: length must equal num_pieces")
def set_priority(self, piece_index: int, priority: int) -> None:
if self._lib.peer_set_priority(self._h, piece_index, priority) != 0:
raise ValueError(f"set_priority({piece_index}) out of range")
def request_piece(self, piece_index: int) -> None:
"""Re-arm a piece for (re-)download (e.g. after a hash failure)."""
if self._lib.peer_request_piece(self._h, piece_index) != 0:
raise ValueError(f"request_piece({piece_index}) out of range")
def poll_ready(self):
"""Return a list of BlockDesc for completed blocks (may be empty)."""
n = self._lib.peer_poll_ready(self._h, self._descs, self._batch)
return [self._descs[i] for i in range(n)]
def block_data(self, slot: int, length: int) -> memoryview:
off = slot * BLOCK_SIZE
return self.arena[off:off + length]
def release(self, slot: int) -> None:
self._lib.peer_release_slot(self._h, slot)
def wait(self, timeout_ms: int) -> int:
return self._lib.peer_wait(self._h, timeout_ms)
def status(self) -> PeerStatus:
st = PeerStatus()
self._lib.peer_get_status(self._h, C.byref(st))
return st
def stop(self) -> None:
if self._h:
self._lib.peer_stop(self._h)
def close(self) -> None:
if self._h:
# Drop the memoryview before freeing the arena it points into.
self.arena.release()
del self._arena
self._lib.peer_destroy(self._h)
self._h = None
def __enter__(self):
return self
def __exit__(self, *exc):
self.close()

364
harness/seed_server.py Normal file
View file

@ -0,0 +1,364 @@
"""
seed_server.py - A lightweight emulated BitTorrent client (seed side) for
load-testing the peer/engine against "as many peers as possible".
Rather than spinning up N heavyweight libtorrent sessions, this serves one
torrent's data from a single asyncio event loop across many listener sockets.
Each listener is a distinct endpoint, so from the engine's point of view each is
a separate peer: one `add_peer(tid, ip, port)` per listener.
It speaks the plaintext BitTorrent v1 wire protocol a real client would on the
seed side: validates the handshake + info-hash, advertises a full bitfield,
unchokes, and answers `request` messages with `piece` data read (mmap'd) from
disk. That is exactly the subset the engine drives, and it is plaintext because
the engine does not negotiate MSE encryption yet.
Usage
-----
Serve an existing torrent (data already on disk under --data):
python harness/seed_server.py some.torrent --data /path/to/datadir --peers 64
Generate a random test torrent, serve it, and print the .torrent path:
python harness/seed_server.py --generate 256M --peers 64 --out /tmp/seedtest
Generate + serve + drive the engine against every peer and report throughput:
python harness/seed_server.py --generate 256M --peers 64 --out /tmp/seedtest --self-test
In a test, use the SeedSwarm class directly to start listeners in-process and
read `swarm.endpoints`.
"""
from __future__ import annotations
import argparse
import asyncio
import mmap
import os
import secrets
import struct
import threading
import time
PSTR = b"BitTorrent protocol"
HANDSHAKE_LEN = 68
# Wire message ids.
MSG_CHOKE, MSG_UNCHOKE, MSG_INTERESTED, MSG_NOT_INTERESTED = 0, 1, 2, 3
MSG_HAVE, MSG_BITFIELD, MSG_REQUEST, MSG_PIECE, MSG_CANCEL = 4, 5, 6, 7, 8
DRAIN_HIGH_WATER = 1 << 20 # let blocks queue up, apply backpressure past 1 MiB
MAX_BLOCK = 1 << 17 # reject absurd request lengths (128 KiB)
# --------------------------------------------------------------------------- #
# Data source: read (offset, length) from the torrent's concatenated files.
# --------------------------------------------------------------------------- #
class DataSource:
"""Maps the linear piece space onto one or more on-disk files (BT v1 lays
files out back-to-back). Files are mmap'd for cheap repeated reads."""
def __init__(self, files: list[tuple[str, int]]):
self._maps = [] # (global_offset, size, mmap_or_none)
self._handles = []
off = 0
for path, size in files:
mm = None
if size > 0:
fh = open(path, "rb")
self._handles.append(fh)
mm = mmap.mmap(fh.fileno(), size, prot=mmap.PROT_READ)
self._maps.append((off, size, mm))
off += size
self.total = off
self._single = self._maps[0][2] if len(self._maps) == 1 else None
def read(self, offset: int, length: int) -> bytes:
if self._single is not None: # fast path: one file
return self._single[offset:offset + length]
out = bytearray()
remaining = length
for foff, size, mm in self._maps:
if remaining <= 0:
break
if offset >= foff + size or offset < foff:
continue
local = offset - foff
take = min(size - local, remaining)
out += mm[local:local + take]
offset += take
remaining -= take
return bytes(out)
def close(self):
for _, _, mm in self._maps:
if mm is not None:
mm.close()
for fh in self._handles:
fh.close()
def _full_bitfield(num_pieces: int) -> bytes:
nbytes = (num_pieces + 7) // 8
bf = bytearray(b"\xff" * nbytes)
rem = num_pieces & 7
if rem: # clear pad bits past the end
bf[-1] = (0xFF << (8 - rem)) & 0xFF
return bytes(bf)
def _msg(mid: int, payload: bytes = b"") -> bytes:
return struct.pack(">IB", 1 + len(payload), mid) + payload
# --------------------------------------------------------------------------- #
# The swarm: many listeners sharing one event loop, on a background thread.
# --------------------------------------------------------------------------- #
class SeedSwarm:
def __init__(self, info_hash: bytes, piece_length: int, num_pieces: int,
source: DataSource, host: str = "127.0.0.1"):
self.info_hash = info_hash
self.piece_length = piece_length
self.num_pieces = num_pieces
self.source = source
self.host = host
self.bitfield = _full_bitfield(num_pieces)
self.endpoints: list[tuple[str, int]] = []
self.served_bytes = 0
self._loop = asyncio.new_event_loop()
self._servers: list[asyncio.AbstractServer] = []
self._thread = threading.Thread(target=self._run, daemon=True)
# -- connection handler (one per accepted peer) ------------------------- #
async def _handle(self, reader: asyncio.StreamReader,
writer: asyncio.StreamWriter):
try:
hs = await reader.readexactly(HANDSHAKE_LEN)
if hs[1:20] != PSTR or hs[28:48] != self.info_hash:
writer.close()
return
peer_id = b"-SD0001-" + secrets.token_bytes(12)
writer.write(bytes([len(PSTR)]) + PSTR + b"\x00" * 8 +
self.info_hash + peer_id)
writer.write(_msg(MSG_BITFIELD, self.bitfield))
writer.write(_msg(MSG_UNCHOKE))
await writer.drain()
while True:
(length,) = struct.unpack(">I", await reader.readexactly(4))
if length == 0:
continue # keep-alive
body = await reader.readexactly(length)
mid = body[0]
if mid == MSG_REQUEST and length >= 13:
index, begin, blen = struct.unpack(">III", body[1:13])
if blen > MAX_BLOCK:
continue
data = self.source.read(index * self.piece_length + begin, blen)
writer.write(struct.pack(">IB", 9 + len(data), MSG_PIECE) +
struct.pack(">II", index, begin) + data)
self.served_bytes += len(data)
if writer.transport.get_write_buffer_size() > DRAIN_HIGH_WATER:
await writer.drain()
# interested / not-interested / cancel / choke: nothing to do —
# we are already unchoked and answer requests as they arrive.
except (asyncio.IncompleteReadError, ConnectionResetError,
BrokenPipeError, ConnectionError):
pass
finally:
try:
writer.close()
except Exception:
pass
async def _make_servers(self, count: int):
servers, endpoints = [], []
for _ in range(count):
srv = await asyncio.start_server(self._handle, self.host, 0)
servers.append(srv)
endpoints.append((self.host, srv.sockets[0].getsockname()[1]))
return servers, endpoints
def _run(self):
asyncio.set_event_loop(self._loop)
self._loop.run_forever()
def start(self, count: int):
self._thread.start()
fut = asyncio.run_coroutine_threadsafe(self._make_servers(count), self._loop)
self._servers, self.endpoints = fut.result(timeout=15)
return self.endpoints
def stop(self):
def _shutdown():
for srv in self._servers:
srv.close()
self._loop.stop()
if self._loop.is_running():
self._loop.call_soon_threadsafe(_shutdown)
self._thread.join(timeout=5)
try:
self._loop.close()
except Exception:
pass
self.source.close()
# --------------------------------------------------------------------------- #
# Torrent loading / generation.
# --------------------------------------------------------------------------- #
def load_torrent(torrent_path: str, data_dir: str):
"""Return (info_hash, piece_length, num_pieces, total_size, DataSource)."""
from torrent_meta import load_torrent as parse_torrent
tf = parse_torrent(torrent_path)
files = []
for path, size in tf.files:
files.append((os.path.join(data_dir, path), size))
missing = [p for p, _ in files if not os.path.exists(p)]
if missing:
raise FileNotFoundError(f"data files not found under {data_dir}: {missing}")
meta = tf.metadata
return (meta.info_hash, meta.piece_length, meta.num_pieces,
meta.total_size, DataSource(files))
def generate_torrent(out_dir: str, size: int, piece_size: int = 256 * 1024):
"""Create a random data file + .torrent under out_dir. Returns
(torrent_path, info_hash, piece_length, num_pieces, total_size, DataSource)."""
import libtorrent as lt
os.makedirs(out_dir, exist_ok=True)
data_path = os.path.join(out_dir, "data.bin")
with open(data_path, "wb") as f:
rem = size
while rem > 0:
n = min(rem, 8 << 20)
f.write(os.urandom(n))
rem -= n
fs = lt.file_storage()
lt.add_files(fs, data_path)
t = lt.create_torrent(fs, piece_size=piece_size)
t.set_priv(False)
lt.set_piece_hashes(t, out_dir)
torrent_path = os.path.join(out_dir, "test.torrent")
with open(torrent_path, "wb") as f:
f.write(lt.bencode(t.generate()))
ih, pl, npc, total, src = load_torrent(torrent_path, out_dir)
return torrent_path, ih, pl, npc, total, src
def parse_size(s: str) -> int:
s = s.strip().upper()
mult = 1
if s and s[-1] in "KMG":
mult = {"K": 1024, "M": 1024**2, "G": 1024**3}[s[-1]]
s = s[:-1]
return int(float(s) * mult)
# --------------------------------------------------------------------------- #
# Optional: drive the engine against every peer and verify + report rate.
# --------------------------------------------------------------------------- #
def _self_test(torrent_path: str, data_dir: str, endpoints, timeout: float):
import hashlib
from engine_ffi import Engine, EngineConfig, STATE_ERROR, ERROR_NAMES
from harness import load_metadata
meta = load_metadata(torrent_path)
lib = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"build", "libtorrentpeer.so")
loops = min(8, max(1, (os.cpu_count() or 1)))
with Engine(EngineConfig(loop_count=loops, slots_per_loop=4096,
max_pipeline=1024), lib_path=lib) as eng:
tid = eng.add_torrent(meta.info_hash, b"-PC0001-" + secrets.token_bytes(12),
meta.piece_length, meta.total_size, meta.num_pieces)
eng.set_priorities(tid, [1] * meta.num_pieces)
for ip, port in endpoints:
eng.add_peer(tid, ip, port)
buffers = [bytearray(meta.piece_len(i)) for i in range(meta.num_pieces)]
received = [0] * meta.num_pieces
done = bytearray(meta.num_pieces)
done_count = 0
t0 = time.time()
deadline = t0 + timeout
while done_count < meta.num_pieces:
st = eng.status(tid)
if st.state == STATE_ERROR:
raise RuntimeError(f"engine error: {ERROR_NAMES[st.error]}")
descs = eng.poll_ready()
if not descs:
eng.wait(200)
if time.time() > deadline:
raise TimeoutError(f"stalled at {done_count}/{meta.num_pieces}")
continue
for x in descs:
buf = buffers[x.piece]
buf[x.begin:x.begin + x.len] = eng.block_data(x.loop, x.slot, x.len)
eng.release(x.loop, x.slot)
received[x.piece] += x.len
if not done[x.piece] and received[x.piece] >= meta.piece_len(x.piece):
if hashlib.sha1(bytes(buf)).digest() != meta.piece_hashes[x.piece]:
raise ValueError(f"piece {x.piece} hash mismatch")
done[x.piece] = 1
done_count += 1
eng.set_priority(tid, x.piece, 0)
dt = time.time() - t0
mb = meta.total_size / 1e6
print(f"self-test OK: {mb:.1f} MB from {len(endpoints)} peers "
f"in {dt:.2f}s = {mb/dt:.1f} MB/s (peers={eng.status(tid).peers})")
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("torrent", nargs="?", help="path to an existing .torrent")
ap.add_argument("--data", help="directory (or file) holding the torrent's data")
ap.add_argument("--generate", metavar="SIZE",
help="generate a random torrent of this size (e.g. 256M, 1G)")
ap.add_argument("--out", default="/tmp/seedtest",
help="output dir for --generate (default: /tmp/seedtest)")
ap.add_argument("--peers", type=int, default=32, help="number of listeners")
ap.add_argument("--host", default="127.0.0.1")
ap.add_argument("--piece-size", type=int, default=256 * 1024)
ap.add_argument("--ports-file", help="write the endpoints (one ip:port/line) here")
ap.add_argument("--self-test", action="store_true",
help="drive the engine against all peers, verify, report rate")
ap.add_argument("--timeout", type=float, default=300.0)
args = ap.parse_args()
torrent_path = args.torrent
if args.generate:
size = parse_size(args.generate)
torrent_path, ih, pl, npc, total, src = generate_torrent(
args.out, size, args.piece_size)
data_dir = args.out
print(f"generated {torrent_path} ({total} bytes, {npc} pieces x {pl})")
else:
if not torrent_path or not args.data:
ap.error("provide a .torrent and --data, or use --generate SIZE")
data_dir = args.data if os.path.isdir(args.data) else os.path.dirname(args.data)
ih, pl, npc, total, src = load_torrent(torrent_path, data_dir)
swarm = SeedSwarm(ih, pl, npc, src, host=args.host)
swarm.start(args.peers)
print(f"seeding {total} bytes on {len(swarm.endpoints)} peers "
f"({args.host}:{swarm.endpoints[0][1]}..{swarm.endpoints[-1][1]})")
if args.ports_file:
with open(args.ports_file, "w") as f:
f.writelines(f"{ip}:{port}\n" for ip, port in swarm.endpoints)
print(f"endpoints written to {args.ports_file}")
try:
if args.self_test:
_self_test(torrent_path, data_dir, swarm.endpoints, args.timeout)
else:
print("serving; press Ctrl-C to stop")
while True:
time.sleep(1.0)
except KeyboardInterrupt:
pass
finally:
swarm.stop()
return 0
if __name__ == "__main__":
raise SystemExit(main())

543
harness/swarm_download.py Normal file
View file

@ -0,0 +1,543 @@
"""
swarm_download.py - Download a real torrent from a real swarm using the engine,
to measure how it performs against actual peers.
Division of labour: for .torrent files, this harness parses metainfo locally and
uses the sibling torrent-tracker library for DHT get_peers and HTTP/UDP tracker
announces. Magnet metadata resolution still falls back to libtorrent. Every
discovered peer endpoint is fed to the engine, which does all the data transfer.
Pieces are reassembled and SHA-1-verified here; the engine never hashes or
persists anything.
Reality check (important for interpreting the numbers): a real public swarm
contains unreachable peers, peers behind NAT, peers that only accept a different
transport/encryption combination, and peers that do not actually have useful
pieces. Those show up as "failed". The headline metric this prints is therefore
how many discovered peers were actually usable, and the sustained rate across
them. That is the honest "how well does it work today" answer.
Usage
-----
python harness/swarm_download.py path/to/file.torrent
python harness/swarm_download.py 'magnet:?xt=urn:btih:...'
python harness/swarm_download.py file.torrent --max-peers 200 --output /tmp/out --timeout 600
python harness/swarm_download.py file.torrent --output /tmp/out --resume
Pick a well-seeded torrent (e.g. a current Linux distro ISO) for a meaningful
test; obscure or dead torrents will show few usable peers regardless.
"""
from __future__ import annotations
import argparse
import hashlib
import os
import secrets
import sys
import tempfile
import time
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(ROOT, "harness"))
import libtorrent as lt # noqa: E402 # only used for magnet metadata fallback
from harness import load_metadata # noqa: E402
from engine_ffi import (BLOCK_SIZE, Engine, EngineConfig, STATE_NAMES, # noqa: E402
ERROR_NAMES)
from torrent_meta import load_torrent # noqa: E402
from tracker_ffi import DHTClient, TrackerClient # noqa: E402
LIB = os.path.join(ROOT, "build", "libtorrentpeer.so")
def _discovery_session() -> lt.session:
s = lt.session({
"listen_interfaces": "0.0.0.0:0,[::]:0",
"enable_dht": True, "enable_lsd": True,
"enable_upnp": True, "enable_natpmp": True,
"alert_mask": 0,
})
for host, port in (("router.bittorrent.com", 6881),
("dht.transmissionbt.com", 6881),
("router.utorrent.com", 6881)):
try:
s.add_dht_node((host, port))
except Exception:
pass
return s
class LibtorrentDiscovery:
def __init__(self, handle, session):
self.handle = handle
self.session = session
def collect(self, _max_peers: int) -> set[tuple[str, int]]:
eps = set()
try:
for pi in self.handle.get_peer_info():
ip = pi.ip
if isinstance(ip, tuple) and len(ip) == 2 and ip[1]:
eps.add((ip[0], int(ip[1])))
except Exception:
pass
return eps
def close(self):
try:
self.session.remove_torrent(self.handle)
except Exception:
pass
class TrackerDiscovery:
def __init__(self, trackers: list[str], meta, *, timeout: float,
max_trackers: int, use_dht: bool, dht_timeout: float,
dht_queries: int):
self.trackers = trackers[:max_trackers] if max_trackers > 0 else trackers
self.meta = meta
self.timeout = timeout
self.client = TrackerClient()
self.dht = DHTClient() if use_dht else None
self.dht_timeout = dht_timeout
self.dht_queries = dht_queries
self.dht_done = False
self.peer_id = b"-PC0001-" + secrets.token_bytes(12)
self.key = secrets.randbits(32)
self.endpoints: set[tuple[str, int]] = set()
self.index = 0
self.next_cycle_at = 0.0
def collect(self, max_peers: int) -> set[tuple[str, int]]:
now = time.time()
if len(self.endpoints) >= max_peers:
return set(self.endpoints)
if self.dht and not self.dht_done:
self.dht_done = True
result = self.dht.lookup(
self.meta.info_hash,
timeout=self.dht_timeout,
max_queries=self.dht_queries,
max_peers=max_peers - len(self.endpoints),
)
self.endpoints.update(result.peers)
msg = (f"dht: {len(result.peers)} peers, "
f"{result.nodes_queried} queried/{result.nodes_discovered} learned "
f"({result.elapsed_ms:.0f} ms)")
if result.error:
msg += f": {result.error}"
print(msg, flush=True)
if len(self.endpoints) >= max_peers:
return set(self.endpoints)
if self.index >= len(self.trackers):
if now < self.next_cycle_at:
return set(self.endpoints)
self.index = 0
if not self.trackers:
return set()
url = self.trackers[self.index]
self.index += 1
if self.index >= len(self.trackers):
self.next_cycle_at = now + 60.0
result = self.client.announce(
url, self.meta, self.peer_id, port=6881, key=self.key,
numwant=max(1, min(200, max_peers - len(self.endpoints))),
event="started", timeout=self.timeout)
if result.ok:
self.endpoints.update(result.peers)
print(f"tracker: {url} returned {len(result.peers)} peers "
f"({result.elapsed_ms:.0f} ms)", flush=True)
else:
print(f"tracker: {url} failed: {result.error}", flush=True)
return set(self.endpoints)
def close(self):
pass
def _resolve_magnet(arg: str, scratch: str):
ses = _discovery_session()
params = lt.parse_magnet_uri(arg)
params.save_path = scratch
params.flags |= lt.torrent_flags.upload_mode
h = ses.add_torrent(params)
print("resolving magnet metadata from the swarm using libtorrent...", flush=True)
deadline = time.time() + 120
while time.time() < deadline and not h.status().has_metadata:
time.sleep(0.5)
if not h.status().has_metadata:
raise TimeoutError("could not fetch metadata for magnet within 120s")
tpath = os.path.join(scratch, "resolved.torrent")
with open(tpath, "wb") as f:
f.write(lt.bencode(lt.create_torrent(h.torrent_file()).generate()))
return tpath, LibtorrentDiscovery(h, ses)
def _completed_bytes(meta, done: bytearray) -> int:
return sum(meta.piece_len(i) for i, is_done in enumerate(done) if is_done)
def verify_existing_output(out_fh, meta) -> tuple[bytearray, int, int]:
"""Hash pieces already present in the output file.
Returns (done_bitfield, done_count, verified_bytes). Only pieces whose full
range exists and whose SHA-1 matches the torrent metadata are marked done.
This intentionally ignores partial pieces because the engine currently only
persists data after a whole piece has passed verification.
"""
done = bytearray(meta.num_pieces)
done_count = 0
file_size = os.fstat(out_fh.fileno()).st_size
for piece in range(meta.num_pieces):
piece_len = meta.piece_len(piece)
offset = piece * meta.piece_length
if offset + piece_len > file_size:
continue
out_fh.seek(offset)
data = out_fh.read(piece_len)
if len(data) != piece_len:
continue
if hashlib.sha1(data).digest() != meta.piece_hashes[piece]:
continue
done[piece] = 1
done_count += 1
return done, done_count, _completed_bytes(meta, done)
class PieceAssembler:
"""Reassemble pieces while ignoring duplicate blocks.
Endgame mode deliberately re-requests unfinished pieces from multiple peers.
That means the same block may arrive more than once. Counting raw bytes would
mark a piece complete too early, so completion is based on unique block
offsets within each piece.
"""
def __init__(self, meta, done: bytearray):
self.meta = meta
self.done = done
self.buffers: dict[int, bytearray] = {}
self.received = [0] * meta.num_pieces
self.seen: dict[int, bytearray] = {}
def _block_count(self, piece: int) -> int:
piece_len = self.meta.piece_len(piece)
return (piece_len + BLOCK_SIZE - 1) // BLOCK_SIZE
def reset_piece(self, piece: int) -> int:
previous = self.received[piece]
self.received[piece] = 0
self.buffers.pop(piece, None)
self.seen.pop(piece, None)
return previous
def add_block(self, piece: int, begin: int, data) -> tuple[bool, bool]:
if self.done[piece]:
return False, False
if begin % BLOCK_SIZE != 0:
raise ValueError(f"unaligned block for piece {piece}: begin={begin}")
piece_len = self.meta.piece_len(piece)
length = len(data)
if begin + length > piece_len:
raise ValueError(
f"block overruns piece {piece}: begin={begin} len={length} "
f"piece_len={piece_len}")
buf = self.buffers.get(piece)
if buf is None:
buf = bytearray(piece_len)
self.buffers[piece] = buf
seen = self.seen.get(piece)
if seen is None:
seen = bytearray(self._block_count(piece))
self.seen[piece] = seen
block = begin // BLOCK_SIZE
buf[begin:begin + length] = data
if seen[block]:
return False, self.received[piece] >= piece_len
seen[block] = 1
self.received[piece] += length
return True, self.received[piece] >= piece_len
def piece_bytes(self, piece: int) -> bytes:
return bytes(self.buffers[piece])
def finish_piece(self, piece: int) -> None:
self.done[piece] = 1
self.buffers.pop(piece, None)
self.seen.pop(piece, None)
class EndgameController:
"""Conservative piece-level endgame for slow tail pieces."""
def __init__(self, meta, *, min_pieces: int, peer_factor: float,
interval: float):
self.meta = meta
self.min_pieces = min_pieces
self.peer_factor = peer_factor
self.interval = interval
self.active = False
self.last_rearm = 0.0
def maybe_rearm(self, eng: Engine, tid: int, done: bytearray,
done_count: int, connected: int, now: float) -> None:
remaining = self.meta.num_pieces - done_count
if remaining <= 0:
return
threshold = max(self.min_pieces,
int(max(1, connected) * self.peer_factor))
if remaining > threshold:
return
if not self.active:
self.active = True
print(f"endgame: {remaining} pieces left; duplicating tail requests",
flush=True)
for piece, is_done in enumerate(done):
if not is_done:
eng.set_priority(tid, piece, 255)
self.last_rearm = 0.0
if now - self.last_rearm < self.interval:
return
for piece, is_done in enumerate(done):
if not is_done:
eng.request_piece(tid, piece)
self.last_rearm = now
def open_output(path: str, meta, resume: bool):
if resume:
existed = os.path.exists(path)
out_fh = open(path, "r+b" if existed else "w+b")
if existed:
print("resume: verifying existing output pieces...", flush=True)
done, done_count, verified_bytes = verify_existing_output(out_fh, meta)
pct = 100.0 * done_count / meta.num_pieces if meta.num_pieces else 100.0
print(f"resume: found {done_count}/{meta.num_pieces} verified pieces "
f"({pct:4.1f}%, {verified_bytes/1e6:.1f} MB)",
flush=True)
else:
done = bytearray(meta.num_pieces)
done_count = 0
print("resume: output file does not exist yet; starting fresh",
flush=True)
else:
out_fh = open(path, "w+b")
done = bytearray(meta.num_pieces)
done_count = 0
out_fh.truncate(meta.total_size)
return out_fh, done, done_count
def main() -> int:
ap = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("torrent", help="path to a .torrent file or a magnet: URI")
ap.add_argument("--max-peers", type=int, default=100,
help="cap on peers fed to the engine (default 100)")
ap.add_argument("--output", "-o", help="write verified data here (else discard)")
ap.add_argument("--resume", action="store_true",
help="resume from an existing --output file by hashing "
"verified pieces and skipping them")
ap.add_argument("--timeout", type=float, default=600.0,
help="overall stall timeout in seconds")
ap.add_argument("--tracker-timeout", type=float, default=4.0,
help="per-tracker announce timeout for direct tracker "
"discovery (default 4)")
ap.add_argument("--max-trackers", type=int, default=16,
help="maximum trackers to announce to from a .torrent "
"(0 = all, default 16)")
ap.add_argument("--no-dht", action="store_true",
help="disable DHT get_peers discovery for .torrent files")
ap.add_argument("--dht-timeout", type=float, default=6.0,
help="total DHT lookup budget in seconds (default 6)")
ap.add_argument("--dht-queries", type=int, default=32,
help="maximum DHT nodes to query per torrent (default 32)")
ap.add_argument("--lib", default=LIB)
ap.add_argument("--loops", type=int, default=0, help="engine loops (0=auto)")
ap.add_argument("--encryption", type=int, default=1, choices=[0, 1, 2],
help="0=plaintext, 1=MSE offer RC4+plaintext (default), "
"2=MSE require RC4. Encryption reaches far more of a "
"real swarm.")
ap.add_argument("--utp", type=int, default=0, choices=[0, 1],
help="0=TCP (default), 1=µTP/UDP. A swarm has a mix; this "
"selects which transport the engine dials with.")
ap.add_argument("--no-endgame", action="store_true",
help="disable duplicate tail-piece requests")
ap.add_argument("--endgame-min-pieces", type=int, default=16,
help="enter endgame when remaining pieces are at or below "
"this count, also scaled by connected peers "
"(default 16)")
ap.add_argument("--endgame-peer-factor", type=float, default=2.0,
help="also enter endgame below peers*factor remaining "
"pieces (default 2.0)")
ap.add_argument("--endgame-interval", type=float, default=3.0,
help="seconds between tail-piece re-arms in endgame "
"(default 3.0)")
args = ap.parse_args()
if args.resume and not args.output:
ap.error("--resume requires --output")
scratch = tempfile.mkdtemp(prefix="swarm_dl_")
if args.torrent.startswith("magnet:"):
tpath, discovery = _resolve_magnet(args.torrent, scratch)
else:
torrent_file = load_torrent(args.torrent)
tpath = args.torrent
discovery = TrackerDiscovery(
torrent_file.trackers,
torrent_file.metadata,
timeout=args.tracker_timeout,
max_trackers=args.max_trackers,
use_dht=not args.no_dht,
dht_timeout=args.dht_timeout,
dht_queries=args.dht_queries,
)
meta = load_metadata(tpath)
print(f"torrent: {meta.name} {meta.total_size/1e6:.1f} MB "
f"{meta.num_pieces} pieces x {meta.piece_length}", flush=True)
out_fh = None
done = bytearray(meta.num_pieces)
done_count = 0
if args.output:
out_fh, done, done_count = open_output(args.output, meta, args.resume)
cfg = EngineConfig(loop_count=args.loops, slots_per_loop=4096,
max_pipeline=1024, encryption=args.encryption,
utp=args.utp)
print(f"transport: {'µTP' if args.utp else 'TCP'}, "
f"encryption={['off','offer','require'][args.encryption]}", flush=True)
eng = Engine(cfg, lib_path=args.lib)
tid = eng.add_torrent(meta.info_hash, b"-PC0001-" + secrets.token_bytes(12),
meta.piece_length, meta.total_size, meta.num_pieces)
eng.set_priorities(tid, [0 if done[i] else 1
for i in range(meta.num_pieces)])
assembler = PieceAssembler(meta, done)
endgame = None if args.no_endgame else EndgameController(
meta,
min_pieces=max(1, args.endgame_min_pieces),
peer_factor=max(1.0, args.endgame_peer_factor),
interval=max(0.5, args.endgame_interval),
)
added: set = set()
peak_connected = 0
useful_bytes = _completed_bytes(meta, done)
t0 = time.time()
deadline = t0 + args.timeout
last_print = 0.0
last_useful_bytes = useful_bytes
last_progress_t = t0
try:
while done_count < meta.num_pieces:
# Feed any newly-discovered peers to the engine, up to the cap.
if len(added) < args.max_peers:
for ip, port in discovery.collect(args.max_peers):
if (ip, port) in added:
continue
added.add((ip, port))
try:
eng.add_peer(tid, ip, port)
except Exception:
pass
if len(added) >= args.max_peers:
break
st = eng.status(tid)
peak_connected = max(peak_connected, st.peers_connected)
now = time.time()
if endgame:
endgame.maybe_rearm(
eng, tid, done, done_count, st.peers_connected, now)
descs = eng.poll_ready()
if not descs:
eng.wait(200)
now = time.time()
if useful_bytes != last_useful_bytes:
last_useful_bytes = useful_bytes
last_progress_t = now
if now - last_print >= 1.0:
last_print = now
pct = 100.0 * done_count / meta.num_pieces
print(f" {done_count}/{meta.num_pieces} pieces ({pct:4.1f}%) "
f"{st.rate_bps/1e6:6.1f} MB/s "
f"peers {st.peers_connected} up / {st.peers_failed} failed "
f"/ {len(added)} tried outstanding={st.outstanding}",
flush=True)
if now > deadline or (now - last_progress_t) > args.timeout:
print("stalled; giving up", flush=True)
break
continue
for x in descs:
if done[x.piece]:
# A block can arrive after the piece was completed and
# de-prioritized because it was already in flight. Drop it.
eng.release(x.loop, x.slot)
continue
block = bytes(eng.block_data(x.loop, x.slot, x.len))
eng.release(x.loop, x.slot)
added_unique, complete = assembler.add_block(
x.piece, x.begin, block)
if added_unique:
useful_bytes += x.len
if complete:
piece_data = assembler.piece_bytes(x.piece)
if hashlib.sha1(piece_data).digest() != meta.piece_hashes[x.piece]:
# Corrupt/garbage block from a misbehaving peer: re-arm.
useful_bytes -= assembler.reset_piece(x.piece)
eng.request_piece(tid, x.piece)
continue
done[x.piece] = 1
done_count += 1
eng.set_priority(tid, x.piece, 0)
if out_fh:
out_fh.seek(x.piece * meta.piece_length)
out_fh.write(piece_data)
out_fh.flush()
assembler.finish_piece(x.piece) # free as we go
dt = time.time() - t0
st = eng.status(tid)
mb = _completed_bytes(meta, done) / 1e6
print("-" * 70)
print(f"downloaded {done_count}/{meta.num_pieces} pieces "
f"({mb:.1f} MB) in {dt:.1f}s = {mb/dt:.1f} MB/s" if dt > 0 else "")
print(f"peers: {len(added)} discovered+tried, "
f"{peak_connected} usable at peak, "
f"{st.peers_failed} failed (likely incompatible transport/"
f"encryption or unreachable)")
if done_count < meta.num_pieces:
print(f"engine state: {STATE_NAMES[st.state]} "
f"err={ERROR_NAMES[st.error]}")
return 0 if done_count == meta.num_pieces else 2
finally:
if out_fh:
out_fh.close()
eng.close()
discovery.close()
if __name__ == "__main__":
raise SystemExit(main())

223
harness/torrent_meta.py Normal file
View file

@ -0,0 +1,223 @@
from __future__ import annotations
import hashlib
import os
from dataclasses import dataclass
from typing import Any
class BencodeError(ValueError):
pass
class BDecoder:
def __init__(self, data: bytes):
self.data = data
self.info_span: tuple[int, int] | None = None
def parse(self) -> Any:
value, pos = self._value(0, top=True)
if pos != len(self.data):
raise BencodeError(f"trailing data at byte {pos}")
return value
def _value(self, pos: int, *, top: bool = False) -> tuple[Any, int]:
if pos >= len(self.data):
raise BencodeError("unexpected end of bencode")
c = self.data[pos]
if c == ord("i"):
return self._int(pos)
if c == ord("l"):
return self._list(pos)
if c == ord("d"):
return self._dict(pos, top=top)
if ord("0") <= c <= ord("9"):
return self._bytes(pos)
raise BencodeError(f"invalid bencode byte {c!r} at {pos}")
def _int(self, pos: int) -> tuple[int, int]:
end = self.data.find(b"e", pos)
if end < 0:
raise BencodeError("unterminated integer")
raw = self.data[pos + 1:end]
if not raw:
raise BencodeError("empty integer")
return int(raw), end + 1
def _bytes(self, pos: int) -> tuple[bytes, int]:
colon = self.data.find(b":", pos)
if colon < 0:
raise BencodeError("unterminated byte string length")
n = int(self.data[pos:colon])
start = colon + 1
end = start + n
if end > len(self.data):
raise BencodeError("byte string exceeds input")
return self.data[start:end], end
def _list(self, pos: int) -> tuple[list[Any], int]:
out: list[Any] = []
pos += 1
while pos < len(self.data) and self.data[pos] != ord("e"):
value, pos = self._value(pos)
out.append(value)
if pos >= len(self.data):
raise BencodeError("unterminated list")
return out, pos + 1
def _dict(self, pos: int, *, top: bool = False) -> tuple[dict[bytes, Any], int]:
out: dict[bytes, Any] = {}
pos += 1
while pos < len(self.data) and self.data[pos] != ord("e"):
key, pos = self._bytes(pos)
value_start = pos
value, pos = self._value(pos)
if top and key == b"info":
self.info_span = (value_start, pos)
out[key] = value
if pos >= len(self.data):
raise BencodeError("unterminated dict")
return out, pos + 1
@dataclass
class Metadata:
info_hash: bytes
piece_length: int
total_size: int
num_pieces: int
piece_hashes: list[bytes]
name: str
def piece_len(self, index: int) -> int:
if index + 1 == self.num_pieces:
return self.total_size - index * self.piece_length
return self.piece_length
@dataclass
class TorrentFile:
path: str
raw: bytes
metainfo: dict[bytes, Any]
info: dict[bytes, Any]
info_raw: bytes
metadata: Metadata
trackers: list[str]
files: list[tuple[str, int]]
def _text(value: Any, default: str = "") -> str:
if isinstance(value, bytes):
return value.decode("utf-8", "replace")
return default
def _file_tree_size(node: Any) -> int:
if not isinstance(node, dict):
return 0
total = 0
file_marker = node.get(b"")
if isinstance(file_marker, dict):
total += int(file_marker.get(b"length", 0))
for key, child in node.items():
if key != b"":
total += _file_tree_size(child)
return total
def _total_size(info: dict[bytes, Any]) -> int:
if b"length" in info:
return int(info[b"length"])
if b"files" in info:
return sum(int(f.get(b"length", 0)) for f in info[b"files"])
if b"file tree" in info:
return _file_tree_size(info[b"file tree"])
return 0
def _trackers(meta: dict[bytes, Any]) -> list[str]:
urls: list[str] = []
announce = meta.get(b"announce")
if isinstance(announce, bytes):
urls.append(_text(announce))
tiers = meta.get(b"announce-list")
if isinstance(tiers, list):
for tier in tiers:
if not isinstance(tier, list):
continue
for item in tier:
if isinstance(item, bytes):
urls.append(_text(item))
seen: set[str] = set()
out: list[str] = []
for url in urls:
if url and url not in seen:
seen.add(url)
out.append(url)
return out
def _path_text(parts: list[Any]) -> str:
decoded = []
for part in parts:
if not isinstance(part, bytes):
raise BencodeError("file path component is not bytes")
decoded.append(part.decode("utf-8", "replace"))
return os.path.join(*decoded) if decoded else ""
def _files(info: dict[bytes, Any]) -> list[tuple[str, int]]:
name = _text(info.get(b"name"), "")
if b"length" in info:
return [(name, int(info[b"length"]))]
files = info.get(b"files")
if isinstance(files, list):
out = []
for entry in files:
if not isinstance(entry, dict):
continue
path = entry.get(b"path")
if not isinstance(path, list):
continue
out.append((os.path.join(name, _path_text(path)),
int(entry.get(b"length", 0))))
return out
return []
def load_torrent(path: str) -> TorrentFile:
raw = open(path, "rb").read()
dec = BDecoder(raw)
meta = dec.parse()
if not isinstance(meta, dict) or dec.info_span is None:
raise BencodeError("metainfo does not contain a top-level info dict")
info = meta.get(b"info")
if not isinstance(info, dict):
raise BencodeError("metainfo info value is not a dict")
info_raw = raw[dec.info_span[0]:dec.info_span[1]]
pieces = info.get(b"pieces")
if not isinstance(pieces, bytes) or len(pieces) % 20 != 0:
raise BencodeError("only v1/hybrid torrents with a valid pieces string are supported")
piece_length = int(info.get(b"piece length", 0))
if piece_length <= 0:
raise BencodeError("missing or invalid piece length")
piece_hashes = [pieces[i:i + 20] for i in range(0, len(pieces), 20)]
total_size = _total_size(info)
if total_size <= 0:
raise BencodeError("missing torrent payload size")
metadata = Metadata(
info_hash=hashlib.sha1(info_raw).digest(),
piece_length=piece_length,
total_size=total_size,
num_pieces=len(piece_hashes),
piece_hashes=piece_hashes,
name=_text(info.get(b"name"), os.path.basename(path)),
)
return TorrentFile(path, raw, meta, info, info_raw, metadata, _trackers(meta),
_files(info))
def load_metadata(path: str) -> Metadata:
return load_torrent(path).metadata

498
harness/tracker_ffi.py Normal file
View file

@ -0,0 +1,498 @@
from __future__ import annotations
import ctypes as C
import os
import random
import socket
import ssl
import struct
import time
from dataclasses import dataclass
from urllib.parse import urlsplit, urlunsplit
from urllib.request import Request, urlopen
TRACKER_OK = 0
TRACKER_EVENT_NONE = 0
TRACKER_EVENT_COMPLETED = 1
TRACKER_EVENT_STARTED = 2
TRACKER_EVENT_STOPPED = 3
TRACKER_ADDR_IPV4 = 4
TRACKER_ADDR_IPV6 = 6
TRACKER_MAX_PEERS = 256
TRACKER_MAX_URL_DATA = 512
DHT_MAX_TRANSACTION = 16
DHT_MAX_TOKEN = 64
DHT_MAX_NODES = 256
DHT_MAX_ERROR = 128
DHT_MSG_QUERY = 1
DHT_MSG_RESPONSE = 2
DHT_QUERY_GET_PEERS = 3
DEFAULT_DHT_BOOTSTRAP = (
("router.bittorrent.com", 6881),
("dht.transmissionbt.com", 6881),
("router.utorrent.com", 6881),
)
class TrackerPeer(C.Structure):
_fields_ = [
("family", C.c_uint8),
("addr", C.c_uint8 * 16),
("port", C.c_uint16),
("peer_id", C.c_uint8 * 20),
("has_peer_id", C.c_uint8),
]
class TrackerAnnounceRequest(C.Structure):
_fields_ = [
("info_hash", C.c_uint8 * 20),
("peer_id", C.c_uint8 * 20),
("port", C.c_uint16),
("uploaded", C.c_uint64),
("downloaded", C.c_uint64),
("left", C.c_uint64),
("numwant", C.c_int32),
("key", C.c_uint32),
("ip4", C.c_uint32),
("event", C.c_int),
("compact", C.c_uint8),
("no_peer_id", C.c_uint8),
("has_key", C.c_uint8),
("has_ip4", C.c_uint8),
("ip", C.c_char * 64),
("tracker_id", C.c_char * 128),
("url_data", C.c_char * TRACKER_MAX_URL_DATA),
]
class TrackerAnnounceResponse(C.Structure):
_fields_ = [
("interval", C.c_uint32),
("min_interval", C.c_uint32),
("complete", C.c_uint32),
("incomplete", C.c_uint32),
("tracker_id", C.c_char_p),
("peers", C.POINTER(TrackerPeer)),
("peer_count", C.c_size_t),
("compact", C.c_uint8),
]
class DHTNode(C.Structure):
_fields_ = [
("id", C.c_uint8 * 20),
("family", C.c_uint8),
("addr", C.c_uint8 * 16),
("port", C.c_uint16),
]
class DHTMessage(C.Structure):
_fields_ = [
("type", C.c_int),
("query", C.c_int),
("transaction", C.c_uint8 * DHT_MAX_TRANSACTION),
("transaction_len", C.c_size_t),
("id", C.c_uint8 * 20),
("target", C.c_uint8 * 20),
("info_hash", C.c_uint8 * 20),
("port", C.c_uint16),
("implied_port", C.c_uint8),
("want_ipv4", C.c_uint8),
("want_ipv6", C.c_uint8),
("token", C.c_uint8 * DHT_MAX_TOKEN),
("token_len", C.c_size_t),
("nodes", DHTNode * DHT_MAX_NODES),
("node_count", C.c_size_t),
("peers", TrackerPeer * TRACKER_MAX_PEERS),
("peer_count", C.c_size_t),
("error_code", C.c_int),
("error_message", C.c_char * DHT_MAX_ERROR),
]
@dataclass
class TrackerResult:
tracker: str
ok: bool
protocol: str
peers: list[tuple[str, int]]
interval: int = 0
complete: int = 0
incomplete: int = 0
error: str = ""
elapsed_ms: float = 0.0
@dataclass
class DHTResult:
peers: list[tuple[str, int]]
nodes_queried: int
nodes_discovered: int
elapsed_ms: float
error: str = ""
def _default_lib_path() -> str:
env = os.environ.get("TORRENT_TRACKER_LIB")
if env:
return env
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
candidates = [
os.path.join(root, "..", "torrent-tracker", "build", "libtorrenttracker.so"),
os.path.join(root, "build", "libtorrenttracker.so"),
]
for path in candidates:
if os.path.exists(path):
return os.path.abspath(path)
return os.path.abspath(candidates[0])
def _load(lib_path: str | None = None) -> C.CDLL:
lib = C.CDLL(lib_path or _default_lib_path())
lib.tracker_http_write_announce_query.restype = C.c_int
lib.tracker_http_write_announce_query.argtypes = [
C.POINTER(TrackerAnnounceRequest), C.c_char_p, C.c_size_t, C.POINTER(C.c_size_t)]
lib.tracker_http_parse_announce_response.restype = C.c_int
lib.tracker_http_parse_announce_response.argtypes = [
C.c_void_p, C.c_size_t, C.POINTER(TrackerPeer), C.c_size_t,
C.POINTER(TrackerAnnounceResponse)]
lib.tracker_udp_write_connect_request.restype = C.c_int
lib.tracker_udp_write_connect_request.argtypes = [
C.c_uint32, C.c_void_p, C.c_size_t, C.POINTER(C.c_size_t)]
lib.tracker_udp_parse_connect_response.restype = C.c_int
lib.tracker_udp_parse_connect_response.argtypes = [
C.c_void_p, C.c_size_t, C.c_uint32, C.POINTER(C.c_uint64)]
lib.tracker_udp_write_announce_request.restype = C.c_int
lib.tracker_udp_write_announce_request.argtypes = [
C.c_uint64, C.c_uint32, C.POINTER(TrackerAnnounceRequest),
C.c_void_p, C.c_size_t, C.POINTER(C.c_size_t)]
lib.tracker_udp_parse_announce_response.restype = C.c_int
lib.tracker_udp_parse_announce_response.argtypes = [
C.c_void_p, C.c_size_t, C.c_uint32, C.c_int, C.POINTER(TrackerPeer),
C.c_size_t, C.POINTER(TrackerAnnounceResponse)]
lib.dht_write_get_peers_query.restype = C.c_int
lib.dht_write_get_peers_query.argtypes = [
C.c_void_p, C.c_size_t, C.c_void_p, C.c_void_p, C.c_uint8, C.c_uint8,
C.c_void_p, C.c_size_t, C.POINTER(C.c_size_t)]
lib.dht_parse_message.restype = C.c_int
lib.dht_parse_message.argtypes = [C.c_void_p, C.c_size_t, C.POINTER(DHTMessage)]
lib.dht_write_peers_response.restype = C.c_int
lib.dht_write_peers_response.argtypes = [
C.c_void_p, C.c_size_t, C.c_void_p, C.c_void_p, C.c_size_t,
C.POINTER(TrackerPeer), C.c_size_t, C.c_void_p, C.c_size_t,
C.POINTER(C.c_size_t)]
return lib
class TrackerClient:
def __init__(self, lib_path: str | None = None):
self.lib = _load(lib_path)
def _request(self, meta, peer_id: bytes, port: int, key: int, numwant: int,
event: str) -> TrackerAnnounceRequest:
req = TrackerAnnounceRequest()
C.memset(C.byref(req), 0, C.sizeof(req))
req.info_hash[:] = meta.info_hash
req.peer_id[:] = peer_id
req.port = port
req.left = meta.total_size
req.numwant = numwant
req.key = key
req.has_key = 1
req.compact = 1
req.no_peer_id = 1
req.event = {
"completed": TRACKER_EVENT_COMPLETED,
"started": TRACKER_EVENT_STARTED,
"stopped": TRACKER_EVENT_STOPPED,
}.get(event, TRACKER_EVENT_NONE)
return req
@staticmethod
def _peers(peers, count: int) -> list[tuple[str, int]]:
out: list[tuple[str, int]] = []
for i in range(count):
p = peers[i]
if p.family == TRACKER_ADDR_IPV4:
host = socket.inet_ntop(socket.AF_INET, bytes(p.addr[:4]))
elif p.family == TRACKER_ADDR_IPV6:
host = socket.inet_ntop(socket.AF_INET6, bytes(p.addr[:16]))
else:
continue
out.append((host, int(p.port)))
return out
def announce_http(self, url: str, meta, peer_id: bytes, port: int, key: int,
numwant: int, event: str, timeout: float) -> TrackerResult:
start = time.monotonic()
try:
req = self._request(meta, peer_id, port, key, numwant, event)
query = C.create_string_buffer(2048)
written = C.c_size_t()
rc = self.lib.tracker_http_write_announce_query(
C.byref(req), query, C.sizeof(query), C.byref(written))
if rc != TRACKER_OK:
raise RuntimeError(f"tracker_http_write_announce_query failed: {rc}")
parts = urlsplit(url)
q = parts.query
suffix = query.value.decode("ascii")
q = f"{q}&{suffix}" if q else suffix
announce_url = urlunsplit((parts.scheme, parts.netloc, parts.path, q,
parts.fragment))
request = Request(announce_url,
headers={"User-Agent": "torrent-peer/0.1"})
ctx = ssl.create_default_context()
with urlopen(request, timeout=timeout, context=ctx) as resp:
raw = resp.read(2 * 1024 * 1024)
raw_buf = C.create_string_buffer(raw, len(raw))
out_peers = (TrackerPeer * TRACKER_MAX_PEERS)()
parsed = TrackerAnnounceResponse()
rc = self.lib.tracker_http_parse_announce_response(
raw_buf, len(raw), out_peers, TRACKER_MAX_PEERS, C.byref(parsed))
if rc != TRACKER_OK:
raise RuntimeError(f"tracker_http_parse_announce_response failed: {rc}")
return TrackerResult(
tracker=url,
ok=True,
protocol=parts.scheme,
peers=self._peers(out_peers, parsed.peer_count),
interval=int(parsed.interval),
complete=int(parsed.complete),
incomplete=int(parsed.incomplete),
elapsed_ms=(time.monotonic() - start) * 1000.0,
)
except Exception as exc:
return TrackerResult(url, False, "http", [], error=str(exc),
elapsed_ms=(time.monotonic() - start) * 1000.0)
@staticmethod
def _url_data(url: str) -> bytes:
parts = urlsplit(url)
data = (parts.path or "").encode("utf-8")
if parts.query:
data += b"?" + parts.query.encode("utf-8")
return data[:TRACKER_MAX_URL_DATA - 1]
@staticmethod
def _roundtrip(sock: socket.socket, packet: bytes, txid: int,
timeout: float) -> bytes:
deadline = time.monotonic() + timeout
delay = min(timeout, 1.0)
while True:
sock.send(packet)
remaining = deadline - time.monotonic()
if remaining <= 0:
raise TimeoutError("UDP tracker timed out")
sock.settimeout(min(delay, remaining))
try:
raw = sock.recv(65535)
except socket.timeout:
delay = min(delay * 2.0, 8.0)
continue
if len(raw) >= 8 and struct.unpack_from("!I", raw, 4)[0] == txid:
return raw
def announce_udp(self, url: str, meta, peer_id: bytes, port: int, key: int,
numwant: int, event: str, timeout: float) -> TrackerResult:
start = time.monotonic()
parts = urlsplit(url)
if not parts.hostname:
return TrackerResult(url, False, "udp", [], error="missing UDP tracker host")
tracker_port = parts.port or 80
try:
infos = socket.getaddrinfo(parts.hostname, tracker_port, 0,
socket.SOCK_DGRAM)
last_error: Exception | None = None
for family, socktype, proto, _canon, sockaddr in infos:
if family not in (socket.AF_INET, socket.AF_INET6):
continue
try:
with socket.socket(family, socktype, proto) as sock:
sock.connect(sockaddr)
txid = random.getrandbits(32)
buf = C.create_string_buffer(2048)
written = C.c_size_t()
rc = self.lib.tracker_udp_write_connect_request(
txid, buf, C.sizeof(buf), C.byref(written))
if rc != TRACKER_OK:
raise RuntimeError(f"connect request failed: {rc}")
raw = self._roundtrip(sock, buf.raw[:written.value], txid, timeout)
conn_id = C.c_uint64()
raw_buf = C.create_string_buffer(raw, len(raw))
rc = self.lib.tracker_udp_parse_connect_response(
raw_buf, len(raw), txid, C.byref(conn_id))
if rc != TRACKER_OK:
raise RuntimeError(f"connect response failed: {rc}")
req = self._request(meta, peer_id, port, key, numwant, event)
url_data = self._url_data(url)
if url_data:
req.url_data = url_data
txid = random.getrandbits(32)
rc = self.lib.tracker_udp_write_announce_request(
conn_id.value, txid, C.byref(req), buf, C.sizeof(buf),
C.byref(written))
if rc != TRACKER_OK:
raise RuntimeError(f"announce request failed: {rc}")
raw = self._roundtrip(sock, buf.raw[:written.value], txid, timeout)
raw_buf = C.create_string_buffer(raw, len(raw))
out_peers = (TrackerPeer * TRACKER_MAX_PEERS)()
parsed = TrackerAnnounceResponse()
tracker_family = (TRACKER_ADDR_IPV6 if family == socket.AF_INET6
else TRACKER_ADDR_IPV4)
rc = self.lib.tracker_udp_parse_announce_response(
raw_buf, len(raw), txid, tracker_family, out_peers,
TRACKER_MAX_PEERS, C.byref(parsed))
if rc != TRACKER_OK:
raise RuntimeError(f"announce response failed: {rc}")
return TrackerResult(
tracker=url,
ok=True,
protocol="udp",
peers=self._peers(out_peers, parsed.peer_count),
interval=int(parsed.interval),
complete=int(parsed.complete),
incomplete=int(parsed.incomplete),
elapsed_ms=(time.monotonic() - start) * 1000.0,
)
except Exception as exc:
last_error = exc
continue
raise RuntimeError(str(last_error or "no usable tracker address"))
except Exception as exc:
return TrackerResult(url, False, "udp", [], error=str(exc),
elapsed_ms=(time.monotonic() - start) * 1000.0)
def announce(self, url: str, meta, peer_id: bytes, port: int, key: int,
numwant: int = 50, event: str = "started",
timeout: float = 8.0) -> TrackerResult:
scheme = urlsplit(url).scheme.lower()
if scheme in ("http", "https"):
return self.announce_http(url, meta, peer_id, port, key, numwant,
event, timeout)
if scheme == "udp":
return self.announce_udp(url, meta, peer_id, port, key, numwant,
event, timeout)
return TrackerResult(url, False, scheme or "unknown", [],
error=f"unsupported tracker scheme {scheme!r}")
class DHTClient:
def __init__(self, lib_path: str | None = None,
bootstrap: tuple[tuple[str, int], ...] = DEFAULT_DHT_BOOTSTRAP):
self.lib = _load(lib_path)
self.bootstrap = bootstrap
self.node_id = os.urandom(20)
self._tx = random.randrange(1, 0xffff)
@staticmethod
def _peer_endpoint(peer: TrackerPeer) -> tuple[str, int] | None:
if peer.family == TRACKER_ADDR_IPV4:
host = socket.inet_ntop(socket.AF_INET, bytes(peer.addr[:4]))
elif peer.family == TRACKER_ADDR_IPV6:
host = socket.inet_ntop(socket.AF_INET6, bytes(peer.addr[:16]))
else:
return None
return host, int(peer.port)
@staticmethod
def _node_endpoint(node: DHTNode) -> tuple[str, int] | None:
if node.family == TRACKER_ADDR_IPV4:
host = socket.inet_ntop(socket.AF_INET, bytes(node.addr[:4]))
elif node.family == TRACKER_ADDR_IPV6:
host = socket.inet_ntop(socket.AF_INET6, bytes(node.addr[:16]))
else:
return None
return host, int(node.port)
def _next_tx(self) -> bytes:
self._tx = (self._tx + 1) & 0xffff
return self._tx.to_bytes(2, "big")
def _get_peers_packet(self, info_hash: bytes, tx: bytes) -> bytes:
buf = C.create_string_buffer(2048)
written = C.c_size_t()
tx_buf = C.create_string_buffer(tx, len(tx))
id_buf = C.create_string_buffer(self.node_id, len(self.node_id))
hash_buf = C.create_string_buffer(info_hash, len(info_hash))
rc = self.lib.dht_write_get_peers_query(
tx_buf, len(tx), id_buf, hash_buf, 1, 1, buf, C.sizeof(buf),
C.byref(written))
if rc != TRACKER_OK:
raise RuntimeError(f"dht_write_get_peers_query failed: {rc}")
return buf.raw[:written.value]
def lookup(self, info_hash: bytes, *, timeout: float = 6.0,
max_queries: int = 32, max_peers: int = 100) -> DHTResult:
start = time.monotonic()
deadline = start + timeout
peers: set[tuple[str, int]] = set()
queue: list[tuple[str, int]] = list(self.bootstrap)
seen_nodes: set[tuple[str, int]] = set()
queried = 0
discovered = 0
last_error = ""
while queue and queried < max_queries and len(peers) < max_peers:
if time.monotonic() >= deadline:
break
host, port = queue.pop(0)
if (host, port) in seen_nodes:
continue
seen_nodes.add((host, port))
queried += 1
remaining = max(0.05, deadline - time.monotonic())
try:
infos = socket.getaddrinfo(host, port, 0, socket.SOCK_DGRAM)
except OSError as exc:
last_error = str(exc)
continue
for family, socktype, proto, _canon, sockaddr in infos:
if family not in (socket.AF_INET, socket.AF_INET6):
continue
tx = self._next_tx()
packet = self._get_peers_packet(info_hash, tx)
try:
with socket.socket(family, socktype, proto) as sock:
sock.settimeout(min(1.0, remaining))
sock.sendto(packet, sockaddr)
raw, _addr = sock.recvfrom(4096)
except OSError as exc:
last_error = str(exc)
continue
msg = DHTMessage()
raw_buf = C.create_string_buffer(raw, len(raw))
rc = self.lib.dht_parse_message(raw_buf, len(raw), C.byref(msg))
if rc != TRACKER_OK:
last_error = f"dht_parse_message failed: {rc}"
continue
got_tx = bytes(msg.transaction[:msg.transaction_len])
if got_tx != tx or msg.type != DHT_MSG_RESPONSE:
continue
for i in range(int(msg.peer_count)):
ep = self._peer_endpoint(msg.peers[i])
if ep:
peers.add(ep)
for i in range(int(msg.node_count)):
ep = self._node_endpoint(msg.nodes[i])
if ep and ep not in seen_nodes and ep not in queue:
queue.append(ep)
discovered += 1
break
return DHTResult(
peers=sorted(peers),
nodes_queried=queried,
nodes_discovered=discovered,
elapsed_ms=(time.monotonic() - start) * 1000.0,
error="" if peers else last_error,
)