Cieric/Naut-Peer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Initial commit: multi-peer torrent download engine

Browse source 
Reactor/loop-pool engine with TCP/µTP/MSE transports, per-connection
pipelining, priority-driven piece selection with endgame, and the Python
FFI test harness.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
ookami125 2026-06-21 23:12:32 -04:00
commit d8208685a2
55 changed files with 9989 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

206
tests/test_engine.py Normal file
View file

@ -0,0 +1,206 @@
"""
Engine-level tests for the multi-peer / multi-torrent download engine
(include/engine.h, harness/engine_ffi.py):
* test_two_torrents - one engine, two torrents downloaded concurrently;
each is pinned to a loop and verified byte-for-byte.
* test_two_peers - one torrent fed by two independent seeds; both
connections share the torrent's "requested" state, so
work is split with no duplicate requests, and the file
still verifies.
These exercise the keystone restructure (reactor/loop-pool + transport vtable)
directly, alongside the legacy single-peer path covered by test_localseed.py.
Run with: python tests/test_engine.py
"""
from __future__ import annotations
import hashlib
import os
import sys
import tempfile
import time
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(ROOT, "harness"))
import libtorrent as lt # noqa: E402
from harness import load_metadata # noqa: E402
from engine_ffi import Engine, EngineConfig, STATE_ERROR, ERROR_NAMES # noqa: E402
from test_localseed import ensure_built, make_torrent, start_full_seed # noqa: E402
LIB = os.path.join(ROOT, "build", "libtorrentpeer.so")
def make_peer_id() -> bytes:
return b"-PC0001-" + os.urandom(12)
class TorrentDrive:
"""Per-torrent reassembly + verification state."""
def __init__(self, meta):
self.meta = meta
self.buffers = [bytearray(meta.piece_len(i)) for i in range(meta.num_pieces)]
self.seen = [set() for _ in range(meta.num_pieces)]
self.received = [0] * meta.num_pieces
self.done = bytearray(meta.num_pieces)
self.done_count = 0
self.order = []
def drive_engine(eng: Engine, drives: dict[int, TorrentDrive], timeout=60.0):
"""Pump the engine until every torrent's pieces are verified."""
want = {tid: d.meta.num_pieces for tid, d in drives.items()}
deadline = time.time() + timeout
while any(drives[tid].done_count < want[tid] for tid in drives):
for tid, d in drives.items():
st = eng.status(tid)
if st.state == STATE_ERROR:
raise RuntimeError(f"torrent {tid} error: {ERROR_NAMES[st.error]}")
descs = eng.poll_ready()
if not descs:
eng.wait(100)
if time.time() > deadline:
raise TimeoutError(
{tid: drives[tid].done_count for tid in drives})
continue
for x in descs:
d = drives[x.torrent]
meta = d.meta
buf = d.buffers[x.piece]
first_copy = x.begin not in d.seen[x.piece]
if first_copy:
buf[x.begin:x.begin + x.len] = eng.block_data(x.loop, x.slot, x.len)
d.seen[x.piece].add(x.begin)
d.received[x.piece] += x.len
eng.release(x.loop, x.slot)
if (not d.done[x.piece]
and d.received[x.piece] >= meta.piece_len(x.piece)):
if hashlib.sha1(bytes(buf)).digest() != meta.piece_hashes[x.piece]:
raise ValueError(f"torrent {x.torrent} piece {x.piece} mismatch")
d.done[x.piece] = 1
d.done_count += 1
d.order.append(x.piece)
eng.set_priority(x.torrent, x.piece, 0)
def test_two_torrents():
ensure_built()
size = 6 * 1024 * 1024
with tempfile.TemporaryDirectory() as root_a, \
tempfile.TemporaryDirectory() as root_b:
ta, data_a, _ = make_torrent(root_a, size)
tb, data_b, _ = make_torrent(root_b, size)
meta_a, meta_b = load_metadata(ta), load_metadata(tb)
ses_a, h_a, port_a = start_full_seed(root_a, ta)
ses_b, h_b, port_b = start_full_seed(root_b, tb)
try:
with Engine(EngineConfig(loop_count=2, slots_per_loop=1024,
max_pipeline=512), lib_path=LIB) as eng:
pid = make_peer_id()
tid_a = eng.add_torrent(meta_a.info_hash, pid, meta_a.piece_length,
meta_a.total_size, meta_a.num_pieces)
tid_b = eng.add_torrent(meta_b.info_hash, pid, meta_b.piece_length,
meta_b.total_size, meta_b.num_pieces)
eng.set_priorities(tid_a, [1] * meta_a.num_pieces)
eng.set_priorities(tid_b, [1] * meta_b.num_pieces)
eng.add_peer(tid_a, "127.0.0.1", port_a)
eng.add_peer(tid_b, "127.0.0.1", port_b)
drives = {tid_a: TorrentDrive(meta_a),
tid_b: TorrentDrive(meta_b)}
t0 = time.time()
drive_engine(eng, drives, timeout=60.0)
dt = time.time() - t0
got_a = b"".join(bytes(b) for b in drives[tid_a].buffers)
got_b = b"".join(bytes(b) for b in drives[tid_b].buffers)
assert got_a == data_a, "torrent A bytes differ"
assert got_b == data_b, "torrent B bytes differ"
finally:
ses_a.remove_torrent(h_a)
ses_b.remove_torrent(h_b)
print(f"two torrents OK: 2 x {size/1e6:.1f} MB in {dt:.2f}s")
def test_two_peers():
ensure_built()
size = 8 * 1024 * 1024
with tempfile.TemporaryDirectory() as root:
torrent, original, _ = make_torrent(root, size)
meta = load_metadata(torrent)
# Two independent seeds of the same content -> two peers for one torrent.
ses1, h1, port1 = start_full_seed(root, torrent)
ses2, h2, port2 = start_full_seed(root, torrent)
try:
with Engine(EngineConfig(loop_count=1, slots_per_loop=1024,
max_pipeline=256), lib_path=LIB) as eng:
tid = eng.add_torrent(meta.info_hash, make_peer_id(),
meta.piece_length, meta.total_size,
meta.num_pieces)
eng.set_priorities(tid, [1] * meta.num_pieces)
eng.add_peer(tid, "127.0.0.1", port1)
eng.add_peer(tid, "127.0.0.1", port2)
drives = {tid: TorrentDrive(meta)}
drive_engine(eng, drives, timeout=60.0)
got = b"".join(bytes(b) for b in drives[tid].buffers)
assert got == original, "two-peer download bytes differ"
st = eng.status(tid)
assert st.peers == 2, f"expected 2 peers, got {st.peers}"
finally:
ses1.remove_torrent(h1)
ses2.remove_torrent(h2)
print(f"two peers OK: {size/1e6:.1f} MB via 2 connections, no duplicate blocks")
def test_peer_failure_releases_claims():
ensure_built()
size = 4 * 1024 * 1024
with tempfile.TemporaryDirectory() as root:
torrent, original, _ = make_torrent(root, size)
meta = load_metadata(torrent)
ses1, h1, port1 = start_full_seed(root, torrent)
ses2, h2, port2 = start_full_seed(root, torrent)
removed1 = False
try:
with Engine(EngineConfig(loop_count=1, slots_per_loop=64,
max_pipeline=4), lib_path=LIB) as eng:
tid = eng.add_torrent(meta.info_hash, make_peer_id(),
meta.piece_length, meta.total_size,
meta.num_pieces)
eng.set_priorities(tid, [1] * meta.num_pieces)
eng.add_peer(tid, "127.0.0.1", port1)
deadline = time.time() + 10.0
while time.time() < deadline:
st = eng.status(tid)
if st.outstanding > 0:
break
eng.wait(20)
else:
raise TimeoutError("first peer never issued requests")
ses1.remove_torrent(h1)
removed1 = True
eng.add_peer(tid, "127.0.0.1", port2)
drives = {tid: TorrentDrive(meta)}
drive_engine(eng, drives, timeout=60.0)
got = b"".join(bytes(b) for b in drives[tid].buffers)
assert got == original, "download after peer failure differs"
finally:
if not removed1:
ses1.remove_torrent(h1)
ses2.remove_torrent(h2)
print("peer failure recovery OK: dead-peer claims were released")
if __name__ == "__main__":
test_two_torrents()
test_two_peers()
test_peer_failure_releases_claims()