Initial commit: multi-peer torrent download engine
Reactor/loop-pool engine with TCP/µTP/MSE transports, per-connection pipelining, priority-driven piece selection with endgame, and the Python FFI test harness. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
ookami125
commit
d8208685a2
55 changed files with 9989 additions and 0 deletions
157
include/engine.h
Normal file
157
include/engine.h
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
/*
|
||||
* engine.h - Public ABI for the multi-peer download engine.
|
||||
*
|
||||
* The engine owns all peer connections via a fixed pool of event loops (one OS
|
||||
* thread each). Torrents are pinned to a loop ("affinity"); every connection of
|
||||
* a torrent lives on that loop, so each loop thread is the sole owner of its
|
||||
* connections, its arena, and its torrents' piece state. The hot path
|
||||
* (recv -> parse -> handoff -> schedule) is therefore lock-free.
|
||||
*
|
||||
* Data plane: each loop owns one arena slab and an SPSC ready-ring; the consumer
|
||||
* thread drains completed blocks across loops with engine_poll_ready() and
|
||||
* returns spent slots with engine_release_slot(). Control plane (add torrent,
|
||||
* add peer, set priorities) is delivered to the owning loop via a command queue.
|
||||
*
|
||||
* The legacy single-peer peer_* API (peer.h) is a thin wrapper over a 1-loop /
|
||||
* 1-torrent / 1-peer engine.
|
||||
*/
|
||||
#ifndef TORRENT_ENGINE_H
|
||||
#define TORRENT_ENGINE_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define PEER_BLOCK_SIZE 16384u /* BitTorrent block size */
|
||||
|
||||
/* Connection/torrent state, mirrored to status. */
|
||||
typedef enum {
|
||||
PEER_STATE_IDLE = 0,
|
||||
PEER_STATE_CONNECTING = 1,
|
||||
PEER_STATE_HANDSHAKE = 2,
|
||||
PEER_STATE_CHOKED = 3,
|
||||
PEER_STATE_RUNNING = 4,
|
||||
PEER_STATE_STOPPED = 5,
|
||||
PEER_STATE_ERROR = 6
|
||||
} peer_state;
|
||||
|
||||
typedef enum {
|
||||
PEER_OK = 0,
|
||||
PEER_ERR_CONNECT = 1,
|
||||
PEER_ERR_HANDSHAKE = 2,
|
||||
PEER_ERR_CLOSED = 3,
|
||||
PEER_ERR_PROTOCOL = 4,
|
||||
PEER_ERR_IO = 5,
|
||||
PEER_ERR_NOMEM = 6
|
||||
} peer_error;
|
||||
|
||||
typedef struct {
|
||||
uint32_t loop_count; /* event-loop threads (0 => min(ncpu, 8)) */
|
||||
uint32_t slots_per_loop; /* arena depth per loop in 16 KiB slots (0=>def)*/
|
||||
uint32_t max_pipeline; /* per-connection outstanding-request cap (0=>def)*/
|
||||
uint32_t request_timeout_ms;/* re-request a block after this long (0=>def) */
|
||||
uint32_t recv_buffer_bytes; /* SO_RCVBUF override; 0 => kernel autotuning */
|
||||
uint32_t encryption; /* 0 = plaintext only; 1 = MSE, offer RC4 +
|
||||
* plaintext (most compatible); 2 = MSE, require
|
||||
* RC4 (refuse plaintext) */
|
||||
uint32_t utp; /* 0 = TCP; 1 = µTP (UDP). May combine with
|
||||
* encryption to run MSE over µTP. */
|
||||
uint32_t connect_timeout_ms;/* drop a peer that hasn't finished connecting +
|
||||
* handshaking within this long (0 => 10000).
|
||||
* Reclaims unreachable/silent peers instead of
|
||||
* leaving them stuck. */
|
||||
uint32_t fallback; /* 1 => if a peer fails before the BitTorrent
|
||||
* handshake, retry the same endpoint over the
|
||||
* next transport/encryption combo (TCP+MSE ->
|
||||
* TCP+plain -> µTP+MSE -> µTP+plain, ordered by
|
||||
* the utp/encryption prefs above). Reaches far
|
||||
* more of a real swarm. 0 => single attempt. */
|
||||
} engine_config;
|
||||
|
||||
/*
|
||||
* One delivered block. Payload lives at:
|
||||
* (uint8_t*)engine_arena_base(e, loop) + (uint64_t)slot * PEER_BLOCK_SIZE
|
||||
* valid until returned via engine_release_slot(e, loop, slot).
|
||||
*/
|
||||
typedef struct {
|
||||
uint32_t torrent; /* torrent id this block belongs to */
|
||||
uint32_t piece;
|
||||
uint32_t begin;
|
||||
uint32_t len;
|
||||
uint32_t loop; /* arena that holds the slot */
|
||||
uint32_t slot; /* slot index within that loop's arena */
|
||||
} engine_block;
|
||||
|
||||
/* Aggregated status for one torrent. */
|
||||
typedef struct {
|
||||
int32_t state; /* peer_state (best connection's state) */
|
||||
int32_t error; /* peer_error of a failed connection, if any */
|
||||
uint64_t bytes_received;
|
||||
uint64_t blocks_received;
|
||||
uint32_t peers; /* connections attached (incl. failed) */
|
||||
uint32_t peers_connected; /* handshake completed (choked or running) */
|
||||
uint32_t peers_failed; /* connections that errored out */
|
||||
uint32_t outstanding; /* in-flight requests summed across peers */
|
||||
uint32_t free_slots; /* free arena slots on the torrent's loop */
|
||||
uint32_t pipeline_target; /* summed adaptive target across peers */
|
||||
double rate_bps; /* summed download rate */
|
||||
double rtt_min_ms; /* smallest observed request->block RTT */
|
||||
} torrent_status;
|
||||
|
||||
typedef struct engine engine;
|
||||
|
||||
/* Lifecycle. */
|
||||
engine *engine_create(const engine_config *cfg);
|
||||
void engine_destroy(engine *e);
|
||||
|
||||
/* Register a torrent. Returns its id (>= 0) or -1 on error. The arrays are
|
||||
* copied. piece_length/total_size/num_pieces describe the torrent geometry. */
|
||||
int32_t engine_add_torrent(engine *e, const uint8_t info_hash[20],
|
||||
const uint8_t peer_id[20], uint64_t piece_length,
|
||||
uint64_t total_size, uint32_t num_pieces);
|
||||
|
||||
/* Open a connection to a peer for a torrent (ip = dotted-quad or IPv6 literal). */
|
||||
int engine_add_peer(engine *e, uint32_t torrent_id, const char *ip, uint16_t port);
|
||||
|
||||
/* Priority vector / single priority / re-arm — see peer.h docs for semantics. */
|
||||
int engine_set_priorities(engine *e, uint32_t torrent_id,
|
||||
const uint8_t *priorities, uint32_t count);
|
||||
int engine_set_priority(engine *e, uint32_t torrent_id, uint32_t piece,
|
||||
uint8_t priority);
|
||||
int engine_request_piece(engine *e, uint32_t torrent_id, uint32_t piece);
|
||||
|
||||
/* Engine-wide download throttle in bytes/sec; 0 = unlimited (default). Bounds
|
||||
* the aggregate receive rate by gating outgoing block requests. Safe to call at
|
||||
* any time from any thread. */
|
||||
void engine_set_download_rate(engine *e, uint64_t bytes_per_sec);
|
||||
|
||||
/* Data plane (single consumer thread). */
|
||||
uint32_t engine_poll_ready(engine *e, engine_block *out, uint32_t max);
|
||||
void engine_release_slot(engine *e, uint32_t loop, uint32_t slot);
|
||||
int engine_wait(engine *e, int timeout_ms);
|
||||
|
||||
void *engine_arena_base(engine *e, uint32_t loop);
|
||||
uint64_t engine_arena_bytes(engine *e, uint32_t loop);
|
||||
uint32_t engine_loop_count(engine *e);
|
||||
|
||||
void engine_torrent_status(engine *e, uint32_t torrent_id, torrent_status *out);
|
||||
|
||||
/* Diagnostic: write a human-readable dump of one torrent's piece-selection and
|
||||
* per-connection state to `out`. Reports, for every still-wanted piece
|
||||
* (priority > 0), whether a peer has claimed it (requested), how many connected
|
||||
* peers advertise it (availability), and how many in-flight block requests it
|
||||
* has across all peers — the data needed to tell apart a stuck piece no peer
|
||||
* has, one a dead/idle peer claimed but never delivered, and one the scheduler
|
||||
* is simply not picking. Reads loop-owned state from the caller's thread without
|
||||
* locking (like engine_torrent_status), so it is a best-effort snapshot meant
|
||||
* for debugging, not control. */
|
||||
void engine_dump_torrent(engine *e, uint32_t torrent_id, FILE *out);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* TORRENT_ENGINE_H */
|
||||
134
include/peer.h
Normal file
134
include/peer.h
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
/*
|
||||
* peer.h - Legacy single-peer ABI, now a thin compatibility shim over the
|
||||
* multi-peer engine (engine.h). A peer_handle is a private 1-loop / 1-torrent /
|
||||
* 1-peer engine; the semantics below are unchanged so existing callers and
|
||||
* tests keep working. New code should use engine.h directly.
|
||||
*/
|
||||
#ifndef TORRENT_PEER_H
|
||||
#define TORRENT_PEER_H
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "engine.h" /* PEER_BLOCK_SIZE, peer_state, peer_error */
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Immutable configuration passed to peer_create(). The arrays are copied, so
|
||||
* the caller need not keep them alive afterwards.
|
||||
*/
|
||||
typedef struct {
|
||||
uint8_t info_hash[20]; /* torrent info-hash (BitTorrent v1, SHA-1) */
|
||||
uint8_t peer_id[20]; /* our 20-byte peer id */
|
||||
uint64_t piece_length; /* bytes per piece (last piece may be shorter) */
|
||||
uint64_t total_size; /* total torrent payload size */
|
||||
uint32_t num_pieces; /* number of pieces */
|
||||
uint32_t num_slots; /* arena depth in 16 KiB slots (0 => default) */
|
||||
uint32_t max_pipeline; /* cap on outstanding block requests (0 => def) */
|
||||
uint32_t request_timeout_ms; /* re-request a block after this long with no
|
||||
* reply (0 => default). Guards against silent
|
||||
* request drops. */
|
||||
uint32_t recv_buffer_bytes; /* SO_RCVBUF override; 0 => leave kernel
|
||||
* autotuning alone (recommended). */
|
||||
} peer_config;
|
||||
|
||||
/*
|
||||
* One received block. The payload lives at:
|
||||
* (uint8_t*)peer_arena_base(h) + (uint64_t)slot * PEER_BLOCK_SIZE
|
||||
* and stays valid until the slot is returned with peer_release_slot().
|
||||
* 16 bytes, trivially copyable, no pointers (ABI/relocation friendly).
|
||||
*/
|
||||
typedef struct {
|
||||
uint32_t piece; /* piece index */
|
||||
uint32_t begin; /* byte offset of this block within the piece */
|
||||
uint32_t len; /* block length in bytes (<= PEER_BLOCK_SIZE) */
|
||||
uint32_t slot; /* arena slot holding the payload */
|
||||
} block_desc;
|
||||
|
||||
/* Snapshot of peer progress; filled by peer_get_status(). */
|
||||
typedef struct {
|
||||
int32_t state; /* peer_state */
|
||||
int32_t error; /* peer_error */
|
||||
uint64_t bytes_received; /* total payload bytes delivered to ready ring*/
|
||||
uint64_t blocks_received; /* total blocks delivered */
|
||||
uint32_t outstanding; /* requests sent but not yet received */
|
||||
uint32_t free_slots; /* arena slots currently available */
|
||||
uint32_t pipeline_target; /* current adaptive in-flight target (blocks) */
|
||||
double rate_bps; /* EWMA download rate, bytes/sec */
|
||||
double rtt_min_ms; /* smallest observed request->block RTT */
|
||||
} peer_status;
|
||||
|
||||
typedef struct peer_handle peer_handle;
|
||||
|
||||
/* Lifecycle ------------------------------------------------------------- */
|
||||
|
||||
/* Allocate a peer and its arena/rings. Returns NULL on bad config or OOM. */
|
||||
peer_handle *peer_create(const peer_config *cfg);
|
||||
|
||||
/* Connect + handshake on a new network thread. ip is dotted-quad IPv4.
|
||||
* Returns 0 if the thread launched, negative on immediate failure. The actual
|
||||
* connection result surfaces asynchronously via peer_get_status(). */
|
||||
int peer_start(peer_handle *h, const char *ip, uint16_t port);
|
||||
|
||||
/*
|
||||
* Piece selection is priority-driven. The harness supplies one priority byte
|
||||
* per piece; the peer always works on the highest-priority piece that
|
||||
* (a) the remote peer actually HAS (per its bitfield/have messages), and
|
||||
* (b) has not already been fully requested,
|
||||
* breaking ties toward the lowest piece index. Priority 0 means "do not
|
||||
* request". Re-evaluation happens at every piece boundary, so updating
|
||||
* priorities while running steers the download live (sequential, rarest-first,
|
||||
* deadline ramps, or any mix). The peer never assumes the remote has a piece it
|
||||
* has not advertised, which is the key fix over a fixed in-order schedule.
|
||||
*/
|
||||
|
||||
/* Replace the whole priority vector. `count` must equal num_pieces. Does not
|
||||
* touch the "already requested" state. Thread-safe. Returns 0, or negative if
|
||||
* count != num_pieces. */
|
||||
int peer_set_priorities(peer_handle *h, const uint8_t *priorities, uint32_t count);
|
||||
|
||||
/* Update one piece's priority. Thread-safe. Negative if out of range. */
|
||||
int peer_set_priority(peer_handle *h, uint32_t piece_index, uint8_t priority);
|
||||
|
||||
/* Re-arm a piece for (re-)download, clearing its internal "already requested"
|
||||
* mark so it becomes selectable again. Newly created peers start fully armed,
|
||||
* so this is only needed to retry a piece that failed hash verification.
|
||||
* Thread-safe. Negative if out of range. */
|
||||
int peer_request_piece(peer_handle *h, uint32_t piece_index);
|
||||
|
||||
/* Stop the network thread and close the socket. Idempotent. */
|
||||
void peer_stop(peer_handle *h);
|
||||
|
||||
/* Free the peer and its arena. The arena pointer is invalid afterwards. */
|
||||
void peer_destroy(peer_handle *h);
|
||||
|
||||
/* Data plane ------------------------------------------------------------ */
|
||||
|
||||
/* Base of the block arena. Wrap [base, base+peer_arena_bytes()) in a Python
|
||||
* memoryview once and slice it per block for zero-copy reads. */
|
||||
void *peer_arena_base(const peer_handle *h);
|
||||
uint64_t peer_arena_bytes(const peer_handle *h);
|
||||
|
||||
/* Drain up to max completed blocks into out[]. Returns the count (0..max),
|
||||
* never blocks. Call from the single harness consumer thread. */
|
||||
uint32_t peer_poll_ready(peer_handle *h, block_desc *out, uint32_t max);
|
||||
|
||||
/* Return a consumed slot so the peer can reuse it. This is what unblocks new
|
||||
* requests (credit-based flow control). */
|
||||
void peer_release_slot(peer_handle *h, uint32_t slot);
|
||||
|
||||
/* Block until ready blocks are available or timeout_ms elapses (negative =
|
||||
* wait forever). Returns 1 if readable, 0 on timeout, negative on error.
|
||||
* Optional: callers may instead poll peer_poll_ready() directly. */
|
||||
int peer_wait(peer_handle *h, int timeout_ms);
|
||||
|
||||
/* Fill *out with a status snapshot. */
|
||||
void peer_get_status(const peer_handle *h, peer_status *out);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* TORRENT_PEER_H */
|
||||
Loading…
Add table
Add a link
Reference in a new issue