Cieric/Naut
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

Initial commit: Naut-Torrent — from-scratch 10 GbE BitTorrent client

Browse source 
A maintainable, extensible BitTorrent client (C11, Linux/io_uring) targeting
10 GbE saturation. All torrent functionality is built from scratch; liburing
is the only linked third-party dependency on the data path.

Implements Phases 1-7 of the roadmap:
- core: page-aligned buffer pool, MPMC/Treiber queues, bitfields, worker pool
- crypto: SHA-1/256 (SHA-NI + scalar), Merkle (BEP-52), RC4 (MSE)
- bencode/metainfo: zero-copy parser, v1/v2/hybrid .torrent + magnet
- peer: sans-IO wire codec, MSE/PE handshake state machine, BEP-10, ut_metadata, PEX
- piece/storage: block-level multi-peer engine, rarest-first + endgame,
  per-file completion events + single-file relocate (move-as-you-finish)
- tracker/dht: HTTP + UDP (BEP-15) trackers, BEP-5 KRPC iterative lookup
- platform: io_uring reactor (SQPOLL, registered buffers, SEND_ZC)
- surface: versioned RPC, native plugin ABI, sandboxed Lua scripting, nautd/nautctl

Verified against libtorrent (single/multi/hybrid, MSE, magnet-via-DHT, swarm);
unit + interop tests green; ASan/UBSan/TSan clean. Scripting reference in
docs/scripting.md.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
ookami125 2026-06-15 12:12:00 -04:00
commit 2178d6a70c
121 changed files with 12644 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

214
apps/echo/main.c Normal file
View file

@ -0,0 +1,214 @@
/* naut_echo — Phase 1 gate.
*
* A single-reactor io_uring echo server that proves the foundation works end to
* end: multishot accept, recv/send driven entirely off the page-aligned buffer
* pool with ZERO per-operation allocation in steady state. Throughput on
* loopback should be limited by memory bandwidth / the single core, not by the
* allocator or syscalls.
*
* It is intentionally one-in-flight-op-per-connection (recv -> send -> recv).
* The real peer reactor (later phase) uses multishot recv + provided buffers
* and pipelines; this is the minimal honest exercise of the primitives.
*
* usage: naut_echo [port] (default 9000)
*/
#include "naut/uring.h"
#include "naut/net.h"
#include "naut/buf.h"
#include "naut/log.h"
#include "naut/system.h"
#include <liburing.h>
#include <errno.h>
#include <signal.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/socket.h>
#define ECHO_BLOCK (128u * 1024u)
#define ECHO_BUFS 4096u
#define RING_ENTRIES 4096u
/* user_data tagging: low 3 bits = op, high bits = conn* (16-byte aligned). */
enum { TAG_ACCEPT = 1, TAG_RECV = 2, TAG_SEND = 3 };
#define UD(p, tag) ((__u64)(uintptr_t)(p) | (unsigned)(tag))
#define UD_TAG(ud) ((unsigned)((ud) & 0x7u))
#define UD_PTR(ud) ((conn *)(uintptr_t)((ud) & ~(__u64)0x7u))
typedef struct conn {
int fd;
uint32_t sent; /* bytes of buf->len already written (partial sends) */
naut_buf *buf;
bool awaiting_notif;
bool recv_fixed; /* the in-flight recv used the fixed buffer */
} conn;
static volatile sig_atomic_t g_stop = 0;
static void on_signal(int s) { (void)s; g_stop = 1; }
static naut_bufpool *g_pool;
static _Atomic uint64_t g_bytes = 0, g_conns = 0, g_zc_copied = 0;
static void arm_recv(naut_ring *owner, conn *c) {
struct io_uring_sqe *sqe = io_uring_get_sqe(&owner->ring);
c->recv_fixed =
naut_ring_prep_recv(owner, sqe, c->fd, c->buf->data, c->buf->cap, 0);
io_uring_sqe_set_data64(sqe, UD(c, TAG_RECV));
}
static void arm_send(naut_ring *owner, conn *c) {
struct io_uring_sqe *sqe = io_uring_get_sqe(&owner->ring);
c->awaiting_notif = naut_ring_prep_send(
owner, sqe, c->fd, c->buf->data + c->sent,
c->buf->len - c->sent, MSG_NOSIGNAL, true);
io_uring_sqe_set_data64(sqe, UD(c, TAG_SEND));
}
static void conn_close(conn *c) {
close(c->fd);
naut_buf_put(c->buf);
free(c);
}
int main(int argc, char **argv) {
uint16_t port = (argc > 1) ? (uint16_t)atoi(argv[1]) : 9000;
int cpu = getenv("NAUT_CPU") ? atoi(getenv("NAUT_CPU")) : -1;
int numa_node =
getenv("NAUT_NUMA_NODE") ? atoi(getenv("NAUT_NUMA_NODE")) : -1;
bool sqpoll = getenv("NAUT_SQPOLL") != NULL;
bool hugepages = getenv("NAUT_HUGEPAGES") != NULL;
signal(SIGINT, on_signal);
signal(SIGTERM, on_signal);
signal(SIGPIPE, SIG_IGN);
if (cpu >= 0 && naut_pin_current_thread(cpu) != NAUT_OK)
NAUT_WARN("failed to pin reactor to CPU %d", cpu);
naut_ring r;
if (naut_ring_init_cpu(&r, RING_ENTRIES, sqpoll, cpu) != NAUT_OK)
return 1;
if (naut_ring_probe(&r) != NAUT_OK) { naut_ring_close(&r); return 1; }
struct io_uring *ring = &r.ring;
int lfd = naut_net_listen(port, 1024, true);
if (lfd < 0) { naut_ring_close(&r); return 1; }
g_pool = naut_bufpool_create_on_node(
ECHO_BLOCK, ECHO_BUFS, hugepages, numa_node);
if (!g_pool) { close(lfd); naut_ring_close(&r); return 1; }
(void)naut_ring_register_bufpool(&r, g_pool);
/* prime the multishot accept */
struct io_uring_sqe *sqe = io_uring_get_sqe(ring);
io_uring_prep_multishot_accept(sqe, lfd, NULL, NULL, 0);
io_uring_sqe_set_data64(sqe, UD(NULL, TAG_ACCEPT));
NAUT_INFO("echo listening on :%u", port);
while (!g_stop) {
int rc = io_uring_submit_and_wait(ring, 1);
if (rc < 0 && rc != -EINTR) { NAUT_ERROR("submit_and_wait: %s", strerror(-rc)); break; }
unsigned head, count = 0;
struct io_uring_cqe *cqe;
io_uring_for_each_cqe(ring, head, cqe) {
count++;
__u64 ud = cqe->user_data;
int res = cqe->res;
switch (UD_TAG(ud)) {
case TAG_ACCEPT: {
if (res < 0) {
if (res != -ECANCELED) NAUT_WARN("accept: %s", strerror(-res));
} else {
int cfd = res;
naut_net_tune_peer(cfd);
naut_buf *b = naut_buf_get(g_pool);
if (!b) { NAUT_WARN("pool exhausted, dropping conn"); close(cfd); }
else {
conn *c = calloc(1, sizeof(*c));
c->fd = cfd; c->buf = b;
atomic_fetch_add(&g_conns, 1);
arm_recv(&r, c);
}
}
/* re-arm if the kernel dropped the multishot registration */
if (!(cqe->flags & IORING_CQE_F_MORE)) {
struct io_uring_sqe *s = io_uring_get_sqe(ring);
io_uring_prep_multishot_accept(s, lfd, NULL, NULL, 0);
io_uring_sqe_set_data64(s, UD(NULL, TAG_ACCEPT));
}
break;
}
case TAG_RECV: {
conn *c = UD_PTR(ud);
if (res <= 0) {
/* A fixed-buffer recv rejected with -EINVAL means this
* kernel doesn't support IORING_RECVSEND_FIXED_BUF on plain
* recv. Disable it ring-wide and retry THIS connection
* unfixed. We key off the per-conn flag, not the ring flag,
* so every connection that armed a fixed recv before the
* flag flipped recovers too (otherwise all but the first
* would be torn down). */
if (res == -EINVAL && c->recv_fixed) {
if (r.recv_fixed) {
NAUT_WARN("fixed-buffer recv unsupported at runtime; "
"falling back to normal recv");
r.recv_fixed = false;
}
arm_recv(&r, c);
break;
}
if (res < 0)
NAUT_WARN("recv completion: %s", strerror(-res));
conn_close(c);
break;
}
c->buf->len = (uint32_t)res;
c->sent = 0;
arm_send(&r, c);
break;
}
case TAG_SEND: {
conn *c = UD_PTR(ud);
if (cqe->flags & IORING_CQE_F_NOTIF) {
if (res & IORING_NOTIF_USAGE_ZC_COPIED) {
uint64_t copied =
atomic_fetch_add(&g_zc_copied, 1) + 1;
if (copied == 8) {
NAUT_WARN("SEND_ZC is copying on this transport; "
"disabling it for this ring");
r.send_zc = false;
}
}
c->awaiting_notif = false;
if (c->sent < c->buf->len) arm_send(&r, c);
else { c->buf->len = 0; arm_recv(&r, c); }
break;
}
if (res <= 0) { conn_close(c); break; }
c->sent += (uint32_t)res;
atomic_fetch_add(&g_bytes, (uint64_t)res);
if (!c->awaiting_notif) {
if (c->sent < c->buf->len) arm_send(&r, c);
else { c->buf->len = 0; arm_recv(&r, c); }
}
break;
}
default:
NAUT_PANIC("bad user_data tag %u", UD_TAG(ud));
}
}
io_uring_cq_advance(ring, count);
}
NAUT_INFO("shutting down: %llu conns, %llu bytes echoed, %llu SEND_ZC copied notifications",
(unsigned long long)atomic_load(&g_conns),
(unsigned long long)atomic_load(&g_bytes),
(unsigned long long)atomic_load(&g_zc_copied));
close(lfd);
naut_ring_unregister_buffers(&r);
naut_bufpool_destroy(g_pool);
naut_ring_close(&r);
return 0;
}

212
apps/leech/main.c Normal file
View file

@ -0,0 +1,212 @@
/* naut_leech — Phase 3 gate: download a torrent from a single peer and write a
* byte-correct, hash-verified file to disk.
*
* Blocking-socket driver around the sans-IO peer codec + download engine. The
* point of this phase is protocol correctness and interop (it downloads from a
* libtorrent seed in the integration test), not peak throughput — the io_uring
* reactor that drives thousands of these comes in Phase 6.
*
* usage: naut_leech [--mse] <file.torrent> <output-dir> <ip> <port>
*/
#include "naut/metainfo.h"
#include "naut/storage.h"
#include "naut/piece.h"
#include "naut/peer.h"
#include "naut/mse.h"
#include "naut/log.h"
#include <arpa/inet.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#define PIPELINE_DEPTH 512 /* outstanding requests (~8 MiB in flight) */
static double now(void) {
struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t);
return t.tv_sec + t.tv_nsec * 1e-9;
}
static uint8_t *slurp(const char *path, size_t *len) {
FILE *f = fopen(path, "rb");
if (!f) { NAUT_ERROR("open %s: %s", path, strerror(errno)); return NULL; }
fseek(f, 0, SEEK_END); long n = ftell(f); fseek(f, 0, SEEK_SET);
uint8_t *b = malloc(n);
if (fread(b, 1, n, f) != (size_t)n) { fclose(f); free(b); return NULL; }
fclose(f); *len = (size_t)n; return b;
}
static int connect_peer(const char *ip, uint16_t port) {
int fd = socket(AF_INET, SOCK_STREAM, 0);
if (fd < 0) return -1;
struct sockaddr_in a; memset(&a, 0, sizeof a);
a.sin_family = AF_INET; a.sin_port = htons(port);
if (inet_pton(AF_INET, ip, &a.sin_addr) != 1) { close(fd); return -1; }
if (connect(fd, (struct sockaddr *)&a, sizeof a) != 0) {
NAUT_ERROR("connect %s:%u: %s", ip, port, strerror(errno));
close(fd); return -1;
}
int one = 1; setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof one);
return fd;
}
/* send up to PIPELINE_DEPTH outstanding requests */
static bool refill(int fd, naut_mse_stream *mse,
naut_download *d, int *outstanding) {
uint32_t idx, begin, len;
while (*outstanding < PIPELINE_DEPTH) {
if (!naut_download_next_request(d, &idx, &begin, &len)) break;
uint8_t req[17];
naut_peer_msg_request(req, idx, begin, len);
if (!naut_mse_send_all(fd, mse, req, sizeof req)) return false;
(*outstanding)++;
}
return true;
}
int main(int argc, char **argv) {
bool use_mse = argc > 1 && strcmp(argv[1], "--mse") == 0;
int arg = use_mse ? 2 : 1;
if (argc - arg != 4) {
fprintf(stderr, "usage: %s [--mse] <file.torrent> <output-dir> <ip> <port>\n",
argv[0]);
return 2;
}
naut_log_set_level(NAUT_LOG_INFO);
size_t tlen;
uint8_t *tor = slurp(argv[arg], &tlen);
if (!tor) return 1;
naut_metainfo mi;
if (naut_metainfo_parse(tor, tlen, &mi) != NAUT_OK) { NAUT_ERROR("bad torrent"); return 1; }
free(tor);
char hex[41]; naut_infohash_v1_hex(&mi, hex);
NAUT_INFO("torrent '%s': %u pieces, %lld bytes, infohash %s",
mi.name, mi.num_pieces, (long long)mi.total_length, hex);
naut_err err;
naut_storage_opts storage_opts = {
.direct_io = getenv("NAUT_DIRECT_IO") != NULL,
.preallocate = true,
};
naut_storage *st = naut_storage_open_opts(
mi.files, mi.num_files, argv[arg + 1], &storage_opts, &err);
if (!st) { NAUT_ERROR("storage: %s", naut_strerror(err)); return 1; }
naut_download *d = naut_download_create(&mi, st);
if (!d) return 1;
int fd = connect_peer(argv[arg + 2], (uint16_t)atoi(argv[arg + 3]));
if (fd < 0) return 1;
/* handshake */
uint8_t peerid[20]; memcpy(peerid, "-NT0001-", 8);
for (int i = 8; i < 20; i++) peerid[i] = (uint8_t)(rand() & 0xff);
uint8_t hs[NAUT_HANDSHAKE_LEN];
naut_peer_handshake_build(hs, mi.infohash_v1, peerid, 0);
naut_mse_stream mse = {0};
uint8_t remote_hs[NAUT_HANDSHAKE_LEN];
bool hs_done = false;
if (use_mse) {
naut_err mse_err = naut_mse_client_handshake(
fd, mi.infohash_v1, peerid, 0, &mse, remote_hs);
if (mse_err != NAUT_OK) {
NAUT_ERROR("MSE handshake failed: %s", naut_strerror(mse_err));
return 1;
}
hs_done = true;
NAUT_INFO("MSE/RC4 peer transport established");
uint8_t intr[5];
naut_peer_msg_simple(intr, NAUT_MSG_INTERESTED);
if (!naut_mse_send_all(fd, &mse, intr, sizeof intr)) {
NAUT_ERROR("interested send failed");
return 1;
}
} else if (!naut_mse_send_all(fd, &mse, hs, sizeof hs)) {
NAUT_ERROR("handshake send failed");
return 1;
}
/* recv buffer */
size_t cap = 4u << 20, len = 0;
uint8_t *buf = malloc(cap);
bool unchoked = false;
int outstanding = 0;
double t0 = now();
while (!naut_download_complete(d)) {
if (len == cap) { cap *= 2; buf = realloc(buf, cap); }
ssize_t r = naut_mse_recv(fd, &mse, buf + len, cap - len);
if (r < 0) { NAUT_ERROR("recv: %s", strerror(errno)); break; }
if (r == 0) { NAUT_ERROR("peer closed (%.1f%% done)",
100.0 * naut_download_pieces_done(d) / mi.num_pieces); break; }
len += (size_t)r;
size_t pos = 0;
if (!hs_done) {
if (len < NAUT_HANDSHAKE_LEN) continue;
uint8_t ih[20], pid[20];
if (!naut_peer_handshake_parse(buf, ih, pid, NULL) ||
memcmp(ih, mi.infohash_v1, 20) != 0) {
NAUT_ERROR("handshake mismatch"); break;
}
pos = NAUT_HANDSHAKE_LEN;
hs_done = true;
uint8_t intr[5]; naut_peer_msg_simple(intr, NAUT_MSG_INTERESTED);
if (!naut_mse_send_all(fd, &mse, intr, 5)) break;
}
/* parse all complete messages */
for (;;) {
naut_msg m;
int c = naut_peer_msg_parse(buf + pos, len - pos, &m);
if (c == 0) break;
if (c < 0) { NAUT_ERROR("protocol error"); goto done; }
pos += (size_t)c;
switch (m.type) {
case NAUT_MSG_UNCHOKE: unchoked = true; break;
case NAUT_MSG_CHOKE: unchoked = false; break;
case NAUT_MSG_PIECE: {
outstanding--;
bool pdone = false;
naut_err e = naut_download_on_block(d, m.index, m.begin, m.payload,
(uint32_t)m.payload_len, &pdone);
if (e != NAUT_OK) { NAUT_ERROR("block rejected: %s", naut_strerror(e)); goto done; }
break;
}
default: break; /* bitfield/have/keepalive/port: ignore for a seed */
}
}
/* compact consumed bytes */
memmove(buf, buf + pos, len - pos);
len -= pos;
if (unchoked && !refill(fd, &mse, d, &outstanding)) {
NAUT_ERROR("request send failed"); break;
}
}
done:;
double dt = now() - t0;
bool ok = naut_download_complete(d);
if (ok) {
double mb = (double)mi.total_length / 1e6;
NAUT_INFO("COMPLETE: %u/%u pieces, %.1f MB in %.2fs (%.1f MB/s), all SHA-1 verified",
naut_download_pieces_done(d), mi.num_pieces, mb, dt, mb / dt);
} else {
NAUT_ERROR("INCOMPLETE: %u/%u pieces", naut_download_pieces_done(d), mi.num_pieces);
}
naut_storage_sync(st);
close(fd);
naut_download_destroy(d);
naut_storage_close(st);
naut_metainfo_free(&mi);
free(buf);
return ok ? 0 : 1;
}

93
apps/nautctl/main.c Normal file
View file

@ -0,0 +1,93 @@
#include "naut/rpc.h"
#include <jansson.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#define DEFAULT_SOCKET "/tmp/nautd.sock"
static void usage(const char *program) {
fprintf(stderr,
"usage: %s [--socket PATH] METHOD [PARAMS_JSON]\n"
" %s [--socket PATH] events\n", program, program);
}
static json_t *parse_params(const char *text) {
if (!text) return json_object();
json_error_t error;
return json_loads(text, JSON_REJECT_DUPLICATES | JSON_DECODE_ANY, &error);
}
static int print_json(json_t *json) {
if (!json) return 1;
if (json_dumpf(json, stdout, JSON_INDENT(2) | JSON_SORT_KEYS) != 0)
return 1;
putchar('\n');
return 0;
}
static int stream_events(const char *socket_path) {
int fd = naut_rpc_connect_unix(socket_path);
if (fd < 0) return 1;
json_t *request = json_pack("{s:s,s:o}", "method", "subscribe",
"params", json_object());
if (!request ||
naut_rpc_send_json(fd, NAUT_RPC_REQUEST, request) != NAUT_OK) {
json_decref(request);
close(fd);
return 1;
}
json_decref(request);
for (;;) {
naut_rpc_frame_type type;
json_t *payload = NULL;
if (naut_rpc_recv_json(fd, &type, &payload) != NAUT_OK) {
close(fd);
return 1;
}
print_json(payload);
fflush(stdout);
json_decref(payload);
}
}
int main(int argc, char **argv) {
const char *socket_path = getenv("NAUT_SOCKET");
if (!socket_path || !*socket_path) socket_path = DEFAULT_SOCKET;
int arg = 1;
if (arg < argc && strcmp(argv[arg], "--socket") == 0) {
if (arg + 1 >= argc) {
usage(argv[0]);
return 2;
}
socket_path = argv[arg + 1];
arg += 2;
}
if (arg >= argc || arg + 2 < argc) {
usage(argv[0]);
return 2;
}
signal(SIGPIPE, SIG_IGN);
const char *method = argv[arg++];
if (strcmp(method, "events") == 0)
return stream_events(socket_path);
json_t *params = parse_params(arg < argc ? argv[arg] : NULL);
if (!params) {
fprintf(stderr, "nautctl: invalid JSON parameters\n");
return 2;
}
json_t *reply = NULL;
naut_err error = naut_rpc_call(socket_path, method, params, &reply);
json_decref(params);
if (error != NAUT_OK) {
fprintf(stderr, "nautctl: RPC failed: %s\n", naut_strerror(error));
return 1;
}
int result = print_json(reply);
bool ok = json_is_true(json_object_get(reply, "ok"));
json_decref(reply);
return result || !ok;
}

507
apps/nautd/main.c Normal file
View file

@ -0,0 +1,507 @@
#include "naut/event.h"
#include "naut/log.h"
#include "naut/metainfo.h"
#include "naut/plugin.h"
#include "naut/rpc.h"
#include "naut/script.h"
#include "naut/session.h"
#include "naut/storage.h"
#include <errno.h>
#include <fcntl.h>
#include <limits.h>
#include <poll.h>
#include <pthread.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <unistd.h>
#define DEFAULT_SOCKET "/tmp/nautd.sock"
#define MOVE_QUEUE_CAPACITY 64
#define MAX_SUBSCRIBERS 64
typedef struct {
uint64_t torrent_id;
uint32_t file_index;
char destination[PATH_MAX];
} move_command;
typedef struct {
naut_event_bus *events;
naut_rpc_registry *rpc;
naut_plugin_manager *plugins;
naut_script *script;
naut_session *session;
pthread_mutex_t move_lock;
move_command moves[MOVE_QUEUE_CAPACITY];
size_t move_head;
size_t move_count;
uint64_t moves_processed;
pthread_mutex_t subscriber_lock;
int subscribers[MAX_SUBSCRIBERS];
size_t subscriber_count;
bool stopping;
} daemon_state;
static volatile sig_atomic_t interrupted;
static void on_signal(int signal_number) {
(void)signal_number;
interrupted = 1;
}
static json_t *rpc_ping(void *opaque, const json_t *params, naut_err *error) {
(void)opaque;
(void)params;
json_t *result = json_object();
if (!result) {
*error = NAUT_ERR_NOMEM;
return NULL;
}
json_object_set_new(result, "protocol", json_integer(NAUT_RPC_VERSION));
json_object_set_new(result, "service", json_string("nautd"));
*error = NAUT_OK;
return result;
}
static json_t *rpc_status(void *opaque, const json_t *params,
naut_err *error) {
(void)params;
daemon_state *state = opaque;
naut_script_stats stats = {0};
if (state->script) naut_script_get_stats(state->script, &stats);
pthread_mutex_lock(&state->move_lock);
uint64_t moves = state->moves_processed;
size_t pending = state->move_count;
pthread_mutex_unlock(&state->move_lock);
json_t *result = json_object();
json_t *script = json_object();
if (!result || !script) {
json_decref(result);
json_decref(script);
*error = NAUT_ERR_NOMEM;
return NULL;
}
json_object_set_new(result, "protocol", json_integer(NAUT_RPC_VERSION));
json_object_set_new(result, "plugins",
json_integer((json_int_t)naut_plugin_count(
state->plugins)));
json_object_set_new(result, "storage_backends",
json_integer((json_int_t)naut_plugin_storage_count(
state->plugins)));
json_object_set_new(script, "queued", json_integer(stats.queued));
json_object_set_new(script, "handled", json_integer(stats.handled));
json_object_set_new(script, "dropped", json_integer(stats.dropped));
json_object_set_new(script, "errors", json_integer(stats.errors));
json_object_set_new(script, "move_requests",
json_integer(stats.move_requests));
json_object_set_new(result, "script", script);
json_object_set_new(result, "move_commands", json_integer(moves));
json_object_set_new(result, "pending_move_commands",
json_integer((json_int_t)pending));
*error = NAUT_OK;
return result;
}
static json_t *rpc_plugins(void *opaque, const json_t *params,
naut_err *error) {
(void)params;
daemon_state *state = opaque;
json_t *plugins = json_array();
json_t *storage = json_array();
if (!plugins || !storage) {
json_decref(plugins);
json_decref(storage);
*error = NAUT_ERR_NOMEM;
return NULL;
}
for (size_t i = 0; i < naut_plugin_count(state->plugins); i++)
json_array_append_new(plugins,
json_string(naut_plugin_name(state->plugins, i)));
for (size_t i = 0; i < naut_plugin_storage_count(state->plugins); i++)
json_array_append_new(storage,
json_string(naut_plugin_storage_name(state->plugins, i)));
json_t *result = json_object();
if (!result) {
json_decref(plugins);
json_decref(storage);
*error = NAUT_ERR_NOMEM;
return NULL;
}
json_object_set_new(result, "plugins", plugins);
json_object_set_new(result, "storage_backends", storage);
*error = NAUT_OK;
return result;
}
static json_t *rpc_emit(void *opaque, const json_t *params, naut_err *error) {
daemon_state *state = opaque;
if (!json_is_object(params)) {
*error = NAUT_ERR_INVAL;
return NULL;
}
const char *type_name = json_string_value(json_object_get(params, "type"));
naut_event_type type;
if (!type_name || !naut_event_type_parse(type_name, &type)) {
*error = NAUT_ERR_INVAL;
return NULL;
}
json_int_t torrent_id =
json_integer_value(json_object_get(params, "torrent_id"));
json_int_t index = json_integer_value(json_object_get(params, "index"));
if (torrent_id < 0 || index < 0 || (uint64_t)index > UINT32_MAX) {
*error = NAUT_ERR_RANGE;
return NULL;
}
naut_event event = {
.type = type,
.torrent_id = (uint64_t)torrent_id,
.index = (uint32_t)index,
.message = json_string_value(json_object_get(params, "message")),
.path = json_string_value(json_object_get(params, "path")),
};
naut_event_emit(state->events, &event);
*error = NAUT_OK;
return json_true();
}
static json_t *rpc_shutdown(void *opaque, const json_t *params,
naut_err *error) {
(void)params;
daemon_state *state = opaque;
state->stopping = true;
*error = NAUT_OK;
return json_true();
}
static uint8_t *read_file(const char *path, size_t *len) {
FILE *f = fopen(path, "rb");
if (!f) return NULL;
if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return NULL; }
long n = ftell(f);
if (n < 0 || fseek(f, 0, SEEK_SET) != 0) { fclose(f); return NULL; }
uint8_t *buf = malloc((size_t)n);
if (!buf) { fclose(f); return NULL; }
if (fread(buf, 1, (size_t)n, f) != (size_t)n) {
free(buf); fclose(f); return NULL;
}
fclose(f);
*len = (size_t)n;
return buf;
}
/* add_torrent {torrent_id, torrent: <.torrent path>, root: <output dir>} opens
* the torrent's storage and registers it so move_file can later relocate one of
* its files. This is the control-plane seam that binds a script's move command
* to a concrete naut_storage; it runs on the daemon owner thread. */
static json_t *rpc_add_torrent(void *opaque, const json_t *params,
naut_err *error) {
daemon_state *state = opaque;
if (!json_is_object(params)) { *error = NAUT_ERR_INVAL; return NULL; }
json_int_t torrent_id =
json_integer_value(json_object_get(params, "torrent_id"));
const char *torrent_path =
json_string_value(json_object_get(params, "torrent"));
const char *root = json_string_value(json_object_get(params, "root"));
if (torrent_id < 0 || !torrent_path || !root) {
*error = NAUT_ERR_INVAL;
return NULL;
}
if (naut_session_has(state->session, (uint64_t)torrent_id)) {
*error = NAUT_ERR_INVAL;
return NULL;
}
size_t len = 0;
uint8_t *raw = read_file(torrent_path, &len);
if (!raw) { *error = NAUT_ERR_IO; return NULL; }
naut_metainfo mi;
naut_err err = naut_metainfo_parse(raw, len, &mi);
free(raw);
if (err != NAUT_OK) { *error = err; return NULL; }
naut_storage *storage =
naut_storage_open(mi.files, mi.num_files, root, &err);
if (!storage) {
naut_metainfo_free(&mi);
*error = err != NAUT_OK ? err : NAUT_ERR_IO;
return NULL;
}
naut_metainfo_free(&mi);
err = naut_session_add(state->session, (uint64_t)torrent_id, storage);
if (err != NAUT_OK) {
naut_storage_close(storage);
*error = err;
return NULL;
}
*error = NAUT_OK;
return json_true();
}
static naut_err queue_move(void *opaque, uint64_t torrent_id,
uint32_t file_index, const char *destination) {
daemon_state *state = opaque;
pthread_mutex_lock(&state->move_lock);
if (state->move_count == MOVE_QUEUE_CAPACITY) {
pthread_mutex_unlock(&state->move_lock);
return NAUT_ERR_FULL;
}
size_t tail = (state->move_head + state->move_count) %
MOVE_QUEUE_CAPACITY;
state->moves[tail] = (move_command) {
.torrent_id = torrent_id,
.file_index = file_index,
};
snprintf(state->moves[tail].destination,
sizeof state->moves[tail].destination, "%s", destination);
state->move_count++;
pthread_mutex_unlock(&state->move_lock);
return NAUT_OK;
}
static void drain_moves(daemon_state *state) {
/* Copy each pending command out under the lock, then perform the relocate
* with the lock released (so the script thread can keep enqueuing). All
* relocates run on this, the owner thread, as the session requires. */
for (;;) {
move_command command;
pthread_mutex_lock(&state->move_lock);
if (state->move_count == 0) {
pthread_mutex_unlock(&state->move_lock);
return;
}
command = state->moves[state->move_head];
state->move_head = (state->move_head + 1) % MOVE_QUEUE_CAPACITY;
state->move_count--;
state->moves_processed++;
pthread_mutex_unlock(&state->move_lock);
naut_err err = naut_session_move_file(state->session,
command.torrent_id,
command.file_index,
command.destination);
if (err == NAUT_OK)
NAUT_INFO("moved torrent=%llu file=%u -> %s",
(unsigned long long)command.torrent_id,
command.file_index, command.destination);
else
NAUT_WARN("move torrent=%llu file=%u -> %s failed: %s",
(unsigned long long)command.torrent_id,
command.file_index, command.destination,
naut_strerror(err));
}
}
static void broadcast_event(void *opaque, const naut_event *event) {
daemon_state *state = opaque;
json_t *payload = naut_rpc_event_json(event);
if (!payload) return;
pthread_mutex_lock(&state->subscriber_lock);
for (size_t i = 0; i < state->subscriber_count;) {
if (naut_rpc_send_json(state->subscribers[i], NAUT_RPC_EVENT,
payload) == NAUT_OK) {
i++;
continue;
}
close(state->subscribers[i]);
state->subscribers[i] =
state->subscribers[--state->subscriber_count];
}
pthread_mutex_unlock(&state->subscriber_lock);
json_decref(payload);
}
static int listen_unix(const char *path) {
int fd = socket(AF_UNIX, SOCK_STREAM, 0);
if (fd < 0) return -1;
struct sockaddr_un address;
memset(&address, 0, sizeof address);
address.sun_family = AF_UNIX;
if (strlen(path) >= sizeof address.sun_path) {
close(fd);
return -1;
}
strcpy(address.sun_path, path);
unlink(path);
if (bind(fd, (struct sockaddr *)&address, sizeof address) != 0 ||
listen(fd, 32) != 0) {
close(fd);
return -1;
}
return fd;
}
static json_t *response(bool ok, json_t *result, naut_err error) {
json_t *reply = json_object();
if (!reply) return NULL;
json_object_set_new(reply, "ok", json_boolean(ok));
if (ok) {
json_object_set(reply, "result", result ? result : json_null());
} else {
json_object_set_new(reply, "code", json_integer(error));
json_object_set_new(reply, "error",
json_string(naut_strerror(error)));
}
return reply;
}
static void add_subscriber(daemon_state *state, int fd) {
int flags = fcntl(fd, F_GETFL, 0);
if (flags >= 0) fcntl(fd, F_SETFL, flags | O_NONBLOCK);
pthread_mutex_lock(&state->subscriber_lock);
if (state->subscriber_count < MAX_SUBSCRIBERS) {
state->subscribers[state->subscriber_count++] = fd;
fd = -1;
}
pthread_mutex_unlock(&state->subscriber_lock);
if (fd >= 0) close(fd);
}
static void handle_client(daemon_state *state, int fd) {
struct timeval timeout = {.tv_sec = 2};
setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout);
naut_rpc_frame_type type;
json_t *request = NULL;
naut_err error = naut_rpc_recv_json(fd, &type, &request);
if (error != NAUT_OK || type != NAUT_RPC_REQUEST ||
!json_is_object(request)) {
json_decref(request);
close(fd);
return;
}
const char *method =
json_string_value(json_object_get(request, "method"));
json_t *params = json_object_get(request, "params");
if (method && strcmp(method, "subscribe") == 0) {
json_t *subscribed = json_string("subscribed");
json_t *reply = response(true, subscribed, NAUT_OK);
json_decref(subscribed);
if (reply &&
naut_rpc_send_json(fd, NAUT_RPC_RESPONSE, reply) == NAUT_OK)
add_subscriber(state, fd);
else
close(fd);
json_decref(reply);
json_decref(request);
return;
}
if (!method) error = NAUT_ERR_INVAL;
json_t *result = method
? naut_rpc_dispatch(state->rpc, method, params, &error) : NULL;
json_t *reply = response(error == NAUT_OK && result, result, error);
json_decref(result);
if (reply) {
naut_rpc_send_json(fd, NAUT_RPC_RESPONSE, reply);
json_decref(reply);
}
json_decref(request);
close(fd);
}
static bool register_commands(daemon_state *state) {
return naut_rpc_register(state->rpc, "ping", rpc_ping, state) == NAUT_OK &&
naut_rpc_register(state->rpc, "status", rpc_status, state) == NAUT_OK &&
naut_rpc_register(state->rpc, "plugins", rpc_plugins, state) == NAUT_OK &&
naut_rpc_register(state->rpc, "emit", rpc_emit, state) == NAUT_OK &&
naut_rpc_register(state->rpc, "add_torrent", rpc_add_torrent, state) == NAUT_OK &&
naut_rpc_register(state->rpc, "shutdown", rpc_shutdown, state) == NAUT_OK;
}
static void usage(const char *program) {
fprintf(stderr,
"usage: %s [--socket PATH] [--plugin PATH]... [--script PATH]\n",
program);
}
int main(int argc, char **argv) {
const char *socket_path = DEFAULT_SOCKET;
const char *script_path = NULL;
const char *plugin_paths[64];
size_t plugin_count = 0;
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--socket") == 0 && i + 1 < argc)
socket_path = argv[++i];
else if (strcmp(argv[i], "--plugin") == 0 && i + 1 < argc &&
plugin_count < NAUT_ARRAY_LEN(plugin_paths))
plugin_paths[plugin_count++] = argv[++i];
else if (strcmp(argv[i], "--script") == 0 && i + 1 < argc)
script_path = argv[++i];
else {
usage(argv[0]);
return 2;
}
}
signal(SIGINT, on_signal);
signal(SIGTERM, on_signal);
signal(SIGPIPE, SIG_IGN);
daemon_state state = {0};
pthread_mutex_init(&state.move_lock, NULL);
pthread_mutex_init(&state.subscriber_lock, NULL);
state.events = naut_event_bus_create();
state.rpc = naut_rpc_registry_create();
state.plugins = naut_plugin_manager_create(state.rpc, state.events);
state.session = naut_session_create();
if (!state.events || !state.rpc || !state.plugins || !state.session ||
!register_commands(&state)) {
fprintf(stderr, "nautd: failed to initialize control plane\n");
return 1;
}
for (size_t i = 0; i < plugin_count; i++) {
if (naut_plugin_load(state.plugins, plugin_paths[i]) != NAUT_OK) {
fprintf(stderr, "nautd: failed to load plugin %s\n",
plugin_paths[i]);
return 1;
}
}
if (script_path) {
naut_err error;
state.script = naut_script_create(state.events, script_path, 256,
queue_move, &state, &error);
if (!state.script) {
fprintf(stderr, "nautd: failed to load script %s: %s\n",
script_path, naut_strerror(error));
return 1;
}
}
uint64_t event_subscription;
if (naut_event_subscribe(state.events, broadcast_event, &state,
&event_subscription) != NAUT_OK)
return 1;
int listener = listen_unix(socket_path);
if (listener < 0) {
perror("nautd: listen");
return 1;
}
NAUT_INFO("nautd listening on %s", socket_path);
while (!state.stopping && !interrupted) {
struct pollfd pollfd = {.fd = listener, .events = POLLIN};
int ready = poll(&pollfd, 1, 100);
if (ready > 0 && (pollfd.revents & POLLIN)) {
int client = accept(listener, NULL, NULL);
if (client >= 0) handle_client(&state, client);
} else if (ready < 0 && errno != EINTR) {
break;
}
drain_moves(&state);
}
close(listener);
unlink(socket_path);
naut_event_unsubscribe(state.events, event_subscription);
pthread_mutex_lock(&state.subscriber_lock);
for (size_t i = 0; i < state.subscriber_count; i++)
close(state.subscribers[i]);
pthread_mutex_unlock(&state.subscriber_lock);
naut_script_destroy(state.script); /* joins the script thread */
drain_moves(&state); /* flush any moves it left queued */
naut_plugin_manager_destroy(state.plugins);
naut_rpc_registry_destroy(state.rpc);
naut_session_destroy(state.session);
naut_event_bus_destroy(state.events);
pthread_mutex_destroy(&state.subscriber_lock);
pthread_mutex_destroy(&state.move_lock);
return 0;
}

801
apps/swarm/main.c Normal file
View file

@ -0,0 +1,801 @@
/* naut_swarm — Phase 4 gate: download from a SWARM of peers concurrently.
*
* A poll()-based multi-socket driver around the same sans-IO peer codec and the
* multi-peer engine (rarest-first + bounded endgame duplication). Peers can be
* supplied explicitly for deterministic testing, or discovered from the
* torrent's HTTP/UDP trackers.
*
* usage: naut_swarm <file.torrent|magnet-uri> <output-dir> [<ip:port> ...]
*/
#include "naut/dht.h"
#include "naut/extension.h"
#include "naut/metainfo.h"
#include "naut/storage.h"
#include "naut/piece.h"
#include "naut/peer.h"
#include "naut/tracker.h"
#include "naut/bitfield.h"
#include "naut/log.h"
#include "naut/pipeline.h"
#include "naut/system.h"
#include "naut/worker.h"
#include <arpa/inet.h>
#include <errno.h>
#include <poll.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#define REQUEST_TIMEOUT 15.0
#define EXT_RESERVED 0x0000000000100000ULL
typedef struct {
uint32_t piece, begin, length;
double sent_at;
} req_t;
typedef struct {
naut_peer_addr addr;
char name[32];
} endpoint_t;
typedef struct {
int fd;
char name[40];
uint8_t *rbuf; size_t rcap, rlen;
bool hs_done, peer_choking;
naut_bitfield have;
req_t *inflight; size_t nflight, cflight;
uint64_t blocks_received;
naut_ext_handshake extensions;
uint64_t pex_received;
naut_pipeline pipeline;
bool dead, availability_removed;
} peer_t;
static double now(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec*1e-9; }
static uint8_t *slurp(const char *path, size_t *len) {
FILE *f = fopen(path, "rb"); if (!f) return NULL;
fseek(f, 0, SEEK_END); long n = ftell(f); fseek(f, 0, SEEK_SET);
uint8_t *b = malloc(n);
if (fread(b, 1, n, f) != (size_t)n) { fclose(f); free(b); return NULL; }
fclose(f); *len = (size_t)n; return b;
}
static bool send_all(int fd, const void *p, size_t n) {
const uint8_t *b = p;
while (n) { ssize_t w = send(fd, b, n, MSG_NOSIGNAL);
if (w < 0) { if (errno == EINTR) continue; return false; } b += w; n -= (size_t)w; }
return true;
}
static bool endpoint_add(endpoint_t **v, size_t *n, size_t *cap,
const naut_peer_addr *addr) {
if (addr->port == 0) return true;
for (size_t i = 0; i < *n; i++)
if ((*v)[i].addr.port == addr->port &&
memcmp((*v)[i].addr.ip, addr->ip, sizeof addr->ip) == 0)
return true;
if (*n == *cap) {
size_t newcap = *cap ? *cap * 2 : 32;
endpoint_t *next = realloc(*v, newcap * sizeof(*next));
if (!next) return false;
*v = next;
*cap = newcap;
}
endpoint_t *ep = &(*v)[(*n)++];
ep->addr = *addr;
snprintf(ep->name, sizeof ep->name, "%u.%u.%u.%u:%u",
addr->ip[0], addr->ip[1], addr->ip[2], addr->ip[3], addr->port);
return true;
}
static bool endpoint_parse(const char *s, naut_peer_addr *out) {
char host[INET_ADDRSTRLEN];
const char *colon = strrchr(s, ':');
if (!colon || colon == s || (size_t)(colon - s) >= sizeof host) return false;
char *end = NULL;
unsigned long port = strtoul(colon + 1, &end, 10);
if (!end || *end || port == 0 || port > UINT16_MAX) return false;
memcpy(host, s, (size_t)(colon - s));
host[colon - s] = 0;
struct in_addr ip;
if (inet_pton(AF_INET, host, &ip) != 1) return false;
memcpy(out->ip, &ip, sizeof out->ip);
out->port = (uint16_t)port;
return true;
}
static bool parse_udp_tracker(const char *url, char *host, size_t hostsz,
uint16_t *port) {
if (strncmp(url, "udp://", 6) != 0) return false;
const char *start = url + 6;
const char *slash = strchr(start, '/');
const char *end = slash ? slash : start + strlen(start);
const char *colon = memchr(start, ':', (size_t)(end - start));
if (!colon || colon == start) return false;
size_t hlen = (size_t)(colon - start);
if (hlen >= hostsz) return false;
char pbuf[16];
size_t plen = (size_t)(end - colon - 1);
if (plen == 0 || plen >= sizeof pbuf) return false;
memcpy(host, start, hlen);
host[hlen] = 0;
memcpy(pbuf, colon + 1, plen);
pbuf[plen] = 0;
char *tail = NULL;
unsigned long parsed = strtoul(pbuf, &tail, 10);
if (!tail || *tail || parsed == 0 || parsed > UINT16_MAX) return false;
*port = (uint16_t)parsed;
return true;
}
static bool discover_trackers(const uint8_t info_hash[20], uint64_t total_length,
char *const *trackers, size_t num_trackers,
const uint8_t peerid[20],
endpoint_t **eps, size_t *neps, size_t *cap) {
naut_announce_req req;
memset(&req, 0, sizeof req);
memcpy(req.info_hash, info_hash, sizeof req.info_hash);
memcpy(req.peer_id, peerid, sizeof req.peer_id);
req.port = 6881;
req.left = total_length;
req.event = NAUT_TEV_STARTED;
req.numwant = 100;
req.key = (uint32_t)rand();
for (size_t i = 0; i < num_trackers; i++) {
const char *tracker = trackers[i];
naut_tracker_response response;
naut_err e = NAUT_ERR_INVAL;
if (strncmp(tracker, "http://", 7) == 0) {
char url[4096];
if (naut_tracker_http_url(tracker, &req, url, sizeof url) != 0)
e = naut_tracker_announce_http(url, &response);
} else if (strncmp(tracker, "udp://", 6) == 0) {
char host[256];
uint16_t port;
if (parse_udp_tracker(tracker, host, sizeof host, &port))
e = naut_tracker_announce_udp(host, port, &req, &response);
} else {
NAUT_WARN("tracker scheme unsupported: %s", tracker);
continue;
}
if (e != NAUT_OK) {
NAUT_WARN("tracker announce failed: %s", tracker);
continue;
}
NAUT_INFO("tracker %s returned %zu peers", tracker, response.num_peers);
for (size_t p = 0; p < response.num_peers; p++) {
if (!endpoint_add(eps, neps, cap, &response.peers[p])) {
naut_tracker_response_free(&response);
return false;
}
}
naut_tracker_response_free(&response);
}
return true;
}
static bool discover_dht(const uint8_t info_hash[20],
endpoint_t **eps, size_t *neps, size_t *cap) {
static const char *defaults[] = {
"router.bittorrent.com:6881",
"router.utorrent.com:6881",
"dht.transmissionbt.com:6881",
};
const char *const *bootstrap = defaults;
size_t num_bootstrap = NAUT_ARRAY_LEN(defaults);
char *copy = NULL;
const char *custom[32];
const char *env = getenv("NAUT_DHT_BOOTSTRAP");
if (env && *env) {
copy = strdup(env);
if (!copy) return false;
num_bootstrap = 0;
char *save = NULL;
for (char *part = strtok_r(copy, ",", &save);
part && num_bootstrap < NAUT_ARRAY_LEN(custom);
part = strtok_r(NULL, ",", &save))
custom[num_bootstrap++] = part;
bootstrap = custom;
}
naut_peer_addr *peers = NULL;
size_t num_peers = 0;
naut_err e = num_bootstrap
? naut_dht_get_peers(bootstrap, num_bootstrap, info_hash,
&peers, &num_peers)
: NAUT_ERR_INVAL;
free(copy);
if (e != NAUT_OK) {
NAUT_WARN("DHT lookup found no peers");
return true;
}
NAUT_INFO("DHT returned %zu peers", num_peers);
for (size_t i = 0; i < num_peers; i++) {
if (!endpoint_add(eps, neps, cap, &peers[i])) {
free(peers);
return false;
}
}
free(peers);
return true;
}
static bool peer_reserve_inflight(peer_t *p) {
if (p->nflight == p->cflight) {
size_t cap = p->cflight ? p->cflight * 2 : 64;
req_t *v = realloc(p->inflight, cap * sizeof(*v));
if (!v) return false;
p->inflight = v;
p->cflight = cap;
}
return true;
}
static void peer_add_inflight(peer_t *p, uint32_t piece, uint32_t begin,
uint32_t length) {
p->inflight[p->nflight].piece = piece;
p->inflight[p->nflight].begin = begin;
p->inflight[p->nflight].length = length;
p->inflight[p->nflight].sent_at = now();
p->nflight++;
}
static bool peer_del_inflight(peer_t *p, uint32_t piece, uint32_t begin,
req_t *removed) {
for (size_t i = 0; i < p->nflight; i++)
if (p->inflight[i].piece == piece && p->inflight[i].begin == begin) {
if (removed) *removed = p->inflight[i];
p->inflight[i] = p->inflight[--p->nflight];
return true;
}
return false;
}
static bool peer_has_request(void *ctx, uint32_t piece, uint32_t begin) {
peer_t *p = ctx;
for (size_t i = 0; i < p->nflight; i++)
if (p->inflight[i].piece == piece && p->inflight[i].begin == begin)
return true;
return false;
}
/* return remaining inflight blocks to the picker (choke / disconnect) */
static void peer_release(naut_download *d, peer_t *p) {
for (size_t i = 0; i < p->nflight; i++)
naut_download_unrequest(d, p->inflight[i].piece, p->inflight[i].begin);
p->nflight = 0;
}
static void peer_drop(naut_download *d, peer_t *p) {
if (!p->availability_removed) {
naut_download_remove_bitfield(d, &p->have);
p->availability_removed = true;
}
peer_release(d, p);
if (p->fd >= 0) close(p->fd);
p->fd = -1;
p->dead = true;
}
static bool refill_one(naut_download *d, peer_t *p) {
if (p->dead || !p->hs_done || p->peer_choking ||
p->nflight >= naut_pipeline_depth(&p->pipeline))
return false;
if (!peer_reserve_inflight(p)) {
peer_drop(d, p);
return false;
}
uint32_t i, b, l;
if (!naut_download_pick_for_peer(d, &p->have, peer_has_request, p,
&i, &b, &l))
return false;
uint8_t req[17];
naut_peer_msg_request(req, i, b, l);
if (!send_all(p->fd, req, sizeof req)) {
naut_download_unrequest(d, i, b);
peer_drop(d, p);
return false;
}
peer_add_inflight(p, i, b, l);
return true;
}
static void expire_requests(naut_download *d, peer_t *p, double t) {
size_t i = 0;
while (i < p->nflight) {
if (t - p->inflight[i].sent_at < REQUEST_TIMEOUT) {
i++;
continue;
}
naut_download_unrequest(d, p->inflight[i].piece, p->inflight[i].begin);
p->inflight[i] = p->inflight[--p->nflight];
}
}
static int connect_to(const endpoint_t *ep) {
int fd = socket(AF_INET, SOCK_STREAM, 0);
if (fd < 0) return -1;
struct sockaddr_in a; memset(&a, 0, sizeof a);
a.sin_family = AF_INET;
a.sin_port = htons(ep->addr.port);
memcpy(&a.sin_addr, ep->addr.ip, sizeof ep->addr.ip);
if (connect(fd, (struct sockaddr *)&a, sizeof a) != 0) { close(fd); return -1; }
int one = 1; setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof one);
return fd;
}
/* process all complete messages currently buffered for peer p */
static void cancel_block(naut_download *d, peer_t *peers, int npeers,
peer_t *source, uint32_t piece, uint32_t begin) {
for (int i = 0; i < npeers; i++) {
peer_t *p = &peers[i];
if (p == source || p->dead) continue;
req_t old;
if (!peer_del_inflight(p, piece, begin, &old)) continue;
uint8_t msg[17];
naut_peer_msg_cancel(msg, old.piece, old.begin, old.length);
if (!send_all(p->fd, msg, sizeof msg)) peer_drop(d, p);
}
}
static void cancel_piece(naut_download *d, peer_t *peers, int npeers,
uint32_t piece) {
for (int i = 0; i < npeers; i++) {
peer_t *p = &peers[i];
size_t j = 0;
while (j < p->nflight) {
req_t old = p->inflight[j];
if (old.piece != piece) {
j++;
continue;
}
p->inflight[j] = p->inflight[--p->nflight];
naut_download_unrequest(d, old.piece, old.begin);
if (!p->dead) {
uint8_t msg[17];
naut_peer_msg_cancel(msg, old.piece, old.begin, old.length);
if (!send_all(p->fd, msg, sizeof msg)) {
peer_drop(d, p);
break;
}
}
}
}
}
static void merge_bitfield(naut_download *d, const naut_metainfo *mi,
peer_t *p, const uint8_t *wire, size_t wire_len) {
naut_bitfield incoming;
if (naut_bitfield_init(&incoming, mi->num_pieces) != NAUT_OK) {
peer_drop(d, p);
return;
}
naut_bitfield_from_wire(&incoming, wire, wire_len);
for (uint32_t piece = 0; piece < mi->num_pieces; piece++) {
if (naut_bitfield_test(&incoming, piece) &&
!naut_bitfield_test(&p->have, piece)) {
naut_bitfield_set(&p->have, piece);
naut_download_inc_avail(d, piece);
}
}
naut_bitfield_free(&incoming);
}
static naut_err peer_process(naut_download *d, const naut_metainfo *mi,
peer_t *peers, int npeers, peer_t *p) {
size_t pos = 0;
if (!p->hs_done) {
if (p->rlen < NAUT_HANDSHAKE_LEN) return NAUT_OK;
uint8_t ih[20], pid[20];
if (!naut_peer_handshake_parse(p->rbuf, ih, pid, NULL) ||
memcmp(ih, mi->infohash_v1, 20) != 0) {
p->dead = true;
return NAUT_OK;
}
pos = NAUT_HANDSHAKE_LEN;
p->hs_done = true;
}
for (;;) {
naut_msg m;
int c = naut_peer_msg_parse(p->rbuf + pos, p->rlen - pos, &m);
if (c == 0) break;
if (c < 0) { p->dead = true; break; }
pos += (size_t)c;
switch (m.type) {
case NAUT_MSG_BITFIELD:
merge_bitfield(d, mi, p, m.payload, m.payload_len);
if (p->dead) goto parsed;
break;
case NAUT_MSG_HAVE:
if (m.index < mi->num_pieces && !naut_bitfield_test(&p->have, m.index)) {
naut_bitfield_set(&p->have, m.index);
naut_download_inc_avail(d, m.index);
}
break;
case NAUT_MSG_UNCHOKE: p->peer_choking = false; break;
case NAUT_MSG_CHOKE: p->peer_choking = true; peer_release(d, p); break;
case NAUT_MSG_EXTENDED:
if (m.payload_len < 1) {
peer_drop(d, p);
goto parsed;
}
if (m.payload[0] == 0) {
if (naut_ext_parse_handshake(
m.payload + 1, m.payload_len - 1,
&p->extensions) != NAUT_OK) {
peer_drop(d, p);
goto parsed;
}
} else if (m.payload[0] == NAUT_EXT_UT_PEX) {
naut_pex_msg pex;
if (naut_pex_parse(m.payload + 1, m.payload_len - 1,
&pex) != NAUT_OK) {
peer_drop(d, p);
goto parsed;
}
p->pex_received += pex.num_added;
naut_pex_free(&pex);
}
break;
case NAUT_MSG_PIECE: {
req_t request;
bool expected = peer_del_inflight(p, m.index, m.begin, &request);
if (expected) {
naut_pipeline_on_block(&p->pipeline, request.length,
request.sent_at, now());
naut_download_unrequest(d, m.index, m.begin);
if (request.length != m.payload_len) {
peer_drop(d, p);
goto parsed;
}
}
bool pdone = false;
naut_err e = naut_download_on_block(d, m.index, m.begin, m.payload,
(uint32_t)m.payload_len, &pdone);
if (e == NAUT_OK) {
p->blocks_received++;
cancel_block(d, peers, npeers, p, m.index, m.begin);
} else if (e == NAUT_ERR_PROTO) {
if (expected) cancel_piece(d, peers, npeers, m.index);
else peer_drop(d, p);
} else {
return e;
}
break;
}
default: break;
}
}
parsed:
memmove(p->rbuf, p->rbuf + pos, p->rlen - pos);
p->rlen -= pos;
return NAUT_OK;
}
int main(int argc, char **argv) {
if (argc < 3) {
fprintf(stderr,
"usage: %s <file.torrent|magnet-uri> <out-dir> [ip:port ...]\n",
argv[0]);
return 2;
}
naut_log_set_level(NAUT_LOG_INFO);
uint8_t peerid[20]; memcpy(peerid, "-NT0001-", 8);
srand((unsigned)time(NULL) ^ (unsigned)getpid());
for (int i = 8; i < 20; i++) peerid[i] = (uint8_t)(rand() & 0xff);
bool from_magnet = strncmp(argv[1], "magnet:?", 8) == 0;
naut_metainfo mi;
memset(&mi, 0, sizeof mi);
naut_magnet magnet;
memset(&magnet, 0, sizeof magnet);
if (from_magnet) {
if (naut_magnet_parse(argv[1], &magnet) != NAUT_OK ||
!magnet.has_v1) {
NAUT_ERROR("magnet must contain a v1 btih hash");
naut_magnet_free(&magnet);
return 1;
}
} else {
size_t tlen;
uint8_t *tor = slurp(argv[1], &tlen);
if (!tor) { NAUT_ERROR("read torrent"); return 1; }
if (naut_metainfo_parse(tor, tlen, &mi) != NAUT_OK) {
NAUT_ERROR("parse torrent");
free(tor);
return 1;
}
free(tor);
}
endpoint_t *endpoints = NULL;
size_t neps = 0, epcap = 0;
if (argc > 3) {
for (int i = 3; i < argc; i++) {
naut_peer_addr addr;
if (!endpoint_parse(argv[i], &addr)) {
NAUT_WARN("invalid peer address: %s", argv[i]);
continue;
}
if (!endpoint_add(&endpoints, &neps, &epcap, &addr)) {
NAUT_ERROR("out of memory collecting peers");
naut_metainfo_free(&mi);
naut_magnet_free(&magnet);
free(endpoints);
return 1;
}
}
} else {
const uint8_t *hash =
from_magnet ? magnet.infohash_v1 : mi.infohash_v1;
char *const *trackers =
from_magnet ? magnet.trackers : mi.trackers;
size_t num_trackers =
from_magnet ? magnet.num_trackers : mi.num_trackers;
uint64_t total = from_magnet ? 0 : (uint64_t)mi.total_length;
if (!discover_trackers(hash, total, trackers, num_trackers, peerid,
&endpoints, &neps, &epcap) ||
(neps == 0 &&
!discover_dht(hash, &endpoints, &neps, &epcap))) {
NAUT_ERROR("out of memory collecting discovered peers");
naut_metainfo_free(&mi);
naut_magnet_free(&magnet);
free(endpoints);
return 1;
}
}
if (from_magnet && neps) {
uint8_t *info = NULL;
size_t info_len = 0;
naut_err metadata_error = NAUT_ERR_EMPTY;
for (size_t i = 0; i < neps; i++) {
metadata_error = naut_metadata_fetch(
&endpoints[i].addr, magnet.infohash_v1, peerid,
&info, &info_len);
if (metadata_error == NAUT_OK) break;
NAUT_WARN("metadata fetch from %s failed", endpoints[i].name);
}
if (metadata_error != NAUT_OK ||
naut_metainfo_parse_info(
info, info_len, (const char *const *)magnet.trackers,
magnet.num_trackers, &mi) != NAUT_OK) {
NAUT_ERROR("unable to fetch valid magnet metadata");
free(info);
naut_magnet_free(&magnet);
free(endpoints);
return 1;
}
free(info);
NAUT_INFO("magnet metadata verified: %u pieces, %lld bytes",
mi.num_pieces, (long long)mi.total_length);
}
naut_magnet_free(&magnet);
if (neps == 0) {
NAUT_ERROR(argc > 3 ? "no valid peer addresses" :
"tracker and DHT discovery returned no peers");
naut_metainfo_free(&mi);
free(endpoints);
return 1;
}
naut_err err;
naut_storage_opts storage_opts = {
.direct_io = getenv("NAUT_DIRECT_IO") != NULL,
.preallocate = true,
};
naut_storage *st = naut_storage_open_opts(
mi.files, mi.num_files, argv[2], &storage_opts, &err);
if (!st) {
NAUT_ERROR("storage: %s", naut_strerror(err));
naut_metainfo_free(&mi);
free(endpoints);
return 1;
}
naut_download *d = naut_download_create(&mi, st);
if (!d) {
naut_storage_close(st);
naut_metainfo_free(&mi);
free(endpoints);
return 1;
}
int online_cpus = naut_online_cpus();
uint32_t worker_count =
(uint32_t)NAUT_MAX(1, NAUT_MIN(8, online_cpus / 2));
if (getenv("NAUT_WORKERS")) {
unsigned long configured = strtoul(getenv("NAUT_WORKERS"), NULL, 10);
if (configured > 0 && configured <= 256)
worker_count = (uint32_t)configured;
}
int worker_cpu_base = getenv("NAUT_WORKER_CPU_BASE")
? atoi(getenv("NAUT_WORKER_CPU_BASE")) : -1;
naut_worker_pool *workers =
naut_worker_pool_create(worker_count, 1024, worker_cpu_base);
if (!workers) {
NAUT_ERROR("unable to create hash worker pool");
naut_download_destroy(d);
naut_storage_close(st);
naut_metainfo_free(&mi);
free(endpoints);
return 1;
}
naut_download_set_worker_pool(d, workers);
int npeers = (int)neps;
peer_t *peers = calloc(neps, sizeof(*peers));
struct pollfd *pfd = calloc(neps + 1, sizeof(*pfd));
int *idx_map = calloc(neps + 1, sizeof(*idx_map));
if (!peers || !pfd || !idx_map) {
NAUT_ERROR("out of memory creating swarm");
free(peers); free(pfd); free(idx_map);
naut_worker_pool_destroy(workers);
naut_download_destroy(d);
naut_storage_close(st);
naut_metainfo_free(&mi);
free(endpoints);
return 1;
}
for (int i = 0; i < npeers; i++) peers[i].fd = -1;
int active = 0;
for (int i = 0; i < npeers; i++) {
int fd = connect_to(&endpoints[i]);
if (fd < 0) {
NAUT_WARN("connect %s failed", endpoints[i].name);
peers[i].dead = true;
continue;
}
peer_t *p = &peers[i];
p->fd = fd;
snprintf(p->name, sizeof p->name, "%s", endpoints[i].name);
p->peer_choking = true;
naut_pipeline_init(&p->pipeline, NAUT_BLOCK, 4, 1024, 32);
p->rcap = 1 << 18;
p->rbuf = malloc(p->rcap);
if (!p->rbuf || naut_bitfield_init(&p->have, mi.num_pieces) != NAUT_OK) {
free(p->rbuf);
p->rbuf = NULL;
close(fd);
p->fd = -1;
p->dead = true;
continue;
}
uint8_t hs[NAUT_HANDSHAKE_LEN];
naut_peer_handshake_build(hs, mi.infohash_v1, peerid, EXT_RESERVED);
uint8_t intr[5]; naut_peer_msg_simple(intr, NAUT_MSG_INTERESTED);
uint8_t *ext = NULL;
size_t ext_len = 0;
naut_err ext_error = naut_ext_build_handshake(
NAUT_EXT_UT_METADATA, NAUT_EXT_UT_PEX, 0, 0, &ext, &ext_len);
bool sent = ext_error == NAUT_OK &&
send_all(fd, hs, sizeof hs) &&
send_all(fd, ext, ext_len) &&
send_all(fd, intr, 5);
free(ext);
if (!sent) {
peer_drop(d, p);
continue;
}
active++;
}
free(endpoints);
if (!active) {
NAUT_ERROR("no peers reachable");
goto done;
}
NAUT_INFO("swarm: %d peers, %u pieces, %lld bytes", active, mi.num_pieces, (long long)mi.total_length);
double t0 = now();
naut_err run_error = NAUT_OK;
while (!naut_download_complete(d) && run_error == NAUT_OK) {
int nf = 0;
for (int i = 0; i < npeers; i++) {
if (peers[i].dead) continue;
pfd[nf].fd = peers[i].fd; pfd[nf].events = POLLIN; pfd[nf].revents = 0;
idx_map[nf] = i; nf++;
}
int live_peers = nf;
pfd[nf].fd = naut_worker_eventfd(workers);
pfd[nf].events = POLLIN;
pfd[nf].revents = 0;
idx_map[nf] = -1;
nf++;
if (live_peers == 0) {
uint32_t completed = 0;
run_error = naut_download_poll(d, &completed);
if (naut_download_complete(d)) break;
NAUT_ERROR("all peers gone (%.0f%% done)",
100.0 * naut_download_pieces_done(d) / mi.num_pieces);
break;
}
int r = poll(pfd, nf, 2000);
if (r < 0) { if (errno == EINTR) continue; break; }
for (int k = 0; k < nf; k++) {
if (idx_map[k] < 0) {
if (pfd[k].revents & POLLIN) {
uint64_t count;
(void)read(pfd[k].fd, &count, sizeof count);
uint32_t completed = 0;
run_error = naut_download_poll(d, &completed);
}
continue;
}
peer_t *p = &peers[idx_map[k]];
if (!(pfd[k].revents & (POLLIN | POLLHUP | POLLERR))) continue;
if (p->rlen == p->rcap) {
size_t cap = p->rcap * 2;
uint8_t *buf = realloc(p->rbuf, cap);
if (!buf) { peer_drop(d, p); continue; }
p->rbuf = buf;
p->rcap = cap;
}
ssize_t got = recv(p->fd, p->rbuf + p->rlen, p->rcap - p->rlen, 0);
if (got <= 0) { peer_drop(d, p); continue; }
p->rlen += (size_t)got;
run_error = peer_process(d, &mi, peers, npeers, p);
if (run_error != NAUT_OK) break;
if (p->dead) peer_drop(d, p);
}
if (run_error != NAUT_OK) break;
double t = now();
for (int i = 0; i < npeers; i++) {
peer_t *p = &peers[i];
if (!p->dead) expire_requests(d, p, t);
}
for (;;) {
bool sent = false;
for (int i = 0; i < npeers; i++)
if (refill_one(d, &peers[i])) sent = true;
if (!sent) break;
}
}
double dt = now() - t0;
bool ok = naut_download_complete(d);
if (ok) {
double mb = (double)mi.total_length / 1e6;
NAUT_INFO("COMPLETE: %u/%u pieces from swarm in %.2fs (%.1f MB/s), all SHA-1 verified%s",
naut_download_pieces_done(d), mi.num_pieces, dt, mb/dt,
naut_download_in_endgame(d) ? " (passed through endgame)" : "");
} else {
if (run_error != NAUT_OK)
NAUT_ERROR("swarm stopped: %s", naut_strerror(run_error));
NAUT_ERROR("INCOMPLETE: %u/%u pieces", naut_download_pieces_done(d), mi.num_pieces);
}
done:
ok = naut_download_complete(d);
naut_storage_sync(st);
for (int i = 0; i < npeers; i++) {
if (peers[i].blocks_received)
NAUT_INFO("peer %s delivered %llu blocks (pipeline %u, RTT %.1f ms, %.1f MB/s)",
peers[i].name,
(unsigned long long)peers[i].blocks_received,
naut_pipeline_depth(&peers[i].pipeline),
peers[i].pipeline.rtt_seconds * 1000.0,
peers[i].pipeline.bytes_per_second / 1e6);
if (peers[i].pex_received)
NAUT_INFO("peer %s advertised %llu peers through PEX",
peers[i].name,
(unsigned long long)peers[i].pex_received);
if (!peers[i].dead) peer_drop(d, &peers[i]);
free(peers[i].rbuf); free(peers[i].inflight);
if (peers[i].have.words) naut_bitfield_free(&peers[i].have);
}
free(peers); free(pfd); free(idx_map);
naut_worker_pool_destroy(workers);
naut_download_destroy(d); naut_storage_close(st); naut_metainfo_free(&mi);
return ok ? 0 : 1;
}