diff --git a/apps/nautd/main.c b/apps/nautd/main.c index e99b421..b03c28b 100644 --- a/apps/nautd/main.c +++ b/apps/nautd/main.c @@ -256,6 +256,28 @@ static json_t *torrent_json(torrent_task *task) { } json_object_set_new(result, "peer_list", peer_list); } + json_t *trackers = json_array(); + if (trackers) { + uint32_t tracker_count = task->stats.tracker_count; + if (tracker_count > NAUT_SWARM_MAX_TRACKER_STATS) + tracker_count = NAUT_SWARM_MAX_TRACKER_STATS; + for (uint32_t i = 0; i < tracker_count; i++) { + const naut_swarm_tracker_stats *tracker = + &task->stats.tracker_stats[i]; + json_t *item = json_pack( + "{s:s,s:i,s:s,s:i,s:i,s:i,s:i,s:s}", + "url", tracker->url, + "tier", tracker->tier, + "status", tracker->status, + "seeds", tracker->seeds, + "peers", tracker->peers, + "leeches", tracker->leeches, + "downloaded", tracker->downloaded, + "message", tracker->message); + if (item) json_array_append_new(trackers, item); + } + json_object_set_new(result, "trackers", trackers); + } json_object_set_new(result, "elapsed_seconds", json_real(task->stats.elapsed_seconds)); json_object_set_new(result, "pending_moves", diff --git a/apps/swarm/main.c b/apps/swarm/main.c index 258ce53..16219ea 100644 --- a/apps/swarm/main.c +++ b/apps/swarm/main.c @@ -41,6 +41,10 @@ #define EXT_RESERVED 0x0000000000100000ULL #define DEFAULT_TARGET_PEERS 80 #define MAX_TARGET_PEERS 512 +#define TRACKER_DEFAULT_INTERVAL 1800.0 +#define TRACKER_MIN_INTERVAL 60.0 +#define TRACKER_FAILURE_RETRY_INTERVAL 300.0 +#define DHT_REFRESH_INTERVAL 300.0 typedef struct { uint32_t piece, begin, length; @@ -85,6 +89,13 @@ static uint32_t target_peer_count(void) { return (uint32_t)NAUT_MIN(value, MAX_TARGET_PEERS); } +static double tracker_delay_seconds(int32_t interval) { + if (interval <= 0) return TRACKER_DEFAULT_INTERVAL; + if (interval < (int32_t)TRACKER_MIN_INTERVAL) + return TRACKER_MIN_INTERVAL; + return (double)interval; +} + static void random_bytes(uint8_t *output, size_t length) { int fd = open("/dev/urandom", O_RDONLY); size_t offset = 0; @@ -225,12 +236,63 @@ static void snapshot_peer_stats(naut_swarm_stats *stats, } } +static uint32_t init_tracker_stats(char *const *trackers, + const uint32_t *tracker_tiers, + size_t num_trackers, + naut_swarm_tracker_stats *stats, + uint32_t capacity) { + uint32_t count = 0; + if (!stats) return 0; + for (size_t i = 0; i < num_trackers && count < capacity; i++) { + naut_swarm_tracker_stats *out = &stats[count++]; + memset(out, 0, sizeof(*out)); + snprintf(out->url, sizeof out->url, "%s", trackers[i]); + out->tier = tracker_tiers ? (int32_t)tracker_tiers[i] : (int32_t)i; + snprintf(out->status, sizeof out->status, "not contacted"); + out->seeds = -1; + out->peers = -1; + out->leeches = -1; + out->downloaded = -1; + } + return count; +} + +static naut_swarm_tracker_stats *tracker_stat_for( + naut_swarm_tracker_stats *stats, uint32_t count, const char *url) { + if (!stats || !url) return NULL; + for (uint32_t i = 0; i < count; i++) + if (strcmp(stats[i].url, url) == 0) return &stats[i]; + return NULL; +} + +static void tracker_set_status(naut_swarm_tracker_stats *tracker, + const char *status, const char *message) { + if (!tracker) return; + snprintf(tracker->status, sizeof tracker->status, "%s", + status ? status : ""); + snprintf(tracker->message, sizeof tracker->message, "%s", + message ? message : ""); +} + +static void snapshot_tracker_stats(naut_swarm_stats *stats, + const naut_swarm_tracker_stats *trackers, + uint32_t tracker_count) { + if (!stats || !trackers) return; + if (tracker_count > NAUT_SWARM_MAX_TRACKER_STATS) + tracker_count = NAUT_SWARM_MAX_TRACKER_STATS; + stats->tracker_count = tracker_count; + for (uint32_t i = 0; i < tracker_count; i++) + stats->tracker_stats[i] = trackers[i]; +} + static void report_progress(const naut_swarm_config *config, const naut_download *download, const naut_metainfo *metainfo, uint32_t peers_total, uint32_t peers_connecting, uint32_t peers_active, uint32_t peers_failed, peer_t *peers, int npeers, + const naut_swarm_tracker_stats *trackers, + uint32_t tracker_count, double started_at) { if (!config->on_progress) return; naut_swarm_stats stats = { @@ -245,6 +307,7 @@ static void report_progress(const naut_swarm_config *config, .elapsed_seconds = now() - started_at, }; snapshot_peer_stats(&stats, peers, npeers, metainfo->num_pieces); + snapshot_tracker_stats(&stats, trackers, tracker_count); if (download) { stats.piece_state_count = (uint32_t)naut_download_piece_states( download, stats.piece_states, NAUT_SWARM_MAX_PIECE_STATS); @@ -340,16 +403,24 @@ static bool discover_trackers(const uint8_t info_hash[20], uint64_t total_length char *const *trackers, size_t num_trackers, const uint32_t *tracker_tiers, const uint8_t peerid[20], - endpoint_t **eps, size_t *neps, size_t *cap) { + uint64_t downloaded, uint64_t left, + naut_tracker_event event, + endpoint_t **eps, size_t *neps, size_t *cap, + int32_t *announce_interval, + naut_swarm_tracker_stats *tracker_stats, + uint32_t tracker_count) { naut_announce_req req; memset(&req, 0, sizeof req); memcpy(req.info_hash, info_hash, sizeof req.info_hash); memcpy(req.peer_id, peerid, sizeof req.peer_id); req.port = 6881; + req.downloaded = downloaded; req.left = total_length; - req.event = NAUT_TEV_STARTED; + if (left <= total_length) req.left = left; + req.event = event; req.numwant = 100; memcpy(&req.key, peerid + 8, sizeof req.key); + if (announce_interval) *announce_interval = 0; size_t tier_start = 0; while (tier_start < num_trackers) { @@ -370,6 +441,10 @@ static bool discover_trackers(const uint8_t info_hash[20], uint64_t total_length size_t i = tier_start + (first + n) % tier_count; const char *tracker = trackers[i]; naut_tracker_response response; + memset(&response, 0, sizeof response); + response.seeders = response.leechers = -1; + naut_swarm_tracker_stats *tracker_stat = + tracker_stat_for(tracker_stats, tracker_count, tracker); naut_err e = NAUT_ERR_INVAL; if (strncmp(tracker, "http://", 7) == 0) { char url[4096]; @@ -382,14 +457,30 @@ static bool discover_trackers(const uint8_t info_hash[20], uint64_t total_length if (parse_udp_tracker(tracker, host, sizeof host, &port)) e = naut_tracker_announce_udp(host, port, &req, &response); } else { + tracker_set_status(tracker_stat, "unsupported", + "unsupported tracker scheme"); NAUT_WARN("tracker scheme unsupported: %s", tracker); continue; } if (e != NAUT_OK) { + tracker_set_status(tracker_stat, "error", + response.failure + ? response.failure : "announce failed"); NAUT_WARN("tracker announce failed: %s", tracker); + naut_tracker_response_free(&response); continue; } tier_succeeded = true; + if (announce_interval && response.interval > 0) + *announce_interval = response.interval; + if (tracker_stat) { + tracker_set_status(tracker_stat, "working", ""); + tracker_stat->seeds = response.seeders; + tracker_stat->leeches = response.leechers; + tracker_stat->peers = response.num_peers > INT32_MAX + ? INT32_MAX : (int32_t)response.num_peers; + tracker_stat->downloaded = -1; + } NAUT_INFO("tracker %s returned %zu peers", tracker, response.num_peers); for (size_t p = 0; p < response.num_peers; p++) { @@ -934,6 +1025,9 @@ naut_err naut_swarm_run(const naut_swarm_config *config) { endpoint_t *endpoints = NULL; size_t neps = 0, epcap = 0; uint32_t target_peers = target_peer_count(); + int32_t tracker_interval = 0; + naut_swarm_tracker_stats tracker_stats[NAUT_SWARM_MAX_TRACKER_STATS]; + uint32_t tracker_count = 0; if (config->num_peers > 0) { for (size_t i = 0; i < config->num_peers; i++) { naut_peer_addr addr; @@ -958,10 +1052,15 @@ naut_err naut_swarm_run(const naut_swarm_config *config) { from_magnet ? NULL : mi.tracker_tiers; size_t num_trackers = from_magnet ? magnet.num_trackers : mi.num_trackers; + tracker_count = init_tracker_stats(trackers, tracker_tiers, + num_trackers, tracker_stats, + NAUT_SWARM_MAX_TRACKER_STATS); uint64_t total = from_magnet ? 0 : (uint64_t)mi.total_length; if (!discover_trackers(hash, total, trackers, num_trackers, - tracker_tiers, peerid, &endpoints, &neps, - &epcap) || + tracker_tiers, peerid, 0, total, + NAUT_TEV_STARTED, &endpoints, &neps, &epcap, + &tracker_interval, tracker_stats, + tracker_count) || (neps < target_peers && !discover_dht(hash, &endpoints, &neps, &epcap))) { NAUT_ERROR("out of memory collecting discovered peers"); @@ -1088,6 +1187,11 @@ naut_err naut_swarm_run(const naut_swarm_config *config) { for (int i = 0; i < npeers; i++) peers[i].fd = -1; double t0 = now(); + double next_tracker_announce = + t0 + (tracker_interval > 0 + ? tracker_delay_seconds(tracker_interval) + : TRACKER_FAILURE_RETRY_INTERVAL); + double next_dht_lookup = t0 + DHT_REFRESH_INTERVAL; naut_err run_error = NAUT_OK; bool cancelled = false; uint32_t connecting = 0; @@ -1096,7 +1200,7 @@ naut_err naut_swarm_run(const naut_swarm_config *config) { struct pollfd *connect_fds = pfd; for (int i = 0; i < npeers; i++) connect_fds[i].fd = -1; report_progress(config, d, &mi, (uint32_t)npeers, 0, 0, 0, - peers, npeers, t0); + peers, npeers, tracker_stats, tracker_count, t0); for (int i = 0; i < npeers; i++) { bool connected = false; int fd = connect_start(&endpoints[i], &connected); @@ -1130,7 +1234,8 @@ naut_err naut_swarm_run(const naut_swarm_config *config) { while (connecting > 0 && now() < connect_deadline && !stop_requested(config)) { report_progress(config, d, &mi, (uint32_t)npeers, connecting, - active, failed, peers, npeers, t0); + active, failed, peers, npeers, + tracker_stats, tracker_count, t0); int ready = poll(connect_fds, (nfds_t)neps, 100); if (ready < 0) { if (errno == EINTR) continue; @@ -1181,13 +1286,53 @@ naut_err naut_swarm_run(const naut_swarm_config *config) { emit_event(config, NAUT_EVENT_TORRENT_ADDED, 0, NULL, NULL); report_progress(config, d, &mi, (uint32_t)npeers, 0, active, failed, - peers, npeers, t0); + peers, npeers, tracker_stats, tracker_count, t0); while (!naut_download_complete(d) && run_error == NAUT_OK) { service_control(config, st); if (stop_requested(config)) { cancelled = true; break; } + if (config->num_peers == 0 && npending == 0 && + peer_count_active(peers, npeers, true) < target_peers) { + double t = now(); + if (mi.num_trackers > 0 && t >= next_tracker_announce) { + uint64_t downloaded = naut_download_bytes_done(d); + uint64_t left = (uint64_t)mi.total_length > downloaded + ? (uint64_t)mi.total_length - downloaded : 0; + int32_t interval = 0; + size_t before = npending; + if (!discover_trackers(mi.infohash_v1, + (uint64_t)mi.total_length, + mi.trackers, mi.num_trackers, + mi.tracker_tiers, peerid, + downloaded, left, NAUT_TEV_NONE, + &pending, &npending, &pending_cap, + &interval, tracker_stats, + tracker_count)) { + run_error = NAUT_ERR_NOMEM; + break; + } + next_tracker_announce = t + (interval > 0 + ? tracker_delay_seconds(interval) + : TRACKER_FAILURE_RETRY_INTERVAL); + if (npending > before) + NAUT_INFO("tracker refresh queued %zu peers", + npending - before); + } + if (npending == 0 && t >= next_dht_lookup) { + size_t before = npending; + if (!discover_dht(mi.infohash_v1, &pending, &npending, + &pending_cap)) { + run_error = NAUT_ERR_NOMEM; + break; + } + next_dht_lookup = t + DHT_REFRESH_INTERVAL; + if (npending > before) + NAUT_INFO("DHT refresh queued %zu peers", + npending - before); + } + } if (!connect_pending_peers(config, d, &mi, peerid, &peers, &npeers, &peer_capacity, &pfd, &idx_map, pending, &npending, target_peers, &failed)) { @@ -1226,7 +1371,7 @@ naut_err naut_swarm_run(const naut_swarm_config *config) { (uint32_t)connecting_peers, (uint32_t)live_peers, failed, - peers, npeers, t0); + peers, npeers, tracker_stats, tracker_count, t0); int r = poll(pfd, nf, 200); if (r < 0) { if (errno == EINTR) continue; @@ -1304,7 +1449,8 @@ naut_err naut_swarm_run(const naut_swarm_config *config) { naut_download_pieces_done(d), mi.num_pieces, dt, mb/dt, naut_download_in_endgame(d) ? " (passed through endgame)" : ""); report_progress(config, d, &mi, (uint32_t)npeers, 0, 0, - (uint32_t)npeers, peers, npeers, t0); + (uint32_t)npeers, peers, npeers, + tracker_stats, tracker_count, t0); emit_event(config, NAUT_EVENT_TORRENT_FINISHED, 0, NULL, NULL); while (config->keep_alive && !stop_requested(config)) { service_control(config, st); diff --git a/include/naut/swarm.h b/include/naut/swarm.h index 952c80e..092547d 100644 --- a/include/naut/swarm.h +++ b/include/naut/swarm.h @@ -13,6 +13,7 @@ #define NAUT_SWARM_MAX_PEER_STATS 64 #define NAUT_SWARM_MAX_PIECE_STATS 4000 +#define NAUT_SWARM_MAX_TRACKER_STATS 32 typedef struct { char ip[46]; @@ -28,6 +29,17 @@ typedef struct { uint64_t uploaded; } naut_swarm_peer_stats; +typedef struct { + char url[256]; + int32_t tier; + char status[32]; + int32_t seeds; + int32_t peers; + int32_t leeches; + int32_t downloaded; + char message[128]; +} naut_swarm_tracker_stats; + typedef struct { uint64_t total_bytes; uint64_t bytes_done; @@ -40,6 +52,8 @@ typedef struct { double elapsed_seconds; uint32_t peer_count; naut_swarm_peer_stats peer_stats[NAUT_SWARM_MAX_PEER_STATS]; + uint32_t tracker_count; + naut_swarm_tracker_stats tracker_stats[NAUT_SWARM_MAX_TRACKER_STATS]; uint32_t piece_state_count; uint8_t piece_states[NAUT_SWARM_MAX_PIECE_STATS]; } naut_swarm_stats; diff --git a/plugins/webui/webui.c b/plugins/webui/webui.c index 817c1c0..7e34b43 100644 --- a/plugins/webui/webui.c +++ b/plugins/webui/webui.c @@ -574,6 +574,53 @@ static json_t *tracker_hosts(json_t *trackers) { return hosts; } +static bool tracker_host(char out[256], const char *url) { + if (!url || strstr(url, "**")) return false; + const char *start = strstr(url, "://"); + start = start ? start + 3 : url; + size_t len = strcspn(start, "/"); + if (len == 0 || len >= 256) return false; + snprintf(out, 256, "%.*s", (int)len, start); + return true; +} + +static json_t *tracker_summary(json_t *torrents) { + json_t *summary = json_array(); + if (!summary || !json_is_array(torrents)) return summary; + size_t tindex; + json_t *torrent; + json_array_foreach(torrents, tindex, torrent) { + json_t *trackers = json_object_get(torrent, "trackers"); + if (!json_is_array(trackers)) continue; + size_t index; + json_t *tracker; + json_array_foreach(trackers, index, tracker) { + int64_t tier = json_integer_value(json_object_get(tracker, "tier")); + if (tier < 0) continue; + char host[256]; + if (!tracker_host(host, json_string_value( + json_object_get(tracker, "url")))) + continue; + bool found = false; + size_t hindex; + json_t *entry; + json_array_foreach(summary, hindex, entry) { + if (strcmp(json_string_or(entry, "host", ""), host) != 0) + continue; + json_int_t count = + json_integer_value(json_object_get(entry, "count")); + json_object_set_new(entry, "count", json_integer(count + 1)); + found = true; + break; + } + if (!found) + json_array_append_new(summary, json_pack( + "{s:s,s:i}", "host", host, "count", 1)); + } + } + return summary; +} + static json_t *map_peer_list(json_t *torrent) { json_t *out = json_array(); json_t *peers = json_object_get(torrent, "peer_list"); @@ -799,15 +846,11 @@ static json_t *map_torrent(json_t *torrent, bool detail, double dlspeed) { json_t *files = NULL; json_t *peers_list = NULL; json_t *hosts = NULL; + json_t *source_trackers = json_object_get(torrent, "trackers"); + hosts = tracker_hosts(source_trackers); if (detail) { - trackers = json_array(); - if (trackers) - json_array_append_new(trackers, json_pack( - "{s:s,s:i,s:s,s:i,s:i,s:i,s:i,s:s}", - "url", "** [DHT] **", "tier", -1, "status", "working", - "seeds", (int)json_u64(torrent, "peers_discovered"), - "peers", (int)json_u64(torrent, "peers"), - "leeches", -1, "downloaded", -1, "message", "")); + trackers = json_is_array(source_trackers) + ? json_deep_copy(source_trackers) : json_array(); files = json_array(); if (files) json_array_append_new(files, json_pack( @@ -815,7 +858,6 @@ static json_t *map_torrent(json_t *torrent, bool detail, double dlspeed) { "size", (json_int_t)total, "progress", progress, "priority", 1, "availability", 1.0)); peers_list = map_peer_list(torrent); - hosts = tracker_hosts(trackers); } uint64_t discovered = json_u64(torrent, "peers_discovered"); @@ -825,11 +867,11 @@ static json_t *map_torrent(json_t *torrent, bool detail, double dlspeed) { json_t *out = json_object(); if (!out) { free(name); + json_decref(hosts); if (detail) { json_decref(trackers); json_decref(files); json_decref(peers_list); - json_decref(hosts); } return NULL; } @@ -1013,7 +1055,12 @@ static void api_meta(int fd) { json_deep_copy(g_webui.categories)); json_object_set_new(json, "tags", json_deep_copy(g_webui.tags)); pthread_mutex_unlock(&g_webui.meta_lock); - json_object_set_new(json, "trackers", json_array()); + json_t *params = json_object(); + json_t *torrents = rpc_call_json("torrents", params); + json_decref(params); + json_t *trackers = tracker_summary(torrents); + json_decref(torrents); + json_object_set_new(json, "trackers", trackers ? trackers : json_array()); json_object_set_new(preferences, "save_path", json_string(getenv("NAUT_WEBUI_SAVE_PATH") ? getenv("NAUT_WEBUI_SAVE_PATH") : "."));