Limit active pieces and expose file list
This commit is contained in:
parent
ff8e0f5f5b
commit
6dc711cf57
6 changed files with 117 additions and 3 deletions
|
|
@ -278,6 +278,25 @@ static json_t *torrent_json(torrent_task *task) {
|
||||||
}
|
}
|
||||||
json_object_set_new(result, "trackers", trackers);
|
json_object_set_new(result, "trackers", trackers);
|
||||||
}
|
}
|
||||||
|
json_t *files = json_array();
|
||||||
|
if (files) {
|
||||||
|
uint32_t file_count = task->stats.file_count;
|
||||||
|
if (file_count > NAUT_SWARM_MAX_FILE_STATS)
|
||||||
|
file_count = NAUT_SWARM_MAX_FILE_STATS;
|
||||||
|
for (uint32_t i = 0; i < file_count; i++) {
|
||||||
|
const naut_swarm_file_stats *file =
|
||||||
|
&task->stats.file_stats[i];
|
||||||
|
json_t *item = json_pack(
|
||||||
|
"{s:s,s:I,s:f,s:i,s:f}",
|
||||||
|
"name", file->path,
|
||||||
|
"size", (json_int_t)file->size,
|
||||||
|
"progress", file->progress,
|
||||||
|
"priority", file->priority,
|
||||||
|
"availability", file->availability);
|
||||||
|
if (item) json_array_append_new(files, item);
|
||||||
|
}
|
||||||
|
json_object_set_new(result, "files", files);
|
||||||
|
}
|
||||||
json_object_set_new(result, "elapsed_seconds",
|
json_object_set_new(result, "elapsed_seconds",
|
||||||
json_real(task->stats.elapsed_seconds));
|
json_real(task->stats.elapsed_seconds));
|
||||||
json_object_set_new(result, "pending_moves",
|
json_object_set_new(result, "pending_moves",
|
||||||
|
|
|
||||||
|
|
@ -285,6 +285,51 @@ static void snapshot_tracker_stats(naut_swarm_stats *stats,
|
||||||
stats->tracker_stats[i] = trackers[i];
|
stats->tracker_stats[i] = trackers[i];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void snapshot_file_stats(naut_swarm_stats *stats,
|
||||||
|
const naut_download *download,
|
||||||
|
const naut_metainfo *metainfo) {
|
||||||
|
if (!stats || !metainfo || !metainfo->files) return;
|
||||||
|
uint64_t offset = 0;
|
||||||
|
uint32_t count = 0;
|
||||||
|
for (size_t i = 0;
|
||||||
|
i < metainfo->num_files && count < NAUT_SWARM_MAX_FILE_STATS;
|
||||||
|
i++) {
|
||||||
|
const naut_file *file = &metainfo->files[i];
|
||||||
|
uint64_t size = file->length > 0 ? (uint64_t)file->length : 0;
|
||||||
|
uint64_t done = 0;
|
||||||
|
if (download && size > 0) {
|
||||||
|
uint64_t start = offset;
|
||||||
|
uint64_t end = offset + size;
|
||||||
|
uint32_t first = (uint32_t)(start / (uint64_t)metainfo->piece_length);
|
||||||
|
uint32_t last = (uint32_t)((end - 1) /
|
||||||
|
(uint64_t)metainfo->piece_length);
|
||||||
|
for (uint32_t p = first; p <= last; p++) {
|
||||||
|
if (!naut_download_have(download, p)) continue;
|
||||||
|
uint64_t piece_start = (uint64_t)p *
|
||||||
|
(uint64_t)metainfo->piece_length;
|
||||||
|
uint64_t piece_end = piece_start +
|
||||||
|
(uint64_t)metainfo->piece_length;
|
||||||
|
if (piece_end > (uint64_t)metainfo->total_length)
|
||||||
|
piece_end = (uint64_t)metainfo->total_length;
|
||||||
|
uint64_t lo = piece_start > start ? piece_start : start;
|
||||||
|
uint64_t hi = piece_end < end ? piece_end : end;
|
||||||
|
if (hi > lo) done += hi - lo;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
naut_swarm_file_stats *out = &stats->file_stats[count++];
|
||||||
|
memset(out, 0, sizeof(*out));
|
||||||
|
snprintf(out->path, sizeof out->path, "%s",
|
||||||
|
file->path ? file->path : "");
|
||||||
|
out->size = size;
|
||||||
|
out->progress = size ? (double)done / (double)size : 1.0;
|
||||||
|
if (out->progress > 1.0) out->progress = 1.0;
|
||||||
|
out->priority = 1;
|
||||||
|
out->availability = 1.0;
|
||||||
|
offset += size;
|
||||||
|
}
|
||||||
|
stats->file_count = count;
|
||||||
|
}
|
||||||
|
|
||||||
static void report_progress(const naut_swarm_config *config,
|
static void report_progress(const naut_swarm_config *config,
|
||||||
const naut_download *download,
|
const naut_download *download,
|
||||||
const naut_metainfo *metainfo,
|
const naut_metainfo *metainfo,
|
||||||
|
|
@ -308,6 +353,7 @@ static void report_progress(const naut_swarm_config *config,
|
||||||
};
|
};
|
||||||
snapshot_peer_stats(&stats, peers, npeers, metainfo->num_pieces);
|
snapshot_peer_stats(&stats, peers, npeers, metainfo->num_pieces);
|
||||||
snapshot_tracker_stats(&stats, trackers, tracker_count);
|
snapshot_tracker_stats(&stats, trackers, tracker_count);
|
||||||
|
snapshot_file_stats(&stats, download, metainfo);
|
||||||
if (download) {
|
if (download) {
|
||||||
stats.piece_state_count = (uint32_t)naut_download_piece_states(
|
stats.piece_state_count = (uint32_t)naut_download_piece_states(
|
||||||
download, stats.piece_states, NAUT_SWARM_MAX_PIECE_STATS);
|
download, stats.piece_states, NAUT_SWARM_MAX_PIECE_STATS);
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,7 @@
|
||||||
#define NAUT_SWARM_MAX_PEER_STATS 64
|
#define NAUT_SWARM_MAX_PEER_STATS 64
|
||||||
#define NAUT_SWARM_MAX_PIECE_STATS 4000
|
#define NAUT_SWARM_MAX_PIECE_STATS 4000
|
||||||
#define NAUT_SWARM_MAX_TRACKER_STATS 32
|
#define NAUT_SWARM_MAX_TRACKER_STATS 32
|
||||||
|
#define NAUT_SWARM_MAX_FILE_STATS 1024
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
char ip[46];
|
char ip[46];
|
||||||
|
|
@ -40,6 +41,14 @@ typedef struct {
|
||||||
char message[128];
|
char message[128];
|
||||||
} naut_swarm_tracker_stats;
|
} naut_swarm_tracker_stats;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
char path[512];
|
||||||
|
uint64_t size;
|
||||||
|
double progress;
|
||||||
|
int32_t priority;
|
||||||
|
double availability;
|
||||||
|
} naut_swarm_file_stats;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
uint64_t total_bytes;
|
uint64_t total_bytes;
|
||||||
uint64_t bytes_done;
|
uint64_t bytes_done;
|
||||||
|
|
@ -54,6 +63,8 @@ typedef struct {
|
||||||
naut_swarm_peer_stats peer_stats[NAUT_SWARM_MAX_PEER_STATS];
|
naut_swarm_peer_stats peer_stats[NAUT_SWARM_MAX_PEER_STATS];
|
||||||
uint32_t tracker_count;
|
uint32_t tracker_count;
|
||||||
naut_swarm_tracker_stats tracker_stats[NAUT_SWARM_MAX_TRACKER_STATS];
|
naut_swarm_tracker_stats tracker_stats[NAUT_SWARM_MAX_TRACKER_STATS];
|
||||||
|
uint32_t file_count;
|
||||||
|
naut_swarm_file_stats file_stats[NAUT_SWARM_MAX_FILE_STATS];
|
||||||
uint32_t piece_state_count;
|
uint32_t piece_state_count;
|
||||||
uint8_t piece_states[NAUT_SWARM_MAX_PIECE_STATS];
|
uint8_t piece_states[NAUT_SWARM_MAX_PIECE_STATS];
|
||||||
} naut_swarm_stats;
|
} naut_swarm_stats;
|
||||||
|
|
|
||||||
|
|
@ -847,12 +847,14 @@ static json_t *map_torrent(json_t *torrent, bool detail, double dlspeed) {
|
||||||
json_t *peers_list = NULL;
|
json_t *peers_list = NULL;
|
||||||
json_t *hosts = NULL;
|
json_t *hosts = NULL;
|
||||||
json_t *source_trackers = json_object_get(torrent, "trackers");
|
json_t *source_trackers = json_object_get(torrent, "trackers");
|
||||||
|
json_t *source_files = json_object_get(torrent, "files");
|
||||||
hosts = tracker_hosts(source_trackers);
|
hosts = tracker_hosts(source_trackers);
|
||||||
if (detail) {
|
if (detail) {
|
||||||
trackers = json_is_array(source_trackers)
|
trackers = json_is_array(source_trackers)
|
||||||
? json_deep_copy(source_trackers) : json_array();
|
? json_deep_copy(source_trackers) : json_array();
|
||||||
files = json_array();
|
files = json_is_array(source_files)
|
||||||
if (files)
|
? json_deep_copy(source_files) : json_array();
|
||||||
|
if (files && json_array_size(files) == 0 && total > 0)
|
||||||
json_array_append_new(files, json_pack(
|
json_array_append_new(files, json_pack(
|
||||||
"{s:s,s:I,s:f,s:i,s:f}", "name", name,
|
"{s:s,s:I,s:f,s:i,s:f}", "name", name,
|
||||||
"size", (json_int_t)total, "progress", progress,
|
"size", (json_int_t)total, "progress", progress,
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@
|
||||||
#define ENDGAME_BLOCKS 8 /* switch to endgame when this few remain */
|
#define ENDGAME_BLOCKS 8 /* switch to endgame when this few remain */
|
||||||
#define ENDGAME_COPIES 2 /* at most two peers race a missing block */
|
#define ENDGAME_COPIES 2 /* at most two peers race a missing block */
|
||||||
#define PIECE_INFLIGHT_SOFT_CAP 64
|
#define PIECE_INFLIGHT_SOFT_CAP 64
|
||||||
|
#define ACTIVE_PIECE_SOFT_CAP 64
|
||||||
|
|
||||||
/* per-piece in-progress state, lazily allocated and freed on completion */
|
/* per-piece in-progress state, lazily allocated and freed on completion */
|
||||||
typedef struct {
|
typedef struct {
|
||||||
|
|
@ -36,6 +37,7 @@ struct naut_download {
|
||||||
naut_bitfield have;
|
naut_bitfield have;
|
||||||
uint32_t *avail; /* [num_pieces] swarm availability count */
|
uint32_t *avail; /* [num_pieces] swarm availability count */
|
||||||
pstate **ps; /* [num_pieces] in-progress state or NULL */
|
pstate **ps; /* [num_pieces] in-progress state or NULL */
|
||||||
|
uint32_t active_pieces;
|
||||||
|
|
||||||
uint32_t cur_piece; /* sequential cursor for next_request() */
|
uint32_t cur_piece; /* sequential cursor for next_request() */
|
||||||
uint32_t pick_cursor; /* rotating start point for rarest-first ties */
|
uint32_t pick_cursor; /* rotating start point for rarest-first ties */
|
||||||
|
|
@ -88,6 +90,7 @@ static pstate *ensure_ps(naut_download *d, uint32_t p) {
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
d->ps[p] = s;
|
d->ps[p] = s;
|
||||||
|
d->active_pieces++;
|
||||||
return s;
|
return s;
|
||||||
}
|
}
|
||||||
static void free_ps(naut_download *d, uint32_t p) {
|
static void free_ps(naut_download *d, uint32_t p) {
|
||||||
|
|
@ -95,6 +98,7 @@ static void free_ps(naut_download *d, uint32_t p) {
|
||||||
if (!s) return;
|
if (!s) return;
|
||||||
free(s->recv_bits); free(s->req_count); free(s->buf); free(s);
|
free(s->recv_bits); free(s->req_count); free(s->buf); free(s);
|
||||||
d->ps[p] = NULL;
|
d->ps[p] = NULL;
|
||||||
|
if (d->active_pieces) d->active_pieces--;
|
||||||
}
|
}
|
||||||
|
|
||||||
naut_download *naut_download_create(const naut_metainfo *mi, naut_storage *st) {
|
naut_download *naut_download_create(const naut_metainfo *mi, naut_storage *st) {
|
||||||
|
|
@ -230,13 +234,15 @@ bool naut_download_pick_for_peer(naut_download *d, const naut_bitfield *peer_hav
|
||||||
uint32_t p = (d->pick_cursor + n) % d->num_pieces;
|
uint32_t p = (d->pick_cursor + n) % d->num_pieces;
|
||||||
if (naut_bitfield_test(&d->have, p) || !d->ps[p]) continue;
|
if (naut_bitfield_test(&d->have, p) || !d->ps[p]) continue;
|
||||||
if (p >= peer_have->nbits || !naut_bitfield_test(peer_have, p)) continue;
|
if (p >= peer_have->nbits || !naut_bitfield_test(peer_have, p)) continue;
|
||||||
if (!d->endgame &&
|
if (!d->endgame && d->active_pieces < ACTIVE_PIECE_SOFT_CAP &&
|
||||||
piece_inflight(d->ps[p]) >= PIECE_INFLIGHT_SOFT_CAP)
|
piece_inflight(d->ps[p]) >= PIECE_INFLIGHT_SOFT_CAP)
|
||||||
continue;
|
continue;
|
||||||
uint32_t b = first_unreq(d->ps[p]);
|
uint32_t b = first_unreq(d->ps[p]);
|
||||||
if (b != UINT32_MAX) return hand_out(d, p, b, index, begin, length);
|
if (b != UINT32_MAX) return hand_out(d, p, b, index, begin, length);
|
||||||
}
|
}
|
||||||
/* pass 2: start the rarest new piece the peer has */
|
/* pass 2: start the rarest new piece the peer has */
|
||||||
|
if (!d->endgame && d->active_pieces >= ACTIVE_PIECE_SOFT_CAP)
|
||||||
|
return false;
|
||||||
uint32_t best = UINT32_MAX, best_av = UINT32_MAX;
|
uint32_t best = UINT32_MAX, best_av = UINT32_MAX;
|
||||||
for (uint32_t n = 0; n < d->num_pieces; n++) {
|
for (uint32_t n = 0; n < d->num_pieces; n++) {
|
||||||
uint32_t p = (d->pick_cursor + n) % d->num_pieces;
|
uint32_t p = (d->pick_cursor + n) % d->num_pieces;
|
||||||
|
|
|
||||||
|
|
@ -90,6 +90,36 @@ int main(void) {
|
||||||
|
|
||||||
naut_download_destroy(d);
|
naut_download_destroy(d);
|
||||||
|
|
||||||
|
/* The picker should not open the entire torrent at once. A large swarm can
|
||||||
|
* keep many requests in flight, but new-piece fanout is bounded so the
|
||||||
|
* piece map does not show most pieces "downloading" while few verify. */
|
||||||
|
enum { CAP_NP = 80 };
|
||||||
|
uint64_t cap_total = (uint64_t)CAP_NP * NAUT_BLOCK;
|
||||||
|
uint8_t *cap_hashes = malloc(CAP_NP * NAUT_SHA1_LEN);
|
||||||
|
CHECK(cap_hashes != NULL);
|
||||||
|
for (int p = 0; p < CAP_NP; p++)
|
||||||
|
naut_sha1(data, NAUT_BLOCK, cap_hashes + p * NAUT_SHA1_LEN);
|
||||||
|
naut_file cap_file[1] = { { (char *)"cap.bin", (int64_t)cap_total } };
|
||||||
|
naut_metainfo cap_mi; memset(&cap_mi, 0, sizeof cap_mi);
|
||||||
|
cap_mi.has_v1 = true; cap_mi.num_pieces = CAP_NP;
|
||||||
|
cap_mi.piece_length = NAUT_BLOCK; cap_mi.total_length = cap_total;
|
||||||
|
cap_mi.piece_hashes = cap_hashes; cap_mi.files = cap_file;
|
||||||
|
cap_mi.num_files = 1; cap_mi.name = (char *)"cap";
|
||||||
|
d = naut_download_create(&cap_mi, st);
|
||||||
|
CHECK(d != NULL);
|
||||||
|
naut_bitfield all;
|
||||||
|
naut_bitfield_init(&all, CAP_NP);
|
||||||
|
for (int p = 0; p < CAP_NP; p++) naut_bitfield_set(&all, p);
|
||||||
|
naut_download_add_bitfield(d, &all);
|
||||||
|
int opened = 0;
|
||||||
|
while (naut_download_pick(d, &all, &idx, &begin, &len))
|
||||||
|
opened++;
|
||||||
|
CHECK(opened > 0);
|
||||||
|
CHECK(opened < CAP_NP);
|
||||||
|
naut_bitfield_free(&all);
|
||||||
|
naut_download_destroy(d);
|
||||||
|
free(cap_hashes);
|
||||||
|
|
||||||
/* full multi-peer download: alternate peers, all pieces verify */
|
/* full multi-peer download: alternate peers, all pieces verify */
|
||||||
d = naut_download_create(&mi, st);
|
d = naut_download_create(&mi, st);
|
||||||
naut_download_add_bitfield(d, &hb); /* one peer that has everything */
|
naut_download_add_bitfield(d, &hb); /* one peer that has everything */
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue