From b9b4b9cca2a1e87e5d3d27c241a49afaef8ddb23 Mon Sep 17 00:00:00 2001 From: Mauro Mombelli Date: Sun, 25 Feb 2018 16:26:07 +0100 Subject: [PATCH] First implementation of parallel download --- lib/libalpm/sync.c | 94 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 82 insertions(+), 12 deletions(-) diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c index 73120b63..b728bead 100644 --- a/lib/libalpm/sync.c +++ b/lib/libalpm/sync.c @@ -28,6 +28,7 @@ #include /* intmax_t */ #include #include +#include /* libalpm */ #include "sync.h" @@ -933,29 +934,96 @@ static int download_single_file(alpm_handle_t *handle, struct dload_payload *pay payload->handle = handle; payload->allow_resume = 1; - EVENT(handle, &event); + EVENT(handle, &event); /* TODO: is handle/EVENT thread-safe? */ for(server = payload->servers; server; server = server->next) { const char *server_url = server->data; size_t len; /* print server + filename into a buffer */ len = strlen(server_url) + strlen(payload->remote_name) + 2; - MALLOC(payload->fileurl, len, RET_ERR(handle, ALPM_ERR_MEMORY, -1)); + MALLOC(payload->fileurl, len, RET_ERR(handle, ALPM_ERR_MEMORY, -1)); /* TODO: is handle/RET_ERR thread-safe? */ snprintf(payload->fileurl, len, "%s/%s", server_url, payload->remote_name); if(_alpm_download(payload, cachedir, NULL, NULL) != -1) { event.type = ALPM_EVENT_PKGDOWNLOAD_DONE; - EVENT(handle, &event); + EVENT(handle, &event); /* TODO: is handle/event thread-safe? */ return 0; } _alpm_dload_payload_reset_for_retry(payload); } event.type = ALPM_EVENT_PKGDOWNLOAD_FAILED; - EVENT(handle, &event); + EVENT(handle, &event); /* TODO: is handle/event thread-safe? */ return -1; } +void serial_download_files(alpm_handle_t *handle, const int num_files, alpm_list_t *files, const char * const cachedir, int * const errors, alpm_event_t *event) +{ + (void)num_files; /* ignore parameter */ + + alpm_list_t *i; + for(i = files; i; i = i->next) { + if(download_single_file(handle, i->data, cachedir) == -1) { + (*errors)++; + event->type = ALPM_EVENT_RETRIEVE_FAILED; + _alpm_log(handle, ALPM_LOG_WARNING, _("failed to retrieve some files\n")); + } + } +} + +struct download_single_file_argument{ + alpm_handle_t *handle; + struct dload_payload *payload; + const char *cachedir; +}; + +static void* download_single_file_wrapper(void * const arg_void) +{ + struct download_single_file_argument * const arg = (struct download_single_file_argument * const)arg_void; + int * const ris = malloc(sizeof(int)); + *ris = download_single_file(arg->handle, arg->payload, arg->cachedir); + return ris; +} + +static void parallel_download_files(alpm_handle_t *handle, const int num_files, alpm_list_t *files, const char * const cachedir, int * const errors, alpm_event_t *event) +{ + alpm_list_t *i; + pthread_t download_thread[num_files]; + struct download_single_file_argument download_thread_argument[num_files]; + size_t thread_idx; + for(i = files, thread_idx = 0; i; i = i->next) { + download_thread_argument[thread_idx].cachedir = cachedir; + download_thread_argument[thread_idx].handle = handle; + download_thread_argument[thread_idx].payload = i->data; + const int err = pthread_create( &download_thread[thread_idx], NULL, download_single_file_wrapper, (void*) &download_thread_argument[thread_idx]); + if(err == 0) { + thread_idx++; + } else { + (*errors)++; + event->type = ALPM_EVENT_RETRIEVE_FAILED; + _alpm_log(handle, ALPM_LOG_WARNING, _("failed to start some download thread\n")); + } + } + for(; thread_idx; thread_idx--) { + int * function_return_err; + const int err = pthread_join(download_thread[thread_idx - 1], (void**)&function_return_err); + if ( err || (*function_return_err) ){ + (*errors)++; + event->type = ALPM_EVENT_RETRIEVE_FAILED; + _alpm_log(handle, ALPM_LOG_WARNING, _("failed to retrieve some files\n")); + } + if (err == 0){ + if (*function_return_err){ + _alpm_log(handle, ALPM_LOG_WARNING, _("failed to download\n")); + printf("err: %d\n", *function_return_err); + } + free(function_return_err); + }else{ + _alpm_log(handle, ALPM_LOG_WARNING, _("failed to join some download thread\n")); + } + } +} + static int download_files(alpm_handle_t *handle, alpm_list_t **deltas) { const char *cachedir; @@ -984,15 +1052,15 @@ static int download_files(alpm_handle_t *handle, alpm_list_t **deltas) } if(files) { + const size_t num_files = alpm_list_count(files); /* check for necessary disk space for download */ if(handle->checkspace) { off_t *file_sizes; - size_t idx, num_files; + size_t idx; int ret; _alpm_log(handle, ALPM_LOG_DEBUG, "checking available disk space for download\n"); - num_files = alpm_list_count(files); CALLOC(file_sizes, num_files, sizeof(off_t), goto finish); for(i = files, idx = 0; i; i = i->next, idx++) { @@ -1012,13 +1080,15 @@ static int download_files(alpm_handle_t *handle, alpm_list_t **deltas) event.type = ALPM_EVENT_RETRIEVE_START; EVENT(handle, &event); event.type = ALPM_EVENT_RETRIEVE_DONE; - for(i = files; i; i = i->next) { - if(download_single_file(handle, i->data, cachedir) == -1) { - errors++; - event.type = ALPM_EVENT_RETRIEVE_FAILED; - _alpm_log(handle, ALPM_LOG_WARNING, _("failed to retrieve some files\n")); - } + + /* currently internal curl implementation does NOT support parallel download */ + int download_parallel = 1; /* TODO: read from pacman.conf */ + if (download_parallel && handle->fetchcb != NULL) { + parallel_download_files(handle, num_files, files, cachedir, &errors, &event); + } else { + serial_download_files(handle, num_files, files, cachedir, &errors, &event); } + EVENT(handle, &event); } -- 2.16.2