From 4040fc736ae78450d5f9437413b8f810db7e21a6 Mon Sep 17 00:00:00 2001 From: Mauro Mombelli Date: Thu, 1 Mar 2018 19:33:40 +0100 Subject: [PATCH] tested working parallel download with lock --- lib/libalpm/alpm.h | 10 ++++++++++ lib/libalpm/dload.c | 23 +++++++++++++++++++++-- lib/libalpm/dload.h | 2 ++ lib/libalpm/handle.c | 13 +++++++++++++ lib/libalpm/handle.h | 1 + lib/libalpm/sync.c | 21 +++++++++++++++++---- src/pacman/conf.c | 35 +++++++++++++++++++++++++++++++++-- src/pacman/conf.h | 1 + src/pacman/pacman-conf.c | 3 +++ 9 files changed, 101 insertions(+), 8 deletions(-) diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h index 2637a055..371b9f13 100644 --- a/lib/libalpm/alpm.h +++ b/lib/libalpm/alpm.h @@ -756,6 +756,11 @@ typedef void (*alpm_cb_totaldl)(off_t total); typedef int (*alpm_cb_fetch)(const char *url, const char *localpath, int force); +/** A callback for waiting for download of file + * @return 0 on success, -1 on error. + */ +typedef int (*alpm_cb_fetch_lock)(); + /** Fetch a remote pkg. * @param handle the context handle * @param url URL of the package to download @@ -783,6 +788,11 @@ alpm_cb_fetch alpm_option_get_fetchcb(alpm_handle_t *handle); /** Sets the downloading callback. */ int alpm_option_set_fetchcb(alpm_handle_t *handle, alpm_cb_fetch cb); +/** Returns the downloading lock callback. */ +alpm_cb_fetch_lock alpm_option_get_fetch_lockcb(alpm_handle_t *handle); +/** Sets the downloading lock callback. */ +int alpm_option_set_fetch_lockcb(alpm_handle_t *handle, alpm_cb_fetch_lock cb); + /** Returns the callback used to report total download size. */ alpm_cb_totaldl alpm_option_get_totaldlcb(alpm_handle_t *handle); /** Sets the callback used to report total download size. */ diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 44db5f88..bd425081 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -586,14 +586,14 @@ cleanup: } #endif -/** Download a file given by a URL to a local directory. +/** Download a file given by a URL to a local directory, it will NOT wait for the download to be completed * Does not overwrite an existing file if the download fails. * @param payload the payload context * @param localpath the directory to save the file in * @param final_file the real name of the downloaded file (may be NULL) * @return 0 on success, -1 on error (pm_errno is set accordingly if errors_ok == 0) */ -int _alpm_download(struct dload_payload *payload, const char *localpath, +int _alpm_download_unlock(struct dload_payload *payload, const char *localpath, char **final_file, const char **final_url) { alpm_handle_t *handle = payload->handle; @@ -616,6 +616,25 @@ int _alpm_download(struct dload_payload *payload, const char *localpath, } } +/** Download a file given by a URL to a local directory. + * Does not overwrite an existing file if the download fails. + * @param payload the payload context + * @param localpath the directory to save the file in + * @param final_file the real name of the downloaded file (may be NULL) + * @return 0 on success, -1 on error (pm_errno is set accordingly if errors_ok == 0) + */ +int _alpm_download(struct dload_payload *payload, const char *localpath, + char **final_file, const char **final_url) +{ + alpm_handle_t *handle = payload->handle; + _alpm_log(handle, ALPM_LOG_WARNING, _("_alpm_download\n")); + int err = _alpm_download_unlock(payload, localpath, final_file, final_url); + if (err == 0){ + err = handle->fetch_lockcb(); + } + return err; +} + static char *filecache_find_url(alpm_handle_t *handle, const char *url) { const char *filebase = strrchr(url, '/'); diff --git a/lib/libalpm/dload.h b/lib/libalpm/dload.h index ac948528..40a9a0e6 100644 --- a/lib/libalpm/dload.h +++ b/lib/libalpm/dload.h @@ -52,6 +52,8 @@ void _alpm_dload_payload_reset_for_retry(struct dload_payload *payload); int _alpm_download(struct dload_payload *payload, const char *localpath, char **final_file, const char **final_url); +int _alpm_download_unlock(struct dload_payload *payload, const char *localpath, + char **final_file, const char **final_url); #endif /* ALPM_DLOAD_H */ diff --git a/lib/libalpm/handle.c b/lib/libalpm/handle.c index 08625e3a..593f437a 100644 --- a/lib/libalpm/handle.c +++ b/lib/libalpm/handle.c @@ -189,6 +189,12 @@ alpm_cb_fetch SYMEXPORT alpm_option_get_fetchcb(alpm_handle_t *handle) return handle->fetchcb; } +alpm_cb_fetch_lock SYMEXPORT alpm_option_get_fetch_lockcb(alpm_handle_t *handle) +{ + CHECK_HANDLE(handle, return NULL); + return handle->fetch_lockcb; +} + alpm_cb_totaldl SYMEXPORT alpm_option_get_totaldlcb(alpm_handle_t *handle) { CHECK_HANDLE(handle, return NULL); @@ -342,6 +348,13 @@ int SYMEXPORT alpm_option_set_fetchcb(alpm_handle_t *handle, alpm_cb_fetch cb) return 0; } +int SYMEXPORT alpm_option_set_fetch_lockcb(alpm_handle_t *handle, alpm_cb_fetch_lock cb) +{ + CHECK_HANDLE(handle, return -1); + handle->fetch_lockcb = cb; + return 0; +} + int SYMEXPORT alpm_option_set_totaldlcb(alpm_handle_t *handle, alpm_cb_totaldl cb) { CHECK_HANDLE(handle, return -1); diff --git a/lib/libalpm/handle.h b/lib/libalpm/handle.h index 115c3481..a9df3be7 100644 --- a/lib/libalpm/handle.h +++ b/lib/libalpm/handle.h @@ -72,6 +72,7 @@ struct __alpm_handle_t { alpm_cb_download dlcb; /* Download callback function */ alpm_cb_totaldl totaldlcb; /* Total download callback function */ alpm_cb_fetch fetchcb; /* Download file callback function */ + alpm_cb_fetch_lock fetch_lockcb; /* Download lock file callback function */ alpm_cb_event eventcb; alpm_cb_question questioncb; alpm_cb_progress progresscb; diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c index 73120b63..8099e8e0 100644 --- a/lib/libalpm/sync.c +++ b/lib/libalpm/sync.c @@ -943,10 +943,18 @@ static int download_single_file(alpm_handle_t *handle, struct dload_payload *pay MALLOC(payload->fileurl, len, RET_ERR(handle, ALPM_ERR_MEMORY, -1)); snprintf(payload->fileurl, len, "%s/%s", server_url, payload->remote_name); - if(_alpm_download(payload, cachedir, NULL, NULL) != -1) { - event.type = ALPM_EVENT_PKGDOWNLOAD_DONE; - EVENT(handle, &event); - return 0; + if (handle->fetch_lockcb == NULL) { + if (_alpm_download(payload, cachedir, NULL, NULL) != -1) { + event.type = ALPM_EVENT_PKGDOWNLOAD_DONE; + EVENT(handle, &event); + return 0; + } + }else{ + if (_alpm_download_unlock(payload, cachedir, NULL, NULL) != -1) { + event.type = ALPM_EVENT_PKGDOWNLOAD_DONE; + EVENT(handle, &event); + return 0; + } } _alpm_dload_payload_reset_for_retry(payload); } @@ -1019,6 +1027,11 @@ static int download_files(alpm_handle_t *handle, alpm_list_t **deltas) _alpm_log(handle, ALPM_LOG_WARNING, _("failed to retrieve some files\n")); } } + + if (handle->fetch_lockcb != NULL) { + handle->fetch_lockcb(); + } + EVENT(handle, &event); } diff --git a/src/pacman/conf.c b/src/pacman/conf.c index 1ad75405..9a977c43 100644 --- a/src/pacman/conf.c +++ b/src/pacman/conf.c @@ -152,6 +152,7 @@ int config_free(config_t *oldconfig) FREELIST(oldconfig->hookdirs); FREELIST(oldconfig->cachedirs); free(oldconfig->xfercommand); + free(oldconfig->xferlockcommand); free(oldconfig->print_format); free(oldconfig->arch); free(oldconfig); @@ -235,8 +236,13 @@ static int download_with_xfercommand(const char *url, const char *localpath, tempcmd = strdup(config->xfercommand); /* replace all occurrences of %o with fn.part */ if(strstr(tempcmd, "%o")) { - usepart = 1; - parsedcmd = strreplace(tempcmd, "%o", tempfile); + //in unlock mode, the management of the temp part is left to the external program + if (config->xferlockcommand == NULL) { + usepart = 1; + parsedcmd = strreplace(tempcmd, "%o", tempfile); + }else { + parsedcmd = strreplace(tempcmd, "%o", destfile); + } free(tempcmd); tempcmd = parsedcmd; } @@ -303,6 +309,22 @@ cleanup: return ret; } +/** External fetch lock callback */ +static int download_lock_with_xfercommand() +{ + if(!config->xferlockcommand) { + return -1; + } + + int retval = system(config->xferlockcommand); + + if(retval == -1) { + pm_printf(ALPM_LOG_WARNING, _("running XferCommand: fork failed!\n")); + } + + return retval; +} + int config_set_arch(const char *arch) { @@ -571,6 +593,9 @@ static int _parse_options(const char *key, char *value, } else if(strcmp(key, "XferCommand") == 0) { config->xfercommand = strdup(value); pm_printf(ALPM_LOG_DEBUG, "config: xfercommand: %s\n", value); + } else if(strcmp(key, "XferLockCommand") == 0) { + config->xferlockcommand = strdup(value); + pm_printf(ALPM_LOG_DEBUG, "config: xferlockcommand: %s\n", value); } else if(strcmp(key, "CleanMethod") == 0) { alpm_list_t *methods = NULL; setrepeatingoption(value, "CleanMethod", &methods); @@ -760,6 +785,12 @@ static int setup_libalpm(void) pm_printf(ALPM_LOG_WARNING, _("no '%s' configured\n"), "XferCommand"); } + if(config->xferlockcommand) { + alpm_option_set_fetch_lockcb(handle, download_lock_with_xfercommand); + } else if(!(alpm_capabilities() & ALPM_CAPABILITY_DOWNLOADER)) { + pm_printf(ALPM_LOG_WARNING, _("no '%s' configured\n"), "XferLockCommand"); + } + if(config->totaldownload) { alpm_option_set_totaldlcb(handle, cb_dl_total); } diff --git a/src/pacman/conf.h b/src/pacman/conf.h index 786eec6e..f7b238e3 100644 --- a/src/pacman/conf.h +++ b/src/pacman/conf.h @@ -126,6 +126,7 @@ typedef struct __config_t { alpm_list_t *noextract; alpm_list_t *overwrite_files; char *xfercommand; + char *xferlockcommand; /* our connection to libalpm */ alpm_handle_t *handle; diff --git a/src/pacman/pacman-conf.c b/src/pacman/pacman-conf.c index 1e6f55f9..15ff66d1 100644 --- a/src/pacman/pacman-conf.c +++ b/src/pacman/pacman-conf.c @@ -254,6 +254,7 @@ static void dump_config(void) show_str("Architecture", config->arch); show_str("XferCommand", config->xfercommand); + show_str("XferLockCommand", config->xferlockcommand); show_bool("UseSyslog", config->usesyslog); show_bool("Color", config->color); @@ -359,6 +360,8 @@ static int list_directives(void) show_str("Architecture", config->arch); } else if(strcasecmp(i->data, "XferCommand") == 0) { show_str("XferCommand", config->xfercommand); + } else if(strcasecmp(i->data, "XferLockCommand") == 0) { + show_str("XferLockCommand", config->xferlockcommand); } else if(strcasecmp(i->data, "UseSyslog") == 0) { show_bool("UseSyslog", config->usesyslog); -- 2.16.2