folkslinux
/
tdnf
mirror of https://github.com/vmware/tdnf.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #196 from oliverkurth/topic/okurth/download-options 

Implement download options retries, timeout, minrate and throttle
This commit is contained in:
Oliver Kurth 2020-12-09 13:57:57 -08:00 committed by GitHub
parent 9aa8834b78 b99f39aed6
commit fe2f372804
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 275 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
client/defines.h
View File

@ -91,6 +91,10 @@ typedef enum
#define TDNF_REPO_KEY_USERNAME "username"
#define TDNF_REPO_KEY_PASSWORD "password"
#define TDNF_REPO_KEY_METADATA_EXPIRE "metadata_expire"
#define TDNF_REPO_KEY_TIMEOUT "timeout"
#define TDNF_REPO_KEY_RETRIES "retries"
#define TDNF_REPO_KEY_MINRATE "minrate"
#define TDNF_REPO_KEY_THROTTLE "throttle"
#define TDNF_REPO_KEY_SSL_VERIFY "sslverify"
#define TDNF_REPO_KEY_SSL_CA_CERT "sslcacert"
#define TDNF_REPO_KEY_SSL_CLI_CERT "sslclientcert"

21
client/prototypes.h
View File

@ -180,6 +180,12 @@ TDNFRemoveSolvCache(
const char* pszRepoId
);
uint32_t
TDNFRepoApplyDownloadSettings(
PTDNF_REPO_DATA_INTERNAL pRepo,
CURL *pCurl
);
uint32_t
TDNFRepoApplyProxySettings(
PTDNF_CONF pConf,
@ -188,11 +194,22 @@ TDNFRepoApplyProxySettings(
uint32_t
TDNFRepoApplySSLSettings(
PTDNF pTdnf,
const char* pszRepo,
PTDNF_REPO_DATA_INTERNAL pRepo,
CURL *pCurl
);
uint32_t
TDNFFindRepoById(
PTDNF pTdnf,
const char* pszRepo,
PTDNF_REPO_DATA_INTERNAL* ppRepo
);
uint32_t
TDNFCurlErrorIsFatal(
CURLcode curlError
);
//remoterepo.c
uint32_t
TDNFCheckRepoMDFileHashFromMetalink(

62
client/remoterepo.c
View File

@ -725,8 +725,11 @@ TDNFDownloadFile(
CURL *pCurl = NULL;
FILE *fp = NULL;
char *pszUserPass = NULL;
char *pszFileTmp = NULL;
/* lStatus reads CURLINFO_RESPONSE_CODE. Must be long */
long lStatus = 0;
PTDNF_REPO_DATA_INTERNAL pRepo;
int i;
//If TDNF install is invoked with quiet argument,
//pszProgressData will be NULL
@ -763,10 +766,16 @@ TDNFDownloadFile(
BAIL_ON_TDNF_ERROR(dwError);
}
dwError = TDNFFindRepoById(pTdnf, pszRepo, &pRepo);
BAIL_ON_TDNF_ERROR(dwError);
dwError = TDNFRepoApplyProxySettings(pTdnf->pConf, pCurl);
BAIL_ON_TDNF_ERROR(dwError);
dwError = TDNFRepoApplySSLSettings(pTdnf, pszRepo, pCurl);
dwError = TDNFRepoApplyDownloadSettings(pRepo, pCurl);
BAIL_ON_TDNF_ERROR(dwError);
dwError = TDNFRepoApplySSLSettings(pRepo, pCurl);
BAIL_ON_TDNF_ERROR(dwError);
dwError = curl_easy_setopt(pCurl, CURLOPT_URL, pszFileUrl);
@ -785,19 +794,42 @@ TDNFDownloadFile(
}
}
fp = fopen(pszFile, "wb");
if(!fp)
dwError = TDNFAllocateStringPrintf(&pszFileTmp,
"%s.tmp",
pszFile);
BAIL_ON_TDNF_ERROR(dwError);
for(i = 0; i <= pRepo->nRetries; i++)
{
dwError = errno;
BAIL_ON_TDNF_SYSTEM_ERROR(dwError);
fp = fopen(pszFileTmp, "wb");
if(!fp)
{
dwError = errno;
BAIL_ON_TDNF_SYSTEM_ERROR(dwError);
}
dwError = curl_easy_setopt(pCurl, CURLOPT_WRITEDATA, fp);
BAIL_ON_TDNF_CURL_ERROR(dwError);
if (i > 0)
{
printf("\nretrying %d/%d\n", i, pRepo->nRetries);
}
dwError = curl_easy_perform(pCurl);
if (dwError == CURLE_OK)
{
fclose(fp);
fp = NULL;
break;
}
if (i == pRepo->nRetries || TDNFCurlErrorIsFatal(dwError))
{
BAIL_ON_TDNF_CURL_ERROR(dwError);
}
fclose(fp);
fp = NULL;
}
dwError = curl_easy_setopt(pCurl, CURLOPT_WRITEDATA, fp);
BAIL_ON_TDNF_CURL_ERROR(dwError);
dwError = curl_easy_perform(pCurl);
BAIL_ON_TDNF_CURL_ERROR(dwError);
dwError = curl_easy_getinfo(pCurl,
CURLINFO_RESPONSE_CODE,
&lStatus);
@ -814,6 +846,9 @@ TDNFDownloadFile(
}
else
{
dwError = rename(pszFileTmp, pszFile);
BAIL_ON_TDNF_ERROR(dwError);
if(is_metalink)
{
if(fp)
@ -827,6 +862,7 @@ TDNFDownloadFile(
}
cleanup:
TDNF_SAFE_FREE_MEMORY(pszUserPass);
TDNF_SAFE_FREE_MEMORY(pszFileTmp);
if(fp)
{
fclose(fp);
@ -843,9 +879,9 @@ error:
fclose(fp);
fp = NULL;
}
if(!IsNullOrEmptyString(pszFile))
if(!IsNullOrEmptyString(pszFileTmp))
{
unlink(pszFile);
unlink(pszFileTmp);
}
if(pCurl && TDNFIsCurlError(dwError))

28
client/repolist.c
View File

@ -334,6 +334,34 @@ TDNFLoadReposFromFile(
&pRepo->pszPass);
BAIL_ON_TDNF_ERROR(dwError);
dwError = TDNFReadKeyValueInt(
pSections,
TDNF_REPO_KEY_TIMEOUT,
0,
&pRepo->nTimeout);
BAIL_ON_TDNF_ERROR(dwError);
dwError = TDNFReadKeyValueInt(
pSections,
TDNF_REPO_KEY_RETRIES,
10,
&pRepo->nRetries);
BAIL_ON_TDNF_ERROR(dwError);
dwError = TDNFReadKeyValueInt(
pSections,
TDNF_REPO_KEY_MINRATE,
0,
&pRepo->nMinrate);
BAIL_ON_TDNF_ERROR(dwError);
dwError = TDNFReadKeyValueInt(
pSections,
TDNF_REPO_KEY_THROTTLE,
0,
&pRepo->nThrottle);
BAIL_ON_TDNF_ERROR(dwError);
dwError = TDNFReadKeyValueBoolean(
pSections,
TDNF_REPO_KEY_SSL_VERIFY,

228
client/repoutils.c
View File

@ -701,17 +701,149 @@ error:
goto cleanup;
}
uint32_t
TDNFRepoApplyDownloadSettings(
PTDNF_REPO_DATA_INTERNAL pRepo,
CURL *pCurl
)
{
CURLcode curlError = CURLE_OK;
uint32_t dwError = 0;
if(!pRepo || !pCurl)
{
dwError = ERROR_TDNF_INVALID_PARAMETER;
BAIL_ON_TDNF_ERROR(dwError);
}
if((curlError = curl_easy_setopt(
pCurl,
CURLOPT_TIMEOUT,
pRepo->nTimeout)) != CURLE_OK)
{
dwError = ERROR_TDNF_CURL_BASE + curlError;
BAIL_ON_TDNF_ERROR(dwError);
}
if((curlError = curl_easy_setopt(
pCurl,
CURLOPT_LOW_SPEED_TIME,
pRepo->nTimeout)) != CURLE_OK)
{
dwError = ERROR_TDNF_CURL_BASE + curlError;
BAIL_ON_TDNF_ERROR(dwError);
}
if((curlError = curl_easy_setopt(
pCurl,
CURLOPT_LOW_SPEED_LIMIT,
pRepo->nMinrate)) != CURLE_OK)
{
dwError = ERROR_TDNF_CURL_BASE + curlError;
BAIL_ON_TDNF_ERROR(dwError);
}
if((curlError = curl_easy_setopt(
pCurl,
CURLOPT_MAX_RECV_SPEED_LARGE,
pRepo->nThrottle)) != CURLE_OK)
{
dwError = ERROR_TDNF_CURL_BASE + curlError;
BAIL_ON_TDNF_ERROR(dwError);
}
cleanup:
return dwError;
error:
goto cleanup;
}
uint32_t
TDNFRepoApplySSLSettings(
PTDNF_REPO_DATA_INTERNAL pRepo,
CURL *pCurl
)
{
uint32_t dwError = 0;
if(!pRepo || !pCurl)
{
dwError = ERROR_TDNF_INVALID_PARAMETER;
BAIL_ON_TDNF_ERROR(dwError);
}
if(curl_easy_setopt(
pCurl,
CURLOPT_SSL_VERIFYPEER,
((pRepo->nSSLVerify) ? 1 : 0)) != CURLE_OK)
{
dwError = ERROR_TDNF_SET_SSL_SETTINGS;
BAIL_ON_TDNF_ERROR(dwError);
}
if(curl_easy_setopt(
pCurl,
CURLOPT_SSL_VERIFYHOST,
((pRepo->nSSLVerify) ? 2 : 0)) != CURLE_OK)
{
dwError = ERROR_TDNF_SET_SSL_SETTINGS;
BAIL_ON_TDNF_ERROR(dwError);
}
if(!IsNullOrEmptyString(pRepo->pszSSLCaCert))
{
if(curl_easy_setopt(
pCurl,
CURLOPT_CAINFO,
pRepo->pszSSLCaCert) != CURLE_OK)
{
dwError = ERROR_TDNF_SET_SSL_SETTINGS;
BAIL_ON_TDNF_ERROR(dwError);
}
}
if(!IsNullOrEmptyString(pRepo->pszSSLClientCert))
{
if(curl_easy_setopt(
pCurl,
CURLOPT_SSLCERT,
pRepo->pszSSLClientCert) != CURLE_OK)
{
dwError = ERROR_TDNF_SET_SSL_SETTINGS;
BAIL_ON_TDNF_ERROR(dwError);
}
}
if(!IsNullOrEmptyString(pRepo->pszSSLClientKey))
{
if(curl_easy_setopt(
pCurl,
CURLOPT_SSLKEY,
pRepo->pszSSLClientKey) != CURLE_OK)
{
dwError = ERROR_TDNF_SET_SSL_SETTINGS;
BAIL_ON_TDNF_ERROR(dwError);
}
}
cleanup:
return dwError;
error:
goto cleanup;
}
uint32_t
TDNFFindRepoById(
PTDNF pTdnf,
const char* pszRepo,
CURL *pCurl
PTDNF_REPO_DATA_INTERNAL* ppRepo
)
{
uint32_t dwError = 0;
PTDNF_REPO_DATA_INTERNAL pRepos = NULL;
if(!pTdnf || IsNullOrEmptyString(pszRepo) || !pCurl)
if(!pTdnf || IsNullOrEmptyString(pszRepo))
{
dwError = ERROR_TDNF_INVALID_PARAMETER;
BAIL_ON_TDNF_ERROR(dwError);
@ -737,64 +869,46 @@ TDNFRepoApplySSLSettings(
dwError = ERROR_TDNF_REPO_NOT_FOUND;
BAIL_ON_TDNF_ERROR(dwError);
}
if(curl_easy_setopt(
pCurl,
CURLOPT_SSL_VERIFYPEER,
((pRepos->nSSLVerify) ? 1 : 0)) != CURLE_OK)
{
dwError = ERROR_TDNF_SET_SSL_SETTINGS;
BAIL_ON_TDNF_ERROR(dwError);
}
if(curl_easy_setopt(
pCurl,
CURLOPT_SSL_VERIFYHOST,
((pRepos->nSSLVerify) ? 2 : 0)) != CURLE_OK)
{
dwError = ERROR_TDNF_SET_SSL_SETTINGS;
BAIL_ON_TDNF_ERROR(dwError);
}
if(!IsNullOrEmptyString(pRepos->pszSSLCaCert))
{
if(curl_easy_setopt(
pCurl,
CURLOPT_CAINFO,
pRepos->pszSSLCaCert) != CURLE_OK)
{
dwError = ERROR_TDNF_SET_SSL_SETTINGS;
BAIL_ON_TDNF_ERROR(dwError);
}
}
if(!IsNullOrEmptyString(pRepos->pszSSLClientCert))
{
if(curl_easy_setopt(
pCurl,
CURLOPT_SSLCERT,
pRepos->pszSSLClientCert) != CURLE_OK)
{
dwError = ERROR_TDNF_SET_SSL_SETTINGS;
BAIL_ON_TDNF_ERROR(dwError);
}
}
if(!IsNullOrEmptyString(pRepos->pszSSLClientKey))
{
if(curl_easy_setopt(
pCurl,
CURLOPT_SSLKEY,
pRepos->pszSSLClientKey) != CURLE_OK)
{
dwError = ERROR_TDNF_SET_SSL_SETTINGS;
BAIL_ON_TDNF_ERROR(dwError);
}
}
*ppRepo = pRepos;
cleanup:
return dwError;
error:
goto cleanup;
}
uint32_t
TDNFCurlErrorIsFatal(
CURLcode curlError
)
{
uint32_t dwError = 0;
switch(curlError) {
case CURLE_UNSUPPORTED_PROTOCOL:
case CURLE_FAILED_INIT:
case CURLE_URL_MALFORMAT:
case CURLE_FILE_COULDNT_READ_FILE:
case CURLE_FUNCTION_NOT_FOUND:
case CURLE_UNKNOWN_OPTION:
case CURLE_SSL_ENGINE_NOTFOUND:
case CURLE_RECURSIVE_API_CALL:
case CURLE_ABORTED_BY_CALLBACK:
case CURLE_BAD_FUNCTION_ARGUMENT:
case CURLE_CONV_REQD:
case CURLE_COULDNT_RESOLVE_PROXY:
case CURLE_FILESIZE_EXCEEDED:
case CURLE_INTERFACE_FAILED:
case CURLE_NOT_BUILT_IN:
case CURLE_OUT_OF_MEMORY:
case CURLE_SSL_CACERT_BADFILE:
case CURLE_SSL_CRL_BADFILE:
case CURLE_WRITE_ERROR:
dwError = curlError;
break;
default:
break;
}
return dwError;
}

4
client/structs.h
View File

@ -48,6 +48,10 @@ typedef struct _TDNF_REPO_DATA_INTERNAL_
char** ppszUrlGPGKeys;
char* pszUser;
char* pszPass;
int nTimeout;
int nMinrate;
int nThrottle;
int nRetries;
struct _TDNF_REPO_DATA_INTERNAL_* pNext;
}TDNF_REPO_DATA_INTERNAL, *PTDNF_REPO_DATA_INTERNAL;

4
include/tdnftypes.h
View File

@ -268,15 +268,11 @@ typedef struct _TDNF_REPO_DATA
int nEnabled;
int nSkipIfUnavailable;
int nGPGCheck;
int nSSLVerify;
long lMetadataExpire;
char* pszId;
char* pszName;
char* pszBaseUrl;
char* pszMetaLink;
char* pszSSLCaCert;
char* pszSSLClientCert;
char* pszSSLClientKey;
char** ppszUrlGPGKeys;
struct _TDNF_REPO_DATA* pNext;