libzypp  17.32.5
MediaCurl.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <iostream>
14 #include <chrono>
15 #include <list>
16 
17 #include <zypp/base/Logger.h>
18 #include <zypp/ExternalProgram.h>
19 #include <zypp/base/String.h>
20 #include <zypp/base/Gettext.h>
21 #include <utility>
22 #include <zypp-core/parser/Sysconfig>
23 #include <zypp/base/Gettext.h>
24 
25 #include <zypp/media/MediaCurl.h>
28 #include <zypp-curl/ProxyInfo>
29 #include <zypp-curl/auth/CurlAuthData>
30 #include <zypp-media/auth/CredentialManager>
31 #include <zypp-curl/CurlConfig>
33 #include <zypp/Target.h>
34 #include <zypp/ZYppFactory.h>
35 #include <zypp/ZConfig.h>
36 #include <zypp/zypp_detail/ZYppImpl.h> // for zypp_poll
37 
38 #include <cstdlib>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <sys/mount.h>
42 #include <dirent.h>
43 #include <unistd.h>
44 #include <glib.h>
45 
46 using std::endl;
47 
48 namespace internal {
49  using namespace zypp;
53  struct OptionalDownloadProgressReport : public callback::ReceiveReport<media::DownloadProgressReport>
54  {
55  using TimePoint = std::chrono::steady_clock::time_point;
56 
57  OptionalDownloadProgressReport( bool isOptional=false )
58  : _oldRec { Distributor::instance().getReceiver() }
59  , _isOptional { isOptional }
60  { connect(); }
61 
66 
68  if (_oldRec)
69  Distributor::instance().setReceiver(*_oldRec);
70  else
71  Distributor::instance().noReceiver();
72  }
73 
74  void reportbegin() override
75  { if ( _oldRec ) _oldRec->reportbegin(); }
76 
77  void reportend() override
78  { if ( _oldRec ) _oldRec->reportend(); }
79 
80  void report( const UserData & userData_r = UserData() ) override
81  { if ( _oldRec ) _oldRec->report( userData_r ); }
82 
83 
84  void start( const Url & file_r, Pathname localfile_r ) override
85  {
86  if ( not _oldRec ) return;
87  if ( _isOptional ) {
88  // delay start until first data are received.
89  _startFile = file_r;
90  _startLocalfile = std::move(localfile_r);
91  return;
92  }
93  _oldRec->start( file_r, localfile_r );
94  }
95 
96  bool progress( int value_r, const Url & file_r, double dbps_avg_r = -1, double dbps_current_r = -1 ) override
97  {
98  if ( not _oldRec ) return true;
99  if ( notStarted() ) {
100  if ( not ( value_r || dbps_avg_r || dbps_current_r ) )
101  return true;
102  sendStart();
103  }
104 
105  //static constexpr std::chrono::milliseconds minfequency { 1000 }; only needed if we'd avoid sending reports without change
106  static constexpr std::chrono::milliseconds maxfequency { 100 };
107  TimePoint now { TimePoint::clock::now() };
108  TimePoint::duration elapsed { now - _lastProgressSent };
109  if ( elapsed < maxfequency )
110  return true; // continue
111  _lastProgressSent = now;
112  return _oldRec->progress( value_r, file_r, dbps_avg_r, dbps_current_r );
113  }
114 
115  Action problem( const Url & file_r, Error error_r, const std::string & description_r ) override
116  {
117  if ( not _oldRec || notStarted() ) return ABORT;
118  return _oldRec->problem( file_r, error_r, description_r );
119  }
120 
121  void finish( const Url & file_r, Error error_r, const std::string & reason_r ) override
122  {
123  if ( not _oldRec || notStarted() ) return;
124  _oldRec->finish( file_r, error_r, reason_r );
125  }
126 
127  private:
128  // _isOptional also indicates the delayed start
129  bool notStarted() const
130  { return _isOptional; }
131 
132  void sendStart()
133  {
134  if ( _isOptional ) {
135  // we know _oldRec is valid...
136  _oldRec->start( _startFile, std::move(_startLocalfile) );
137  _isOptional = false;
138  }
139  }
140 
141  private:
142  Receiver *const _oldRec;
145  Pathname _startLocalfile;
147  };
148 
150  {
151  ProgressData( CURL *curl, time_t timeout = 0, zypp::Url url = zypp::Url(),
152  zypp::ByteCount expectedFileSize_r = 0,
154 
155  void updateStats( curl_off_t dltotal = 0.0, curl_off_t dlnow = 0.0 );
156 
157  int reportProgress() const;
158 
159  CURL * curl()
160  { return _curl; }
161 
162  bool timeoutReached() const
163  { return _timeoutReached; }
164 
165  bool fileSizeExceeded() const
166  { return _fileSizeExceeded; }
167 
169  { return _expectedFileSize; }
170 
171  void expectedFileSize( ByteCount newval_r )
172  { _expectedFileSize = newval_r; }
173 
174  private:
175  CURL * _curl;
177  time_t _timeout;
182 
183  time_t _timeStart = 0;
184  time_t _timeLast = 0;
185  time_t _timeRcv = 0;
186  time_t _timeNow = 0;
187 
188  curl_off_t _dnlTotal = 0.0;
189  curl_off_t _dnlLast = 0.0;
190  curl_off_t _dnlNow = 0.0;
191 
192  int _dnlPercent= 0;
193 
194  double _drateTotal= 0.0;
195  double _drateLast = 0.0;
196  };
197 
198 
199 
201  : _curl( curl )
202  , _url(std::move( url ))
203  , _timeout( timeout )
204  , _timeoutReached( false )
205  , _fileSizeExceeded ( false )
206  , _expectedFileSize( expectedFileSize_r )
207  , report( _report )
208  {}
209 
210  void ProgressData::updateStats( curl_off_t dltotal, curl_off_t dlnow )
211  {
212  time_t now = _timeNow = time(0);
213 
214  // If called without args (0.0), recompute based on the last values seen
215  if ( dltotal && dltotal != _dnlTotal )
216  _dnlTotal = dltotal;
217 
218  if ( dlnow && dlnow != _dnlNow )
219  {
220  _timeRcv = now;
221  _dnlNow = dlnow;
222  }
223 
224  // init or reset if time jumps back
225  if ( !_timeStart || _timeStart > now )
226  _timeStart = _timeLast = _timeRcv = now;
227 
228  // timeout condition
229  if ( _timeout )
230  _timeoutReached = ( (now - _timeRcv) > _timeout );
231 
232  // check if the downloaded data is already bigger than what we expected
233  _fileSizeExceeded = _expectedFileSize > 0 && _expectedFileSize < static_cast<ByteCount::SizeType>(_dnlNow);
234 
235  // percentage:
236  if ( _dnlTotal )
237  _dnlPercent = int( _dnlNow * 100 / _dnlTotal );
238 
239  // download rates:
240  _drateTotal = double(_dnlNow) / std::max( int(now - _timeStart), 1 );
241 
242  if ( _timeLast < now )
243  {
244  _drateLast = double(_dnlNow - _dnlLast) / int(now - _timeLast);
245  // start new period
246  _timeLast = now;
247  _dnlLast = _dnlNow;
248  }
249  else if ( _timeStart == _timeLast )
251  }
252 
254  {
255  if ( _fileSizeExceeded )
256  return 1;
257  if ( _timeoutReached )
258  return 1; // no-data timeout
259  if ( report && !(*report)->progress( _dnlPercent, _url, _drateTotal, _drateLast ) )
260  return 1; // user requested abort
261  return 0;
262  }
263 
264  const char * anonymousIdHeader()
265  {
266  // we need to add the release and identifier to the
267  // agent string.
268  // The target could be not initialized, and then this information
269  // is guessed.
270  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
271  static const std::string _value( str::trim( str::form(
272  "X-ZYpp-AnonymousId: %s",
273  Target::anonymousUniqueId( Pathname()/*guess root*/ ).c_str()
274  )));
275  return _value.c_str();
276  }
277 
279  {
280  // we need to add the release and identifier to the
281  // agent string.
282  // The target could be not initialized, and then this information
283  // is guessed.
284  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
285  static const std::string _value( str::trim( str::form(
286  "X-ZYpp-DistributionFlavor: %s",
287  Target::distributionFlavor( Pathname()/*guess root*/ ).c_str()
288  )));
289  return _value.c_str();
290  }
291 
292  const char * agentString()
293  {
294  // we need to add the release and identifier to the
295  // agent string.
296  // The target could be not initialized, and then this information
297  // is guessed.
298  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
299  static const std::string _value( str::trim( str::form(
300  "ZYpp " LIBZYPP_VERSION_STRING " (curl %s) %s"
301  , curl_version_info(CURLVERSION_NOW)->version
302  , Target::targetDistribution( Pathname()/*guess root*/ ).c_str()
303  )));
304  return _value.c_str();
305  }
306 
311  {
312  public:
314  const std::string & err_r,
315  const std::string & msg_r )
316  : media::MediaCurlException( url_r, err_r, msg_r )
317  {}
318  //~MediaCurlExceptionMayRetryInternaly() noexcept {}
319  };
320 
321 }
322 
323 
324 using namespace internal;
325 using namespace zypp::base;
326 
327 namespace zypp {
328 
329  namespace media {
330 
331 Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
332 
333 // we use this define to unbloat code as this C setting option
334 // and catching exception is done frequently.
336 #define SET_OPTION(opt,val) do { \
337  ret = curl_easy_setopt ( _curl, opt, val ); \
338  if ( ret != 0) { \
339  ZYPP_THROW(MediaCurlSetOptException(_url, _curlError)); \
340  } \
341  } while ( false )
342 
343 #define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
344 #define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
345 #define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
346 
347 MediaCurl::MediaCurl( const Url & url_r,
348  const Pathname & attach_point_hint_r )
349  : MediaNetworkCommonHandler( url_r, attach_point_hint_r,
350  "/", // urlpath at attachpoint
351  true ), // does_download
352  _curl( NULL ),
353  _customHeaders(0L)
354 {
355  _curlError[0] = '\0';
356 
357  MIL << "MediaCurl::MediaCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
358 
360 
361  if( !attachPoint().empty())
362  {
363  PathInfo ainfo(attachPoint());
364  Pathname apath(attachPoint() + "XXXXXX");
365  char *atemp = ::strdup( apath.asString().c_str());
366  char *atest = NULL;
367  if( !ainfo.isDir() || !ainfo.userMayRWX() ||
368  atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
369  {
370  WAR << "attach point " << ainfo.path()
371  << " is not useable for " << url_r.getScheme() << endl;
372  setAttachPoint("", true);
373  }
374  else if( atest != NULL)
375  ::rmdir(atest);
376 
377  if( atemp != NULL)
378  ::free(atemp);
379  }
380 }
381 
383 {
385 }
386 
387 void MediaCurl::setCookieFile( const Pathname &fileName )
388 {
389  _cookieFile = fileName;
390 }
391 
393 
394 void MediaCurl::checkProtocol(const Url &url) const
395 {
396  curl_version_info_data *curl_info = NULL;
397  curl_info = curl_version_info(CURLVERSION_NOW);
398  // curl_info does not need any free (is static)
399  if (curl_info->protocols)
400  {
401  const char * const *proto = nullptr;
402  std::string scheme( url.getScheme());
403  bool found = false;
404  for(proto=curl_info->protocols; !found && *proto; ++proto)
405  {
406  if( scheme == std::string((const char *)*proto))
407  found = true;
408  }
409  if( !found)
410  {
411  std::string msg("Unsupported protocol '");
412  msg += scheme;
413  msg += "'";
415  }
416  }
417 }
418 
420 {
422 
423  curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
424  curl_easy_setopt(_curl, CURLOPT_HEADERDATA, &_lastRedirect);
425  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_ERRORBUFFER, _curlError );
426  if ( ret != 0 ) {
427  ZYPP_THROW(MediaCurlSetOptException(_url, "Error setting error buffer"));
428  }
429 
430  SET_OPTION(CURLOPT_FAILONERROR, 1L);
431  SET_OPTION(CURLOPT_NOSIGNAL, 1L);
432 
433  // create non persistant settings
434  // so that we don't add headers twice
435  TransferSettings vol_settings(_settings);
436 
437  // add custom headers for download.opensuse.org (bsc#955801)
438  if ( _url.getHost() == "download.opensuse.org" )
439  {
440  vol_settings.addHeader(anonymousIdHeader());
441  vol_settings.addHeader(distributionFlavorHeader());
442  }
443  vol_settings.addHeader("Pragma:");
444 
446 
447  // fill some settings from url query parameters
448  try
449  {
451  }
452  catch ( const MediaException &e )
453  {
454  disconnectFrom();
455  ZYPP_RETHROW(e);
456  }
457  // if the proxy was not set (or explicitly unset) by url, then look...
458  if ( _settings.proxy().empty() )
459  {
460  // ...at the system proxy settings
462  }
463 
465  switch ( env::ZYPP_MEDIA_CURL_IPRESOLVE() )
466  {
467  case 4: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); break;
468  case 6: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); break;
469  }
470 
474  SET_OPTION(CURLOPT_CONNECTTIMEOUT, _settings.connectTimeout());
475  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
476  // just in case curl does not trigger its progress callback frequently
477  // enough.
478  if ( _settings.timeout() )
479  {
480  SET_OPTION(CURLOPT_TIMEOUT, 3600L);
481  }
482 
483  // follow any Location: header that the server sends as part of
484  // an HTTP header (#113275)
485  SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
486  // 3 redirects seem to be too few in some cases (bnc #465532)
487  SET_OPTION(CURLOPT_MAXREDIRS, 6L);
488 
489  if ( _url.getScheme() == "https" )
490  {
491  if ( :: internal::setCurlRedirProtocols ( _curl ) != CURLE_OK ) {
493  }
494 
497  {
499  }
500 
502  {
503  SET_OPTION(CURLOPT_SSLCERT, _settings.clientCertificatePath().c_str());
504  }
505  if( ! _settings.clientKeyPath().empty() )
506  {
507  SET_OPTION(CURLOPT_SSLKEY, _settings.clientKeyPath().c_str());
508  }
509 
510 #ifdef CURLSSLOPT_ALLOW_BEAST
511  // see bnc#779177
512  ret = curl_easy_setopt( _curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
513  if ( ret != 0 ) {
514  disconnectFrom();
516  }
517 #endif
518  SET_OPTION(CURLOPT_SSL_VERIFYPEER, _settings.verifyPeerEnabled() ? 1L : 0L);
519  SET_OPTION(CURLOPT_SSL_VERIFYHOST, _settings.verifyHostEnabled() ? 2L : 0L);
520  // bnc#903405 - POODLE: libzypp should only talk TLS
521  SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
522  }
523 
524  SET_OPTION(CURLOPT_USERAGENT, _settings.userAgentString().c_str() );
525 
526  /* Fixes bsc#1174011 "auth=basic ignored in some cases"
527  * We should proactively add the password to the request if basic auth is configured
528  * and a password is available in the credentials but not in the URL.
529  *
530  * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
531  * and ask the server first about the auth method
532  */
533  if ( _settings.authType() == "basic"
534  && _settings.username().size()
535  && !_settings.password().size() ) {
536 
537  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
538  const auto cred = cm.getCred( _url );
539  if ( cred && cred->valid() ) {
540  if ( !_settings.username().size() )
541  _settings.setUsername(cred->username());
542  _settings.setPassword(cred->password());
543  }
544  }
545 
546  /*---------------------------------------------------------------*
547  CURLOPT_USERPWD: [user name]:[password]
548 
549  Url::username/password -> CURLOPT_USERPWD
550  If not provided, anonymous FTP identification
551  *---------------------------------------------------------------*/
552 
553  if ( _settings.userPassword().size() )
554  {
555  SET_OPTION(CURLOPT_USERPWD, _settings.userPassword().c_str());
556  std::string use_auth = _settings.authType();
557  if (use_auth.empty())
558  use_auth = "digest,basic"; // our default
559  long auth = CurlAuthData::auth_type_str2long(use_auth);
560  if( auth != CURLAUTH_NONE)
561  {
562  DBG << "Enabling HTTP authentication methods: " << use_auth
563  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
564  SET_OPTION(CURLOPT_HTTPAUTH, auth);
565  }
566  }
567 
568  if ( _settings.proxyEnabled() && ! _settings.proxy().empty() )
569  {
570  DBG << "Proxy: '" << _settings.proxy() << "'" << endl;
571  SET_OPTION(CURLOPT_PROXY, _settings.proxy().c_str());
572  SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
573  /*---------------------------------------------------------------*
574  * CURLOPT_PROXYUSERPWD: [user name]:[password]
575  *
576  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
577  * If not provided, $HOME/.curlrc is evaluated
578  *---------------------------------------------------------------*/
579 
580  std::string proxyuserpwd = _settings.proxyUserPassword();
581 
582  if ( proxyuserpwd.empty() )
583  {
584  CurlConfig curlconf;
585  CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
586  if ( curlconf.proxyuserpwd.empty() )
587  DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
588  else
589  {
590  proxyuserpwd = curlconf.proxyuserpwd;
591  DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
592  }
593  }
594  else
595  {
596  DBG << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << endl;
597  }
598 
599  if ( ! proxyuserpwd.empty() )
600  {
601  SET_OPTION(CURLOPT_PROXYUSERPWD, curlUnEscape( proxyuserpwd ).c_str());
602  }
603  }
604 #if CURLVERSION_AT_LEAST(7,19,4)
605  else if ( _settings.proxy() == EXPLICITLY_NO_PROXY )
606  {
607  // Explicitly disabled in URL (see fillSettingsFromUrl()).
608  // This should also prevent libcurl from looking into the environment.
609  DBG << "Proxy: explicitly NOPROXY" << endl;
610  SET_OPTION(CURLOPT_NOPROXY, "*");
611  }
612 #endif
613  else
614  {
615  DBG << "Proxy: not explicitly set" << endl;
616  DBG << "Proxy: libcurl may look into the environment" << endl;
617  }
618 
620  if ( _settings.minDownloadSpeed() != 0 )
621  {
622  SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, _settings.minDownloadSpeed());
623  // default to 10 seconds at low speed
624  SET_OPTION(CURLOPT_LOW_SPEED_TIME, 60L);
625  }
626 
627 #if CURLVERSION_AT_LEAST(7,15,5)
628  if ( _settings.maxDownloadSpeed() != 0 )
629  SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, _settings.maxDownloadSpeed());
630 #endif
631 
632  /*---------------------------------------------------------------*
633  *---------------------------------------------------------------*/
634 
636  if ( ::geteuid() == 0 || PathInfo(_currentCookieFile).owner() == ::geteuid() )
638 
639  const auto &cookieFileParam = _url.getQueryParam( "cookies" );
640  if ( !cookieFileParam.empty() && str::strToBool( cookieFileParam, true ) )
641  SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
642  else
643  MIL << "No cookies requested" << endl;
644  SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
645  SET_OPTION(CURLOPT_XFERINFOFUNCTION, &progressCallback );
646  SET_OPTION(CURLOPT_NOPROGRESS, 0L);
647 
648 #if CURLVERSION_AT_LEAST(7,18,0)
649  // bnc #306272
650  SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
651 #endif
652  // Append settings custom headers to curl.
653  // TransferSettings assert strings are trimmed (HTTP/2 RFC 9113)
654  for ( const auto &header : vol_settings.headers() ) {
655  _customHeaders = curl_slist_append(_customHeaders, header.c_str());
656  if ( !_customHeaders )
658  }
659  SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
660 }
661 
663 
664 
665 void MediaCurl::attachTo (bool next)
666 {
667  if ( next )
669 
670  if ( !_url.isValid() )
672 
675  {
677  }
678 
679  disconnectFrom(); // clean _curl if needed
680  _curl = curl_easy_init();
681  if ( !_curl ) {
683  }
684  try
685  {
686  setupEasy();
687  }
688  catch (Exception & ex)
689  {
690  disconnectFrom();
691  ZYPP_RETHROW(ex);
692  }
693 
694  // FIXME: need a derived class to propelly compare url's
696  setMediaSource(media);
697 }
698 
699 bool
701 {
702  return MediaHandler::checkAttachPoint( apoint, true, true);
703 }
704 
706 
708 {
709  if ( _customHeaders )
710  {
711  curl_slist_free_all(_customHeaders);
712  _customHeaders = 0L;
713  }
714 
715  if ( _curl )
716  {
717  // bsc#1201092: Strange but within global_dtors we may exceptions here.
718  try { curl_easy_cleanup( _curl ); }
719  catch (...) { ; }
720  _curl = NULL;
721  }
722 }
723 
725 
726 void MediaCurl::releaseFrom( const std::string & ejectDev )
727 {
728  disconnect();
729 }
730 
732 
733 void MediaCurl::getFile( const OnMediaLocation &file ) const
734 {
735  // Use absolute file name to prevent access of files outside of the
736  // hierarchy below the attach point.
737  getFileCopy( file, localPath(file.filename()).absolutename() );
738 }
739 
741 
742 void MediaCurl::getFileCopy( const OnMediaLocation & srcFile , const Pathname & target ) const
743 {
744 
745  const auto &filename = srcFile.filename();
746 
747  // Optional files will send no report until data are actually received (we know it exists).
748  OptionalDownloadProgressReport reportfilter( srcFile.optional() );
750 
751  Url fileurl(getFileUrl(filename));
752 
753  bool firstAuth = true; // bsc#1210870: authenticate must not return stored credentials more than once.
754  unsigned internalTry = 0;
755  static constexpr unsigned maxInternalTry = 3;
756 
757  do
758  {
759  try
760  {
761  doGetFileCopy( srcFile, target, report );
762  break; // success!
763  }
764  // retry with proper authentication data
765  catch (MediaUnauthorizedException & ex_r)
766  {
767  if ( authenticate(ex_r.hint(), firstAuth) ) {
768  firstAuth = false; // must not return stored credentials again
769  continue; // retry
770  }
771 
772  report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
773  ZYPP_RETHROW(ex_r);
774  }
775  // unexpected exception
776  catch (MediaException & excpt_r)
777  {
778  if ( typeid(excpt_r) == typeid( MediaCurlExceptionMayRetryInternaly ) ) {
779  ++internalTry;
780  if ( internalTry < maxInternalTry ) {
781  // just report (NO_ERROR); no interactive request to the user
782  report->problem(fileurl, media::DownloadProgressReport::NO_ERROR, excpt_r.asUserHistory()+_("Will try again..."));
783  continue; // retry
784  }
785  excpt_r.addHistory( str::Format(_("Giving up after %1% attempts.")) % maxInternalTry );
786  }
787 
789  if( typeid(excpt_r) == typeid( media::MediaFileNotFoundException ) ||
790  typeid(excpt_r) == typeid( media::MediaNotAFileException ) )
791  {
793  }
794  report->finish(fileurl, reason, excpt_r.asUserHistory());
795  ZYPP_RETHROW(excpt_r);
796  }
797  }
798  while ( true );
799  report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
800 }
801 
803 
804 bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
805 {
806  bool retry = false;
807 
808  do
809  {
810  try
811  {
812  return doGetDoesFileExist( filename );
813  }
814  // authentication problem, retry with proper authentication data
815  catch (MediaUnauthorizedException & ex_r)
816  {
817  if(authenticate(ex_r.hint(), !retry))
818  retry = true;
819  else
820  ZYPP_RETHROW(ex_r);
821  }
822  // unexpected exception
823  catch (MediaException & excpt_r)
824  {
825  ZYPP_RETHROW(excpt_r);
826  }
827  }
828  while (retry);
829 
830  return false;
831 }
832 
834 
836  CURLcode code,
837  bool timeout_reached) const
838 {
839  if ( code != 0 )
840  {
841  Url url;
842  if (filename.empty())
843  url = _url;
844  else
845  url = getFileUrl(filename);
846 
847  std::string err;
848  {
849  switch ( code )
850  {
851  case CURLE_UNSUPPORTED_PROTOCOL:
852  err = " Unsupported protocol";
853  if ( !_lastRedirect.empty() )
854  {
855  err += " or redirect (";
856  err += _lastRedirect;
857  err += ")";
858  }
859  break;
860  case CURLE_URL_MALFORMAT:
861  case CURLE_URL_MALFORMAT_USER:
862  err = " Bad URL";
863  break;
864  case CURLE_LOGIN_DENIED:
865  ZYPP_THROW(
866  MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
867  break;
868  case CURLE_HTTP_RETURNED_ERROR:
869  {
870  long httpReturnCode = 0;
871  CURLcode infoRet = curl_easy_getinfo( _curl,
872  CURLINFO_RESPONSE_CODE,
873  &httpReturnCode );
874  if ( infoRet == CURLE_OK )
875  {
876  std::string msg = "HTTP response: " + str::numstring( httpReturnCode );
877  switch ( httpReturnCode )
878  {
879  case 401:
880  {
881  std::string auth_hint = getAuthHint();
882 
883  DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
884  DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
885 
887  url, "Login failed.", _curlError, auth_hint
888  ));
889  }
890 
891  case 502: // bad gateway (bnc #1070851)
892  case 503: // service temporarily unavailable (bnc #462545)
894  case 504: // gateway timeout
896  case 403:
897  {
898  std::string msg403;
899  if ( url.getHost().find(".suse.com") != std::string::npos )
900  msg403 = _("Visit the SUSE Customer Center to check whether your registration is valid and has not expired.");
901  else if (url.asString().find("novell.com") != std::string::npos)
902  msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
904  }
905  case 404:
906  case 410:
908  }
909 
910  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
912  }
913  else
914  {
915  std::string msg = "Unable to retrieve HTTP response:";
916  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
918  }
919  }
920  break;
921  case CURLE_FTP_COULDNT_RETR_FILE:
922 #if CURLVERSION_AT_LEAST(7,16,0)
923  case CURLE_REMOTE_FILE_NOT_FOUND:
924 #endif
925  case CURLE_FTP_ACCESS_DENIED:
926  case CURLE_TFTP_NOTFOUND:
927  err = "File not found";
929  break;
930  case CURLE_BAD_PASSWORD_ENTERED:
931  case CURLE_FTP_USER_PASSWORD_INCORRECT:
932  err = "Login failed";
933  break;
934  case CURLE_COULDNT_RESOLVE_PROXY:
935  case CURLE_COULDNT_RESOLVE_HOST:
936  case CURLE_COULDNT_CONNECT:
937  case CURLE_FTP_CANT_GET_HOST:
938  err = "Connection failed";
939  break;
940  case CURLE_WRITE_ERROR:
941  err = "Write error";
942  break;
943  case CURLE_PARTIAL_FILE:
944  case CURLE_OPERATION_TIMEDOUT:
945  timeout_reached = true; // fall though to TimeoutException
946  // fall though...
947  case CURLE_ABORTED_BY_CALLBACK:
948  if( timeout_reached )
949  {
950  err = "Timeout reached";
952  }
953  else
954  {
955  err = "User abort";
956  }
957  break;
958 
959  default:
960  err = "Curl error " + str::numstring( code );
961  break;
962  }
963 
964  // uhm, no 0 code but unknown curl exception
966  }
967  }
968  else
969  {
970  // actually the code is 0, nothing happened
971  }
972 }
973 
975 
976 bool MediaCurl::doGetDoesFileExist( const Pathname & filename ) const
977 {
978  DBG << filename.asString() << endl;
979 
980  if(!_url.isValid())
982 
983  if(_url.getHost().empty())
985 
986  Url url(getFileUrl(filename));
987 
988  DBG << "URL: " << url.asString() << endl;
989  // Use URL without options and without username and passwd
990  // (some proxies dislike them in the URL).
991  // Curl seems to need the just scheme, hostname and a path;
992  // the rest was already passed as curl options (in attachTo).
993  Url curlUrl( clearQueryString(url) );
994 
995  //
996  // See also Bug #154197 and ftp url definition in RFC 1738:
997  // The url "ftp://user@host/foo/bar/file" contains a path,
998  // that is relative to the user's home.
999  // The url "ftp://user@host//foo/bar/file" (or also with
1000  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1001  // contains an absolute path.
1002  //
1003  _lastRedirect.clear();
1004  std::string urlBuffer( curlUrl.asString());
1005  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1006  urlBuffer.c_str() );
1007  if ( ret != 0 ) {
1009  }
1010 
1011  // If no head requests allowed (?head_requests=no):
1012  // Instead of returning no data with NOBODY, we return
1013  // little data, that works with broken servers, and
1014  // works for ftp as well, because retrieving only headers
1015  // ftp will return always OK code ?
1016  // See http://curl.haxx.se/docs/knownbugs.html #58
1018  struct TempSetHeadRequest
1019  {
1020  TempSetHeadRequest(CURL *curl_r, bool doHttpHeadRequest_r)
1021  : _curl{curl_r}, _doHttpHeadRequest{doHttpHeadRequest_r} {
1022  if ( _doHttpHeadRequest ) {
1023  curl_easy_setopt( _curl, CURLOPT_NOBODY, 1L );
1024  } else {
1025  curl_easy_setopt( _curl, CURLOPT_RANGE, "0-1" );
1026  }
1027  }
1028  TempSetHeadRequest(const TempSetHeadRequest &) = delete;
1029  TempSetHeadRequest(TempSetHeadRequest &&) = delete;
1030  TempSetHeadRequest &operator=(const TempSetHeadRequest &) = delete;
1031  TempSetHeadRequest &operator=(TempSetHeadRequest &&) = delete;
1032  ~TempSetHeadRequest() {
1033  if ( _doHttpHeadRequest ) {
1034  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1035  /* yes, this is why we never got to get NOBODY working before,
1036  because setting it changes this option too, and we also*
1037  need to reset it
1038  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1039  */
1040  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
1041  } else {
1042  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
1043  }
1044  }
1045  private:
1046  CURL * _curl;
1047  bool _doHttpHeadRequest;
1048  } _guard( _curl, (_url.getScheme() == "http" || _url.getScheme() == "https") && _settings.headRequestsAllowed() );
1049 
1050 
1051  AutoFILE file { ::fopen( "/dev/null", "w" ) };
1052  if ( !file ) {
1053  ERR << "fopen failed for /dev/null" << endl;
1054  ZYPP_THROW(MediaWriteException("/dev/null"));
1055  }
1056 
1057  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, (*file) );
1058  if ( ret != 0 ) {
1060  }
1061 
1062  CURLcode ok = executeCurl();
1063  MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
1064 
1065  // as we are not having user interaction, the user can't cancel
1066  // the file existence checking, a callback or timeout return code
1067  // will be always a timeout.
1068  try {
1069  evaluateCurlCode( filename, ok, true /* timeout */);
1070  }
1071  catch ( const MediaFileNotFoundException &e ) {
1072  // if the file did not exist then we can return false
1073  return false;
1074  }
1075  catch ( const MediaException &e ) {
1076  // some error, we are not sure about file existence, rethrw
1077  ZYPP_RETHROW(e);
1078  }
1079  // exists
1080  return ( ok == CURLE_OK );
1081 }
1082 
1084 
1085 void MediaCurl::doGetFileCopy( const OnMediaLocation &srcFile , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1086 {
1087  Pathname dest = target.absolutename();
1088  if( assert_dir( dest.dirname() ) )
1089  {
1090  DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1091  ZYPP_THROW( MediaSystemException(getFileUrl(srcFile.filename()), "System error on " + dest.dirname().asString()) );
1092  }
1093 
1094  ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
1095  AutoFILE file;
1096  {
1097  AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
1098  if( ! buf )
1099  {
1100  ERR << "out of memory for temp file name" << endl;
1101  ZYPP_THROW(MediaSystemException(getFileUrl(srcFile.filename()), "out of memory for temp file name"));
1102  }
1103 
1104  AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1105  if( tmp_fd == -1 )
1106  {
1107  ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1108  ZYPP_THROW(MediaWriteException(destNew));
1109  }
1110  destNew = ManagedFile( (*buf), filesystem::unlink );
1111 
1112  file = ::fdopen( tmp_fd, "we" );
1113  if ( ! file )
1114  {
1115  ERR << "fopen failed for file '" << destNew << "'" << endl;
1116  ZYPP_THROW(MediaWriteException(destNew));
1117  }
1118  tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
1119  }
1120 
1121  DBG << "dest: " << dest << endl;
1122  DBG << "temp: " << destNew << endl;
1123 
1124  // set IFMODSINCE time condition (no download if not modified)
1125  if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
1126  {
1127  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1128  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
1129  }
1130  else
1131  {
1132  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1133  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1134  }
1135  try
1136  {
1137  doGetFileCopyFile( srcFile, dest, file, report, options);
1138  }
1139  catch (Exception &e)
1140  {
1141  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1142  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1143  ZYPP_RETHROW(e);
1144  }
1145 
1146  long httpReturnCode = 0;
1147  CURLcode infoRet = curl_easy_getinfo(_curl,
1148  CURLINFO_RESPONSE_CODE,
1149  &httpReturnCode);
1150  bool modified = true;
1151  if (infoRet == CURLE_OK)
1152  {
1153  DBG << "HTTP response: " + str::numstring(httpReturnCode);
1154  if ( httpReturnCode == 304
1155  || ( httpReturnCode == 213 && (_url.getScheme() == "ftp" || _url.getScheme() == "tftp") ) ) // not modified
1156  {
1157  DBG << " Not modified.";
1158  modified = false;
1159  }
1160  DBG << endl;
1161  }
1162  else
1163  {
1164  WAR << "Could not get the response code." << endl;
1165  }
1166 
1167  if (modified || infoRet != CURLE_OK)
1168  {
1169  // apply umask
1170  if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
1171  {
1172  ERR << "Failed to chmod file " << destNew << endl;
1173  }
1174 
1175  file.resetDispose(); // we're going to close it manually here
1176  if ( ::fclose( file ) )
1177  {
1178  ERR << "Fclose failed for file '" << destNew << "'" << endl;
1179  ZYPP_THROW(MediaWriteException(destNew));
1180  }
1181 
1182  // move the temp file into dest
1183  if ( rename( destNew, dest ) != 0 ) {
1184  ERR << "Rename failed" << endl;
1186  }
1187  destNew.resetDispose(); // no more need to unlink it
1188  }
1189 
1190  DBG << "done: " << PathInfo(dest) << endl;
1191 }
1192 
1194 
1195 void MediaCurl::doGetFileCopyFile( const OnMediaLocation & srcFile, const Pathname & dest, FILE *file, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1196 {
1197  DBG << srcFile.filename().asString() << endl;
1198 
1199  if(!_url.isValid())
1201 
1202  if(_url.getHost().empty())
1204 
1205  Url url(getFileUrl(srcFile.filename()));
1206 
1207  DBG << "URL: " << url.asString() << endl;
1208  // Use URL without options and without username and passwd
1209  // (some proxies dislike them in the URL).
1210  // Curl seems to need the just scheme, hostname and a path;
1211  // the rest was already passed as curl options (in attachTo).
1212  Url curlUrl( clearQueryString(url) );
1213 
1214  //
1215  // See also Bug #154197 and ftp url definition in RFC 1738:
1216  // The url "ftp://user@host/foo/bar/file" contains a path,
1217  // that is relative to the user's home.
1218  // The url "ftp://user@host//foo/bar/file" (or also with
1219  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1220  // contains an absolute path.
1221  //
1222  _lastRedirect.clear();
1223  std::string urlBuffer( curlUrl.asString());
1224  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1225  urlBuffer.c_str() );
1226  if ( ret != 0 ) {
1228  }
1229 
1230  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1231  if ( ret != 0 ) {
1233  }
1234 
1235  // Set callback and perform.
1236  internal::ProgressData progressData(_curl, _settings.timeout(), url, srcFile.downloadSize(), &report);
1237  if (!(options & OPTION_NO_REPORT_START))
1238  report->start(url, dest);
1239  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1240  WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1241  }
1242 
1243  ret = executeCurl();
1244 #if CURLVERSION_AT_LEAST(7,19,4)
1245  // bnc#692260: If the client sends a request with an If-Modified-Since header
1246  // with a future date for the server, the server may respond 200 sending a
1247  // zero size file.
1248  // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
1249  if ( ftell(file) == 0 && ret == 0 )
1250  {
1251  long httpReturnCode = 33;
1252  if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
1253  {
1254  long conditionUnmet = 33;
1255  if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
1256  {
1257  WAR << "TIMECONDITION unmet - retry without." << endl;
1258  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1259  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1260  ret = executeCurl();
1261  }
1262  }
1263  }
1264 #endif
1265 
1266  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1267  WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1268  }
1269 
1270  if ( ret != 0 )
1271  {
1272  ERR << "curl error: " << ret << ": " << _curlError
1273  << ", temp file size " << ftell(file)
1274  << " bytes." << endl;
1275 
1276  // the timeout is determined by the progress data object
1277  // which holds whether the timeout was reached or not,
1278  // otherwise it would be a user cancel
1279  try {
1280 
1281  if ( progressData.fileSizeExceeded() )
1282  ZYPP_THROW(MediaFileSizeExceededException(url, progressData.expectedFileSize()));
1283 
1284  evaluateCurlCode( srcFile.filename(), ret, progressData.timeoutReached() );
1285  }
1286  catch ( const MediaException &e ) {
1287  // some error, we are not sure about file existence, rethrw
1288  ZYPP_RETHROW(e);
1289  }
1290  }
1291 }
1292 
1294 
1295 void MediaCurl::getDir( const Pathname & dirname, bool recurse_r ) const
1296 {
1297  filesystem::DirContent content;
1298  getDirInfo( content, dirname, /*dots*/false );
1299 
1300  for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1301  Pathname filename = dirname + it->name;
1302  int res = 0;
1303 
1304  switch ( it->type ) {
1305  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
1306  case filesystem::FT_FILE:
1307  getFile( OnMediaLocation( filename ) );
1308  break;
1309  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
1310  if ( recurse_r ) {
1311  getDir( filename, recurse_r );
1312  } else {
1313  res = assert_dir( localPath( filename ) );
1314  if ( res ) {
1315  WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
1316  }
1317  }
1318  break;
1319  default:
1320  // don't provide devices, sockets, etc.
1321  break;
1322  }
1323  }
1324 }
1325 
1327 
1328 void MediaCurl::getDirInfo( std::list<std::string> & retlist,
1329  const Pathname & dirname, bool dots ) const
1330 {
1331  getDirectoryYast( retlist, dirname, dots );
1332 }
1333 
1335 
1337  const Pathname & dirname, bool dots ) const
1338 {
1339  getDirectoryYast( retlist, dirname, dots );
1340 }
1341 
1343 //
1344 int MediaCurl::aliveCallback( void *clientp, curl_off_t /*dltotal*/, curl_off_t dlnow, curl_off_t /*ultotal*/, curl_off_t /*ulnow*/ )
1345 {
1346  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1347  if( pdata )
1348  {
1349  // Do not propagate dltotal in alive callbacks. MultiCurl uses this to
1350  // prevent a percentage raise while downloading a metalink file. Download
1351  // activity however is indicated by propagating the download rate (via dlnow).
1352  pdata->updateStats( 0.0, dlnow );
1353  return pdata->reportProgress();
1354  }
1355  return 0;
1356 }
1357 
1358 int MediaCurl::progressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow )
1359 {
1360  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1361  if( pdata )
1362  {
1363  // work around curl bug that gives us old data
1364  long httpReturnCode = 0;
1365  if ( curl_easy_getinfo( pdata->curl(), CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0 )
1366  return aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1367 
1368  pdata->updateStats( dltotal, dlnow );
1369  return pdata->reportProgress();
1370  }
1371  return 0;
1372 }
1373 
1375 {
1376  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>(clientp);
1377  return pdata ? pdata->curl() : 0;
1378 }
1379 
1381 
1382 std::string MediaCurl::getAuthHint() const
1383 {
1384  long auth_info = CURLAUTH_NONE;
1385 
1386  CURLcode infoRet =
1387  curl_easy_getinfo(_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1388 
1389  if(infoRet == CURLE_OK)
1390  {
1391  return CurlAuthData::auth_type_long2str(auth_info);
1392  }
1393 
1394  return "";
1395 }
1396 
1401 void MediaCurl::resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
1402 {
1403  internal::ProgressData *data = reinterpret_cast<internal::ProgressData *>(clientp);
1404  if ( data ) {
1405  data->expectedFileSize( expectedFileSize );
1406  }
1407 }
1408 
1414 CURLcode MediaCurl::executeCurl() const
1415 {
1416  // initialize our helpers
1418  internal::CurlPollHelper::CurlPoll{ curl_multi_init() }
1419  ,[](auto &releaseMe ){ if (releaseMe._multi) curl_multi_cleanup(releaseMe._multi); }
1420  );
1421 
1422  if (!cMulti->_multi)
1424 
1425  // we could derive from that, but currently that does not make a lot of sense
1426  internal::CurlPollHelper _curlHelper(cMulti.value());
1427 
1428  // add the easy handle to the multi instance
1429  if ( curl_multi_add_handle( cMulti->_multi, _curl ) != CURLM_OK )
1430  ZYPP_THROW(MediaCurlException( _url, "curl_multi_add_handle", "unknown error"));
1431 
1432  // make sure the handle is cleanly removed from the multi handle
1433  OnScopeExit autoRemove([&](){ curl_multi_remove_handle( cMulti->_multi, _curl ); });
1434 
1435  // kickstart curl, this will cause libcurl to go over the added handles and register sockets and timeouts
1436  CURLMcode mcode = _curlHelper.handleTimout();
1437  if (mcode != CURLM_OK)
1438  ZYPP_THROW(MediaCurlException( _url, "curl_multi_socket_action", "unknown error"));
1439 
1440  bool canContinue = true;
1441  while ( canContinue ) {
1442 
1443  // copy watched sockets in case curl changes the vector as we go over the events later
1444  std::vector<GPollFD> requestedFds = _curlHelper.socks;
1445 
1446  int r = zypp_detail::zypp_poll( requestedFds, _curlHelper.timeout_ms.value_or( -1 ) );
1447  if ( r == -1 )
1448  ZYPP_THROW( MediaCurlException(_url, "zypp_poll() failed", "unknown error") );
1449 
1450  // run curl
1451  if ( r == 0 ) {
1452  CURLMcode mcode = _curlHelper.handleTimout();
1453  if (mcode != CURLM_OK)
1454  ZYPP_THROW(MediaCurlException(_url, "curl_multi_socket_action", "unknown error"));
1455  } else {
1456  CURLMcode mcode = _curlHelper.handleSocketActions( requestedFds );
1457  if (mcode != CURLM_OK)
1458  ZYPP_THROW(MediaCurlException(_url, "curl_multi_socket_action", "unknown error"));
1459  }
1460 
1461  CURLMsg *msg = nullptr;
1462  int nqueue = 0;
1463  while ((msg = curl_multi_info_read( cMulti->_multi, &nqueue)) != 0) {
1464  if ( msg->msg != CURLMSG_DONE ) continue;
1465  if ( msg->easy_handle != _curl ) continue;
1466 
1467  return msg->data.result;
1468  }
1469  }
1470  return CURLE_OK;
1471 }
1472 
1474 
1475 bool MediaCurl::authenticate(const std::string & availAuthTypes, bool firstTry) const
1476 {
1478  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
1479  CurlAuthData_Ptr credentials;
1480 
1481  // get stored credentials
1482  AuthData_Ptr cmcred = cm.getCred(_url);
1483 
1484  if (cmcred && firstTry)
1485  {
1486  credentials.reset(new CurlAuthData(*cmcred));
1487  DBG << "got stored credentials:" << endl << *credentials << endl;
1488  }
1489  // if not found, ask user
1490  else
1491  {
1492 
1493  CurlAuthData_Ptr curlcred;
1494  curlcred.reset(new CurlAuthData());
1496 
1497  // preset the username if present in current url
1498  if (!_url.getUsername().empty() && firstTry)
1499  curlcred->setUsername(_url.getUsername());
1500  // if CM has found some credentials, preset the username from there
1501  else if (cmcred)
1502  curlcred->setUsername(cmcred->username());
1503 
1504  // indicate we have no good credentials from CM
1505  cmcred.reset();
1506 
1507  std::string prompt_msg = str::Format(_("Authentication required for '%s'")) % _url.asString();
1508 
1509  // set available authentication types from the exception
1510  // might be needed in prompt
1511  curlcred->setAuthType(availAuthTypes);
1512 
1513  // ask user
1514  if (auth_report->prompt(_url, prompt_msg, *curlcred))
1515  {
1516  DBG << "callback answer: retry" << endl
1517  << "CurlAuthData: " << *curlcred << endl;
1518 
1519  if (curlcred->valid())
1520  {
1521  credentials = curlcred;
1522  // if (credentials->username() != _url.getUsername())
1523  // _url.setUsername(credentials->username());
1531  }
1532  }
1533  else
1534  {
1535  DBG << "callback answer: cancel" << endl;
1536  }
1537  }
1538 
1539  // set username and password
1540  if (credentials)
1541  {
1542  // HACK, why is this const?
1543  const_cast<MediaCurl*>(this)->_settings.setUsername(credentials->username());
1544  const_cast<MediaCurl*>(this)->_settings.setPassword(credentials->password());
1545 
1546  // set username and password
1547  CURLcode ret = curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
1549 
1550  // set available authentication types from the exception
1551  if (credentials->authType() == CURLAUTH_NONE)
1552  credentials->setAuthType(availAuthTypes);
1553 
1554  // set auth type (seems this must be set _after_ setting the userpwd)
1555  if (credentials->authType() != CURLAUTH_NONE)
1556  {
1557  // FIXME: only overwrite if not empty?
1558  const_cast<MediaCurl*>(this)->_settings.setAuthType(credentials->authTypeAsString());
1559  ret = curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, credentials->authType());
1561  }
1562 
1563  if (!cmcred)
1564  {
1565  credentials->setUrl(_url);
1566  cm.addCred(*credentials);
1567  cm.save();
1568  }
1569 
1570  return true;
1571  }
1572 
1573  return false;
1574 }
1575 
1576 //need a out of line definiton, otherwise vtable is emitted for every translation unit
1578 
1579  } // namespace media
1580 } // namespace zypp
1581 //
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:537
long timeout() const
transfer timeout
void globalInitCurlOnce()
Definition: curlhelper.cc:64
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
bool checkAttachPoint(const Pathname &apoint) const override
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
Definition: MediaCurl.cc:700
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:320
MediaCurlExceptionMayRetryInternaly(const Url &url_r, const std::string &err_r, const std::string &msg_r)
Definition: MediaCurl.cc:313
std::string targetDistribution() const
This is register.target attribute of the installed base product.
Definition: Target.cc:102
Interface to gettext.
#define SET_OPTION_OFFT(opt, val)
Definition: MediaCurl.cc:343
#define MIL
Definition: Logger.h:96
std::string curlUnEscape(const std::string &text_r)
Definition: curlhelper.cc:366
const Pathname & clientCertificatePath() const
SSL client certificate file.
bool progress(int value_r, const Url &file_r, double dbps_avg_r=-1, double dbps_current_r=-1) override
Download progress.
Definition: MediaCurl.cc:96
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar" (trims)
#define _(MSG)
Definition: Gettext.h:37
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: curlhelper.cc:143
ProgressData()
Ctor no range [0,0](0).
Definition: progressdata.h:174
const Pathname & path() const
Return current Pathname.
Definition: PathInfo.h:247
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:429
Describes a resource file located on a medium.
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:925
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:394
bool authenticate(const std::string &availAuthTypes, bool firstTry) const
Definition: MediaCurl.cc:1475
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:31
void setPassword(const std::string &val_r)
sets the auth password
std::chrono::steady_clock::time_point TimePoint
Definition: MediaCurl.cc:55
ByteCount _expectedFileSize
Definition: MediaCurl.cc:180
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
const char * distributionFlavorHeader()
Definition: MediaCurl.cc:278
Store and operate with byte count.
Definition: ByteCount.h:30
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:43
Holds transfer setting.
void save()
Saves any unsaved credentials added via addUserCred() or addGlobalCred() methods. ...
const std::string & authType() const
get the allowed authentication types
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & proxyUsername() const
proxy auth username
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:173
int reportProgress() const
Definition: MediaCurl.cc:253
Url clearQueryString(const Url &url)
Definition: curlhelper.cc:373
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
void start(const Url &file_r, Pathname localfile_r) override
Definition: MediaCurl.cc:84
const char * anonymousIdHeader()
Definition: MediaCurl.cc:264
const char * c_str() const
String representation.
Definition: Pathname.h:110
void addHistory(const std::string &msg_r)
Add some message text to the history.
Definition: Exception.cc:159
bool isUseableAttachPoint(const Pathname &path, bool mtab=true) const
Ask media manager, if the specified path is already used as attach point or if there are another atta...
void setUsername(const std::string &val_r)
sets the auth username
Action problem(const Url &file_r, Error error_r, const std::string &description_r) override
Definition: MediaCurl.cc:115
void getFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename) const override
Definition: MediaCurl.cc:742
MediaCurlException(const Url &url_r, std::string err_r, std::string msg_r)
time_t _timeNow
Now.
Definition: MediaCurl.cc:186
Definition: Arch.h:363
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:419
bool timeoutReached() const
Definition: MediaCurl.cc:162
Convenient building of std::string with boost::format.
Definition: String.h:252
Structure holding values of curlrc options.
Definition: curlconfig.h:26
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:37
CURLcode executeCurl() const
Definition: MediaCurl.cc:1414
Edition * _value
Definition: SysContent.cc:311
AutoDispose<int> calling ::close
Definition: AutoDispose.h:309
std::string _currentCookieFile
Definition: MediaCurl.h:163
const std::string & password() const
auth password
void getDir(const Pathname &dirname, bool recurse_r) const override
Call concrete handler to provide directory content (not recursive!) below attach point.
Definition: MediaCurl.cc:1295
void getFile(const OnMediaLocation &file) const override
Call concrete handler to provide file below attach point.
Definition: MediaCurl.cc:733
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:519
#define ERR
Definition: Logger.h:98
const std::string & username() const
auth username
bool fileSizeExceeded() const
Definition: MediaCurl.cc:165
Pathname localPath(const Pathname &pathname) const
Files provided will be available at &#39;localPath(filename)&#39;.
const Headers & headers() const
returns a list of all added headers (trimmed)
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
static void setCookieFile(const Pathname &)
Definition: MediaCurl.cc:387
void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
Definition: MediaCurl.cc:726
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition: MediaCurl.cc:1401
const std::string & hint() const
comma separated list of available authentication types
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
zypp::callback::UserData UserData
Definition: userrequest.h:18
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:441
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1206
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:501
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:664
Bottleneck filtering all DownloadProgressReport issued from Media[Muli]Curl.
Definition: MediaCurl.cc:53
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:382
void setAuthType(const std::string &val_r)
set the allowed authentication types
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:224
void finish(const Url &file_r, Error error_r, const std::string &reason_r) override
Definition: MediaCurl.cc:121
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:701
const Url _url
Url to handle.
Definition: MediaHandler.h:113
void setMediaSource(const MediaSourceRef &ref)
Set new media source reference.
const std::string & asString() const
String representation.
Definition: Pathname.h:91
int rename(const Pathname &oldpath, const Pathname &newpath)
Like &#39;rename&#39;.
Definition: PathInfo.cc:743
Just inherits Exception to separate media exceptions.
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:835
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition: Exception.cc:110
const ByteCount & downloadSize() const
The size of the resource on the server.
void disconnect()
Use concrete handler to isconnect media.
long connectTimeout() const
connection timeout
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:124
do not send a start ProgressReport
Definition: MediaCurl.h:45
#define WAR
Definition: Logger.h:97
shared_ptr< AuthData > AuthData_Ptr
Definition: authdata.h:81
void getDirectoryYast(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Retrieve and if available scan dirname/directory.yast.
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: curlhelper.cc:183
static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback reporting download progress.
Definition: MediaCurl.cc:1358
Receiver * _oldRec
std::string proxyuserpwd
Definition: curlconfig.h:49
bool isValid() const
Verifies the Url.
Definition: Url.cc:493
virtual bool doGetDoesFileExist(const Pathname &filename) const
Definition: MediaCurl.cc:976
const Pathname & clientKeyPath() const
SSL client key file.
void expectedFileSize(ByteCount newval_r)
Definition: MediaCurl.cc:171
const Pathname & filename() const
The path to the resource on the medium.
void attachTo(bool next=false) override
Call concrete handler to attach the media.
Definition: MediaCurl.cc:665
std::string numstring(char n, int w=0)
Definition: String.h:289
Common baseclass for MediaCurl and MediaNetwork.
bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly, or user cancels the operation.
Definition: MediaCurl.cc:804
const char * agentString()
Definition: MediaCurl.cc:292
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:171
void doGetFileCopyFile(const OnMediaLocation &srcFile, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1195
Media source internally used by MediaManager and MediaHandler.
Definition: MediaSource.h:37
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
Definition: MediaCurl.cc:192
curl_slist * _customHeaders
Definition: MediaCurl.h:171
bool proxyEnabled() const
proxy is enabled
int rmdir(const Pathname &path)
Like &#39;rmdir&#39;.
Definition: PathInfo.cc:367
constexpr std::string_view FILE("file")
#define SET_OPTION(opt, val)
Definition: MediaCurl.cc:336
Pathname absolutename() const
Return this path, adding a leading &#39;/&#39; if relative.
Definition: Pathname.h:139
Base class for Exception.
Definition: Exception.h:146
Pathname attachPoint() const
Return the currently used attach point.
std::string _lastRedirect
to log/report redirections
Definition: MediaCurl.h:166
curl_off_t _dnlNow
Bytes downloaded now.
Definition: MediaCurl.cc:190
Url url() const
Url used.
Definition: MediaHandler.h:503
shared_ptr< CurlAuthData > CurlAuthData_Ptr
Definition: curlauthdata.h:102
curl_off_t _dnlTotal
Bytes to download or 0 if unknown.
Definition: MediaCurl.cc:188
double _drateLast
Download rate in last period.
Definition: MediaCurl.cc:195
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:592
time_t _timeStart
Start total stats.
Definition: MediaCurl.cc:183
void getDirInfo(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const override
Call concrete handler to provide a content list of directory on media via retlist.
Definition: MediaCurl.cc:1328
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:130
void disconnectFrom() override
Definition: MediaCurl.cc:707
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1374
Typesafe passing of user data via callbacks.
Definition: UserData.h:38
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3" (trims)
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:221
curl_off_t _dnlLast
Bytes downloaded at period start.
Definition: MediaCurl.cc:189
void report(const UserData &userData_r=UserData()) override
The most generic way of sending/receiving data.
Definition: MediaCurl.cc:80
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:320
static Pathname _cookieFile
Definition: MediaCurl.h:164
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: curlhelper.cc:331
double _drateTotal
Download rate so far.
Definition: MediaCurl.cc:194
std::string anonymousUniqueId() const
anonymous unique id
Definition: Target.cc:132
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:798
std::string userPassword() const
returns the user and password as a user:pass string
std::string getAuthHint() const
Return a comma separated list of available authentication methods supported by server.
Definition: MediaCurl.cc:1382
time_t _timeRcv
Start of no-data timeout.
Definition: MediaCurl.cc:185
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:23
zypp::callback::SendReport< zypp::KeyRingReport > _report
Definition: keyringwf.cc:457
Pathname createAttachPoint() const
Try to create a default / temporary attach point.
int zypp_poll(std::vector< GPollFD > &fds, int timeout)
Small wrapper around g_poll that additionally listens to the shutdown FD returned by ZYpp::shutdownSi...
Definition: ZYppImpl.cc:313
virtual void doGetFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1085
void updateStats(curl_off_t dltotal=0.0, curl_off_t dlnow=0.0)
Definition: MediaCurl.cc:210
void addCred(const AuthData &cred)
Add new credentials with user callbacks.
Easy-to use interface to the ZYPP dependency resolver.
Definition: Application.cc:19
Curl HTTP authentication data.
Definition: curlauthdata.h:22
std::string distributionFlavor() const
This is flavor attribute of the installed base product but does not require the target to be loaded a...
Definition: Target.cc:127
OptionalDownloadProgressReport(bool isOptional=false)
Definition: MediaCurl.cc:57
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:170
ByteCount expectedFileSize() const
Definition: MediaCurl.cc:168
CURLcode setCurlRedirProtocols(CURL *curl)
Definition: curlhelper.cc:512
const std::string & proxy() const
proxy host
bool optional() const
Whether this is an optional resource.
time_t _timeLast
Start last period(~1sec)
Definition: MediaCurl.cc:184
const char * c_str() const
Definition: IdStringType.h:107
TrueBool _guard
Definition: TargetImpl.cc:1608
static int aliveCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback sending just an alive trigger to the UI, without stats (e.g.
Definition: MediaCurl.cc:1344
bool userMayRWX() const
Definition: PathInfo.h:354
const std::string & userAgentString() const
user agent string (trimmed)
Url manipulation class.
Definition: Url.h:91
bool headRequestsAllowed() const
whether HEAD requests are allowed
#define DBG
Definition: Logger.h:95
Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy E.g. ...
Definition: MediaCurl.cc:310
zypp::callback::SendReport< zypp::media::DownloadProgressReport > * report
Definition: MediaCurl.cc:181
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:576