libzypp  17.32.5
susetags.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
9 #include "susetags.h"
10 #include "zypp-core/base/Regex.h"
11 #include <zypp-core/zyppng/ui/ProgressObserver>
12 #include <zypp-media/ng/ProvideSpec>
13 #include <zypp/ng/Context>
14 
15 #include <zypp-core/parser/ParseException>
16 
19 
24 
26 
27  namespace {
28 
29  using namespace zyppng::operators;
30 
36  template<class Executor, class OpType>
37  struct StatusLogic : public LogicBase<Executor, OpType>{
38  ZYPP_ENABLE_LOGIC_BASE(Executor, OpType);
39 
40  public:
41 
42  using DlContextRefType = std::conditional_t<zyppng::detail::is_async_op_v<OpType>, repo::AsyncDownloadContextRef, repo::SyncDownloadContextRef>;
43  using ZyppContextType = typename DlContextRefType::element_type::ContextType;
44  using ProvideType = typename ZyppContextType::ProvideType;
45  using MediaHandle = typename ProvideType::MediaHandle;
46  using ProvideRes = typename ProvideType::Res;
47 
48  StatusLogic( DlContextRefType ctx, MediaHandle &&media )
49  : _ctx(std::move(ctx))
50  , _handle(std::move(media))
51  {}
52 
53  MaybeAsyncRef<expected<zypp::RepoStatus>> execute() {
54  return _ctx->zyppContext()->provider()->provide( _handle, _ctx->repoInfo().path() / "content" , ProvideFileSpec() )
55  | [this]( expected<ProvideRes> contentFile ) {
56 
57  // mandatory master index is missing -> stay empty
58  if ( !contentFile )
60 
61  zypp::RepoStatus status ( contentFile->file() );
62 
63  if ( !status.empty() /* && _ctx->repoInfo().requireStatusWithMediaFile() */ ) {
64  return _ctx->zyppContext()->provider()->provide( _handle, "/media.1/media" , ProvideFileSpec())
65  | [status = std::move(status)]( expected<ProvideRes> mediaFile ) mutable {
66  if ( mediaFile ) {
67  return make_expected_success(status && zypp::RepoStatus( mediaFile->file()) );
68  }
69  return make_expected_success( std::move(status) );
70  };
71  }
72  return makeReadyResult( make_expected_success(std::move(status)) );
73  };
74  }
75 
76  DlContextRefType _ctx;
77  MediaHandle _handle;
78  };
79  }
80 
81  AsyncOpRef<expected<zypp::RepoStatus> > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
82  {
83  return SimpleExecutor< StatusLogic, AsyncOp<expected<zypp::RepoStatus>> >::run( std::move(dl), std::move(mediaHandle) );
84  }
85 
86  expected<zypp::RepoStatus> repoStatus(repo::SyncDownloadContextRef dl, SyncMediaHandle mediaHandle)
87  {
88  return SimpleExecutor< StatusLogic, SyncOp<expected<zypp::RepoStatus>> >::run( std::move(dl), std::move(mediaHandle) );
89  }
90 
91 
92 
93  namespace {
94 
95  using namespace zyppng::operators;
96 
97  // search old repository file file to run the delta algorithm on
98  static zypp::Pathname search_deltafile( const zypp::Pathname &dir, const zypp::Pathname &file )
99  {
100  zypp::Pathname deltafile(dir + file.basename());
102  return deltafile;
103  return zypp::Pathname();
104  }
105 
106 
107  template<class Executor, class OpType>
108  struct DlLogic : public LogicBase<Executor, OpType> {
109 
110  ZYPP_ENABLE_LOGIC_BASE(Executor, OpType);
111  public:
112 
113  using DlContextRefType = std::conditional_t<zyppng::detail::is_async_op_v<OpType>, repo::AsyncDownloadContextRef, repo::SyncDownloadContextRef>;
114  using ZyppContextType = typename DlContextRefType::element_type::ContextType;
115  using ProvideType = typename ZyppContextType::ProvideType;
116  using MediaHandle = typename ProvideType::MediaHandle;
117  using ProvideRes = typename ProvideType::Res;
118 
119  DlLogic( DlContextRefType ctx, MediaHandle &&mediaHandle, ProgressObserverRef &&progressObserver )
120  : _ctx( std::move(ctx))
121  , _mediaHandle(std::move(mediaHandle))
122  , _progressObserver(std::move(progressObserver))
123  {}
124 
125  auto execute() {
126  // download media file here
128  | [this]( expected<zypp::ManagedFile> &&mediaInfo ) {
129 
130  // remember the media info if we had one
131  if ( mediaInfo ) _ctx->files().push_back ( std::move(mediaInfo.get()) );
132 
133  if ( _progressObserver ) _progressObserver->inc();
134 
135  return RepoDownloaderWorkflow::downloadMasterIndex( _ctx, _mediaHandle, _ctx->repoInfo().path() / "content" )
137  | and_then( [this] ( DlContextRefType && ) {
138 
139  zypp::Pathname contentPath = _ctx->files().front();
140  std::vector<zypp::OnMediaLocation> requiredFiles;
141 
142  // Content file first to get the repoindex
143  try {
144  const zypp::Pathname &inputfile = contentPath;
146  content.setRepoIndexConsumer( [this]( const auto & data_r ) {
147  MIL << "Consuming repo index" << std::endl;
148  _repoindex = data_r;
149  });
150  content.parse( inputfile );
151 
152  if ( ! _repoindex ) {
153  ZYPP_THROW( zypp::parser::ParseException( ( _ctx->destDir() / _ctx->repoInfo().path() ).asString() + ": " + "No repository index in content file." ) );
154  }
155 
156  MIL << "RepoIndex: " << _repoindex << std::endl;
157  if ( _repoindex->metaFileChecksums.empty() ) {
158  ZYPP_THROW( zypp::parser::ParseException( ( _ctx->destDir() / _ctx->repoInfo().path() ).asString() + ": " + "No metadata checksums in content file." ) );
159  }
160 
161  if ( _repoindex->signingKeys.empty() ) {
162  WAR << "No signing keys defined." << std::endl;
163  }
164 
165  // Prepare parsing
166  zypp::Pathname descr_dir = _repoindex->descrdir; // path below reporoot
167  //_datadir = _repoIndex->datadir; // path below reporoot
168 
169  std::map<std::string,zypp::parser::susetags::RepoIndex::FileChecksumMap::const_iterator> availablePackageTranslations;
170 
171  for_( it, _repoindex->metaFileChecksums.begin(), _repoindex->metaFileChecksums.end() )
172  {
173  // omit unwanted translations
174  if ( zypp::str::hasPrefix( it->first, "packages" ) )
175  {
176  static const zypp::str::regex rx_packages( "^packages((.gz)?|(.([^.]*))(.gz)?)$" );
177  zypp::str::smatch what;
178 
179  if ( zypp::str::regex_match( it->first, what, rx_packages ) ) {
180  if ( what[4].empty() // packages(.gz)?
181  || what[4] == "DU"
182  || what[4] == "en" )
183  { ; /* always downloaded */ }
184  else if ( what[4] == "FL" )
185  { continue; /* never downloaded */ }
186  else
187  {
188  // remember and decide later
189  availablePackageTranslations[what[4]] = it;
190  continue;
191  }
192  }
193  else
194  continue; // discard
195 
196  } else if ( it->first == "patterns.pat"
197  || it->first == "patterns.pat.gz" ) {
198  // take all patterns in one go
199 
200  } else if ( zypp::str::endsWith( it->first, ".pat" )
201  || zypp::str::endsWith( it->first, ".pat.gz" ) ) {
202 
203  // *** see also zypp/parser/susetags/RepoParser.cc ***
204 
205  // omit unwanted patterns, see https://bugzilla.novell.com/show_bug.cgi?id=298716
206  // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
207  // split at dots, take .pat or .pat.gz into account
208 
209  std::vector<std::string> patparts;
210  unsigned archpos = 2;
211  // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
212  unsigned count = zypp::str::split( it->first, std::back_inserter(patparts), "." );
213  if ( patparts[count-1] == "gz" )
214  archpos++;
215 
216  if ( count > archpos ) {
217  try { // might by an invalid architecture
218  zypp::Arch patarch( patparts[count-archpos] );
219  if ( !patarch.compatibleWith( zConfig().systemArchitecture() ) ) {
220  // discard, if not compatible
221  MIL << "Discarding pattern " << it->first << std::endl;
222  continue;
223  }
224 
225  } catch ( const zypp::Exception & excpt ) {
226  WAR << "Pattern file name does not contain recognizable architecture: " << it->first << std::endl;
227  // keep .pat file if it doesn't contain an recognizable arch
228  }
229  }
230  }
231 
232  MIL << "adding job " << it->first << std::endl;
233  auto location = zypp::OnMediaLocation( repoInfo().path() + descr_dir + it->first, 1 )
234  .setChecksum( it->second )
235  .setDeltafile( search_deltafile( _ctx->deltaDir() + descr_dir, it->first) );
236 
237  requiredFiles.push_back( std::move(location) );
238  }
239 
240 
241  // check whether to download more package translations:
242  {
243  auto fnc_checkTransaltions( [&]( const zypp::Locale & locale_r ) {
244  for ( zypp::Locale toGet( locale_r ); toGet; toGet = toGet.fallback() ) {
245  auto it( availablePackageTranslations.find( toGet.code() ) );
246  if ( it != availablePackageTranslations.end() ) {
247  auto mit( it->second );
248  MIL << "adding job " << mit->first << std::endl;
249  requiredFiles.push_back( zypp::OnMediaLocation( repoInfo().path() + descr_dir + mit->first, 1 )
250  .setChecksum( mit->second )
251  .setDeltafile( search_deltafile( deltaDir() + descr_dir, mit->first) ));
252  break;
253  }
254  }
255  });
256 
257  for ( const zypp::Locale & it : zConfig().repoRefreshLocales() ) {
258  fnc_checkTransaltions( it );
259  }
260  fnc_checkTransaltions( zConfig().textLocale() );
261  }
262 
263  for( const auto &it : _repoindex->mediaFileChecksums ) {
264  // Repo adopts license files listed in HASH
265  if ( it.first != "license.tar.gz" )
266  continue;
267 
268  MIL << "adding job " << it.first << std::endl;
269  requiredFiles.push_back( zypp::OnMediaLocation ( repoInfo().path() + it.first, 1 )
270  .setChecksum( it.second )
271  .setDeltafile( search_deltafile( deltaDir(), it.first ) ));
272  }
273 
274  for( const auto &it : _repoindex->signingKeys ) {
275  MIL << "adding job " << it.first << std::endl;
276  zypp::OnMediaLocation location( repoInfo().path() + it.first, 1 );
277  location.setChecksum( it.second );
278  requiredFiles.push_back( std::move(location) );
279  }
280 
281  } catch ( const zypp::Exception &e ) {
282  ZYPP_CAUGHT( e );
283  return makeReadyResult(expected<DlContextRefType>::error( std::make_exception_ptr(e) ) );
284  } catch ( ... ) {
285  return makeReadyResult(expected<DlContextRefType>::error( std::current_exception() ) );
286  }
287 
288  // add the required files to the base steps
289  if ( _progressObserver ) _progressObserver->setBaseSteps ( _progressObserver->baseSteps () + requiredFiles.size() );
290 
291  return transform_collect ( std::move(requiredFiles), [this]( zypp::OnMediaLocation file ) {
292 
293  return DownloadWorkflow::provideToCacheDir( _ctx, _mediaHandle, file.filename(), ProvideFileSpec(file) )
295 
296  }) | and_then ( [this]( std::vector<zypp::ManagedFile> &&dlFiles ) {
297  auto &downloadedFiles = _ctx->files();
298  downloadedFiles.insert( downloadedFiles.end(), std::make_move_iterator(dlFiles.begin()), std::make_move_iterator(dlFiles.end()) );
299  return expected<DlContextRefType>::success( std::move(_ctx) );
300  });
301  });
302 
304  }
305 
306  private:
307 
308  const zypp::RepoInfo &repoInfo() const {
309  return _ctx->repoInfo();
310  }
311 
312  const zypp::filesystem::Pathname &deltaDir() const {
313  return _ctx->deltaDir();
314  }
315 
316  zypp::ZConfig &zConfig() {
317  return _ctx->zyppContext()->config();
318  }
319 
320  DlContextRefType _ctx;
321  zypp::parser::susetags::RepoIndex_Ptr _repoindex;
322  MediaHandle _mediaHandle;
323  ProgressObserverRef _progressObserver;
324  };
325  }
326 
327  AsyncOpRef<expected<repo::AsyncDownloadContextRef> > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
328  {
329  return SimpleExecutor< DlLogic, AsyncOp<expected<repo::AsyncDownloadContextRef>> >::run( std::move(dl), std::move(mediaHandle), std::move(progressObserver) );
330  }
331 
332  expected<repo::SyncDownloadContextRef> download(repo::SyncDownloadContextRef dl, SyncMediaHandle mediaHandle, ProgressObserverRef progressObserver)
333  {
334  return SimpleExecutor< DlLogic, SyncOp<expected<repo::SyncDownloadContextRef>> >::run( std::move(dl), std::move(mediaHandle), std::move(progressObserver) );
335  }
336 
337 }
#define MIL
Definition: Logger.h:96
Pathname deltafile
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:429
Describes a resource file located on a medium.
Regular expression.
Definition: Regex.h:94
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
Definition: susetags.cc:327
expected< T, E > inspect(expected< T, E > exp, Function &&f)
Definition: expected.h:462
Architecture.
Definition: Arch.h:36
A ProvideRes object is a reference counted ownership of a resource in the cache provided by a Provide...
Definition: provideres.h:35
Locale fallback() const
Return the fallback locale for this locale, if no fallback exists the empty Locale::noCode.
Definition: Locale.cc:208
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition: Easy.h:28
OnMediaLocation & setChecksum(CheckSum val_r)
Set the checksum.
Definition: Arch.h:363
What is known about a repository.
Definition: RepoInfo.h:71
static expected< std::decay_t< Type >, Err > make_expected_success(Type &&t)
Definition: expected.h:341
ProgressObserverRef _progressObserver
Definition: susetags.cc:323
DlContextRefType _ctx
Definition: susetags.cc:76
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:128
MediaHandle _handle
Definition: susetags.cc:77
std::string asString(TInt val, char zero='0', char one='1')
For printing bits.
Definition: Bit.h:57
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \, const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition: String.h:531
auto transform_collect(Container< Msg, CArgs... > &&in, Transformation &&f)
Definition: expected.h:597
bool isExist() const
Return whether valid stat info exists.
Definition: PathInfo.h:282
AsyncOpRef< expected< zypp::ManagedFile > > provideToCacheDir(AsyncCacheProviderContextRef cacheContext, ProvideMediaHandle medium, zypp::Pathname file, ProvideFileSpec filespec)
Definition: downloadwf.cc:209
Interim helper class to collect global options and settings.
Definition: ZConfig.h:63
#define WAR
Definition: Logger.h:97
Parse repoindex part from a content file.
#define ZYPP_ENABLE_LOGIC_BASE(Executor, OpType)
Definition: logichelpers.h:199
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > downloadMasterIndex(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, zypp::filesystem::Pathname masterIndex_r)
void setRepoIndexConsumer(const RepoIndexConsumer &fnc_r)
Consumer to call when repo index was parsed.
typename conditional< B, T, F >::type conditional_t
Definition: TypeTraits.h:39
std::conditional_t< isAsync, AsyncOpRef< T >, T > makeReadyResult(T &&result)
Definition: asyncop.h:297
const Pathname & filename() const
The path to the resource on the medium.
bool endsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasSuffix
Definition: String.h:1092
static expected success(ConsParams &&...params)
Definition: expected.h:115
OnMediaLocation & setDeltafile(Pathname path)
Set the deltafile.
&#39;Language[_Country]&#39; codes.
Definition: Locale.h:49
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:437
std::shared_ptr< AsyncOp< T > > AsyncOpRef
Definition: asyncop.h:255
Regular expression match result.
Definition: Regex.h:167
zypp::parser::susetags::RepoIndex_Ptr _repoindex
Definition: susetags.cc:321
Base class for Exception.
Definition: Exception.h:146
AsyncOpRef< expected< zypp::RepoStatus > > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
Definition: susetags.cc:81
MediaHandle _mediaHandle
Definition: susetags.cc:322
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:221
bool regex_match(const std::string &s, smatch &matches, const regex &regex)
regex ZYPP_STR_REGEX regex ZYPP_STR_REGEX
Definition: Regex.h:70
auto finishProgress(ProgressObserverRef progressObserver)
ResultType and_then(const expected< T, E > &exp, Function &&f)
Definition: expected.h:367
Track changing files or directories.
Definition: RepoStatus.h:40
Download workflow namespace for SUSETags (YaST2) repositories Encapsulates all the knowledge of which...
Definition: susetags.cc:25
auto downloadMediaInfo(MediaHandle &&mediaHandle, const zypp::filesystem::Pathname &destdir)
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition: String.h:1027
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
auto incProgress(ProgressObserverRef progressObserver, double progrIncrease=1.0, std::optional< std::string > newStr={})