XRootD
XrdPfcConfiguration.cc
Go to the documentation of this file.
1 #include "XrdPfc.hh"
2 #include "XrdPfcTrace.hh"
3 #include "XrdPfcInfo.hh"
4 
5 #include "XrdOss/XrdOss.hh"
6 
7 #include "XrdOuc/XrdOucEnv.hh"
8 #include "XrdOuc/XrdOucUtils.hh"
9 #include "XrdOuc/XrdOucStream.hh"
11 #include "XrdOuc/XrdOuca2x.hh"
12 
13 #include "XrdOfs/XrdOfsConfigPI.hh"
14 #include "XrdVersion.hh"
15 
16 #include <fcntl.h>
17 
18 using namespace XrdPfc;
19 
21 
23  m_hdfsmode(false),
24  m_allow_xrdpfc_command(false),
25  m_data_space("public"),
26  m_meta_space("public"),
27  m_diskTotalSpace(-1),
28  m_diskUsageLWM(-1),
29  m_diskUsageHWM(-1),
30  m_fileUsageBaseline(-1),
31  m_fileUsageNominal(-1),
32  m_fileUsageMax(-1),
33  m_purgeInterval(300),
34  m_purgeColdFilesAge(-1),
35  m_purgeAgeBasedPeriod(10),
36  m_accHistorySize(20),
37  m_dirStatsMaxDepth(-1),
38  m_dirStatsStoreDepth(0),
39  m_bufferSize(128*1024),
40  m_RamAbsAvailable(0),
41  m_RamKeepStdBlocks(0),
42  m_wqueue_blocks(16),
43  m_wqueue_threads(4),
44  m_prefetch_max_blocks(10),
45  m_hdfsbsize(128*1024*1024),
46  m_flushCnt(2000),
47  m_cs_UVKeep(-1),
48  m_cs_Chk(CSChk_Net),
49  m_cs_ChkTLS(false),
50  m_onlyIfCachedMinSize(1024*1024),
51  m_onlyIfCachedMinFrac(1.0)
52 {}
53 
54 
55 bool Cache::cfg2bytes(const std::string &str, long long &store, long long totalSpace, const char *name)
56 {
57  char errStr[1024];
58  snprintf(errStr, 1024, "ConfigParameters() Error parsing parameter %s", name);
59 
60  if (::isalpha(*(str.rbegin())))
61  {
62  if (XrdOuca2x::a2sz(m_log, errStr, str.c_str(), &store, 0, totalSpace))
63  {
64  return false;
65  }
66  }
67  else
68  {
69  char *eP;
70  errno = 0;
71  double frac = strtod(str.c_str(), &eP);
72  if (errno || eP == str.c_str())
73  {
74  m_log.Emsg(errStr, str.c_str());
75  return false;
76  }
77 
78  store = static_cast<long long>(totalSpace * frac + 0.5);
79  }
80 
81  if (store < 0 || store > totalSpace)
82  {
83  snprintf(errStr, 1024, "ConfigParameters() Error: parameter %s should be between 0 and total available disk space (%lld) - it is %lld (given as %s)",
84  name, totalSpace, store, str.c_str());
85  m_log.Emsg(errStr, "");
86  return false;
87  }
88 
89  return true;
90 }
91 
92 /* Function: xcschk
93 
94  Purpose: To parse the directive: cschk <parms>
95 
96  parms: [[no]net] [[no]tls] [[no]cache] [uvkeep <arg>]
97 
98  all Checksum check on cache & net transfers.
99  cache Checksum check on cache only, 'no' turns it off.
100  net Checksum check on net transfers 'no' turns it off.
101  tls use TLS if server doesn't support checksums 'no' turns it off.
102  uvkeep Maximum amount of time a cached file make be kept if it
103  contains unverified checksums as n[d|h|m|s], where 'n'
104  is a non-negative integer. A value of 0 prohibits disk
105  caching unless the checksum can be verified. You can
106  also specify "lru" which means the standard purge policy
107  is to be used.
108 
109  Output: true upon success or false upon failure.
110  */
111 bool Cache::xcschk(XrdOucStream &Config)
112 {
113  const char *val, *val2;
114  struct cschkopts {const char *opname; int opval;} csopts[] =
115  {
116  {"off", CSChk_None},
117  {"cache", CSChk_Cache},
118  {"net", CSChk_Net},
119  {"tls", CSChk_TLS}
120  };
121  int i, numopts = sizeof(csopts)/sizeof(struct cschkopts);
122  bool isNo;
123 
124  if (! (val = Config.GetWord()))
125  {m_log.Emsg("Config", "cschk parameter not specified"); return false; }
126 
127  while(val)
128  {
129  if ((isNo = strncmp(val, "no", 2) == 0))
130  val2 = val + 2;
131  else
132  val2 = val;
133  for (i = 0; i < numopts; i++)
134  {
135  if (!strcmp(val2, csopts[i].opname))
136  {
137  if (isNo)
138  m_configuration.m_cs_Chk &= ~csopts[i].opval;
139  else if (csopts[i].opval)
140  m_configuration.m_cs_Chk |= csopts[i].opval;
141  else
142  m_configuration.m_cs_Chk = csopts[i].opval;
143  break;
144  }
145  }
146  if (i >= numopts)
147  {
148  if (strcmp(val, "uvkeep"))
149  {
150  m_log.Emsg("Config", "invalid cschk option -", val);
151  return false;
152  }
153  if (!(val = Config.GetWord()))
154  {
155  m_log.Emsg("Config", "cschk uvkeep value not specified");
156  return false;
157  }
158  if (!strcmp(val, "lru"))
159  m_configuration.m_cs_UVKeep = -1;
160  else
161  {
162  int uvkeep;
163  if (XrdOuca2x::a2tm(m_log, "uvkeep time", val, &uvkeep, 0))
164  return false;
165  m_configuration.m_cs_UVKeep = uvkeep;
166  }
167  }
168  val = Config.GetWord();
169  }
170  // Decompose into separate TLS state, it is only passed on to psx
171  m_configuration.m_cs_ChkTLS = m_configuration.m_cs_Chk & CSChk_TLS;
172  m_configuration.m_cs_Chk &= ~CSChk_TLS;
173 
174  m_env->Put("psx.CSNet", m_configuration.is_cschk_net() ? (m_configuration.m_cs_ChkTLS ? "2" : "1") : "0");
175 
176  return true;
177 }
178 
179 
180 /* Function: xdlib
181 
182  Purpose: To parse the directive: decisionlib <path> [<parms>]
183 
184  <path> the path of the decision library to be used.
185  <parms> optional parameters to be passed.
186 
187 
188  Output: true upon success or false upon failure.
189  */
190 bool Cache::xdlib(XrdOucStream &Config)
191 {
192  const char* val;
193 
194  std::string libp;
195  if (! (val = Config.GetWord()) || ! val[0])
196  {
197  TRACE(Info," Cache::Config() decisionlib not specified; always caching files");
198  return true;
199  }
200  else
201  {
202  libp = val;
203  }
204 
205  char params[4096];
206  if (val[0])
207  Config.GetRest(params, 4096);
208  else
209  params[0] = 0;
210 
211  XrdOucPinLoader* myLib = new XrdOucPinLoader(&m_log, 0, "decisionlib",
212  libp.c_str());
213 
214  Decision *(*ep)(XrdSysError&);
215  ep = (Decision *(*)(XrdSysError&))myLib->Resolve("XrdPfcGetDecision");
216  if (! ep) {myLib->Unload(true); return false; }
217 
218  Decision * d = ep(m_log);
219  if (! d)
220  {
221  TRACE(Error, "Config() decisionlib was not able to create a decision object");
222  return false;
223  }
224  if (params[0])
225  d->ConfigDecision(params);
226 
227  m_decisionpoints.push_back(d);
228  return true;
229 }
230 
231 /* Function: xtrace
232 
233  Purpose: To parse the directive: trace <level>
234  Output: true upon success or false upon failure.
235  */
236 bool Cache::xtrace(XrdOucStream &Config)
237 {
238  char *val;
239  static struct traceopts {const char *opname; int opval; } tropts[] =
240  {
241  {"none", 0},
242  {"error", 1},
243  {"warning", 2},
244  {"info", 3},
245  {"debug", 4},
246  {"dump", 5},
247  {"dumpxl", 6}
248  };
249  int numopts = sizeof(tropts)/sizeof(struct traceopts);
250 
251  if (! (val = Config.GetWord()))
252  {m_log.Emsg("Config", "trace option not specified"); return 1; }
253 
254  for (int i = 0; i < numopts; i++)
255  {
256  if (! strcmp(val, tropts[i].opname))
257  {
258  m_trace->What = tropts[i].opval;
259  return true;
260  }
261  }
262  m_log.Emsg("Config", "invalid trace option -", val);
263  return false;
264 }
265 
266 
267 //______________________________________________________________________________
268 /* Function: Config
269 
270  Purpose: To parse configuration file and configure Cache instance.
271  Output: true upon success or false upon failure.
272  */
273 bool Cache::Config(const char *config_filename, const char *parameters)
274 {
275  // Indicate whether or not we are a client instance
276  const char *theINS = getenv("XRDINSTANCE");
277  m_isClient = (theINS != 0 && strncmp("*client ", theINS, 8) == 0);
278 
279  // Tell everyone else we are a caching proxy
280  XrdOucEnv::Export("XRDPFC", 1);
281 
282  XrdOucEnv myEnv;
283  XrdOucStream Config(&m_log, theINS, &myEnv, "=====> ");
284 
285  if (! config_filename || ! *config_filename)
286  {
287  TRACE(Error, "Config() configuration file not specified.");
288  return false;
289  }
290 
291  int fd;
292  if ( (fd = open(config_filename, O_RDONLY, 0)) < 0)
293  {
294  TRACE( Error, "Config() can't open configuration file " << config_filename);
295  return false;
296  }
297 
298  Config.Attach(fd);
299  static const char *cvec[] = { "*** pfc plugin config:", 0 };
300  Config.Capture(cvec);
301 
302  // Obtain OFS configurator for OSS plugin.
303  XrdOfsConfigPI *ofsCfg = XrdOfsConfigPI::New(config_filename,&Config,&m_log,
304  &XrdVERSIONINFOVAR(XrdOucGetCache));
305  if (! ofsCfg) return false;
306 
307  TmpConfiguration tmpc;
308 
309  // Adjust default parameters for client/serverless caching
310  if (m_isClient)
311  {
312  m_configuration.m_bufferSize = 128 * 1024; // same as normal.
313  m_configuration.m_wqueue_blocks = 8;
314  m_configuration.m_wqueue_threads = 1;
315  }
316 
317  // If network checksum processing is the default, indicate so.
318  if (m_configuration.is_cschk_net()) m_env->Put("psx.CSNet", m_configuration.m_cs_ChkTLS ? "2" : "1");
319 
320  // Actual parsing of the config file.
321  bool retval = true, aOK = true;
322  char *var;
323  while ((var = Config.GetMyFirstWord()))
324  {
325  if (! strcmp(var,"pfc.osslib"))
326  {
327  retval = ofsCfg->Parse(XrdOfsConfigPI::theOssLib);
328  }
329  else if (! strcmp(var,"pfc.cschk"))
330  {
331  retval = xcschk(Config);
332  }
333  else if (! strcmp(var,"pfc.decisionlib"))
334  {
335  retval = xdlib(Config);
336  }
337  else if (! strcmp(var,"pfc.trace"))
338  {
339  retval = xtrace(Config);
340  }
341  else if (! strcmp(var,"pfc.allow_xrdpfc_command"))
342  {
343  m_configuration.m_allow_xrdpfc_command = true;
344  }
345  else if (! strncmp(var,"pfc.", 4))
346  {
347  retval = ConfigParameters(std::string(var+4), Config, tmpc);
348  }
349 
350  if ( ! retval)
351  {
352  TRACE(Error, "Config() error in parsing");
353  aOK = false;
354  }
355  }
356 
357  Config.Close();
358 
359  // Load OSS plugin.
360  myEnv.Put("oss.runmode", "pfc");
361  if (m_configuration.is_cschk_cache())
362  {
363  char csi_conf[128];
364  if (snprintf(csi_conf, 128, "space=%s nofill", m_configuration.m_meta_space.c_str()) < 128)
365  {
366  ofsCfg->Push(XrdOfsConfigPI::theOssLib, "libXrdOssCsi.so", csi_conf);
367  } else {
368  TRACE(Error, "Config() buffer too small for libXrdOssCsi params.");
369  return false;
370  }
371  }
372  if (ofsCfg->Load(XrdOfsConfigPI::theOssLib, &myEnv))
373  {
374  ofsCfg->Plugin(m_oss);
375  }
376  else
377  {
378  TRACE(Error, "Config() Unable to create an OSS object");
379  return false;
380  }
381 
382  // sets default value for disk usage
383  XrdOssVSInfo sP;
384  {
385  if (m_oss->StatVS(&sP, m_configuration.m_data_space.c_str(), 1) < 0)
386  {
387  m_log.Emsg("ConfigParameters()", "error obtaining stat info for data space ", m_configuration.m_data_space.c_str());
388  return false;
389  }
390  if (sP.Total < 10ll << 20)
391  {
392  m_log.Emsg("ConfigParameters()", "available data space is less than 10 MB (can be due to a mistake in oss.localroot directive) for space ",
393  m_configuration.m_data_space.c_str());
394  return false;
395  }
396 
397  m_configuration.m_diskTotalSpace = sP.Total;
398 
399  if (cfg2bytes(tmpc.m_diskUsageLWM, m_configuration.m_diskUsageLWM, sP.Total, "lowWatermark") &&
400  cfg2bytes(tmpc.m_diskUsageHWM, m_configuration.m_diskUsageHWM, sP.Total, "highWatermark"))
401  {
402  if (m_configuration.m_diskUsageLWM >= m_configuration.m_diskUsageHWM) {
403  m_log.Emsg("ConfigParameters()", "pfc.diskusage should have lowWatermark < highWatermark.");
404  aOK = false;
405  }
406  }
407  else aOK = false;
408 
409  if ( ! tmpc.m_fileUsageMax.empty())
410  {
411  if (cfg2bytes(tmpc.m_fileUsageBaseline, m_configuration.m_fileUsageBaseline, sP.Total, "files baseline") &&
412  cfg2bytes(tmpc.m_fileUsageNominal, m_configuration.m_fileUsageNominal, sP.Total, "files nominal") &&
413  cfg2bytes(tmpc.m_fileUsageMax, m_configuration.m_fileUsageMax, sP.Total, "files max"))
414  {
415  if (m_configuration.m_fileUsageBaseline >= m_configuration.m_fileUsageNominal ||
416  m_configuration.m_fileUsageBaseline >= m_configuration.m_fileUsageMax ||
417  m_configuration.m_fileUsageNominal >= m_configuration.m_fileUsageMax)
418  {
419  m_log.Emsg("ConfigParameters()", "pfc.diskusage files should have baseline < nominal < max.");
420  aOK = false;
421  }
422  }
423  else aOK = false;
424  }
425  }
426  // sets flush frequency
427  if ( ! tmpc.m_flushRaw.empty())
428  {
429  if (::isalpha(*(tmpc.m_flushRaw.rbegin())))
430  {
431  if (XrdOuca2x::a2sz(m_log, "Error getting number of bytes written before flush", tmpc.m_flushRaw.c_str(),
432  &m_configuration.m_flushCnt,
433  100 * m_configuration.m_bufferSize , 100000 * m_configuration.m_bufferSize))
434  {
435  return false;
436  }
437  m_configuration.m_flushCnt /= m_configuration.m_bufferSize;
438  }
439  else
440  {
441  if (XrdOuca2x::a2ll(m_log, "Error getting number of blocks written before flush", tmpc.m_flushRaw.c_str(),
442  &m_configuration.m_flushCnt, 100, 100000))
443  {
444  return false;
445  }
446  }
447  }
448 
449  // get number of available RAM blocks after process configuration
450  if (m_configuration.m_RamAbsAvailable == 0)
451  {
452  m_configuration.m_RamAbsAvailable = m_isClient ? 256ll * 1024 * 1024 : 1024ll * 1024 * 1024;
453  char buff[1024];
454  snprintf(buff, sizeof(buff), "RAM usage pfc.ram is not specified. Default value %s is used.", m_isClient ? "256m" : "1g");
455  m_log.Say("Config info: ", buff);
456  }
457  // Setup number of standard-size blocks not released back to the system to 5% of total RAM.
458  m_configuration.m_RamKeepStdBlocks = (m_configuration.m_RamAbsAvailable / m_configuration.m_bufferSize + 1) * 5 / 100;
459 
460 
461  // Set tracing to debug if this is set in environment
462  char* cenv = getenv("XRDDEBUG");
463  if (cenv && ! strcmp(cenv,"1") && m_trace->What < 4) m_trace->What = 4;
464 
465  if (aOK)
466  {
467  int loff = 0;
468 // 000 001 010
469  const char *csc[] = {"off", "cache nonet", "nocache net notls",
470 // 011
471  "cache net notls",
472 // 100 101 110
473  "off", "cache nonet", "nocache net tls",
474 // 111
475  "cache net tls"};
476  char buff[8192], uvk[32];
477  if (m_configuration.m_cs_UVKeep < 0)
478  strcpy(uvk, "lru");
479  else
480  sprintf(uvk, "%lld", (long long) m_configuration.m_cs_UVKeep);
481  float rg = (m_configuration.m_RamAbsAvailable) / float(1024*1024*1024);
482  loff = snprintf(buff, sizeof(buff), "Config effective %s pfc configuration:\n"
483  " pfc.cschk %s uvkeep %s\n"
484  " pfc.blocksize %lld\n"
485  " pfc.prefetch %d\n"
486  " pfc.ram %.fg\n"
487  " pfc.writequeue %d %d\n"
488  " # Total available disk: %lld\n"
489  " pfc.diskusage %lld %lld files %lld %lld %lld purgeinterval %d purgecoldfiles %d\n"
490  " pfc.spaces %s %s\n"
491  " pfc.trace %d\n"
492  " pfc.flush %lld\n"
493  " pfc.acchistorysize %d\n"
494  " pfc.onlyIfCachedMinBytes %lld\n"
495  " pfc.onlyIfCachedMinFrac %.2f\n",
496  config_filename,
497  csc[int(m_configuration.m_cs_Chk)], uvk,
498  m_configuration.m_bufferSize,
499  m_configuration.m_prefetch_max_blocks,
500  rg,
501  m_configuration.m_wqueue_blocks, m_configuration.m_wqueue_threads,
502  sP.Total,
503  m_configuration.m_diskUsageLWM, m_configuration.m_diskUsageHWM,
504  m_configuration.m_fileUsageBaseline, m_configuration.m_fileUsageNominal, m_configuration.m_fileUsageMax,
505  m_configuration.m_purgeInterval, m_configuration.m_purgeColdFilesAge,
506  m_configuration.m_data_space.c_str(),
507  m_configuration.m_meta_space.c_str(),
508  m_trace->What,
509  m_configuration.m_flushCnt,
510  m_configuration.m_accHistorySize,
511  m_configuration.m_onlyIfCachedMinSize,
512  m_configuration.m_onlyIfCachedMinFrac);
513 
514  if (m_configuration.is_dir_stat_reporting_on())
515  {
516  loff += snprintf(buff + loff, sizeof(buff) - loff,
517  " pfc.dirstats maxdepth %d ((internal: store_depth %d, size_of_dirlist %d, size_of_globlist %d))\n",
518  m_configuration.m_dirStatsMaxDepth, m_configuration.m_dirStatsStoreDepth,
519  (int) m_configuration.m_dirStatsDirs.size(), (int) m_configuration.m_dirStatsDirGlobs.size());
520  loff += snprintf(buff + loff, sizeof(buff) - loff, " dirlist:\n");
521  for (std::set<std::string>::iterator i = m_configuration.m_dirStatsDirs.begin(); i != m_configuration.m_dirStatsDirs.end(); ++i)
522  loff += snprintf(buff + loff, sizeof(buff) - loff, " %s\n", i->c_str());
523  loff += snprintf(buff + loff, sizeof(buff) - loff, " globlist:\n");
524  for (std::set<std::string>::iterator i = m_configuration.m_dirStatsDirGlobs.begin(); i != m_configuration.m_dirStatsDirGlobs.end(); ++i)
525  loff += snprintf(buff + loff, sizeof(buff) - loff, " %s/*\n", i->c_str());
526  }
527 
528  if (m_configuration.m_hdfsmode)
529  {
530  loff += snprintf(buff + loff, sizeof(buff) - loff, " pfc.hdfsmode hdfsbsize %lld\n", m_configuration.m_hdfsbsize);
531  }
532 
533  if (m_configuration.m_username.empty())
534  {
535  char unameBuff[256];
536  XrdOucUtils::UserName(getuid(), unameBuff, sizeof(unameBuff));
537  m_configuration.m_username = unameBuff;
538  }
539  else
540  {
541  loff += snprintf(buff + loff, sizeof(buff) - loff, " pfc.user %s\n", m_configuration.m_username.c_str());
542  }
543 
544  m_log.Say(buff);
545 
546  m_env->Put("XRDPFC.SEGSIZE", std::to_string(m_configuration.m_bufferSize).c_str());
547  }
548 
549  // Derived settings
550  m_prefetch_enabled = m_configuration.m_prefetch_max_blocks > 0;
551  Info::s_maxNumAccess = m_configuration.m_accHistorySize;
552 
553  m_gstream = (XrdXrootdGStream*) m_env->GetPtr("pfc.gStream*");
554 
555  m_log.Say("Config Proxy File Cache g-stream has", m_gstream ? "" : " NOT", " been configured via xrootd.monitor directive");
556 
557  m_log.Say("------ Proxy File Cache configuration parsing ", aOK ? "completed" : "failed");
558 
559  if (ofsCfg) delete ofsCfg;
560 
561  // XXXX-CKSUM Testing. To be removed after OssPgi is also merged and valildated.
562  // Building of xrdpfc_print fails when this is enabled.
563 #ifdef XRDPFC_CKSUM_TEST
564  {
565  int xxx = m_configuration.m_cs_Chk;
566 
567  for (m_configuration.m_cs_Chk = CSChk_None; m_configuration.m_cs_Chk <= CSChk_Both; ++m_configuration.m_cs_Chk)
568  {
569  Info::TestCksumStuff();
570  }
571 
572  m_configuration.m_cs_Chk = xxx;
573  }
574 #endif
575 
576  return aOK;
577 }
578 
579 //------------------------------------------------------------------------------
580 
581 bool Cache::ConfigParameters(std::string part, XrdOucStream& config, TmpConfiguration &tmpc)
582 {
583  struct ConfWordGetter
584  {
585  XrdOucStream &m_config;
586  char *m_last_word;
587 
588  ConfWordGetter(XrdOucStream& c) : m_config(c), m_last_word((char*)1) {}
589 
590  const char* GetWord() { if (HasLast()) m_last_word = m_config.GetWord(); return HasLast() ? m_last_word : ""; }
591  bool HasLast() { return (m_last_word != 0); }
592  };
593 
594  ConfWordGetter cwg(config);
595 
596  XrdSysError err(0, "");
597  if ( part == "user" )
598  {
599  m_configuration.m_username = cwg.GetWord();
600  if ( ! cwg.HasLast())
601  {
602  m_log.Emsg("Config", "Error: pfc.user requires a parameter.");
603  return false;
604  }
605  }
606  else if ( part == "diskusage" )
607  {
608  tmpc.m_diskUsageLWM = cwg.GetWord();
609  tmpc.m_diskUsageHWM = cwg.GetWord();
610 
611  if (tmpc.m_diskUsageHWM.empty())
612  {
613  m_log.Emsg("Config", "Error: pfc.diskusage parameter requires at least two arguments.");
614  return false;
615  }
616 
617  const char *p = 0;
618  while ((p = cwg.GetWord()) && cwg.HasLast())
619  {
620  if (strcmp(p, "files") == 0)
621  {
622  tmpc.m_fileUsageBaseline = cwg.GetWord();
623  tmpc.m_fileUsageNominal = cwg.GetWord();
624  tmpc.m_fileUsageMax = cwg.GetWord();
625 
626  if ( ! cwg.HasLast())
627  {
628  m_log.Emsg("Config", "Error: pfc.diskusage files directive requires three arguments.");
629  return false;
630  }
631  }
632  else if (strcmp(p, "sleep") == 0 || strcmp(p, "purgeinterval") == 0)
633  {
634  if (strcmp(p, "sleep") == 0) m_log.Emsg("Config", "warning sleep directive is deprecated in pfc.diskusage. Please use purgeinterval instead.");
635 
636  if (XrdOuca2x::a2tm(m_log, "Error getting purgeinterval", cwg.GetWord(), &m_configuration.m_purgeInterval, 60, 3600))
637  {
638  return false;
639  }
640  }
641  else if (strcmp(p, "purgecoldfiles") == 0)
642  {
643  if (XrdOuca2x::a2tm(m_log, "Error getting purgecoldfiles age", cwg.GetWord(), &m_configuration.m_purgeColdFilesAge, 3600, 3600*24*360))
644  {
645  return false;
646  }
647  if (XrdOuca2x::a2i(m_log, "Error getting purgecoldfiles period", cwg.GetWord(), &m_configuration.m_purgeAgeBasedPeriod, 1, 1000))
648  {
649  return false;
650  }
651  }
652  else
653  {
654  m_log.Emsg("Config", "Error: diskusage stanza contains unknown directive", p);
655  }
656  }
657  }
658  else if ( part == "acchistorysize" )
659  {
660  if ( XrdOuca2x::a2i(m_log, "Error getting access-history-size", cwg.GetWord(), &m_configuration.m_accHistorySize, 20, 200))
661  {
662  return false;
663  }
664  }
665  else if ( part == "dirstats" )
666  {
667  const char *p = 0;
668  while ((p = cwg.GetWord()) && cwg.HasLast())
669  {
670  if (strcmp(p, "maxdepth") == 0)
671  {
672  if (XrdOuca2x::a2i(m_log, "Error getting maxdepth value", cwg.GetWord(), &m_configuration.m_dirStatsMaxDepth, 0, 16))
673  {
674  return false;
675  }
676  m_configuration.m_dirStatsStoreDepth = std::max(m_configuration.m_dirStatsStoreDepth, m_configuration.m_dirStatsMaxDepth);
677  }
678  else if (strcmp(p, "dir") == 0)
679  {
680  p = cwg.GetWord();
681  if (p && p[0] == '/')
682  {
683  // XXX -- should we just store them as sets of PathTokenizer objects, not strings?
684 
685  char d[1024]; d[0] = 0;
686  int depth = 0;
687  { // Compress multiple slashes and "measure" depth
688  const char *pp = p;
689  char *pd = d;
690  *(pd++) = *(pp++);
691  while (*pp != 0)
692  {
693  if (*(pd - 1) == '/')
694  {
695  if (*pp == '/')
696  {
697  ++pp; continue;
698  }
699  ++depth;
700  }
701  *(pd++) = *(pp++);
702  }
703  *(pd--) = 0;
704  // remove trailing but but not leading /
705  if (*pd == '/' && pd != d) *pd = 0;
706  }
707  int ld = strlen(d);
708  if (ld >= 2 && d[ld-1] == '*' && d[ld-2] == '/')
709  {
710  d[ld-2] = 0;
711  ld -= 2;
712  m_configuration.m_dirStatsDirGlobs.insert(d);
713  printf("Glob %s -> %s -- depth = %d\n", p, d, depth);
714  }
715  else
716  {
717  m_configuration.m_dirStatsDirs.insert(d);
718  printf("Dir %s -> %s -- depth = %d\n", p, d, depth);
719  }
720 
721  m_configuration.m_dirStatsStoreDepth = std::max(m_configuration.m_dirStatsStoreDepth, depth);
722  }
723  else
724  {
725  m_log.Emsg("Config", "Error: dirstats dir parameter requires a directory argument starting with a '/'.");
726  return false;
727  }
728  }
729  else
730  {
731  m_log.Emsg("Config", "Error: dirstats stanza contains unknown directive '", p, "'");
732  return false;
733  }
734  }
735  }
736  else if ( part == "blocksize" )
737  {
738  long long minBSize = 4 * 1024;
739  long long maxBSize = 512 * 1024 * 1024;
740  if (XrdOuca2x::a2sz(m_log, "Error reading block-size", cwg.GetWord(), &m_configuration.m_bufferSize, minBSize, maxBSize))
741  {
742  return false;
743  }
744  if (m_configuration.m_bufferSize & 0xFFF)
745  {
746  m_configuration.m_bufferSize &= ~0x0FFF;
747  m_configuration.m_bufferSize += 0x1000;
748  m_log.Emsg("Config", "pfc.blocksize must be a multiple of 4 kB. Rounded up.");
749  }
750  }
751  else if ( part == "prefetch" || part == "nramprefetch" )
752  {
753  if (part == "nramprefetch")
754  {
755  m_log.Emsg("Config", "pfc.nramprefetch is deprecated, please use pfc.prefetch instead. Replacing the directive internally.");
756  }
757 
758  if (XrdOuca2x::a2i(m_log, "Error setting prefetch block count", cwg.GetWord(), &m_configuration.m_prefetch_max_blocks, 0, 128))
759  {
760  return false;
761  }
762 
763  }
764  else if ( part == "nramread" )
765  {
766  m_log.Emsg("Config", "pfc.nramread is deprecated, please use pfc.ram instead. Ignoring this directive.");
767  cwg.GetWord(); // Ignoring argument.
768  }
769  else if ( part == "ram" )
770  {
771  long long minRAM = m_isClient ? 256 * 1024 * 1024 : 1024 * 1024 * 1024;
772  long long maxRAM = 256 * minRAM;
773  if ( XrdOuca2x::a2sz(m_log, "get RAM available", cwg.GetWord(), &m_configuration.m_RamAbsAvailable, minRAM, maxRAM))
774  {
775  return false;
776  }
777  }
778  else if ( part == "writequeue")
779  {
780  if (XrdOuca2x::a2i(m_log, "Error getting pfc.writequeue num-blocks", cwg.GetWord(), &m_configuration.m_wqueue_blocks, 1, 1024))
781  {
782  return false;
783  }
784  if (XrdOuca2x::a2i(m_log, "Error getting pfc.writequeue num-threads", cwg.GetWord(), &m_configuration.m_wqueue_threads, 1, 64))
785  {
786  return false;
787  }
788  }
789  else if ( part == "spaces" )
790  {
791  m_configuration.m_data_space = cwg.GetWord();
792  m_configuration.m_meta_space = cwg.GetWord();
793  if ( ! cwg.HasLast())
794  {
795  m_log.Emsg("Config", "spacenames requires two parameters: <data-space> <metadata-space>.");
796  return false;
797  }
798  }
799  else if ( part == "hdfsmode" )
800  {
801  m_log.Emsg("Config", "pfc.hdfsmode is currently unsupported.");
802  return false;
803 
804  m_configuration.m_hdfsmode = true;
805 
806  const char* params = cwg.GetWord();
807  if (params)
808  {
809  if (! strncmp("hdfsbsize", params, 9))
810  {
811  long long minBlSize = 32 * 1024;
812  long long maxBlSize = 128 * 1024 * 1024;
813  if ( XrdOuca2x::a2sz(m_log, "Error getting file fragment size", cwg.GetWord(), &m_configuration.m_hdfsbsize, minBlSize, maxBlSize))
814  {
815  return false;
816  }
817  }
818  else
819  {
820  m_log.Emsg("Config", "Error setting the fragment size parameter name");
821  return false;
822  }
823  }
824  }
825  else if ( part == "flush" )
826  {
827  tmpc.m_flushRaw = cwg.GetWord();
828  if ( ! cwg.HasLast())
829  {
830  m_log.Emsg("Config", "Error: pfc.flush requires a parameter.");
831  return false;
832  }
833  }
834  else if ( part == "onlyifcached" )
835  {
836  const char *p = 0;
837  while ((p = cwg.GetWord()) && cwg.HasLast())
838  {
839  if (strcmp(p, "minsize") == 0)
840  {
841  std::string minBytes = cwg.GetWord();
842  long long minBytesTop = 1024 * 1024 * 1024;
843  if (::isalpha(*(minBytes.rbegin())))
844  {
845  if (XrdOuca2x::a2sz(m_log, "Error in parsing minsize value for onlyifcached parameter", minBytes.c_str(), &m_configuration.m_onlyIfCachedMinSize, 0, minBytesTop))
846  {
847  return false;
848  }
849  }
850  else
851  {
852  if (XrdOuca2x::a2ll(m_log, "Error in parsing numeric minsize value for onlyifcached parameter", minBytes.c_str(),&m_configuration.m_onlyIfCachedMinSize, 0, minBytesTop))
853  {
854  return false;
855  }
856  }
857  }
858  if (strcmp(p, "minfrac") == 0)
859  {
860  std::string minFrac = cwg.GetWord();
861  char *eP;
862  errno = 0;
863  double frac = strtod(minFrac.c_str(), &eP);
864  if (errno || eP == minFrac.c_str())
865  {
866  m_log.Emsg("Config", "Error setting fraction for only-if-cached directive");
867  return false;
868  }
869  m_configuration.m_onlyIfCachedMinFrac = frac;
870  }
871  else
872  {
873  m_log.Emsg("Config", "Error: onlyifcached stanza contains unknown directive", p);
874  }
875  }
876  }
877  else
878  {
879  m_log.Emsg("ConfigParameters() unmatched pfc parameter", part.c_str());
880  return false;
881  }
882 
883  return true;
884 }
XrdVERSIONINFO(XrdOucGetCache, XrdPfc)
XrdOucCache * XrdOucGetCache(XrdSysLogger *logger, const char *config_filename, const char *parameters, XrdOucEnv *env)
Definition: XrdPfc.cc:81
int open(const char *path, int oflag,...)
int isNo(int dflt, const char *Msg1, const char *Msg2, const char *Msg3)
if(Avsz)
#define TRACE(act, x)
Definition: XrdTrace.hh:63
bool Parse(TheLib what)
bool Plugin(XrdAccAuthorize *&piP)
Get Authorization plugin.
static XrdOfsConfigPI * New(const char *cfn, XrdOucStream *cfgP, XrdSysError *errP, XrdVersionInfo *verP=0, XrdSfsFileSystem *sfsP=0)
bool Load(int what, XrdOucEnv *envP=0)
bool Push(TheLib what, const char *plugP, const char *parmP=0)
@ theOssLib
Oss plugin.
long long Total
Definition: XrdOssVS.hh:90
virtual int StatVS(XrdOssVSInfo *vsP, const char *sname=0, int updt=0)
Definition: XrdOss.cc:117
static int Export(const char *Var, const char *Val)
Definition: XrdOucEnv.cc:170
void * GetPtr(const char *varname)
Definition: XrdOucEnv.cc:263
void Put(const char *varname, const char *value)
Definition: XrdOucEnv.hh:85
void * Resolve(const char *symbl, int mcnt=1)
void Unload(bool dodel=false)
char * GetWord(int lowcase=0)
static int UserName(uid_t uID, char *uName, int uNsz)
static int a2i(XrdSysError &, const char *emsg, const char *item, int *val, int minv=-1, int maxv=-1)
Definition: XrdOuca2x.cc:45
static int a2sz(XrdSysError &, const char *emsg, const char *item, long long *val, long long minv=-1, long long maxv=-1)
Definition: XrdOuca2x.cc:257
static int a2ll(XrdSysError &, const char *emsg, const char *item, long long *val, long long minv=-1, long long maxv=-1)
Definition: XrdOuca2x.cc:70
static int a2tm(XrdSysError &, const char *emsg, const char *item, int *val, int minv=-1, int maxv=-1)
Definition: XrdOuca2x.cc:288
bool Config(const char *config_filename, const char *parameters)
Parse configuration file.
Base class for selecting which files should be cached.
virtual bool ConfigDecision(const char *params)
Status of cached file. Can be read from and written into a binary file.
Definition: XrdPfcInfo.hh:45
static size_t s_maxNumAccess
Definition: XrdPfcInfo.hh:315
int Emsg(const char *esfx, int ecode, const char *text1, const char *text2=0)
Definition: XrdSysError.cc:95
void Say(const char *text1, const char *text2=0, const char *txt3=0, const char *text4=0, const char *text5=0, const char *txt6=0)
Definition: XrdSysError.cc:141
XrdCmsConfig Config
Definition: XrdPfc.hh:41
@ CSChk_Both
Definition: XrdPfcTypes.hh:23
@ CSChk_Net
Definition: XrdPfcTypes.hh:23
@ CSChk_TLS
Definition: XrdPfcTypes.hh:24
@ CSChk_Cache
Definition: XrdPfcTypes.hh:23
@ CSChk_None
Definition: XrdPfcTypes.hh:23
long long m_hdfsbsize
used with m_hdfsmode, default 128MB
Definition: XrdPfc.hh:108
long long m_RamAbsAvailable
available from configuration
Definition: XrdPfc.hh:102
long long m_flushCnt
nuber of unsynced blcoks on disk before flush is called
Definition: XrdPfc.hh:109
int m_accHistorySize
max number of entries in access history part of cinfo file
Definition: XrdPfc.hh:94
int m_wqueue_threads
number of threads writing blocks to disk
Definition: XrdPfc.hh:105
long long m_diskTotalSpace
total disk space on configured partition or oss space
Definition: XrdPfc.hh:85
long long m_fileUsageMax
cache purge - files usage maximum
Definition: XrdPfc.hh:90
long long m_fileUsageBaseline
cache purge - files usage baseline
Definition: XrdPfc.hh:88
int m_dirStatsStoreDepth
depth to which statistics should be collected
Definition: XrdPfc.hh:99
bool m_allow_xrdpfc_command
flag for enabling access to /xrdpfc-command/ functionality.
Definition: XrdPfc.hh:79
long long m_diskUsageHWM
cache purge - disk usage high water mark
Definition: XrdPfc.hh:87
bool is_cschk_cache() const
Definition: XrdPfc.hh:69
std::set< std::string > m_dirStatsDirGlobs
directory globs for which stat reporting was requested
Definition: XrdPfc.hh:97
int m_prefetch_max_blocks
maximum number of blocks to prefetch per file
Definition: XrdPfc.hh:106
bool m_cs_ChkTLS
Allow TLS.
Definition: XrdPfc.hh:113
long long m_fileUsageNominal
cache purge - files usage nominal
Definition: XrdPfc.hh:89
int m_cs_Chk
Checksum check.
Definition: XrdPfc.hh:112
int m_purgeAgeBasedPeriod
peform cold file / uvkeep purge every this many purge cycles
Definition: XrdPfc.hh:93
bool m_hdfsmode
flag for enabling block-level operation
Definition: XrdPfc.hh:78
int m_purgeColdFilesAge
purge files older than this age
Definition: XrdPfc.hh:92
std::string m_data_space
oss space for data files
Definition: XrdPfc.hh:82
std::set< std::string > m_dirStatsDirs
directories for which stat reporting was requested
Definition: XrdPfc.hh:96
long long m_diskUsageLWM
cache purge - disk usage low water mark
Definition: XrdPfc.hh:86
int m_RamKeepStdBlocks
number of standard-sized blocks kept after release
Definition: XrdPfc.hh:103
long long m_bufferSize
prefetch buffer size, default 1MB
Definition: XrdPfc.hh:101
std::string m_meta_space
oss space for metadata files (cinfo)
Definition: XrdPfc.hh:83
int m_wqueue_blocks
maximum number of blocks written per write-queue loop
Definition: XrdPfc.hh:104
std::string m_username
username passed to oss plugin
Definition: XrdPfc.hh:81
bool is_cschk_net() const
Definition: XrdPfc.hh:70
double m_onlyIfCachedMinFrac
minimum fraction of downloaded file, used by only-if-cached CGI option
Definition: XrdPfc.hh:116
time_t m_cs_UVKeep
unverified checksum cache keep
Definition: XrdPfc.hh:111
int m_dirStatsMaxDepth
maximum depth for statistics write out
Definition: XrdPfc.hh:98
int m_purgeInterval
sleep interval between cache purges
Definition: XrdPfc.hh:91
long long m_onlyIfCachedMinSize
minumum size of downloaded file, used by only-if-cached CGI option
Definition: XrdPfc.hh:115
bool is_dir_stat_reporting_on() const
Definition: XrdPfc.hh:62
std::string m_diskUsageLWM
Definition: XrdPfc.hh:123
std::string m_diskUsageHWM
Definition: XrdPfc.hh:124
std::string m_fileUsageBaseline
Definition: XrdPfc.hh:125
std::string m_fileUsageNominal
Definition: XrdPfc.hh:126
std::string m_flushRaw
Definition: XrdPfc.hh:128
std::string m_fileUsageMax
Definition: XrdPfc.hh:127