XRootD
XrdPfc::IOFileBlock Class Reference

Downloads original file into multiple files, chunked into blocks. Only blocks that are asked for are downloaded. Handles read requests as they come along. More...

#include <XrdPfcIOFileBlock.hh>

+ Inheritance diagram for XrdPfc::IOFileBlock:
+ Collaboration diagram for XrdPfc::IOFileBlock:

Public Member Functions

 IOFileBlock (XrdOucCacheIO *io, Cache &cache)
 
 ~IOFileBlock ()
 
void DetachFinalize () override
 Abstract virtual method of XrdPfc::IO Called to destruct the IO object after it is no longer used. More...
 
long long FSize () override
 
int Fstat (struct stat &sbuff) override
 
bool ioActive () override
 Abstract virtual method of XrdPfc::IO Called to check if destruction needs to be done in a separate task. More...
 
virtual int Read (char *buff, long long offs, int rlen)=0
 Pass Read request to the corresponding File object. More...
 
int Read (char *Buffer, long long Offset, int Length) override
 
virtual void Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int rlen)
 Pass Read request to the corresponding File object. More...
 
void Update (XrdOucCacheIO &iocp) override
 
- Public Member Functions inherited from XrdPfc::IO
 IO (XrdOucCacheIO *io, Cache &cache)
 
virtual XrdOucCacheIOBase ()
 Original data source. More...
 
bool Detach (XrdOucCacheIOCD &iocdP) final
 
XrdOucCacheIOGetInput ()
 
const char * GetLocation ()
 
XrdSysTraceGetTrace ()
 
const char * Path () override
 Original data source URL. More...
 
int Sync () override
 
virtual int Sync ()=0
 
virtual void Sync (XrdOucCacheIOCB &iocb)
 
virtual int Trunc (long long offs)=0
 
int Trunc (long long Offset) override
 
virtual void Trunc (XrdOucCacheIOCB &iocb, long long offs)
 
virtual int Write (char *buff, long long offs, int wlen)=0
 
int Write (char *Buffer, long long Offset, int Length) override
 
virtual void Write (XrdOucCacheIOCB &iocb, char *buff, long long offs, int wlen)
 
- Public Member Functions inherited from XrdOucCacheIO
 XrdOucCacheIO ()
 Construct and Destructor. More...
 
virtual const char * Location (bool refresh=false)
 
virtual int pgRead (char *buff, long long offs, int rdlen, std::vector< uint32_t > &csvec, uint64_t opts=0, int *csfix=0)
 
virtual void pgRead (XrdOucCacheIOCB &iocb, char *buff, long long offs, int rdlen, std::vector< uint32_t > &csvec, uint64_t opts=0, int *csfix=0)
 
virtual int pgWrite (char *buff, long long offs, int wrlen, std::vector< uint32_t > &csvec, uint64_t opts=0, int *csfix=0)
 
virtual void pgWrite (XrdOucCacheIOCB &iocb, char *buff, long long offs, int wrlen, std::vector< uint32_t > &csvec, uint64_t opts=0, int *csfix=0)
 
virtual void Preread (aprParms &Parms)
 
virtual void Preread (long long offs, int rlen, int opts=0)
 
virtual void Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int rlen)
 
virtual int ReadV (const XrdOucIOVec *readV, int rnum)
 
virtual void ReadV (XrdOucCacheIOCB &iocb, const XrdOucIOVec *readV, int rnum)
 
virtual void Sync (XrdOucCacheIOCB &iocb)
 
virtual void Trunc (XrdOucCacheIOCB &iocb, long long offs)
 
virtual void Write (XrdOucCacheIOCB &iocb, char *buff, long long offs, int wlen)
 
virtual int WriteV (const XrdOucIOVec *writV, int wnum)
 
virtual void WriteV (XrdOucCacheIOCB &iocb, const XrdOucIOVec *writV, int wnum)
 

Additional Inherited Members

- Static Public Attributes inherited from XrdOucCacheIO
static const uint64_t forceCS = 0x0000000000000001ULL
 
static const int SingleUse = 0x0001
 Mark pages for single use. More...
 
- Protected Member Functions inherited from XrdPfc::IO
std::string GetFilename ()
 
const char * GetPath ()
 
unsigned short ObtainReadSid ()
 
const char * RefreshLocation ()
 
bool register_block_error (int res)
 
bool register_incomplete_read ()
 
- Protected Member Functions inherited from XrdOucCacheIO
virtual ~XrdOucCacheIO ()
 
- Protected Attributes inherited from XrdPfc::IO
RAtomic_int m_active_read_reqs
 number of active read requests More...
 
Cachem_cache
 reference to Cache object More...
 
std::map< int, int > m_error_counts
 
int m_incomplete_count {0}
 
const char * m_traceID
 

Detailed Description

Downloads original file into multiple files, chunked into blocks. Only blocks that are asked for are downloaded. Handles read requests as they come along.

Definition at line 38 of file XrdPfcIOFileBlock.hh.

Constructor & Destructor Documentation

◆ IOFileBlock()

IOFileBlock::IOFileBlock ( XrdOucCacheIO io,
Cache cache 
)

Definition at line 39 of file XrdPfcIOFileBlock.cc.

39  :
40  IO(io, cache), m_localStat(0), m_info(cache.GetTrace(), false), m_info_file(0)
41 {
43  GetBlockSizeFromPath();
44  initLocalStat();
45 }
XrdSysTrace * GetTrace()
Definition: XrdPfc.hh:402
const Configuration & RefConfiguration() const
Reference XrdPfc configuration.
Definition: XrdPfc.hh:319
static Cache & GetInstance()
Singleton access.
Definition: XrdPfc.cc:163
IO(XrdOucCacheIO *io, Cache &cache)
Definition: XrdPfcIO.cc:6
long long m_hdfsbsize
used with m_hdfsmode, default 128MB
Definition: XrdPfc.hh:108

References XrdPfc::Cache::GetInstance(), XrdPfc::Configuration::m_hdfsbsize, and XrdPfc::Cache::RefConfiguration().

+ Here is the call graph for this function:

◆ ~IOFileBlock()

IOFileBlock::~IOFileBlock ( )

Definition at line 48 of file XrdPfcIOFileBlock.cc.

49 {
50  // called from Detach() if no sync is needed or
51  // from Cache's sync thread
52 
53  TRACEIO(Debug, "deleting IOFileBlock");
54 }
#define TRACEIO(act, x)
Definition: XrdPfcTrace.hh:63

References Macaroons::Debug, and TRACEIO.

Member Function Documentation

◆ DetachFinalize()

void IOFileBlock::DetachFinalize ( )
overridevirtual

Abstract virtual method of XrdPfc::IO Called to destruct the IO object after it is no longer used.

Implements XrdPfc::IO.

Definition at line 101 of file XrdPfcIOFileBlock.cc.

102 {
103  // Effectively a destructor.
104 
105  TRACEIO(Info, "DetachFinalize() " << this);
106 
107  CloseInfoFile();
108  {
109  XrdSysMutexHelper lock(&m_mutex);
110  for (std::map<int, File*>::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it)
111  {
112  if (it->second)
113  {
114  it->second->RequestSyncOfDetachStats();
115  m_cache.ReleaseFile(it->second, this);
116  }
117  }
118  }
119 
120  delete this;
121 }
void ReleaseFile(File *, IO *)
Definition: XrdPfc.cc:497
Cache & m_cache
reference to Cache object
Definition: XrdPfcIO.hh:52
Status of cached file. Can be read from and written into a binary file.
Definition: XrdPfcInfo.hh:45

References XrdPfc::IO::m_cache, XrdPfc::Cache::ReleaseFile(), and TRACEIO.

+ Here is the call graph for this function:

◆ FSize()

long long IOFileBlock::FSize ( )
overridevirtual

Obtain size of the file.

Returns
Size of the file in bytes.

Implements XrdOucCacheIO.

Definition at line 206 of file XrdPfcIOFileBlock.cc.

207 {
208  if ( ! m_localStat) return -ENOENT;
209 
210  return m_localStat->st_size;
211 }

Referenced by Read().

+ Here is the caller graph for this function:

◆ Fstat()

int IOFileBlock::Fstat ( struct stat sbuff)
overridevirtual

Perform an fstat() operation (defaults to passthrough).

Parameters
sbuffreference to the stat buffer to be filled in. Only fields st_size, st_blocks, st_mtime (st_atime and st_ctime may be set to st_mtime), st_ino, and st_mode need to be set. All other fields are preset and should not be changed.
Returns
<0 - fstat failed, value is -errno. =0 - fstat succeeded, sbuff holds stat information. >0 - fstat could not be done, forward operation to next level.

Reimplemented from XrdOucCacheIO.

Definition at line 194 of file XrdPfcIOFileBlock.cc.

195 {
196  // local stat is create in constructor. if file was on disk before
197  // attach that the only way stat was not successful is becuse there
198  // were info file read errors
199  if ( ! m_localStat) return -ENOENT;
200 
201  memcpy(&sbuff, m_localStat, sizeof(struct stat));
202  return 0;
203 }
int stat(const char *path, struct stat *buf)

References stat().

+ Here is the call graph for this function:

◆ ioActive()

bool IOFileBlock::ioActive ( )
overridevirtual

Abstract virtual method of XrdPfc::IO Called to check if destruction needs to be done in a separate task.

Implements XrdPfc::IO.

Definition at line 77 of file XrdPfcIOFileBlock.cc.

78 {
79  // Called from XrdPosixFile when local connection is closed.
80 
82 
83  bool active = false;
84  {
85  XrdSysMutexHelper lock(&m_mutex);
86 
87  for (std::map<int, File*>::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it)
88  {
89  // Need to initiate stop on all File / block objects.
90  if (it->second && it->second->ioActive(this))
91  {
92  active = true;
93  }
94  }
95  }
96 
97  return active;
98 }
const char * RefreshLocation()
Definition: XrdPfcIO.hh:57

References XrdPfc::IO::RefreshLocation().

+ Here is the call graph for this function:

◆ Read() [1/3]

virtual int XrdOucCacheIO::Read

Pass Read request to the corresponding File object.

◆ Read() [2/3]

int IOFileBlock::Read ( char *  buff,
long long  offs,
int  rlen 
)
overridevirtual

Perform an synchronous read.

Parameters
buffpointer to the buffer to receive the results. The buffer must remain valid until the callback is invoked.
offsthe offset into the file.
rlenthe number of bytes to read.
Returns
< 0 - Read failed, value is -errno. >=0 - Read succeeded, value is number of bytes read.

Implements XrdOucCacheIO.

Definition at line 286 of file XrdPfcIOFileBlock.cc.

287 {
288  // protect from reads over the file size
289 
290  long long fileSize = FSize();
291 
292  if (off >= fileSize)
293  return 0;
294  if (off < 0)
295  {
296  return -EINVAL;
297  }
298  if (off + size > fileSize)
299  size = fileSize - off;
300 
301  long long off0 = off;
302  int idx_first = off0 / m_blocksize;
303  int idx_last = (off0 + size - 1) / m_blocksize;
304  int bytes_read = 0;
305  TRACEIO(Dump, "Read() "<< off << "@" << size << " block range ["<< idx_first << ", " << idx_last << "]");
306 
307  for (int blockIdx = idx_first; blockIdx <= idx_last; ++blockIdx)
308  {
309  // locate block
310  File *fb;
311  m_mutex.Lock();
312  std::map<int, File*>::iterator it = m_blocks.find(blockIdx);
313  if (it != m_blocks.end())
314  {
315  fb = it->second;
316  }
317  else
318  {
319  size_t pbs = m_blocksize;
320  // check if this is last block
321  int lastIOFileBlock = (fileSize-1)/m_blocksize;
322  if (blockIdx == lastIOFileBlock )
323  {
324  pbs = fileSize - blockIdx*m_blocksize;
325  // TRACEIO(Dump, "Read() last block, change output file size to " << pbs);
326  }
327 
328  // Note: File* can be 0 and stored as 0 if local open fails!
329  fb = newBlockFile(blockIdx*m_blocksize, pbs);
330  m_blocks.insert(std::make_pair(blockIdx, fb));
331  }
332  m_mutex.UnLock();
333 
334  // edit size if read request is reaching more than a block
335  int readBlockSize = size;
336  if (idx_first != idx_last)
337  {
338  if (blockIdx == idx_first)
339  {
340  readBlockSize = (blockIdx + 1) * m_blocksize - off0;
341  TRACEIO(Dump, "Read partially till the end of the block");
342  }
343  else if (blockIdx == idx_last)
344  {
345  readBlockSize = (off0 + size) - blockIdx * m_blocksize;
346  TRACEIO(Dump, "Read partially till the end of the block");
347  }
348  else
349  {
350  readBlockSize = m_blocksize;
351  }
352  }
353 
354  TRACEIO(Dump, "Read() block[ " << blockIdx << "] read-block-size[" << readBlockSize << "], offset[" << readBlockSize << "] off = " << off );
355 
356  int retvalBlock;
357  if (fb != 0)
358  {
359  struct ZHandler : public ReadReqRH
360  { using ReadReqRH::ReadReqRH;
361  XrdSysCondVar m_cond {0};
362  int m_retval {0};
363 
364  void Done(int result) override
365  { m_cond.Lock(); m_retval = result; m_cond.Signal(); m_cond.UnLock(); }
366  };
367 
368  ReadReqRHCond rh(ObtainReadSid(), nullptr);
369 
370  rh.m_cond.Lock();
371  retvalBlock = fb->Read(this, buff, off, readBlockSize, &rh);
372  if (retvalBlock == -EWOULDBLOCK)
373  {
374  rh.m_cond.Wait();
375  retvalBlock = rh.m_retval;
376  }
377  rh.m_cond.UnLock();
378  }
379  else
380  {
381  retvalBlock = GetInput()->Read(buff, off, readBlockSize);
382  }
383 
384  TRACEIO(Dump, "Read() Block read returned " << retvalBlock);
385  if (retvalBlock == readBlockSize)
386  {
387  bytes_read += retvalBlock;
388  buff += retvalBlock;
389  off += retvalBlock;
390  }
391  else if (retvalBlock >= 0)
392  {
393  TRACEIO(Warning, "Read() incomplete read, missing bytes " << readBlockSize-retvalBlock);
394  return -EIO;
395  }
396  else
397  {
398  TRACEIO(Error, "Read() read error, retval" << retvalBlock);
399  return retvalBlock;
400  }
401  }
402 
403  return bytes_read;
404 }
@ Warning
virtual int Read(char *buff, long long offs, int rlen)=0
int Read(IO *io, char *buff, long long offset, int size, ReadReqRH *rh)
Normal read.
Definition: XrdPfcFile.cc:676
long long FSize() override
XrdOucCacheIO * GetInput()
Definition: XrdPfcIO.cc:30
unsigned short ObtainReadSid()
Definition: XrdPfcIO.hh:59
ReadReqRH(unsigned short sid, XrdOucCacheIOCB *iocb)
Definition: XrdPfcFile.hh:67

References XrdPfc::ReadReqRH::ReadReqRH(), Macaroons::Error, FSize(), XrdPfc::IO::GetInput(), XrdSysCondVar::Lock(), XrdSysMutex::Lock(), XrdPfc::IO::ReadReqRHCond::m_cond, XrdPfc::IO::ReadReqRHCond::m_retval, XrdPfc::IO::ObtainReadSid(), XrdOucCacheIO::Read(), XrdPfc::File::Read(), TRACEIO, XrdSysCondVar::UnLock(), XrdSysMutex::UnLock(), XrdSysCondVar::Wait(), and Warning.

+ Here is the call graph for this function:

◆ Read() [3/3]

virtual void XrdOucCacheIO::Read
inline

Pass Read request to the corresponding File object.

Definition at line 333 of file XrdOucCache.hh.

334  {iocb.Done(Read(buff, offs, rlen));}
virtual int Read(char *buff, long long offs, int rlen)=0
Pass Read request to the corresponding File object.

◆ Update()

void IOFileBlock::Update ( XrdOucCacheIO iocp)
overridevirtual

Update the originally passed XrdOucCacheIO object with the object passed. All future uses underlying XrdOucCacheIO object must now use this object. Update() is called when Prepare() indicated that the file should not be physically opened and a file method was invoked in the XrdOucCacheIO passed to Attach(). When this occurs, the file is actually opened and Update() called to replace the original XrdOucCacheIO object with one that uses the newly opened file.

Parameters
iocpreference to the new XrdOucCacheIO object.

Reimplemented from XrdPfc::IO.

Definition at line 62 of file XrdPfcIOFileBlock.cc.

63 {
64  IO::Update(iocp);
65  {
66  XrdSysMutexHelper lock(&m_mutex);
67 
68  for (std::map<int, File*>::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it)
69  {
70  // Need to update all File / block objects.
71  if (it->second) it->second->ioUpdated(this);
72  }
73  }
74 }
void Update(XrdOucCacheIO &iocp) override
Definition: XrdPfcIO.cc:16

References XrdPfc::IO::Update().

+ Here is the call graph for this function:

The documentation for this class was generated from the following files: