XRootD
XrdClHttp::CurlOperation Class Referenceabstract

#include <XrdClHttpOps.hh>

+ Inheritance diagram for XrdClHttp::CurlOperation:
+ Collaboration diagram for XrdClHttp::CurlOperation:

Public Types

using HeaderList = std::vector< std::pair< std::string, std::string > >
 
enum class  HttpVerb {
  COPY ,
  DELETE ,
  HEAD ,
  GET ,
  MKCOL ,
  OPTIONS ,
  PROPFIND ,
  PUT ,
  Count
}
 
enum  OpError {
  ErrNone ,
  ErrHeaderTimeout ,
  ErrCallback ,
  ErrOperationTimeout ,
  ErrTransferClientStall ,
  ErrTransferStall ,
  ErrTransferSlow
}
 
enum class  RedirectAction {
  Fail ,
  Reinvoke ,
  ReinvokeAfterAllow
}
 

Public Member Functions

 CurlOperation (const CurlOperation &)=delete
 
 CurlOperation (XrdCl::ResponseHandler *handler, const std::string &url, std::chrono::steady_clock::time_point expiry, XrdCl::Log *log, CreateConnCalloutType, HeaderCallout *header_callout)
 
 CurlOperation (XrdCl::ResponseHandler *handler, const std::string &url, struct timespec timeout, XrdCl::Log *log, CreateConnCalloutType, HeaderCallout *header_callout)
 
virtual ~CurlOperation ()
 
virtual bool ContinueHandle ()
 
virtual void Fail (uint16_t errCode, uint32_t errNum, const std::string &)
 
bool FinishSetup (CURL *curl)
 
std::pair< XErrorCode, std::string > GetCallbackError () const
 
CreateConnCalloutType GetConnCalloutFunc () const
 
std::string GetCurlErrorMessage () const
 
CURLGetCurlHandle () const
 
OpError GetError () const
 
std::chrono::steady_clock::time_point GetHeaderExpiry () const
 
std::chrono::steady_clock::time_point GetOperationExpiry ()
 
std::unique_ptr< ResponseInfoGetResponseInfo ()
 
int GetStatusCode () const
 
std::string GetStatusMessage () const
 
bool GetTriedBoker () const
 
const std::string & GetUrl () const
 
virtual HttpVerb GetVerb () const =0
 
bool HasFailed () const
 
bool HeaderTimeoutExpired (const std::chrono::steady_clock::time_point &now)
 
bool IsDone () const
 
bool IsPaused () const
 
bool IsRedirect () const
 
std::unique_ptr< ResponseInfoMoveResponseInfo ()
 
bool OperationTimeoutExpired (const std::chrono::steady_clock::time_point &now)
 
virtual void OptionsDone ()
 
virtual RedirectAction Redirect (std::string &target)
 
virtual void ReleaseHandle ()
 
virtual bool RequiresOptions () const
 
virtual void SetContinueQueue (std::shared_ptr< XrdClHttp::HandlerQueue > queue)
 
void SetTriedBoker ()
 
virtual bool Setup (CURL *curl, CurlWorker &)
 
bool StartConnectionCallout (std::string &err)
 
std::tuple< uint64_t, std::chrono::steady_clock::duration, std::chrono::steady_clock::duration, std::chrono::steady_clock::duration > StatisticsReset ()
 
virtual void Success ()=0
 
bool TransferStalled (uint64_t xfer_bytes, const std::chrono::steady_clock::time_point &now)
 
bool UseConnectionCallout ()
 
virtual int WaitSocket ()
 
virtual int WaitSocketCallback (std::string &err)
 

Static Public Member Functions

static void CleanupDnsCache ()
 
static int GetDefaultSlowRateBytesSec ()
 
static int GetDefaultStallTimeout ()
 
static const std::string GetVerbString (HttpVerb)
 
static void SetSlowRateBytesSec (int rate)
 
static void SetStallTimeout (const std::chrono::steady_clock::duration &stall_interval)
 
static void SetStallTimeout (int stall_interval)
 

Protected Member Functions

int FailCallback (XErrorCode ecode, const std::string &emsg)
 
void SetDone (bool has_failed)
 
void SetPaused (bool paused)
 
void UpdateBytes (uint64_t bytes)
 

Protected Attributes

std::unique_ptr< CURL, void(*)(CURL *)> m_curl
 
XrdCl::ResponseHandlerm_handler {nullptr}
 
HeaderCalloutm_header_callout
 
std::chrono::steady_clock::time_point m_header_expiry
 
HeaderParser m_headers
 
std::vector< std::pair< std::string, std::string > > m_headers_list
 
XrdCl::Logm_logger
 
int m_minimum_rate {m_minimum_transfer_rate}
 
std::chrono::steady_clock::time_point m_operation_expiry
 
const std::string m_url
 

Static Protected Attributes

static constexpr int m_default_minimum_rate {1024 * 256}
 
static int m_minimum_transfer_rate {CurlOperation::m_default_minimum_rate}
 

Detailed Description

Definition at line 56 of file XrdClHttpOps.hh.

Member Typedef Documentation

◆ HeaderList

using XrdClHttp::CurlOperation::HeaderList = std::vector<std::pair<std::string, std::string> >

Definition at line 58 of file XrdClHttpOps.hh.

Member Enumeration Documentation

◆ HttpVerb

enum XrdClHttp::CurlOperation::HttpVerb
strong
Enumerator
COPY 
DELETE 
HEAD 
GET 
MKCOL 
OPTIONS 
PROPFIND 
PUT 
Count 

Definition at line 60 of file XrdClHttpOps.hh.

60  {
61  COPY,
62  DELETE,
63  HEAD,
64  GET,
65  MKCOL,
66  OPTIONS,
67  PROPFIND,
68  PUT,
69  Count
70  };

◆ OpError

enum XrdClHttp::CurlOperation::OpError
Enumerator
ErrNone 
ErrHeaderTimeout 
ErrCallback 
ErrOperationTimeout 
ErrTransferClientStall 
ErrTransferStall 
ErrTransferSlow 

Definition at line 206 of file XrdClHttpOps.hh.

206  {
207  ErrNone, // No error
208  ErrHeaderTimeout, // Header was not sent back in time
209  ErrCallback, // Error in the read/write callback (e.g., response too large for propfind)
210  ErrOperationTimeout, // Entire curl request operation has timed out
211  ErrTransferClientStall, // Transfer stalled while client had paused it (no data was available)
212  ErrTransferStall, // Transfer has stalled, not receiving any data within 60 seconds
213  ErrTransferSlow, // Average transfer rate is below the minimum
214  };

◆ RedirectAction

enum XrdClHttp::CurlOperation::RedirectAction
strong
Enumerator
Fail 
Reinvoke 
ReinvokeAfterAllow 

Definition at line 135 of file XrdClHttpOps.hh.

135  {
136  Fail, // The redirect parsing failed and Fail() was called
137  Reinvoke, // Reinvoke the curl handle, following redirect
138  ReinvokeAfterAllow, // Reinvoke the Redirect function once the allowed verbs are known.
139  };
virtual void Fail(uint16_t errCode, uint32_t errNum, const std::string &)

Constructor & Destructor Documentation

◆ CurlOperation() [1/3]

XrdClHttp::CurlOperation::CurlOperation ( XrdCl::ResponseHandler handler,
const std::string &  url,
struct timespec  timeout,
XrdCl::Log log,
CreateConnCalloutType  ,
HeaderCallout header_callout 
)

◆ CurlOperation() [2/3]

XrdClHttp::CurlOperation::CurlOperation ( XrdCl::ResponseHandler handler,
const std::string &  url,
std::chrono::steady_clock::time_point  expiry,
XrdCl::Log log,
CreateConnCalloutType  ,
HeaderCallout header_callout 
)

◆ ~CurlOperation()

CurlOperation::~CurlOperation ( )
virtual

Definition at line 188 of file XrdClHttpOps.cc.

188 {}

◆ CurlOperation() [3/3]

XrdClHttp::CurlOperation::CurlOperation ( const CurlOperation )
delete

Member Function Documentation

◆ CleanupDnsCache()

void CurlOperation::CleanupDnsCache ( )
static

Definition at line 694 of file XrdClHttpOps.cc.

695 {
696  auto now = std::chrono::steady_clock::now();
697  for (auto it = fake_dns_refcount.begin(); it != fake_dns_refcount.end(); ) {
698  if (it->second->count <= 0 && it->second->IsExpired(now)) {
699  auto rev_iter = reverse_fake_dns_map.find(*it->first);
700  if (rev_iter != reverse_fake_dns_map.end()) {
701  fake_dns_map.erase(rev_iter->second.first);
702  reverse_fake_dns_map.erase(rev_iter);
703  }
704  it = fake_dns_refcount.erase(it);
705  } else {
706  ++it;
707  }
708  }
709 }

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ ContinueHandle()

virtual bool XrdClHttp::CurlOperation::ContinueHandle ( )
inlinevirtual

Reimplemented in XrdClHttp::CurlPutOp, and XrdClHttp::CurlReadOp.

Definition at line 129 of file XrdClHttpOps.hh.

129 {return true;}

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ Fail()

virtual void XrdClHttp::CurlOperation::Fail ( uint16_t  errCode,
uint32_t  errNum,
const std::string &   
)
virtual

◆ FailCallback()

int CurlOperation::FailCallback ( XErrorCode  ecode,
const std::string &  emsg 
)
protected

Definition at line 207 of file XrdClHttpOps.cc.

207  {
208  m_callback_error_code = ecode;
209  m_callback_error_str = emsg;
210  m_error = OpError::ErrCallback;
211  m_logger->Debug(kLogXrdClHttp, "%s", emsg.c_str());
212  return 0;
213 }
int emsg(int rc, char *msg)
void Debug(uint64_t topic, const char *format,...)
Print a debug message.
Definition: XrdClLog.cc:282
const uint64_t kLogXrdClHttp

References emsg().

Referenced by XrdClHttp::CurlVectorReadOp::Write().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ FinishSetup()

bool CurlOperation::FinishSetup ( CURL curl)

Definition at line 216 of file XrdClHttpOps.cc.

217 {
218  if (!m_header_callout) {
219  m_header_slist.reset();
220  for (const auto &header : m_headers_list) {
221  m_header_slist.reset(curl_slist_append(m_header_slist.release(),
222  (header.first + ": " + header.second).c_str()));
223  }
224  return curl_easy_setopt(curl, CURLOPT_HTTPHEADER, m_header_slist.get()) == CURLE_OK;
225  }
226  const auto &verb = GetVerbString(GetVerb());
227 
228  auto extra_headers = m_header_callout->GetHeaders(verb, m_url, m_headers_list);
229  if (!extra_headers) {
230  m_logger->Error(kLogXrdClHttp, "Failed to get headers from header callout for %s", m_url.c_str());
231  return false;
232  }
233  m_header_slist.reset();
234  for (const auto &header : *extra_headers) {
235  if (!strcasecmp(header.first.c_str(), "Content-Length")) {
236  auto upload_size = std::stoull(header.second);
237  curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, upload_size);
238  continue;
239  }
240  m_header_slist.reset(curl_slist_append(m_header_slist.release(),
241  (header.first + ": " + header.second).c_str()));
242  }
243  return curl_easy_setopt(curl, CURLOPT_HTTPHEADER, m_header_slist.get()) == CURLE_OK;
244 }
static unsigned long long int stoull(const std::string &s)
simple integer parsing, to be replaced by std::stoll when C++11 can be used
const std::string m_url
static const std::string GetVerbString(HttpVerb)
virtual HttpVerb GetVerb() const =0
std::vector< std::pair< std::string, std::string > > m_headers_list
HeaderCallout * m_header_callout
virtual std::shared_ptr< HeaderList > GetHeaders(const std::string &verb, const std::string &url, const HeaderList &headers)=0
void Error(uint64_t topic, const char *format,...)
Report an error.
Definition: XrdClLog.cc:231

References stoull().

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ GetCallbackError()

std::pair<XErrorCode, std::string> XrdClHttp::CurlOperation::GetCallbackError ( ) const
inline

Definition at line 226 of file XrdClHttpOps.hh.

226 {return std::make_pair(m_callback_error_code, m_callback_error_str);}

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ GetConnCalloutFunc()

CreateConnCalloutType XrdClHttp::CurlOperation::GetConnCalloutFunc ( ) const
inline

Definition at line 99 of file XrdClHttpOps.hh.

99 {return m_conn_callout;}

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ GetCurlErrorMessage()

std::string XrdClHttp::CurlOperation::GetCurlErrorMessage ( ) const
inline

Definition at line 252 of file XrdClHttpOps.hh.

252  {
253  if (m_curl_error_buffer[0] != '\0')
254  return m_curl_error_buffer;
255  return "";
256  }

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ GetCurlHandle()

CURL* XrdClHttp::CurlOperation::GetCurlHandle ( ) const
inline

Definition at line 217 of file XrdClHttpOps.hh.

217 {return m_curl.get();}
std::unique_ptr< CURL, void(*)(CURL *)> m_curl

References m_curl.

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ GetDefaultSlowRateBytesSec()

static int XrdClHttp::CurlOperation::GetDefaultSlowRateBytesSec ( )
inlinestatic

Definition at line 278 of file XrdClHttpOps.hh.

279  {
280  return m_default_minimum_rate;
281  }
static constexpr int m_default_minimum_rate

References m_default_minimum_rate.

◆ GetDefaultStallTimeout()

static int XrdClHttp::CurlOperation::GetDefaultStallTimeout ( )
inlinestatic

Definition at line 272 of file XrdClHttpOps.hh.

273  {
274  return std::chrono::duration_cast<std::chrono::seconds>(m_default_stall_interval).count();
275  }

◆ GetError()

OpError XrdClHttp::CurlOperation::GetError ( ) const
inline

Definition at line 220 of file XrdClHttpOps.hh.

220 {return m_error;}

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ GetHeaderExpiry()

std::chrono::steady_clock::time_point XrdClHttp::CurlOperation::GetHeaderExpiry ( ) const
inline

Definition at line 110 of file XrdClHttpOps.hh.

110 {return m_header_expiry;}
std::chrono::steady_clock::time_point m_header_expiry

References m_header_expiry.

Referenced by GetOperationExpiry().

+ Here is the caller graph for this function:

◆ GetOperationExpiry()

std::chrono::steady_clock::time_point XrdClHttp::CurlOperation::GetOperationExpiry ( )
inline

Definition at line 113 of file XrdClHttpOps.hh.

113  {
114  if (m_last_xfer == std::chrono::steady_clock::time_point()) {
115  return GetHeaderExpiry();
116  }
117  return m_last_xfer + m_stall_interval;
118  }
std::chrono::steady_clock::time_point GetHeaderExpiry() const

References GetHeaderExpiry().

+ Here is the call graph for this function:

◆ GetResponseInfo()

std::unique_ptr<ResponseInfo> XrdClHttp::CurlOperation::GetResponseInfo ( )

◆ GetStatusCode()

int XrdClHttp::CurlOperation::GetStatusCode ( ) const
inline

Definition at line 229 of file XrdClHttpOps.hh.

229 {return m_headers.GetStatusCode();}

References XrdClHttp::HeaderParser::GetStatusCode(), and m_headers.

Referenced by XrdClHttp::CurlMkcolOp::Fail(), XrdClHttp::CurlWorker::Run(), and XrdClHttp::CurlVectorReadOp::Write().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ GetStatusMessage()

std::string XrdClHttp::CurlOperation::GetStatusMessage ( ) const
inline

Definition at line 232 of file XrdClHttpOps.hh.

232 {return m_headers.GetStatusMessage();}
std::string GetStatusMessage() const

References XrdClHttp::HeaderParser::GetStatusMessage(), and m_headers.

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ GetTriedBoker()

bool XrdClHttp::CurlOperation::GetTriedBoker ( ) const
inline

Definition at line 167 of file XrdClHttpOps.hh.

167 {return m_tried_broker;} // Returns true if the connection broker has been tried.

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ GetUrl()

const std::string& XrdClHttp::CurlOperation::GetUrl ( ) const
inline

Definition at line 177 of file XrdClHttpOps.hh.

177 {return m_url;}

References m_url.

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ GetVerb()

◆ GetVerbString()

const std::string CurlOperation::GetVerbString ( CurlOperation::HttpVerb  verb)
static

Definition at line 247 of file XrdClHttpOps.cc.

248 {
249  switch (verb) {
250  case HttpVerb::COPY:
251  return "COPY";
252  case HttpVerb::DELETE:
253  return "DELETE";
254  case HttpVerb::GET:
255  return "GET";
256  case HttpVerb::HEAD:
257  return "HEAD";
258  case HttpVerb::MKCOL:
259  return "MKCOL";
260  case HttpVerb::OPTIONS:
261  return "OPTIONS";
262  case HttpVerb::PROPFIND:
263  return "PROPFIND";
264  case HttpVerb::PUT:
265  return "PUT";
266  case HttpVerb::Count:
267  return "UNKNOWN";
268  }
269  return "UNKNOWN";
270 }

Referenced by XrdClHttp::CurlWorker::GetMonitoringJson().

+ Here is the caller graph for this function:

◆ HasFailed()

bool XrdClHttp::CurlOperation::HasFailed ( ) const
inline

Definition at line 241 of file XrdClHttpOps.hh.

241 {return m_has_failed.load(std::memory_order_acquire);}

◆ HeaderTimeoutExpired()

bool CurlOperation::HeaderTimeoutExpired ( const std::chrono::steady_clock::time_point &  now)

Definition at line 450 of file XrdClHttpOps.cc.

450  {
451  if (m_received_header) return false;
452 
453  if (now > m_header_expiry) {
454  if (m_error == OpError::ErrNone) m_error = OpError::ErrHeaderTimeout;
455  return true;
456  }
457  return false;
458 }

◆ IsDone()

bool XrdClHttp::CurlOperation::IsDone ( ) const
inline

Definition at line 235 of file XrdClHttpOps.hh.

235 {return m_done;}

Referenced by XrdClHttp::CurlReadOp::ContinueHandle(), and XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ IsPaused()

bool XrdClHttp::CurlOperation::IsPaused ( ) const
inline

Definition at line 238 of file XrdClHttpOps.hh.

238 {return m_is_paused;}

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ IsRedirect()

bool XrdClHttp::CurlOperation::IsRedirect ( ) const
inline

Definition at line 151 of file XrdClHttpOps.hh.

151 {return m_headers.GetStatusCode() >= 300 && m_headers.GetStatusCode() < 400;}

References XrdClHttp::HeaderParser::GetStatusCode(), and m_headers.

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ MoveResponseInfo()

std::unique_ptr<ResponseInfo> XrdClHttp::CurlOperation::MoveResponseInfo ( )
inline

Definition at line 223 of file XrdClHttpOps.hh.

223 {return std::move(m_response_info);}

Referenced by XrdClHttp::CurlChecksumOp::Success(), XrdClHttp::CurlDeleteOp::Success(), XrdClHttp::CurlMkcolOp::Success(), XrdClHttp::CurlListdirOp::Success(), and XrdClHttp::CurlStatOp::SuccessImpl().

+ Here is the caller graph for this function:

◆ OperationTimeoutExpired()

bool CurlOperation::OperationTimeoutExpired ( const std::chrono::steady_clock::time_point &  now)

Definition at line 461 of file XrdClHttpOps.cc.

461  {
462  if (m_operation_expiry == std::chrono::steady_clock::time_point{} ||
463  !m_received_header) {
464  return false;
465  }
466 
467  if (now > m_operation_expiry) {
468  if (m_error == OpError::ErrNone) m_error = OpError::ErrOperationTimeout;
469  return true;
470  }
471  return false;
472 }
std::chrono::steady_clock::time_point m_operation_expiry

◆ OptionsDone()

virtual void XrdClHttp::CurlOperation::OptionsDone ( )
inlinevirtual

Reimplemented in XrdClHttp::CurlChecksumOp, and XrdClHttp::CurlStatOp.

Definition at line 174 of file XrdClHttpOps.hh.

174 {}

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ Redirect()

CurlOperation::RedirectAction CurlOperation::Redirect ( std::string &  target)
virtual

Reimplemented in XrdClHttp::CurlChecksumOp, and XrdClHttp::CurlStatOp.

Definition at line 305 of file XrdClHttpOps.cc.

306 {
307  m_callout.reset();
308  m_conn_callout_result = -1;
309  m_conn_callout_listener = -1;
310  m_tried_broker = false;
311 
312  auto location = m_headers.GetLocation();
313  if (location.empty()) {
314  m_logger->Warning(kLogXrdClHttp, "After request to %s, server returned a redirect with no new location", m_url.c_str());
315  Fail(XrdCl::errErrorResponse, kXR_ServerError, "Server returned redirect without updated location");
316  return RedirectAction::Fail;
317  }
318  if (location.size() && location[0] == '/') { // hostname not included in the location - redirect to self.
319  std::string_view orig_url(m_url);
320  auto scheme_loc = orig_url.find("://");
321  if (scheme_loc == std::string_view::npos) {
322  Fail(XrdCl::errErrorResponse, kXR_ServerError, "Server returned a location with unknown hostname");
323  return RedirectAction::Fail;
324  }
325  auto path_loc = orig_url.find('/', scheme_loc + 3);
326  if (path_loc == std::string_view::npos) {
327  location = m_url + location;
328  } else {
329  location = std::string(orig_url.substr(0, path_loc)) + location;
330  }
331  }
332  m_logger->Debug(kLogXrdClHttp, "Request for %s redirected to %s", m_url.c_str(), location.c_str());
333  target = location;
334  curl_easy_setopt(m_curl.get(), CURLOPT_URL, location.c_str());
335  int disable_x509;
336  auto env = XrdCl::DefaultEnv::GetEnv();
337  if (env->GetInt("HttpDisableX509", disable_x509) && !disable_x509) {
338  std::string cert, key;
339  env->GetString("HttpClientCertFile", cert);
340  env->GetString("HttpClientKeyFile", key);
341  if (!cert.empty())
342  curl_easy_setopt(m_curl.get(), CURLOPT_SSLCERT, cert.c_str());
343  if (!key.empty())
344  curl_easy_setopt(m_curl.get(), CURLOPT_SSLKEY, key.c_str());
345  }
346  m_headers = HeaderParser();
347 
348  if (m_conn_callout) {
349  auto conn_callout = m_conn_callout(location, *m_response_info);
350  if (conn_callout != nullptr) {
351 
352  auto [host, port] = ParseHostPort(location);
353  if (host.empty() || port == -1) {
354  Fail(XrdCl::errInternal, 0, "Failed to parse host and port from URL " + location);
355  return RedirectAction::Fail;
356  }
357  auto fake_addr = GetFakeEndpointForHost(host, port);
358  if (!fake_addr || fake_addr->empty()) {
359  Fail(XrdCl::errInternal, 0, "Failed to generate a fake address for host " + host);
360  return RedirectAction::Fail;
361  }
362  m_resolve_slist.reset(curl_slist_append(m_resolve_slist.release(),
363  (host + ":" + std::to_string(port) + ":" + *fake_addr).c_str()));
364  m_logger->Debug(kLogXrdClHttp, "For connection callout in redirect, mapping %s:%d -> %s", host.c_str(), port, fake_addr->c_str());
365 
366  m_callout.reset(conn_callout);
367  std::string err;
368  SetTriedBoker();
369  if ((m_conn_callout_listener = m_callout->BeginCallout(err, m_header_expiry)) == -1) {
370  auto errMsg = "Failed to start a connection callout request: " + err;
371  Fail(XrdCl::errInternal, 0, errMsg.c_str());
372  return RedirectAction::Fail;
373  }
374  curl_easy_setopt(m_curl.get(), CURLOPT_OPENSOCKETFUNCTION, CurlOperation::OpenSocketCallback);
375  curl_easy_setopt(m_curl.get(), CURLOPT_CLOSESOCKETFUNCTION, CurlOperation::CloseSocketCallback);
376  curl_easy_setopt(m_curl.get(), CURLOPT_OPENSOCKETDATA, this);
377  curl_easy_setopt(m_curl.get(), CURLOPT_CLOSESOCKETDATA, fake_addr);
378  curl_easy_setopt(m_curl.get(), CURLOPT_SOCKOPTFUNCTION, CurlOperation::SockOptCallback);
379  curl_easy_setopt(m_curl.get(), CURLOPT_SOCKOPTDATA, this);
380  curl_easy_setopt(m_curl.get(), CURLOPT_CONNECT_TO, m_resolve_slist.get());
381  }
382  }
383  m_received_header = false;
384 
385  m_last_header_reset = m_last_reset = m_header_start = m_start_op = m_header_lastop = std::chrono::steady_clock::now();
386  return RedirectAction::Reinvoke;
387 }
@ kXR_ServerError
Definition: XProtocol.hh:1044
const std::string & GetLocation() const
static Env * GetEnv()
Get default client environment.
void Warning(uint64_t topic, const char *format,...)
Report a warning.
Definition: XrdClLog.cc:248
const uint16_t errErrorResponse
Definition: XrdClStatus.hh:105
const uint16_t errInternal
Internal error.
Definition: XrdClStatus.hh:56

References XrdCl::errErrorResponse, XrdCl::errInternal, XrdCl::DefaultEnv::GetEnv(), and kXR_ServerError.

Referenced by XrdClHttp::CurlStatOp::Redirect(), XrdClHttp::CurlChecksumOp::Redirect(), and XrdClHttp::CurlWorker::Run().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ ReleaseHandle()

void CurlOperation::ReleaseHandle ( )
virtual

Reimplemented in XrdClHttp::CurlPutOp, XrdClHttp::CurlCopyOp, XrdClHttp::CurlListdirOp, XrdClHttp::CurlVectorReadOp, XrdClHttp::CurlReadOp, XrdClHttp::CurlMkcolOp, XrdClHttp::CurlDeleteOp, XrdClHttp::CurlChecksumOp, XrdClHttp::CurlOpenOp, XrdClHttp::CurlStatOp, and XrdClHttp::CurlOptionsOp.

Definition at line 603 of file XrdClHttpOps.cc.

604 {
605  m_conn_callout_listener = -1;
606  m_conn_callout_result = -1;
607  m_tried_broker = false;
608  m_callout.reset();
609 
610  if (m_curl == nullptr) return;
611  curl_easy_setopt(m_curl.get(), CURLOPT_OPENSOCKETFUNCTION, nullptr);
612  curl_easy_setopt(m_curl.get(), CURLOPT_CLOSESOCKETFUNCTION, nullptr);
613  curl_easy_setopt(m_curl.get(), CURLOPT_OPENSOCKETDATA, nullptr);
614  curl_easy_setopt(m_curl.get(), CURLOPT_CLOSESOCKETDATA, nullptr);
615  curl_easy_setopt(m_curl.get(), CURLOPT_SOCKOPTFUNCTION, nullptr);
616  curl_easy_setopt(m_curl.get(), CURLOPT_SOCKOPTDATA, nullptr);
617  curl_easy_setopt(m_curl.get(), CURLOPT_SSLCERT, nullptr);
618  curl_easy_setopt(m_curl.get(), CURLOPT_SSLKEY, nullptr);
619  curl_easy_setopt(m_curl.get(), CURLOPT_HTTPHEADER, nullptr);
620  curl_easy_setopt(m_curl.get(), CURLOPT_CONNECT_TO, nullptr);
621  m_header_slist.reset();
622  m_curl.release();
623 }

Referenced by XrdClHttp::CurlOptionsOp::ReleaseHandle(), XrdClHttp::CurlStatOp::ReleaseHandle(), XrdClHttp::CurlDeleteOp::ReleaseHandle(), XrdClHttp::CurlMkcolOp::ReleaseHandle(), XrdClHttp::CurlReadOp::ReleaseHandle(), XrdClHttp::CurlVectorReadOp::ReleaseHandle(), XrdClHttp::CurlListdirOp::ReleaseHandle(), XrdClHttp::CurlCopyOp::ReleaseHandle(), XrdClHttp::CurlPutOp::ReleaseHandle(), and XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ RequiresOptions()

virtual bool XrdClHttp::CurlOperation::RequiresOptions ( ) const
inlinevirtual

Reimplemented in XrdClHttp::CurlStatOp.

Definition at line 171 of file XrdClHttpOps.hh.

171 {return false;}

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ SetContinueQueue()

virtual void XrdClHttp::CurlOperation::SetContinueQueue ( std::shared_ptr< XrdClHttp::HandlerQueue queue)
inlinevirtual

Reimplemented in XrdClHttp::CurlPutOp, and XrdClHttp::CurlReadOp.

Definition at line 133 of file XrdClHttpOps.hh.

133 {}

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ SetDone()

void XrdClHttp::CurlOperation::SetDone ( bool  has_failed)
inlineprotected

◆ SetPaused()

void CurlOperation::SetPaused ( bool  paused)
protected

Definition at line 400 of file XrdClHttpOps.cc.

400  {
401  m_is_paused = paused;
402  if (m_is_paused) {
403  m_pause_start = std::chrono::steady_clock::now();
404  } else if (m_pause_start != std::chrono::steady_clock::time_point{}) {
405  m_pause_duration += std::chrono::steady_clock::now() - m_pause_start;
406  m_pause_start = std::chrono::steady_clock::time_point{};
407  }
408 }

Referenced by XrdClHttp::CurlReadOp::ContinueHandle(), XrdClHttp::CurlPutOp::ContinueHandle(), XrdClHttp::CurlReadOp::Pause(), and XrdClHttp::CurlPutOp::Pause().

+ Here is the caller graph for this function:

◆ SetSlowRateBytesSec()

static void XrdClHttp::CurlOperation::SetSlowRateBytesSec ( int  rate)
inlinestatic

Definition at line 284 of file XrdClHttpOps.hh.

285  {
287  }
static int m_minimum_transfer_rate

References m_minimum_transfer_rate.

◆ SetStallTimeout() [1/2]

static void XrdClHttp::CurlOperation::SetStallTimeout ( const std::chrono::steady_clock::duration &  stall_interval)
inlinestatic

Definition at line 266 of file XrdClHttpOps.hh.

267  {
268  m_stall_interval = stall_interval;
269  }

◆ SetStallTimeout() [2/2]

static void XrdClHttp::CurlOperation::SetStallTimeout ( int  stall_interval)
inlinestatic

Definition at line 259 of file XrdClHttpOps.hh.

260  {
261  std::chrono::seconds seconds{stall_interval};
262  m_stall_interval = std::chrono::duration_cast<std::chrono::steady_clock::duration>(seconds);
263  }

◆ SetTriedBoker()

void XrdClHttp::CurlOperation::SetTriedBoker ( )
inline

Definition at line 168 of file XrdClHttpOps.hh.

168 {m_tried_broker = true;} // Note that the connection broker has been attempted.

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ Setup()

bool CurlOperation::Setup ( CURL curl,
CurlWorker worker 
)
virtual

Reimplemented in XrdClHttp::CurlPutOp, XrdClHttp::CurlCopyOp, XrdClHttp::CurlListdirOp, XrdClHttp::CurlVectorReadOp, XrdClHttp::CurlReadOp, XrdClHttp::CurlMkcolOp, XrdClHttp::CurlDeleteOp, XrdClHttp::CurlChecksumOp, XrdClHttp::CurlStatOp, and XrdClHttp::CurlOptionsOp.

Definition at line 523 of file XrdClHttpOps.cc.

524 {
525  if (curl == nullptr) {
526  throw std::runtime_error("Unable to setup curl operation with no handle");
527  }
528  struct timespec now;
529  if (clock_gettime(CLOCK_MONOTONIC, &now) == -1) {
530  throw std::runtime_error("Unable to get current time");
531  }
532 
533  m_pause_start = {};
534  m_last_header_reset = m_last_reset = m_start_op = m_header_start = m_header_lastop = std::chrono::steady_clock::now();
535 
536  m_curl.reset(curl);
537  m_curl_error_buffer[0] = '\0';
538  curl_easy_setopt(m_curl.get(), CURLOPT_URL, m_url.c_str());
539  curl_easy_setopt(m_curl.get(), CURLOPT_ERRORBUFFER, m_curl_error_buffer);
540  curl_easy_setopt(m_curl.get(), CURLOPT_HEADERFUNCTION, CurlStatOp::HeaderCallback);
541  curl_easy_setopt(m_curl.get(), CURLOPT_HEADERDATA, this);
542  curl_easy_setopt(m_curl.get(), CURLOPT_WRITEFUNCTION, NullCallback);
543  curl_easy_setopt(m_curl.get(), CURLOPT_WRITEDATA, nullptr);
544  curl_easy_setopt(m_curl.get(), CURLOPT_XFERINFOFUNCTION, CurlOperation::XferInfoCallback);
545  curl_easy_setopt(m_curl.get(), CURLOPT_XFERINFODATA, this);
546  curl_easy_setopt(m_curl.get(), CURLOPT_NOPROGRESS, 0L);
547  // Note: libcurl is not threadsafe unless this option is set.
548  // Before we set it, we saw deadlocks (and partial deadlocks) in practice.
549  curl_easy_setopt(m_curl.get(), CURLOPT_NOSIGNAL, 1L);
550 
551  m_parsed_url.reset(new XrdCl::URL(m_url));
552  auto env = XrdCl::DefaultEnv::GetEnv();
553  int disable_x509;
554  if ((env->GetInt("HttpDisableX509", disable_x509) && !disable_x509)) {
555  auto [cert, key] = worker.ClientX509CertKeyFile();
556  if (!cert.empty()) {
557  m_logger->Debug(kLogXrdClHttp, "Using client X.509 credential found at %s", cert.c_str());
558  curl_easy_setopt(m_curl.get(), CURLOPT_SSLCERT, cert.c_str());
559  if (key.empty()) {
560  m_logger->Error(kLogXrdClHttp, "X.509 client credential specified but not the client key");
561  } else {
562  curl_easy_setopt(m_curl.get(), CURLOPT_SSLKEY, key.c_str());
563  }
564  }
565  }
566 
567  if (m_conn_callout) {
568  ResponseInfo info;
569  auto callout = m_conn_callout(m_url, info);
570  if (callout) {
571  m_callout.reset(callout);
572  m_conn_callout_listener = -1;
573  m_conn_callout_result = -1;
574  m_tried_broker = false;
575 
576  auto [host, port] = ParseHostPort(m_url);
577  if (host.empty() || port == -1) {
578  throw std::runtime_error ("Failed to parse host and port from URL " + m_url);
579  }
580  auto fake_addr = GetFakeEndpointForHost(host, port);
581  if (!fake_addr || fake_addr->empty()) {
582  throw std::runtime_error("Failed to generate a fake address for host " + host);
583  }
584  m_resolve_slist.reset(curl_slist_append(m_resolve_slist.release(),
585  (host + ":" + std::to_string(port) + ":" + *fake_addr).c_str()));
586  m_logger->Debug(kLogXrdClHttp, "For connection callout in operation setup, mapping %s:%d -> %s", host.c_str(), port, fake_addr->c_str());
587 
588  curl_easy_setopt(m_curl.get(), CURLOPT_CONNECT_TO, m_resolve_slist.get());
589 
590  curl_easy_setopt(m_curl.get(), CURLOPT_OPENSOCKETFUNCTION, CurlOperation::OpenSocketCallback);
591  curl_easy_setopt(m_curl.get(), CURLOPT_CLOSESOCKETFUNCTION, CurlOperation::CloseSocketCallback);
592  curl_easy_setopt(m_curl.get(), CURLOPT_OPENSOCKETDATA, this);
593  curl_easy_setopt(m_curl.get(), CURLOPT_CLOSESOCKETDATA, fake_addr);
594  curl_easy_setopt(m_curl.get(), CURLOPT_SOCKOPTFUNCTION, CurlOperation::SockOptCallback);
595  curl_easy_setopt(m_curl.get(), CURLOPT_SOCKOPTDATA, this);
596  }
597  }
598 
599  return true;
600 }
URL representation.
Definition: XrdClURL.hh:31

References XrdClHttp::CurlWorker::ClientX509CertKeyFile(), and XrdCl::DefaultEnv::GetEnv().

Referenced by XrdClHttp::CurlWorker::Run(), XrdClHttp::CurlOptionsOp::Setup(), XrdClHttp::CurlStatOp::Setup(), XrdClHttp::CurlDeleteOp::Setup(), XrdClHttp::CurlMkcolOp::Setup(), XrdClHttp::CurlReadOp::Setup(), XrdClHttp::CurlVectorReadOp::Setup(), XrdClHttp::CurlListdirOp::Setup(), XrdClHttp::CurlCopyOp::Setup(), and XrdClHttp::CurlPutOp::Setup().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ StartConnectionCallout()

bool CurlOperation::StartConnectionCallout ( std::string &  err)

Definition at line 411 of file XrdClHttpOps.cc.

412 {
413  if ((m_conn_callout_listener = m_callout->BeginCallout(err, m_header_expiry)) == -1) {
414  err = "Failed to start a callout for a socket connection: " + err;
415  Fail(XrdCl::errInternal, 1, err.c_str());
416  return false;
417  }
418  return true;
419 }

References XrdCl::errInternal.

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ StatisticsReset()

std::tuple< uint64_t, std::chrono::steady_clock::duration, std::chrono::steady_clock::duration, std::chrono::steady_clock::duration > CurlOperation::StatisticsReset ( )

Definition at line 422 of file XrdClHttpOps.cc.

422  {
423  auto now = std::chrono::steady_clock::now();
424  std::chrono::steady_clock::duration pre_header{}, post_header{}, pause_duration{};
425  if (m_received_header) {
426  if (m_last_header_reset < m_header_start) {
427  pre_header = m_header_start - m_last_header_reset;
428  m_last_header_reset = m_header_start;
429  }
430  post_header = now - ((m_last_reset < m_header_start) ? m_header_start : m_last_reset);
431  m_last_reset = now;
432  } else {
433  pre_header = now - m_last_header_reset;
434  m_last_header_reset = now;
435  }
436  if (IsPaused()) {
437  m_pause_duration += now - m_pause_start;
438  m_pause_start = now;
439  }
440  if (m_pause_duration != std::chrono::steady_clock::duration::zero()) {
441  pause_duration = m_pause_duration;
442  m_pause_duration = std::chrono::steady_clock::duration::zero();
443  }
444  auto bytes = m_bytes;
445  m_bytes = 0;
446  return {bytes, pre_header, post_header, pause_duration};
447 }

◆ Success()

virtual void XrdClHttp::CurlOperation::Success ( )
pure virtual

◆ TransferStalled()

bool CurlOperation::TransferStalled ( uint64_t  xfer_bytes,
const std::chrono::steady_clock::time_point &  now 
)

Definition at line 475 of file XrdClHttpOps.cc.

476 {
477  // First, check to see how long it's been since any data was sent.
478  if (m_last_xfer == std::chrono::steady_clock::time_point()) {
479  m_last_xfer = m_header_lastop;
480  }
481  auto elapsed = now - m_last_xfer;
482  uint64_t xfer_diff = 0;
483  if (xfer > m_last_xfer_count) {
484  xfer_diff = xfer - m_last_xfer_count;
485  m_last_xfer_count = xfer;
486  m_last_xfer = now;
487  }
488 
489  // If progress is made in this callback do not classify as stalled
490  if (elapsed > m_stall_interval && xfer_diff == 0) {
491  if (m_error == OpError::ErrNone) m_error = IsPaused() ? OpError::ErrTransferClientStall : OpError::ErrTransferStall;
492  return true;
493  }
494 
495  // Curl updated us with new timing but the byte count hasn't changed; no need to update the EMA.
496  if (xfer_diff == 0) {
497  return false;
498  }
499 
500  // If the transfer is not stalled, then we check to see if the exponentially-weighted
501  // moving average of the transfer rate is below the minimum.
502 
503  // If the stall interval since the last header hasn't passed, then we don't check for slow transfers.
504  auto elapsed_since_last_headerop = now - m_header_lastop;
505  if (elapsed_since_last_headerop < m_stall_interval) {
506  return false;
507  } else if (m_ema_rate < 0) {
508  m_ema_rate = xfer / std::chrono::duration<double>(elapsed_since_last_headerop).count();
509  }
510  // Calculate the exponential moving average of the transfer rate.
511  double elapsed_seconds = std::chrono::duration<double>(elapsed).count();
512  auto recent_rate = static_cast<double>(xfer_diff) / elapsed_seconds;
513  auto alpha = 1.0 - exp(-elapsed_seconds / std::chrono::duration<double>(m_stall_interval).count());
514  m_ema_rate = (1.0 - alpha) * m_ema_rate + alpha * recent_rate;
515  if (m_ema_rate < static_cast<double>(m_minimum_rate)) {
516  if (m_error == OpError::ErrNone) m_error = OpError::ErrTransferSlow;
517  return true;
518  }
519  return false;
520 }

◆ UpdateBytes()

void XrdClHttp::CurlOperation::UpdateBytes ( uint64_t  bytes)
inlineprotected

Definition at line 292 of file XrdClHttpOps.hh.

292 {m_bytes += bytes;}

Referenced by XrdClHttp::CurlVectorReadOp::Write().

+ Here is the caller graph for this function:

◆ UseConnectionCallout()

bool XrdClHttp::CurlOperation::UseConnectionCallout ( )
inline

Definition at line 166 of file XrdClHttpOps.hh.

166 {return m_callout.get();} // Returns true if the callout should be tried.

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ WaitSocket()

virtual int XrdClHttp::CurlOperation::WaitSocket ( )
inlinevirtual

Definition at line 154 of file XrdClHttpOps.hh.

154 {return m_conn_callout_listener;}

Referenced by XrdClHttp::CurlWorker::Run().

+ Here is the caller graph for this function:

◆ WaitSocketCallback()

int CurlOperation::WaitSocketCallback ( std::string &  err)
virtual

Definition at line 727 of file XrdClHttpOps.cc.

728 {
729  m_conn_callout_result = m_callout ? m_callout->FinishCallout(err) : -1;
730  if (m_callout && m_conn_callout_result == -1) {
731  m_logger->Error(kLogXrdClHttp, "Error when getting socket callout: %s", err.c_str());
732  } else if (m_callout) {
733  m_logger->Debug(kLogXrdClHttp, "Got callback socket %d", m_conn_callout_result);
734  }
735  return m_conn_callout_result;
736 }

Member Data Documentation

◆ m_curl

◆ m_default_minimum_rate

constexpr int XrdClHttp::CurlOperation::m_default_minimum_rate {1024 * 256}
staticconstexprprotected

Definition at line 304 of file XrdClHttpOps.hh.

Referenced by GetDefaultSlowRateBytesSec().

◆ m_handler

◆ m_header_callout

HeaderCallout* XrdClHttp::CurlOperation::m_header_callout
protected

Definition at line 321 of file XrdClHttpOps.hh.

◆ m_header_expiry

std::chrono::steady_clock::time_point XrdClHttp::CurlOperation::m_header_expiry
protected

Definition at line 318 of file XrdClHttpOps.hh.

Referenced by XrdClHttp::CurlStatOp::CurlStatOp(), and GetHeaderExpiry().

◆ m_headers

◆ m_headers_list

◆ m_logger

◆ m_minimum_rate

int XrdClHttp::CurlOperation::m_minimum_rate {m_minimum_transfer_rate}
protected

◆ m_minimum_transfer_rate

int CurlOperation::m_minimum_transfer_rate {CurlOperation::m_default_minimum_rate}
staticprotected

Definition at line 309 of file XrdClHttpOps.hh.

Referenced by SetSlowRateBytesSec().

◆ m_operation_expiry

std::chrono::steady_clock::time_point XrdClHttp::CurlOperation::m_operation_expiry
protected

Definition at line 315 of file XrdClHttpOps.hh.

Referenced by XrdClHttp::CurlStatOp::CurlStatOp().

◆ m_url


The documentation for this class was generated from the following files: