XrdHttpTpcTPC.hh

Go to the documentation of this file.

#ifndef XRD_HTTP_TPC_TPC_HH
#define XRD_HTTP_TPC_TPC_HH
#include <memory>
#include <string>
#include <vector>
#include <sys/time.h>
 
#include "XrdSys/XrdSysPthread.hh"
 
#include "XrdHttp/XrdHttpExtHandler.hh"
#include "XrdHttp/XrdHttpUtils.hh"
 
#include "XrdTls/XrdTlsTempCA.hh"
#include "XrdHttpTpcPMarkManager.hh"
 
#include <curl/curl.h>
#include <openssl/ssl.h>
 
class XrdOucErrInfo;
class XrdOucStream;
class XrdSfsFile;
class XrdSfsFileSystem;
class XrdXrootdTpcMon;
typedef void CURL;
 
namespace TPC {
class State;
 
enum LogMask {
    Debug   = 0x01,
    Info    = 0x02,
    Warning = 0x04,
    Error   = 0x08,
    All     = 0xff
};
 
enum class TpcType {
  Pull,
  Push
};
 
struct CurlDeleter {
    void operator()(CURL *curl);
};
using ManagedCurlHandle = std::unique_ptr<CURL, CurlDeleter>;
 
 
class TPCHandler : public XrdHttpExtHandler {
public:
    TPCHandler(XrdSysError *log, const char *config, XrdOucEnv *myEnv);
    virtual ~TPCHandler();
 
    virtual bool MatchesPath(const char *verb, const char *path);
    virtual int ProcessReq(XrdHttpExtReq &req);
    // Abstract method in the base class, but does not seem to be used
    virtual int Init(const char *cfgfile) {return 0;}
    static constexpr std::string_view OSS_TASK_OPAQUE = "oss.task=httptpc";
private:
 
    static int sockopt_callback(void * clientp, curl_socket_t curlfd, curlsocktype purpose);
    static int opensocket_callback(void *clientp,
                                   curlsocktype purpose,
                                   struct curl_sockaddr *address);
 
    static int closesocket_callback(void *clientp, curl_socket_t fd);
    static int ssl_ctx_callback(CURL *curl, void *ssl_ctx, void *clientp);
    static int verify_callback(int preverify_ok, X509_STORE_CTX* ctx);
 
    struct TPCLogRecord {
 
        TPCLogRecord(XrdHttpExtReq &req, const TpcType tpcType)
          : bytes_transferred(-1), status(-1), tpc_status(-1), streams(1), isIPv6(false),
            allow_local(false), allow_private(false), mReq(req), pmarkManager(mReq, tpcType), mTpcType(tpcType)
        {
          gettimeofday(&begT, 0); // Set effective start time
        }
 
       ~TPCLogRecord();
 
        std::string log_prefix;
        std::string local;
        std::string remote;
        std::string name;
        std::string clID;
        // Keeps the shared CA/CRL store alive for the duration of the transfer.
        // Declared here, rather than alongside the curl handle, so that it outlives
        // every handle the request creates -- including the duplicates that
        // multi-stream transfers make, which inherit the pointer to this record.
        std::shared_ptr<X509_STORE> ca_store;
        static XrdXrootdTpcMon* tpcMonitor;
        timeval     begT;
        off_t bytes_transferred;
        int status;
        int tpc_status;
        unsigned int streams;
        bool isIPv6;
        bool allow_local;
        bool allow_private;
        XrdHttpExtReq & mReq;
        XrdHttpTpc::PMarkManager pmarkManager;
        XrdSysError * m_log;
        TpcType mTpcType;
    };
 
    int ProcessOptionsReq(XrdHttpExtReq &req);
 
    static std::string GetAuthz(XrdHttpExtReq &req);
 
    // Configure curl handle's CA settings.  The CA files present here should
    // be valid for the lifetime of the process.  The record takes a reference to
    // the shared CA store, so it must outlive the curl handle.
    //
    // Returns false if no trust anchors could be configured, in which case the
    // transfer must be failed: letting it proceed would leave libcurl verifying
    // against its built-in default CA bundle instead of the configured one.
    bool ConfigureCurlCA(CURL *curl, TPCLogRecord &rec);
 
    // Configure the minimum transfer rate and the permitted time below it.
    void ConfigureCurlLowSpeed(CURL *curl);
 
    // Redirect the transfer according to the contents of an XrdOucErrInfo object.
    int RedirectTransfer(CURL *curl, const std::string &redirect_resource, XrdHttpExtReq &req,
        XrdOucErrInfo &error, TPCLogRecord &);
 
    int OpenWaitStall(XrdSfsFile &fh, const std::string &resource, int mode,
                      int openMode, const XrdSecEntity &sec,
                      const std::string &authz);
 
    int DetermineXferSize(CURL *curl, XrdHttpExtReq &req, TPC::State &state,
                          bool &success, TPCLogRecord &, bool shouldReturnErrorToClient = true);
 
    int GetContentLengthTPCPull(CURL *curl, XrdHttpExtReq &req, uint64_t & contentLength, bool & success, TPCLogRecord &rec);
 
    // Send a 'performance marker' back to the TPC client, informing it of our
    // progress.  The TPC client will use this information to determine whether
    // the transfer is making sufficient progress and/or other monitoring info
    // (such as whether the transfer is happening over IPv4, IPv6, or both).
    int SendPerfMarker(XrdHttpExtReq &req, TPCLogRecord &rec, TPC::State &state);
    int SendPerfMarker(XrdHttpExtReq &req, TPCLogRecord &rec, std::vector<State*> &state,
        off_t bytes_transferred);
 
    // Perform the libcurl transfer, periodically sending back chunked updates.
    int RunCurlWithUpdates(CURL *curl, XrdHttpExtReq &req, TPC::State &state,
                           TPCLogRecord &rec);
 
    // Experimental multi-stream version of RunCurlWithUpdates
    int RunCurlWithStreams(XrdHttpExtReq &req, TPC::State &state,
                           size_t streams, TPCLogRecord &rec);
    int RunCurlWithStreamsImpl(XrdHttpExtReq &req, TPC::State &state,
                           size_t streams, std::vector<TPC::State*> &streams_handles,
                           std::vector<ManagedCurlHandle> &curl_handles,
                           TPCLogRecord &rec);
 
    int ProcessPushReq(const std::string & resource, XrdHttpExtReq &req);
    int ProcessPullReq(const std::string &resource, XrdHttpExtReq &req);
 
    bool ConfigureFSLib(XrdOucStream &Config, std::string &path1, bool &path1_alt,
                        std::string &path2, bool &path2_alt);
    bool Configure(const char *configfn, XrdOucEnv *myEnv);
    bool ConfigureLogger(XrdOucStream &Config);
 
    // Generate a consistently-formatted log message.
    void logTransferEvent(LogMask lvl, const TPCLogRecord &record,
        const std::string &event, const std::string &message="");
 
    std::string generateClientErr(std::stringstream &err_ss, const TPCLogRecord &rec, CURLcode cCode = CURLcode::CURLE_OK);
 
    std::string prepareURL(XrdHttpExtReq &req);
 
    static int m_marker_period;
    static size_t m_block_size;
    static size_t m_small_block_size;
    bool m_allow_local;
    bool m_allow_private;
    bool m_desthttps;
    bool m_fixed_route;  // If 'true' the Destination IP in an HTTP-TPC is forced to be the same as the IP used to contact the server
                           // when 'false' any IP available can be selected
    long m_low_speed_limit; // Minimum transfer rate in bytes per second; zero disables the check.
    long m_low_speed_time; // Time the transfer may remain below m_low_speed_limit before it is aborted.
    int m_timeout; // the 'timeout interval'; if no bytes have been received during this time period, abort the transfer.
    int m_first_timeout; // the 'first timeout interval'; the amount of time we're willing to wait to get the first byte.
                         // Unless explicitly specified, this is 2x the timeout interval.
    std::string m_cadir;  // The directory to use for CAs.
    std::string m_cafile; // The file to use for CAs in libcurl
    static XrdSysMutex m_monid_mutex;
    static uint64_t m_monid;
    XrdSysError m_log;
    XrdSfsFileSystem *m_sfs;
    std::shared_ptr<XrdTlsTempCA> m_ca_file;
 
    // 16 blocks in flight at 16 MB each, meaning that there will be up to 256MB
    // in flight; this is equal to the bandwidth delay product of a 200ms transcontinental
    // connection at 10Gbps.
    static const int m_pipelining_multiplier = 16;
 
    bool usingEC; // indicate if XrdEC is used
 
    static bool allowMissingCRL;
 
    // Whether the libcurl we are linked against supports CURLOPT_SSL_CTX_FUNCTION,
    // which only its OpenSSL-family TLS backends do.  Probed once at configuration
    // time because without it the shared CA store cannot be installed.
    bool m_sslctx_supported{false};
 
    // Time to connect the curl socket to the remote server uses the linux's default value
    // of 60 seconds
    static const long CONNECT_TIMEOUT = 60;
 
    // hdr2cgimap
    std::map<std::string,std::string> hdr2cgimap;
};
}
#endif