path: root/src/httpaction.cc

//
// Created by luqiu on 2018-5-17.
//

#include <cassert>
#include <vector>
#include <pcrecpp.h>

#include "httpaction.h"
#include "http.h"
#include "util.h"
#include "logger.h"

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// 结构化日志记录
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
HttpLogger::HttpLogger(StructLogger * st_logger, unsigned int service_id, unsigned int cfg_id)
    : st_logger_(st_logger), service_id_(service_id), cfg_id_(cfg_id)
{
    return;
}

HttpLogger::~HttpLogger()
{
    if (auto_sendlog_ && need_to_sendlog_) SendLog();
}

/* From: PanGu数据库表设计文件
 * Version:

=======================================================================================================================
数据名称        数据类型    字段说明               可空      备注
=======================================================================================================================
url             string      url地址                N
req_hdr_file    string      请求头转储文件         Y       后缀名为”.txt”
req_body_file   string      请求体转储文件         Y       无后缀名
res_hdr_file    string      应答头转储文件         Y       后缀名为”.txt”
res_body_file   string      应答体体转储文件       Y       无后缀名

======================================================================================================================
数据名称            数据类型    字段说明            可空  备注
=======================================================================================================================
isn                 int         TCP初始序列号         Y       用于单向流对准
proxy_flag          int         是否HTTP代理标志  	 Y       用于单向流对准
http_seq            int         HTTP会话序列号       Y       用于单向流对准
req_line            string      请求行               Y
res_line            string      应答行               Y
cookie              string      Cookie值             Y
referer             string      Referer值            Y
user_agent          string      UA值                 Y
user_define_key     string      用户自定义头域名称    Y       记录命中的用户自定义域名称，如X-header
user_define_value   string      用户自定义域值        Y       记录命中的用户自定义域值，如xxx.cloudfront.net
*/

void HttpLogger::ConstructByConnection(const HttpConnection & conn)
{
    sk_src_ = conn.SockAddrSource();
    sk_dst_ = conn.SockAddrDest();
}

void HttpLogger::ConstructByRequestHeader(const HttpRequest & request)
{
    /* 请求侧数据 */
    st_log_["url"] = request.Url();
    const auto & c_headers = request.cHeaders();

    /* 取头部特定字段 */
    c_headers.ForEachValueOfHeader("Cookie", [this](const std::string & field, const std::string & value)
    {
        st_log_["cookie"] = value;
        return false;
    });

    c_headers.ForEachValueOfHeader("Referer", [this](const std::string & field, const std::string & value)
    {
        st_log_["referer"] = value;
        return false;
    });

    c_headers.ForEachValueOfHeader("User-Agent", [this](const std::string & field, const std::string & value)
    {
        st_log_["user_agent"] = value;
        return false;
    });

    std::string str_req_header;
    c_headers.ForEachHeader([&str_req_header, this](const std::string & field, const std::string & value)
    {
        str_req_header += field + ":" + value + "\r\n";
        return true;
    });

    st_log_["req_header"] = str_req_header;
    need_to_sendlog_ = true;
}

void HttpLogger::ConstructByRequestBody(const HttpRequest & request)
{
    return;
}

void HttpLogger::ConstructByResponseHeader(const HttpResponse & resp)
{
    const auto & c_headers = resp.cHeaders();

    /* 取头部特定字段 */
    c_headers.ForEachValueOfHeader("Content-Length", [this](const std::string & field, const std::string & value)
    {
        st_log_["content_len"] = value;
        return false;
    });

    c_headers.ForEachValueOfHeader("Content-Type", [this](const std::string & field, const std::string & value)
    {
        st_log_["content_type"] = value;
        return false;
    });

    c_headers.ForEachValueOfHeader("Set-Cookie", [this](const std::string & field, const std::string & value)
    {
        st_log_["set_cookie"] = value;
        return false;
    });

    std::string str_resp_header;
    c_headers.ForEachHeader([&str_resp_header, this](const std::string & field, const std::string & value)
    {
        str_resp_header += field + ":" + value + "\r\n";
        return true;
    });

    st_log_["resp_header"] = str_resp_header;
    need_to_sendlog_ = true;
}

void HttpLogger::ConstructByResponseBody(const HttpResponse & resp)
{
    return;
}

void HttpLogger::SendLog()
{
    st_logger_->CommonLogMake(sk_src_, sk_dst_, service_id_, cfg_id_, st_log_);
    st_logger_->SendLog(topic_id_, std::move(st_log_));
}

class HttpActionCommon : public HttpAction
{
public:
    void OnRequestHeader(HttpSession * session) override
    {
        if (logger_ != nullptr)
        {
            logger_->ConstructByRequestHeader(session->request());
        }

        return __on_request_header(session);
    }

    void OnRequestBody(HttpSession * session) override
    {
        if (logger_ != nullptr)
        {
            logger_->ConstructByRequestBody(session->request());
        }

        return __on_request_body(session);
    }

    void OnResponseHeader(HttpSession * session) override
    {
        if (logger_ != nullptr)
        {
            logger_->ConstructByResponseHeader(session->response());
        }

        return __on_response_header(session);
    }

    void OnResponseBody(HttpSession * session) override
    {
        if (logger_ != nullptr)
        {
            logger_->ConstructByResponseBody(session->response());
        }

        return __on_response_body(session);
    }

    void LoggerSetup(std::unique_ptr<HttpLogger> logger) override
    { logger_ = std::move(logger); }

    void LoggerClear() override
    { logger_ = nullptr; }

    std::string DumpToString() override
    { return str_dump_;}

protected:
    std::unique_ptr<HttpLogger> logger_{};
    std::string str_dump_;

protected:
    virtual void __on_request_header(HttpSession * session) = 0;
    virtual void __on_request_body(HttpSession * session) = 0;
    virtual void __on_response_header(HttpSession * session) = 0;
    virtual void __on_response_body(HttpSession * session) = 0;
};

class HttpActionMonitor : public HttpActionCommon
{
public:
    virtual void Construct(const std::string & str_service_define)
    { str_dump_ = "ACTION-MONITOR"; }

private:
    void __on_request_header(HttpSession * session) override
    { return; }
    virtual void __on_request_body(HttpSession * session)
    { return; }
    virtual void __on_response_header(HttpSession * session)
    { return; }
    virtual void __on_response_body(HttpSession * session)
    { return; }
};

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// HTTP白名单，不进行任何处理
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

class HttpActionBypass : public HttpAction
{
public:
    void Construct(const std::string & str_service_define) override
    {}

    std::string DumpToString() override
    { return "ACTION-WHITELIST"; }

    void OnRequestHeader(HttpSession * session) override
    { return __set_session_bypass(session); }
    void OnRequestBody(HttpSession * session) override
    { return __set_session_bypass(session); }
    void OnResponseHeader(HttpSession * session) override
    { return __set_session_bypass(session); }
    void OnResponseBody(HttpSession * session) override
    { return __set_session_bypass(session); }

    void LoggerSetup(std::unique_ptr<HttpLogger> logger) override
    {}
    void LoggerClear() override
    {}

private:
    void __set_session_bypass(HttpSession * session)
    { session->Bypass(); }
};

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// HTTP重定向
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

class HttpActionRedirect : public HttpActionCommon
{
public:
    void Construct(const std::string & str_service_define) override;

protected:
    void __on_request_header(HttpSession * session) override
    { return do_redirect_action(session); }

    void __on_request_body(HttpSession * session) override
    { return do_redirect_action(session); }

    void __on_response_header(HttpSession * session) override
    {};

    void __on_response_body(HttpSession * session) override
    {};

private:
    unsigned int resp_code_{500};

    bool is_resp_location_set{false};
    std::string resp_location_{};

    bool is_resp_content_set{false};
    std::string resp_content_{};

    void do_redirect_action(HttpSession * session);
};

void HttpActionRedirect::Construct(const std::string & str_kv)
{
    std::vector<std::string> __split_token;
    tokenize(str_kv, __split_token, ";", true);

    if (__split_token.size() != 2)
    {
        throw std::invalid_argument(string_format(
            "Not enough tokens: %s, need two tokens.", str_kv.c_str()));
    }

    for (const auto & kv_iterate : __split_token)
    {
        std::vector<std::string> __kv_tokens;
        tokenize(kv_iterate, __kv_tokens, "=", false);

        if (__kv_tokens.size() != 2)
        {
            throw std::invalid_argument(string_format(
                "Token %s must be conposed by key and value", kv_iterate.c_str()));
        }

        const std::string & __str_key = __kv_tokens[0];
        const std::string & __str_value = __kv_tokens[1];

        if (__str_key == "code")
        {
            resp_code_ = static_cast<unsigned int>(std::stoul(__str_value));
            continue;
        }

        if (__str_key == "url")
        {
            resp_location_ = __str_value;
            is_resp_location_set = true;
            continue;
        }

        if (__str_key == "content")
        {
            resp_content_ = __str_value;
            is_resp_content_set = true;
            continue;
        }

        throw std::invalid_argument(string_format(
            "Illegal redirect keyrules: %s", __str_key.c_str()));
    }

    str_dump_ = "ACTION-REDIRECT ";
}

void HttpActionRedirect::do_redirect_action(HttpSession * session)
{
    auto & http_connection = session->connection();

    /* 创建新的HttpResponse */
    auto http_session = std::make_unique<HttpSession>(http_connection);

    /* 根据当前的Http协议版本构造新的请求、应答 */
    auto request_version = session->request().Version();
    auto version_major = std::get<0>(request_version);
    auto version_minor = std::get<1>(request_version);

    http_session->request(HttpRequestFactory(version_major, version_minor));
    http_session->response(HttpResponseFactory(version_major, version_minor));

    /* 构建Redirect Response */
    auto & http_response = http_session->response();

    http_response.ResponseCode(resp_code_);

    /* Location字段 */
    if (is_resp_location_set)
    {
        http_response.Headers().Add("Location", resp_location_);
    }

    /* 应答内容 */
    if (is_resp_content_set)
    {
        auto body_segment_ptr = std::make_unique<HttpResponse::body_content_t>();
        body_segment_ptr->insert(body_segment_ptr->end(), resp_content_.begin(), resp_content_.end());

        auto body_segment_vec = std::vector<decltype(body_segment_ptr)>();
        body_segment_vec.push_back(std::move(body_segment_ptr));

        http_response.Body(std::move(body_segment_vec));
    }

    /* 调试记录 */
    str_dump_ = string_format("%d %s", resp_code_, resp_location_.c_str());

    http_response.Construct();
    http_connection.Write(std::move(http_session));

    /* 丢弃当前的HTTP Session */
    session->Drop();
    return;
}

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// HTTP连接阻断
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

class HttpActionBlock : public HttpActionCommon
{
public:
    void Construct(const std::string & str_service_define) override
    {}

protected:
    void __on_request_header(HttpSession * session) override
    { do_block_action(session); }

    void __on_request_body(HttpSession * session) override
    { do_block_action(session); }

    void __on_response_header(HttpSession * session) override
    { do_block_action(session); };

    void __on_response_body(HttpSession * session) override
    { do_block_action(session); };

private:
    void do_block_action(HttpSession * session);
};

void HttpActionBlock::do_block_action(HttpSession * session)
{
    session->connection().Close();
}

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class HttpActionDrop : public HttpActionCommon
{
public:
    void Construct(const std::string & str_service_define) override
    {}

protected:
    void __on_request_header(HttpSession * session) override
    { do_drop_action(session); }

    void __on_request_body(HttpSession * session) override
    { do_drop_action(session); }

    void __on_response_header(HttpSession * session) override
    { do_drop_action(session); };

    void __on_response_body(HttpSession * session) override
    { do_drop_action(session); };

private:
    void do_drop_action(HttpSession * session);
};

void HttpActionDrop::do_drop_action(HttpSession * session)
{
    session->Drop();
};

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// HTTP内容编辑功能实现
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

class HttpActionReplace : public HttpActionCommon
{
public:
    void Construct(const std::string & str_kv) override;

protected:
    void __on_request_header(HttpSession * session) override;
    void __on_request_body(HttpSession * session) override;
    void __on_response_header(HttpSession * session) override;
    void __on_response_body(HttpSession * session) override;

private:
    enum edit_zone
    {
        kZoneRequestUri,
        kZoneRequestHeaders,
        kZoneRequestBody,
        kZoneResponseHeader,
        kZoneResponseBody,
        kZoneMax
    };

    struct edit_rule
    {
        pcrecpp::RE regex;
        std::string fmt;
    };

    /* Edit Zone to string, define in maat service define */
    static std::array<std::string, kZoneMax> map_edit_zone_to_str;
    std::array<std::vector<struct edit_rule>, kZoneMax> edit_rules_{};

    /* Scan and Replace */
    bool __scan_and_replace(enum edit_zone, std::string & raw);
    bool __is_replaced{false};
};

std::array<std::string, HttpActionReplace::kZoneMax> HttpActionReplace::map_edit_zone_to_str =
    {
        "http_req_uri",
        "http_req_headers",
        "http_req_body",
        "http_resp_headers",
        "http_resp_body"
    };

void HttpActionReplace::Construct(const std::string & str_kv)
{
    std::vector<std::string> __split_token;
    tokenize(str_kv, __split_token, ";", false, [](const std::string & str, std::string::size_type pos)
    {
        return (pos == 0 || str[pos - 1] != '\\');
    });

    if (__split_token.size() % 2 != 0)
    {
        throw std::invalid_argument(string_format(
            "Invalid edit rule: %s, the count of tokens must be even numbers.", str_kv.c_str()));
    }

    for (auto __iterate_zone = __split_token.cbegin(), __iterate_regex = __split_token.cbegin() + 1;
         __iterate_zone != __split_token.cend(); __iterate_zone += 2, __iterate_regex += 2)
    {
        std::vector<std::string> __kv_tokens_zone;
        std::vector<std::string> __kv_tokens_regex;

        tokenize(*__iterate_zone, __kv_tokens_zone, "=");
        tokenize(*__iterate_regex, __kv_tokens_regex, "=");

        if (__kv_tokens_zone.size() != 2 || __kv_tokens_regex.size() != 2)
        {
            throw std::invalid_argument(string_format(
                "Invalid edit rule: %s, the tokens must composed by key-value. ", str_kv.c_str()));
        }

        if (__kv_tokens_zone[0] != "zone" || __kv_tokens_regex[0] != "regex")
        {
            throw std::invalid_argument(string_format(
                "Invalid edit rule: %s, the tokens' key must be 'zone' or 'regex'", str_kv.c_str()));
        }

        const std::string & __kv_zone = __kv_tokens_zone[1];
        const std::string & __kv_regex = __kv_tokens_regex[1];

        LOG(DEBUG) << "KvZone = " << __kv_zone << " " << "KvRegex = " << __kv_regex;

        /* 查找strZone对应的Zone数值 */
        auto zone_iterate = std::find(map_edit_zone_to_str.cbegin(), map_edit_zone_to_str.cend(), __kv_zone);
        if (zone_iterate == map_edit_zone_to_str.cend())
        {
            throw std::invalid_argument("Invalid edit rule: %s, illegal zone. ");
        }

        auto zone_id = static_cast<edit_zone>(zone_iterate - map_edit_zone_to_str.cbegin());
        auto & zone_rule_ref = edit_rules_[zone_id];

        std::vector<std::string> __token_regex;

        /* 按'/'拆分正则串，判断转义字符，若/前面是\，则不认为该字符为拆分字符 */
        tokenize(__kv_regex, __token_regex, "/", true, [](const std::string & str, std::string::size_type pos)
        {
            return (pos == 0 || str[pos - 1] != '\\');
        });

        if (__token_regex.size() != 2)
        {
            throw std::invalid_argument(string_format(
                "Invalid edit rule: %s, the regex must composed by two args.", str_kv.c_str()));
        }

        const std::string & __kv_regex_base = __token_regex[0];
        const std::string & __kv_regex_fmt = __token_regex[1];

        /* 编译正则表达式 */
        pcrecpp::RE __re(__kv_regex_base);
        if (__re.error() != "")
        {
            throw std::invalid_argument(string_format(
                "Invalid edit rule: %s, illegal regex expr, %s", str_kv.c_str(), __re.error().c_str()));
        }

        struct edit_rule __temp_edit_rule = {std::move(__re), std::move(__kv_regex_fmt)};
        zone_rule_ref.push_back(std::move(__temp_edit_rule));
    }

    str_dump_ = "ACTION-REPLACE ";
    return;
}

void HttpActionReplace::__on_request_header(HttpSession * session)
{
    auto & request = session->request();
    bool replaced = false;

    /* 对URI替换 */
    std::string str_uri = request.Uri();
    if (__scan_and_replace(kZoneRequestUri, str_uri))
    {
        request.Uri(str_uri);
        replaced = true;
    }

    /* 对Headers替换 */
    request.Headers().ForEachHeader([this, &request, &replaced]
        (const std::string & str_field, const std::string str_value) -> bool
    {
        /* Field, Value组合成字符串，整体调用正则扫描 */
        std::string __combine_header = str_field + ":" + str_value;

        /* 没有命中，继续调用本函数处理后面的Headers */
        if (!__scan_and_replace(kZoneRequestHeaders, __combine_header))
            return true;

        /* 替换标志 */
        replaced = true;

        /* 命中，若替换后的长度为0，删除该头部 */
        if (__combine_header.length() == 0)
        {
            request.Headers().Remove(str_field);
            return true;
        }

        /* 按分号拆分成Field和Value */
        auto first_comma_pos = __combine_header.find_first_of(':');
        if (first_comma_pos == std::string::npos)
        {
            throw std::runtime_error("Invalid regex replacement for http headers, no comma found.");
        }

        auto __replaced_field = __combine_header.substr(0, first_comma_pos);
        auto __replaced_value = __combine_header.substr(first_comma_pos + 1);

        request.Headers().Set(__replaced_field, __replaced_value);
        return true;
    });

    if (replaced)
    {
        str_dump_ += "REQ-HEADER ";
        request.Construct();
    }

    return;
}

void HttpActionReplace::__on_request_body(HttpSession * session)
{
    /* Request Object */
    auto & request = session->request();
    /* Request Body */
    auto * request_body = request.Body();
    /* Replace Result */
    std::string str_output(request_body->begin(), request_body->end());

    /* Replace Tag */
    if (__scan_and_replace(kZoneRequestBody, str_output))
    {
        auto new_body_content = std::make_unique<HttpRequest::body_content_t>();
        std::copy(str_output.cbegin(), str_output.cend(), std::back_inserter(*new_body_content));

        request.Headers().Set("Content-Length", std::to_string(new_body_content->size()));
        request.Body(std::move(new_body_content));
        request.Construct();

        str_dump_ += "REQ-BODY ";
    }

    return;
}

void HttpActionReplace::__on_response_header(HttpSession * session)
{
    auto & response = session->response();

    /* 对Headers替换 */
    response.Headers().ForEachHeader([this, &response]
        (const std::string & str_field, const std::string str_value) -> bool
    {
        /* Field, Value组合成字符串，整体调用正则扫描 */
        std::string __combine_header = str_field + ":" + str_value;

        /* 没有命中，继续调用本函数处理后面的Headers */
        if (!__scan_and_replace(kZoneResponseHeader, __combine_header))
            return true;

        /* 替换标志 */
        __is_replaced = true;

        /* 命中，若替换后的长度为0，删除该头部 */
        if (__combine_header.length() == 0)
        {
            response.Headers().Remove(str_field);
            return true;
        }

        /* 按分号拆分成Field和Value */
        auto first_comma_pos = __combine_header.find_first_of(':');
        if (first_comma_pos == std::string::npos)
        {
            throw std::runtime_error("Invalid regex replacement for http headers, no comma found.");
        }

        auto __replaced_field = __combine_header.substr(0, first_comma_pos);
        auto __replaced_value = __combine_header.substr(first_comma_pos + 1);

        response.Headers().Set(__replaced_field, __replaced_value);
        return true;
    });

    return;
}

void HttpActionReplace::__on_response_body(HttpSession * session)
{
    auto & response = session->response();
    auto resp_bodys = response.StolenBody();

    size_t total_body_bytes = 0;
    std::vector<HttpResponse::body_content_ptr_t> body_replaced_segments;

    for (auto & body_raw_ptr : resp_bodys)
    {
        std::string str_body_raw = std::string(body_raw_ptr->begin(), body_raw_ptr->end());
        std::unique_ptr<std::vector<char>> body_replaced_ptr;

        if (__scan_and_replace(kZoneResponseBody, str_body_raw))
        {
            body_replaced_ptr = std::make_unique<std::vector<char>>();
            body_replaced_ptr->insert(body_replaced_ptr->end(), str_body_raw.begin(), str_body_raw.end());
            __is_replaced = true;
        }
        else
        {
            body_replaced_ptr = std::move(body_raw_ptr);
        }

        total_body_bytes += body_replaced_ptr->size();
        body_replaced_segments.push_back(std::move(body_replaced_ptr));
    }

    response.Body(std::move(body_replaced_segments));

    /* 发生替换后，重新构造应答内容 */
    if (__is_replaced)
    {
        bool is_content_length_set = false;

        /* 原始头部是否有Content-Length字段 */
        response.cHeaders().ForEachValueOfHeader("Content-Length", [&is_content_length_set](
            const std::string _, const std::string __)
        {
            is_content_length_set = true;
            return false;
        });

        /* 如果有，需要根据替换后的内容校正该数值 */
        if (is_content_length_set)
        {
            response.Headers().Set("Content-Length", std::to_string(total_body_bytes));
        }

        /* 构建应答内容 */
        response.Construct();
        str_dump_ += "RESP-BODY";
    }

    return;
}

bool HttpActionReplace::__scan_and_replace(enum edit_zone zone, std::string & raw)
{
    /* 扫描规则库，逐条执行 */
    const auto & edit_rule = edit_rules_[zone];
    bool replaced = false;

    for (const auto & edit_rule_iter : edit_rule)
    {
        /* 正则表达式，确定要替换的内容 */
        const auto & __edit_regex = edit_rule_iter.regex;
        /* 替换表达式，替换方法 */
        const auto & __edit_fmt = edit_rule_iter.fmt;
        /* 执行替换 */
        replaced = __edit_regex.Replace(__edit_fmt, &raw);
        if (replaced) break;
    }

    return replaced;
}

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// 工厂函数
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
std::shared_ptr<HttpAction> HttpActionFactory(enum HttpActionType type, std::string str_service_define)
{
    std::shared_ptr<HttpAction> __http_action_object;

    switch (type)
    {
        case kActionBypass:
        {
            __http_action_object = std::make_shared<HttpActionBypass>();
            break;
        }
        case kActionMonitor:
        {
            __http_action_object = std::make_shared<HttpActionMonitor>();
            break;
        }
        case kActionReplace:
        {
            __http_action_object = std::make_shared<HttpActionReplace>();
            break;
        }
        case kActionBlock:
        {
            __http_action_object = std::make_shared<HttpActionBlock>();
            break;
        }
        case kActionDrop:
        {
            __http_action_object = std::make_shared<HttpActionDrop>();
            break;
        }
        case kActionRedirect:
        {
            __http_action_object = std::make_shared<HttpActionRedirect>();
            break;
        }
        default: assert(0);
    }

    __http_action_object->Construct(str_service_define);
    return std::move(__http_action_object);
}

std::unique_ptr<HttpLogger> HttpLoggerFactory(int service_id, int cfg_id)
{
    return std::make_unique<HttpLogger>(g_tfe_instance->struct_logger_module.get(),
        static_cast<unsigned int>(service_id), static_cast<unsigned int>(cfg_id));
}