From 4636f27eec7ea502735a599242c7cbc216da613d Mon Sep 17 00:00:00 2001 From: 崔一鸣 Date: Tue, 21 May 2019 16:54:19 +0800 Subject: 修改目录结构, 添加文档 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 2 +- README.md | 38 ++ adapter/CMakeLists.txt | 3 + adapter/http_adapter/conf/http_adapter.inf | 9 + adapter/http_adapter/lua/conflist_lua.inf | 2 + .../http_adapter/lua/http_request/http_request.inf | 3 + .../http_adapter/lua/http_request/http_request.lua | 87 ++++ .../lua/http_response/http_response.inf | 3 + .../lua/http_response/http_response.lua | 57 +++ adapter/http_adapter/script/deploy.sh | 11 + adapter/http_adapter/src/adapter.cpp | 458 ++++++++++++++++++++ adapter/http_adapter/src/adapter.h | 112 +++++ picture/platform.png | Bin 0 -> 39382 bytes picture/plugin.png | Bin 0 -> 174449 bytes test/CMakeLists.txt | 15 - test/new_http_service/adapter.cpp | 471 --------------------- test/new_http_service/adapter.h | 112 ----- test/new_http_service/http_request.lua | 87 ---- test/new_http_service/http_response.lua | 57 --- test/test_coroutine/main.cpp | 226 ---------- test/test_coroutine/test.lua | 63 --- test/test_plugin/http_service.cpp | 183 -------- test/test_plugin/http_service.h | 51 --- vendor/CMakeLists.txt | 4 +- 24 files changed, 786 insertions(+), 1268 deletions(-) create mode 100644 README.md create mode 100644 adapter/CMakeLists.txt create mode 100644 adapter/http_adapter/conf/http_adapter.inf create mode 100644 adapter/http_adapter/lua/conflist_lua.inf create mode 100644 adapter/http_adapter/lua/http_request/http_request.inf create mode 100644 adapter/http_adapter/lua/http_request/http_request.lua create mode 100644 adapter/http_adapter/lua/http_response/http_response.inf create mode 100644 adapter/http_adapter/lua/http_response/http_response.lua create mode 100644 adapter/http_adapter/script/deploy.sh create mode 100644 adapter/http_adapter/src/adapter.cpp create mode 100644 adapter/http_adapter/src/adapter.h create mode 100644 picture/platform.png create mode 100644 picture/plugin.png delete mode 100644 test/CMakeLists.txt delete mode 100644 test/new_http_service/adapter.cpp delete mode 100644 test/new_http_service/adapter.h delete mode 100644 test/new_http_service/http_request.lua delete mode 100644 test/new_http_service/http_response.lua delete mode 100644 test/test_coroutine/main.cpp delete mode 100644 test/test_coroutine/test.lua delete mode 100644 test/test_plugin/http_service.cpp delete mode 100644 test/test_plugin/http_service.h diff --git a/CMakeLists.txt b/CMakeLists.txt index bbf8ca0..90a2387 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,4 +17,4 @@ if (CMAKE_BUILD_TYPE STREQUAL Debug) endif() add_subdirectory(vendor) -add_subdirectory(test) \ No newline at end of file +add_subdirectory(adapter) \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..bdc7924 --- /dev/null +++ b/README.md @@ -0,0 +1,38 @@ +* 简介 + * lua_sapp框架可以看成是lua和sapp平台的一个Adapter。可以使sapp的业务层插件使用lua语言编写。这样做有以下优点: + * lua是高级语言，语法简单，有自动垃圾回收，对程序员要求低 + * 业务层逻辑由异步回调变成同步阻塞(利用lua的协程提供了一组同步api供业务层调用) + * 平台内部的数据结构(如stream_info)对业务层透明 + * 增加了程序的健壮性，每个业务层插件运行在自己的lua虚拟机内，各个插件之间相互隔离。 + * 平台层次结构如下图所示 + * ![平台结构图](./picture/platform.png) + + +* 设计思路 + * 基本思路 + * 在设计上，lua_sapp由多个adapter组成，如http_adapter, ssl_adapter，dns_adapter等。 + * 每个adapter都是一个业务层插件，从对应的解析层获取流量(如http_adapter从http解析层获取流量)，然后调用注册了这个adapter的lua脚本。 + * lua脚本在./plug/lua下，通过配置文件中entry_type注册对应的adapter + * 一个典型的插件组织如下: + * ![插件组织](./picture/plugin.png) + + + * 线程/协程模型 + * 每个流(如http stream)对应一个协程 + * m个线程，n个业务层插件，对应 m * n个lua虚拟机 +* 安装配置 + * 一键脚本 + + ```bash + git clone git@git.mesalab.cn:cuiyiming/lua_sapp.git + cd lua_sapp && mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Debug ../ && make + sh ../adapter/http_adapter/script/deploy.sh + ``` +* 测试环境 + * 目前192.168.11.137:/home/mesasoft/sapp_run1下面有一套配好的测试环境，直接运行sapp即可看到效果 + * 日志路径: ./log/lua_sapp +* TODO + * 目前只写了http协议的adapter, 其他如ssl, dns协议的adapter还未编写 + * 目前处于原型阶段，所以大量使用了C++的STL库，需要改成C语言 + * 代码缺少注释，但是可读性还不错，基本可以自解释 \ No newline at end of file diff --git a/adapter/CMakeLists.txt b/adapter/CMakeLists.txt new file mode 100644 index 0000000..92dedbf --- /dev/null +++ b/adapter/CMakeLists.txt @@ -0,0 +1,3 @@ +### http_adapter +add_library(http_adapter SHARED http_adapter/src/adapter.cpp) +target_link_libraries(http_adapter lua-static dl MESA_handle_logger MESA_htable MESA_prof_load) \ No newline at end of file diff --git a/adapter/http_adapter/conf/http_adapter.inf b/adapter/http_adapter/conf/http_adapter.inf new file mode 100644 index 0000000..d9b827c --- /dev/null +++ b/adapter/http_adapter/conf/http_adapter.inf @@ -0,0 +1,9 @@ +[PLUGINFO] +PLUGNAME = http_adapter +SO_PATH = ./plug/business/http_adapter/libhttp_adapter.so +INIT_FUNC = http_adapter_init +DESTROY_FUNC = http_adapter_destroy + +[HTTP] +FUNC_FLAG = ALL +FUNC_NAME = http_adapter_http_entry diff --git a/adapter/http_adapter/lua/conflist_lua.inf b/adapter/http_adapter/lua/conflist_lua.inf new file mode 100644 index 0000000..401ea6b --- /dev/null +++ b/adapter/http_adapter/lua/conflist_lua.inf @@ -0,0 +1,2 @@ +./plug/lua/http_request/http_request.inf +./plug/lua/http_response/http_response.inf diff --git a/adapter/http_adapter/lua/http_request/http_request.inf b/adapter/http_adapter/lua/http_request/http_request.inf new file mode 100644 index 0000000..d8d37d6 --- /dev/null +++ b/adapter/http_adapter/lua/http_request/http_request.inf @@ -0,0 +1,3 @@ +[main] +file_path = ./plug/lua/http_request/http_request.lua +entry_type = http \ No newline at end of file diff --git a/adapter/http_adapter/lua/http_request/http_request.lua b/adapter/http_adapter/lua/http_request/http_request.lua new file mode 100644 index 0000000..9d9197f --- /dev/null +++ b/adapter/http_adapter/lua/http_request/http_request.lua @@ -0,0 +1,87 @@ +---http_request.lua: 结果打印出来 +function printf(s,...) + io.write(s:format(...)) + io.flush() +end + +function format_write_file(file, s, ...) + file:write(s:format(...)) + file:flush() +end + +function init() + ---init something +end + +function print_header(header) + for k, v in pairs(header) do + if k == "HTTP_OTHER_REGIONS" then + format_write_file(file, "%s: ", k) + for k, v in pairs(header["HTTP_OTHER_REGIONS"]) do + format_write_file(file, "{%s} ", v) + end + format_write_file(file, "\n") + else + format_write_file(file, "%s: %s\n", k, v) + end + end +end + +function process() + printf("\nprint stream: \n") + stream_info = get_stream_info() + for k, v in pairs(stream_info) do + printf("%s: %s\n", k, v) + end + + printf("\nprint header:\n") + local regions = {"ALL"} + header = get_http_request_header(regions) + for k, v in pairs(header) do + printf("%s: %s\n", k, v) + end + + printf("\nprint body: \n") + while true do + body = get_http_request_body() + for k, v in pairs(body) do + printf("%s: %s\n", k, v) + end + if body['data_end'] == true then + break + end + end + + --[[ + file = io.open("./log/lua_sapp/http_adapter/lua/http_request.log", "a+") + format_write_file(file, "lua: call process\n") + + format_write_file(file, "\nprint stream info: \n") + stream_info = get_stream_info() + for k, v in pairs(stream_info) do + format_write_file(file, "%s: %s\n", k, v) + end + + format_write_file(file, "\nprint request headers: \n") + local req_regions = {"HTTP_HOST", "HTTP_REQ_LINE", "HTTP_OTHER_REGIONS"} + req_header = get_http_request_header(req_regions) + print_header(req_header) + + + format_write_file(file, "\nprint request body: \n") + while true do + body = get_http_request_body() + for k, v in pairs(body) do + format_write_file(file, "%s: %s\n", k, v) + end + if body['data_end'] == true then + format_write_file(file, "request body end\n") + break + end + format_write_file(file, "\n"); + end + format_write_file(file, "\nlua: process end\n") + io.close(file) + ]] + +end \ No newline at end of file diff --git a/adapter/http_adapter/lua/http_response/http_response.inf b/adapter/http_adapter/lua/http_response/http_response.inf new file mode 100644 index 0000000..e80eef9 --- /dev/null +++ b/adapter/http_adapter/lua/http_response/http_response.inf @@ -0,0 +1,3 @@ +[main] +file_path = ./plug/lua/http_response/http_response.lua +entry_type = http \ No newline at end of file diff --git a/adapter/http_adapter/lua/http_response/http_response.lua b/adapter/http_adapter/lua/http_response/http_response.lua new file mode 100644 index 0000000..7e23fa3 --- /dev/null +++ b/adapter/http_adapter/lua/http_response/http_response.lua @@ -0,0 +1,57 @@ + +---结果输出到文件中: ./log/lua_sapp/http_adapter/lua/http_response.log +function printf(s,...) + io.write(s:format(...)) + io.flush() +end + +function format_write_file(file, s, ...) + file:write(s:format(...)) + file:flush() +end + +function init() + ---init something +end + +function print_header(header) + for k, v in pairs(header) do + if k == "HTTP_OTHER_REGIONS" then + format_write_file(file, "%s: ", k) + for k, v in pairs(header["HTTP_OTHER_REGIONS"]) do + format_write_file(file, "{%s} ", v) + end + format_write_file(file, "\n") + else + format_write_file(file, "%s: %s\n", k, v) + end + end +end + +function process() + file = io.open("./log/lua_sapp/http_adapter/lua/http_response.log", "a+") + format_write_file(file, "lua: call process\n") + format_write_file(file, "\nprint stream info: \n") + stream_info = get_stream_info() + for k, v in pairs(stream_info) do + format_write_file(file, "%s: %s\n", k, v) + end + + local resp_regions = {"HTTP_SERVER", "HTTP_RES_LINE", "HTTP_OTHER_REGIONS"} + format_write_file(file, "\nprint response headers: \n") + resp_header = get_http_response_header(resp_regions) + print_header(resp_header) + + format_write_file(file, "\nprint response body: \n") + while true do + body = get_http_response_body() + for k, v in pairs(body) do + format_write_file(file, "%s: %s\n", k, v) + end + if body['data_end'] == true then + format_write_file(file, "response body end\n") + break + end + end + format_write_file(file, "\nlua: process end\n") +end \ No newline at end of file diff --git a/adapter/http_adapter/script/deploy.sh b/adapter/http_adapter/script/deploy.sh new file mode 100644 index 0000000..574960a --- /dev/null +++ b/adapter/http_adapter/script/deploy.sh @@ -0,0 +1,11 @@ +SAPP_RUN=$1 +if [ "${SAPP_RUN}" = "" ]; then +echo "USAGE: sh build_rpm.sh sapp_path" && exit 0 +fi +mkdir -p $SAPP_RUN/plug/business/http_adapter +/bin/cp -f adapter/libhttp_adapter.so $SAPP_RUN/plug/business/http_adapter/libhttp_adapter.so +/bin/cp -f ../adapter/http_adapter/conf/http_adapter.inf $SAPP_RUN/plug/business/http_adapter/http_adapter.inf +/bin/cp -rf ../adapter/http_adapter/lua $SAPP_RUN/plug +mkdir -p $SAPP_RUN/log/lua_sapp/http_adapter/lua/ +cat $SAPP_RUN/plug/business/conflist_business.inf | grep "http_adapter.inf" >/dev/null 2>&1 && exit +echo ./plug/business/http_adapter/http_adapter.inf >> $SAPP_RUN/plug/business/conflist_business.inf \ No newline at end of file diff --git a/adapter/http_adapter/src/adapter.cpp b/adapter/http_adapter/src/adapter.cpp new file mode 100644 index 0000000..94cd70f --- /dev/null +++ b/adapter/http_adapter/src/adapter.cpp @@ -0,0 +1,458 @@ +#include "adapter.h" + +//TODO: +//每个函数都要判断返回值 done +// lua错误处理，比如函数没有注册，resume的返回值判断然后打印错误, 不再resume该协程 done +//压栈之前要check_stack done +//流结束clear_ctx之外还要释放该协程，自动垃圾回收？做实验验证一下 done +//destroy，释放lua_state done +//多线程 m * n个虚拟机(m是线程数量，n是lua插件数量) done +//错误处理，日志, 工程化，日志完备 + +static void* g_logger = NULL; +static std::vector> g_http_lua_states; +extern int g_iThreadNum; + +static void lua_traceback(const char *func_name, lua_State *lua_state, int ret){ + int n = lua_gettop(lua_state); + MESA_handle_runtime_log(g_logger, RLOG_LV_FATAL, LOG_MODULE_NAME, "%s error: ret is %d, traceback is: ", func_name, ret); + for(int i = -1; i >= 0 - n; i--){ + std::string type = std::string(lua_typename(lua_state, lua_type(lua_state, i))); + if(type == "string"){ + MESA_handle_runtime_log(g_logger, RLOG_LV_FATAL, LOG_MODULE_NAME, "%s", lua_tostring(lua_state, i)); + } + } + lua_pop(lua_state, n); +} + +std::string _inet_ntoa(uint32_t ip) +{ + const char *_ip = inet_ntoa(*(struct in_addr *)&ip); + return std::string(_ip); +} + +//copy from suricata +static int lua_push_string_buffer(lua_State *lua_state, const char *input, size_t input_len) +{ + if (input_len % 4 != 0) { + /* we're using a buffer sized at a multiple of 4 as lua_pushlstring generates + * invalid read errors in valgrind otherwise. Adding in a nul to be sure. + * Buffer size = len + 1 (for nul) + whatever makes it a multiple of 4 */ + size_t buflen = input_len + 1 + ((input_len + 1) % 4); + char buf[buflen]; + memset(buf, 0x00, buflen); + memcpy(buf, input, input_len); + buf[input_len] = '\0'; + /* return value through lua_state, as a luastring */ + lua_pushlstring(lua_state, buf, input_len); + } else { + lua_pushlstring(lua_state, input, input_len); + } + return 1; +} + +static int get_stream_tuple5(lua_State *lua_state){ + lua_getglobal(lua_state, "cur_http_sess_ctx"); + http_sess_ctx *sess_ctx = (http_sess_ctx *)lua_touserdata(lua_state, -1); + lua_pop(lua_state, 1); + if(sess_ctx == NULL){ + MESA_handle_runtime_log(g_logger, RLOG_LV_FATAL, LOG_MODULE_NAME, "sess_ctx is null"); + return -1; + } + stream_tuple5& tuple5 = sess_ctx->tuple5; + if(tuple5.parse_done == true){ + lua_newtable(lua_state); + lua_push_string_buffer(lua_state, "ip_version", std::string("ip_version").length()); + lua_push_string_buffer(lua_state, tuple5.ip_version.c_str(), tuple5.ip_version.length()); + lua_settable(lua_state, -3); + lua_push_string_buffer(lua_state, "stream_type", std::string("stream_type").length()); + lua_push_string_buffer(lua_state, tuple5.stream_type.c_str(), tuple5.stream_type.length()); + lua_settable(lua_state, -3); + lua_push_string_buffer(lua_state, "sip", std::string("sip").length()); + lua_push_string_buffer(lua_state, tuple5.sip.c_str(), tuple5.sip.length()); + lua_settable(lua_state, -3); + lua_push_string_buffer(lua_state, "dip", std::string("dip").length()); + lua_push_string_buffer(lua_state, tuple5.dip.c_str(), tuple5.dip.length()); + lua_settable(lua_state, -3); + lua_push_string_buffer(lua_state, "sport", std::string("sport").length()); + lua_pushinteger(lua_state, tuple5.sport); + lua_settable(lua_state, -3); + lua_push_string_buffer(lua_state, "dport", std::string("dport").length()); + lua_pushinteger(lua_state, tuple5.dport); + lua_settable(lua_state, -3); + return 0; + } + return -1; +} + +static int _get_stream_info(lua_State *lua_state, int status, lua_KContext yieldk_ctx){ + int ret = get_stream_tuple5(lua_state); + if(ret < 0){ + lua_yieldk(lua_state, 0, 0, _get_stream_info); + } + return 1; +} + +static int get_stream_info(lua_State* lua_state){ + MESA_handle_runtime_log(g_logger, RLOG_LV_DEBUG, LOG_MODULE_NAME, "call get_strem_info"); + return _get_stream_info(lua_state, 0, 0); +} + +static int get_http_header(lua_State *lua_state, int type){ + lua_getglobal(lua_state, "cur_http_sess_ctx"); + http_sess_ctx *sess_ctx = (http_sess_ctx *)lua_touserdata(lua_state, -1); + lua_pop(lua_state, 1); + if(sess_ctx == NULL){ + MESA_handle_runtime_log(g_logger, RLOG_LV_FATAL, LOG_MODULE_NAME, "sess_ctx is null"); + return -1; + } + std::unordered_set required_regions; + lua_pushnil(lua_state); + while(lua_next(lua_state, -2) != 0){ + required_regions.insert(std::string(lua_tostring(lua_state, -1))); + lua_pop(lua_state, 1); + } + http_header& header = (type == HTTP_TYPE_REQUEST ? sess_ctx->req_header : sess_ctx->resp_header); + if(header.parse_done == true){ + MESA_handle_runtime_log(g_logger, RLOG_LV_DEBUG, LOG_MODULE_NAME, "header parse done"); + lua_newtable(lua_state); + for(auto region : header.std_regions){ + if(required_regions.find("ALL") != required_regions.end() || required_regions.find(region.first) != required_regions.end()){ + lua_push_string_buffer(lua_state, region.first.c_str(), region.first.length()); + lua_push_string_buffer(lua_state, region.second.c_str(), region.second.length()); + lua_settable(lua_state, -3); + } + } + if(!header.other_regions.empty()){ + if(required_regions.find("ALL") != required_regions.end() || required_regions.find("HTTP_OTHER_REGIONS") != required_regions.end()){ + lua_push_string_buffer(lua_state, "HTTP_OTHER_REGIONS", std::string("HTTP_OTHER_REGIONS").length()); + lua_newtable(lua_state); + int i = 1; + for(auto region : header.other_regions){ + lua_push_string_buffer(lua_state, region.c_str(), region.length()); + lua_rawseti(lua_state, -2, i++); + } + lua_settable(lua_state, -3); + } + } + return 0; + } + return -1; +} + +static int _get_http_request_header(lua_State *lua_state, int status, lua_KContext yieldk_ctx){ + MESA_handle_runtime_log(g_logger, RLOG_LV_DEBUG, LOG_MODULE_NAME, "call get_http_request_header"); + int ret = get_http_header(lua_state, HTTP_TYPE_REQUEST); + if(ret < 0){ + lua_yieldk(lua_state, 0, 0, _get_http_request_header); + } + return 1; +} + +static int get_http_request_header(lua_State* lua_state){ + return _get_http_request_header(lua_state, 0, 0); +} + +static int _get_http_response_header(lua_State *lua_state, int status, lua_KContext yieldk_ctx){ + int ret = get_http_header(lua_state, HTTP_TYPE_RESPONSE); + if(ret < 0){ + lua_yieldk(lua_state, 0, 0, _get_http_response_header); + } + return 1; +} + +static int get_http_response_header(lua_State* lua_state){ + return _get_http_response_header(lua_state, 0, 0); +} + +static int get_http_body(lua_State *lua_state, int type){ + MESA_handle_runtime_log(g_logger, RLOG_LV_DEBUG, LOG_MODULE_NAME, "call _get_http_response_body"); + lua_getglobal(lua_state, "cur_http_sess_ctx"); + http_sess_ctx *sess_ctx = (http_sess_ctx *)lua_touserdata(lua_state, -1); + lua_pop(lua_state, 1); + if(sess_ctx == NULL){ + MESA_handle_runtime_log(g_logger, RLOG_LV_FATAL, LOG_MODULE_NAME, "sess_ctx is null"); + return -1; + } + http_body& body = (type == HTTP_TYPE_REQUEST? sess_ctx->req_body : sess_ctx->resp_body); + if(body.data_end == true || body.buf != nullptr){ + lua_newtable(lua_state); + lua_push_string_buffer(lua_state, "block_id", std::string("block_id").length()); + lua_pushinteger(lua_state, body.block_id); + lua_settable(lua_state, -3); + lua_push_string_buffer(lua_state, "data_end", std::string("data_end").length()); + lua_pushboolean(lua_state, body.data_end); + lua_settable(lua_state, -3); + lua_push_string_buffer(lua_state, "buf", std::string("buf").length()); + lua_pushlightuserdata(lua_state, body.buf); + lua_settable(lua_state, -3); + lua_push_string_buffer(lua_state, "buflen", std::string("buflen").length()); + lua_pushinteger(lua_state, body.buflen); + lua_settable(lua_state, -3); + body.buf = nullptr; + body.buflen = 0; + return 0; + } + return -1; +} + +static int _get_http_request_body(lua_State *lua_state, int status, lua_KContext yieldk_ctx){ + int ret = get_http_body(lua_state, HTTP_TYPE_REQUEST); + if(ret < 0){ + lua_yieldk(lua_state, 0, 0, _get_http_request_body); + } + return 1; +} + +static int get_http_request_body(lua_State* lua_state){ + return _get_http_request_body(lua_state, 0, 0); +} + +static int _get_http_response_body(lua_State *lua_state, int status, lua_KContext yieldk_ctx){ + int ret = get_http_body(lua_state, HTTP_TYPE_RESPONSE); + if(ret < 0){ + lua_yieldk(lua_state, 0, 0, _get_http_response_body); + } + return 1; +} + +static int get_http_response_body(lua_State* lua_state){ + return _get_http_response_body(lua_state, 0, 0); +} + + +static int load_lua_http_plug(const char *profile, int thread_num){ + const char *section = "main"; + char file_path[LUA_SAPP_PATH_MAX] = ""; + char entry_type[LUA_SAPP_SYMBOL_MAX] = ""; + MESA_load_profile_string_def(profile, section, "file_path", file_path, sizeof(file_path), ""); + MESA_load_profile_string_def(profile, section, "entry_type", entry_type, sizeof(entry_type), ""); + MESA_handle_runtime_log(g_logger, RLOG_LV_INFO, LOG_MODULE_NAME, "MESA_prof_load: profile is %s, section is %s," + "file_path is %s, entry_type is %s\n", profile, section, file_path, entry_type); + if(strncmp(entry_type, "http", LUA_SAPP_SYMBOL_MAX) == 0){ + for(int i = 0; i < thread_num; i++){ + lua_State *lua_state = luaL_newstate(); + if(lua_state == NULL){ + MESA_handle_runtime_log(g_logger, RLOG_LV_FATAL, LOG_MODULE_NAME, "failed to LuaL_newstate"); + return -1; + } + luaL_openlibs(lua_state); + int ret = luaL_dofile(lua_state, file_path); + if(ret){ + MESA_handle_runtime_log(g_logger, RLOG_LV_FATAL, LOG_MODULE_NAME, "error: ret is %d, file_path is %s\n", ret, file_path); + return -1; + } + lua_register(lua_state, "get_stream_info", get_stream_info); + lua_register(lua_state, "get_http_request_header", get_http_request_header); + lua_register(lua_state, "get_http_response_header", get_http_response_header); + lua_register(lua_state, "get_http_request_body", get_http_request_body); + lua_register(lua_state, "get_http_response_body", get_http_response_body); + g_http_lua_states[i].push_back(lua_state); + } + } + return 0; +} + +static int process_lua_plug_conflist(const char* filename, int thread_num) +{ + char lua_plug_conf_path[LUA_SAPP_PATH_MAX] = {0}; + FILE* fp = fopen(filename, "r"); + if(fp == NULL){ + MESA_handle_runtime_log(g_logger, RLOG_LV_FATAL, LOG_MODULE_NAME, "process_conflist() fopen %s error!\n", filename); + return -1; + } + while(feof(fp) == 0){ + if((fgets(lua_plug_conf_path, LUA_SAPP_PATH_MAX, fp)) == NULL){ + fclose(fp); + fp=NULL; + return 0; + } + if(lua_plug_conf_path[0]=='#'){ + continue; + } + int len = strnlen(lua_plug_conf_path, LUA_SAPP_PATH_MAX); + lua_plug_conf_path[len - 1] = '\0'; + int ret = load_lua_http_plug(lua_plug_conf_path, thread_num); + if(ret < 0){ + MESA_handle_runtime_log(g_logger, RLOG_LV_FATAL, LOG_MODULE_NAME, "failed to load_lua_plug: conf_path is %s\n", lua_plug_conf_path); + } + } + fclose(fp); + fp=NULL; + return 0; +} + +static http_sess_ctx* init_http_sess_ctx(int thread_seq){ + http_sess_ctx *ctx = new http_sess_ctx(); + for(lua_State *lua_state : g_http_lua_states[thread_seq]){ + lua_State* coroutine = lua_newthread(lua_state); + int ret = lua_checkstack(lua_state, 1); + if(ret != 1){ + MESA_handle_runtime_log(g_logger, RLOG_LV_FATAL, LOG_MODULE_NAME, "do not have enough space, ret is %d\n", ret); + continue; + } + ctx->coroutines.push_back(coroutine); + } + return ctx; +} + +static void clear_http_sess_ctx(http_sess_ctx *ctx){ + delete ctx; + ctx = NULL; +} + + +std::string utils_inet_ntoa(uint32_t ip) +{ + const char *_ip = inet_ntoa(*(struct in_addr *)&ip); + return std::string(_ip); +} + +extern "C" uchar http_adapter_http_entry(stSessionInfo* session_info, void **param, int thread_seq, struct streaminfo *a_tcp, void *a_packet){ + uchar ret = PROT_STATE_GIVEME; + http_sess_ctx *ctx = (http_sess_ctx*)*param; + http_infor *a_http = (http_infor *)(session_info->app_info); + bool stream_begin = false; + if(session_info == nullptr){ + return PROT_STATE_DROPME; + } + if(ctx == nullptr){ + ctx = init_http_sess_ctx(thread_seq); + *param = ctx; + stream_begin = true; + } + if(ctx->tuple5.parse_done == false){ + struct stream_tuple4_v4 *tuple4_v4 = a_tcp->addr.tuple4_v4; + ctx->tuple5.ip_version = "IPV4"; + if(a_tcp->type == STREAM_TYPE_TCP){ + ctx->tuple5.stream_type = "TCP"; + } + if(a_tcp->type == STREAM_TYPE_UDP){ + ctx->tuple5.stream_type = "UDP"; + } + ctx->tuple5.sip = utils_inet_ntoa(tuple4_v4->saddr); + ctx->tuple5.dip = utils_inet_ntoa(tuple4_v4->daddr); + ctx->tuple5.sport = ntohs(tuple4_v4->source); + ctx->tuple5.dport = ntohs(tuple4_v4->dest); + ctx->tuple5.parse_done = true; + } + uchar curdir = a_http->curdir; + switch(session_info->prot_flag) + { + case HTTP_STATE: + case HTTP_CONTENT: + break; + case HTTP_UNGZIP_CONTENT: + if(curdir == DIR_C2S){ + ctx->req_body.block_id++; + ctx->req_body.buf = session_info->buf; + ctx->req_body.buflen = session_info->buflen; + } + if(curdir == DIR_S2C){ + ctx->resp_body.block_id++; + ctx->resp_body.buf = session_info->buf; + ctx->resp_body.buflen = session_info->buflen; + } + break; + default: + std::string key(http_proto_flag2region(session_info->prot_flag)); + std::string value((const char*)(session_info->buf), session_info->buflen); + if(curdir == DIR_C2S){ + if(key == "HTTP_OTHER_REGIONS"){ + ctx->req_header.other_regions.insert(value); + } + else{ + ctx->req_header.std_regions.insert({key, value}); + } + } + else{ + if(key == "HTTP_OTHER_REGIONS"){ + ctx->resp_header.other_regions.insert(value); + } + else{ + ctx->resp_header.std_regions.insert({key, value}); + } + } + break; + } + uchar http_state = a_http->http_state; + //header over + if(http_state == HTTP_DATA_BEGIN){ + if(curdir == DIR_C2S){ + ctx->req_header.parse_done = true; + } + if(curdir == DIR_S2C){ + ctx->resp_header.parse_done = true; + } + } + //data over + if(http_state == HTTP_DATA_END){ + if(curdir == DIR_C2S){ + ctx->req_body.data_end = true; + } + if(curdir == DIR_S2C){ + ctx->resp_body.data_end = true; + } + } + //resume coroutine + for(lua_State*& coroutine : ctx->coroutines){ + if(coroutine == nullptr){ + continue; + } + /* + int ret = lua_checkstack(coroutine, 1); + if(ret != 1){ + //do log + MESA_handle_runtime_log(g_logger, RLOG_LV_FATAL, LOG_MODULE_NAME, "do not have enough space, ret is %d", ret); + continue; + } + */ + lua_pushlightuserdata(coroutine, (void *)ctx); + lua_setglobal(coroutine, "cur_http_sess_ctx"); + if(stream_begin){ + lua_getglobal(coroutine, "process"); + } + ret = lua_resume(coroutine, NULL, 0); + if(ret == LUA_OK){ + coroutine = nullptr; + continue; + } + if(ret != LUA_YIELD){ + lua_traceback("lua_resume", coroutine, ret); + coroutine = nullptr; + } + } + if(session_info->session_state & SESSION_STATE_CLOSE){ + MESA_handle_runtime_log(g_logger, RLOG_LV_INFO, LOG_MODULE_NAME, "close tcp stream"); + clear_http_sess_ctx(ctx); + *param = NULL; + } + return ret; +} + +extern "C" int http_adapter_init(void){ + g_logger = MESA_create_runtime_log_handle("./log/lua_sapp/http_adapter/http_adapter.log", 10); + if(g_logger == NULL){ + MESA_handle_runtime_log(g_logger, RLOG_LV_FATAL, LOG_MODULE_NAME, "%s init : get log handle error!", HTTP_SERVICE_PLUGNAME); + return -1; + } + int thread_num = g_iThreadNum;; + for(int i = 0; i < thread_num; i++){ + g_http_lua_states.push_back(std::vector()); + } + + // get all business lua script which register http + const char *conflist_path = "./plug/lua/conflist_lua.inf"; + process_lua_plug_conflist(conflist_path, thread_num); + return 0; +} + +extern "C" void http_adapter_destroy(void){ + for(auto states : g_http_lua_states){ + for(auto state : states){ + lua_close(state); + } + } + return; +} + diff --git a/adapter/http_adapter/src/adapter.h b/adapter/http_adapter/src/adapter.h new file mode 100644 index 0000000..1c4a790 --- /dev/null +++ b/adapter/http_adapter/src/adapter.h @@ -0,0 +1,112 @@ +/* + * HTTP_Service.h + * + * Created on: 2013-8-19 + * Author: lishu + */ + +#ifndef HTTP_SERVICE_H_ +#define HTTP_SERVICE_H_ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "stream.h" +#include "http.h" +#include "MESA/MESA_prof_load.h" +#include "MESA/MESA_handle_logger.h" + +extern "C" { + #include "lua.h" + #include "lualib.h" + #include "lauxlib.h" +} +#define HTTP_SERVICE_PLUGNAME "new_http_service.so" +#define LOG_PATH "./log/new_http_service/" + +#define LOG_MODULE_NAME "SAPP_LUA" +#define LUA_SAPP_SYMBOL_MAX 64 +#define LUA_SAPP_PATH_MAX 256 +#define LUA_SAPP_STRING_MAX 2048 +#define LUA_ENTRY_TYPE_NUM 8 +enum lua_entry_type{ + LUA_ENTRY_TYPE_IP = 0, + LUA_ENTRY_TYPE_TCP, + LUA_ENTRY_TYPE_UDP, + LUA_ENTRY_TYPE_HTTP, + LUA_ENTRY_TYPE_TLS, + LUA_ENTRY_TYPE_DNS, + LUA_ENTRY_TYPE_MAIL, + LUA_ENTRY_TYPE_FTP +}; + +enum http_type{ + HTTP_TYPE_REQUEST = 0, + HTTP_TYPE_RESPONSE +}; +class http_header{ +public: + bool parse_done; + std::unordered_map std_regions; + std::unordered_set other_regions; + http_header() : parse_done(false){} +}; + +class http_body{ +public: + bool data_end; + int block_id; + void *buf; + int buflen; + http_body() : data_end(false), block_id(-1), buf(nullptr), buflen(0){} +}; + +class stream_tuple5{ +public: + bool parse_done; + std::string ip_version; //IPV4 or IPV6 + std::string stream_type; //TCP or UDP + std::string sip; + std::string dip; + uint16_t sport; + uint16_t dport; + stream_tuple5() : parse_done(false){} +}; + +class http_sess_ctx{ +public: + stream_tuple5 tuple5; + http_header req_header; + http_header resp_header; + http_body req_body; + http_body resp_body; + std::vector coroutines; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +uchar NEW_HTTP_SERVICE_ENTRY(stSessionInfo* session_info, void **param, int thread_seq, struct streaminfo *a_tcp, void *a_packet); +int NEW_HTTP_SERVICE_INIT(void); +void NEW_HTTP_SERVICE_DESTROY(void); + +#ifdef __cplusplus +} +#endif + +#endif /* HTTP_SERVICE_H_ */ diff --git a/picture/platform.png b/picture/platform.png new file mode 100644 index 0000000..25c6c6a Binary files /dev/null and b/picture/platform.png differ diff --git a/picture/plugin.png b/picture/plugin.png new file mode 100644 index 0000000..15cbd89 Binary files /dev/null and b/picture/plugin.png differ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt deleted file mode 100644 index 5797582..0000000 --- a/test/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -### test_coroutine -add_executable(test_coroutine test_coroutine/main.cpp) -#target_include_directories(test_coroutine PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include) -target_link_libraries(test_coroutine lua-static dl) - - -### test_plugin -add_library(http_service SHARED test_plugin/http_service.cpp) -#target_include_directories(http_service PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include) -target_link_libraries(http_service MESA_handle_logger MESA_htable MESA_prof_load) - -### new_http_service -add_library(new_http_service SHARED new_http_service/adapter.cpp) -#target_include_directories(new_http_service PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include) -target_link_libraries(new_http_service lua-static dl MESA_handle_logger MESA_htable MESA_prof_load) \ No newline at end of file diff --git a/test/new_http_service/adapter.cpp b/test/new_http_service/adapter.cpp deleted file mode 100644 index 4a6d64a..0000000 --- a/test/new_http_service/adapter.cpp +++ /dev/null @@ -1,471 +0,0 @@ -#include "adapter.h" - -//TODO: -//每个函数都要判断返回值 done -// lua错误处理，比如函数没有注册，resume的返回值判断然后打印错误, 不再resume该协程 done -//压栈之前要check_stack done -//流结束clear_ctx之外还要释放该协程，自动垃圾回收？做实验验证一下 done -//destroy，释放lua_state done -//多线程 m * n个虚拟机 -//dumpfile改成在线试一下 -//错误处理，日志, 工程化，日志完备 - -static void* g_logger = NULL; -static std::vector> g_http_lua_states; - -static void lua_traceback(const char *func_name, lua_State *lua_state, int ret){ - int n = lua_gettop(lua_state); - printf("%s error: ret is %d, traceback is:\n", func_name, ret); - for(int i = -1; i >= 0 - n; i--){ - std::string type = std::string(lua_typename(lua_state, lua_type(lua_state, i))); - if(type == "string"){ - printf("%s\n", lua_tostring(lua_state, i)); - } - } - lua_pop(lua_state, n); -} - -std::string _inet_ntoa(uint32_t ip) -{ - const char *_ip = inet_ntoa(*(struct in_addr *)&ip); - return std::string(_ip); -} - -//copy from suricata -static int lua_push_string_buffer(lua_State *lua_state, const char *input, size_t input_len) -{ - if (input_len % 4 != 0) { - /* we're using a buffer sized at a multiple of 4 as lua_pushlstring generates - * invalid read errors in valgrind otherwise. Adding in a nul to be sure. - * Buffer size = len + 1 (for nul) + whatever makes it a multiple of 4 */ - size_t buflen = input_len + 1 + ((input_len + 1) % 4); - char buf[buflen]; - memset(buf, 0x00, buflen); - memcpy(buf, input, input_len); - buf[input_len] = '\0'; - /* return value through lua_state, as a luastring */ - lua_pushlstring(lua_state, buf, input_len); - } else { - lua_pushlstring(lua_state, input, input_len); - } - return 1; -} - -static int get_stream_tuple5(lua_State *lua_state){ - lua_getglobal(lua_state, "cur_http_sess_ctx"); - http_sess_ctx *sess_ctx = (http_sess_ctx *)lua_touserdata(lua_state, -1); - lua_pop(lua_state, 1); - if(sess_ctx == NULL){ - //do log - std::cout<<"sess_ctx is null"<tuple5; - if(tuple5.parse_done == true){ - lua_newtable(lua_state); - lua_push_string_buffer(lua_state, "ip_version", std::string("ip_version").length()); - lua_push_string_buffer(lua_state, tuple5.ip_version.c_str(), tuple5.ip_version.length()); - lua_settable(lua_state, -3); - lua_push_string_buffer(lua_state, "stream_type", std::string("stream_type").length()); - lua_push_string_buffer(lua_state, tuple5.stream_type.c_str(), tuple5.stream_type.length()); - lua_settable(lua_state, -3); - lua_push_string_buffer(lua_state, "sip", std::string("sip").length()); - lua_push_string_buffer(lua_state, tuple5.sip.c_str(), tuple5.sip.length()); - lua_settable(lua_state, -3); - lua_push_string_buffer(lua_state, "dip", std::string("dip").length()); - lua_push_string_buffer(lua_state, tuple5.dip.c_str(), tuple5.dip.length()); - lua_settable(lua_state, -3); - lua_push_string_buffer(lua_state, "sport", std::string("sport").length()); - lua_pushinteger(lua_state, tuple5.sport); - lua_settable(lua_state, -3); - lua_push_string_buffer(lua_state, "dport", std::string("dport").length()); - lua_pushinteger(lua_state, tuple5.dport); - lua_settable(lua_state, -3); - return 0; - } - return -1; -} - -static int _get_stream_info(lua_State *lua_state, int status, lua_KContext yieldk_ctx){ - int ret = get_stream_tuple5(lua_state); - if(ret < 0){ - lua_yieldk(lua_state, 0, 0, _get_stream_info); - } - return 1; -} - -static int get_stream_info(lua_State* lua_state){ - //printf("call get_strem_info\n"); - return _get_stream_info(lua_state, 0, 0); -} - -static int get_http_header(lua_State *lua_state, int type){ - lua_getglobal(lua_state, "cur_http_sess_ctx"); - http_sess_ctx *sess_ctx = (http_sess_ctx *)lua_touserdata(lua_state, -1); - lua_pop(lua_state, 1); - if(sess_ctx == NULL){ - //do log - std::cout<<"sess_ctx is null"< required_regions; - lua_pushnil(lua_state); - while(lua_next(lua_state, -2) != 0){ - required_regions.insert(std::string(lua_tostring(lua_state, -1))); - lua_pop(lua_state, 1); - } - http_header& header = (type == HTTP_TYPE_REQUEST ? sess_ctx->req_header : sess_ctx->resp_header); - if(header.parse_done == true){ - //printf("header parse done\n"); - lua_newtable(lua_state); - for(auto region : header.std_regions){ - if(required_regions.find("ALL") != required_regions.end() || required_regions.find(region.first) != required_regions.end()){ - lua_push_string_buffer(lua_state, region.first.c_str(), region.first.length()); - lua_push_string_buffer(lua_state, region.second.c_str(), region.second.length()); - lua_settable(lua_state, -3); - } - } - if(!header.other_regions.empty()){ - if(required_regions.find("ALL") != required_regions.end() || required_regions.find("HTTP_OTHER_REGIONS") != required_regions.end()){ - lua_push_string_buffer(lua_state, "HTTP_OTHER_REGIONS", std::string("HTTP_OTHER_REGIONS").length()); - lua_newtable(lua_state); - int i = 1; - for(auto region : header.other_regions){ - lua_push_string_buffer(lua_state, region.c_str(), region.length()); - lua_rawseti(lua_state, -2, i++); - } - lua_settable(lua_state, -3); - } - } - return 0; - } - return -1; -} - -static int _get_http_request_header(lua_State *lua_state, int status, lua_KContext yieldk_ctx){ - //printf("call get_http_request_header\n"); - int ret = get_http_header(lua_state, HTTP_TYPE_REQUEST); - if(ret < 0){ - lua_yieldk(lua_state, 0, 0, _get_http_request_header); - } - return 1; -} - -static int get_http_request_header(lua_State* lua_state){ - return _get_http_request_header(lua_state, 0, 0); -} - -static int _get_http_response_header(lua_State *lua_state, int status, lua_KContext yieldk_ctx){ - int ret = get_http_header(lua_state, HTTP_TYPE_RESPONSE); - if(ret < 0){ - lua_yieldk(lua_state, 0, 0, _get_http_response_header); - } - return 1; -} - -static int get_http_response_header(lua_State* lua_state){ - return _get_http_response_header(lua_state, 0, 0); -} - -static int get_http_body(lua_State *lua_state, int type){ - //printf("call _get_http_response_body\n"); - lua_getglobal(lua_state, "cur_http_sess_ctx"); - http_sess_ctx *sess_ctx = (http_sess_ctx *)lua_touserdata(lua_state, -1); - lua_pop(lua_state, 1); - if(sess_ctx == NULL){ - //do log - std::cout<<"sess_ctx is null"<req_body : sess_ctx->resp_body); - if(body.data_end == true || body.buf != nullptr){ - lua_newtable(lua_state); - lua_push_string_buffer(lua_state, "block_id", std::string("block_id").length()); - lua_pushinteger(lua_state, body.block_id); - lua_settable(lua_state, -3); - lua_push_string_buffer(lua_state, "data_end", std::string("data_end").length()); - lua_pushboolean(lua_state, body.data_end); - lua_settable(lua_state, -3); - lua_push_string_buffer(lua_state, "buf", std::string("buf").length()); - lua_pushlightuserdata(lua_state, body.buf); - lua_settable(lua_state, -3); - lua_push_string_buffer(lua_state, "buflen", std::string("buflen").length()); - lua_pushinteger(lua_state, body.buflen); - lua_settable(lua_state, -3); - body.buf = nullptr; - body.buflen = 0; - return 0; - } - return -1; -} - -static int _get_http_request_body(lua_State *lua_state, int status, lua_KContext yieldk_ctx){ - int ret = get_http_body(lua_state, HTTP_TYPE_REQUEST); - if(ret < 0){ - lua_yieldk(lua_state, 0, 0, _get_http_request_body); - } - return 1; -} - -static int get_http_request_body(lua_State* lua_state){ - return _get_http_request_body(lua_state, 0, 0); -} - -static int _get_http_response_body(lua_State *lua_state, int status, lua_KContext yieldk_ctx){ - int ret = get_http_body(lua_state, HTTP_TYPE_RESPONSE); - if(ret < 0){ - lua_yieldk(lua_state, 0, 0, _get_http_response_body); - } - return 1; -} - -static int get_http_response_body(lua_State* lua_state){ - return _get_http_response_body(lua_state, 0, 0); -} - - -static int load_lua_http_plug(const char *profile, int thread_num){ - const char *section = "main"; - char file_path[LUA_SAPP_PATH_MAX] = ""; - char entry_type[LUA_SAPP_SYMBOL_MAX] = ""; - MESA_load_profile_string_def(profile, section, "file_path", file_path, sizeof(file_path), ""); - MESA_load_profile_string_def(profile, section, "entry_type", entry_type, sizeof(entry_type), ""); - printf("MESA_prof_load: profile is %s, section is %s, file_path is %s, entry_type is %s\n", - profile, section, file_path, entry_type); - if(strncmp(entry_type, "http", LUA_SAPP_SYMBOL_MAX) == 0){ - for(int i = 0; i < thread_num; i++){ - lua_State *lua_state = luaL_newstate(); - if(lua_state == NULL){ - printf("failed to LuaL_newstate\n"); - return -1; - } - luaL_openlibs(lua_state); - int ret = luaL_dofile(lua_state, file_path); - if(ret){ - //log error - printf("error: ret is %d, file_path is %s\n", ret, file_path); - return -1; - } - lua_register(lua_state, "get_stream_info", get_stream_info); - lua_register(lua_state, "get_http_request_header", get_http_request_header); - lua_register(lua_state, "get_http_response_header", get_http_response_header); - lua_register(lua_state, "get_http_request_body", get_http_request_body); - lua_register(lua_state, "get_http_response_body", get_http_response_body); - g_http_lua_states[i].push_back(lua_state); - } - } - return 0; -} - -static int process_lua_plug_conflist(const char* filename, int thread_num) -{ - char lua_plug_conf_path[LUA_SAPP_PATH_MAX] = {0}; - FILE* fp = fopen(filename, "r"); - if(fp == NULL){ - MESA_handle_runtime_log(g_logger, RLOG_LV_FATAL, LOG_MODULE_NAME, "process_conflist() fopen %s error!\n", filename); - return -1; - } - while(feof(fp) == 0){ - if((fgets(lua_plug_conf_path, LUA_SAPP_PATH_MAX, fp)) == NULL){ - fclose(fp); - fp=NULL; - return 0; - } - if(lua_plug_conf_path[0]=='#'){ - continue; - } - int len = strnlen(lua_plug_conf_path, LUA_SAPP_PATH_MAX); - lua_plug_conf_path[len - 1] = '\0'; - int ret = load_lua_http_plug(lua_plug_conf_path, thread_num); - if(ret < 0){ - printf("failed to load_lua_plug: conf_path is %s\n", lua_plug_conf_path); - } - } - fclose(fp); - fp=NULL; - return 0; -} - -static http_sess_ctx* init_http_sess_ctx(int thread_seq){ - http_sess_ctx *ctx = new http_sess_ctx(); - for(lua_State *lua_state : g_http_lua_states[thread_seq]){ - lua_State* coroutine = lua_newthread(lua_state); - int ret = lua_checkstack(lua_state, 1); - if(ret != 1){ - //do log - printf("do not have enough space, ret is %d\n", ret); - continue; - } - ctx->coroutines.push_back(coroutine); - } - return ctx; -} - -static void clear_http_sess_ctx(http_sess_ctx *ctx){ - delete ctx; - ctx = NULL; -} - - -std::string utils_inet_ntoa(uint32_t ip) -{ - const char *_ip = inet_ntoa(*(struct in_addr *)&ip); - return std::string(_ip); -} - -uchar NEW_HTTP_SERVICE_ENTRY(stSessionInfo* session_info, void **param, int thread_seq, struct streaminfo *a_tcp, void *a_packet){ - //printf("thread_seq is %d\n", thread_seq); - uchar ret = PROT_STATE_GIVEME; - http_sess_ctx *ctx = (http_sess_ctx*)*param; - http_infor *a_http = (http_infor *)(session_info->app_info); - bool stream_begin = false; - if(session_info == nullptr){ - return PROT_STATE_DROPME; - } - if(ctx == nullptr){ - ctx = init_http_sess_ctx(thread_seq); - *param = ctx; - stream_begin = true; - } - if(ctx->tuple5.parse_done == false){ - struct stream_tuple4_v4 *tuple4_v4 = a_tcp->addr.tuple4_v4; - ctx->tuple5.ip_version = "IPV4"; - if(a_tcp->type == STREAM_TYPE_TCP){ - ctx->tuple5.stream_type = "TCP"; - } - if(a_tcp->type == STREAM_TYPE_UDP){ - ctx->tuple5.stream_type = "UDP"; - } - ctx->tuple5.sip = utils_inet_ntoa(tuple4_v4->saddr); - ctx->tuple5.dip = utils_inet_ntoa(tuple4_v4->daddr); - ctx->tuple5.sport = ntohs(tuple4_v4->source); - ctx->tuple5.dport = ntohs(tuple4_v4->dest); - ctx->tuple5.parse_done = true; - } - uchar curdir = a_http->curdir; - switch(session_info->prot_flag) - { - case HTTP_STATE: - case HTTP_CONTENT: - break; - case HTTP_UNGZIP_CONTENT: - if(curdir == DIR_C2S){ - ctx->req_body.block_id++; - ctx->req_body.buf = session_info->buf; - ctx->req_body.buflen = session_info->buflen; - } - if(curdir == DIR_S2C){ - ctx->resp_body.block_id++; - ctx->resp_body.buf = session_info->buf; - ctx->resp_body.buflen = session_info->buflen; - } - break; - default: - std::string key(http_proto_flag2region(session_info->prot_flag)); - std::string value((const char*)(session_info->buf), session_info->buflen); - if(curdir == DIR_C2S){ - if(key == "HTTP_OTHER_REGIONS"){ - ctx->req_header.other_regions.insert(value); - } - else{ - ctx->req_header.std_regions.insert({key, value}); - } - } - else{ - if(key == "HTTP_OTHER_REGIONS"){ - ctx->resp_header.other_regions.insert(value); - } - else{ - ctx->resp_header.std_regions.insert({key, value}); - } - } - break; - } - uchar http_state = a_http->http_state; - //printf("curdir is %d, http_state is %d\n", curdir, http_state); - //header over - if(http_state == HTTP_DATA_BEGIN){ - if(curdir == DIR_C2S){ - ctx->req_header.parse_done = true; - } - if(curdir == DIR_S2C){ - ctx->resp_header.parse_done = true; - } - } - //data over - if(http_state == HTTP_DATA_END){ - if(curdir == DIR_C2S){ - ctx->req_body.data_end = true; - } - if(curdir == DIR_S2C){ - ctx->resp_body.data_end = true; - } - } - //resume coroutine - for(lua_State*& coroutine : ctx->coroutines){ - if(coroutine == nullptr){ - continue; - } - /* - int ret = lua_checkstack(coroutine, 1); - if(ret != 1){ - //do log - printf("do not have enough space, ret is %d\n", ret); - continue; - } - */ - lua_pushlightuserdata(coroutine, (void *)ctx); - lua_setglobal(coroutine, "cur_http_sess_ctx"); - if(stream_begin){ - lua_getglobal(coroutine, "process"); - } - ret = lua_resume(coroutine, NULL, 0); - if(ret == LUA_OK){ - coroutine = nullptr; - continue; - } - if(ret != LUA_YIELD){ - lua_traceback("lua_resume", coroutine, ret); - coroutine = nullptr; - } - } - if(session_info->session_state & SESSION_STATE_CLOSE){ - //printf("close tcp stream\n"); - clear_http_sess_ctx(ctx); - *param = NULL; - } - return ret; -} - -int NEW_HTTP_SERVICE_INIT(void){ - /* - g_logger = MESA_create_runtime_log_handle("./log/http/http_service", 10); - if(g_logger == NULL){ - printf("%s init : get log handle error!\n", HTTP_SERVICE_PLUGNAME); - return -1; - } - */ - int thread_num = 1; - const char *profile = "./conf/main.conf"; - const char *section = "Module"; - MESA_load_profile_int_def(profile, section, "threadnum", (int*)&thread_num, 0); - printf("MESA_prof_load: profile is %s, section is %s, thread_num is %d\n", profile, section, thread_num); - for(int i = 0; i < thread_num; i++){ - g_http_lua_states.push_back(std::vector()); - } - - // get all business lua script which register http - const char *conflist_path = "./plug/lua/conflist_lua.inf"; - process_lua_plug_conflist(conflist_path, thread_num); - return 0; -} - -void NEW_HTTP_SERVICE_DESTROY(void){ - for(auto states : g_http_lua_states){ - for(auto state : states){ - lua_close(state); - } - } - return; -} - diff --git a/test/new_http_service/adapter.h b/test/new_http_service/adapter.h deleted file mode 100644 index 1c4a790..0000000 --- a/test/new_http_service/adapter.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * HTTP_Service.h - * - * Created on: 2013-8-19 - * Author: lishu - */ - -#ifndef HTTP_SERVICE_H_ -#define HTTP_SERVICE_H_ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "stream.h" -#include "http.h" -#include "MESA/MESA_prof_load.h" -#include "MESA/MESA_handle_logger.h" - -extern "C" { - #include "lua.h" - #include "lualib.h" - #include "lauxlib.h" -} -#define HTTP_SERVICE_PLUGNAME "new_http_service.so" -#define LOG_PATH "./log/new_http_service/" - -#define LOG_MODULE_NAME "SAPP_LUA" -#define LUA_SAPP_SYMBOL_MAX 64 -#define LUA_SAPP_PATH_MAX 256 -#define LUA_SAPP_STRING_MAX 2048 -#define LUA_ENTRY_TYPE_NUM 8 -enum lua_entry_type{ - LUA_ENTRY_TYPE_IP = 0, - LUA_ENTRY_TYPE_TCP, - LUA_ENTRY_TYPE_UDP, - LUA_ENTRY_TYPE_HTTP, - LUA_ENTRY_TYPE_TLS, - LUA_ENTRY_TYPE_DNS, - LUA_ENTRY_TYPE_MAIL, - LUA_ENTRY_TYPE_FTP -}; - -enum http_type{ - HTTP_TYPE_REQUEST = 0, - HTTP_TYPE_RESPONSE -}; -class http_header{ -public: - bool parse_done; - std::unordered_map std_regions; - std::unordered_set other_regions; - http_header() : parse_done(false){} -}; - -class http_body{ -public: - bool data_end; - int block_id; - void *buf; - int buflen; - http_body() : data_end(false), block_id(-1), buf(nullptr), buflen(0){} -}; - -class stream_tuple5{ -public: - bool parse_done; - std::string ip_version; //IPV4 or IPV6 - std::string stream_type; //TCP or UDP - std::string sip; - std::string dip; - uint16_t sport; - uint16_t dport; - stream_tuple5() : parse_done(false){} -}; - -class http_sess_ctx{ -public: - stream_tuple5 tuple5; - http_header req_header; - http_header resp_header; - http_body req_body; - http_body resp_body; - std::vector coroutines; -}; - -#ifdef __cplusplus -extern "C" { -#endif - -uchar NEW_HTTP_SERVICE_ENTRY(stSessionInfo* session_info, void **param, int thread_seq, struct streaminfo *a_tcp, void *a_packet); -int NEW_HTTP_SERVICE_INIT(void); -void NEW_HTTP_SERVICE_DESTROY(void); - -#ifdef __cplusplus -} -#endif - -#endif /* HTTP_SERVICE_H_ */ diff --git a/test/new_http_service/http_request.lua b/test/new_http_service/http_request.lua deleted file mode 100644 index 8cc212e..0000000 --- a/test/new_http_service/http_request.lua +++ /dev/null @@ -1,87 +0,0 @@ - -function printf(s,...) - io.write(s:format(...)) - io.flush() -end - -function format_write_file(file, s, ...) - file:write(s:format(...)) - file:flush() -end - -function init() - ---init something -end - -function print_header(header) - for k, v in pairs(header) do - if k == "HTTP_OTHER_REGIONS" then - format_write_file(file, "%s: ", k) - for k, v in pairs(header["HTTP_OTHER_REGIONS"]) do - format_write_file(file, "{%s} ", v) - end - format_write_file(file, "\n") - else - format_write_file(file, "%s: %s\n", k, v) - end - end -end - -function process() - printf("\nprint stream: \n") - stream_info = get_stream_info() - for k, v in pairs(stream_info) do - printf("%s: %s\n", k, v) - end - - printf("\nprint header:\n") - local regions = {"ALL"} - header = get_http_request_header(regions) - for k, v in pairs(header) do - printf("%s: %s\n", k, v) - end - - printf("\nprint body: \n") - while true do - body = get_http_request_body() - for k, v in pairs(body) do - printf("%s: %s\n", k, v) - end - if body['data_end'] == true then - break - end - end - - --[[ - file = io.open("./log/lua/http_request.log", "a+") - format_write_file(file, "lua: call process\n") - - format_write_file(file, "\nprint stream info: \n") - stream_info = get_stream_info() - for k, v in pairs(stream_info) do - format_write_file(file, "%s: %s\n", k, v) - end - - format_write_file(file, "\nprint request headers: \n") - local req_regions = {"HTTP_HOST", "HTTP_REQ_LINE", "HTTP_OTHER_REGIONS"} - req_header = get_http_request_header(req_regions) - print_header(req_header) - - - format_write_file(file, "\nprint request body: \n") - while true do - body = get_http_request_body() - for k, v in pairs(body) do - format_write_file(file, "%s: %s\n", k, v) - end - if body['data_end'] == true then - format_write_file(file, "request body end\n") - break - end - format_write_file(file, "\n"); - end - format_write_file(file, "\nlua: process end\n") - io.close(file) - ]] - -end \ No newline at end of file diff --git a/test/new_http_service/http_response.lua b/test/new_http_service/http_response.lua deleted file mode 100644 index ed6b6df..0000000 --- a/test/new_http_service/http_response.lua +++ /dev/null @@ -1,57 +0,0 @@ - -function printf(s,...) - io.write(s:format(...)) - io.flush() -end - -function format_write_file(file, s, ...) - file:write(s:format(...)) - file:flush() -end - -function init() - ---init something -end - -function print_header(header) - for k, v in pairs(header) do - if k == "HTTP_OTHER_REGIONS" then - format_write_file(file, "%s: ", k) - for k, v in pairs(header["HTTP_OTHER_REGIONS"]) do - format_write_file(file, "{%s} ", v) - end - format_write_file(file, "\n") - else - format_write_file(file, "%s: %s\n", k, v) - end - end -end - -function process() - file = io.open("./log/lua/http_response.log", "a+") - format_write_file(file, "lua: call process\n") - - format_write_file(file, "\nprint stream info: \n") - stream_info = get_stream_info() - for k, v in pairs(stream_info) do - format_write_file(file, "%s: %s\n", k, v) - end - - local resp_regions = {"HTTP_SERVER", "HTTP_RES_LINE", "HTTP_OTHER_REGIONS"} - format_write_file(file, "\nprint response headers: \n") - resp_header = get_http_response_header(resp_regions) - print_header(resp_header) - - format_write_file(file, "\nprint response body: \n") - while true do - body = get_http_response_body() - for k, v in pairs(body) do - format_write_file(file, "%s: %s\n", k, v) - end - if body['data_end'] == true then - format_write_file(file, "response body end\n") - break - end - end - format_write_file(file, "\nlua: process end\n") -end \ No newline at end of file diff --git a/test/test_coroutine/main.cpp b/test/test_coroutine/main.cpp deleted file mode 100644 index 77c258c..0000000 --- a/test/test_coroutine/main.cpp +++ /dev/null @@ -1,226 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -using namespace std; -extern "C" { - #include "lua.h" - #include "lualib.h" - #include "lauxlib.h" -} - -static void output_stack_size(lua_State *L, int line){ - int n = lua_gettop(L); - printf("line %d: stack size is %d\n", line, n); - for(int i = -1; i >= 0 - n; i--){ - string type = string(lua_typename(L, lua_type(L, i))); - string value = "unknown"; - if(type == "string"){ - value = lua_tostring(L, i); - } - printf("index is %d, type is %s, value is %s\n", i, type.c_str(), value.c_str()); - } -} - - -static void lua_traceback(const char *func_name, lua_State *L){ - int n = lua_gettop(L); - printf("%s error: stack size is %d, traceback is:\n", func_name, n); - for(int i = -1; i >= 0 - n; i--){ - string type = string(lua_typename(L, lua_type(L, i))); - if(type == "string"){ - printf("%s\n", lua_tostring(L, i)); - } - } - lua_pop(L, n); -} - -//copy from suricata -static int lua_push_string_buffer(lua_State *lua_state, const char *input, size_t input_len) -{ - if (input_len % 4 != 0) { - /* we're using a buffer sized at a multiple of 4 as lua_pushlstring generates - * invalid read errors in valgrind otherwise. Adding in a nul to be sure. - * Buffer size = len + 1 (for nul) + whatever makes it a multiple of 4 */ - size_t buflen = input_len + 1 + ((input_len + 1) % 4); - char buf[buflen]; - memset(buf, 0x00, buflen); - memcpy(buf, input, input_len); - buf[input_len] = '\0'; - /* return value through lua_state, as a luastring */ - lua_pushlstring(lua_state, buf, input_len); - } else { - lua_pushlstring(lua_state, input, input_len); - } - return 1; -} - -static void test_coroutine(int argc, char *argv[]){ - int max_state_num = 10; - int max_coroutine_num = 100; - if(argc > 2){ - max_state_num = atoi(argv[1]); - max_coroutine_num = atoi(argv[2]); - } - const char *filename = "../../test/test_coroutine/test.lua"; - vector states; - for(int i = 0; i < max_state_num; i++){ - lua_State *L = luaL_newstate(); - luaL_openlibs(L); - luaL_dofile(L, filename); - states.push_back(L); - } - vector> coroutines; - for(auto state : states){ - int i = 0; - for(; i < max_coroutine_num; i++){ - lua_State* l = lua_newthread(state); - int ret = lua_checkstack(state, 1); - if(ret != 1) - printf("do not have enough space, ret is %d\n", ret); - coroutines.push_back({l, state}); - lua_getglobal(l, "test"); - //cout<<"after getglobal, size = "< regions; - output_stack_size(L); - lua_pushnil(L); - while(lua_next(L, -2) != 0){ - printf("%s - %s\n", lua_typename(L, lua_type(L, -2)), lua_typename(L, lua_type(L, -1))); - regions.insert(string(lua_tostring(L, -1))); - lua_pop(L, 1); - } - for(auto region : regions) - cout< -#include -#include -#include -#include -#include -#include -#include "http_service.h" -#include "MESA_handle_logger.h" - -int HTTP_SERVICE_VERSION_1_20160412 = 0; -void http_service_version_1_20160412() -{ - //20160412 create project -} - -void* g_log_handle = NULL; - -int init_pmeinfo(service_pmeinfo **service_pme, int thread_seq) -{ - service_pmeinfo* pme = (service_pmeinfo*)dictator_malloc(thread_seq, sizeof(service_pmeinfo)); - *service_pme = pme; - return 0; -} - -void clear_pmeinfo(service_pmeinfo *service_pme, int thread_seq) -{ - if(service_pme!=NULL) - { - dictator_free(thread_seq, service_pme); - } - service_pme = NULL; -} - -std::string trans_to_binary(unsigned int n){ - std::string res; - while(n){ - res = std::to_string(n % 2) + res; - n /= 2; - } - return res; -} - -std::string _inet_ntoa(uint32_t ip) -{ - const char *_ip = inet_ntoa(*(struct in_addr *)&ip); - return std::string(_ip); -} - - -static int get_stream_tuple4(struct streaminfo *a_tcp){ - struct stream_tuple4_v4 *tuple4_v4 = a_tcp->addr.tuple4_v4; - printf("saddr: %s\n", _inet_ntoa(tuple4_v4->saddr).c_str()); - printf("daddr: %s\n", _inet_ntoa(tuple4_v4->daddr).c_str()); - printf("source: %d\n", ntohs(tuple4_v4->source)); - printf("dest: %d\n", ntohs(tuple4_v4->dest)); - return 0; -} - -uchar HTTP_SERVICE_ENTRY(stSessionInfo* session_info, void **param, int thread_seq, struct streaminfo *a_tcp, void *a_packet) -{ - //get_stream_tuple4(a_tcp); - uchar rec = PROT_STATE_GIVEME; - service_pmeinfo *service_pme = (service_pmeinfo*)*param; - http_infor *a_http = (http_infor *)(session_info->app_info); - char filename[512] = {0}; - const char* region = NULL; - //static int header_len = 0; - //static int content_len = 0; - - uchar http_state = a_http->http_state; - - if(NULL==session_info) - { - return PROT_STATE_DROPME; - } - - if(service_pme == NULL) - { - if(init_pmeinfo(&service_pme, thread_seq) <0) - { - return PROT_STATE_DROPME; - } - *param = service_pme; - } - std::string prot_flag_str = trans_to_binary(session_info->prot_flag); - uchar curdir = a_http->curdir; - printf("curdir is %d, http_state is %d\n", curdir, http_state); - /* - MESA_handle_runtime_log(g_log_handle, RLOG_LV_INFO, HTTP_SERVICE_PLUGNAME, - "call http_service entry, http_state is %02x\n, prot_flag mask is %s : %d", http_state, prot_flag_str.c_str(), prot_flag_str.length() - 1); - - if(curdir == DIR_C2S){ - int i = 0; - printf("data begin: curdir is %d, http_state is %d, prot_flag mask is %d\n", curdir, http_state, prot_flag_str.length() - 1); - } - */ - switch(session_info->prot_flag) - { - case HTTP_STATE: - case HTTP_CONTENT: - break; - case HTTP_UNGZIP_CONTENT: - snprintf(filename, sizeof(filename), "%s/HTTP_%s_%u", LOG_PATH, printaddr(&a_tcp->addr, thread_seq), a_http->http_session_seq); - service_pme->fp = fopen(filename, "a+"); - if(NULL==service_pme->fp) - { - MESA_handle_runtime_log(g_log_handle, RLOG_LV_FATAL, HTTP_SERVICE_PLUGNAME, "%s file open error.", filename); - return PROT_STATE_DROPME; - } - MESA_handle_runtime_log(g_log_handle, RLOG_LV_DEBUG, HTTP_SERVICE_PLUGNAME, "%s file open.",filename); - - fwrite(session_info->buf, session_info->buflen, 1, service_pme->fp); - fflush(service_pme->fp); - //content_len += session_info->buflen; - //printf("content_len:%d\n", content_len); - - snprintf(filename, sizeof(filename), "%s/HTTP_%s_%u", LOG_PATH, printaddr(&a_tcp->addr, thread_seq), a_http->http_session_seq); - MESA_handle_runtime_log(g_log_handle, RLOG_LV_DEBUG, HTTP_SERVICE_PLUGNAME, "%s file close.",filename); - fclose(service_pme->fp); - break; - default: - snprintf(filename, sizeof(filename), "%s/HTTP_%s_%u", LOG_PATH, printaddr(&a_tcp->addr, thread_seq), a_http->http_session_seq); - service_pme->fp = fopen(filename, "a+"); - if(NULL==service_pme->fp) - { - MESA_handle_runtime_log(g_log_handle, RLOG_LV_FATAL, HTTP_SERVICE_PLUGNAME, "%s file open error.", filename); - return PROT_STATE_DROPME; - } - MESA_handle_runtime_log(g_log_handle, RLOG_LV_DEBUG, HTTP_SERVICE_PLUGNAME, "%s file open.",filename); - - region = http_proto_flag2region(session_info->prot_flag); - - fwrite(region, strlen(region), 1, service_pme->fp); - fwrite(":", 1, 1, service_pme->fp); - fwrite(session_info->buf, session_info->buflen, 1, service_pme->fp); - fwrite("\r\n", 2, 1, service_pme->fp); - fflush(service_pme->fp); - //header_len += session_info->buflen; - //printf("header_len:%d\n", header_len); - - snprintf(filename, sizeof(filename), "%s/HTTP_%s_%u", LOG_PATH, printaddr(&a_tcp->addr, thread_seq), a_http->http_session_seq); - MESA_handle_runtime_log(g_log_handle, RLOG_LV_DEBUG, HTTP_SERVICE_PLUGNAME, "%s file close.",filename); - fclose(service_pme->fp); - break; - } - - if(session_info->session_state&SESSION_STATE_CLOSE) - { - if(NULL!=service_pme->fp) - { - - service_pme->fp = NULL; - } - clear_pmeinfo((service_pmeinfo*)*param, thread_seq); - *param = NULL; - } - return rec; -} - -int HTTP_SERVICE_INIT(void) -{ - g_log_handle = MESA_create_runtime_log_handle("./log/http/http_service", 10); - if(g_log_handle == NULL) - { - printf("%s init : get log handle error!\n", HTTP_SERVICE_PLUGNAME); - return -1; - } - return 0; -} - -void HTTP_SERVICE_DESTROY(void) -{ - return ; -} - diff --git a/test/test_plugin/http_service.h b/test/test_plugin/http_service.h deleted file mode 100644 index 5d0d423..0000000 --- a/test/test_plugin/http_service.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * HTTP_Service.h - * - * Created on: 2013-8-19 - * Author: lishu - */ - -#ifndef HTTP_SERVICE_H_ -#define HTTP_SERVICE_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -#include "stream.h" -#include "http.h" - -#define HTTP_SERVICE_PLUGNAME "http_service.so" -#define LOG_PATH "./log/http/" - - -typedef struct service_pmeinfo -{ - FILE* fp; -}service_pmeinfo; - - -#ifdef __cplusplus -extern "C" { -#endif - -uchar HTTP_SERVICE_ENTRY(stSessionInfo* session_info, void **param, int thread_seq, struct streaminfo *a_tcp, void *a_packet); -int HTTP_SERVICE_INIT(void); -void HTTP_SERVICE_DESTROY(void); - -#ifdef __cplusplus -} -#endif - -#endif /* HTTP_SERVICE_H_ */ diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index 33aea31..7154001 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -4,9 +4,9 @@ include(ExternalProject) ExternalProject_Add(lua PREFIX lua URL ${CMAKE_CURRENT_SOURCE_DIR}/lua-5.3.5.tar.gz - URL_MD5 184779090118aebdf125fdb1233335ea + URL_MD5 4f4b4f323fd3514a68e0ab3da8ce3455 CONFIGURE_COMMAND cd ./src - BUILD_COMMAND make linux + BUILD_COMMAND make linux MYCFLAGS="-fPIC" INSTALL_COMMAND make install INSTALL_TOP= BUILD_IN_SOURCE 1) -- cgit v1.2.3