diff options
| -rw-r--r-- | CMakeLists.txt | 7 | ||||
| -rw-r--r-- | common/include/endianness.h | 157 | ||||
| -rw-r--r-- | common/include/tfe_http.h | 2 | ||||
| -rw-r--r-- | common/include/uuid_v4.h | 276 | ||||
| -rw-r--r-- | plugin/business/doh/src/logger.cpp | 77 | ||||
| -rw-r--r-- | plugin/business/tsg-http/src/tsg_logger.cpp | 174 | ||||
| -rw-r--r-- | plugin/protocol/http/src/http_half.cpp | 2 | ||||
| -rw-r--r-- | plugin/protocol/http2/src/http2_stream.cpp | 2 |
8 files changed, 551 insertions, 146 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index b467b2d..81690c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,13 @@ add_definitions(-D_GNU_SOURCE) set(CMAKE_CXX_STANDARD 11) set(CMAKE_C_STANDARD 11) +if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native") +elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") +endif () + if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE RelWithDebInfo) endif() diff --git a/common/include/endianness.h b/common/include/endianness.h new file mode 100644 index 0000000..5a80e1d --- /dev/null +++ b/common/include/endianness.h @@ -0,0 +1,157 @@ +#pragma once + +#if defined(__GLIBC__) || defined(__GNU_LIBRARY__) || defined(__ANDROID__) + #include <endian.h> +#elif defined(__APPLE__) && defined(__MACH__) + #include <machine/endian.h> +#elif defined(BSD) || defined(_SYSTYPE_BSD) + #if defined(__OpenBSD__) + #include <machine/endian.h> + #else + #include <sys/endian.h> + #endif +#endif + +#if defined(__BYTE_ORDER) + #if defined(__BIG_ENDIAN) && (__BYTE_ORDER == __BIG_ENDIAN) + #define BIGENDIAN + #elif defined(__LITTLE_ENDIAN) && (__BYTE_ORDER == __LITTLE_ENDIAN) + #define LITTLEENDIAN + #endif +#elif defined(_BYTE_ORDER) + #if defined(_BIG_ENDIAN) && (_BYTE_ORDER == _BIG_ENDIAN) + #define BIGENDIAN + #elif defined(_LITTLE_ENDIAN) && (_BYTE_ORDER == _LITTLE_ENDIAN) + #define LITTLEENDIAN + #endif +#elif defined(__BIG_ENDIAN__) + #define BIGENDIAN +#elif defined(__LITTLE_ENDIAN__) + #define LITTLEENDIAN +#else + #if defined(__ARMEL__) || \ + defined(__THUMBEL__) || \ + defined(__AARCH64EL__) || \ + defined(_MIPSEL) || \ + defined(__MIPSEL) || \ + defined(__MIPSEL__) || \ + defined(__ia64__) || defined(_IA64) || \ + defined(__IA64__) || defined(__ia64) || \ + defined(_M_IA64) || defined(__itanium__) || \ + defined(i386) || defined(__i386__) || \ + defined(__i486__) || defined(__i586__) || \ + defined(__i686__) || defined(__i386) || \ + defined(_M_IX86) || defined(_X86_) || \ + defined(__THW_INTEL__) || defined(__I86__) || \ + defined(__INTEL__) || \ + defined(__x86_64) || defined(__x86_64__) || \ + defined(__amd64__) || defined(__amd64) || \ + defined(_M_X64) || \ + defined(__bfin__) || defined(__BFIN__) || \ + defined(bfin) || defined(BFIN) + + #define LITTLEENDIAN + #elif defined(__m68k__) || defined(M68000) || \ + defined(__hppa__) || defined(__hppa) || defined(__HPPA__) || \ + defined(__sparc__) || defined(__sparc) || \ + defined(__370__) || defined(__THW_370__) || \ + defined(__s390__) || defined(__s390x__) || \ + defined(__SYSC_ZARCH__) + + #define BIGENDIAN + + #elif defined(__arm__) || defined(__arm64) || defined(__thumb__) || \ + defined(__TARGET_ARCH_ARM) || defined(__TARGET_ARCH_THUMB) || \ + defined(__ARM_ARCH) || \ + defined(_M_ARM) || defined(_M_ARM64) + + #if defined(_WIN32) || defined(_WIN64) || \ + defined(__WIN32__) || defined(__TOS_WIN__) || \ + defined(__WINDOWS__) + + #define LITTLEENDIAN + + #else + #error "Cannot determine system endianness." + #endif + #endif +#endif + + +#if defined(BIGENDIAN) + // Try to use compiler intrinsics + #if defined(__INTEL_COMPILER) || defined(__ICC) + #define betole16(x) _bswap16(x) + #define betole32(x) _bswap(x) + #define betole64(x) _bswap64(x) + #elif defined(__GNUC__) // GCC and CLANG + #define betole16(x) __builtin_bswap16(x) + #define betole32(x) __builtin_bswap32(x) + #define betole64(x) __builtin_bswap64(x) + #elif defined(_MSC_VER) // MSVC + #include <stdlib.h> + #define betole16(x) _byteswap_ushort(x) + #define betole32(x) _byteswap_ulong(x) + #define betole64(x) _byteswap_uint64(x) + #else + #define FALLBACK_SWAP + #define betole16(x) swap_u16(x) + #define betole32(x) swap_u32(x) + #define betole64(x) swap_u64(x) + #endif + #define betole128(x) swap_u128(x) + #define betole256(x) swap_u256(x) +#else + #define betole16(x) (x) + #define betole32(x) (x) + #define betole64(x) (x) + #define betole128(x) (x) + #define betole256(x) (x) +#endif // BIGENDIAN + +#if defined(BIGENDIAN) + #include <emmintrin.h> + #include <smmintrin.h> + #include <immintrin.h> + #include <tmmintrin.h> + + inline __m128i swap_u128(__m128i value) { + const __m128i shuffle = _mm_set_epi64x(0x0001020304050607, 0x08090a0b0c0d0e0f); + return _mm_shuffle_epi8(value, shuffle); + } + + inline __m256i swap_u256(__m256i value) { + const __m256i shuffle = _mm256_set_epi64x(0x0001020304050607, 0x08090a0b0c0d0e0f, 0x0001020304050607, 0x08090a0b0c0d0e0f); + return _mm256_shuffle_epi8(value, shuffle); + } +#endif // BIGENDIAN + +#if defined(FALLBACK_SWAP) + #include <stdint.h> + inline uint16_t swap_u16(uint16_t value) + { + return + ((value & 0xFF00u) >> 8u) | + ((value & 0x00FFu) << 8u); + } + inline uint32_t swap_u32(uint32_t value) + { + return + ((value & 0xFF000000u) >> 24u) | + ((value & 0x00FF0000u) >> 8u) | + ((value & 0x0000FF00u) << 8u) | + ((value & 0x000000FFu) << 24u); + } + inline uint64_t swap_u64(uint64_t value) + { + return + ((value & 0xFF00000000000000u) >> 56u) | + ((value & 0x00FF000000000000u) >> 40u) | + ((value & 0x0000FF0000000000u) >> 24u) | + ((value & 0x000000FF00000000u) >> 8u) | + ((value & 0x00000000FF000000u) << 8u) | + ((value & 0x0000000000FF0000u) << 24u) | + ((value & 0x000000000000FF00u) << 40u) | + ((value & 0x00000000000000FFu) << 56u); + } +#endif // FALLBACK_SWAP diff --git a/common/include/tfe_http.h b/common/include/tfe_http.h index 68d1ad9..2b0c9de 100644 --- a/common/include/tfe_http.h +++ b/common/include/tfe_http.h @@ -268,7 +268,7 @@ struct tfe_http_session unsigned int session_id; short major_version; short minor_version; - time_t start_time; + struct timeval start_time; struct tfe_http_session_ops * ops; struct tfe_http_half * req; diff --git a/common/include/uuid_v4.h b/common/include/uuid_v4.h new file mode 100644 index 0000000..ed44498 --- /dev/null +++ b/common/include/uuid_v4.h @@ -0,0 +1,276 @@ +/* +MIT License + +Copyright (c) 2018 Xavier "Crashoz" Launey +*/ + +#pragma once + +#include <random> +#include <string> +#include <limits> +#include <iostream> +#include <sstream> +#include <cstdint> +#include <memory> + +#include <emmintrin.h> +#include <smmintrin.h> +#include <immintrin.h> + +#include "endianness.h" + +namespace UUIDv4 { + +/* + Converts a 128-bits unsigned int to an UUIDv4 string representation. + Uses SIMD via Intel's AVX2 instruction set. + */ +void inline m128itos(__m128i x, char* mem) { + // Expand each byte in x to two bytes in res + // i.e. 0x12345678 -> 0x0102030405060708 + // Then translate each byte to its hex ascii representation + // i.e. 0x0102030405060708 -> 0x3132333435363738 + const __m256i mask = _mm256_set1_epi8(0x0F); + const __m256i add = _mm256_set1_epi8(0x06); + const __m256i alpha_mask = _mm256_set1_epi8(0x10); + const __m256i alpha_offset = _mm256_set1_epi8(0x57); + + __m256i a = _mm256_castsi128_si256(x); + __m256i as = _mm256_srli_epi64(a, 4); + __m256i lo = _mm256_unpacklo_epi8(as, a); + __m128i hi = _mm256_castsi256_si128(_mm256_unpackhi_epi8(as, a)); + __m256i c = _mm256_inserti128_si256(lo, hi, 1); + __m256i d = _mm256_and_si256(c, mask); + __m256i alpha = _mm256_slli_epi64(_mm256_and_si256(_mm256_add_epi8(d, add), alpha_mask), 3); + __m256i offset = _mm256_blendv_epi8(_mm256_slli_epi64(add, 3), alpha_offset, alpha); + __m256i res = _mm256_add_epi8(d, offset); + + // Add dashes between blocks as specified in RFC-4122 + // 8-4-4-4-12 + const __m256i dash_shuffle = _mm256_set_epi32(0x0b0a0908, 0x07060504, 0x80030201, 0x00808080, 0x0d0c800b, 0x0a090880, 0x07060504, 0x03020100); + const __m256i dash = _mm256_set_epi64x(0x0000000000000000ull, 0x2d000000002d0000ull, 0x00002d000000002d, 0x0000000000000000ull); + + __m256i resd = _mm256_shuffle_epi8(res, dash_shuffle); + resd = _mm256_or_si256(resd, dash); + + _mm256_storeu_si256((__m256i*)mem, betole256(resd)); + *(uint16_t*)(mem+16) = betole16(_mm256_extract_epi16(res, 7)); + *(uint32_t*)(mem+32) = betole32(_mm256_extract_epi32(res, 7)); +} + +/* + Converts an UUIDv4 string representation to a 128-bits unsigned int. + Uses SIMD via Intel's AVX2 instruction set. + */ +__m128i inline stom128i(const char* mem) { + // Remove dashes and pack hex ascii bytes in a 256-bits int + const __m256i dash_shuffle = _mm256_set_epi32(0x80808080, 0x0f0e0d0c, 0x0b0a0908, 0x06050403, 0x80800f0e, 0x0c0b0a09, 0x07060504, 0x03020100); + + __m256i x = betole256(_mm256_loadu_si256((__m256i*)mem)); + x = _mm256_shuffle_epi8(x, dash_shuffle); + x = _mm256_insert_epi16(x, betole16(*(uint16_t*)(mem+16)), 7); + x = _mm256_insert_epi32(x, betole32(*(uint32_t*)(mem+32)), 7); + + // Build a mask to apply a different offset to alphas and digits + const __m256i sub = _mm256_set1_epi8(0x2F); + const __m256i mask = _mm256_set1_epi8(0x20); + const __m256i alpha_offset = _mm256_set1_epi8(0x28); + const __m256i digits_offset = _mm256_set1_epi8(0x01); + const __m256i unweave = _mm256_set_epi32(0x0f0d0b09, 0x0e0c0a08, 0x07050301, 0x06040200, 0x0f0d0b09, 0x0e0c0a08, 0x07050301, 0x06040200); + const __m256i shift = _mm256_set_epi32(0x00000000, 0x00000004, 0x00000000, 0x00000004, 0x00000000, 0x00000004, 0x00000000, 0x00000004); + + // Translate ascii bytes to their value + // i.e. 0x3132333435363738 -> 0x0102030405060708 + // Shift hi-digits + // i.e. 0x0102030405060708 -> 0x1002300450067008 + // Horizontal add + // i.e. 0x1002300450067008 -> 0x12345678 + __m256i a = _mm256_sub_epi8(x, sub); + __m256i alpha = _mm256_slli_epi64(_mm256_and_si256(a, mask), 2); + __m256i sub_mask = _mm256_blendv_epi8(digits_offset, alpha_offset, alpha); + a = _mm256_sub_epi8(a, sub_mask); + a = _mm256_shuffle_epi8(a, unweave); + a = _mm256_sllv_epi32(a, shift); + a = _mm256_hadd_epi32(a, _mm256_setzero_si256()); + a = _mm256_permute4x64_epi64(a, 0b00001000); + + return _mm256_castsi256_si128(a); +} + +/* + * UUIDv4 (random 128-bits) RFC-4122 + */ +class UUID { + public: + UUID() + {} + + UUID(const UUID &other) { + __m128i x = _mm_load_si128((__m128i*)other.data); + _mm_store_si128((__m128i*)data, x); + } + + /* Builds a 128-bits UUID */ + UUID(__m128i uuid) { + _mm_store_si128((__m128i*)data, uuid); + } + + UUID(uint64_t x, uint64_t y) { + __m128i z = _mm_set_epi64x(x, y); + _mm_store_si128((__m128i*)data, z); + } + + UUID(const uint8_t* bytes) { + __m128i x = _mm_loadu_si128((__m128i*)bytes); + _mm_store_si128((__m128i*)data, x); + } + + /* Builds an UUID from a byte string (16 bytes long) */ + explicit UUID(const std::string &bytes) { + __m128i x = betole128(_mm_loadu_si128((__m128i*)bytes.data())); + _mm_store_si128((__m128i*)data, x); + } + + /* Static factory to parse an UUID from its string representation */ + static UUID fromStrFactory(const std::string &s) { + return fromStrFactory(s.c_str()); + } + + static UUID fromStrFactory(const char* raw) { + return UUID(stom128i(raw)); + } + + void fromStr(const char* raw) { + _mm_store_si128((__m128i*)data, stom128i(raw)); + } + + UUID& operator=(const UUID &other) { + if (&other == this) { + return *this; + } + __m128i x = _mm_load_si128((__m128i*)other.data); + _mm_store_si128((__m128i*)data, x); + return *this; + } + + friend bool operator==(const UUID &lhs, const UUID &rhs) { + __m128i x = _mm_load_si128((__m128i*)lhs.data); + __m128i y = _mm_load_si128((__m128i*)rhs.data); + + __m128i neq = _mm_xor_si128(x, y); + return _mm_test_all_zeros(neq, neq); + } + + friend bool operator<(const UUID &lhs, const UUID &rhs) { + // There are no trivial 128-bits comparisons in SSE/AVX + // It's faster to compare two uint64_t + uint64_t *x = (uint64_t*)lhs.data; + uint64_t *y = (uint64_t*)rhs.data; + return *x < *y || (*x == *y && *(x + 1) < *(y + 1)); + } + + friend bool operator!=(const UUID &lhs, const UUID &rhs) { return !(lhs == rhs); } + friend bool operator> (const UUID &lhs, const UUID &rhs) { return rhs < lhs; } + friend bool operator<=(const UUID &lhs, const UUID &rhs) { return !(lhs > rhs); } + friend bool operator>=(const UUID &lhs, const UUID &rhs) { return !(lhs < rhs); } + + /* Serializes the uuid to a byte string (16 bytes) */ + std::string bytes() const { + std::string mem; + bytes(mem); + return mem; + } + + void bytes(std::string &out) const { + out.resize(sizeof(data)); + bytes((char*)out.data()); + } + + void bytes(char* bytes) const { + __m128i x = betole128(_mm_load_si128((__m128i*)data)); + _mm_storeu_si128((__m128i*)bytes, x); + } + + /* Converts the uuid to its string representation */ + std::string str() const { + std::string mem; + str(mem); + return mem; + } + + void str(std::string &s) const { + s.resize(36); + str((char*)s.data()); + } + + void str(char *res) const { + __m128i x = _mm_load_si128((__m128i*)data); + m128itos(x, res); + } + + friend std::ostream& operator<< (std::ostream& stream, const UUID& uuid) { + return stream << uuid.str(); + } + + friend std::istream& operator>> (std::istream& stream, UUID& uuid) { + std::string s; + stream >> s; + uuid = fromStrFactory(s); + return stream; + } + + size_t hash() const { + const uint64_t a = *((uint64_t*)data); + const uint64_t b = *((uint64_t*)&data[8]); + return a ^ (b + 0x9e3779b9 + (a << 6) + (a >> 2)); + } + + private: + alignas(128) uint8_t data[16]; +}; + +/* + Generates UUIDv4 from a provided random generator (c++11 <random> module) + std::mt19937_64 is highly recommended as it has a SIMD implementation that + makes it very fast and it produces high quality randomness. + */ +template <typename RNG> +class UUIDGenerator { + public: + UUIDGenerator() : generator(new RNG(std::random_device()())), distribution(std::numeric_limits<uint64_t>::min(), std::numeric_limits<uint64_t>::max()) + {} + + UUIDGenerator(uint64_t seed) : generator(new RNG(seed)), distribution(std::numeric_limits<uint64_t>::min(), std::numeric_limits<uint64_t>::max()) + {} + + UUIDGenerator(RNG &gen) : generator(gen), distribution(std::numeric_limits<uint64_t>::min(), std::numeric_limits<uint64_t>::max()) + {} + + /* Generates a new UUID */ + UUID getUUID() { + // The two masks set the uuid version (4) and variant (1) + const __m128i and_mask = _mm_set_epi64x(0xFFFFFFFFFFFFFF3Full, 0xFF0FFFFFFFFFFFFFull); + const __m128i or_mask = _mm_set_epi64x(0x0000000000000080ull, 0x0040000000000000ull); + __m128i n = _mm_set_epi64x(distribution(*generator), distribution(*generator)); + __m128i uuid = _mm_or_si128(_mm_and_si128(n, and_mask), or_mask); + + return UUID(uuid); + } + + private: + std::shared_ptr<RNG> generator; + std::uniform_int_distribution<uint64_t> distribution; +}; + +} + +namespace std { + template <> struct hash<UUIDv4::UUID> + { + size_t operator()(const UUIDv4::UUID & uuid) const + { + return uuid.hash(); + } + }; +} diff --git a/plugin/business/doh/src/logger.cpp b/plugin/business/doh/src/logger.cpp index 88526de..b9f986c 100644 --- a/plugin/business/doh/src/logger.cpp +++ b/plugin/business/doh/src/logger.cpp @@ -328,25 +328,20 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c common_obj = cJSON_CreateObject(); cur_time = time(NULL); - cJSON_AddNumberToObject(common_obj, "common_start_time", cur_time); - cJSON_AddNumberToObject(common_obj, "common_end_time", cur_time); + cJSON_AddNumberToObject(common_obj, "start_timestamp_ms", cur_time); + cJSON_AddNumberToObject(common_obj, "end_timestamp_ms", cur_time); cJSON_AddStringToObject(common_obj, "doh_version", app_proto[http->major_version]); - cJSON_AddStringToObject(common_obj, "common_schema_type", "DoH"); + cJSON_AddStringToObject(common_obj, "decoded_as", "DoH"); char opt_val[24] = { 0 }; - uint16_t opt_out_size; unsigned int common_direction=0; + uint16_t opt_out_size; struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream); if (cmsg != NULL) { int ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_STREAM_TRACE_ID, (unsigned char *)opt_val, sizeof(opt_val), &opt_out_size); if (ret == 0) { - cJSON_AddStringToObject(common_obj, "common_stream_trace_id", opt_val); - } - ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_COMMON_DIRECTION, (unsigned char *)&common_direction, sizeof(common_direction), &opt_out_size); - if (ret==0) - { - cJSON_AddNumberToObject(common_obj, "common_direction", common_direction); //0：域内->域外，1：域外->域内，描述的是CLIENT_IP信息 + cJSON_AddStringToObject(common_obj, "session_id", opt_val); } } @@ -371,24 +366,22 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c switch (addr->addrtype) { case TFE_ADDR_STREAM_TUPLE4_V4: - cJSON_AddNumberToObject(common_obj, "common_address_type", 4); + cJSON_AddNumberToObject(common_obj, "address_type", 4); inet_ntop(AF_INET, &addr->tuple4_v4->saddr, src_ip_str, sizeof(src_ip_str)); inet_ntop(AF_INET, &addr->tuple4_v4->daddr, dst_ip_str, sizeof(dst_ip_str)); - cJSON_AddStringToObject(common_obj, "common_client_ip", src_ip_str); - cJSON_AddStringToObject(common_obj, "common_server_ip", dst_ip_str); - cJSON_AddNumberToObject(common_obj, "common_client_port", ntohs(addr->tuple4_v4->source)); - cJSON_AddNumberToObject(common_obj, "common_server_port", ntohs(addr->tuple4_v4->dest)); - cJSON_AddStringToObject(common_obj, "common_l4_protocol", "IPv4_TCP"); + cJSON_AddStringToObject(common_obj, "client_ip", src_ip_str); + cJSON_AddStringToObject(common_obj, "server_ip", dst_ip_str); + cJSON_AddNumberToObject(common_obj, "client_port", ntohs(addr->tuple4_v4->source)); + cJSON_AddNumberToObject(common_obj, "server_port", ntohs(addr->tuple4_v4->dest)); break; case TFE_ADDR_STREAM_TUPLE4_V6: - cJSON_AddNumberToObject(common_obj, "common_address_type", 6); + cJSON_AddNumberToObject(common_obj, "address_type", 6); inet_ntop(AF_INET6, &addr->tuple4_v6->saddr, src_ip_str, sizeof(src_ip_str)); inet_ntop(AF_INET6, &addr->tuple4_v6->daddr, dst_ip_str, sizeof(dst_ip_str)); - cJSON_AddStringToObject(common_obj, "common_client_ip", src_ip_str); - cJSON_AddStringToObject(common_obj, "common_server_ip", dst_ip_str); - cJSON_AddNumberToObject(common_obj, "common_client_port", ntohs(addr->tuple4_v6->source)); - cJSON_AddNumberToObject(common_obj, "common_server_port", ntohs(addr->tuple4_v6->dest)); - cJSON_AddStringToObject(common_obj, "common_l4_protocol", "IPv6_TCP"); + cJSON_AddStringToObject(common_obj, "client_ip", src_ip_str); + cJSON_AddStringToObject(common_obj, "server_ip", dst_ip_str); + cJSON_AddNumberToObject(common_obj, "client_port", ntohs(addr->tuple4_v6->source)); + cJSON_AddNumberToObject(common_obj, "server_port", ntohs(addr->tuple4_v6->dest)); break; default: break; @@ -397,20 +390,21 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c tfe_stream_info_get(stream, INFO_FROM_DOWNSTREAM_RX_OFFSET, &c2s_byte_num, sizeof(c2s_byte_num)); tfe_stream_info_get(stream, INFO_FROM_UPSTREAM_RX_OFFSET, &s2c_byte_num, sizeof(s2c_byte_num)); - cJSON_AddNumberToObject(common_obj, "common_link_id", 0); - cJSON_AddNumberToObject(common_obj, "common_stream_dir", 3); //1:c2s, 2:s2c, 3:double - cJSON_AddStringToObject(common_obj, "common_sled_ip", handle->kafka_logger->local_ip_str); - cJSON_AddNumberToObject(common_obj, "common_t_vsys_id", handle->kafka_logger->t_vsys_id); - cJSON_AddNumberToObject(common_obj, "common_vsys_id", ctx->vsys_id); - cJSON_AddNumberToObject(common_obj, "common_entrance_id", handle->entry_id); - cJSON_AddStringToObject(common_obj, "common_device_id", handle->device_id); - cJSON_AddNumberToObject(common_obj, "common_c2s_byte_num", c2s_byte_num); - cJSON_AddNumberToObject(common_obj, "common_s2c_byte_num", s2c_byte_num); + cJSON_AddNumberToObject(common_obj, "out_link_id", 0); + cJSON_AddNumberToObject(common_obj, "in_link_id", 0); + cJSON_AddStringToObject(common_obj, "sled_ip", handle->kafka_logger->local_ip_str); + cJSON_AddNumberToObject(common_obj, "t_vsys_id", handle->kafka_logger->t_vsys_id); + cJSON_AddNumberToObject(common_obj, "vsys_id", ctx->vsys_id); + cJSON_AddStringToObject(common_obj, "device_id", handle->device_id); + cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num); + cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num); cJSON_AddStringToObject(common_obj, "doh_url", http->req->req_spec.url); cJSON_AddStringToObject(common_obj, "doh_host", http->req->req_spec.host); + cJSON_AddStringToObject(common_obj, "server_fqdn", http->req->req_spec.host); + if(handle->effective_device_tag) { - cJSON_AddStringToObject(common_obj, "common_device_tag", handle->effective_device_tag); + cJSON_AddStringToObject(common_obj, "device_tag", handle->effective_device_tag); } for (size_t i = 0; i < sizeof(req_fields) / sizeof(struct json_spec); i++) @@ -432,19 +426,19 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c if (ctx->location_client) { - cJSON_AddStringToObject(common_obj, "common_client_location", ctx->location_client); + cJSON_AddStringToObject(common_obj, "client_geolocation", ctx->location_client); } if (ctx->location_server) { - cJSON_AddStringToObject(common_obj, "common_server_location", ctx->location_server); + cJSON_AddStringToObject(common_obj, "server_geolocation", ctx->location_server); } if (ctx->asn_client) { - cJSON_AddStringToObject(common_obj, "common_client_asn", ctx->asn_client); + cJSON_AddStringToObject(common_obj, "client_asn", ctx->asn_client); } if (ctx->asn_server) { - cJSON_AddStringToObject(common_obj, "common_server_asn", ctx->asn_server); + cJSON_AddStringToObject(common_obj, "server_asn", ctx->asn_server); } add_dns_info_to_log(common_obj, dns_info); @@ -462,11 +456,14 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c continue; } + cJSON *proxy_rule_list=NULL; + int config_id[1]={0}; + per_hit_obj = cJSON_Duplicate(common_obj, 1); - cJSON_AddNumberToObject(per_hit_obj, "common_policy_id", result[i].config_id); - cJSON_AddNumberToObject(per_hit_obj, "common_service", result[i].service_id); - cJSON_AddNumberToObject(per_hit_obj, "common_action", LG_ACTION_MANIPULATE); - cJSON_AddStringToObject(per_hit_obj, "common_sub_action", "redirect"); + config_id[0]=result[i].config_id; + proxy_rule_list = cJSON_CreateIntArray(config_id, 1); + cJSON_AddItemToObject(per_hit_obj, "proxy_rule_list", proxy_rule_list); + cJSON_AddStringToObject(per_hit_obj, "proxy_action", "redirect"); log_payload = cJSON_PrintUnformatted(per_hit_obj); diff --git a/plugin/business/tsg-http/src/tsg_logger.cpp b/plugin/business/tsg-http/src/tsg_logger.cpp index ec289b4..8114e0f 100644 --- a/plugin/business/tsg-http/src/tsg_logger.cpp +++ b/plugin/business/tsg-http/src/tsg_logger.cpp @@ -43,6 +43,18 @@ enum _log_action //Bigger action number is prior. __LG_ACTION_MAX }; +#define get_time_ms(tv) ((long long)(tv.tv_sec) * 1000 + (long long)(tv.tv_usec) / 1000) + +#include "uuid_v4.h" +UUIDv4::UUIDGenerator<std::mt19937_64> uuidGenerator; + +void get_http_body_uuid(char *uuid) +{ + UUIDv4::UUID uid = uuidGenerator.getUUID(); + uid.str(uuid); + return; +} + struct proxy_logger* proxy_log_handle_create(const char* profile, const char* section, void* local_logger) { struct tango_cache_parameter *log_file_upload_para=NULL; @@ -115,8 +127,7 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) char* log_payload=NULL; int kafka_status=0; int send_cnt=0; - int tmp=0; - time_t cur_time; + struct timeval cur_time; char src_ip_str[MAX(INET6_ADDRSTRLEN,INET_ADDRSTRLEN)] = {0}; char dst_ip_str[MAX(INET6_ADDRSTRLEN,INET_ADDRSTRLEN)] = {0}; @@ -145,14 +156,15 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) } common_obj=cJSON_CreateObject(); - cur_time = time(NULL); + gettimeofday(&cur_time, NULL); - cJSON_AddNumberToObject(common_obj, "common_start_time", http->start_time); - cJSON_AddNumberToObject(common_obj, "common_end_time", cur_time); + cJSON_AddNumberToObject(common_obj, "start_timestamp_ms", get_time_ms(http->start_time)); + cJSON_AddNumberToObject(common_obj, "end_timestamp_ms", get_time_ms(cur_time)); cJSON_AddStringToObject(common_obj, "http_version", app_proto[http->major_version]); - cJSON_AddStringToObject(common_obj, "common_schema_type", "HTTP"); + cJSON_AddStringToObject(common_obj, "decoded_as", "HTTP"); + - unsigned int common_direction=0, category_id_val[64]={0}; + unsigned int category_id_val[64]={0}; char opt_val[24]={0}; uint16_t opt_out_size; struct tfe_cmsg * cmsg = tfe_stream_get0_cmsg(log_msg->stream); if (cmsg!=NULL) @@ -160,17 +172,12 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) int ret=tfe_cmsg_get_value(cmsg, TFE_CMSG_STREAM_TRACE_ID, (unsigned char *) opt_val, sizeof(opt_val), &opt_out_size); if (ret==0) { - cJSON_AddStringToObject(common_obj, "common_stream_trace_id", opt_val); + cJSON_AddStringToObject(common_obj, "session_id", opt_val); } - ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_COMMON_DIRECTION, (unsigned char *)&common_direction, sizeof(common_direction), &opt_out_size); - if (ret==0) - { - cJSON_AddNumberToObject(common_obj, "common_direction", common_direction); //69：域内->域外，73：域外->域内，描述的是CLIENT_IP信息 - } ret = proxy_log_get_fqdn_cat(cmsg, category_id_val, sizeof(category_id_val)); if (ret>0) { - cJSON_AddItemToObject(common_obj, "common_service_category", cJSON_CreateIntArray((const int*)category_id_val, ret)); + cJSON_AddItemToObject(common_obj, "fqdn_category_list", cJSON_CreateIntArray((const int*)category_id_val, ret)); } } @@ -189,30 +196,29 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) struct tfe_http_resp_spec resp_spec=http->resp->resp_spec; asprintf(&response_line, "HTTP/%d.%d %d OK", http->major_version, http->minor_version, resp_spec.resp_code); cJSON_AddStringToObject(common_obj, "http_response_line", response_line); + cJSON_AddNumberToObject(common_obj, "http_status_code", resp_spec.resp_code); free(response_line); } switch(addr->addrtype) { case TFE_ADDR_STREAM_TUPLE4_V4: - cJSON_AddNumberToObject(common_obj, "common_address_type", 4); + cJSON_AddNumberToObject(common_obj, "address_type", 4); inet_ntop(AF_INET, &addr->tuple4_v4->saddr, src_ip_str, sizeof(src_ip_str)); inet_ntop(AF_INET, &addr->tuple4_v4->daddr, dst_ip_str, sizeof(dst_ip_str)); - cJSON_AddStringToObject(common_obj, "common_client_ip", src_ip_str); - cJSON_AddStringToObject(common_obj, "common_server_ip", dst_ip_str); - cJSON_AddNumberToObject(common_obj, "common_client_port", ntohs(addr->tuple4_v4->source)); - cJSON_AddNumberToObject(common_obj, "common_server_port", ntohs(addr->tuple4_v4->dest)); - cJSON_AddStringToObject(common_obj, "common_l4_protocol", "IPv4_TCP"); + cJSON_AddStringToObject(common_obj, "client_ip", src_ip_str); + cJSON_AddStringToObject(common_obj, "server_ip", dst_ip_str); + cJSON_AddNumberToObject(common_obj, "client_port", ntohs(addr->tuple4_v4->source)); + cJSON_AddNumberToObject(common_obj, "server_port", ntohs(addr->tuple4_v4->dest)); break; case TFE_ADDR_STREAM_TUPLE4_V6: - cJSON_AddNumberToObject(common_obj, "common_address_type", 6); + cJSON_AddNumberToObject(common_obj, "address_type", 6); inet_ntop(AF_INET6, &addr->tuple4_v6->saddr, src_ip_str, sizeof(src_ip_str)); inet_ntop(AF_INET6, &addr->tuple4_v6->daddr, dst_ip_str, sizeof(dst_ip_str)); - cJSON_AddStringToObject(common_obj, "common_client_ip", src_ip_str); - cJSON_AddStringToObject(common_obj, "common_server_ip", dst_ip_str); - cJSON_AddNumberToObject(common_obj, "common_client_port", ntohs(addr->tuple4_v6->source)); - cJSON_AddNumberToObject(common_obj, "common_server_port", ntohs(addr->tuple4_v6->dest)); - cJSON_AddStringToObject(common_obj, "common_l4_protocol", "IPv6_TCP"); + cJSON_AddStringToObject(common_obj, "client_ip", src_ip_str); + cJSON_AddStringToObject(common_obj, "server_ip", dst_ip_str); + cJSON_AddNumberToObject(common_obj, "client_port", ntohs(addr->tuple4_v6->source)); + cJSON_AddNumberToObject(common_obj, "server_port", ntohs(addr->tuple4_v6->dest)); break; default: break; @@ -230,19 +236,20 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) s2c_byte_num = log_msg->s2c_byte_num; } - cJSON_AddNumberToObject(common_obj, "common_link_id", 0); - cJSON_AddNumberToObject(common_obj, "common_stream_dir", 3); //1:c2s, 2:s2c, 3:double - cJSON_AddStringToObject(common_obj, "common_sled_ip", handle->kafka_logger->local_ip_str); - cJSON_AddNumberToObject(common_obj, "common_t_vsys_id", handle->kafka_logger->t_vsys_id); - cJSON_AddNumberToObject(common_obj, "common_entrance_id", handle->entry_id); - cJSON_AddStringToObject(common_obj, "common_device_id", handle->device_id); - cJSON_AddNumberToObject(common_obj, "common_c2s_byte_num", c2s_byte_num); - cJSON_AddNumberToObject(common_obj, "common_s2c_byte_num", s2c_byte_num); + cJSON_AddNumberToObject(common_obj, "out_link_id", 0); + cJSON_AddNumberToObject(common_obj, "in_link_id", 0); + cJSON_AddStringToObject(common_obj, "sled_ip", handle->kafka_logger->local_ip_str); + cJSON_AddNumberToObject(common_obj, "t_vsys_id", handle->kafka_logger->t_vsys_id); + cJSON_AddStringToObject(common_obj, "device_id", handle->device_id); + cJSON_AddNumberToObject(common_obj, "sent_bytes", c2s_byte_num); + cJSON_AddNumberToObject(common_obj, "received_bytes", s2c_byte_num); cJSON_AddStringToObject(common_obj, "http_url", http->req->req_spec.url); cJSON_AddStringToObject(common_obj, "http_host", http->req->req_spec.host); + cJSON_AddStringToObject(common_obj, "server_fqdn", http->req->req_spec.host); + if(handle->effective_device_tag) { - cJSON_AddStringToObject(common_obj, "common_device_tag", handle->effective_device_tag); + cJSON_AddStringToObject(common_obj, "device_tag", handle->effective_device_tag); } for(size_t i=0;i<sizeof(req_fields)/sizeof(struct json_spec);i++) @@ -262,11 +269,8 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) } } - char log_file_upload_req_path[TFE_STRING_MAX]={0}, cont_type_whole[TFE_STRING_MAX]={0}; - char log_file_upload_resp_path[TFE_STRING_MAX]={0}; - memset(log_file_upload_req_path, 0, sizeof(log_file_upload_req_path)); - memset(log_file_upload_resp_path, 0, sizeof(log_file_upload_resp_path)); - memset(cont_type_whole, 0, sizeof(cont_type_whole)); + #define FILE_CHUNK_UUID_LEN 40 + char uuid[FILE_CHUNK_UUID_LEN]={0}; for(size_t i=0; i<log_msg->result_num; i++) { @@ -280,81 +284,34 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) continue; } - struct tango_cache_meta_put meta; - char* log_file_key=NULL;; - const char* cont_type_val; if(log_msg->req_body!=NULL) { - if(log_file_upload_req_path[0] != '\0') + if(uuid[0] != '\0') { - cJSON_AddStringToObject(common_obj, "http_request_body", log_file_upload_req_path); + cJSON_AddStringToObject(common_obj, "http_request_body", uuid); } else { - memset(&meta, 0, sizeof(meta)); - asprintf(&log_file_key, "%s.reqbody", http->req->req_spec.url); - meta.url=log_file_key; - cont_type_val=tfe_http_std_field_read(http->req, TFE_HTTP_CONT_TYPE); - if(cont_type_val!=NULL) - { - snprintf(cont_type_whole, sizeof(cont_type_whole), "Content-Type:%s", cont_type_val); - meta.std_hdr[0]=cont_type_whole; - } - meta.user_log_name=1; - tmp=cache_evbase_upload_once_evbuf(handle->log_file_upload_instance, NULL, - log_msg->req_body, - &meta, - log_file_upload_req_path, sizeof(log_file_upload_req_path)); - if(tmp==0) - { - cJSON_AddStringToObject(common_obj, "http_request_body", log_file_upload_req_path); - } - else - { - TFE_LOG_ERROR(handle->local_logger, "Upload req_body failed."); - } - free(log_file_key); + get_http_body_uuid(uuid); + cJSON_AddStringToObject(common_obj, "http_request_body", uuid); } } if(log_msg->resp_body!=NULL) { - if(log_file_upload_resp_path[0] != '\0') + if(uuid[0] != '\0') { - cJSON_AddStringToObject(common_obj, "http_response_body", log_file_upload_resp_path); + cJSON_AddStringToObject(common_obj, "http_response_body", uuid); } else { - memset(&meta, 0, sizeof(meta)); - asprintf(&log_file_key, "%s.respbody", http->req->req_spec.url); - meta.url=log_file_key; - cont_type_val=tfe_http_std_field_read(http->resp, TFE_HTTP_CONT_TYPE); - if(cont_type_val!=NULL) - { - snprintf(cont_type_whole, sizeof(cont_type_whole), "Content-Type:%s", cont_type_val); - meta.std_hdr[0]=cont_type_whole; - } - meta.user_log_name=1; - tmp=cache_evbase_upload_once_evbuf(handle->log_file_upload_instance, NULL, - log_msg->resp_body, - &meta, - log_file_upload_resp_path, sizeof(log_file_upload_resp_path)); - - if(tmp==0) - { - cJSON_AddStringToObject(common_obj, "http_response_body", log_file_upload_resp_path); - } - else - { - TFE_LOG_ERROR(handle->local_logger, "Upload resp_body failed."); - } - free(log_file_key); + get_http_body_uuid(uuid); + cJSON_AddStringToObject(common_obj, "http_response_body", uuid); } } } for(size_t i=0; i<log_msg->result_num; i++) { - TFE_LOG_DEBUG(handle->local_logger, "URL: %s, policy_id: %lld, service: %d, do_log:%d", http->req->req_spec.url, log_msg->result[i].config_id, @@ -366,27 +323,38 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) continue; } + cJSON *proxy_rule_list=NULL; + int config_id[1]={0}; + per_hit_obj=cJSON_Duplicate(common_obj, 1); - cJSON_AddNumberToObject(per_hit_obj, "common_policy_id", log_msg->result[i].config_id); - cJSON_AddNumberToObject(per_hit_obj, "common_service", log_msg->result[i].service_id); - cJSON_AddNumberToObject(per_hit_obj, "common_vsys_id", log_msg->result[i].vsys_id); - cJSON_AddNumberToObject(per_hit_obj, "common_action", LG_ACTION_MANIPULATE); + config_id[0]=log_msg->result[i].config_id; + proxy_rule_list = cJSON_CreateIntArray(config_id, 1); + cJSON_AddItemToObject(per_hit_obj, "proxy_rule_list", proxy_rule_list); + cJSON_AddNumberToObject(per_hit_obj, "vsys_id", log_msg->result[i].vsys_id); if(log_msg->result[i].action == LG_ACTION_MANIPULATE) { - cJSON_AddStringToObject(per_hit_obj, "common_sub_action", manipulate_action_map[log_msg->action]); + cJSON_AddStringToObject(per_hit_obj, "proxy_action", manipulate_action_map[log_msg->action]); cJSON_AddNumberToObject(per_hit_obj, "http_action_file_size", log_msg->inject_sz); } else { - cJSON_AddStringToObject(per_hit_obj, "common_sub_action", panggu_action_map[(unsigned char)(log_msg->result[i].action)]); + cJSON_AddStringToObject(per_hit_obj, "proxy_action", panggu_action_map[(unsigned char)(log_msg->result[i].action)]); } if(log_msg->location_client) { - cJSON_AddStringToObject(per_hit_obj, "common_client_location", log_msg->location_client); + cJSON_AddStringToObject(per_hit_obj, "client_geolocation", log_msg->location_client); } if(log_msg->location_server) { - cJSON_AddStringToObject(per_hit_obj, "common_server_location", log_msg->location_server); + cJSON_AddStringToObject(per_hit_obj, "server_geolocation", log_msg->location_server); + } + if(log_msg->asn_client) + { + cJSON_AddStringToObject(common_obj, "client_asn", log_msg->asn_client); + } + if (log_msg->asn_server) + { + cJSON_AddStringToObject(common_obj, "server_asn", log_msg->asn_server); } log_payload = cJSON_PrintUnformatted(per_hit_obj); diff --git a/plugin/protocol/http/src/http_half.cpp b/plugin/protocol/http/src/http_half.cpp index 3d832b4..c58c83e 100644 --- a/plugin/protocol/http/src/http_half.cpp +++ b/plugin/protocol/http/src/http_half.cpp @@ -1136,7 +1136,7 @@ struct http_session_private * hs_private_create(struct http_connection_private * __hs_private->hs_public.req = hf_private_req != NULL ? to_hf_public(hf_private_req) : NULL; __hs_private->hs_public.resp = hf_private_req != NULL ? to_hf_public(hf_private_resp) : NULL; __hs_private->hs_public.session_id = hc_private->session_id_counter++; - __hs_private->hs_public.start_time=time(NULL); + gettimeofday(&(__hs_private->hs_public.start_time), NULL); /* HS-PRIVATE*/ __hs_private->hc_private = hc_private; diff --git a/plugin/protocol/http2/src/http2_stream.cpp b/plugin/protocol/http2/src/http2_stream.cpp index 13e37e8..15f32fa 100644 --- a/plugin/protocol/http2/src/http2_stream.cpp +++ b/plugin/protocol/http2/src/http2_stream.cpp @@ -1270,7 +1270,7 @@ static int tfe_half_session_init(struct tfe_h2_session *h2_session, int32_t stre struct tfe_h2_half_private *req = h2_session->req; tfe_session->ops = &http2_session_ops; tfe_session->req = &req->half_public; - tfe_session->start_time=time(NULL); + gettimeofday(&tfe_session->start_time, NULL); tfe_session->session_id = stream_id; } |