path: root/test/monitor/gtest_topk.cpp

#include <assert.h>
#include <netinet/in.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <stddef.h>
#include <getopt.h>
#include <sys/time.h>
#include <gtest/gtest.h>
#include <arpa/inet.h>
// #include <fieldstat.h>
#include <fieldstat/fieldstat_exporter.h>

#define MAX_TOPK_CELL_NUM 100
#define MAX_TOPK_METRIC_NUM 255
#define SHOW_TOPK_RESULT_NUM 10
#ifndef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif

#define TOPK_DIMENSION_NUM 8
static const char *g_topk_dimension[TOPK_DIMENSION_NUM] = {
    "top_client_ip",
    "top_server_ip",
    "top_internal_ip",
    "top_external_ip",
    "top_server_fqdn",
    "top_server_domain",
    "top_client_country",
    "top_server_country",
};

#define TOPK_RANK_BY_NUM 3
static const char *g_topk_rank_by[TOPK_RANK_BY_NUM] = {
    "sessions",
    "packets",
    "bytes",
};

#define TOPK_METRIC_NUM 3
/* index corresponding to g_topk_rank_by[] */
static const char *g_metric_name[TOPK_METRIC_NUM] = {
    "sessions",
    "pkts",
    "bytes",
};
struct topk_t
{
    char name[128];
    long long value;
};

static int g_topk_cubeid[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM] = {};                      // [dimension][rank_by]
static int g_topk_counter_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM][TOPK_METRIC_NUM] = {}; // [dimension][rank_by][metric]

static struct fieldstat *g_fs4_ins = NULL;

/* the biggest value index is 0 */
static int qsort_topk_value_cmp_cb(const void *a, const void *b)
{
    struct topk_t *la = (struct topk_t *)a;
    struct topk_t *lb = (struct topk_t *)b;
    return (int)(lb->value - la->value);
}

// lijia temp add for test

static void show_topk_stat(const char *dimension, const char *rank_by, struct topk_t *topk_result, size_t n_result, size_t topk)
{

    if (n_result)
    {
        printf("### top '%s' rank_by '%s' result %zu: \n", dimension, rank_by, topk);
        qsort((void *)topk_result, n_result, sizeof(struct topk_t), qsort_topk_value_cmp_cb);
        int show_num = MIN(topk, n_result);
        for (int i = 0; i < show_num; i++)
        {
            printf("%2d)  %16s\t%8lld\n", i + 1, topk_result[i].name, topk_result[i].value);
        }
    }
}

static int get_topk_cube_id(struct fieldstat *fs4_ins, const char *dimension, const char *rank_by)
{
    struct field find_cube_tmp[2] = {};
    find_cube_tmp[0].key = "name";
    find_cube_tmp[0].type = FIELD_VALUE_CSTRING;
    find_cube_tmp[0].value_str = dimension;
    find_cube_tmp[1].key = "rank_by";
    find_cube_tmp[1].type = FIELD_VALUE_CSTRING;
    find_cube_tmp[1].value_str = rank_by;
    return fieldstat_find_cube(fs4_ins, find_cube_tmp, 2);
}

/*
   dimensions: client_ip, server_ip, internal_ip, external_ip, server_fqdn, server_domain, client_country, server_country
   rank_by: sessions, bytes, packets
   metric: sessions, bytes, packets
*/
static int get_topk_rank_by(struct fieldstat *fs4_ins, const char *dimension, const char *rank_by, const char *metric_name, struct topk_t *topk_array, size_t max_array_num)
{
    int cube_id = get_topk_cube_id(fs4_ins, dimension, rank_by);
    assert(cube_id >= 0);

    int metric_id = fieldstat_cube_get_metric_id_by_name(fs4_ins, cube_id, metric_name);
    assert(metric_id >= 0);

    struct field_list *cell_dimensions_list = NULL;
    size_t n_cell = 0;
    fieldstat_cube_get_cells(fs4_ins, cube_id, &cell_dimensions_list, &n_cell);
    if (NULL == cell_dimensions_list)
    {
        return 0;
    }
    size_t result_min = MIN(max_array_num, n_cell);
    for (size_t ce = 0; ce < result_min; ce++)
    {
        strncpy(topk_array[ce].name, cell_dimensions_list[ce].field->value_str, sizeof(topk_array[ce].name));
        fieldstat_counter_get(fs4_ins, cube_id, &cell_dimensions_list[ce], metric_id, &topk_array[ce].value);
    }
    fieldstat_field_list_arr_free(cell_dimensions_list, n_cell);
    return result_min;
}

static void stm_topk_register_cube(struct fieldstat *fs4_ins, int cube_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM])
{
    struct field reg_cube_tmp[2] = {};
    for (size_t i = 0; i < sizeof(g_topk_dimension) / sizeof(g_topk_dimension[0]); i++)
    {
        for (size_t j = 0; j < sizeof(g_topk_rank_by) / sizeof(g_topk_rank_by[0]); j++)
        {
            reg_cube_tmp[0].key = "name";
            reg_cube_tmp[0].type = FIELD_VALUE_CSTRING;
            reg_cube_tmp[0].value_str = g_topk_dimension[i];
            reg_cube_tmp[1].key = "rank_by";
            reg_cube_tmp[1].type = FIELD_VALUE_CSTRING;
            reg_cube_tmp[1].value_str = g_topk_rank_by[j];
            cube_id[i][j] = fieldstat_cube_create(fs4_ins, reg_cube_tmp, 2);
            assert(cube_id[i][j] >= 0);
        }
    }
}

static void stm_topk_register_counter(struct fieldstat *fs4_ins, int cube_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM],
                                      int counter_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM][TOPK_METRIC_NUM])
{
    for (size_t i = 0; i < sizeof(g_topk_dimension) / sizeof(g_topk_dimension[0]); i++)
    {
        for (size_t j = 0; j < sizeof(g_topk_rank_by) / sizeof(g_topk_rank_by[0]); j++)
        {
            for (size_t k = 0; k < sizeof(g_metric_name) / sizeof(char *); k++)
            {
                counter_id[i][j][k] = fieldstat_register_counter(fs4_ins, cube_id[i][j], g_metric_name[k]);
                assert(counter_id[i][j][k] >= 0);
            }
        }
    }
}

static void stm_topk_set_cube_sampling(struct fieldstat *fs4_ins, enum sampling_mode mode, int cube_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM],
                                       int counter_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM][TOPK_METRIC_NUM])
{
    for (size_t i = 0; i < sizeof(g_topk_dimension) / sizeof(g_topk_dimension[0]); i++)
    {
        for (size_t j = 0; j < sizeof(g_topk_rank_by) / sizeof(g_topk_rank_by[0]); j++)
        {
            fieldstat_cube_set_sampling(fs4_ins, cube_id[i][j], mode, MAX_TOPK_CELL_NUM, counter_id[i][j][j]);
        }
    }
}

static int stm_topk_client_ip_rankby_sessions(void)
{
    struct field ip_tag = {};
    ip_tag.key = "top_client_ip";
    ip_tag.type = FIELD_VALUE_CSTRING;
    unsigned int begin_ip_addr = 0x04030201;
    struct topk_t benchmark_result[MAX_TOPK_METRIC_NUM];
    long long max_value = 0;
    memset(benchmark_result, 0, sizeof(benchmark_result));
    for (int i = 0; i < MAX_TOPK_METRIC_NUM; i++)
    {
        benchmark_result[i].value = rand() % 10000;
        if (benchmark_result[i].value > max_value)
        {
            max_value = benchmark_result[i].value;
        }
        unsigned int ip_addr_net = htonl(begin_ip_addr + i);
        inet_ntop(AF_INET, &ip_addr_net, benchmark_result[i].name, sizeof(benchmark_result[i].name));
        ip_tag.value_str = benchmark_result[i].name;
        fieldstat_counter_incrby(g_fs4_ins, g_topk_cubeid[0][0], g_topk_counter_id[0][0][0], &ip_tag, 1, benchmark_result[i].value);
    }

    /***  query and verify  ***/
    struct topk_t fs4_topk_result[MAX_TOPK_CELL_NUM] = {};
    int num = get_topk_rank_by(g_fs4_ins, "top_client_ip", "sessions", "sessions", fs4_topk_result, sizeof(fs4_topk_result) / sizeof(struct topk_t));
    assert(num > 0);
    printf("topk top_client_ip rankby sessions, max value:%lld\n", max_value);
    show_topk_stat("top_client_ip", "sessions", fs4_topk_result, num, SHOW_TOPK_RESULT_NUM);
    qsort((void *)benchmark_result, MAX_TOPK_METRIC_NUM, sizeof(struct topk_t), qsort_topk_value_cmp_cb);
    assert(max_value == fs4_topk_result[0].value);
    return memcmp(fs4_topk_result, &benchmark_result[0], SHOW_TOPK_RESULT_NUM);
}

static int stm_topk_server_ip_rankby_packets(void)
{
    struct field ip_tag = {};
    ip_tag.key = "top_server_ip";
    ip_tag.type = FIELD_VALUE_CSTRING;
    unsigned int begin_ip_addr = 0x64640101;
    long long max_value = 0;
    struct topk_t benchmark_result[MAX_TOPK_METRIC_NUM];
    memset(benchmark_result, 0, sizeof(benchmark_result));
    for (int i = 0; i < MAX_TOPK_METRIC_NUM; i++)
    {
        benchmark_result[i].value = rand() % 10000;
        if (benchmark_result[i].value > max_value)
        {
            max_value = benchmark_result[i].value;
        }
        unsigned int ip_addr_net = htonl(begin_ip_addr + i);
        inet_ntop(AF_INET, &ip_addr_net, benchmark_result[i].name, sizeof(benchmark_result[i].name));
        ip_tag.value_str = benchmark_result[i].name;
        fieldstat_counter_incrby(g_fs4_ins, g_topk_cubeid[1][1], g_topk_counter_id[1][1][1], &ip_tag, 1, benchmark_result[i].value);
    }

    /***  query and verify  ***/
    struct topk_t fs4_topk_result[MAX_TOPK_CELL_NUM] = {};
    int num = get_topk_rank_by(g_fs4_ins, "top_server_ip", "packets", "pkts", fs4_topk_result, sizeof(fs4_topk_result) / sizeof(struct topk_t));
    assert(num > 0);
    printf("topk top_server_ip rankby packets, max value:%lld\n", max_value);
    show_topk_stat("top_server_ip", "packets", fs4_topk_result, num, SHOW_TOPK_RESULT_NUM);
    qsort((void *)benchmark_result, MAX_TOPK_METRIC_NUM, sizeof(struct topk_t), qsort_topk_value_cmp_cb);
    assert(max_value == fs4_topk_result[0].value);
    return memcmp(fs4_topk_result, &benchmark_result[0], SHOW_TOPK_RESULT_NUM);
}

static int stm_topk_server_domain_rankby_bytes(void)
{
    struct field domain_tag = {};
    domain_tag.key = "top_server_domain";
    domain_tag.type = FIELD_VALUE_CSTRING;
    struct topk_t benchmark_result[MAX_TOPK_METRIC_NUM];
    memset(benchmark_result, 0, sizeof(benchmark_result));
    for (int i = 0; i < MAX_TOPK_METRIC_NUM; i++)
    {
        snprintf(benchmark_result[i].name, sizeof(benchmark_result[i].name), "www.abcd.com");
    }

    for (int i = 0; i < MAX_TOPK_METRIC_NUM; i++)
    {
        benchmark_result[i].value = rand() % 10000;
        benchmark_result[i].name[4] = rand() % 26 + 'a';
        benchmark_result[i].name[5] = rand() % 26 + 'a';
        benchmark_result[i].name[6] = rand() % 26 + 'a';
        benchmark_result[i].name[7] = rand() % 26 + 'a';
        domain_tag.value_str = benchmark_result[i].name;
        fieldstat_counter_incrby(g_fs4_ins, g_topk_cubeid[5][2], g_topk_counter_id[5][2][2], &domain_tag, 1, benchmark_result[i].value);
    }

    /***  query and verify  ***/
    struct topk_t fs4_topk_result[SHOW_TOPK_RESULT_NUM] = {};
    int num = get_topk_rank_by(g_fs4_ins, "top_server_domain", "bytes", "bytes", fs4_topk_result, sizeof(fs4_topk_result) / sizeof(struct topk_t));
    assert(num > 0);
    show_topk_stat("top_server_domain", "bytes", fs4_topk_result, num, SHOW_TOPK_RESULT_NUM);
    qsort((void *)benchmark_result, MAX_TOPK_METRIC_NUM, sizeof(struct topk_t), qsort_topk_value_cmp_cb);
    return memcmp(fs4_topk_result, &benchmark_result[0], SHOW_TOPK_RESULT_NUM);
}

class MONITOR_TOPK : public testing::Test
{
protected:
    static void SetUpTestCase()
    {
        srand(time(NULL));
        g_fs4_ins = fieldstat_new();
        assert(g_fs4_ins != NULL);

        stm_topk_register_cube(g_fs4_ins, g_topk_cubeid);
        stm_topk_register_counter(g_fs4_ins, g_topk_cubeid, g_topk_counter_id);
        stm_topk_set_cube_sampling(g_fs4_ins, SAMPLING_MODE_TOPK, g_topk_cubeid, g_topk_counter_id);
    }
    static void TearDownTestCase()
    {
        fieldstat_reset(g_fs4_ins);
        fieldstat_free(g_fs4_ins);
    }
};

static void generate_random_counter(struct fieldstat *fs4_ins, int cube_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM],
                                    int counter_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM][TOPK_METRIC_NUM])
{
    struct field tag = {};
    tag.type = FIELD_VALUE_CSTRING;
    tag.value_str = "random";

    for (size_t i = 0; i < sizeof(g_topk_dimension) / sizeof(g_topk_dimension[0]); i++)
    {
        for (size_t j = 0; j < sizeof(g_topk_rank_by) / sizeof(g_topk_rank_by[0]); j++)
        {
            for (size_t k = 0; k < sizeof(g_metric_name) / sizeof(char *); k++)
            {
                assert(g_topk_counter_id[i][j][k] >= 0);
                tag.key = g_topk_dimension[i];
                fieldstat_counter_incrby(fs4_ins, cube_id[i][j], counter_id[i][j][k], &tag, 1, random());
            }
        }
    }
}

char **show_topk_get_metrics_name(struct fieldstat *show_fs4_ins, int *metric_array_num)
{
    int *cube_id_array = NULL;
    int n_cube = 0;
    fieldstat_get_cubes(show_fs4_ins, &cube_id_array, &n_cube);
    if (NULL == cube_id_array)
    {
        return NULL;
    }

    struct field_list *cell_dimensions_list = NULL;
    size_t n_cell = 0;
    fieldstat_cube_get_cells(show_fs4_ins, cube_id_array[0], &cell_dimensions_list, &n_cell);
    if (cell_dimensions_list == NULL)
    {
        return NULL;
    }

    /* get metric name  of any valid cube_id */
    int *metric_id_out;
    size_t n_metric;
    char **metric_array = NULL;
    fieldstat_cell_get_metrics(show_fs4_ins, cube_id_array[0], cell_dimensions_list, &metric_id_out, &n_metric);
    if (n_metric == 0)
    {
        fieldstat_field_list_arr_free(cell_dimensions_list, n_cell);
        return NULL;
    }
    metric_array = (char **)malloc(sizeof(char *) * n_metric);
    for (size_t j = 0; j < n_metric; j++)
    {
        metric_array[j] = strdup(fieldstat_metric_get_name(show_fs4_ins, cube_id_array[0], metric_id_out[j]));
    }
    fieldstat_field_list_arr_free(cell_dimensions_list, n_cell);

    *metric_array_num = n_metric;
    return metric_array;
}

TEST(MONITOR, show_all_valid_metrics)
{
    struct fieldstat *show_fs4_ins = fieldstat_new();
    int show_topk_cubeid[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM] = {};                      // [dimension][rank_by]
    int show_topk_counter_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM][TOPK_METRIC_NUM] = {}; // [dimension][rank_by][metric]

    stm_topk_register_cube(show_fs4_ins, show_topk_cubeid);
    stm_topk_register_counter(show_fs4_ins, show_topk_cubeid, show_topk_counter_id);
    stm_topk_set_cube_sampling(show_fs4_ins, SAMPLING_MODE_TOPK, show_topk_cubeid, show_topk_counter_id);

    generate_random_counter(show_fs4_ins, show_topk_cubeid, show_topk_counter_id);
    int metric_array_num = 0;
    char **metric_array = show_topk_get_metrics_name(show_fs4_ins, &metric_array_num);
    EXPECT_TRUE(metric_array_num > 0);
    EXPECT_TRUE(metric_array != NULL);

    for (int i = 0; i < metric_array_num; i++)
    {
        printf("metric name: %s\n", metric_array[i]);
        free(metric_array[i]);
    }
    free(metric_array);

    fieldstat_free(show_fs4_ins);
}

static int charchar_array_exist(char **dimension_array, int dimension_array_num, const char *expect_name)
{
    for (int i = 0; i < dimension_array_num && dimension_array[i] != NULL; i++)
    {
        if (0 == strcmp(dimension_array[i], expect_name))
        {
            return 1;
        }
    }
    return 0;
}

char **show_topk_get_dimensions_name(struct fieldstat *show_fs4_ins, int *dimension_array_result)
{
    int *cube_id_array = NULL;
    int n_cube = 0;
    fieldstat_get_cubes(show_fs4_ins, &cube_id_array, &n_cube);
    if (0 == n_cube)
    {
        return NULL;
    }
    char **dimension_array = NULL;
    int dimension_array_num = 0;

    struct field_list *cell_dimensions_list = NULL;
    for (int i = 0; i < n_cube; i++)
    {
        size_t n_cell = 0;
        fieldstat_cube_get_cells(show_fs4_ins, cube_id_array[i], &cell_dimensions_list, &n_cell);
        if (NULL == cell_dimensions_list)
        {
            continue;
        }
        for (size_t j = 0; j < cell_dimensions_list->n_field; j++)
        {
            if (0 == charchar_array_exist(dimension_array, dimension_array_num, cell_dimensions_list->field[j].key))
            {
                dimension_array = (char **)realloc(dimension_array, sizeof(char *) * (dimension_array_num + 1));
                dimension_array[dimension_array_num] = strdup(cell_dimensions_list->field[j].key);
                dimension_array_num++;
            }
        }
        fieldstat_field_list_arr_free(cell_dimensions_list, n_cell);
    }

    *dimension_array_result = dimension_array_num;
    return dimension_array;
}

TEST(MONITOR, show_all_valid_dimensions)
{
    struct fieldstat *show_fs4_ins = fieldstat_new();
    int show_topk_cubeid[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM] = {};                      // [dimension][rank_by]
    int show_topk_counter_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM][TOPK_METRIC_NUM] = {}; // [dimension][rank_by][metric]

    stm_topk_register_cube(show_fs4_ins, show_topk_cubeid);
    stm_topk_register_counter(show_fs4_ins, show_topk_cubeid, show_topk_counter_id);
    stm_topk_set_cube_sampling(show_fs4_ins, SAMPLING_MODE_TOPK, show_topk_cubeid, show_topk_counter_id);

    generate_random_counter(show_fs4_ins, show_topk_cubeid, show_topk_counter_id);

    int dimension_array_num = 0;
    char **dimension_array = show_topk_get_dimensions_name(show_fs4_ins, &dimension_array_num);
    EXPECT_TRUE(dimension_array_num > 0);
    EXPECT_TRUE(dimension_array != NULL);

    for (int i = 0; i < dimension_array_num; i++)
    {
        printf("dimension name: %s\n", dimension_array[i]);
        free(dimension_array[i]);
    }
    free(dimension_array);
    fieldstat_free(show_fs4_ins);
}

char **show_topk_get_rankby_name(struct fieldstat *show_fs4_ins, int *rankby_array_result)
{
    int *cube_id_array = NULL;
    int n_cube = 0;
    fieldstat_get_cubes(show_fs4_ins, &cube_id_array, &n_cube);
    if (NULL == cube_id_array)
    {
        return NULL;
    }
    char **rankby_array = NULL;
    int rankby_array_num = 0;

    for (int i = 0; i < n_cube; i++)
    {
        struct field_list *flist = fieldstat_cube_get_dimension(show_fs4_ins, cube_id_array[i]);
        if (flist == NULL)
        {
            continue;
        }
        for (size_t j = 0; j < flist->n_field; j++)
        {
            if (0 == strcmp(flist->field[j].key, "rank_by"))
            {
                if (0 == charchar_array_exist(rankby_array, rankby_array_num, flist->field[j].value_str))
                {
                    rankby_array = (char **)realloc(rankby_array, sizeof(char *) * (rankby_array_num + 1));
                    rankby_array[rankby_array_num] = strdup(flist->field[j].value_str);
                    rankby_array_num++;
                }
            }
        }
        fieldstat_field_list_arr_free(flist, 1);
    }
    *rankby_array_result = rankby_array_num;
    return rankby_array;
}

TEST(MONITOR, show_all_valid_rankby)
{
    struct fieldstat *show_fs4_ins = fieldstat_new();
    int show_topk_cubeid[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM] = {};                      // [dimension][rank_by]
    int show_topk_counter_id[TOPK_DIMENSION_NUM][TOPK_RANK_BY_NUM][TOPK_METRIC_NUM] = {}; // [dimension][rank_by][metric]

    stm_topk_register_cube(show_fs4_ins, show_topk_cubeid);
    stm_topk_register_counter(show_fs4_ins, show_topk_cubeid, show_topk_counter_id);
    stm_topk_set_cube_sampling(show_fs4_ins, SAMPLING_MODE_TOPK, show_topk_cubeid, show_topk_counter_id);

    generate_random_counter(show_fs4_ins, show_topk_cubeid, show_topk_counter_id);

    int rankby_array_num = 0;
    char **rankby_array = show_topk_get_rankby_name(show_fs4_ins, &rankby_array_num);
    EXPECT_TRUE(rankby_array_num > 0);
    EXPECT_TRUE(rankby_array != NULL);

    for (int i = 0; i < rankby_array_num; i++)
    {
        printf("rankby name: %s\n", rankby_array[i]);
        free(rankby_array[i]);
    }
    free(rankby_array);

    fieldstat_free(show_fs4_ins);
}

TEST_F(MONITOR_TOPK, show_client_ip_by_sessions)
{
    EXPECT_EQ(0, stm_topk_client_ip_rankby_sessions());
}

TEST_F(MONITOR_TOPK, show_server_ip_by_packets)
{
    EXPECT_EQ(0, stm_topk_server_ip_rankby_packets());
}

TEST_F(MONITOR_TOPK, show_server_domain_by_bytes)
{
    EXPECT_EQ(0, stm_topk_server_domain_rankby_bytes());
}

int main(int argc, char **argv)
{
    testing::InitGoogleTest(&argc, argv);
    int ret = RUN_ALL_TESTS();
    return ret;
}