fix(cache):修复缓存实现方式-guava，支持schema 通过缓存动态获取。（每30分钟更新一次）v3.0.191223-rc3

8 files changed, 120 insertions, 194 deletions

diff --git a/src/main/java/com/mesalab/common/utils/CacheBaseUtils.java b/src/main/java/com/mesalab/common/utils/CacheBaseUtils.java
deleted file mode 100644
index 311e28e2..00000000
--- a/src/main/java/com/mesalab/common/utils/CacheBaseUtils.java
+++ /dev/null
@@ -1,133 +0,0 @@
-package com.mesalab.common.utils;
-
-import java.util.Calendar;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimerTask;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class CacheBaseUtils {
-
-    @SuppressWarnings("rawtypes")
-    private static Map<String, CacheData> cache = new ConcurrentHashMap<>();
-
-    /**
-     * 设置缓存，不过期
-     *
-     * @param key
-     * @param t
-     */
-    public static <T> void set(String key, T t) {
-        cache.put(key, new CacheData<>(t, 0));
-    }
-
-    /**
-     * 设置缓存，指定过期时间expire(单位毫秒)
-     *
-     * @param key
-     * @param t
-     * @param expire 过期时间
-     */
-    public static <T> void set(String key, T t, long expire) {
-        cache.put(key, new CacheData<>(t, expire));
-    }
-
-    /**
-     * 根据key获取指定缓存
-     *
-     * @param key
-     * @return
-     */
-    @SuppressWarnings("unchecked")
-    public static <T> T get(String key) {
-        CacheData<T> data = cache.get(key);
-        if (null == data) {
-            return null;
-        }
-        if (data.isExpire()) {
-            remove(key);
-            return null;
-        }
-        return data.getData();
-    }
-
-    /**
-     * 移除指定key缓存
-     *
-     * @param key
-     */
-    public static void remove(String key) {
-        cache.remove(key);
-    }
-
-    /**
-     * 移除所有缓存
-     */
-    public static void removeAll() {
-        cache.clear();
-    }
-
-    /**
-     * 清理过期数据定时任务
-     */
-    private static class ClearTimerTask extends TimerTask {
-
-        @SuppressWarnings("rawtypes")
-        Map<String, CacheData> cache;
-
-        @SuppressWarnings("rawtypes")
-        public ClearTimerTask(Map<String, CacheData> cache) {
-            this.cache = cache;
-        }
-
-        @Override
-        public void run() {
-            Set<String> keys = cache.keySet();
-            for (String key : keys) {
-                CacheData<?> data = cache.get(key);
-                if (data.expireTime <= 0) {
-                    continue;
-                }
-                if (data.expireTime > Calendar.getInstance().getTimeInMillis()) {
-                    continue;
-                }
-                cache.remove(key);
-            }
-        }
-    }
-
-    private static class CacheData<T> {
-        // 缓存数据
-        private T data;
-        // 过期时间(单位，毫秒)
-        private long expireTime;
-
-        public CacheData(T t, long expire) {
-            this.data = t;
-            if (expire <= 0) {
-                this.expireTime = 0L;
-            } else {
-                this.expireTime = Calendar.getInstance().getTimeInMillis() + expire;
-            }
-        }
-
-        /**
-         * 判断缓存数据是否过期
-         *
-         * @return true表示过期，false表示未过期
-         */
-        public boolean isExpire() {
-            if (expireTime <= 0) {
-                return false;
-            }
-            if (expireTime > Calendar.getInstance().getTimeInMillis()) {
-                return false;
-            }
-            return true;
-        }
-
-        public T getData() {
-            return data;
-        }
-    }
-}
diff --git a/src/main/java/com/mesalab/common/utils/CacheUtils.java b/src/main/java/com/mesalab/common/utils/QueryCacheUtils.java
index 5fcbc99f..07ee3d70 100644
--- a/src/main/java/com/mesalab/common/utils/CacheUtils.java
+++ b/src/main/java/com/mesalab/common/utils/QueryCacheUtils.java
@@ -10,10 +10,11 @@ import org.slf4j.LoggerFactory;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
-public class CacheUtils {
+public class QueryCacheUtils {
 
     private static Cache<String,Object> cache;
-    private static final Logger log = LoggerFactory.getLogger(CacheUtils.class);
+
+    private static final Logger log = LoggerFactory.getLogger(QueryCacheUtils.class);
 
     static {
         cache = CacheBuilder.newBuilder().maximumSize(100000)
@@ -33,6 +34,7 @@ public class CacheUtils {
     public static void put(String key,Object value){
         if(StringUtil.isNotBlank(key) && value != null){
             cache.put(key,value);
+
         }
     }
 
diff --git a/src/main/java/com/mesalab/common/utils/SchemaCacheUtils.java b/src/main/java/com/mesalab/common/utils/SchemaCacheUtils.java
new file mode 100644
index 00000000..8a056a57
--- /dev/null
+++ b/src/main/java/com/mesalab/common/utils/SchemaCacheUtils.java
@@ -0,0 +1,64 @@
+package com.mesalab.common.utils;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.CacheStats;
+import com.google.common.cache.LoadingCache;
+import com.mesalab.qgw.service.MetadataService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+
+public class SchemaCacheUtils {
+
+    private static final Logger log = LoggerFactory.getLogger(QueryCacheUtils.class);
+    private static final  MetadataService metadataService = (MetadataService) SpringContextUtil.getBean("metadataService");
+
+    private static LoadingCache<String, Object> loadingCache = CacheBuilder.newBuilder()
+            .maximumSize(1000)
+            .expireAfterWrite(30, TimeUnit.MINUTES)
+            .recordStats()
+            .build(new CacheLoader<String, Object>() {
+                @Override
+                public Object load(String key) throws Exception{
+                    Object value = metadataService.getAllSchema();
+                     log.info("loading value by key,key:{}, value:{}", key, value);
+
+                     return value;
+                }
+
+            });
+
+    public static Object get(String key) {
+        try {
+           return loadingCache.get(key);
+        } catch(Exception e) {
+            log.warn(" Schema Cache ,get key error ", e);
+            return null;
+        }
+    }
+
+
+    public static void put(String key, Object value) {
+        loadingCache.put(key, value);
+    }
+
+    public static void remove(String key) {
+        loadingCache.invalidate(key);
+    }
+
+
+    public static void removeAll() {
+        loadingCache.invalidateAll();
+    }
+
+    public static CacheStats getStats() {
+        return loadingCache.stats();
+    }
+
+
+
+
+
+}
diff --git a/src/main/java/com/mesalab/qgw/controller/AuditLogAspect.java b/src/main/java/com/mesalab/qgw/controller/AuditLogAspect.java
index c933fcc1..198638ff 100644
--- a/src/main/java/com/mesalab/qgw/controller/AuditLogAspect.java
+++ b/src/main/java/com/mesalab/qgw/controller/AuditLogAspect.java
@@ -1,19 +1,13 @@
 package com.mesalab.qgw.controller;
 
-import com.mesalab.common.enums.DBTypeEnum;
-import com.mesalab.common.utils.CacheUtils;
+import com.mesalab.common.utils.QueryCacheUtils;
 import com.mesalab.common.utils.IPUtil;
 import com.mesalab.common.utils.JsonMapper;
 import com.mesalab.qgw.model.api.ApiParam;
 import com.mesalab.qgw.model.api.AuditLog;
 import com.mesalab.qgw.model.api.AuditServiceLog;
-import com.zdjizhi.utils.CommonUtil;
 import com.zdjizhi.utils.StringUtil;
-import io.swagger.annotations.Api;
-import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.shiro.SecurityUtils;
 import org.aspectj.lang.JoinPoint;
 import org.aspectj.lang.annotation.*;
 import org.aspectj.lang.reflect.MethodSignature;
@@ -25,7 +19,6 @@ import org.springframework.web.context.request.ServletRequestAttributes;
 
 import javax.servlet.http.HttpServletRequest;
 import java.lang.reflect.Method;
-import java.lang.reflect.Parameter;
 import java.util.Arrays;
 
 
@@ -96,7 +89,7 @@ public class AuditLogAspect {
     public void doAfterReturning(Object result) throws Throwable {
 
         if (StringUtil.isNotEmpty(result) ) {
-            CacheUtils.put(auditServiceLog.getQueryKey(), result);
+            QueryCacheUtils.put(auditServiceLog.getQueryKey(), result);
         }
 
         AuditServiceLog logAfter =  log.get();
diff --git a/src/main/java/com/mesalab/qgw/interceptor/QuerySubmitInterceptor.java b/src/main/java/com/mesalab/qgw/interceptor/QuerySubmitInterceptor.java
index d32c3bec..9e5582f6 100644
--- a/src/main/java/com/mesalab/qgw/interceptor/QuerySubmitInterceptor.java
+++ b/src/main/java/com/mesalab/qgw/interceptor/QuerySubmitInterceptor.java
@@ -1,9 +1,7 @@
 package com.mesalab.qgw.interceptor;
 
 import com.google.common.cache.CacheStats;
-import com.mesalab.common.enums.BaseResultEnum;
-import com.mesalab.common.exception.BusinessException;
-import com.mesalab.common.utils.CacheUtils;
+import com.mesalab.common.utils.QueryCacheUtils;
 import com.mesalab.common.utils.JsonMapper;
 import com.mesalab.qgw.model.api.CachedSubmitCheck;
 import com.zdjizhi.utils.StringUtil;
@@ -40,12 +38,12 @@ public class QuerySubmitInterceptor extends HandlerInterceptorAdapter {
             String queryKey = DigestUtils.md5Hex(request.getQueryString());
 
 
-            log.info("Query Submit Check,ClientId:{}, QueryKey:{}", clientId, DigestUtils.md5Hex(queryKey));
-            Object result = CacheUtils.get(queryKey);
+            log.debug("Query Submit Check,ClientId:{}, QueryKey:{}", clientId, DigestUtils.md5Hex(queryKey));
+            Object result = QueryCacheUtils.get(queryKey);
             if (StringUtil.isNotEmpty(result)) {
 
-                CacheStats stats = CacheUtils.getStats();
-                log.info("Repeat Submit Stat,QEQ_count:{},HIT_count:{},HIT_rate:{}, MISS_count:{}",
+                CacheStats stats = QueryCacheUtils.getStats();
+                log.info(" Query Repeat Submit Stat,QEQ_count:{},HIT_count:{},HIT_rate:{}, MISS_count:{}",
                         stats.requestCount(), stats.hitCount(), stats.hitRate(), stats.missCount());
                 response.setCharacterEncoding("UTF-8");
                 response.setContentType("application/json;charset=UTF-8");
diff --git a/src/main/java/com/mesalab/qgw/service/MetadataService.java b/src/main/java/com/mesalab/qgw/service/MetadataService.java
index e1aa509a..307fe6e4 100644
--- a/src/main/java/com/mesalab/qgw/service/MetadataService.java
+++ b/src/main/java/com/mesalab/qgw/service/MetadataService.java
@@ -2,6 +2,9 @@ package com.mesalab.qgw.service;
 
 
 import com.mesalab.common.base.BaseResult;
+import org.apache.avro.JsonProperties;
+
+import java.util.Map;
 
 public interface MetadataService {
 
@@ -19,4 +22,6 @@ public interface MetadataService {
      * @return
      */
     String getPartitionKey(String tableName);
+
+    Map<String, JsonProperties> getAllSchema();
 }
diff --git a/src/main/java/com/mesalab/qgw/service/impl/MetadataServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/MetadataServiceImpl.java
index e29d844f..1f3bab17 100644
--- a/src/main/java/com/mesalab/qgw/service/impl/MetadataServiceImpl.java
+++ b/src/main/java/com/mesalab/qgw/service/impl/MetadataServiceImpl.java
@@ -1,6 +1,5 @@
 package com.mesalab.qgw.service.impl;
 
-import com.google.common.cache.CacheStats;
 import com.mesalab.common.base.BaseResult;
 import com.mesalab.common.base.BaseResultGenerator;
 import com.mesalab.common.configuration.ClickHouseProperties;
@@ -8,8 +7,7 @@ import com.mesalab.common.configuration.DruidIoProperties;
 import com.mesalab.common.enums.BaseResultEnum;
 import com.mesalab.common.enums.QueryOptionEnum;
 import com.mesalab.common.exception.BusinessException;
-import com.mesalab.common.utils.CacheBaseUtils;
-import com.mesalab.common.utils.CacheUtils;
+import com.mesalab.common.utils.SchemaCacheUtils;
 import com.mesalab.qgw.model.api.ApiParam;
 import com.mesalab.qgw.service.MetadataService;
 import com.zdjizhi.utils.JsonMapper;
@@ -27,6 +25,8 @@ import java.util.*;
 @Service("metadataService")
 public class MetadataServiceImpl implements MetadataService {
 
+    private static final String METADATA_SCHEMA_KEY = "metadata:schema";
+
     @Autowired
     ClickHouseProperties clickHouseProperties;
     @Autowired
@@ -61,10 +61,12 @@ public class MetadataServiceImpl implements MetadataService {
      */
     public BaseResult getSchemaResult(String tableName) {
         Map<String, JsonProperties> allSchema;
-        Object allSchemasInfo = CacheBaseUtils.get("metadata:schema");
-        if(StringUtil.isNotEmpty(allSchemasInfo)){
+        Object allSchemasInfo = SchemaCacheUtils.get(METADATA_SCHEMA_KEY);
+        log.info(" Schema Repeat Submit Stat,QEQ_count:{},HIT_count:{},HIT_rate:{}, MISS_count:{}",
+                SchemaCacheUtils.getStats().requestCount(), SchemaCacheUtils.getStats().hitCount(), SchemaCacheUtils.getStats().hitRate(), SchemaCacheUtils.getStats().missCount());
+        if (StringUtil.isNotEmpty(allSchemasInfo)) {
             allSchema = (Map<String, JsonProperties>)allSchemasInfo;
-        }else {
+        } else {
             allSchema = getAllSchema();
         }
         Map resultMap = (Map) JsonMapper.fromJsonString(String.valueOf(allSchema.get(tableName)), Map.class);
@@ -134,24 +136,6 @@ public class MetadataServiceImpl implements MetadataService {
         return druidschemas;
     }
 
-    /**
-     * 获取所有schemas
-     * @return
-     */
-    private Map<String, JsonProperties> getAllSchema(){
-        HashMap<String, JsonProperties> result = new HashMap<>();
-        Map<String, JsonProperties> schemasOfCK = getSchemasOfCK();
-        if(StringUtil.isNotEmpty(schemasOfCK)){
-            result.putAll(schemasOfCK);
-        }
-        Map<String, JsonProperties> schemasOfDruid = getSchemasOfDruid();
-        if(StringUtil.isNotEmpty(schemasOfDruid)){
-            result.putAll(schemasOfDruid);
-        }
-        CacheBaseUtils.set("metadata:schema", result, 1000 * 86400);
-        return result;
-    }
-
 
     /**
      * 获取当前路径下的所有schema信息
@@ -187,24 +171,37 @@ public class MetadataServiceImpl implements MetadataService {
 
     @Override
     public String getPartitionKey(String tableName){
-        String cacheKey = tableName + ":partition_key";
-        Object value = CacheBaseUtils.get(cacheKey);
-        if (value == null) {
-            String v = "";
-            try {
-                BaseResult schemaResult = getSchemaResult(tableName);
-                Map data = (Map)schemaResult.getData();
-                Map doc = (Map)data.get("doc");
-                if(StringUtil.isNotEmpty(doc)){
-                    Object partition_key = doc.get("partition_key");
-                    v = StringUtil.isEmpty(partition_key) ? v : partition_key.toString();
-                }
-                CacheBaseUtils.set(cacheKey, v, 1000 * 86400);
-            }catch (Exception e){
-                log.error("获取partition_key异常", e);
+        String partitionKey = "";
+        try {
+            BaseResult schemaResult = getSchemaResult(tableName);
+            Map data = (Map)schemaResult.getData();
+            Map doc = (Map)data.get("doc");
+            if(StringUtil.isNotEmpty(doc)){
+                Object partition_key = doc.get("partition_key");
+                partitionKey = StringUtil.isEmpty(partition_key) ? "" : partition_key.toString();
             }
-            return v;
+
+        }catch (Exception e){
+            log.error("获取 partition_key异常 ", e);
         }
-        return value.toString();
+
+
+        return partitionKey;
     }
+
+    @Override
+    public Map<String, JsonProperties> getAllSchema() {
+        HashMap<String, JsonProperties> result = new HashMap<>();
+        Map<String, JsonProperties> schemasOfCK = getSchemasOfCK();
+        if(StringUtil.isNotEmpty(schemasOfCK)){
+            result.putAll(schemasOfCK);
+        }
+        Map<String, JsonProperties> schemasOfDruid = getSchemasOfDruid();
+        if(StringUtil.isNotEmpty(schemasOfDruid)){
+            result.putAll(schemasOfDruid);
+        }
+
+        return result;
+    }
+
 }
diff --git a/src/main/java/com/mesalab/qgw/service/impl/TestSqlServiceImpl.java b/src/main/java/com/mesalab/qgw/service/impl/TestSqlServiceImpl.java
index 24f3b426..1e55f110 100644
--- a/src/main/java/com/mesalab/qgw/service/impl/TestSqlServiceImpl.java
+++ b/src/main/java/com/mesalab/qgw/service/impl/TestSqlServiceImpl.java
@@ -8,8 +8,8 @@ import com.mesalab.common.configuration.ClickHouseProperties;
 import com.mesalab.common.configuration.DruidIoProperties;
 import com.mesalab.common.enums.BaseResultEnum;
 import com.mesalab.common.exception.BusinessException;
-import com.mesalab.common.utils.CacheBaseUtils;
 import com.mesalab.common.utils.JsonMapper;
+import com.mesalab.common.utils.SchemaCacheUtils;
 import com.mesalab.qgw.model.api.ClickHouseHttpSource;
 import com.zdjizhi.utils.Encodes;
 import com.zdjizhi.utils.StringUtil;
@@ -105,7 +105,7 @@ public class TestSqlServiceImpl {
      * @return
      */
     public BaseResult runSchema(){
-        CacheBaseUtils.removeAll();
+        SchemaCacheUtils.removeAll();
         long start = System.currentTimeMillis();
         errorDate = new ArrayList();
         checkCKSchema("/config/avro/clickhouse");