diff --git a/ruoyi-admin/src/main/resources/application.yml b/ruoyi-admin/src/main/resources/application.yml index 6c5c7c7..785444f 100644 --- a/ruoyi-admin/src/main/resources/application.yml +++ b/ruoyi-admin/src/main/resources/application.yml @@ -145,6 +145,7 @@ tencent: appKey: c167746e38d64143a874cec3d5de014e clientIP: + gettps: https://tps.kdlapi.com/api/gettps proxyAddr: k136.kdltps.com proxyPort: 15818 proxySocks: 20818 diff --git a/ruoyi-common/src/main/java/com/ruoyi/common/config/ProxyProperties.java b/ruoyi-common/src/main/java/com/ruoyi/common/config/ProxyProperties.java index 0603c7c..ec53aa9 100644 --- a/ruoyi-common/src/main/java/com/ruoyi/common/config/ProxyProperties.java +++ b/ruoyi-common/src/main/java/com/ruoyi/common/config/ProxyProperties.java @@ -2,6 +2,7 @@ package com.ruoyi.common.config; import com.alibaba.fastjson2.JSONObject; import lombok.Data; +import org.apache.http.client.ClientProtocolException; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; @@ -10,6 +11,8 @@ import org.apache.http.util.EntityUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; +import java.io.IOException; + @Data @Configuration public class ProxyProperties { @@ -24,6 +27,8 @@ public class ProxyProperties { // secretId: o7fnvzz6211s5ymtw3nq // secretKey: qg2ehv53xazqj0fa5zer3dcssoj311ff + @Value("${clientIP.gettps}") + private String gettps; @Value("${clientIP.proxyAddr}") private String proxyAddr; @Value("${clientIP.proxyPort}") @@ -62,7 +67,10 @@ public class ProxyProperties { @Value("${clientIP.secretKeySM}") private String secretKeySM; - + /** + * 私密 ip 获取 + * @return + */ public String getUrl() { try { // https://dps.kdlapi.com/api/getdps/?secret_id=ol1ydp86g518q4y607xe&signature=6vwfgvti51u2njgntamc1f23lqq7kave&num=1&pt=1&format=json&sep=1 @@ -95,6 +103,10 @@ public class ProxyProperties { return null; } + /** + * 私密 ip剩余时长判断 + * @return + */ private Boolean getdpsvalidtime(String proxy) { Boolean flag = false; try { @@ -130,11 +142,11 @@ public class ProxyProperties { } return flag; } -// GET https://dps.kdlapi.com/api/checkdpsvalid?secret_id=o1fjh1re9o28876h7c08 -// &proxy=113.120.61.166:22989,123.163.183.200:18635 -// &signature=oxf0n0g59h7wcdyvz2uo68ph2s - + /** + * 私密 ip有效性验证 + * @return + */ private Boolean checkdpsvalid(String proxy) { Boolean flag = false; try { @@ -165,4 +177,41 @@ public class ProxyProperties { } return flag; } + + /** + * 隧道 ip获取 + * @return + */ + public String gettps(){ + CloseableHttpClient httpclient = HttpClients.createDefault(); + try { + String params = "?secret_id=" + secretId + "&num=1&signature=" + secretKey; + HttpGet httpget = new HttpGet(gettps + params); + httpget.addHeader("Accept-Encoding", "gzip"); //使用gzip压缩传输数据让访问更快 + + System.out.println("Executing request " + httpget.getURI()); + CloseableHttpResponse response = httpclient.execute(httpget); + try { + return EntityUtils.toString(response.getEntity()); + } finally { + response.close(); + } + } catch (ClientProtocolException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + httpclient.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + return null; + } + + public static void main(String[] args) { + ProxyProperties proxyProperties = new ProxyProperties(); + System.out.println(proxyProperties.gettps()); + } } diff --git a/ruoyi-common/src/main/java/com/ruoyi/common/utils/SplitListUtils.java b/ruoyi-common/src/main/java/com/ruoyi/common/utils/SplitListUtils.java new file mode 100644 index 0000000..79f0695 --- /dev/null +++ b/ruoyi-common/src/main/java/com/ruoyi/common/utils/SplitListUtils.java @@ -0,0 +1,98 @@ +package com.ruoyi.common.utils; + + +import org.apache.commons.compress.utils.Lists; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * @Description: TODO:拆分工具类 + * 1、获取需要进行批量更新的大集合A,对大集合进行拆分操作,分成N个小集合A-1 ~ A-N; + * 2、开启线程池,针对集合的大小进行调参,对小集合进行批量更新操作; + * 3、对流程进行控制,控制线程执行顺序。按照指定大小拆分集合的工具类 + */ +public class SplitListUtils { + /** + * 功能描述:拆分集合 + * @param 泛型对象 + * @MethodName: split + * @MethodParam: [resList:需要拆分的集合, subListLength:每个子集合的元素个数] + * @Return: java.util.List>:返回拆分后的各个集合组成的列表 + * 代码里面用到了guava和common的结合工具类 + */ + public static List> split(List resList, int subListLength) { + if (CollectionUtils.isEmpty(resList) || subListLength <= 0) { + return Lists.newArrayList(); + } + List> ret = Lists.newArrayList(); + int size = resList.size(); + if (size <= subListLength) { + // 数据量不足 subListLength 指定的大小 + ret.add(resList); + } else { + int pre = size / subListLength; + int last = size % subListLength; + // 前面pre个集合,每个大小都是 subListLength 个元素 + for (int i = 0; i < pre; i++) { + List itemList = Lists.newArrayList(); + for (int j = 0; j < subListLength; j++) { + itemList.add(resList.get(i * subListLength + j)); + } + ret.add(itemList); + } + // last的进行处理 + if (last > 0) { + List itemList = Lists.newArrayList(); + for (int i = 0; i < last; i++) { + itemList.add(resList.get(pre * subListLength + i)); + } + ret.add(itemList); + } + } + return ret; + } + + /** + * 功能描述:方法二:集合切割类,就是把一个大集合切割成多个指定条数的小集合,方便往数据库插入数据 + * 推荐使用 + * @MethodName: pagingList + * @MethodParam:[resList:需要拆分的集合, subListLength:每个子集合的元素个数] + * @Return: java.util.List>:返回拆分后的各个集合组成的列表 + + */ + public static List> pagingList(List resList, int pageSize){ + //判断是否为空 + if (CollectionUtils.isEmpty(resList) || pageSize <= 0) { + return Lists.newArrayList(); + } + int length = resList.size(); + int num = (length+pageSize-1)/pageSize; + List> newList = new ArrayList<>(); + for(int i=0;i list = Lists.newArrayList(); + int size = 19; + for (int i = 0; i < size; i++) { + list.add("hello-" + i); + } + // 大集合里面包含多个小集合 + List> temps = pagingList(list, 2); + int j = 0; + // 对大集合里面的每一个小集合进行操作 + for (List obj : temps) { + System.out.println(String.format("row:%s -> size:%s,data:%s", ++j, obj.size(), obj)); + } + } + +} diff --git a/ruoyi-framework/src/main/java/com/ruoyi/framework/config/ThreadPoolConfig.java b/ruoyi-framework/src/main/java/com/ruoyi/framework/config/ThreadPoolConfig.java index 7840141..5005c81 100644 --- a/ruoyi-framework/src/main/java/com/ruoyi/framework/config/ThreadPoolConfig.java +++ b/ruoyi-framework/src/main/java/com/ruoyi/framework/config/ThreadPoolConfig.java @@ -18,16 +18,16 @@ import java.util.concurrent.ThreadPoolExecutor; public class ThreadPoolConfig { // 核心线程池大小 - private int corePoolSize = 50; + private int corePoolSize = 10; // 最大可创建的线程数 - private int maxPoolSize = 200; + private int maxPoolSize = 20; // 队列最大长度 - private int queueCapacity = 1000; + private int queueCapacity = 500; // 线程池维护线程所允许的空闲时间 - private int keepAliveSeconds = 300; + private int keepAliveSeconds = 60; @Bean(name = "threadPoolTaskExecutor") public ThreadPoolTaskExecutor threadPoolTaskExecutor() @@ -60,4 +60,28 @@ public class ThreadPoolConfig } }; } + //1、自定义asyncServiceExecutor线程池 + @Bean(name = "asyncServiceExecutor") + public ThreadPoolTaskExecutor asyncServiceExecutor() { + System.out.println(("start asyncServiceExecutor......")); + //在这里修改 + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + //配置核心线程数 + executor.setCorePoolSize(corePoolSize); + //配置最大线程数 + executor.setMaxPoolSize(maxPoolSize); + //设置线程空闲等待时间 s + executor.setKeepAliveSeconds(keepAliveSeconds); + //配置队列大小 设置任务等待队列的大小 + executor.setQueueCapacity(queueCapacity); + //配置线程池中的线程的名称前缀 + //设置线程池内线程名称的前缀-------阿里编码规约推荐--方便出错后进行调试 + executor.setThreadNamePrefix("myTread=="); + // rejection-policy:当pool已经达到max size的时候,如何处理新任务 + // CALLER_RUNS:不在新线程中执行任务,而是有调用者所在的线程来执行 + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy()); + //执行初始化 + executor.initialize(); + return executor; + } } diff --git a/ruoyi-system/src/main/java/com/ruoyi/business/service/impl/AsyncTaskImpl.java b/ruoyi-system/src/main/java/com/ruoyi/business/service/impl/AsyncTaskImpl.java new file mode 100644 index 0000000..9b8ca6b --- /dev/null +++ b/ruoyi-system/src/main/java/com/ruoyi/business/service/impl/AsyncTaskImpl.java @@ -0,0 +1,46 @@ +package com.ruoyi.business.service.impl; + +import com.ruoyi.business.domain.BusStoreInfo; +import com.ruoyi.business.mapper.BusStoreInfoMapper; +import com.ruoyi.business.service.IMeituanService; +import com.ruoyi.common.core.domain.R; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +/** + * 自定义线程池 studentService换为ip + */ +@Service +public class AsyncTaskImpl { + + @Autowired + private IMeituanService iMeituanService; + + @Async("asyncServiceExecutor") + public void executeAsync(List busStoreInfoList, + BusStoreInfoMapper studentService, + CountDownLatch countDownLatch) { + try{ + System.out.println("start executeAsync"); + LocalDateTime now = LocalDateTime.now(); + //异步线程要做的事情 + for (BusStoreInfo store : busStoreInfoList) { +// R flag = iMeituanService.orderInfo(store.getStoreCode(),now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")),store.getStoreCookie()); + R flag = R.fail(); + if (flag.getCode() != 200) { + System.out.println(store.getStoreName()+"获取订单信息返回错误..."+store.getStoreCode()); + } + } + System.out.println("end executeAsync"); + }finally { + countDownLatch.countDown();// 很关键, 无论上面程序是否异常必须执行countDown,否则await无法释放 + } + } + +} diff --git a/ruoyi-system/src/main/java/com/ruoyi/business/service/impl/MeituanServiceImpl.java b/ruoyi-system/src/main/java/com/ruoyi/business/service/impl/MeituanServiceImpl.java index 74fc133..a4b2bf7 100644 --- a/ruoyi-system/src/main/java/com/ruoyi/business/service/impl/MeituanServiceImpl.java +++ b/ruoyi-system/src/main/java/com/ruoyi/business/service/impl/MeituanServiceImpl.java @@ -10,8 +10,10 @@ import com.ruoyi.common.config.ProxyProperties; import com.ruoyi.common.core.domain.R; import com.ruoyi.common.exception.ServiceException; import com.ruoyi.common.utils.SecurityUtils; +import com.ruoyi.common.utils.SplitListUtils; import com.ruoyi.common.utils.StringUtils; import com.ruoyi.system.mapper.SysDictDataMapper; +import jdk.nashorn.internal.runtime.logging.Logger; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; @@ -32,9 +34,13 @@ import org.apache.http.util.EntityUtils; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.util.UriUtils; +import javax.annotation.Resource; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; @@ -45,12 +51,14 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.*; +import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; /** * 采集美团数据service */ @Service +@Logger public class MeituanServiceImpl implements IMeituanService { @Autowired @@ -258,6 +266,10 @@ public class MeituanServiceImpl implements IMeituanService { } return "ok"; } +// @Resource(name = "asyncServiceExecutor") +// private ThreadPoolTaskExecutor asyncServiceExecutor; + @Autowired + private AsyncTaskImpl asyncTask; /** * 获取订单批量 * @@ -270,12 +282,27 @@ public class MeituanServiceImpl implements IMeituanService { BusStoreInfo busStoreInfo = new BusStoreInfo(); busStoreInfo.setReturnVisitStatus("1"); List busStoreInfoList = busStoreInfoMapper.selectBusStoreInfoList(busStoreInfo); - for (BusStoreInfo store : busStoreInfoList) { - R flag = orderInfo(store.getStoreCode(),now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")),store.getStoreCookie()); - if (flag.getCode() != 200) { - System.out.println("获取订单信息返回错误..."); - } + long startTime = System.currentTimeMillis(); // 开始时间 + //boolean a=studentService.batchInsert(list); + List> list1= SplitListUtils.pagingList(busStoreInfoList,4); //拆分集合 + CountDownLatch countDownLatch = new CountDownLatch(list1.size()); + for (List list2 : list1) { + asyncTask.executeAsync(list2,busStoreInfoMapper,countDownLatch); } + try { + countDownLatch.await(); //保证之前的所有的线程都执行完成,才会走下面的; + long endTime = System.currentTimeMillis(); //结束时间 + System.out.println(("一共耗时time: " + (endTime - startTime) / 1000 + " s")); + // 这样就可以在下面拿到所有线程执行完的集合结果 + } catch (Exception e) { + System.out.println("阻塞异常:"+e.getMessage()); + } +// for (BusStoreInfo store : busStoreInfoList) { +// R flag = orderInfo(store.getStoreCode(),now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")),store.getStoreCookie()); +// if (flag.getCode() != 200) { +// System.out.println("获取订单信息返回错误..."); +// } +// } return "ok"; } @@ -490,7 +517,7 @@ public class MeituanServiceImpl implements IMeituanService { storeInfo.setStoreCode(wmPoiId); BusStoreInfo storeInfo1 = busStoreInfoMapper.selectBusStoreInfoList(storeInfo).get(0); storeInfo1.setGrantStatus("2"); - busStoreInfoService.updateBusStoreInfo(storeInfo1); + busStoreInfoMapper.updateBusStoreInfo(storeInfo1); System.out.println(jo.getString("msg")); return R.fail(jo.getString("msg")); @@ -502,7 +529,7 @@ public class MeituanServiceImpl implements IMeituanService { storeInfo.setStoreCode(wmPoiId); BusStoreInfo storeInfo1 = busStoreInfoMapper.selectBusStoreInfoList(storeInfo).get(0); storeInfo1.setGrantStatus("2"); - busStoreInfoService.updateBusStoreInfo(storeInfo1); + busStoreInfoMapper.updateBusStoreInfo(storeInfo1); } return R.fail(jsonObject.getString("msg")); } @@ -749,7 +776,7 @@ public class MeituanServiceImpl implements IMeituanService { */ @Override public String mtgsigInfo(String getUrl, String orderId, String regionId, String regionVersion,String dfpid) { - CloseableHttpClient httpClient = proxyHttpClient(); + CloseableHttpClient httpClient = HttpClients.custom().build(); String url = "http://43.140.224.18:12000/get_mtgsig"; HttpPost httpPost = new HttpPost(url); httpPost.setHeader("Content-Type", "application/json"); @@ -765,8 +792,8 @@ public class MeituanServiceImpl implements IMeituanService { JSONObject jsonObject = new JSONObject(); jsonObject.put("url","https://e.waimai.meituan.com/v2/order/history/r/search/ajax?searchItem="+orderId+"®ion_id="+regionId+"®ion_version="+regionVersion+"&yodaReady=h5&csecplatform=4&csecversion=2.4.0"); jsonObject.put("data",""); - jsonObject.put("dfpid",dfpid); - jsonObject.put("method","post"); + jsonObject.put("dfpid",dfpid.split("-")[0]); + jsonObject.put("method","GET"); StringEntity entity = new StringEntity(JSONObject.toJSONString(jsonObject), "utf-8"); // subUrlParams.put("data", ""); @@ -778,51 +805,25 @@ public class MeituanServiceImpl implements IMeituanService { // urlParam = urlParam + stringObjectEntry.getKey() + "=" + stringObjectEntry.getValue() + "&"; // } String result = null; - try { // StringEntity params = new StringEntity(urlParam.substring(0, urlParam.length() - 1)); + try { httpPost.setEntity(entity); CloseableHttpResponse response = httpClient.execute(httpPost); result = EntityUtils.toString(response.getEntity()); - result = URLEncoder.encode(result,"utf-8") - .replace("+", "%20") - .replace("%21", "!") - .replace("%27", "'") - .replace("%28", "(") - .replace("%29", ")") - .replace("%7E", "~"); - } catch (Exception e) { +// result = UriUtils.encode(result,"utf-8"); + result = URLEncoder.encode(result); + response.close(); + } catch (IOException e) { e.printStackTrace(); - }finally { try { httpClient.close(); - } catch (IOException e) { - e.printStackTrace(); + } catch (IOException ex) { + ex.printStackTrace(); } } - return result; } -// public static void main(String[] args) throws Exception { -// String s = "{\n" + -// "\"a1\": \"1.1\",\n" + -// "\"a2\": 1722262786240,\n" + -// "\"a3\": \"1719555287467YYKUWIAfd79fef3d01d5e9aadc18ccd4d0c95077155\",\n" + -// "\"a5\": \"Hb6DmbOppfh+9/IyVMq4\",\n" + -// "\"a6\": \"hs1.4aOG4x69iuIGtADfqn9IKcXWfqxcu6RKBEt0ULtKkr0Pxmrai6YXTRjzK6Sg7d16jqhHlzuHMnrjVQt4PthJFNw==\",\n" + -// "\"x0\": 4,\n" + -// "\"d1\": \"9cff2c0335b0bb892a1bd01fa94f978f\"\n" + -// "}"; -// String ss = URLEncoder.encode(s,"utf-8") -// .replace("+", "%20") -// .replace("%21", "!") -// .replace("%27", "'") -// .replace("%28", "(") -// .replace("%29", ")") -// .replace("%7E", "~"); -// System.out.println(ss); -// } - @Override public void getReturnInfo(Long id) { getComments(null);