wangshuai
/
youshi
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

1; 隧道+线程池

This commit is contained in:
wangshuai 2024-07-30 06:46:35 +08:00
parent e3af04aa1f
commit bbb79118a2
6 changed files with 271 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ruoyi-admin/src/main/resources/application.yml
View File

@ -145,6 +145,7 @@ tencent:
appKey: c167746e38d64143a874cec3d5de014e
clientIP:
gettps: https://tps.kdlapi.com/api/gettps
proxyAddr: k136.kdltps.com
proxyPort: 15818
proxySocks: 20818

59
ruoyi-common/src/main/java/com/ruoyi/common/config/ProxyProperties.java
View File

@ -2,6 +2,7 @@ package com.ruoyi.common.config;
import com.alibaba.fastjson2.JSONObject;
import lombok.Data;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
@ -10,6 +11,8 @@ import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import java.io.IOException;
@Data
@Configuration
public class ProxyProperties {
@ -24,6 +27,8 @@ public class ProxyProperties {
// secretId: o7fnvzz6211s5ymtw3nq
// secretKey: qg2ehv53xazqj0fa5zer3dcssoj311ff
@Value("${clientIP.gettps}")
private String gettps;
@Value("${clientIP.proxyAddr}")
private String proxyAddr;
@Value("${clientIP.proxyPort}")
@ -62,7 +67,10 @@ public class ProxyProperties {
@Value("${clientIP.secretKeySM}")
private String secretKeySM;
/**
* 私密 ip 获取
* @return
*/
public String getUrl() {
try {
// https://dps.kdlapi.com/api/getdps/?secret_id=ol1ydp86g518q4y607xe&signature=6vwfgvti51u2njgntamc1f23lqq7kave&num=1&pt=1&format=json&sep=1
@ -95,6 +103,10 @@ public class ProxyProperties {
return null;
}
/**
* 私密 ip剩余时长判断
* @return
*/
private Boolean getdpsvalidtime(String proxy) {
Boolean flag = false;
try {
@ -130,11 +142,11 @@ public class ProxyProperties {
}
return flag;
}
// GET https://dps.kdlapi.com/api/checkdpsvalid?secret_id=o1fjh1re9o28876h7c08
// &proxy=113.120.61.166:22989,123.163.183.200:18635
// &signature=oxf0n0g59h7wcdyvz2uo68ph2s
/**
* 私密 ip有效性验证
* @return
*/
private Boolean checkdpsvalid(String proxy) {
Boolean flag = false;
try {
@ -165,4 +177,41 @@ public class ProxyProperties {
}
return flag;
}
/**
* 隧道 ip获取
* @return
*/
public String gettps(){
CloseableHttpClient httpclient = HttpClients.createDefault();
try {
String params = "?secret_id=" + secretId + "&num=1&signature=" + secretKey;
HttpGet httpget = new HttpGet(gettps + params);
httpget.addHeader("Accept-Encoding", "gzip"); //使用gzip压缩传输数据让访问更快
System.out.println("Executing request " + httpget.getURI());
CloseableHttpResponse response = httpclient.execute(httpget);
try {
return EntityUtils.toString(response.getEntity());
} finally {
response.close();
}
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
httpclient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}
public static void main(String[] args) {
ProxyProperties proxyProperties = new ProxyProperties();
System.out.println(proxyProperties.gettps());
}
}

98
ruoyi-common/src/main/java/com/ruoyi/common/utils/SplitListUtils.java Normal file
View File

@ -0,0 +1,98 @@
package com.ruoyi.common.utils;
import org.apache.commons.compress.utils.Lists;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
/**
* @Description: TODO：拆分工具类
* 1、获取需要进行批量更新的大集合A，对大集合进行拆分操作，分成N个小集合A-1 ~ A-N;
* 2、开启线程池，针对集合的大小进行调参，对小集合进行批量更新操作;
* 3、对流程进行控制，控制线程执行顺序。按照指定大小拆分集合的工具类
*/
public class SplitListUtils {
/**
* 功能描述:拆分集合
* @param <T> 泛型对象
* @MethodName: split
* @MethodParam: [resList:需要拆分的集合, subListLength:每个子集合的元素个数]
* @Return: java.util.List<java.util.List<T>>:返回拆分后的各个集合组成的列表
* 代码里面用到了guava和common的结合工具类
*/
public static <T> List<List<T>> split(List<T> resList, int subListLength) {
if (CollectionUtils.isEmpty(resList) || subListLength <= 0) {
return Lists.newArrayList();
}
List<List<T>> ret = Lists.newArrayList();
int size = resList.size();
if (size <= subListLength) {
// 数据量不足 subListLength 指定的大小
ret.add(resList);
} else {
int pre = size / subListLength;
int last = size % subListLength;
// 前面pre个集合，每个大小都是 subListLength 个元素
for (int i = 0; i < pre; i++) {
List<T> itemList = Lists.newArrayList();
for (int j = 0; j < subListLength; j++) {
itemList.add(resList.get(i * subListLength + j));
}
ret.add(itemList);
}
// last的进行处理
if (last > 0) {
List<T> itemList = Lists.newArrayList();
for (int i = 0; i < last; i++) {
itemList.add(resList.get(pre * subListLength + i));
}
ret.add(itemList);
}
}
return ret;
}
/**
* 功能描述:方法二：集合切割类，就是把一个大集合切割成多个指定条数的小集合，方便往数据库插入数据
* 推荐使用
* @MethodName: pagingList
* @MethodParam:[resList:需要拆分的集合, subListLength:每个子集合的元素个数]
* @Return: java.util.List<java.util.List<T>>：返回拆分后的各个集合组成的列表
*/
public static <T> List<List<T>> pagingList(List<T> resList, int pageSize){
//判断是否为空
if (CollectionUtils.isEmpty(resList) || pageSize <= 0) {
return Lists.newArrayList();
}
int length = resList.size();
int num = (length+pageSize-1)/pageSize;
List<List<T>> newList = new ArrayList<>();
for(int i=0;i<num;i++){
int fromIndex = i*pageSize;
int toIndex = (i+1)*pageSize<length?(i+1)*pageSize:length;
newList.add(resList.subList(fromIndex,toIndex));
}
return newList;
}
// 运行测试代码 可以按顺序拆分为11个集合
public static void main(String[] args) {
//初始化数据
List<String> list = Lists.newArrayList();
int size = 19;
for (int i = 0; i < size; i++) {
list.add("hello-" + i);
}
// 大集合里面包含多个小集合
List<List<String>> temps = pagingList(list, 2);
int j = 0;
// 对大集合里面的每一个小集合进行操作
for (List<String> obj : temps) {
System.out.println(String.format("row:%s -> size:%s,data:%s", ++j, obj.size(), obj));
}
}
}

32
ruoyi-framework/src/main/java/com/ruoyi/framework/config/ThreadPoolConfig.java
View File

@ -18,16 +18,16 @@ import java.util.concurrent.ThreadPoolExecutor;
public class ThreadPoolConfig
{
// 核心线程池大小
private int corePoolSize = 50;
private int corePoolSize = 10;
// 最大可创建的线程数
private int maxPoolSize = 200;
private int maxPoolSize = 20;
// 队列最大长度
private int queueCapacity = 1000;
private int queueCapacity = 500;
// 线程池维护线程所允许的空闲时间
private int keepAliveSeconds = 300;
private int keepAliveSeconds = 60;
@Bean(name = "threadPoolTaskExecutor")
public ThreadPoolTaskExecutor threadPoolTaskExecutor()
@ -60,4 +60,28 @@ public class ThreadPoolConfig
}
};
}
//1、自定义asyncServiceExecutor线程池
@Bean(name = "asyncServiceExecutor")
public ThreadPoolTaskExecutor asyncServiceExecutor() {
System.out.println(("start asyncServiceExecutor......"));
//在这里修改
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
//配置核心线程数
executor.setCorePoolSize(corePoolSize);
//配置最大线程数
executor.setMaxPoolSize(maxPoolSize);
//设置线程空闲等待时间 s
executor.setKeepAliveSeconds(keepAliveSeconds);
//配置队列大小 设置任务等待队列的大小
executor.setQueueCapacity(queueCapacity);
//配置线程池中的线程的名称前缀
//设置线程池内线程名称的前缀-------阿里编码规约推荐--方便出错后进行调试
executor.setThreadNamePrefix("myTread==");
// rejection-policy：当pool已经达到max size的时候，如何处理新任务
// CALLER_RUNS：不在新线程中执行任务，而是有调用者所在的线程来执行
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
//执行初始化
executor.initialize();
return executor;
}
}

46
ruoyi-system/src/main/java/com/ruoyi/business/service/impl/AsyncTaskImpl.java Normal file
View File

@ -0,0 +1,46 @@
package com.ruoyi.business.service.impl;
import com.ruoyi.business.domain.BusStoreInfo;
import com.ruoyi.business.mapper.BusStoreInfoMapper;
import com.ruoyi.business.service.IMeituanService;
import com.ruoyi.common.core.domain.R;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.concurrent.CountDownLatch;
/**
* 自定义线程池 studentService换为ip
*/
@Service
public class AsyncTaskImpl {
@Autowired
private IMeituanService iMeituanService;
@Async("asyncServiceExecutor")
public void executeAsync(List<BusStoreInfo> busStoreInfoList,
BusStoreInfoMapper studentService,
CountDownLatch countDownLatch) {
try{
System.out.println("start executeAsync");
LocalDateTime now = LocalDateTime.now();
//异步线程要做的事情
for (BusStoreInfo store : busStoreInfoList) {
// R flag = iMeituanService.orderInfo(store.getStoreCode(),now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")),store.getStoreCookie());
R flag = R.fail();
if (flag.getCode() != 200) {
System.out.println(store.getStoreName()+"获取订单信息返回错误..."+store.getStoreCode());
}
}
System.out.println("end executeAsync");
}finally {
countDownLatch.countDown();// 很关键, 无论上面程序是否异常必须执行countDown,否则await无法释放
}
}
}

87
ruoyi-system/src/main/java/com/ruoyi/business/service/impl/MeituanServiceImpl.java
View File

@ -10,8 +10,10 @@ import com.ruoyi.common.config.ProxyProperties;
import com.ruoyi.common.core.domain.R;
import com.ruoyi.common.exception.ServiceException;
import com.ruoyi.common.utils.SecurityUtils;
import com.ruoyi.common.utils.SplitListUtils;
import com.ruoyi.common.utils.StringUtils;
import com.ruoyi.system.mapper.SysDictDataMapper;
import jdk.nashorn.internal.runtime.logging.Logger;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
@ -32,9 +34,13 @@ import org.apache.http.util.EntityUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.util.UriUtils;
import javax.annotation.Resource;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
@ -45,12 +51,14 @@ import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.stream.Collectors;
/**
* 采集美团数据service
*/
@Service
@Logger
public class MeituanServiceImpl implements IMeituanService {
@Autowired
@ -258,6 +266,10 @@ public class MeituanServiceImpl implements IMeituanService {
}
return "ok";
}
// @Resource(name = "asyncServiceExecutor")
// private ThreadPoolTaskExecutor asyncServiceExecutor;
@Autowired
private AsyncTaskImpl asyncTask;
/**
* 获取订单批量
*
@ -270,12 +282,27 @@ public class MeituanServiceImpl implements IMeituanService {
BusStoreInfo busStoreInfo = new BusStoreInfo();
busStoreInfo.setReturnVisitStatus("1");
List<BusStoreInfo> busStoreInfoList = busStoreInfoMapper.selectBusStoreInfoList(busStoreInfo);
for (BusStoreInfo store : busStoreInfoList) {
R flag = orderInfo(store.getStoreCode(),now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")),store.getStoreCookie());
if (flag.getCode() != 200) {
System.out.println("获取订单信息返回错误...");
}
long startTime = System.currentTimeMillis(); // 开始时间
//boolean a=studentService.batchInsert(list);
List<List<BusStoreInfo>> list1= SplitListUtils.pagingList(busStoreInfoList,4); //拆分集合
CountDownLatch countDownLatch = new CountDownLatch(list1.size());
for (List<BusStoreInfo> list2 : list1) {
asyncTask.executeAsync(list2,busStoreInfoMapper,countDownLatch);
}
try {
countDownLatch.await(); //保证之前的所有的线程都执行完成，才会走下面的；
long endTime = System.currentTimeMillis(); //结束时间
System.out.println(("一共耗时time: " + (endTime - startTime) / 1000 + " s"));
// 这样就可以在下面拿到所有线程执行完的集合结果
} catch (Exception e) {
System.out.println("阻塞异常:"+e.getMessage());
}
// for (BusStoreInfo store : busStoreInfoList) {
// R flag = orderInfo(store.getStoreCode(),now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")),store.getStoreCookie());
// if (flag.getCode() != 200) {
// System.out.println("获取订单信息返回错误...");
// }
// }
return "ok";
}
@ -490,7 +517,7 @@ public class MeituanServiceImpl implements IMeituanService {
storeInfo.setStoreCode(wmPoiId);
BusStoreInfo storeInfo1 = busStoreInfoMapper.selectBusStoreInfoList(storeInfo).get(0);
storeInfo1.setGrantStatus("2");
busStoreInfoService.updateBusStoreInfo(storeInfo1);
busStoreInfoMapper.updateBusStoreInfo(storeInfo1);
System.out.println(jo.getString("msg"));
return R.fail(jo.getString("msg"));
@ -502,7 +529,7 @@ public class MeituanServiceImpl implements IMeituanService {
storeInfo.setStoreCode(wmPoiId);
BusStoreInfo storeInfo1 = busStoreInfoMapper.selectBusStoreInfoList(storeInfo).get(0);
storeInfo1.setGrantStatus("2");
busStoreInfoService.updateBusStoreInfo(storeInfo1);
busStoreInfoMapper.updateBusStoreInfo(storeInfo1);
}
return R.fail(jsonObject.getString("msg"));
}
@ -749,7 +776,7 @@ public class MeituanServiceImpl implements IMeituanService {
*/
@Override
public String mtgsigInfo(String getUrl, String orderId, String regionId, String regionVersion,String dfpid) {
CloseableHttpClient httpClient = proxyHttpClient();
CloseableHttpClient httpClient = HttpClients.custom().build();
String url = "http://43.140.224.18:12000/get_mtgsig";
HttpPost httpPost = new HttpPost(url);
httpPost.setHeader("Content-Type", "application/json");
@ -765,8 +792,8 @@ public class MeituanServiceImpl implements IMeituanService {
JSONObject jsonObject = new JSONObject();
jsonObject.put("url","https://e.waimai.meituan.com/v2/order/history/r/search/ajax?searchItem="+orderId+"&region_id="+regionId+"&region_version="+regionVersion+"&yodaReady=h5&csecplatform=4&csecversion=2.4.0");
jsonObject.put("data","");
jsonObject.put("dfpid",dfpid);
jsonObject.put("method","post");
jsonObject.put("dfpid",dfpid.split("-")[0]);
jsonObject.put("method","GET");
StringEntity entity = new StringEntity(JSONObject.toJSONString(jsonObject), "utf-8");
// subUrlParams.put("data", "");
@ -778,51 +805,25 @@ public class MeituanServiceImpl implements IMeituanService {
// urlParam = urlParam + stringObjectEntry.getKey() + "=" + stringObjectEntry.getValue() + "&";
// }
String result = null;
try {
// StringEntity params = new StringEntity(urlParam.substring(0, urlParam.length() - 1));
try {
httpPost.setEntity(entity);
CloseableHttpResponse response = httpClient.execute(httpPost);
result = EntityUtils.toString(response.getEntity());
result = URLEncoder.encode(result,"utf-8")
.replace("+", "%20")
.replace("%21", "!")
.replace("%27", "'")
.replace("%28", "(")
.replace("%29", ")")
.replace("%7E", "~");
} catch (Exception e) {
// result = UriUtils.encode(result,"utf-8");
result = URLEncoder.encode(result);
response.close();
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
} catch (IOException ex) {
ex.printStackTrace();
}
}
return result;
}
// public static void main(String[] args) throws Exception {
// String s = "{\n" +
// "\"a1\": \"1.1\",\n" +
// "\"a2\": 1722262786240,\n" +
// "\"a3\": \"1719555287467YYKUWIAfd79fef3d01d5e9aadc18ccd4d0c95077155\",\n" +
// "\"a5\": \"Hb6DmbOppfh+9/IyVMq4\",\n" +
// "\"a6\": \"hs1.4aOG4x69iuIGtADfqn9IKcXWfqxcu6RKBEt0ULtKkr0Pxmrai6YXTRjzK6Sg7d16jqhHlzuHMnrjVQt4PthJFNw==\",\n" +
// "\"x0\": 4,\n" +
// "\"d1\": \"9cff2c0335b0bb892a1bd01fa94f978f\"\n" +
// "}";
// String ss = URLEncoder.encode(s,"utf-8")
// .replace("+", "%20")
// .replace("%21", "!")
// .replace("%27", "'")
// .replace("%28", "(")
// .replace("%29", ")")
// .replace("%7E", "~");
// System.out.println(ss);
// }
@Override
public void getReturnInfo(Long id) {
getComments(null);