Test_Project_Web/e5
1
0
Fork 0
mirror of https://github.com/luoye663/e5.git synced 2024-12-25 03:08:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

1、取消启动清空redis

Browse Source 
2、将调用日志放到influx，减少mysql压力
This commit is contained in:
LuoYe 2021-08-19 16:10:11 +08:00
parent 1006883e95
commit 5b318e2357
9 changed files with 90 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -10,7 +10,7 @@
</parent> </parent>
<groupId>io.qyi</groupId> <groupId>io.qyi</groupId>
<artifactId>e5</artifactId> <artifactId>e5</artifactId>
<version>1.0.7</version> <version>1.0.8</version>
<name>e5</name> <name>e5</name>
<description>Demo project for Spring Boot</description> <description>Demo project for Spring Boot</description>

2
src/main/java/io/qyi/e5/bean/influx2/InfluxdbConfig.java
View File

@ -17,7 +17,7 @@ public class InfluxdbConfig {
@Bean @Bean
public InfluxDBClient influxDBClient() { public InfluxDBClient influxDBClient() {
InfluxDBClient influxDBClient = InfluxDBClientFactory.create(influxDBUrl, token.toCharArray()); InfluxDBClient influxDBClient = InfluxDBClientFactory.create(influxDBUrl, token.toCharArray());
influxDBClient.setLogLevel(LogLevel.BASIC); influxDBClient.setLogLevel(LogLevel.NONE);
return influxDBClient; return influxDBClient;
} }

14
src/main/java/io/qyi/e5/config/Start.java
View File

@ -38,13 +38,17 @@ public class Start {
@Value("${e5.system.threadPool}") @Value("${e5.system.threadPool}")
Integer poolSize; Integer poolSize;
@Value("${isdebug:true}")
private boolean isdebug;
private ExecutorService threadPool; private ExecutorService threadPool;
@PostConstruct @PostConstruct
public void init() { public void init() {
log.info("清空redis...... "); // log.info("清空redis...... ");
redisUtil.delAll(); // redisUtil.delAll();
threadPool = new ThreadPoolExecutor( threadPool = new ThreadPoolExecutor(
//指定了线程池中的线程数量，它的数量决定了添加的任务是开辟新的线程去执行，还是放到workQueue任务队列中去； //指定了线程池中的线程数量，它的数量决定了添加的任务是开辟新的线程去执行，还是放到workQueue任务队列中去；
poolSize, poolSize,
@ -64,8 +68,12 @@ public class Start {
} }
// @Scheduled(cron = "0 0/1 * * * ? ") @Scheduled(cron = "0 0/1 * * * ? ")
private void distributeTask() { private void distributeTask() {
if (isdebug) {
log.debug("Debug模式，跳过执行");
return;
}
List<Outlook> runOutlookList = outlookService.findRunOutlookList(); List<Outlook> runOutlookList = outlookService.findRunOutlookList();
CountDownLatch cdl = new CountDownLatch(runOutlookList.size()); CountDownLatch cdl = new CountDownLatch(runOutlookList.size());

4
src/main/java/io/qyi/e5/controller/web/WebController.java
View File

@ -59,7 +59,7 @@ public class WebController {
public Result delete(){ public Result delete(){
UsernamePasswordAuthenticationToken authentication = (UsernamePasswordAuthenticationToken) SecurityContextHolder.getContext().getAuthentication(); UsernamePasswordAuthenticationToken authentication = (UsernamePasswordAuthenticationToken) SecurityContextHolder.getContext().getAuthentication();
/*删除数据库信息*/ /*删除数据库信息*/
int outlooklog = iOutlookLogService.deleteInfo(authentication.getGithub_id()); // int outlooklog = iOutlookLogService.deleteInfo(authentication.getGithub_id());
int outlook = outlookService.deleteInfo(authentication.getGithub_id()); int outlook = outlookService.deleteInfo(authentication.getGithub_id());
int github = GithubService.deleteInfo(authentication.getGithub_id()); int github = GithubService.deleteInfo(authentication.getGithub_id());
/*删除redis中信息*/ /*删除redis中信息*/
@ -69,7 +69,7 @@ public class WebController {
} }
/*返回结果信息*/ /*返回结果信息*/
Map<String, Integer> map = new HashMap<>(); Map<String, Integer> map = new HashMap<>();
map.put("outlooklog", outlooklog); // map.put("outlooklog", outlooklog);
map.put("outlook", outlook); map.put("outlook", outlook);
map.put("github", github); map.put("github", github);
return ResultUtil.success(map); return ResultUtil.success(map);

26
src/main/java/io/qyi/e5/outlook_log/controller/OutlookLogController.java
View File

@ -1,15 +1,7 @@
package io.qyi.e5.outlook_log.controller; package io.qyi.e5.outlook_log.controller;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.influxdb.client.InfluxDBClient; import com.influxdb.client.InfluxDBClient;
import com.influxdb.client.QueryApi;
import com.influxdb.client.WriteApi;
import com.influxdb.client.WriteOptions;
import com.influxdb.client.domain.WritePrecision;
import com.influxdb.client.write.Point;
import com.influxdb.query.FluxRecord;
import com.influxdb.query.FluxTable;
import io.qyi.e5.bean.result.Result; import io.qyi.e5.bean.result.Result;
import io.qyi.e5.config.security.UsernamePasswordAuthenticationToken; import io.qyi.e5.config.security.UsernamePasswordAuthenticationToken;
import io.qyi.e5.outlook.service.IOutlookService; import io.qyi.e5.outlook.service.IOutlookService;
@ -17,8 +9,6 @@ import io.qyi.e5.outlook_log.bena.LogVo;
import io.qyi.e5.outlook_log.entity.OutlookLog; import io.qyi.e5.outlook_log.entity.OutlookLog;
import io.qyi.e5.outlook_log.service.IOutlookLogService; import io.qyi.e5.outlook_log.service.IOutlookLogService;
import io.qyi.e5.util.ResultUtil; import io.qyi.e5.util.ResultUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
@ -29,8 +19,6 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseBody;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
@ -73,20 +61,8 @@ public class OutlookLogController {
public Result findLog(@RequestParam int outlookId){ public Result findLog(@RequestParam int outlookId){
UsernamePasswordAuthenticationToken authentication = (UsernamePasswordAuthenticationToken) SecurityContextHolder.getContext().getAuthentication(); UsernamePasswordAuthenticationToken authentication = (UsernamePasswordAuthenticationToken) SecurityContextHolder.getContext().getAuthentication();
int github_id = authentication.getGithub_id(); int github_id = authentication.getGithub_id();
QueryWrapper<OutlookLog> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("github_id", github_id).eq("outlook_id", outlookId).orderByAsc("call_time");
List<OutlookLog> list = outlookLogService.findAllList(github_id, outlookId); List<OutlookLog> list = outlookLogService.findAllList(github_id, outlookId);
Iterator<OutlookLog> iterator = list.iterator(); return ResultUtil.success(list);
List<LogVo> logVo = new LinkedList<>();
while (iterator.hasNext()) {
OutlookLog next = iterator.next();
LogVo vo = new LogVo();
BeanUtils.copyProperties(next,vo);
logVo.add(vo);
}
return ResultUtil.success(logVo);
} }
} }

6
src/main/java/io/qyi/e5/outlook_log/entity/OutlookLog.java
View File

@ -43,14 +43,14 @@ public class OutlookLog {
/** /**
* 调用时间 * 调用时间
*/ */
@Column(timestamp = true) @Column
private Instant callTime; private long callTime;
/** /**
* 调用结果 * 调用结果
*/ */
@Column @Column
private Number resultc; private Number result;
/** /**
* 如果有错误原因则记录 * 如果有错误原因则记录

2
src/main/java/io/qyi/e5/outlook_log/service/IOutlookLogService.java
View File

@ -15,6 +15,6 @@ import java.util.List;
*/ */
public interface IOutlookLogService { public interface IOutlookLogService {
void addLog(int githubId,int outlookId, String msg,int result,String original_msg); void addLog(int githubId,int outlookId, String msg,int result,String original_msg);
int deleteInfo(int github_id); // int deleteInfo(int github_id);
List<OutlookLog> findAllList(int githubId, int outlookId); List<OutlookLog> findAllList(int githubId, int outlookId);
} }

50
src/main/java/io/qyi/e5/outlook_log/service/impl/OutlookLogServiceImpl.java
View File

@ -1,12 +1,15 @@
package io.qyi.e5.outlook_log.service.impl; package io.qyi.e5.outlook_log.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.influxdb.client.InfluxDBClient; import com.influxdb.client.InfluxDBClient;
import com.influxdb.client.InfluxDBClientFactory; import com.influxdb.client.QueryApi;
import com.influxdb.client.WriteApi;
import com.influxdb.client.domain.WritePrecision;
import io.qyi.e5.outlook_log.entity.OutlookLog; import io.qyi.e5.outlook_log.entity.OutlookLog;
import io.qyi.e5.outlook_log.service.IOutlookLogService; import io.qyi.e5.outlook_log.service.IOutlookLogService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.List; import java.util.List;
/** /**
@ -20,33 +23,40 @@ import java.util.List;
@Service @Service
public class OutlookLogServiceImpl implements IOutlookLogService { public class OutlookLogServiceImpl implements IOutlookLogService {
@Resource
private InfluxDBClient influxDBClient;
@Value("${spring.influx.org}")
private String org;
@Value("${spring.influx.bucket}")
private String bucket;
@Override @Override
public void addLog(int githubId, int outlookId, String msg, int result, String original_msg) { public void addLog(int githubId, int outlookId, String msg, int result, String original_msg) {
// OutlookLog outlookLog = new OutlookLog(); try (WriteApi writeApi = influxDBClient.getWriteApi()) {
// outlookLog.setGithubId(githubId) OutlookLog log = new OutlookLog();
// .setOutlookId(outlookId) log.setCallTime(System.currentTimeMillis())
// .setResult(result) .setGithubId(String.valueOf(githubId))
// .setMsg(msg) .setOutlookId(String.valueOf(outlookId))
// .setOriginalMsg(original_msg); .setMsg(msg)
.setOriginalMsg(original_msg)
.setResult(result);
writeApi.writeMeasurement(bucket, org, WritePrecision.NS, log);
// baseMapper.insert(outlookLog); }
}
@Override
public int deleteInfo(int github_id) {
QueryWrapper<OutlookLog> outlookLogQueryWrapper = new QueryWrapper<>();
outlookLogQueryWrapper.eq("github_id", github_id);
// return baseMapper.delete(outlookLogQueryWrapper);
return 0;
} }
@Override @Override
public List<OutlookLog> findAllList(int githubId, int outlookId) { public List<OutlookLog> findAllList(int githubId, int outlookId) {
// return baseMapper.findAllList(githubId, outlookId); String flux = "from(bucket:\"" + bucket + "\") |> range(start: 0)" +
return null; "|> filter(fn: (r) => r[\"_measurement\"] == \"OutlookLog\")" +
"|> filter(fn: (r) => r[\"githubId\"] == \"" + githubId + "\")" +
"|> filter(fn: (r) => r[\"outlookId\"] == \"" + outlookId + "\")" +
"|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")";
QueryApi queryApi = influxDBClient.getQueryApi();
List<OutlookLog> tables = queryApi.query(flux, org, OutlookLog.class);
return tables;
} }
} }

77
src/test/java/influxdb2Test.java
View File

@ -8,11 +8,12 @@ import io.qyi.e5.outlook_log.entity.OutlookLog;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.cglib.beans.BeanMap; import org.springframework.cglib.beans.BeanMap;
import java.time.Instant;
import java.util.*; import java.util.*;
public class influxdb2Test { public class influxdb2Test {
InfluxDBClient influxDBClient = InfluxDBClientFactory.create("http://127.0.0.1:8086", "ko6GtE_P5R2AlMkCBkEgBwW7rVBl46GYx0IoCrG-Dd5VFxTDSnFJ--BB2f8FRFcGd6Tb_yu6-MlMAD-lMSbH6A==".toCharArray() String token = "HquxNXwOyfgW-f8wzkSUuBz0tswWiFPsTbEnr5jKFS3BY3RcKezDdQF0o5yeoNfaiwQUJhy8YJSIUrVrWSQn8Q==";
InfluxDBClient influxDBClient = InfluxDBClientFactory.create("http://127.0.0.1:8086", token.toCharArray()
); );
private String org = "luoye"; private String org = "luoye";
@ -22,7 +23,7 @@ public class influxdb2Test {
.build(); .build();
@Test @Test
public void save(){ public void save() {
influxDBClient.setLogLevel(LogLevel.BASIC); influxDBClient.setLogLevel(LogLevel.BASIC);
WriteOptions writeOptions = WriteOptions.builder() WriteOptions writeOptions = WriteOptions.builder()
@ -36,13 +37,13 @@ public class influxdb2Test {
List<OutlookLog> list = new ArrayList<>(); List<OutlookLog> list = new ArrayList<>();
for (int i = 0; i < 100; i++) { for (int i = 0; i < 100; i++) {
OutlookLog outlookLog = new OutlookLog(); OutlookLog outlookLog = new OutlookLog();
outlookLog.setMsg(i + "- ok").setOriginalMsg("加入成功").setCallTime(Instant.now()); outlookLog.setMsg(i + "- ok").setOriginalMsg("加入成功").setCallTime(System.currentTimeMillis());
list.add(outlookLog); list.add(outlookLog);
} }
try (WriteApi writeApi = influxDBClient.getWriteApi()) { try (WriteApi writeApi = influxDBClient.getWriteApi()) {
// writeApi.writeMeasurement("e5", org ,WritePrecision.NS,outlookLog); // writeApi.writeMeasurement("e5", org ,WritePrecision.NS,outlookLog);
writeApi.writeMeasurements("e5", org ,WritePrecision.NS,list); writeApi.writeMeasurements("e5", org, WritePrecision.NS, list);
List<Point> list1 = new ArrayList<>(); List<Point> list1 = new ArrayList<>();
list.forEach(outlookLog -> { list.forEach(outlookLog -> {
@ -57,70 +58,70 @@ public class influxdb2Test {
System.out.println("list 大小:" + list1.size()); System.out.println("list 大小:" + list1.size());
Map<String, Object> aa = new HashMap<>(); Map<String, Object> aa = new HashMap<>();
aa.put("a1", 1); aa.put("a1", 1);
writeApi.writePoint("e5",org,list1.get(0)); writeApi.writePoint("e5", org, list1.get(0));
} }
influxDBClient.close(); influxDBClient.close();
} }
@Test @Test
public void saveLog(){ public void saveLog() {
influxDBClient.setLogLevel(LogLevel.BASIC); influxDBClient.setLogLevel(LogLevel.BASIC);
// for (int i = 0; i < 1000; i++) {
addLog(1002, 37,"error", 0, "检测到3次连续错误，下次将不再自动调用，请修正错误后再授权开启续订。"); addLog(19658189, 4959, "error", 1, "检测到3次连续错误，下次将不再自动调用，请修正错误后再授权开启续订。");
for (int i = 0; i < 10000; i++) { // }
influxDBClient.close();
}
} }
public void addLog(int githubId, int outlookId, String msg, int result, String original_msg) { public void addLog(int githubId, int outlookId, String msg, int result, String original_msg) {
try (WriteApi writeApi = influxDBClient.getWriteApi()) { try (WriteApi writeApi = influxDBClient.getWriteApi()) {
List<OutlookLog> list = new ArrayList<>(); // List<OutlookLog>
for (int i = 0; i < 10000; i++) { // for (int i = 0; i < 10000; i++) {
OutlookLog log = new OutlookLog(); OutlookLog log = new OutlookLog();
log.setCallTime(Instant.now()) log.setCallTime(System.currentTimeMillis())
.setGithubId(String.valueOf(githubId) ) .setGithubId(String.valueOf(githubId))
.setOutlookId(String.valueOf(outlookId)) .setOutlookId(String.valueOf(outlookId))
.setMsg(msg) .setMsg(msg)
.setOriginalMsg(original_msg) .setOriginalMsg(original_msg)
.setResultc(result) .setResult(result);
.setCallTime(Instant.now()); Thread.sleep(1);
list.add(log); writeApi.writeMeasurement("e5", org, WritePrecision.NS, log);
// }
}
writeApi.writeMeasurements("e5",org, WritePrecision.NS, list); } catch (InterruptedException e) {
influxDBClient.close(); e.printStackTrace();
} }
} }
public void addLog2(int githubId, int outlookId, String msg, int result, String original_msg) { public void addLog2(int githubId, int outlookId, String msg, int result, String original_msg) {
try (WriteApi writeApi = influxDBClient.getWriteApi()) { try (WriteApi writeApi = influxDBClient.getWriteApi()) {
Point point = Point.measurement("OutlookLog") Point point = Point.measurement("OutlookLog")
.addTag("githubId",String.valueOf(githubId)) .addTag("githubId", String.valueOf(githubId))
.addTag("outlookId",String.valueOf(outlookId)) .addTag("outlookId", String.valueOf(outlookId))
.addField("msg", msg) .addField("msg", msg)
.addField("resultc", result) .addField("resultc", result)
.addField("originalMsg", original_msg); .addField("originalMsg", original_msg);
writeApi.writePoint("e5",org,point); writeApi.writePoint("e5", org, point);
} }
} }
@Test @Test
public void find(){ public void find() {
String flux = "from(bucket:\"e5\") |> range(start: 0)" + String flux = "from(bucket:\"e5\") |> range(start: 0)" +
"|> filter(fn: (r) => r[\"_measurement\"] == \"OutlookLog\")" + "|> filter(fn: (r) => r[\"_measurement\"] == \"OutlookLog\")" +
"|> filter(fn: (r) => r[\"githubId\"] == \"1002\")" + "|> filter(fn: (r) => r[\"githubId\"] == \"1002\")" +
"|> filter(fn: (r) => r[\"outlookId\"] == \"37\")" + "|> filter(fn: (r) => r[\"outlookId\"] == \"37\")" +
"|> limit(n: 100)"; "|> limit(n: 100)";
QueryApi queryApi = influxDBClient.getQueryApi(); QueryApi queryApi = influxDBClient.getQueryApi();
List<FluxTable> tables = queryApi.query(flux,org); List<FluxTable> tables = queryApi.query(flux, org);
for (FluxTable fluxTable : tables) { for (FluxTable fluxTable : tables) {
List<FluxRecord> records = fluxTable .getRecords(); List<FluxRecord> records = fluxTable.getRecords();
for (FluxRecord fluxRecord : records) { for (FluxRecord fluxRecord : records) {
// System.out.println(fluxRecord.getField()); // System.out.println(fluxRecord.getField());
@ -132,14 +133,15 @@ public class influxdb2Test {
} }
@Test @Test
public void findPojo(){ public void findPojo() {
String flux = "from(bucket:\"e5\") |> range(start: 0)" + String flux = "from(bucket:\"e5\") |> range(start: 0)" +
"|> filter(fn: (r) => r[\"_measurement\"] == \"OutlookLog\")" + "|> filter(fn: (r) => r[\"_measurement\"] == \"OutlookLog\")" +
"|> filter(fn: (r) => r[\"githubId\"] == \"1002\")" + "|> filter(fn: (r) => r[\"githubId\"] == \"1002\")" +
"|> filter(fn: (r) => r[\"outlookId\"] == \"38\")" +
"|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")"; "|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")";
QueryApi queryApi = influxDBClient.getQueryApi(); QueryApi queryApi = influxDBClient.getQueryApi();
System.out.println(System.currentTimeMillis()); System.out.println(System.currentTimeMillis());
List<OutlookLog> tables = queryApi.query(flux,org,OutlookLog.class); List<OutlookLog> tables = queryApi.query(flux, org, OutlookLog.class);
System.out.println(System.currentTimeMillis()); System.out.println(System.currentTimeMillis());
for (OutlookLog table : tables) { for (OutlookLog table : tables) {
if (table.getMsg() == null) { if (table.getMsg() == null) {
@ -155,13 +157,14 @@ public class influxdb2Test {
public void findPojoAsync() throws InterruptedException { public void findPojoAsync() throws InterruptedException {
String flux = "from(bucket:\"e5\") |> range(start: 0)" + String flux = "from(bucket:\"e5\") |> range(start: 0)" +
"|> filter(fn: (r) => r[\"_measurement\"] == \"OutlookLog\")" + "|> filter(fn: (r) => r[\"_measurement\"] == \"OutlookLog\")" +
"|> filter(fn: (r) => r[\"githubId\"] == \"1002\")" + "|> filter(fn: (r) => r[\"githubId\"] == \"1003\")" +
"|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")" ; "|> filter(fn: (r) => r[\"outlookId\"] == \"39\")" +
"|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")";
QueryApi queryApi = influxDBClient.getQueryApi(); QueryApi queryApi = influxDBClient.getQueryApi();
queryApi.query(flux,org,OutlookLog.class,(cancellable, outlookLog) -> { queryApi.query(flux, org, OutlookLog.class, (cancellable, outlookLog) -> {
if (outlookLog.getMsg() != null) { if (outlookLog.getMsg() != null) {
System.out.println(outlookLog); System.out.println(outlookLog);
} }
}); });
System.out.println("查询完成"); System.out.println("查询完成");