Test_Project_Web/e5
1
0
Fork 0
mirror of https://github.com/luoye663/e5.git synced 2024-12-25 03:08:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

1、取消启动清空redis

Browse Source 
2、将调用日志放到influx，减少mysql压力
This commit is contained in:
LuoYe 2021-08-19 16:10:11 +08:00
parent 1006883e95
commit 5b318e2357
9 changed files with 90 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -10,7 +10,7 @@
</parent>
<groupId>io.qyi</groupId>
<artifactId>e5</artifactId>
<version>1.0.7</version>
<version>1.0.8</version>
<name>e5</name>
<description>Demo project for Spring Boot</description>

2
src/main/java/io/qyi/e5/bean/influx2/InfluxdbConfig.java
View File

@ -17,7 +17,7 @@ public class InfluxdbConfig {
@Bean
public InfluxDBClient influxDBClient() {
InfluxDBClient influxDBClient = InfluxDBClientFactory.create(influxDBUrl, token.toCharArray());
influxDBClient.setLogLevel(LogLevel.BASIC);
influxDBClient.setLogLevel(LogLevel.NONE);
return influxDBClient;
}

14
src/main/java/io/qyi/e5/config/Start.java
View File

@ -38,13 +38,17 @@ public class Start {
@Value("${e5.system.threadPool}")
Integer poolSize;
@Value("${isdebug:true}")
private boolean isdebug;
private ExecutorService threadPool;
@PostConstruct
public void init() {
log.info("清空redis...... ");
redisUtil.delAll();
// log.info("清空redis...... ");
// redisUtil.delAll();
threadPool = new ThreadPoolExecutor(
//指定了线程池中的线程数量，它的数量决定了添加的任务是开辟新的线程去执行，还是放到workQueue任务队列中去；
poolSize,
@ -64,8 +68,12 @@ public class Start {
}
// @Scheduled(cron = "0 0/1 * * * ? ")
@Scheduled(cron = "0 0/1 * * * ? ")
private void distributeTask() {
if (isdebug) {
log.debug("Debug模式，跳过执行");
return;
}
List<Outlook> runOutlookList = outlookService.findRunOutlookList();
CountDownLatch cdl = new CountDownLatch(runOutlookList.size());

4
src/main/java/io/qyi/e5/controller/web/WebController.java
View File

@ -59,7 +59,7 @@ public class WebController {
public Result delete(){
UsernamePasswordAuthenticationToken authentication = (UsernamePasswordAuthenticationToken) SecurityContextHolder.getContext().getAuthentication();
/*删除数据库信息*/
int outlooklog = iOutlookLogService.deleteInfo(authentication.getGithub_id());
// int outlooklog = iOutlookLogService.deleteInfo(authentication.getGithub_id());
int outlook = outlookService.deleteInfo(authentication.getGithub_id());
int github = GithubService.deleteInfo(authentication.getGithub_id());
/*删除redis中信息*/
@ -69,7 +69,7 @@ public class WebController {
}
/*返回结果信息*/
Map<String, Integer> map = new HashMap<>();
map.put("outlooklog", outlooklog);
// map.put("outlooklog", outlooklog);
map.put("outlook", outlook);
map.put("github", github);
return ResultUtil.success(map);

26
src/main/java/io/qyi/e5/outlook_log/controller/OutlookLogController.java
View File

@ -1,15 +1,7 @@
package io.qyi.e5.outlook_log.controller;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.influxdb.client.InfluxDBClient;
import com.influxdb.client.QueryApi;
import com.influxdb.client.WriteApi;
import com.influxdb.client.WriteOptions;
import com.influxdb.client.domain.WritePrecision;
import com.influxdb.client.write.Point;
import com.influxdb.query.FluxRecord;
import com.influxdb.query.FluxTable;
import io.qyi.e5.bean.result.Result;
import io.qyi.e5.config.security.UsernamePasswordAuthenticationToken;
import io.qyi.e5.outlook.service.IOutlookService;
@ -17,8 +9,6 @@ import io.qyi.e5.outlook_log.bena.LogVo;
import io.qyi.e5.outlook_log.entity.OutlookLog;
import io.qyi.e5.outlook_log.service.IOutlookLogService;
import io.qyi.e5.util.ResultUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@ -29,8 +19,6 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@ -73,20 +61,8 @@ public class OutlookLogController {
public Result findLog(@RequestParam int outlookId){
UsernamePasswordAuthenticationToken authentication = (UsernamePasswordAuthenticationToken) SecurityContextHolder.getContext().getAuthentication();
int github_id = authentication.getGithub_id();
QueryWrapper<OutlookLog> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("github_id", github_id).eq("outlook_id", outlookId).orderByAsc("call_time");
List<OutlookLog> list = outlookLogService.findAllList(github_id, outlookId);
Iterator<OutlookLog> iterator = list.iterator();
List<LogVo> logVo = new LinkedList<>();
while (iterator.hasNext()) {
OutlookLog next = iterator.next();
LogVo vo = new LogVo();
BeanUtils.copyProperties(next,vo);
logVo.add(vo);
}
return ResultUtil.success(logVo);
return ResultUtil.success(list);
}
}

6
src/main/java/io/qyi/e5/outlook_log/entity/OutlookLog.java
View File

@ -43,14 +43,14 @@ public class OutlookLog {
/**
* 调用时间
*/
@Column(timestamp = true)
private Instant callTime;
@Column
private long callTime;
/**
* 调用结果
*/
@Column
private Number resultc;
private Number result;
/**
* 如果有错误原因则记录

2
src/main/java/io/qyi/e5/outlook_log/service/IOutlookLogService.java
View File

@ -15,6 +15,6 @@ import java.util.List;
*/
public interface IOutlookLogService {
void addLog(int githubId,int outlookId, String msg,int result,String original_msg);
int deleteInfo(int github_id);
// int deleteInfo(int github_id);
List<OutlookLog> findAllList(int githubId, int outlookId);
}

48
src/main/java/io/qyi/e5/outlook_log/service/impl/OutlookLogServiceImpl.java
View File

@ -1,12 +1,15 @@
package io.qyi.e5.outlook_log.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.influxdb.client.InfluxDBClient;
import com.influxdb.client.InfluxDBClientFactory;
import com.influxdb.client.QueryApi;
import com.influxdb.client.WriteApi;
import com.influxdb.client.domain.WritePrecision;
import io.qyi.e5.outlook_log.entity.OutlookLog;
import io.qyi.e5.outlook_log.service.IOutlookLogService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.List;
/**
@ -20,33 +23,40 @@ import java.util.List;
@Service
public class OutlookLogServiceImpl implements IOutlookLogService {
@Resource
private InfluxDBClient influxDBClient;
@Value("${spring.influx.org}")
private String org;
@Value("${spring.influx.bucket}")
private String bucket;
@Override
public void addLog(int githubId, int outlookId, String msg, int result, String original_msg) {
// OutlookLog outlookLog = new OutlookLog();
// outlookLog.setGithubId(githubId)
// .setOutlookId(outlookId)
// .setResult(result)
// .setMsg(msg)
// .setOriginalMsg(original_msg);
// baseMapper.insert(outlookLog);
try (WriteApi writeApi = influxDBClient.getWriteApi()) {
OutlookLog log = new OutlookLog();
log.setCallTime(System.currentTimeMillis())
.setGithubId(String.valueOf(githubId))
.setOutlookId(String.valueOf(outlookId))
.setMsg(msg)
.setOriginalMsg(original_msg)
.setResult(result);
writeApi.writeMeasurement(bucket, org, WritePrecision.NS, log);
}
@Override
public int deleteInfo(int github_id) {
QueryWrapper<OutlookLog> outlookLogQueryWrapper = new QueryWrapper<>();
outlookLogQueryWrapper.eq("github_id", github_id);
// return baseMapper.delete(outlookLogQueryWrapper);
return 0;
}
@Override
public List<OutlookLog> findAllList(int githubId, int outlookId) {
// return baseMapper.findAllList(githubId, outlookId);
return null;
String flux = "from(bucket:\"" + bucket + "\") |> range(start: 0)" +
"|> filter(fn: (r) => r[\"_measurement\"] == \"OutlookLog\")" +
"|> filter(fn: (r) => r[\"githubId\"] == \"" + githubId + "\")" +
"|> filter(fn: (r) => r[\"outlookId\"] == \"" + outlookId + "\")" +
"|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")";
QueryApi queryApi = influxDBClient.getQueryApi();
List<OutlookLog> tables = queryApi.query(flux, org, OutlookLog.class);
return tables;
}
}

71
src/test/java/influxdb2Test.java
View File

@ -8,11 +8,12 @@ import io.qyi.e5.outlook_log.entity.OutlookLog;
import org.junit.jupiter.api.Test;
import org.springframework.cglib.beans.BeanMap;
import java.time.Instant;
import java.util.*;
public class influxdb2Test {
InfluxDBClient influxDBClient = InfluxDBClientFactory.create("http://127.0.0.1:8086", "ko6GtE_P5R2AlMkCBkEgBwW7rVBl46GYx0IoCrG-Dd5VFxTDSnFJ--BB2f8FRFcGd6Tb_yu6-MlMAD-lMSbH6A==".toCharArray()
String token = "HquxNXwOyfgW-f8wzkSUuBz0tswWiFPsTbEnr5jKFS3BY3RcKezDdQF0o5yeoNfaiwQUJhy8YJSIUrVrWSQn8Q==";
InfluxDBClient influxDBClient = InfluxDBClientFactory.create("http://127.0.0.1:8086", token.toCharArray()
);
private String org = "luoye";
@ -22,7 +23,7 @@ public class influxdb2Test {
.build();
@Test
public void save(){
public void save() {
influxDBClient.setLogLevel(LogLevel.BASIC);
WriteOptions writeOptions = WriteOptions.builder()
@ -36,13 +37,13 @@ public class influxdb2Test {
List<OutlookLog> list = new ArrayList<>();
for (int i = 0; i < 100; i++) {
OutlookLog outlookLog = new OutlookLog();
outlookLog.setMsg(i + "- ok").setOriginalMsg("加入成功").setCallTime(Instant.now());
outlookLog.setMsg(i + "- ok").setOriginalMsg("加入成功").setCallTime(System.currentTimeMillis());
list.add(outlookLog);
}
try (WriteApi writeApi = influxDBClient.getWriteApi()) {
// writeApi.writeMeasurement("e5", org ,WritePrecision.NS,outlookLog);
writeApi.writeMeasurements("e5", org ,WritePrecision.NS,list);
writeApi.writeMeasurements("e5", org, WritePrecision.NS, list);
List<Point> list1 = new ArrayList<>();
list.forEach(outlookLog -> {
@ -57,70 +58,70 @@ public class influxdb2Test {
System.out.println("list 大小:" + list1.size());
Map<String, Object> aa = new HashMap<>();
aa.put("a1", 1);
writeApi.writePoint("e5",org,list1.get(0));
writeApi.writePoint("e5", org, list1.get(0));
}
influxDBClient.close();
}
@Test
public void saveLog(){
public void saveLog() {
influxDBClient.setLogLevel(LogLevel.BASIC);
addLog(1002, 37,"error", 0, "检测到3次连续错误，下次将不再自动调用，请修正错误后再授权开启续订。");
for (int i = 0; i < 10000; i++) {
}
// for (int i = 0; i < 1000; i++) {
addLog(19658189, 4959, "error", 1, "检测到3次连续错误，下次将不再自动调用，请修正错误后再授权开启续订。");
// }
influxDBClient.close();
}
public void addLog(int githubId, int outlookId, String msg, int result, String original_msg) {
try (WriteApi writeApi = influxDBClient.getWriteApi()) {
List<OutlookLog> list = new ArrayList<>();
for (int i = 0; i < 10000; i++) {
// List<OutlookLog>
// for (int i = 0; i < 10000; i++) {
OutlookLog log = new OutlookLog();
log.setCallTime(Instant.now())
.setGithubId(String.valueOf(githubId) )
log.setCallTime(System.currentTimeMillis())
.setGithubId(String.valueOf(githubId))
.setOutlookId(String.valueOf(outlookId))
.setMsg(msg)
.setOriginalMsg(original_msg)
.setResultc(result)
.setCallTime(Instant.now());
list.add(log);
.setResult(result);
Thread.sleep(1);
writeApi.writeMeasurement("e5", org, WritePrecision.NS, log);
// }
}
writeApi.writeMeasurements("e5",org, WritePrecision.NS, list);
influxDBClient.close();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public void addLog2(int githubId, int outlookId, String msg, int result, String original_msg) {
try (WriteApi writeApi = influxDBClient.getWriteApi()) {
Point point = Point.measurement("OutlookLog")
.addTag("githubId",String.valueOf(githubId))
.addTag("outlookId",String.valueOf(outlookId))
.addTag("githubId", String.valueOf(githubId))
.addTag("outlookId", String.valueOf(outlookId))
.addField("msg", msg)
.addField("resultc", result)
.addField("originalMsg", original_msg);
writeApi.writePoint("e5",org,point);
writeApi.writePoint("e5", org, point);
}
}
@Test
public void find(){
public void find() {
String flux = "from(bucket:\"e5\") |> range(start: 0)" +
"|> filter(fn: (r) => r[\"_measurement\"] == \"OutlookLog\")" +
"|> filter(fn: (r) => r[\"githubId\"] == \"1002\")" +
"|> filter(fn: (r) => r[\"outlookId\"] == \"37\")" +
"|> limit(n: 100)";
QueryApi queryApi = influxDBClient.getQueryApi();
List<FluxTable> tables = queryApi.query(flux,org);
List<FluxTable> tables = queryApi.query(flux, org);
for (FluxTable fluxTable : tables) {
List<FluxRecord> records = fluxTable .getRecords();
List<FluxRecord> records = fluxTable.getRecords();
for (FluxRecord fluxRecord : records) {
// System.out.println(fluxRecord.getField());
@ -132,14 +133,15 @@ public class influxdb2Test {
}
@Test
public void findPojo(){
public void findPojo() {
String flux = "from(bucket:\"e5\") |> range(start: 0)" +
"|> filter(fn: (r) => r[\"_measurement\"] == \"OutlookLog\")" +
"|> filter(fn: (r) => r[\"githubId\"] == \"1002\")" +
"|> filter(fn: (r) => r[\"outlookId\"] == \"38\")" +
"|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")";
QueryApi queryApi = influxDBClient.getQueryApi();
System.out.println(System.currentTimeMillis());
List<OutlookLog> tables = queryApi.query(flux,org,OutlookLog.class);
List<OutlookLog> tables = queryApi.query(flux, org, OutlookLog.class);
System.out.println(System.currentTimeMillis());
for (OutlookLog table : tables) {
if (table.getMsg() == null) {
@ -155,10 +157,11 @@ public class influxdb2Test {
public void findPojoAsync() throws InterruptedException {
String flux = "from(bucket:\"e5\") |> range(start: 0)" +
"|> filter(fn: (r) => r[\"_measurement\"] == \"OutlookLog\")" +
"|> filter(fn: (r) => r[\"githubId\"] == \"1002\")" +
"|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")" ;
"|> filter(fn: (r) => r[\"githubId\"] == \"1003\")" +
"|> filter(fn: (r) => r[\"outlookId\"] == \"39\")" +
"|> pivot(rowKey:[\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\")";
QueryApi queryApi = influxDBClient.getQueryApi();
queryApi.query(flux,org,OutlookLog.class,(cancellable, outlookLog) -> {
queryApi.query(flux, org, OutlookLog.class, (cancellable, outlookLog) -> {
if (outlookLog.getMsg() != null) {
System.out.println(outlookLog);
}