From 01d946c5adabb3f8b0e8dd6e35fe4124c3f35107 Mon Sep 17 00:00:00 2001 From: duanledexianxianxian Date: Mon, 3 Aug 2020 13:15:55 +0800 Subject: [PATCH] base project --- demo-parent.iml | 32 ------ kafka/README.md | 23 ---- kafka/kafka.iml | 108 ------------------ kafka/pom.xml | 41 ------- .../demo/kafka/Application.java | 21 ---- .../demo/kafka/consumer/TestConsumer.java | 26 ----- .../kafka/controller/KafkaController.java | 24 ---- .../demo/kafka/listener/AppListener.java | 66 ----------- .../demo/kafka/model/Student.java | 35 ------ kafka/src/main/resources/application.yml | 38 ------ menu/pom.xml | 16 --- .../demo/Application.java | 94 --------------- .../com/duanledexianxianxian/demo/Menu.java | 18 --- .../com/duanledexianxianxian/demo/MenuVO.java | 21 ---- pom.xml | 4 - 15 files changed, 567 deletions(-) delete mode 100644 kafka/README.md delete mode 100644 kafka/kafka.iml delete mode 100644 kafka/pom.xml delete mode 100644 kafka/src/main/java/com/duanledexianxianxian/demo/kafka/Application.java delete mode 100644 kafka/src/main/java/com/duanledexianxianxian/demo/kafka/consumer/TestConsumer.java delete mode 100644 kafka/src/main/java/com/duanledexianxianxian/demo/kafka/controller/KafkaController.java delete mode 100644 kafka/src/main/java/com/duanledexianxianxian/demo/kafka/listener/AppListener.java delete mode 100644 kafka/src/main/java/com/duanledexianxianxian/demo/kafka/model/Student.java delete mode 100644 kafka/src/main/resources/application.yml delete mode 100644 menu/pom.xml delete mode 100644 menu/src/main/java/com/duanledexianxianxian/demo/Application.java delete mode 100644 menu/src/main/java/com/duanledexianxianxian/demo/Menu.java delete mode 100644 menu/src/main/java/com/duanledexianxianxian/demo/MenuVO.java diff --git a/demo-parent.iml b/demo-parent.iml index 3c81f99..28baa0d 100644 --- a/demo-parent.iml +++ b/demo-parent.iml @@ -13,38 +13,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/kafka/README.md b/kafka/README.md deleted file mode 100644 index f5a3232..0000000 --- a/kafka/README.md +++ /dev/null @@ -1,23 +0,0 @@ -##### kafka配置 - -``` -bootstrap.servers:kafka server的地址 - -acks:写入kafka时，leader负责一个该partion读写，当写入partition时，需要将记录同步到repli节点，all是全部同步节点都返回成功，leader才返回ack。 - -retris:写入失败时，重试次数。当leader节点失效，一个repli节点会替代成为leader节点，此时可能出现写入失败，当retris为0时，produce不会重复。retirs重发，此时repli节点完全成为leader节点，不会产生消息丢失。 - -batch.size:produce积累到一定数据，一次发送。 - -buffer.memory: produce积累数据一次发送，缓存大小达到buffer.memory就发送数据。 - -linger.ms :当设置了缓冲区，消息就不会即时发送，如果消息总不够条数、或者消息不够buffer大小就不发送了吗？当消息超过linger时间，也会发送。 - -key/value serializer：序列化类。 -``` - -### 关于kafka序列化/反序列化 - -https://www.jianshu.com/p/5da86afed228 - -自定义Serializer和Deserializer非常痛苦，还有很多类型不支持，非常脆弱。复杂类型的支持更是一件痛苦的事情，不同版本之间的兼容性问题更是一个极大的挑战。由于Serializer和Deserializer影响到上下游系统，导致牵一发而动全身。自定义序列化&反序列化实现不是能力的体现，而是逗比的体现。所以强烈不建议自定义实现序列化&反序列化，推荐直接使用StringSerializer和StringDeserializer，然后使用json作为标准的数据传输格式。 \ No newline at end of file diff --git a/kafka/kafka.iml b/kafka/kafka.iml deleted file mode 100644 index 7fa819a..0000000 --- a/kafka/kafka.iml +++ /dev/null @@ -1,108 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/kafka/pom.xml b/kafka/pom.xml deleted file mode 100644 index 6210217..0000000 --- a/kafka/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - demo-parent - com.duanledexianxianxian.demo - 1.0-SNAPSHOT - - 4.0.0 - - kafka - - - - - org.springframework.kafka - spring-kafka - - - com.alibaba - fastjson - 1.2.56 - - - org.projectlombok - lombok - provided - - - org.springframework.kafka - spring-kafka-test - test - - - org.apache.commons - commons-lang3 - 3.9 - - - diff --git a/kafka/src/main/java/com/duanledexianxianxian/demo/kafka/Application.java b/kafka/src/main/java/com/duanledexianxianxian/demo/kafka/Application.java deleted file mode 100644 index fd0a87d..0000000 --- a/kafka/src/main/java/com/duanledexianxianxian/demo/kafka/Application.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.duanledexianxianxian.demo.kafka; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; - -/** - * @author duanledexianxianxian - * @date 2019/10/18 23:32 - * @since 1.0.0 - */ -@SpringBootApplication -@Slf4j -public class Application { - - - public static void main(String[] args) { - SpringApplication.run(Application.class, args); - - } -} diff --git a/kafka/src/main/java/com/duanledexianxianxian/demo/kafka/consumer/TestConsumer.java b/kafka/src/main/java/com/duanledexianxianxian/demo/kafka/consumer/TestConsumer.java deleted file mode 100644 index 0459282..0000000 --- a/kafka/src/main/java/com/duanledexianxianxian/demo/kafka/consumer/TestConsumer.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.duanledexianxianxian.demo.kafka.consumer; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.springframework.kafka.annotation.KafkaListener; -import org.springframework.stereotype.Component; - -/** - * kafka消费者测试 - * - * @author fengyuchenglun - * @version 1.0.0 - */ -@Component -public class TestConsumer { - - /** - * Listen. - * - * @param record the record - * @throws Exception the exception - */ - @KafkaListener(topics = "topic_second") - public void listen(ConsumerRecord record) throws Exception { - System.out.printf("topic = %s, offset = %d, value = %s \n", record.topic(), record.offset(), record.value()); - } -} diff --git a/kafka/src/main/java/com/duanledexianxianxian/demo/kafka/controller/KafkaController.java b/kafka/src/main/java/com/duanledexianxianxian/demo/kafka/controller/KafkaController.java deleted file mode 100644 index c6878ba..0000000 --- a/kafka/src/main/java/com/duanledexianxianxian/demo/kafka/controller/KafkaController.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.duanledexianxianxian.demo.kafka.controller; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -/** - * @author duanledexianxianxian - * @date 2019/10/18 23:50 - * @since 1.0.0 - */ -@RestController -public class KafkaController { - @Autowired - private KafkaTemplate kafkaTemplate; - - @GetMapping("/message/send") - public boolean send(@RequestParam String message){ - kafkaTemplate.send("testTopic",message); - return true; - } -} diff --git a/kafka/src/main/java/com/duanledexianxianxian/demo/kafka/listener/AppListener.java b/kafka/src/main/java/com/duanledexianxianxian/demo/kafka/listener/AppListener.java deleted file mode 100644 index fee2774..0000000 --- a/kafka/src/main/java/com/duanledexianxianxian/demo/kafka/listener/AppListener.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.duanledexianxianxian.demo.kafka.listener; - -import com.alibaba.fastjson.JSONObject; -import com.duanledexianxianxian.demo.kafka.model.Student; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.RandomUtils; -import org.apache.commons.lang3.concurrent.BasicThreadFactory; -import org.apache.tomcat.util.threads.ThreadPoolExecutor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.stereotype.Component; - -import java.time.LocalDateTime; -import java.util.Random; -import java.util.concurrent.*; - -/** - * @author duanledexianxianxian - * @date 2019/10/19 0:44 - * @since 1.0.0 - */ -@Component -@Slf4j -public class AppListener implements ApplicationListener { - @Autowired - private KafkaTemplate kafkaTemplate; - - - @Override - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - - ThreadFactory namedThreadFactory = new BasicThreadFactory.Builder().namingPattern("producer-kafka-pool-%d").daemon(true).build(); - - ExecutorService pool = new ThreadPoolExecutor(5, 200, - 0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue(1024), namedThreadFactory, new ThreadPoolExecutor.AbortPolicy()); - int i = 0; - while (true) { - int finalI = i; - pool.execute(() -> { - this.sendData(finalI); - // sleep 5 seconds - }); - - try { - Thread.sleep(new Random().nextInt(6) * 1000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - i++; - } - } - - private void sendData(int i) { - Student student = new Student(); - student.setStudentId(String.valueOf(i)); - student.setName("duanledexianxianxian" + new Random().nextInt(20)); - student.setAge(new Random().nextInt(19) + 6); - student.setSex(Byte.valueOf(String.valueOf((new Random().nextInt(2))))); - log.info("Thread:{} Send Data:{}", Thread.currentThread().getName(), student); - kafkaTemplate.send("topic_first", JSONObject.toJSONString(student)); - } -} diff --git a/kafka/src/main/java/com/duanledexianxianxian/demo/kafka/model/Student.java b/kafka/src/main/java/com/duanledexianxianxian/demo/kafka/model/Student.java deleted file mode 100644 index da1a583..0000000 --- a/kafka/src/main/java/com/duanledexianxianxian/demo/kafka/model/Student.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.duanledexianxianxian.demo.kafka.model; - -import lombok.Data; - -import java.io.Serializable; - -/** - * 学生实体类 - * - * @author duanledexianxianxian - * @date 2019/10/19 0:32 - * @since 1.0.0 - */ -@Data -public class Student implements Serializable { - private static final long serialVersionUID = -4572819276161640638L; - /** - * 学生学号 - */ - private String studentId; - /** - * 学生姓名 - */ - private String name; - /** - * 学生年龄 - */ - private Integer age; - /** - * 学生性别 - * 0-男 - * 1-女 - */ - private Byte sex; -} diff --git a/kafka/src/main/resources/application.yml b/kafka/src/main/resources/application.yml deleted file mode 100644 index 9ad2db8..0000000 --- a/kafka/src/main/resources/application.yml +++ /dev/null @@ -1,38 +0,0 @@ -spring: - kafka: - # 指定kafka server的地址，集群配多个，中间，逗号隔开 - bootstrap-servers: 113.105.144.9:9104 - #=============== provider ======================= - producer: - # 写入失败时，重试次数。当leader节点失效，一个repli节点会替代成为leader节点，此时可能出现写入失败， - # 当retris为0时，produce不会重复。retirs重发，此时repli节点完全成为leader节点，不会产生消息丢失。 - retries: 0 - # 每次批量发送消息的数量,produce积累到一定数据，一次发送 - batch-size: 16384 - # produce积累数据一次发送，缓存大小达到buffer.memory就发送数据 - buffer-memory: 33554432 - # 指定消息key和消息体的编解码方式 - key-serializer: org.apache.kafka.common.serialization.StringSerializer - value-serializer: org.apache.kafka.common.serialization.StringSerializer - properties: - #当设置了缓冲区，消息就不会即时发送，如果消息总不够条数、或者消息不够buffer大小就不发送了吗？当消息超过linger时间，也会发送。 - linger.ms: 1 - #procedure要求leader在考虑完成请求之前收到的确认数，用于控制发送记录在服务端的持久化，其值可以为如下： - #acks = 0 如果设置为零，则生产者将不会等待来自服务器的任何确认，该记录将立即添加到套接字缓冲区并视为已发送。在这种情况下，无法保证服务器已收到记录，并且重试配置将不会生效（因为客户端通常不会知道任何故障），为每条记录返回的偏移量始终设置为-1。 - #acks = 1 这意味着leader会将记录写入其本地日志，但无需等待所有副本服务器的完全确认即可做出回应，在这种情况下，如果leader在确认记录后立即失败，但在将数据复制到所有的副本服务器之前，则记录将会丢失。 - #acks = all 这意味着leader将等待完整的同步副本集以确认记录，这保证了只要至少一个同步副本服务器仍然存活，记录就不会丢失，这是最强有力的保证，这相当于acks = -1的设置。 - #可以设置的值为：all, -1, 0, 1 - acks: 1 - - #=============== consumer ======================= - consumer: - group-id: test - # enable.auto.commit:true --> 设置自动提交offset - enable-auto-commit: true - # auto-commit'为true，则消费者偏移自动提交给Kafka的频率（以毫秒为单位），默认值为5000。: - auto-commit-interval: 100ms - # 指定消息key和消息体的编解码方式 - key-deserializer: org.apache.kafka.common.serialization.StringDeserializer - value-deserializer: org.apache.kafka.common.serialization.StringDeserializer - properties: - session.timeout.ms: 15000 \ No newline at end of file diff --git a/menu/pom.xml b/menu/pom.xml deleted file mode 100644 index 6aba2a9..0000000 --- a/menu/pom.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - demo-parent - com.duanledexianxianxian.demo - 1.0-SNAPSHOT - - 4.0.0 - - menu - - - - \ No newline at end of file diff --git a/menu/src/main/java/com/duanledexianxianxian/demo/Application.java b/menu/src/main/java/com/duanledexianxianxian/demo/Application.java deleted file mode 100644 index 47325f1..0000000 --- a/menu/src/main/java/com/duanledexianxianxian/demo/Application.java +++ /dev/null @@ -1,94 +0,0 @@ -package com.duanledexianxianxian.demo; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; -import org.springframework.beans.BeanUtils; -import org.springframework.util.CollectionUtils; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * @author duanledexianxianxian - * @date 2020/1/12 23:51 - * @since 1.0.0 - */ -public class Application { - private static String[][] menus = { - {null, "0001", "用户管理", "/urlMgr"}, - {"0001", "00010001", "用户管理-添加用户", "/urlMgr/add"}, - {"0001", "00010002", "用户管理-修改用户", "/urlMgr/edit"}, - {"0001", "00010003", "用户管理-删除用户", "/urlMgr/delete"}, - {"0001", "00010004", "用户管理-查询用户", "/urlMgr/query"}, - {null, "0002", "系统管理", "/systemMgr"}, - {"0002", "00020001", "系统管理-系统管理0001", "/systemMgr/0001"}, - {"00020001", "000200010001", "系统管理-系统管理0001-系统管理0001", "/systemMgr/0001/0001"}, - {"000200010001", "0002000100010001", "系统管理-系统管理0001-系统管理0001-系统管理0001", "/systemMgr/0001/0001/0001"}, - }; - - public static void main(String[] args) { - List menuList = Lists.newArrayList(); - Arrays.asList(menus).forEach(x -> { - Menu menu = new Menu(); - menu.setParentMenuCode(x[0]); - menu.setMenuCode(x[1]); - menu.setMenuTitle(x[2]); - menu.setMenuUrl(x[3]); - menuList.add(menu); - }); - - menuList.forEach(x -> System.out.println(x.toString())); - MenuVO menuVO = getTreeMenuVO(menuList); - ObjectMapper mapper = new ObjectMapper(); - try { - System.out.println(mapper.writeValueAsString(menuVO)); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } - } - - /** - * 平级列表转成树结构 - * @param menuList - * @return - */ - private static MenuVO getTreeMenuVO(List menuList) { - if (CollectionUtils.isEmpty(menuList)) { - return null; - } - List menuVOList = Lists.newArrayList(); - // 类型转换 一层循环 - menuList.forEach(x -> { - MenuVO menuVO = new MenuVO(); - BeanUtils.copyProperties(x, menuVO); - menuVOList.add(menuVO); - }); - // 转成map 二层循环 一层与二层可以合并 - Map menuMap = menuVOList.stream().collect(Collectors.toMap(MenuVO::getMenuCode, x -> x)); - // 构建根节点 - MenuVO menuVO = new MenuVO(); - menuVO.setParentMenuCode(null); - menuVO.setMenuCode("root"); - menuVO.setMenuTitle("根节点"); - - // 转换逻辑 三层循环 - for (MenuVO menu : menuVOList) { - if (menuMap.containsKey(menu.getParentMenuCode())) { - if (menuMap.get(menu.getParentMenuCode()).getChildren() == null) { - menuMap.get(menu.getParentMenuCode()).setChildren(Lists.newArrayList()); - } - menuMap.get(menu.getParentMenuCode()).getChildren().add(menu); - - } else { - if (menuVO.getChildren() == null) { - menuVO.setChildren(Lists.newArrayList()); - } - menuVO.getChildren().add(menu); - } - } - return menuVO; - } -} diff --git a/menu/src/main/java/com/duanledexianxianxian/demo/Menu.java b/menu/src/main/java/com/duanledexianxianxian/demo/Menu.java deleted file mode 100644 index b5dac88..0000000 --- a/menu/src/main/java/com/duanledexianxianxian/demo/Menu.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.duanledexianxianxian.demo; - -import lombok.Data; - -/** - * 菜单 - * - * @author duanledexianxianxian - * @date 2020/1/12 23:49 - * @since 1.0.0 - */ -@Data -public class Menu { - private String parentMenuCode; - private String menuCode; - private String menuTitle; - private String menuUrl; -} diff --git a/menu/src/main/java/com/duanledexianxianxian/demo/MenuVO.java b/menu/src/main/java/com/duanledexianxianxian/demo/MenuVO.java deleted file mode 100644 index ece311a..0000000 --- a/menu/src/main/java/com/duanledexianxianxian/demo/MenuVO.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.duanledexianxianxian.demo; - -import lombok.Data; - -import java.util.List; - -/** - * 菜单 - * - * @author duanledexianxianxian - * @date 2020/1/12 23:49 - * @since 1.0.0 - */ -@Data -public class MenuVO { - private String parentMenuCode; - private String menuCode; - private String menuTitle; - private String menuUrl; - private List children; -} diff --git a/pom.xml b/pom.xml index 06796d7..2c2bd81 100644 --- a/pom.xml +++ b/pom.xml @@ -8,10 +8,6 @@ demo-parent pom 1.0-SNAPSHOT - - kafka - menu - org.springframework.boot -- GitLab