diff --git a/data-process/pom.xml b/data-process/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..4c6ec0bf60a9c0236ddacc4221f82b5a8bfc8be4 --- /dev/null +++ b/data-process/pom.xml @@ -0,0 +1,40 @@ + + + 4.0.0 + + com.duanledexianxianxian.data.process + data-process + 1.0.0-SNAPSHOT + + + + + commons-io + commons-io + 2.6 + + + org.apache.commons + commons-lang3 + 3.10 + + + org.apache.commons + commons-collections4 + 4.4 + + + com.google.guava + guava + 23.0 + + + org.projectlombok + lombok + 1.18.12 + provided + + + \ No newline at end of file diff --git a/data-process/src/main/java/com/duanledexianxianxian/data/process/Application.java b/data-process/src/main/java/com/duanledexianxianxian/data/process/Application.java new file mode 100644 index 0000000000000000000000000000000000000000..81e51771eabf806ef7280203618a4c68dcbe121c --- /dev/null +++ b/data-process/src/main/java/com/duanledexianxianxian/data/process/Application.java @@ -0,0 +1,116 @@ +package com.duanledexianxianxian.data.process; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.sun.org.apache.regexp.internal.RE; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.io.IOException; +import java.util.*; + +/** + * @author duanledexianxianxian + */ +public class Application { + private static Set errorSet = Sets.newHashSet(); + + public static void main(String[] args) throws IOException { + List lines = readData(); + + // 中文数据 + String zh = processData(lines, 0); + writeData("F:\\@project@\\@dianli@\\java-demo\\data-process\\src\\main\\resources\\zh.sql", zh); + // 英文数据 + String en = processData(lines, 1); + writeData("F:\\@project@\\@dianli@\\java-demo\\data-process\\src\\main\\resources\\en.sql", en); + + System.out.println("error data"); + for (Model model:errorSet){ + System.out.println(model); + } + } + + private static void writeData(String fileName, String content) throws IOException { + FileUtils.write(new File(fileName), content, "UTF-8"); + } + + private static List readData() throws IOException { + File file = new File("F:\\@project@\\@dianli@\\java-demo\\data-process\\src\\main\\resources\\省市区码表.csv"); + List lines = FileUtils.readLines(file, "UTF-8"); + System.out.println("lines:" + lines); + return lines; + } + + + private static String processData(List lines, int lang) { + List modelList = Lists.newLinkedList(); + Map provinceMap = Maps.newTreeMap(); + Map allMap = Maps.newHashMap(); + int[] index = {lang == 0 ? 100000 : 200000}; + if (CollectionUtils.isNotEmpty(lines)) { + lines.forEach(x -> { + String[] words = StringUtils.split(x, "@"); + System.out.println("words:" + Arrays.toString(words)); + Model model; + model = new Model(); + model.setId((long) index[0]); + model.setGrade(words[0]); + model.setType(Integer.parseInt(words[0])); + model.setNameCn(words[1]); + model.setNameEn(words[2]); + model.setOrders(words[3]); + model.setRegCode(words[4]); + if (words.length <= 5) { + model.setPId(lang == 0 ? 1L : 242L); + provinceMap.put(model.getRegCode(), model); + } else { + model.setBelongTo(words[5]); + model.setPRegCode(words[6]); + } + allMap.put(model.getRegCode(), model); + modelList.add(model); + index[0]++; + }); + } + + System.out.println("modelList:" + modelList); + System.out.println("provinceMap:" + provinceMap); + System.out.println("allMap:" + allMap); + + modelList.forEach(x -> { + if (allMap.containsKey(x.getPRegCode())) { + x.setPId(allMap.get(x.getPRegCode()).getId()); + } + }); + + // 排个序 + System.out.println("modelList:" + modelList); + Collections.sort(modelList, (o1, o2) -> { + Integer result = o1.getType() - o2.getType(); + if (result.equals(0)) { + return o1.getRegCode().compareTo(o2.getRegCode()); + } + return result; + }); + System.out.println("sort modelList" + modelList); + + StringBuffer sb = new StringBuffer(); + modelList.forEach(x -> { + if (null != x.getPId()) { + sb.append(String.format("INSERT INTO `tsys_business_parameter_geography`(`id`, `pId`, `local`, `district_code`, `district_name`, `type`, `is_delete`, `creator_id`, `create_time`, `editor_id`, `edit_time`) VALUES " + + "(\"%d\", \"%d\", \"%s\", \"%s\", \"%s\", \"%d\", 0, 1, now(), NULL, NULL);\n", + x.getId(), x.getPId(), lang == 0 ? "zh-cn" : "en-us", x.getRegCode(), lang == 0 ? x.getNameCn() : x.getNameEn(), x.getType())); + + } else { + errorSet.add(x); + } + }); + System.out.println(sb); + return sb.toString(); + } + +} diff --git a/data-process/src/main/java/com/duanledexianxianxian/data/process/Model.java b/data-process/src/main/java/com/duanledexianxianxian/data/process/Model.java new file mode 100644 index 0000000000000000000000000000000000000000..16621221f9c5e4f7813cd53908d0303a5bb7aaa8 --- /dev/null +++ b/data-process/src/main/java/com/duanledexianxianxian/data/process/Model.java @@ -0,0 +1,52 @@ +package com.duanledexianxianxian.data.process; + +import com.google.common.base.Objects; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.io.Serializable; + +/** + * The type Model. + * + * @author duanledexianxianxian + */ +@Data +public class Model implements Serializable { + private Long id; + private Long pId; + private String grade; + private String nameCn; + private String nameEn; + private String orders; + private String regCode; + private String belongTo; + private String pRegCode; + /** + * 0-国家 + * 1-省 + * 2-地级市/区 + * 3-县/区 + */ + private Integer type; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Model model = (Model) o; + return Objects.equal(grade, model.grade) && + Objects.equal(nameCn, model.nameCn) && + Objects.equal(nameEn, model.nameEn) && + Objects.equal(orders, model.orders) && + Objects.equal(regCode, model.regCode) && + Objects.equal(belongTo, model.belongTo) && + Objects.equal(pRegCode, model.pRegCode) && + Objects.equal(type, model.type); + } + + @Override + public int hashCode() { + return Objects.hashCode(grade, nameCn, nameEn, orders, regCode, belongTo, pRegCode, type); + } +}