diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml new file mode 100644 index 0000000..91106d3 --- /dev/null +++ b/.github/workflows/maven.yml @@ -0,0 +1,17 @@ +name: Java CI + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build with Maven + run: mvn -B package --file pom.xml diff --git a/.gitignore b/.gitignore index 3cd2d05..409684a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,41 +1,44 @@ -/target/ -/classes/ -/log/ -/logs/ -.classpath -.project -.settings -.myeclipse - ##filter databfilesln file## -*.mdb -*.ldb -*.sln -##class file## -*.com -*.class -*.dll -*.exe -*.o -*.so -# compression file -*.7z -*.dmg -*.gz -*.iso -*.jar -*.rar -*.tar -*.zip -*.via -*.tmp -*.err -*.log -# OS generated files # -.DS_Store -.DS_Store? -._* -.Spotlight-V100 -.Trashes -Icon? -ehthumbs.db -Thumbs.db \ No newline at end of file +/target/ +/classes/ +/log/ +/logs/ +.classpath +.project +.settings +.myeclipse + ##filter databfile��sln file## +*.mdb +*.ldb +*.sln +##class file## +*.com +*.class +*.dll +*.exe +*.o +*.so +# compression file +*.7z +*.dmg +*.gz +*.iso +*.jar +*.rar +*.tar +*.zip +*.via +*.iml +*.tmp +*.err +*.log +# OS generated files # +/.idea +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +Icon? +ehthumbs.db +Thumbs.db +/.github/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1 @@ + diff --git a/README.md b/README.md index e6d95f9..ae50e7e 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,45 @@ +
+ +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](http://github.com/hhyo/archery/blob/master/LICENSE) +[![996.icu](https://img.shields.io/badge/link-996.icu-red.svg)](https://996.icu) + +
+ ## java-study **介绍** -[java-study](https://github.com/xuwujing/java-study) 是本人学习Java过程中记录的一些代码!从Java基础的数据类型、jdk1.8的使用、IO、集合、线程等等技术以及一些常用框架,netty、mina、springboot、kafka、storm、zookeeper、redis、hbase、hive等等。 + +[java-study](https://github.com/xuwujing/java-study) 是本人学习Java过程中记录的一些代码!从Java基础的数据类型、jdk1.8的使用、IO、集合、线程等等技术以及一些常用框架,netty、mina、springboot、kafka、storm、zookeeper、es、redis、hbase、hive等等。 **使用** 下载: - + git clone https://github.com/xuwujing/java-study 然后使用maven方式导入IDE中运行main方法即可。 - ## 项目结构 - com.pancm.arithmetic - 一些算法相关类 com.pancm.basics - 一些Java基础相关类 主要是三大特性、修饰符、io、集合、反射、克隆等等相关代码 com.pancm.bigdata - 大数据相关的类 主要是hbase、storm、zookeeper等等相关的代码 com.pancm.commons - 一些第三方工具类的测试用例 主要是apache commons、apache lang、google common、google guava、joda等等一些工具包测试使用代码 - com.pancm.design - 设计模式相关的示例类 包含常用的23种设计模式 + com.pancm.design - 设计模式相关的示例类 包含常用的23种设计模式 + com.pancm.elasticsearch - elasticsearch相关使用的测试用例,包括索引mapping的创建、全文检索、聚合查询等等 com.pancm.jdk8 - jdk1.8相关的类 主要是lambda、stream以及LocalDateTime等等测试代码 - com.pancm.mq - 一些消息中间件的类,主要包含kafka、rabbitmq相关的测试代码 - com.pancm.nio - 一些nio框架,主要是netty和mina - com.pancm.others - 一些不知道怎么定义的测试类,Jsoup(爬虫)、logback、lombok等等测试代码 - com.pancm.pojo - 实体相关类 - com.pancm.question - 一些面试可能会问的问题的类 - com.pancm.sql - 一些数据库相关的类,包括非关系型数据库 - com.pancm.thread - 一些线程相关的类 从基本的使用到各种并发的测试类 - com.pancm.utils - 一些常用的工具类 主要是Json数据转换,日期转换,二维码图片生成工具类,常用的AES、MD5、BASE64等等编码解码工具类,redis、kafka、zookeeper等等工具类 - - - - + com.pancm.mq - 一些消息中间件的类,主要包含kafka、rabbitmq相关的测试代码 + com.pancm.nio - 一些nio框架,主要是netty和mina + com.pancm.others - 一些不知道怎么定义的测试类,Jsoup(爬虫)、logback、lombok等等测试代码 + com.pancm.pojo - 实体相关类 + com.pancm.question - 一些面试可能会问的问题的类 + com.pancm.redis - redis相关使用的类 + com.pancm.sql - 一些数据库相关的类 + com.pancm.thread - 一些线程相关的类 从基本的使用到各种并发的测试类 + com.pancm.utils - 一些常用的工具类 主要是Json数据转换,日期转换,二维码图片生成工具类,常用的AES、MD5、BASE64等等编码解码工具类,redis、kafka、zookeeper等等工具类 ## 相关文章 + 这里介绍的文章主要是本人写的一些博客。博客主要发布在[个人博客](http://www.panchengming.com)、[CSDN](https://blog.csdn.net/qazwsxpcm)、[博客园](https://www.cnblogs.com/xuwujing/)等,但是由于个人博客在github上,访问可能较慢,CSDN目前观感体验不好,所以以下链接主要就在博客园中了。 **Java基础相关:** @@ -47,7 +52,6 @@ - [IO流](https://www.cnblogs.com/xuwujing/p/9191546.html) - [总结篇](https://www.cnblogs.com/xuwujing/p/9236376.html) - **设计模式:** - [单例模式](https://www.cnblogs.com/xuwujing/p/9277266.html) @@ -64,12 +68,61 @@ - [观察者模式和空对象模式](https://www.cnblogs.com/xuwujing/p/10036204.html) - [总结篇](https://www.cnblogs.com/xuwujing/p/10134494.html) +**JAVA进阶相关:** +- [JDK1.8的Lambda、Stream和日期的使用详解](https://www.cnblogs.com/xuwujing/p/10145691.html) +**大数据相关:** +- [大数据学习系列之三 ----- HBase Java Api 图文详解](https://www.cnblogs.com/xuwujing/p/8039175.html) +- [Kafka 使用Java实现数据的生产和消费demo](https://www.cnblogs.com/xuwujing/p/8371127.html) +- [关于Kafka 的 consumer 消费者手动提交详解](https://www.cnblogs.com/xuwujing/p/8432984.html) +- [Storm 入门的Demo教程](https://www.cnblogs.com/xuwujing/p/8584684.html) -## 其他 +**ElasticSearch相关:** +- [ElasticSearch实战系列一: ElasticSearch集群+Kibana安装教程](https://www.cnblogs.com/xuwujing/p/11385255.html) +- [ElasticSearch实战系列二: ElasticSearch的DSL语句使用教程---图文详解](https://www.cnblogs.com/xuwujing/p/11567053.html) +- [ElasticSearch实战系列三: ElasticSearch的JAVA API使用教程](https://www.cnblogs.com/xuwujing/p/11645630.html) +- [ElasticSearch实战系列四: ElasticSearch理论知识介绍](https://www.cnblogs.com/xuwujing/p/12093933.html) +- [ElasticSearch实战系列五: ElasticSearch的聚合查询基础使用教程之度量(Metric)聚合](https://www.cnblogs.com/xuwujing/p/12385903.html) +- [ElasticSearch实战系列六: Logstash快速入门](https://www.cnblogs.com/xuwujing/p/13412108.html) +- [ElasticSearch实战系列七: Logstash实战使用-图文讲解](https://www.cnblogs.com/xuwujing/p/13520666.html) +- [ElasticSearch实战系列八: Filebeat快速入门和使用---图文详解](https://www.cnblogs.com/xuwujing/p/13532125.html) +- [ElasticSearch实战系列九: ELK日志系统介绍和安装](https://www.cnblogs.com/xuwujing/p/13870806.html) +- [ElasticSearch实战系列十: ElasticSearch冷热分离架构](https://www.cnblogs.com/xuwujing/p/14599290.html) +- [ElasticSearch实战系列十一: ElasticSearch错误问题解决方案](https://www.cnblogs.com/xuwujing/p/14806392.html) + +**手记系列:** + + +- [手记系列之一 ----- 关于微信公众号和小程序的开发流程](https://www.cnblogs.com/xuwujing/p/16841577.html) +- [手记系列之二 ----- 关于IDEA的一些使用方法经验](https://www.cnblogs.com/xuwujing/p/16862451.html) +- [手记系列之三 ----- 关于使用Nginx的一些使用方法和经验](https://www.cnblogs.com/xuwujing/p/16885964.html) +- [手记系列之四 ----- 关于使用MySql的经验](https://www.cnblogs.com/xuwujing/p/17356379.html) +- [手记系列之五 ----- SQL使用经验分享](https://www.cnblogs.com/xuwujing/p/17444266.html) +- [手记系列之六 ----- 分享个人使用kafka经验](https://www.cnblogs.com/xuwujing/p/17466519.html) +- [手记系列之七 ----- 分享Linux使用经验](https://www.cnblogs.com/xuwujing/p/17807802.html) + + + +**其他博客:** + +- [两年JAVA程序员的面试总结](https://www.cnblogs.com/xuwujing/p/7613084.html) +- [一个两年java程序猿的2017个人总结](https://www.cnblogs.com/xuwujing/p/8158716.html) +- [写了一年的博客,我收获了什么](https://www.cnblogs.com/xuwujing/p/8747769.html) +- [给刚工作不久的程序猿同学的一封信](https://www.cnblogs.com/xuwujing/p/9665966.html) +- [一个平凡但不平庸的程序猿2018个人总结](https://www.cnblogs.com/xuwujing/p/9665966.html) +- [个人收集的资源分享](https://www.cnblogs.com/xuwujing/p/10393111.html) +- [一个毕业三年的程序猿对于提升自我的一些建议](https://www.cnblogs.com/xuwujing/p/11735726.html) +- [认清自我,不在迷茫!2019个人年终总结!](https://www.cnblogs.com/xuwujing/p/12174112.html) +- [纵然前路坎坷,也要毅然前行!2020年终总结!](https://www.cnblogs.com/xuwujing/p/14233270.html) +- [有一点思考的2021年终总结!](https://www.cnblogs.com/xuwujing/p/15746791.html) +- [一个想活得简单的程序猿的2022年终总结!](https://www.cnblogs.com/xuwujing/p/17060965.html) +- [写给步入三十的自己,2023年终总结!](https://www.cnblogs.com/xuwujing/p/17868627.html) + +## 其他 -在这些代码中,虽然大部分都是自己写的,但是也有不少是在学习过程中从网上或书上直接摘抄的,当时有些并未标明出处,现在由于忘了出处,有些代码并未标明,若有冒犯,忘请见谅! +在这些代码中,虽然大部分都是自己写的,但是也有不少是在学习过程中从网上或书上直接摘抄的,当时有些并未标明出处,现在由于忘了出处,有些代码并未标明,若有冒犯,请见谅! +Thanks to [Jetbrains](https://www.jetbrains.com/?from=java-study) for IDE support! \ No newline at end of file diff --git a/pom.xml b/pom.xml index a78a143..2029aef 100644 --- a/pom.xml +++ b/pom.xml @@ -1,428 +1,425 @@ - - 4.0.0 - - 1.0.0 - java_study - 0.0.1-SNAPSHOT - jar - - java_study - http://maven.apache.org - - - UTF-8 - UTF-8 - 1.8 - 1.8 - 1.8 - - - - - - - - - junit - junit - 4.12 - - - - - jdk.tools - jdk.tools - 1.8 - system - ${JAVA_HOME}/lib/tools.jar - - - - - - org.slf4j - slf4j-api - 1.7.25 - - - - - ch.qos.logback - logback-classic - 1.2.3 - - - - ch.qos.logback - logback-core - 1.2.3 - - - - - - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - 2.9.6 - - - - - com.google.code.gson - gson - 2.8.5 - - - - - com.alibaba - fastjson - 1.2.49 - - - - commons-codec - commons-codec - 1.11 - - - - - - commons-lang - commons-lang - 2.6 - - - - org.apache.commons - commons-lang3 - 3.7 - - - - - org.apache.commons - commons-compress - 1.18 - - - - - - - org.apache.poi - poi - 3.9 - - - - org.apache.poi - poi-ooxml - 3.17 - - - - - com.google.zxing - core - 3.0.0 - - - com.google.zxing - javase - 3.0.0 - - - - - - com.github.pagehelper - pagehelper - 4.1.0 - - - - - org.projectlombok - lombok - 1.16.18 - - - - - com.geccocrawler - gecco - 1.2.8 - - - - - - com.google.protobuf - protobuf-java - 3.5.1 - - - - - - org.quartz-scheduler - quartz - 2.3.0 - - - - org.quartz-scheduler - quartz-jobs - 2.3.0 - - - - - - - - - - - redis.clients - jedis - 2.9.0 - - - - - mysql - mysql-connector-java - 5.1.41 - - - - - org.xerial - sqlite-jdbc - 3.20.1 - - - - - com.microsoft.sqlserver - sqljdbc4 - 4.0 - - - - - - - - - - - io.netty - netty-all - 4.1.17.Final - - - - - org.apache.mina - mina-core - 2.0.16 - - - - - com.github.kevinsawicki - http-request - 6.0 - - - - - - - - - - - - - com.rabbitmq - amqp-client - 4.3.0 - - - - - org.apache.kafka - kafka_2.12 - 1.0.0 - - - org.apache.zookeeper - zookeeper - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - - - - - org.apache.kafka - kafka-clients - 1.0.0 - - - - org.apache.kafka - kafka-streams - 1.0.0 - - - - - - - - - - org.apache.zookeeper - zookeeper - 3.4.10 - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - - - - - com.101tec - zkclient - 0.10 - - - - - - org.apache.hadoop - hadoop-common - 2.8.2 - - - org.apache.hadoop - hadoop-client - 2.8.2 - - - org.apache.hadoop - hadoop-hdfs - 2.8.2 - - - org.apache.hadoop - hadoop-mapreduce-client-core - 2.8.2 - - - org.apache.hadoop - hadoop-yarn-common - 2.8.2 - - - - - org.apache.hbase - hbase-hadoop-compat - 1.3.1 - - - org.apache.hbase - hbase-server - 1.4.8 - - - org.apache.hbase - hbase-client - 1.4.8 - - - org.apache.hbase - hbase-common - 1.4.8 - - - - - - org.apache.hive - hive-jdbc - 3.1.1 - - - - - org.apache.storm - storm-core - 1.2.2 - - - - org.apache.storm - storm-kafka - 1.2.2 - - - - - - org.apache.spark - spark-core_2.11 - 2.4.0 - - - org.apache.spark - spark-sql_2.11 - 2.4.0 - - - org.apache.spark - spark-streaming_2.11 - 2.4.0 - provided - - - com.sparkjava - spark-core - 2.8.0 - - - org.apache.spark - spark-mllib_2.11 - 2.4.0 - provided - - - - - - - + + 4.0.0 + + 1.0.0 + java-study + 0.0.1-SNAPSHOT + jar + + java-study + http://maven.apache.org + + + UTF-8 + UTF-8 + 1.8 + 1.8 + 1.8 + + + + + + + + + junit + junit + 4.13.1 + + + + + + + + + org.slf4j + slf4j-api + 1.7.25 + + + + + ch.qos.logback + logback-classic + 1.2.3 + + + + ch.qos.logback + logback-core + 1.2.3 + + + + + + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider + 2.9.6 + + + + + com.google.code.gson + gson + 2.8.5 + + + + + com.alibaba + fastjson + 1.2.49 + + + + commons-codec + commons-codec + 1.11 + + + + + + commons-lang + commons-lang + 2.6 + + + + org.apache.commons + commons-lang3 + 3.7 + + + + + org.apache.commons + commons-compress + 1.18 + + + + + + + org.apache.poi + poi + 3.17 + + + + org.apache.poi + poi-ooxml + 3.17 + + + + + com.google.zxing + core + 3.0.0 + + + com.google.zxing + javase + 3.0.0 + + + + + + com.github.pagehelper + pagehelper + 4.1.0 + + + + + org.projectlombok + lombok + 1.16.18 + + + + + com.geccocrawler + gecco + 1.2.8 + + + + + + com.google.protobuf + protobuf-java + 3.5.1 + + + + + + org.quartz-scheduler + quartz + 2.3.0 + + + + org.quartz-scheduler + quartz-jobs + 2.3.0 + + + + org.springframework + spring-beans + 5.2.1.RELEASE + + + + + com.google.guava + guava + 27.1-jre + + + + + + + + + + + redis.clients + jedis + 2.9.0 + + + + + mysql + mysql-connector-java + 5.1.41 + + + + + org.xerial + sqlite-jdbc + 3.20.1 + + + + + + + com.microsoft.sqlserver + mssql-jdbc + 6.2.0.jre8 + test + + + + + + + + + + io.netty + netty-all + 4.1.42.Final + + + + + org.apache.mina + mina-core + 2.0.16 + + + + + com.github.kevinsawicki + http-request + 6.0 + + + + + + + + + + + + + com.rabbitmq + amqp-client + 4.3.0 + + + + + org.apache.kafka + kafka_2.12 + 1.0.0 + + + org.apache.zookeeper + zookeeper + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + + + + + org.apache.kafka + kafka-clients + 1.0.0 + + + + org.apache.kafka + kafka-streams + 1.0.0 + + + + + + + + + + org.apache.zookeeper + zookeeper + 3.4.10 + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + + + + + com.101tec + zkclient + 0.10 + + + + + org.apache.hbase + hbase-client + 2.1.4 + + + org.apache.hbase + hbase-common + 2.1.4 + + + + org.apache.hadoop + hadoop-client + 3.2.0 + + + + org.apache.spark + spark-core_2.12 + 2.4.0 + + + + org.apache.spark + spark-sql_2.12 + 2.4.0 + + + + + org.apache.storm + storm-core + 1.2.2 + provided + + + + org.apache.storm + storm-kafka + 1.2.3 + provided + + + + + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 6.6.1 + + + + + io.searchbox + jest + 6.3.1 + + + + + com.alibaba + easyexcel + 2.2.7 + + + + + + + + + + spring-milestone + http://repo.spring.io/libs-release + + + diff --git a/src/main/java/com/pancm/arithmetic/jzoffer/DuplicateNumberInArray.java b/src/main/java/com/pancm/arithmetic/jzoffer/DuplicateNumberInArray.java new file mode 100644 index 0000000..8f7734b --- /dev/null +++ b/src/main/java/com/pancm/arithmetic/jzoffer/DuplicateNumberInArray.java @@ -0,0 +1,147 @@ +package com.pancm.arithmetic.jzoffer; + +/** + * https://www.nowcoder.com/practice/623a5ac0ea5b4e5f95552655361ae0a8?tpId=13&tqId=11203&tPage=1&rp=1&ru=/ta/coding-interviews&qru=/ta/coding-interviews/question-ranking + * + * @description: + * 3.数组中的重复数字 + * 在一个长度为 n 的数组里的所有数字都在 0 到 n-1 的范围内。 + * 数组中某些数字是重复的,但不知道有几个数字是重复的,也不知道每个数字重复几次。 + * 请找出数组中任意一个重复的数字。 + * + * @author: Zhoust + * @date: 2019/04/28 10:58 + * @version: V1.0 + */ +public class DuplicateNumberInArray { + + /** + * 一种简单的实现思路,借助外部数组保存 numbers 中每个值的出现次数,即 numbers 中的值作为数组 array 的下标。 + * 题目中说了长度为 n 的数组中每个值都在 0 到 n-1 范围内,因此遍历数组 numbers 存入对应的 array 不会出现数组下标越界的问题 + * @param numbers + * @param length numbers 数组的长度 + * @param duplication (Output) the duplicated number in the array number,length of duplication array is 1,so using duplication[0] = ? in implementation; + * 这里要特别注意~返回任意重复的一个,赋值duplication[0] + * @return true if the input is valid, and there are some duplications in the array number, otherwise false + */ + public boolean duplicate(int[] numbers, int length, int[] duplication) { + if (numbers == null) { + return false; + } + boolean result = false; + int[] array = new int[length]; + for (int i = 0; i < length; i++) { + if (++array[numbers[i]] > 1 && !result) { + result = true; + duplication[0] = numbers[i]; + } + } + return result; + } + + /** + * 先排序,然后找到排序后数组中第一个重复的数字(主要就是练习一下快排) + * @param numbers + * @param length + * @param duplication + * @return + */ + public boolean sortThenFindFirstDuplicateNumber(int[] numbers, int length, int[] duplication) { + if (numbers == null) { + return false; + } + quickSort(numbers, 0, length-1); + int temp = numbers[0]; + boolean result = false; + + for (int i = 1; i < length; i++) { + if (numbers[i] == temp) { + result = true; + duplication[0] = temp; + break; + } + temp = numbers[i]; + } + return result; + } + + /** + * 将值为 i 的元素调整到下标为 i 的位置,如果该位置上的数已经等于 i 了,就说明 i 已经重复了 + * @param numbers + * @param length + * @param duplication + * @return + */ + public static boolean jz(int[] numbers, int length, int[] duplication) { + for (int i = 0; i < length; i++) { + //第 i 个位置上的数 numbers[i] 若不与下标 i 相等,就把 numbers[i] 放到下标为 numbers[i] 的位置 + while (numbers[i] != i) { + //在交换位置之前还要检查 numbers[i] 位置上的数字是否已经是 numbers[i],如果是就已经找到了重复的数字 + if (numbers[numbers[i]] == numbers[i]) { + duplication[0] = numbers[i]; + //注意这里不是 break + return true; + } + + //交换下标为 i、numbers[i] 两个位置上的数字(保证下标为 numbers[i] 的位置,放置的数字是 numbers[i]) + swap(numbers, i, numbers[i]); + } + } + return false; + } + + /** + * 交换下标为 i、j 的两个数 + * @param numbers + * @param i + * @param j + */ + public static void swap(int[] numbers, int i, int j) { + int swap = numbers[i]; + numbers[i] = numbers[j]; + numbers[j] = swap; + } + + /** + * 快排 + * @param array + * @param length + */ + @SuppressWarnings("all") + public static void quickSort(int[] array, int low, int high) { + //这里必须首先限制 low temp && h > l) { + h--; + } + array[l] = array[h]; + while (array[l] <= temp && l < h) { + l++; + } + array[h] = array[l]; + } + array[l] = temp; + quickSort(array, low, h-1); + quickSort(array, l+1, high); + } + } + + public static void main(String[] args) { + int[] result = new int[1]; + jz(new int[]{2,4,2,1,4}, 5, result); + System.out.println(result[0]); +// int[] d = new int[1]; +// jz(new int[]{2,1,3,0,4}, 5, d); +// int[] array = new int[]{1,4,4,4,12,78,8,45,67,39,20,90,65,34,34,6,7,88,9,1,-3,-88,-2}; + int[] array = new int[]{2,4,3,1,4}; + quickSort(array, 0, array.length-1); + for (int i = 0; i < array.length; i++) { + System.out.println(array[i]); + } + + } + +} \ No newline at end of file diff --git a/src/main/java/com/pancm/arithmetic/jzoffer/package-info.java b/src/main/java/com/pancm/arithmetic/jzoffer/package-info.java new file mode 100644 index 0000000..5a7d751 --- /dev/null +++ b/src/main/java/com/pancm/arithmetic/jzoffer/package-info.java @@ -0,0 +1,7 @@ +/** + * @description: 剑指 Offer 中的题目 + * @author: Zhoust + * @date: 2019/04/28 10:57 + * @version: V1.0 + */ +package com.pancm.arithmetic.jzoffer; \ No newline at end of file diff --git a/src/main/java/com/pancm/basics/ListTest.java b/src/main/java/com/pancm/basics/ListTest.java index 41640dd..b863c96 100644 --- a/src/main/java/com/pancm/basics/ListTest.java +++ b/src/main/java/com/pancm/basics/ListTest.java @@ -1,292 +1,295 @@ -package com.pancm.basics; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Vector; -import java.util.stream.Collectors; - -/** - * -* @Title: ListTest -* @Description:关于list测试 -* @Version:1.0.0 -* @author pancm -* @date 2017年10月13日 - */ -public class ListTest { - - private final static int count=50000; - - private static ArrayList arrayList = new ArrayList<>(); - private static LinkedList linkedList = new LinkedList<>(); - private static Vector vector = new Vector<>(); - - - public static void main(String[] args) { -// test1(); -// test2(); -// test3(); - test4(); -// test5(); - - Vector v=new Vector<>(); - } - - - /** - * 遍历方法 - */ - private static void test1() { - List list=new ArrayList(); - list.add("a"); - list.add("b"); - list.add("c"); - - System.out.println("list截取:"+list.subList(0, 1)); - System.out.println("list截取:"+list.subList(1, 3)); - - - //第一种使用 - for(int i=0;i iterator=list.iterator(); - while(iterator.hasNext())//判断下一个元素之后有值 - { - System.out.println(iterator.next()); - } - } - - /** - * 数组变化遍历 - */ - private static void test2() { - - List list1 = new ArrayList(); - list1.add("1"); - list1.add("2"); - System.out.println("list1遍历之前:"+list1); - Iterator iterator = list1.iterator(); - while (iterator.hasNext()) { - String item = iterator.next(); - if ("2".equals(item)) { - iterator.remove(); - } - } - System.out.println("list1遍历之后:"+list1); - - List list = new ArrayList(); - list.add("1"); - list.add("2"); - System.out.println("list遍历之前:"+list); - for (String item : list) { - if ("2".equals(item)) { - list.remove(item); - //如果这里不适用break的话,会直接报错的 - break; - } - } - System.out.println("list遍历之后:"+list); - - - } - - - /** - * 常用方法 - */ - private static void test3() { - List list=initData(100); - System.out.println("list:"+list); - System.out.println("removeList:"+removeList(list,10,20)); - list.subList(10, 20).clear(); - System.out.println("subList:"+list); - - List ls1=new ArrayList(); - List ls2=new ArrayList(); - ls1.add("a"); - ls1.add("b"); - ls1.add("c"); - ls2.add("a"); - ls2.add("d"); - ls2.add("e"); -// System.out.println("合集:"+addAll(ls1,ls2)); - System.out.println("交集 :"+retainAll(ls1,ls2)); - System.out.println("差集 :"+removeAll(ls1,ls2)); - System.out.println("并集 :"+andAll(ls1,ls2)); - - } - - private static void test4() { - LinkedList list=new LinkedList(); - list.add(3); - list.add(5); - list.add(4); - System.out.println(list); - list.addFirst(2); - list.addLast(4); - System.out.println(list); - - LinkedList list2=new LinkedList(); - list2.add(1); - list2.add(2); - list2.add(4); - list2.add(4); - list2.add(6); - list2.add(6); - list2.add(5); - System.out.println("去重之前:"+list2); - //jdk1.8去重 - List newList = list2.stream().distinct().collect(Collectors.toList()); - System.out.println("去重之后:"+newList); - - } - - private static void test5() { - insertList(arrayList); - insertList(linkedList); - insertList(vector); - - System.out.println("--------------------"); - - readList(arrayList); - readList(linkedList); - readList(vector); - - System.out.println("--------------------"); - - delList(arrayList); - delList(linkedList); - delList(vector); - - } - - - private static void insertList(List list){ - long start=System.currentTimeMillis(); - Object o = new Object(); - for(int i=0;i removeList(List list,int s,int d){ - for(int i=0,j=list.size();i=s&&i initData(int j){ - List list=new ArrayList(); - for(int i=1;i<=j;i++){ - list.add(i); - } - return list; - } - - /** - * 数组合集 - * @param ls1 - * @param ls2 - * @return - */ - private static List addAll(List ls1,Listls2){ - ls1.addAll(ls2); - return ls1; - } - - /** - * 数组交集 (retainAll 会删除 ls1在ls2中没有的元素) - * @param ls1 - * @param ls2 - * @return - */ - private static List retainAll(List ls1,Listls2){ - System.out.println("ls1:"+ls1+";ls2:"+ls2); - ls1.retainAll(ls2); - return ls1; - } - - /** - * 差集 (删除ls2中没有ls1中的元素) - * @param ls1 - * @param ls2 - * @return - */ - private static List removeAll(List ls1,Listls2){ - ls1.removeAll(ls2); - System.out.println(ls1+";ls2:"+ls2); - return ls1; - } - - /** - * 无重复的并集 (ls1和ls2中并集,并无重复) - * @param ls1 - * @param ls2 - * @return - */ - private static List andAll(List ls1,Listls2){ - //删除在ls1中出现的元素 - ls2.removeAll(ls1); - //将剩余的ls2中的元素添加到ls1中 - ls1.addAll(ls2); - System.out.println(ls1+";ls2:"+ls2); - return ls1; - } -} +package com.pancm.basics; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Vector; +import java.util.stream.Collectors; + +/** + * +* @Title: ListTest +* @Description:关于list测试 +* @Version:1.0.0 +* @author pancm +* @date 2017年10月13日 + */ +public class ListTest { + + private final static int count=50000; + + private static ArrayList arrayList = new ArrayList<>(); + private static LinkedList linkedList = new LinkedList<>(); + private static Vector vector = new Vector<>(); + + + public static void main(String[] args) { +// test1(); +// test2(); + test3(); +// test4(); +// test5(); + + + Vector v=new Vector<>(); + } + + + /** + * 遍历方法 + */ + private static void test1() { + List list=new ArrayList(); + list.add("a"); + list.add("b"); + list.add("c"); + + System.out.println("list截取:"+list.subList(0, 1)); + System.out.println("list截取:"+list.subList(1, 3)); + + + //第一种使用 + for(int i=0;i iterator=list.iterator(); + while(iterator.hasNext())//判断下一个元素之后有值 + { + System.out.println(iterator.next()); + } + } + + /** + * 数组变化遍历 + */ + private static void test2() { + + List list1 = new ArrayList(); + list1.add("1"); + list1.add("2"); + System.out.println("list1遍历之前:"+list1); + Iterator iterator = list1.iterator(); + while (iterator.hasNext()) { + String item = iterator.next(); + if ("2".equals(item)) { + iterator.remove(); + } + } + System.out.println("list1遍历之后:"+list1); + + List list = new ArrayList(); + list.add("1"); + list.add("2"); + System.out.println("list遍历之前:"+list); + for (String item : list) { + if ("2".equals(item)) { + list.remove(item); + //如果这里不适用break的话,会直接报错的 + break; + } + } + System.out.println("list遍历之后:"+list); + + + } + + + /** + * 常用方法 + */ + private static void test3() { + List list=initData(100); + System.out.println("list:"+list); + System.out.println("removeList:"+removeList(list,10,20)); + list.subList(10, 20).clear(); + System.out.println("subList:"+list); + + List ls1=new ArrayList(); + List ls2=new ArrayList(); + ls1.add("a"); + ls1.add("b"); + ls1.add("c"); + ls2.add("a"); + ls2.add("b"); + ls2.add("c"); + ls2.add("e"); +// System.out.println("合集:"+addAll(ls1,ls2)); +// System.out.println("交集 :"+retainAll(ls1,ls2)); + System.out.println("差集 :"+removeAll(ls2,ls1)); +// System.out.println("并集 :"+andAll(ls1,ls2)); + + } + + private static void test4() { + LinkedList list=new LinkedList(); + list.add(3); + list.add(5); + list.add(4); + System.out.println(list); + list.addFirst(2); + list.addLast(4); + System.out.println(list); + + LinkedList list2=new LinkedList(); + list2.add(1); + list2.add(2); + list2.add(4); + list2.add(4); + list2.add(6); + list2.add(6); + list2.add(5); + System.out.println("去重之前:"+list2); + //jdk1.8去重 + List newList = list2.stream().distinct().collect(Collectors.toList()); + System.out.println("去重之后:"+newList); + + } + + private static void test5() { + insertList(arrayList); + insertList(linkedList); + insertList(vector); + + System.out.println("--------------------"); + + readList(arrayList); + readList(linkedList); + readList(vector); + + System.out.println("--------------------"); + + delList(arrayList); + delList(linkedList); + delList(vector); + + } + + + private static void insertList(List list){ + long start=System.currentTimeMillis(); + Object o = new Object(); + for(int i=0;i removeList(List list,int s,int d){ + for(int i=0,j=list.size();i=s&&i initData(int j){ + List list=new ArrayList(); + for(int i=1;i<=j;i++){ + list.add(i); + } + return list; + } + + /** + * 数组合集 + * @param ls1 + * @param ls2 + * @return + */ + private static List addAll(List ls1,Listls2){ + ls1.addAll(ls2); + return ls1; + } + + /** + * 数组交集 (retainAll 会删除 ls1在ls2中没有的元素) + * @param ls1 + * @param ls2 + * @return + */ + private static List retainAll(List ls1,Listls2){ + System.out.println("ls1:"+ls1+";ls2:"+ls2); + ls1.retainAll(ls2); + return ls1; + } + + /** + * 差集 (删除ls2中没有ls1中的元素) + * @param ls1 + * @param ls2 + * @return + */ + private static List removeAll(List ls1,Listls2){ + System.out.println("ls1:"+ls1+";ls2:"+ls2); + ls1.removeAll(ls2); + return ls1; + } + + /** + * 无重复的并集 (ls1和ls2中并集,并无重复) + * @param ls1 + * @param ls2 + * @return + */ + private static List andAll(List ls1,Listls2){ + //删除在ls1中出现的元素 + ls2.removeAll(ls1); + //将剩余的ls2中的元素添加到ls1中 + ls1.addAll(ls2); + System.out.println(ls1+";ls2:"+ls2); + return ls1; + } +} diff --git a/src/main/java/com/pancm/design/mediator/MediatorTest.java b/src/main/java/com/pancm/design/mediator/MediatorTest.java index 02c80c4..e582d55 100644 --- a/src/main/java/com/pancm/design/mediator/MediatorTest.java +++ b/src/main/java/com/pancm/design/mediator/MediatorTest.java @@ -29,9 +29,8 @@ public static void main(String[] args) { XuWuJing xwj = new XuWuJing("xuwujing", jq); jq.setZs(zs); jq.setXwj(xwj); - - zs.talk("大家好!我是张三!");; - xwj.talk("欢迎你!张三!"); + zs.exchange("大家好!我是张三!"); + xwj.exchange("欢迎你!张三!"); /* diff --git a/src/main/java/com/pancm/design/responsibility/ResponsibilityTest.java b/src/main/java/com/pancm/design/responsibility/ResponsibilityTest.java index b810e58..fa1b4a3 100644 --- a/src/main/java/com/pancm/design/responsibility/ResponsibilityTest.java +++ b/src/main/java/com/pancm/design/responsibility/ResponsibilityTest.java @@ -22,19 +22,19 @@ public static void main(String[] args) { String name = "xuwujing"; String something = "去聚餐"; String something2 = "去旅游"; - Learder learder1 =new Supervisor(name, something); - Learder learder2 =new BranchManager(name, something); - Learder learder3 =new GeneralManager(name, something); - learder1.setLearder(learder2); - learder2.setLearder(learder3); - learder1.handler(1); + Leader leader1 =new Supervisor(name, something); + Leader leader2 =new BranchManager(name, something); + Leader leader3 =new GeneralManager(name, something); + leader1.setLeader(leader2); + leader2.setLeader(leader3); + leader1.handler(1); - Learder learder4 =new Supervisor(name, something2); - Learder learder5 =new BranchManager(name, something2); - Learder learder6 =new GeneralManager(name, something2); - learder4.setLearder(learder5); - learder5.setLearder(learder6); - learder4.handler(0); + Leader leader4 =new Supervisor(name, something2); + Leader leader5 =new BranchManager(name, something2); + Leader leader6 =new GeneralManager(name, something2); + leader4.setLeader(leader5); + leader5.setLeader(leader6); + leader4.handler(0); @@ -76,24 +76,24 @@ public void handleRequest() { } -abstract class Learder{ +abstract class Leader { - protected Learder learder; + protected Leader leader; - protected void setLearder(Learder learder){ - this.learder=learder; + protected void setLeader(Leader leader){ + this.leader = leader; } - protected Learder getLearder(){ - return learder; + protected Leader getLeader(){ + return leader; } abstract void handler(int level); } //主管 -class Supervisor extends Learder{ +class Supervisor extends Leader { private String name; private String something; public Supervisor(String name,String something) { @@ -108,13 +108,13 @@ void handler(int level) { System.out.println("主管处理了 "+name+"所述的<"+something+">事情!"); }else{ System.out.println("主管未能处理 "+name+"所述的<"+something+">事情!转交给上级!"); - getLearder().handler(level); + getLeader().handler(level); } } } //部门经理 -class BranchManager extends Learder{ +class BranchManager extends Leader { private String name; private String something; public BranchManager(String name,String something) { @@ -129,13 +129,13 @@ void handler(int level) { System.out.println("部门经理处理了 "+name+"所述的<"+something+">事情!"); }else{ System.out.println("部门经理未能处理 "+name+"所述的<"+something+">事情!转交给上级!"); - getLearder().handler(level); + getLeader().handler(level); } } } //总经理 -class GeneralManager extends Learder{ +class GeneralManager extends Leader { private String name; private String something; public GeneralManager(String name,String something) { @@ -150,7 +150,7 @@ void handler(int level) { System.out.println("总经理处理了 "+name+"所述的<"+something+">事情!"); }else{ System.out.println("总经理未能处理 "+name+"所述的<"+something+">事情!转交给上级!"); - getLearder().handler(level); + getLeader().handler(level); } } } diff --git a/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java b/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java new file mode 100644 index 0000000..22b3049 --- /dev/null +++ b/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java @@ -0,0 +1,604 @@ +package com.pancm.elasticsearch; + +import org.apache.http.HttpHost; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.avg.Avg; +import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.max.Max; +import org.elasticsearch.search.aggregations.metrics.min.Min; +import org.elasticsearch.search.aggregations.metrics.sum.Sum; +import org.elasticsearch.search.aggregations.metrics.tophits.TopHits; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders; +import org.elasticsearch.search.aggregations.pipeline.bucketselector.BucketSelectorPipelineAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; + +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author pancm + * @Title: pancm_project + * @Description:ES聚合查询测试用例 + * @Version:1.0.0 + * @Since:jdk1.8 + * @date 2019/4/2 + */ +public class EsAggregationSearchTest { + + + + private static String elasticIp = "192.169.0.23"; + private static int elasticPort = 9200; + private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestSearchTest.class); + + private static RestHighLevelClient client = null; + + /** + * @param args + */ + public static void main(String[] args) { + + try { + init(); + createIndex(); + bulk(); + groupbySearch(); + avgSearch(); + maxSearch(); + sumSearch(); + avgGroupSearch(); + maxGroupSearch(); + sumGroupSearch(); + topSearch(); + } catch (Exception e) { + e.printStackTrace(); + }finally { + close(); + } + + } + + + /* + * 初始化服务 + */ + private static void init() { + RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(elasticIp, elasticPort)); + client = new RestHighLevelClient(restClientBuilder); + } + + /* + * 关闭服务 + */ + private static void close() { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + e.printStackTrace(); + }finally{ + client=null; + } + } + } + + /** + * 创建索引 + * + * @throws IOException + */ + private static void createIndex() throws IOException { + + // 类型 + String type = "_doc"; + String index = "student"; + // setting 的值 + Map setmapping = new HashMap<>(); + // 分区数、副本数、缓存刷新时间 + setmapping.put("number_of_shards", 10); + setmapping.put("number_of_replicas", 1); + setmapping.put("refresh_interval", "5s"); + Map keyword = new HashMap<>(); + //设置类型 + keyword.put("type", "keyword"); + Map lon = new HashMap<>(); + //设置类型 + lon.put("type", "long"); + Map date = new HashMap<>(); + //设置类型 + date.put("type", "date"); + date.put("format", "yyyy-MM-dd"); + + Map date2 = new HashMap<>(); + //设置类型 + date2.put("type", "date"); + date2.put("format", "yyyy-MM-dd HH:mm:ss.SSS"); + Map jsonMap2 = new HashMap<>(); + Map properties = new HashMap<>(); + //设置字段message信息 + properties.put("uid", lon); + properties.put("grade", lon); + properties.put("class", lon); + properties.put("age", lon); + properties.put("name", keyword); + properties.put("createtm", date); + properties.put("updatetm", date2); + Map mapping = new HashMap<>(); + mapping.put("properties", properties); + jsonMap2.put(type, mapping); + + GetIndexRequest getRequest = new GetIndexRequest(); + getRequest.indices(index); + getRequest.types(type); + getRequest.local(false); + getRequest.humanReadable(true); + boolean exists2 = client.indices().exists(getRequest, RequestOptions.DEFAULT); + //如果存在就不创建了 + if(exists2) { + System.out.println(index+"索引库已经存在!"); + return; + } + // 开始创建库 + CreateIndexRequest request = new CreateIndexRequest(index); + try { + // 加载数据类型 + request.settings(setmapping); + //设置mapping参数 + request.mapping(type, jsonMap2); + //设置别名 + request.alias(new Alias("pancm_alias")); + CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); + boolean falg = createIndexResponse.isAcknowledged(); + if(falg){ + System.out.println("创建索引库:"+index+"成功!" ); + } + } catch (IOException e) { + e.printStackTrace(); + } + + } + + + + /** + * 批量操作示例 + * + * @throws InterruptedException + */ + private static void bulk() throws IOException{ + // 类型 + String type = "_doc"; + String index = "student"; + + BulkRequest request = new BulkRequest(); + int k =10; + List> mapList = new ArrayList<>(); + LocalDateTime ldt = LocalDateTime.now(); + for (int i = 1; i <=k ; i++) { + Map map = new HashMap<>(); + map.put("uid",i); + map.put("age",i); + map.put("name","虚无境"+(i%3)); + map.put("class",i%10); + map.put("grade",400+i); + map.put("createtm",ldt.plusDays(i).format(DateTimeFormatter.ofPattern("yyyy-MM-dd"))); + map.put("updatetm",ldt.plusDays(i).format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"))); + if(i==5){ + map.put("updatetm","2019-11-31 21:04:55.268"); + } + mapList.add(map); + } + + + for (int i = 0; i map = mapList.get(i); + String id = map.get("uid").toString(); + // 可以进行修改/删除/新增 操作 + //docAsUpsert 为true表示存在更新,不存在插入,为false表示不存在就是不做更新 + request.add(new UpdateRequest(index, type, id).doc(map, XContentType.JSON).docAsUpsert(true).retryOnConflict(5)); + } + + client.bulk(request, RequestOptions.DEFAULT); + System.out.println("批量执行成功!"); + } + + /** + * @Author pancm + * @Description 多个聚合条件测试 + * SQL: select age, name, count(*) as count1 from student group by age, name; + * @Date 2019/7/3 + * @Param [] + * @return void + **/ + private static void groupbySearch() throws IOException{ + String buk="group"; + AggregationBuilder aggregation = AggregationBuilders.terms("age").field("age"); + AggregationBuilder aggregation2 = AggregationBuilders.terms("name").field("name"); + //根据创建时间按天分组 + AggregationBuilder aggregation3 = AggregationBuilders.dateHistogram("createtm") + .field("createtm") + .format("yyyy-MM-dd") + .dateHistogramInterval(DateHistogramInterval.DAY); + + aggregation2.subAggregation(aggregation3); + aggregation.subAggregation(aggregation2); + agg(aggregation,buk); + } + + /** + * @Author pancm + * @Description 平均聚合查询测试用例 + * @Date 2019/4/1 + * @Param [] + * @return void + **/ + private static void avgSearch() throws IOException { + + String buk="t_grade_avg"; + //直接求平均数 + AggregationBuilder aggregation = AggregationBuilders.avg(buk).field("grade"); + logger.info("求班级的平均分数:"); + agg(aggregation,buk); + + } + + private static void maxSearch() throws IOException{ + String buk="t_grade"; + AggregationBuilder aggregation = AggregationBuilders.max(buk).field("grade"); + logger.info("求班级的最高分数:"); + agg(aggregation,buk); + } + + private static void sumSearch() throws IOException{ + String buk="t_grade"; + AggregationBuilder aggregation = AggregationBuilders.sum(buk).field("grade"); + logger.info("求班级的总分数:"); + agg(aggregation,buk); + } + + /** + * @Author pancm + * @Description 平均聚合查询测试用例 + * @Date 2019/4/1 + * @Param [] + * @return void + **/ + private static void avgGroupSearch() throws IOException { + + + String agg="t_class_avg"; + String buk="t_grade"; + //terms 就是分组统计 根据student的grade成绩进行分组并创建一个新的聚合 + TermsAggregationBuilder aggregation = AggregationBuilders.terms(agg).field("class"); + aggregation.subAggregation(AggregationBuilders.avg(buk).field("grade")); + + logger.info("根据班级求平均分数:"); + agg(aggregation,agg,buk); + + } + + + private static void maxGroupSearch() throws IOException{ + + String agg="t_class_max"; + String buk="t_grade"; + //terms 就是分组统计 根据student的grade成绩进行分组并创建一个新的聚合 + TermsAggregationBuilder aggregation = AggregationBuilders.terms(agg).field("class"); + aggregation.subAggregation(AggregationBuilders.max(buk).field("grade")); + logger.info("根据班级求最大分数:"); + agg(aggregation,agg,buk); + } + + + private static void sumGroupSearch() throws IOException{ + String agg="t_class_sum"; + String buk="t_grade"; + //terms 就是分组统计 根据student的grade成绩进行分组并创建一个新的聚合 + TermsAggregationBuilder aggregation = AggregationBuilders.terms(agg).field("class"); + aggregation.subAggregation(AggregationBuilders.sum(buk).field("grade")); + + logger.info("根据班级求总分:"); + agg(aggregation,agg,buk); + } + + + protected static void agg(AggregationBuilder aggregation, String buk) throws IOException{ + SearchResponse searchResponse = search(aggregation); + if(RestStatus.OK.equals(searchResponse.status())) { + // 获取聚合结果 + Aggregations aggregations = searchResponse.getAggregations(); + + if(buk.contains("avg")){ + //取子聚合 + Avg ba = aggregations.get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + }else if(buk.contains("max")){ + //取子聚合 + Max ba = aggregations.get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + + }else if(buk.contains("min")){ + //取子聚合 + Min ba = aggregations.get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + }else if(buk.contains("sum")){ + //取子聚合 + Sum ba = aggregations.get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + }else if(buk.contains("top")){ + //取子聚合TopHits + TopHits ba = aggregations.get(buk); + logger.info(buk+":" + ba.getHits().totalHits); + logger.info("------------------------------------"); + }else if (buk.contains("group")){ + Map map = new HashMap<>(); + List> list = new ArrayList<>(); + agg(map,list,aggregations); + logger.info("聚合查询结果:"+list); + logger.info("------------------------------------"); + } + + } + } + + private static void agg(Map map, List> list, Aggregations aggregations) { + aggregations.forEach(aggregation -> { + String name = aggregation.getName(); + Terms genders = aggregations.get(name); + for (Terms.Bucket entry : genders.getBuckets()) { + String key = entry.getKey().toString(); + long t = entry.getDocCount(); + map.put(name,key); + map.put(name+"_"+"count",t); + + //判断里面是否还有嵌套的数据 + List list2 = entry.getAggregations().asList(); + if (list2.isEmpty()) { + Map map2 = new HashMap<>(); + BeanUtils.copyProperties(map,map2); + list.add(map2); + }else{ + agg(map, list, entry.getAggregations()); + } + } + }); + } + + private static void agg(List> list, Aggregations aggregations) { + aggregations.forEach(aggregation -> { + String name = aggregation.getName(); + Terms genders = aggregations.get(name); + for (Terms.Bucket entry : genders.getBuckets()) { + String key = entry.getKey().toString(); + long t = entry.getDocCount(); + Map map =new HashMap<>(); + map.put(name,key); + map.put(name+"_"+"count",t); + //判断里面是否还有嵌套的数据 + List list2 = entry.getAggregations().asList(); + if (list2.isEmpty()) { + list.add(map); + }else{ + agg(list, entry.getAggregations()); + } + } + }); + System.out.println(list); + } + + + + private static SearchResponse search(AggregationBuilder aggregation) throws IOException { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices("student"); + searchRequest.types("_doc"); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + //不需要解释 + searchSourceBuilder.explain(false); + //不需要原始数据 + searchSourceBuilder.fetchSource(false); + //不需要版本号 + searchSourceBuilder.version(false); + searchSourceBuilder.aggregation(aggregation); + logger.info("查询的语句:"+searchSourceBuilder.toString()); + searchRequest.source(searchSourceBuilder); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + return searchResponse; + } + + /** + * @Author pancm + * @Description 进行聚合 + * @Date 2019/4/2 + * @Param [] + * @return void + **/ + protected static void agg(AggregationBuilder aggregation, String agg, String buk) throws IOException{ + + // 同步查询 + SearchResponse searchResponse = search(aggregation); + + //4、处理响应 + //搜索结果状态信息 + if(RestStatus.OK.equals(searchResponse.status())) { + // 获取聚合结果 + Aggregations aggregations = searchResponse.getAggregations(); + + + //分组 + Terms byAgeAggregation = aggregations.get(agg); + logger.info(agg+" 结果"); + logger.info("name: " + byAgeAggregation.getName()); + logger.info("type: " + byAgeAggregation.getType()); + logger.info("sumOfOtherDocCounts: " + byAgeAggregation.getSumOfOtherDocCounts()); + + logger.info("------------------------------------"); + for(Terms.Bucket buck : byAgeAggregation.getBuckets()) { + logger.info("key: " + buck.getKeyAsNumber()); + logger.info("docCount: " + buck.getDocCount()); + logger.info("docCountError: " + buck.getDocCountError()); + + + if(agg.contains("avg")){ + //取子聚合 + Avg ba = buck.getAggregations().get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + }else if(agg.contains("max")){ + //取子聚合 + Max ba = buck.getAggregations().get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + + }else if(agg.contains("min")){ + //取子聚合 + Min ba = buck.getAggregations().get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + }else if(agg.contains("sum")){ + //取子聚合 + Sum ba = buck.getAggregations().get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + } + } + } + } + + /** + * @Author pancm + * @Description having + * @Date 2020/8/21 + * @Param [] + * @return void + **/ + private static void havingSearch() throws IOException{ + String index=""; + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.indices(index); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); + String alias_name = "nas_ip_address_group"; + String group_name = "nas_ip_address"; + String query_name = "acct_start_time"; + String query_type = "gte,lte"; + String query_name_value="2020-08-05 13:25:55,2020-08-20 13:26:55"; + String[] query_types= query_type.split(","); + String[] query_name_values= query_name_value.split(","); + + for (int i = 0; i < query_types.length; i++) { + if("gte".equals(query_types[i])){ + boolQueryBuilder.must(QueryBuilders.rangeQuery(query_name).gte(query_name_values[i])); + } + if("lte".equals(query_types[i])){ + boolQueryBuilder.must(QueryBuilders.rangeQuery(query_name).lte(query_name_values[i])); + } + } + + AggregationBuilder aggregationBuilder = AggregationBuilders.terms(alias_name).field(group_name).size(Integer.MAX_VALUE); + + //声明BucketPath,用于后面的bucket筛选 + Map bucketsPathsMap = new HashMap<>(8); + bucketsPathsMap.put("groupCount", "_count"); + //设置脚本 + Script script = new Script("params.groupCount >= 1000"); + //构建bucket选择器 + BucketSelectorPipelineAggregationBuilder bs = + PipelineAggregatorBuilders.bucketSelector("having", bucketsPathsMap, script); + aggregationBuilder.subAggregation(bs); + sourceBuilder.aggregation(aggregationBuilder); + //不需要解释 + sourceBuilder.explain(false); + //不需要原始数据 + sourceBuilder.fetchSource(false); + //不需要版本号 + sourceBuilder.version(false); + sourceBuilder.query(boolQueryBuilder); + searchRequest.source(sourceBuilder); + + + System.out.println(sourceBuilder); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + // 查询条数 + long count = searchResponse.getHits().getHits().length; + Aggregations aggregations = searchResponse.getAggregations(); +// agg(aggregations); + Map map =new HashMap<>(); + List> list =new ArrayList<>(); + agg(list,aggregations); +// System.out.println(map); + System.out.println(list); + } + + + /** + * @Author pancm + * @Description 去重 + * @Date 2020/8/26 + * @Param [] + * @return void + **/ + private static void distinctSearch() throws IOException{ + String buk="group"; + String distinctName="name"; + AggregationBuilder aggregation = AggregationBuilders.terms("age").field("age"); + CardinalityAggregationBuilder cardinalityBuilder = AggregationBuilders.cardinality(distinctName).field(distinctName); + //根据创建时间按天分组 +// AggregationBuilder aggregation3 = AggregationBuilders.dateHistogram("createtm") +// .field("createtm") +// .format("yyyy-MM-dd") +// .dateHistogramInterval(DateHistogramInterval.DAY); +// +// aggregation2.subAggregation(aggregation3); + aggregation.subAggregation(cardinalityBuilder); + agg(aggregation,buk); + } + + + + private static void topSearch() throws IOException{ + + + } + + +} diff --git a/src/main/java/com/pancm/elasticsearch/EsHighLevelRestSearchTest.java b/src/main/java/com/pancm/elasticsearch/EsHighLevelRestSearchTest.java new file mode 100644 index 0000000..0ef3d42 --- /dev/null +++ b/src/main/java/com/pancm/elasticsearch/EsHighLevelRestSearchTest.java @@ -0,0 +1,613 @@ +package com.pancm.elasticsearch; + +import org.apache.http.HttpHost; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.avg.Avg; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.suggest.SuggestBuilders; +import org.elasticsearch.search.suggest.SuggestionBuilder; +import org.elasticsearch.search.suggest.term.TermSuggestion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; + + +/** + * @author pancm + * @Title: EsHighLevelRestSearchTest + * @Description: Java High Level REST Client Es高级客户端查询使用使用教程 (Search查询使用教程) + * 官方文档地址: + * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high.html + * @Version:1.0.0 + * @date 2019年3月12日 + */ +public class EsHighLevelRestSearchTest { + + private static String elasticIp = "192.169.0.23"; + private static int elasticPort = 9200; + private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestSearchTest.class); + + private static RestHighLevelClient client = null; + + /** + * @param args + */ + public static void main(String[] args) { + + try { + init(); + allSearch(); + //普通查询 + genSearch(); + orSearch(); + likeSearch(); +// inSearch(); + existSearch(); + rangeSearch(); + regexpSearch(); + boolSearch(); +// search(); +// search2(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + close(); + } + + /** + * test1 + *{ + * "settings" : { + * "number_of_shards" : 10, + * "refresh_interval" : "1s" + * }, + * "mappings" : { + * "_doc" : { + * "properties" : { + * "uid" : { "type" : "long" }, + * "phone" : { "type" : "long" }, + * "userid" : { "type" : "keyword" }, + * + * "message" : { "type" : "keyword" }, + * "msgcode" : { "type" : "long" }, + * "price" : { "type" : "double","index": "false" }, + * "sendtime" : { + * "type" : "date", + * "format" : "yyyy-MM-dd HH:mm:ss.SSS" + * }, + * "sendtime2" : { + * "type" : "date", + * "format" : "yyyy-MM-dd HH:mm:ss.SSS" + * }, + * "sendtm" : { "type" : "long" }, + * "sendtm2" : { "type" : "long" } + * } + * } + * } + * } + */ + + } + + + /** + * @Author pancm + * @Description 组合查询 + * @Date 2019/9/30 + * @Param [] + * @return void + **/ + private static void boolSearch() throws IOException{ + String type = "_doc"; + String index = "test1"; + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.types(type); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.must(QueryBuilders.termQuery("uid",12345)); + boolQueryBuilder.must(QueryBuilders.termQuery("msgcode",1)); + // 设置查询条件 + sourceBuilder.query(boolQueryBuilder); + searchRequest.source(sourceBuilder); + System.out.println("组合查询的DSL语句:"+sourceBuilder.toString()); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + // 结果 + searchResponse.getHits().forEach(hit -> { + String string = hit.getSourceAsString(); + System.out.println("组合查询的String结果:" + string); + }); + System.out.println("\n=================\n"); + } + + /** + * @Author pancm + * @Description 正则查询 + * @Date 2019/9/30 + * @Param [] + * @return void + **/ + private static void regexpSearch() throws IOException{ + String type = "_doc"; + String index = "test1"; + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.types(type); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + // 设置查询条件 + sourceBuilder.query(QueryBuilders.regexpQuery("message","xu[0-9]")); + searchRequest.source(sourceBuilder); + System.out.println("正则查询的DSL语句:"+sourceBuilder.toString()); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + // 结果 + searchResponse.getHits().forEach(hit -> { + Map map = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("正则查询的Map结果:" + map); + System.out.println("正则查询的String结果:" + string); + }); + + System.out.println("\n=================\n"); + } + + /** + * @Author pancm + * @Description 范围查询 + * @Date 2019/9/30 + * @Param [] + * @return void + **/ + private static void rangeSearch() throws IOException{ + String type = "_doc"; + String index = "test1"; + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.types(type); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + + // 设置查询条件 + sourceBuilder.query(QueryBuilders.rangeQuery("sendtime").gte("2019-01-01 00:00:00").lte("2019-12-31 23:59:59")); + searchRequest.source(sourceBuilder); + System.out.println("范围查询的DSL语句:"+sourceBuilder.toString()); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + // 结果 + searchResponse.getHits().forEach(hit -> { + String string = hit.getSourceAsString(); + System.out.println("范围查询的String结果:" + string); + }); + System.out.println("\n=================\n"); + } + + /** + * @return void + * @Author pancm + * @Description exist查询 + * @Date 2019/9/17 + * @Param [] + **/ + private static void existSearch() throws IOException { + String type = "_doc"; + String index = "test1"; + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.types(type); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + + // 设置查询条件 + sourceBuilder.query(QueryBuilders.existsQuery("msgcode")); + searchRequest.source(sourceBuilder); + System.out.println("存在查询的DSL语句:"+sourceBuilder.toString()); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + // 结果 + searchResponse.getHits().forEach(hit -> { + Map map = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("存在查询的Map结果:" + map); + System.out.println("存在查询的String结果:" + string); + }); + System.out.println("\n=================\n"); + } + + /** + * @return void + * @Author pancm + * @Description in查询 + * @Date 2019/9/16 + * @Param [] + **/ + private static void inSearch() throws IOException { + String type = "_doc"; + String index = "test1"; + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest(index,type); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + /** + * SELECT * FROM p_test where uid in (1,2) + * */ + // 设置查询条件 +// sourceBuilder.query(QueryBuilders.termsQuery("uid", 1, 2)); + searchRequest.source(sourceBuilder); + System.out.println("in查询的DSL语句:"+sourceBuilder.toString()); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + // 结果 + searchResponse.getHits().forEach(hit -> { + Map map = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("in查询的Map结果:" + map); + System.out.println("in查询的String结果:" + string); + }); + + System.out.println("\n=================\n"); + } + + /** + * @return void + * @Author pancm + * @Description 模糊查询 + * @Date 2019/9/12 + * @Param [] + **/ + private static void likeSearch() throws IOException { + String type = "_doc"; + String index = "test1"; + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(index); + searchRequest.types(type); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + /** + * SELECT * FROM p_test where message like '%xu%'; + * */ + boolQueryBuilder.must(QueryBuilders.wildcardQuery("message", "*xu*")); + searchSourceBuilder.query(boolQueryBuilder); + System.out.println("模糊查询语句:" + searchSourceBuilder.toString()); + searchRequest.source(searchSourceBuilder); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + searchResponse.getHits().forEach(documentFields -> { + System.out.println("模糊查询结果:" + documentFields.getSourceAsMap()); + }); + System.out.println("\n=================\n"); + } + + + /** + * @return void + * @Author pancm + * @Description 普通查询 + * @Date 2019/9/12 + * @Param [] + **/ + private static void genSearch() throws IOException { + String type = "_doc"; + String index = "test1"; + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.types(type); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + // 设置查询条件 + sourceBuilder.query(QueryBuilders.termQuery("uid", "1234")); + // 设置起止和结束 + sourceBuilder.from(0); + sourceBuilder.size(5); + sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS)); + // 设置路由 +// searchRequest.routing("routing"); + // 设置索引库表达式 + searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); + // 查询选择本地分片,默认是集群分片 + searchRequest.preference("_local"); + + // 排序 + // 根据默认值进行降序排序 +// sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); + // 根据字段进行升序排序 +// sourceBuilder.sort(new FieldSortBuilder("id").order(SortOrder.ASC)); + + // 关闭suorce查询 +// sourceBuilder.fetchSource(false); + + String[] includeFields = new String[]{"title", "user", "innerObject.*"}; + String[] excludeFields = new String[]{"_type"}; + // 包含或排除字段 +// sourceBuilder.fetchSource(includeFields, excludeFields); + + searchRequest.source(sourceBuilder); + System.out.println("普通查询的DSL语句:"+sourceBuilder.toString()); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + + // HTTP状态代码、执行时间或请求是否提前终止或超时 + RestStatus status = searchResponse.status(); + TimeValue took = searchResponse.getTook(); + Boolean terminatedEarly = searchResponse.isTerminatedEarly(); + boolean timedOut = searchResponse.isTimedOut(); + + // 供关于受搜索影响的切分总数的统计信息,以及成功和失败的切分 + int totalShards = searchResponse.getTotalShards(); + int successfulShards = searchResponse.getSuccessfulShards(); + int failedShards = searchResponse.getFailedShards(); + // 失败的原因 + for (ShardSearchFailure failure : searchResponse.getShardFailures()) { + // failures should be handled here + } + // 结果 + searchResponse.getHits().forEach(hit -> { + Map map = hit.getSourceAsMap(); + System.out.println("普通查询的结果:" + map); + }); + System.out.println("\n=================\n"); + } + + private static void allSearch() throws IOException { + /* + * 查询集群所有的索引 + * + */ + SearchRequest searchRequestAll = new SearchRequest(); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchRequestAll.source(searchSourceBuilder); + // 同步查询 + SearchResponse searchResponseAll = client.search(searchRequestAll, RequestOptions.DEFAULT); + System.out.println("所有查询总数:" + searchResponseAll.getHits().getTotalHits()); + } + + /** + * @return void + * @Author pancm + * @Description 或查询 + * @Date 2019/9/6 + * @Param [] + **/ + private static void orSearch() throws IOException { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices("test1"); + searchRequest.types("_doc"); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + BoolQueryBuilder boolQueryBuilder2 = new BoolQueryBuilder(); + /** + * SELECT * FROM test1 where (uid = 1234 or uid =12345) and phone = 12345678909 + * */ + boolQueryBuilder2.should(QueryBuilders.termQuery("uid", 1234)); + boolQueryBuilder2.should(QueryBuilders.termQuery("uid", 12345)); + boolQueryBuilder.must(boolQueryBuilder2); + boolQueryBuilder.must(QueryBuilders.termQuery("phone", "12345678909")); + searchSourceBuilder.query(boolQueryBuilder); + System.out.println("或查询语句:" + searchSourceBuilder.toString()); + searchRequest.source(searchSourceBuilder); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + searchResponse.getHits().forEach(documentFields -> { + System.out.println("或查询结果:" + documentFields.getSourceAsMap()); + }); + System.out.println("\n=================\n"); + } + + /* + * 初始化服务 + */ + private static void init() { + RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(elasticIp, elasticPort)); + client = new RestHighLevelClient(restClientBuilder); + + } + + /* + * 关闭服务 + */ + private static void close() { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + client = null; + } + } + } + + + /** + * search查询使用示例 + * + * @throws IOException + */ + private static void search() throws IOException { + + + + /* + * 全文查询使用示例 + */ + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest("user"); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + // 搜索字段user为pancm的数据 + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("user", "pancm"); + + // 设置模糊查询 + matchQueryBuilder.fuzziness(Fuzziness.AUTO); + // 设置前缀长度 + matchQueryBuilder.prefixLength(3); + // 设置最大扩展选项来控制查询的模糊过程 + matchQueryBuilder.maxExpansions(10); + + /* + * QueryBuilder也可以 + */ + +// QueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("user", "kimchy") +// .fuzziness(Fuzziness.AUTO) +// .prefixLength(3) +// .maxExpansions(10); + + SearchSourceBuilder searchSourceBuilder2 = new SearchSourceBuilder(); + searchSourceBuilder2.query(matchQueryBuilder); + + SearchRequest searchRequest2 = new SearchRequest(); + searchRequest2.source(searchSourceBuilder2); + // 同步查询 + SearchResponse searchResponse2 = client.search(searchRequest, RequestOptions.DEFAULT); + + SearchHits hits = searchResponse2.getHits(); + //总条数和分值 + long totalHits = hits.getTotalHits(); + float maxScore = hits.getMaxScore(); + + + hits.forEach(hit -> { + + String index = hit.getIndex(); + String type = hit.getType(); + String id = hit.getId(); + float score = hit.getScore(); + + + Map sourceAsMap = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("Match查询的Map结果:" + sourceAsMap); + System.out.println("Match查询的String结果:" + string); + + String documentTitle = (String) sourceAsMap.get("title"); +// List users = (List) sourceAsMap.get("user"); + Map innerObject = + (Map) sourceAsMap.get("innerObject"); + }); + + + System.out.println("\n=================\n"); + + /* + * 高亮查询 + */ + + SearchSourceBuilder searchSourceBuilder3 = new SearchSourceBuilder(); + HighlightBuilder highlightBuilder = new HighlightBuilder(); + HighlightBuilder.Field highlightTitle = new HighlightBuilder.Field("title"); + // 设置字段高亮字体 + highlightTitle.highlighterType("user"); + highlightBuilder.field(highlightTitle); + HighlightBuilder.Field highlightUser = new HighlightBuilder.Field("user"); + highlightBuilder.field(highlightUser); + searchSourceBuilder3.highlighter(highlightBuilder); + + SearchRequest searchRequest3 = new SearchRequest(); + searchRequest3.source(searchSourceBuilder3); + + // 同步查询 + SearchResponse searchResponse3 = client.search(searchRequest3, RequestOptions.DEFAULT); + + searchResponse3.getHits().forEach(hit -> { + + Map map = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("Highlight查询的Map结果:" + map); + System.out.println("Highlight查询的String结果:" + string); + }); + + System.out.println("\n=================\n"); + + /** + * 聚合查询 + */ + + SearchSourceBuilder searchSourceBuilder4 = new SearchSourceBuilder(); + + //terms 就是分组统计 根据user进行分组并创建一个新的聚合user_ + TermsAggregationBuilder aggregation = AggregationBuilders.terms("user_").field("user"); + aggregation.subAggregation(AggregationBuilders.avg("average_age").field("age")); + searchSourceBuilder4.aggregation(aggregation); + + SearchRequest searchRequest4 = new SearchRequest(); + searchRequest4.source(searchSourceBuilder4); + + // 同步查询 + SearchResponse searchResponse4 = client.search(searchRequest4, RequestOptions.DEFAULT); + //聚合查询返回条件 + Aggregations aggregations = searchResponse4.getAggregations(); + System.out.println("聚合查询"); + for (Aggregation agg : aggregations) { + String type = agg.getType(); + String name = agg.getName(); + + Terms terms = (Terms) aggregations.get(name); + for (Bucket bucket : terms.getBuckets()) { + System.out.println("条数:" + bucket.getDocCount()); + System.out.println("key:" + bucket.getKey()); + System.out.println("num:" + bucket.getKeyAsNumber()); + Avg avg = bucket.getAggregations().get("average_age"); + System.out.println("value:" + avg.getValue()); + } + if (type.equals(TermsAggregationBuilder.NAME)) { + Bucket elasticBucket = ((Terms) agg).getBucketByKey("average_age"); + long numberOfDocs = elasticBucket.getDocCount(); + System.out.println("条数:" + numberOfDocs); + } + } + + + + + + /* + * 建议查询 + */ + SearchSourceBuilder searchSourceBuilder5 = new SearchSourceBuilder(); + SuggestionBuilder termSuggestionBuilder = SuggestBuilders.termSuggestion("user").text("pancm"); + SuggestBuilder suggestBuilder = new SuggestBuilder(); + suggestBuilder.addSuggestion("suggest_user", termSuggestionBuilder); + searchSourceBuilder5.suggest(suggestBuilder); + + SearchRequest searchRequest5 = new SearchRequest(); + searchRequest5.source(searchSourceBuilder5); + + // 同步查询 + SearchResponse searchResponse5 = client.search(searchRequest5, RequestOptions.DEFAULT); + + Suggest suggest = searchResponse5.getSuggest(); + TermSuggestion termSuggestion = suggest.getSuggestion("suggest_user"); + for (TermSuggestion.Entry entry : termSuggestion.getEntries()) { + for (TermSuggestion.Entry.Option option : entry) { + String suggestText = option.getText().string(); + System.out.println("返回结果:" + suggestText); + } + } + + } + + +} diff --git a/src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest1.java b/src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest1.java new file mode 100644 index 0000000..0238621 --- /dev/null +++ b/src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest1.java @@ -0,0 +1,650 @@ +package com.pancm.elasticsearch; + +import org.apache.http.HttpHost; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.bulk.*; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.support.replication.ReplicationResponse; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.index.reindex.ScrollableHitSource; +import org.elasticsearch.index.reindex.UpdateByQueryRequest; +import org.elasticsearch.rest.RestStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; + +/** + * @Title: EsHighLevelRestTest1 + * @Description: Java High Level REST Client Es高级客户端使用教程一 (基本CRUD使用) 官方文档地址: + * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high.html + * @since jdk 1.8 + * @Version:1.0.0 + * @author pancm + * @date 2019年3月5日 + */ +public class EsHighLevelRestTest1 { + + private static String elasticIp = "192.169.0.23"; + private static int elasticPort = 9200; + + private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestTest1.class); + + private static RestHighLevelClient client = null; + + public static void main(String[] args) { + try { + init(); + createIndex(); + insert(); + queryById(); + exists(); + update(); +// deleteByQuery(); +// deleteIndex(); +// delete(); +// bulk(); + close(); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + private static void insert() throws IOException { + String index = "test1"; + String type = "_doc"; + // 唯一编号 + String id = "1"; + IndexRequest request = new IndexRequest(index, type, id); + /* + * 第一种方式,通过jsonString进行创建 + */ + // json + String jsonString = "{" + "\"uid\":\"1234\","+ "\"phone\":\"12345678909\","+ "\"msgcode\":\"1\"," + "\"sendtime\":\"2019-03-14 01:57:04\"," + + "\"message\":\"xuwujing study Elasticsearch\"" + "}"; + request.source(jsonString, XContentType.JSON); + + /* + * 第二种方式,通过map创建,,会自动转换成json的数据 + */ + Map jsonMap = new HashMap<>(); + jsonMap.put("uid", 1234); + jsonMap.put("phone", 12345678909L); + jsonMap.put("msgcode", 1); + jsonMap.put("sendtime", "2019-03-14 01:57:04"); + jsonMap.put("message", "xuwujing study Elasticsearch"); + request.source(jsonMap); + + /* + * 第三种方式 : 通过XContentBuilder对象进行创建 + */ + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.field("uid", 1234); + builder.field("phone", 12345678909L); + builder.field("msgcode", 1); + builder.timeField("sendtime", "2019-03-14 01:57:04"); + builder.field("message", "xuwujing study Elasticsearch"); + } + builder.endObject(); + request.source(builder); + + IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT); + + //如果是200则表示成功,否则就是失败 + if(200 == indexResponse.status().getStatus()){ + + } + + // 对响应结果进行处理 + String index1 = indexResponse.getIndex(); + String type1 = indexResponse.getType(); + String id1 = indexResponse.getId(); + long version = indexResponse.getVersion(); + // 如果是新增/修改的话的话 + if (indexResponse.getResult() == DocWriteResponse.Result.CREATED) { + + } else if (indexResponse.getResult() == DocWriteResponse.Result.UPDATED) { + + } + ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo(); + if (shardInfo.getTotal() != shardInfo.getSuccessful()) { + + } + if (shardInfo.getFailed() > 0) { + for (ReplicationResponse.ShardInfo.Failure failure : shardInfo.getFailures()) { + String reason = failure.reason(); + } + } + } + + /* + * 初始化服务 + */ + private static void init() { + + client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort))); + + } + + /* + * 关闭服务 + */ + private static void close() { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + /** + * 创建索引 + * + * @throws IOException + */ + private static void createIndex() throws IOException { + + // 类型 + String type = "_doc"; + String index = "test1"; + // setting 的值 + Map setmapping = new HashMap<>(); + // 分区数、副本数、缓存刷新时间 + setmapping.put("number_of_shards", 10); + setmapping.put("number_of_replicas", 1); + setmapping.put("refresh_interval", "5s"); + Map keyword = new HashMap<>(); + //设置类型 + keyword.put("type", "keyword"); + Map lon = new HashMap<>(); + //设置类型 + lon.put("type", "long"); + Map date = new HashMap<>(); + //设置类型 + date.put("type", "date"); + date.put("format", "yyyy-MM-dd HH:mm:ss"); + + Map jsonMap2 = new HashMap<>(); + Map properties = new HashMap<>(); + //设置字段message信息 + properties.put("uid", lon); + properties.put("phone", lon); + properties.put("msgcode", lon); + properties.put("message", keyword); + properties.put("sendtime", date); + Map mapping = new HashMap<>(); + mapping.put("properties", properties); + jsonMap2.put(type, mapping); + + GetIndexRequest getRequest = new GetIndexRequest(); + getRequest.indices(index); + getRequest.types(type); + getRequest.local(false); + getRequest.humanReadable(true); + boolean exists2 = client.indices().exists(getRequest, RequestOptions.DEFAULT); + //如果存在就不创建了 + if(exists2) { + System.out.println(index+"索引库已经存在!"); + return; + } + // 开始创建库 + CreateIndexRequest request = new CreateIndexRequest(index); + try { + // 加载数据类型 + request.settings(setmapping); + //设置mapping参数 + request.mapping(type, jsonMap2); + //设置别名 + request.alias(new Alias("pancm_alias")); + CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); + boolean falg = createIndexResponse.isAcknowledged(); + if(falg){ + System.out.println("创建索引库:"+index+"成功!" ); + } + } catch (IOException e) { + e.printStackTrace(); + } + + } + + + /** + * 删除索引 + * + * @throws IOException + */ + private static void deleteIndex() throws IOException { + String index = "userindex"; + DeleteIndexRequest request = new DeleteIndexRequest(index); + // 同步删除 + client.indices().delete(request,RequestOptions.DEFAULT); + System.out.println("删除索引库成功!"+index); + + } + + /** + * 查询数据 + * + * @throws IOException + */ + private static void queryById() { + String type = "_doc"; + String index = "test1"; + // 唯一编号 + String id = "1"; + // 创建查询请求 + GetRequest getRequest = new GetRequest(index, type, id); + + GetResponse getResponse = null; + try { + getResponse = client.get(getRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ElasticsearchException e) { + // 如果是索引不存在 + if (e.status() == RestStatus.NOT_FOUND) { + System.out.println("该索引库不存在!" + index); + } + + } + // 如果存在该数据则返回对应的结果 + if (getResponse.isExists()) { + long version = getResponse.getVersion(); + String sourceAsString = getResponse.getSourceAsString(); + Map sourceAsMap = getResponse.getSourceAsMap(); + byte[] sourceAsBytes = getResponse.getSourceAsBytes(); + System.out.println("查询返回结果String:" + sourceAsString); + System.out.println("查询返回结果Map:" + sourceAsMap); + } else { + System.out.println("没有找到该数据!"); + } + } + + /** + * 是否存在 + * + * @throws IOException + */ + private static void exists() throws IOException { + String type = "_doc"; + String index = "test1"; + // 唯一编号 + String id = "1"; + // 创建查询请求 + GetRequest getRequest = new GetRequest(index, type, id); + + + boolean exists = client.exists(getRequest, RequestOptions.DEFAULT); + + ActionListener listener = new ActionListener() { + @Override + public void onResponse(Boolean exists) { + System.out.println("==" + exists); + } + + @Override + public void onFailure(Exception e) { + System.out.println("失败的原因:" + e.getMessage()); + } + }; + // 进行异步监听 +// client.existsAsync(getRequest, RequestOptions.DEFAULT, listener); + + System.out.println("数据是否存在:" + exists); + } + + /** + * 更新操作 + * + * @throws IOException + */ + private static void update() throws IOException { + String type = "_doc"; + String index = "test1"; + // 唯一编号 + String id = "1"; + UpdateRequest upateRequest = new UpdateRequest(); + upateRequest.id(id); + upateRequest.index(index); + upateRequest.type(type); + + // 依旧可以使用Map这种集合作为更新条件 + Map jsonMap = new HashMap<>(); + jsonMap.put("uid", 12345); + jsonMap.put("phone", 123456789019L); + jsonMap.put("msgcode", 2); + jsonMap.put("sendtime", "2019-03-14 01:57:04"); + jsonMap.put("message", "xuwujing study Elasticsearch"); + upateRequest.doc(jsonMap); + // upsert 方法表示如果数据不存在,那么就新增一条 + upateRequest.docAsUpsert(true); + client.update(upateRequest, RequestOptions.DEFAULT); + System.out.println("更新成功!"); + + } + + + + /** + * 根据查询条件更新 + * + * @throws IOException + */ + private static void updateByQuery() throws IOException { + String type = "_doc"; + String index = "test1"; + // + UpdateByQueryRequest request = new UpdateByQueryRequest(index,type); + // 设置查询条件 + request.setQuery(new TermQueryBuilder("user", "pancm")); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + + // 设置复制文档的数量 + request.setSize(10); + // 设置一次批量处理的条数,默认是1000 + request.setBatchSize(100); + //设置超时时间 + request.setTimeout(TimeValue.timeValueMinutes(2)); + //索引选项 + request.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); + + // 同步执行 + BulkByScrollResponse bulkResponse = client.updateByQuery(request, RequestOptions.DEFAULT); + + // 异步执行 +// client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener); + + // 返回结果 + TimeValue timeTaken = bulkResponse.getTook(); + boolean timedOut = bulkResponse.isTimedOut(); + long totalDocs = bulkResponse.getTotal(); + long updatedDocs = bulkResponse.getUpdated(); + long deletedDocs = bulkResponse.getDeleted(); + long batches = bulkResponse.getBatches(); + long noops = bulkResponse.getNoops(); + long versionConflicts = bulkResponse.getVersionConflicts(); + long bulkRetries = bulkResponse.getBulkRetries(); + long searchRetries = bulkResponse.getSearchRetries(); + TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); + TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); + List searchFailures = bulkResponse.getSearchFailures(); + List bulkFailures = bulkResponse.getBulkFailures(); + System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒,总条数:" + totalDocs + ",更新数:" + updatedDocs); + + } + + /** + * 删除 + * + * @throws IOException + * + */ + private static void delete() throws IOException { + + String type = "_doc"; + String index = "test1"; + // 唯一编号 + String id = "1"; + DeleteRequest deleteRequest = new DeleteRequest(); + deleteRequest.id(id); + deleteRequest.index(index); + deleteRequest.type(type); + // 设置超时时间 + deleteRequest.timeout(TimeValue.timeValueMinutes(2)); + // 设置刷新策略"wait_for" + // 保持此请求打开,直到刷新使此请求的内容可以搜索为止。此刷新策略与高索引和搜索吞吐量兼容,但它会导致请求等待响应,直到发生刷新 + deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); + // 同步删除 + DeleteResponse deleteResponse = client.delete(deleteRequest, RequestOptions.DEFAULT); + + /* + * 异步删除操作 + */ + + // 进行监听 + ActionListener listener = new ActionListener() { + @Override + public void onResponse(DeleteResponse deleteResponse) { + System.out.println("响应:" + deleteResponse); + } + + @Override + public void onFailure(Exception e) { + System.out.println("删除监听异常:" + e.getMessage()); + } + }; + + // 异步删除 +// client.deleteAsync(deleteRequest, RequestOptions.DEFAULT, listener); + + ReplicationResponse.ShardInfo shardInfo = deleteResponse.getShardInfo(); + // 如果处理成功碎片的数量少于总碎片的情况,说明还在处理或者处理发生异常 + if (shardInfo.getTotal() != shardInfo.getSuccessful()) { + System.out.println("需要处理的碎片总量:" + shardInfo.getTotal()); + System.out.println("处理成功的碎片总量:" + shardInfo.getSuccessful()); + } + + if (shardInfo.getFailed() > 0) { + for (ReplicationResponse.ShardInfo.Failure failure : shardInfo.getFailures()) { + String reason = failure.reason(); + } + } + System.out.println("删除成功!"); + } + + + /** + * 根据查询条件删除 + * + * @throws IOException + */ + private static void deleteByQuery() throws IOException { + String type = "_doc"; + String index = "test1"; + DeleteByQueryRequest request = new DeleteByQueryRequest(index,type); + // 设置查询条件 + request.setQuery(QueryBuilders.termQuery("uid",1234)); + // 同步执行 + BulkByScrollResponse bulkResponse = client.deleteByQuery(request, RequestOptions.DEFAULT); + + // 异步执行 +// client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener); + + // 返回结果 + TimeValue timeTaken = bulkResponse.getTook(); + boolean timedOut = bulkResponse.isTimedOut(); + long totalDocs = bulkResponse.getTotal(); + long updatedDocs = bulkResponse.getUpdated(); + long deletedDocs = bulkResponse.getDeleted(); + long batches = bulkResponse.getBatches(); + long noops = bulkResponse.getNoops(); + long versionConflicts = bulkResponse.getVersionConflicts(); + long bulkRetries = bulkResponse.getBulkRetries(); + long searchRetries = bulkResponse.getSearchRetries(); + TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); + TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); + List searchFailures = bulkResponse.getSearchFailures(); + List bulkFailures = bulkResponse.getBulkFailures(); + System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒,总条数:" + totalDocs + ",更新数:" + updatedDocs); + + } + + /** + * 批量操作示例 + * + * @throws InterruptedException + */ + private static void bulk() throws IOException, InterruptedException { + String index = "estest"; + String type = "estest"; + + BulkRequest request = new BulkRequest(); + // 批量新增,存在会直接覆盖 + request.add(new IndexRequest(index, type, "1").source(XContentType.JSON, "field", "foo")); + request.add(new IndexRequest(index, type, "2").source(XContentType.JSON, "field", "bar")); + request.add(new IndexRequest(index, type, "3").source(XContentType.JSON, "field", "baz")); + + // 可以进行修改/删除/新增 操作 + //docAsUpsert 为true表示存在更新,不存在插入,为false表示不存在就是不做更新 + request.add(new UpdateRequest(index, type, "2").doc(XContentType.JSON, "field", "test").docAsUpsert(true)); + request.add(new DeleteRequest(index, type, "3")); + request.add(new IndexRequest(index, type, "4").source(XContentType.JSON, "field", "baz")); + + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + + + ActionListener listener3 = new ActionListener() { + @Override + public void onResponse(BulkResponse response) { + System.out.println("===="+response.buildFailureMessage()); + } + + @Override + public void onFailure(Exception e) { + System.out.println("====---"+e.getMessage()); + } + }; + + client.bulkAsync(request, RequestOptions.DEFAULT,listener3); + + // 可以快速检查一个或多个操作是否失败 true是有至少一个失败! + if (bulkResponse.hasFailures()) { + System.out.println("有一个操作失败!"); + } + + // 对处理结果进行遍历操作并根据不同的操作进行处理 + for (BulkItemResponse bulkItemResponse : bulkResponse) { + DocWriteResponse itemResponse = bulkItemResponse.getResponse(); + + // 操作失败的进行处理 + if (bulkItemResponse.isFailed()) { + BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); + + } + + if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.INDEX + || bulkItemResponse.getOpType() == DocWriteRequest.OpType.CREATE) { + IndexResponse indexResponse = (IndexResponse) itemResponse; + System.out.println("新增失败!"+indexResponse.toString()); + + } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.UPDATE) { + UpdateResponse updateResponse = (UpdateResponse) itemResponse; + System.out.println("更新失败!"+updateResponse.toString()); + + } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.DELETE) { + DeleteResponse deleteResponse = (DeleteResponse) itemResponse; + System.out.println("删除失败!"+deleteResponse.toString()); + + } + } + + System.out.println("批量执行成功!"); + + /* + * 批量执行处理器相关示例代码 + */ + + // 批量处理器的监听器设置 + + BulkProcessor.Listener listener = new BulkProcessor.Listener() { + + // 在执行BulkRequest的每次执行之前调用,这个方法允许知道将要在BulkRequest中执行的操作的数量 + @Override + public void beforeBulk(long executionId, BulkRequest request) { + int numberOfActions = request.numberOfActions(); + logger.debug("Executing bulk [{}] with {} requests", executionId, numberOfActions); + } + + // 在每次执行BulkRequest之后调用,这个方法允许知道BulkResponse是否包含错误 + @Override + public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + if (response.hasFailures()) { + logger.warn("Bulk [{}] executed with failures", executionId); + } else { + logger.debug("Bulk [{}] completed in {} milliseconds", executionId, response.getTook().getMillis()); + } + } + + // 如果BulkRequest失败,则调用该方法,该方法允许知道失败 + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + logger.error("Failed to execute bulk", failure); + } + }; + + BiConsumer> bulkConsumer = (request2, bulkListener) -> client + .bulkAsync(request, RequestOptions.DEFAULT, bulkListener); + // 创建一个批量执行的处理器 + BulkProcessor bulkProcessor = BulkProcessor.builder(bulkConsumer, listener).build(); + BulkProcessor.Builder builder = BulkProcessor.builder(bulkConsumer, listener); + // 根据当前添加的操作数量设置刷新新批量请求的时间(默认为1000,使用-1禁用它) + builder.setBulkActions(500); + // 根据当前添加的操作大小设置刷新新批量请求的时间(默认为5Mb,使用-1禁用) + builder.setBulkSize(new ByteSizeValue(1L, ByteSizeUnit.MB)); + // 设置允许执行的并发请求数量(默认为1,使用0只允许执行单个请求) + builder.setConcurrentRequests(0); + // 设置刷新间隔如果间隔通过,则刷新任何挂起的BulkRequest(默认为未设置) + builder.setFlushInterval(TimeValue.timeValueSeconds(10L)); + // 设置一个常量后退策略,该策略最初等待1秒并重试最多3次。 + builder.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3)); + + IndexRequest one = new IndexRequest(index, type, "1").source(XContentType.JSON, "title", + "In which order are my Elasticsearch queries executed?"); + IndexRequest two = new IndexRequest(index, type, "2").source(XContentType.JSON, "title", + "Current status and upcoming changes in Elasticsearch"); + IndexRequest three = new IndexRequest(index, type, "3").source(XContentType.JSON, "title", + "The Future of Federated Search in Elasticsearch"); + bulkProcessor.add(one); + bulkProcessor.add(two); + bulkProcessor.add(three); + + // 如果所有大容量请求都已完成,则该方法返回true;如果在所有大容量请求完成之前的等待时间已经过去,则返回false + boolean terminated = bulkProcessor.awaitClose(30L, TimeUnit.SECONDS); + + System.out.println("请求的响应结果:" + terminated); + + } + +} diff --git a/src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest2.java b/src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest2.java new file mode 100644 index 0000000..a42595b --- /dev/null +++ b/src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest2.java @@ -0,0 +1,289 @@ +package com.pancm.elasticsearch; + +import org.apache.http.HttpHost; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetItemResponse; +import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.RethrottleRequest; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.index.reindex.ReindexRequest; +import org.elasticsearch.index.reindex.ScrollableHitSource; +import org.elasticsearch.tasks.TaskId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertNull; + +/** + * @Title: EsHighLevelRestTest2 + * @Description: Java High Level REST Client Es高级客户端使用教程二 (关于组合使用) + * @since jdk 1.8 + * @Version:1.0.0 + * @author pancm + * @date 2019年3月11日 + */ +public class EsHighLevelRestTest2 { + + private static String elasticIp = "192.169.0.23"; + private static int elasticPort = 9200; + + private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestTest2.class); + + private static RestHighLevelClient client = null; + + /** + * @param args + */ + public static void main(String[] args) { + try { + init(); + multiGet(); + reindex(); + deleteByQuery(); + rethrottleByQuery(); + close(); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + /* + * 初始化服务 + */ + private static void init() { + client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort, "http"))); + + } + + /* + * 关闭服务 + */ + private static void close() { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + /** + * 多查询使用 + * + * @throws IOException + */ + private static void multiGet() throws IOException { + + MultiGetRequest request = new MultiGetRequest(); + request.add(new MultiGetRequest.Item("estest", "estest", "1")); + request.add(new MultiGetRequest.Item("user", "userindex", "2")); + // 禁用源检索,默认启用 +// request.add(new MultiGetRequest.Item("user", "userindex", "2").fetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE)); + // 同步构建 + MultiGetResponse response = client.mget(request, RequestOptions.DEFAULT); + // 异步构建 +// MultiGetResponse response2 = client.mgetAsync(request, RequestOptions.DEFAULT, listener); + /* + * 返回的MultiGetResponse包含在' getResponses中的MultiGetItemResponse的列表,其顺序与请求它们的顺序相同。 + * 如果成功,MultiGetItemResponse包含GetResponse或MultiGetResponse。如果失败了就失败。 + * 成功看起来就像一个正常的GetResponse + */ + + for (MultiGetItemResponse item : response.getResponses()) { + assertNull(item.getFailure()); + GetResponse get = item.getResponse(); + String index = item.getIndex(); + String type = item.getType(); + String id = item.getId(); + // 如果请求存在 + if (get.isExists()) { + long version = get.getVersion(); + String sourceAsString = get.getSourceAsString(); + Map sourceAsMap = get.getSourceAsMap(); + byte[] sourceAsBytes = get.getSourceAsBytes(); + System.out.println("查询的结果:" + sourceAsMap); + } else { + System.out.println("没有找到该文档!"); + } + + } + + } + + /** + * 索引复制 + * + * @throws IOException + */ + private static void reindex() throws IOException { + // 创建索引复制请求并进行索引复制 + ReindexRequest request = new ReindexRequest(); + // 需要复制的索引 + request.setSourceIndices("user"); + // 复制的目标索引 + request.setDestIndex("dest_test"); + + // 表示如果在复制索引的时候有缺失的文档的话会进行创建,默认是index + request.setDestOpType("create"); + // 如果在复制的过程中发现版本冲突,那么会继续进行复制 + request.setConflicts("proceed"); + + // 只复制文档类型为 userindex 的数据 + request.setSourceDocTypes("userindex"); + // 只复制 pancm 用户的数据 + request.setSourceQuery(new TermQueryBuilder("user", "pancm")); + // 设置复制文档的数量 + request.setSize(10); + // 设置一次批量处理的条数,默认是1000 + request.setSourceBatchSize(100); + + // 进行排序 +// request.addSortField("postDate", SortOrder.DESC); + + // 指定切片大小 + request.setSlices(2); + + //设置超时时间 + request.setTimeout(TimeValue.timeValueMinutes(2)); + //允许刷新 + request.setRefresh(true); + + // 同步执行 + BulkByScrollResponse bulkResponse = client.reindex(request, RequestOptions.DEFAULT); + + // 异步执行 +// client.reindexAsync(request, RequestOptions.DEFAULT, listener); + + // 响应结果处理 + + TimeValue timeTaken = bulkResponse.getTook(); + boolean timedOut = bulkResponse.isTimedOut(); + long totalDocs = bulkResponse.getTotal(); + long updatedDocs = bulkResponse.getUpdated(); + long createdDocs = bulkResponse.getCreated(); + long deletedDocs = bulkResponse.getDeleted(); + long batches = bulkResponse.getBatches(); + long noops = bulkResponse.getNoops(); + long versionConflicts = bulkResponse.getVersionConflicts(); + long bulkRetries = bulkResponse.getBulkRetries(); + long searchRetries = bulkResponse.getSearchRetries(); + TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); + TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); + List searchFailures = bulkResponse.getSearchFailures(); + List bulkFailures = bulkResponse.getBulkFailures(); + + System.out.println("索引复制总共花费了:" + timeTaken.getMillis() + " 毫秒,总条数:" + totalDocs + ",创建数:" + createdDocs + + ",更新数:" + updatedDocs); + } + + + + + + /** + * 根据查询条件删除 + * @throws IOException + */ + private static void deleteByQuery() throws IOException { + // + DeleteByQueryRequest request = new DeleteByQueryRequest("user"); + + // 设置查询条件 + request.setQuery(new TermQueryBuilder("user", "pancm")); + + // 设置复制文档的数量 + request.setSize(10); + // 设置一次批量处理的条数,默认是1000 + request.setBatchSize(100); + //设置路由 + request.setRouting("=cat"); + //设置超时时间 + request.setTimeout(TimeValue.timeValueMinutes(2)); + //允许刷新 + request.setRefresh(true); + //索引选项 + request.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); + + // 同步执行 + BulkByScrollResponse bulkResponse = client.deleteByQuery(request, RequestOptions.DEFAULT); + + // 异步执行 +// client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener); + + // 返回结果 + TimeValue timeTaken = bulkResponse.getTook(); + boolean timedOut = bulkResponse.isTimedOut(); + long totalDocs = bulkResponse.getTotal(); + long deletedDocs = bulkResponse.getDeleted(); + long batches = bulkResponse.getBatches(); + long noops = bulkResponse.getNoops(); + long versionConflicts = bulkResponse.getVersionConflicts(); + long bulkRetries = bulkResponse.getBulkRetries(); + long searchRetries = bulkResponse.getSearchRetries(); + TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); + TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); + List searchFailures = bulkResponse.getSearchFailures(); + List bulkFailures = bulkResponse.getBulkFailures(); + System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒,总条数:" + totalDocs + ",删除数:" + deletedDocs); + + } + + + /** + * 用于更改正在运行的重索引、逐查询更新或逐查询删除任务的当前节流,或完全禁用任务的节流。 + * @throws IOException + */ + private static void rethrottleByQuery() throws IOException { + TaskId taskId=new TaskId("1"); + //用于更改正在运行的重索引、逐查询更新或逐查询删除任务的当前节流,或完全禁用任务的节流。 + //并且将请求将任务的节流更改为每秒100个请求 + RethrottleRequest request = new RethrottleRequest(taskId,100.0f); + + // 同步设置需要更改的流 +// client.reindexRethrottle(request, RequestOptions.DEFAULT); +// client.updateByQueryRethrottle(request, RequestOptions.DEFAULT); +// client.deleteByQueryRethrottle(request, RequestOptions.DEFAULT); + + + ActionListener listener = new ActionListener() { + @Override + public void onResponse(ListTasksResponse response) { + System.out.println("===="+response.getTasks().toString()); + } + + @Override + public void onFailure(Exception e) { + System.out.println("====---"+e.getMessage()); + } + }; + + // 异步设置要更改的流 + client.reindexRethrottleAsync(request, RequestOptions.DEFAULT, listener); + client.updateByQueryRethrottleAsync(request, RequestOptions.DEFAULT, listener); + client.deleteByQueryRethrottleAsync(request, RequestOptions.DEFAULT, listener); + + System.out.println("已成功设置!"); + + } + + + + +} diff --git a/src/main/java/com/pancm/elasticsearch/EsUtil.java b/src/main/java/com/pancm/elasticsearch/EsUtil.java new file mode 100644 index 0000000..31b2dcc --- /dev/null +++ b/src/main/java/com/pancm/elasticsearch/EsUtil.java @@ -0,0 +1,1043 @@ +package com.pancm.elasticsearch; + + +import org.apache.http.HttpHost; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.index.reindex.ReindexRequest; +import org.elasticsearch.index.reindex.UpdateByQueryRequest; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.SortOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.Serializable; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/** + * @author pancm + * @Title: EsUtil + * @Description: ES的工具类 + * @Version:1.0.0 + * @date 2019年3月19日 + */ +public final class EsUtil { + private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestSearchTest.class); + + private EsUtil() { + + } + + + /** + * @param args + * @throws IOException + */ + public static void main(String[] args) { + + try { + + EsUtil.build("192.169.0.23:9200"); + System.out.println("ES连接初始化成功!"); +// createIndexTest(); +// System.out.println("ES索引库创建成功!"); + String index = "student"; + String type = "_doc"; + List> list = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + Map map = new HashMap<>(); + map.put("id", i); + map.put("name", "张三" + i); + map.put("age", 10 + i); + list.add(map); + } + EsUtil.setIsAutoClose(false); + saveBulk(list, index, type, "id"); + System.out.println("批量写入成功!"); + System.out.println("查询的结果1:" + queryById(index, type, "1")); + QueryBuilder queryBuilder = new TermQueryBuilder("name", "xuwujing"); + System.out.println("更新的结果:" + updateByQuery(index, type, queryBuilder)); + System.out.println("查询的结果2:" + queryById(index, type, "1")); + QueryBuilder queryBuilder3 = QueryBuilders.matchAllQuery(); + System.out.println("查询的结果3:" + query(index, type, queryBuilder3)); + QueryBuilder queryBuilder4 = QueryBuilders.rangeQuery("age").from(15); + QueryBuilder queryBuilder5 = QueryBuilders.rangeQuery("id").from(5); + System.out.println("查询的结果4:" + query(index, type, queryBuilder4,queryBuilder5)); + EsQueryCondition esQueryCondition = new EsQueryCondition(); + esQueryCondition.setCloseSource(true); + esQueryCondition.setIndex(1); + esQueryCondition.setPagesize(3); + esQueryCondition.setOrder("desc"); + esQueryCondition.setOrderField(new String[]{"age"}); + String [] incStrings = new String[]{"age","name"}; + esQueryCondition.setIncludeFields(incStrings); + esQueryCondition.setExcludeFields(new String[]{"id"}); + System.out.println("查询的结果5:" + query(index, type,esQueryCondition, queryBuilder4)); + + + // TODO: + + } catch (IOException e) { + e.printStackTrace(); + + } finally { + // TODO: handle finally clause + try { + close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + + private static void createIndexTest() throws IOException { + // setting 的值 + Map setmapping = new HashMap<>(); + + // 分区数、路由分片数、副本数、缓存刷新时间 + setmapping.put("number_of_shards", 12); + setmapping.put("number_of_routing_shards", 24); + setmapping.put("number_of_replicas", 1); + setmapping.put("refresh_interval", "5s"); + + String index = "test5"; + String type = "test5"; + String alias = "test"; + + Map jsonMap2 = new HashMap<>(); + Map message = new HashMap<>(); + // 设置类型 + message.put("type", "text"); + Map properties = new HashMap<>(); + // 设置字段message信息 + properties.put("msg", message); + Map mapping = new HashMap<>(); + mapping.put("properties", properties); + jsonMap2.put(type, mapping); + + String mappings = jsonMap2.toString(); + + EsBasicModelConfig esBasicModelConfig = new EsBasicModelConfig(); + esBasicModelConfig.setIndex(index); + esBasicModelConfig.setType(type); + esBasicModelConfig.setMappings(mappings); + esBasicModelConfig.setSettings(setmapping); + esBasicModelConfig.setAlias(alias); + + EsUtil.creatIndex(esBasicModelConfig); + } + + + /** + * 创建链接 + * + * @param nodes + * @return + */ + public static void build(String... nodes) { + Objects.requireNonNull(nodes, "hosts can not null"); + ArrayList ahosts = new ArrayList(); + for (String host : nodes) { + IpHandler addr = new IpHandler(); + addr.IpPortFromUrl(host); + ahosts.add(new HttpHost(addr.getIp(), addr.getPort())); + } + httpHosts = ahosts.toArray(new HttpHost[0]); + init(); + } + + + /** + * @return boolean + * @Author pancm + * @Description //创建索引库(指定Mpping类型) + * @Date 2019/3/21 + * @Param [esBasicModelConfig] + **/ + public static boolean creatIndex(EsBasicModelConfig esBasicModelConfig) throws IOException { + boolean falg = true; + Objects.requireNonNull(esBasicModelConfig, "esBasicModelConfig is not null"); + String type = Objects.requireNonNull(esBasicModelConfig.getType(), "type is not null"); + String index = Objects.requireNonNull(esBasicModelConfig.getIndex(), "index is not null"); + if (exitsIndex(index)) { + logger.warn("索引库{}已经存在!无需在进行创建!", index); + return true; + } + String mapping = esBasicModelConfig.getMappings(); + Map setting = esBasicModelConfig.getSettings(); + String alias = esBasicModelConfig.getAlias(); + // 开始创建库 + CreateIndexRequest request = new CreateIndexRequest(index); + try { + if (Objects.nonNull(mapping)) { + // 加载数据类型 + request.mapping(type, mapping); + } + if (Objects.nonNull(setting)) { + // 分片数 + request.settings(setting); + } + if (Objects.nonNull(alias)) { + // 别名 + request.alias(new Alias(alias)); + } + + CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); + falg = createIndexResponse.isAcknowledged(); + } catch (IOException e) { + throw e; + } finally { + if (isAutoClose) { + close(); + } + } + return falg; + + } + + + /** + * 判断索引库是否存在 + * + * @param index + * @return + * @throws IOException + */ + public static boolean exitsIndex(String index) throws IOException { + try { + GetIndexRequest getRequest = new GetIndexRequest(); + getRequest.indices(index); + getRequest.local(false); + getRequest.humanReadable(true); + return client.indices().exists(getRequest, RequestOptions.DEFAULT); + } finally { + if (isAutoClose) { + close(); + } + } + } + + + /** + * @return boolean + * @Author pancm + * @Description 单条新增/更新数据 + * @Date 2019/6/5 + * @Param [mapList, index, type] + **/ + public static boolean save(Map map, String index, String type) throws IOException { + List> mapList = new ArrayList<>(); + mapList.add(map); + return saveBulk(mapList, index, type, null); + } + + /** + * @return boolean + * @Author pancm + * @Description 批量新增/更新数据 + * @Date 2019/6/5 + * @Param [mapList, index, type] + **/ + public static boolean saveBulk(List> mapList, String index, String type) throws IOException { + return saveBulk(mapList, index, type, null); + } + + /** + * @return boolean + * @Author pancm + * @Description 批量新增/更新数据 + * @Date 2019/3/21 + * @Param [mapList:存储参数, index:索引库名, type:索引库类型,key:存储的主键,为空表示使用ES主键] + **/ + public static boolean saveBulk(List> mapList, String index, String type, String key) throws IOException { + + if (mapList == null || mapList.size() == 0) { + return true; + } + if (index == null || index.trim().length() == 0 || type == null || type.trim().length() == 0) { + return false; + } + try { + BulkRequest request = new BulkRequest(); + mapList.forEach(map -> { + if (key != null) { + String id = map.get(key) + ""; + if (id == null || id.trim().length() == 0) { + request.add(new IndexRequest(index, type).source(map, XContentType.JSON)); + } else { + request.add(new IndexRequest(index, type, id).source(map, XContentType.JSON)); + } + } else { + request.add(new IndexRequest(index, type).source(map, XContentType.JSON)); + } + }); + + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + //说明至少有一个失败了,这里就直接返回false + if (bulkResponse.hasFailures()) { + return false; + } + + return true; + } finally { + if (isAutoClose) { + close(); + } + } + } + + + /** + * @return boolean + * @Author pancm + * @Description //删除数据 + * 根据ID进行单条删除 + * @Date 2019/3/21 + * @Param [] + **/ + public static boolean deleteById(String index, String type, String id) throws IOException { + if (index == null || type == null || id == null) { + return true; + } + try { + DeleteRequest deleteRequest = new DeleteRequest(); + deleteRequest.id(id); + deleteRequest.index(index); + deleteRequest.type(type); + // 同步删除 + client.delete(deleteRequest, RequestOptions.DEFAULT); + } finally { + if (isAutoClose) { + close(); + } + } + return true; + } + + + /** + * @return boolean + * @Author pancm + * @Description //批量删除数据 + * 根据ID进行批量删除 + * @Date 2019/3/21 + * @Param [] + **/ + public static boolean deleteByIds(String index, String type, Set ids) throws IOException { + if (index == null || type == null || ids == null) { + return true; + } + try { + BulkRequest requestBulk = new BulkRequest(); + ids.forEach(id -> { + DeleteRequest deleteRequest = new DeleteRequest(index, type, id); + requestBulk.add(deleteRequest); + }); + // 同步删除 + client.bulk(requestBulk, RequestOptions.DEFAULT); + } finally { + if (isAutoClose) { + close(); + } + } + return false; + } + + /** + * @return boolean + * @Author pancm + * @Description 根据id查询 + * @Date 2019/3/21 + * @Param [] + **/ + public static Map queryById(String index, String type, String id) throws IOException { + if (index == null || type == null) { + return null; + } + Map map = new HashMap<>(); + try { + GetRequest request = new GetRequest(); + request.index(index); + request.type(type); + request.id(id); + GetResponse getResponse = client.get(request, RequestOptions.DEFAULT); + // 如果存在该数据则返回对应的结果 + if (getResponse.isExists()) { + map = getResponse.getSourceAsMap(); + } + } finally { + if (isAutoClose) { + close(); + } + } + return map; + } + + + public static List> query(String index, String type, QueryBuilder... queryBuilders) throws IOException { + return query(index, type, null , queryBuilders); + } + + /** + * @return boolean + * @Author pancm + * @Description 根据条件查询 + * @Date 2019/3/21 + * @Param [] + **/ + public static List> query(String index, String type, EsQueryCondition esQueryCondition, QueryBuilder... queryBuilders) throws IOException { + if (index == null || type == null) { + return null; + } + List> list = new ArrayList<>(); + try { + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.types(type); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + + if (esQueryCondition != null) { + Integer form = esQueryCondition.getIndex(); + Integer pagesize = esQueryCondition.getPagesize(); + if (form != null && form > 0 && pagesize != null && pagesize > 0) { + form = (form - 1) * pagesize; + pagesize = form + pagesize; + // 设置起止和结束 + sourceBuilder.from(form); + sourceBuilder.size(pagesize); + } + String routing = esQueryCondition.getRouting(); + if (routing != null && routing.length() > 0) { + // 设置路由 + searchRequest.routing(routing); + } + + // 设置索引库表达式 + searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); + + //设置排序 + String order = esQueryCondition.getOrder(); + if (order != null) { + String[] orderField = esQueryCondition.getOrderField(); + SortOrder order2 = order.equals(SortOrder.DESC) ? SortOrder.DESC : SortOrder.ASC; + //如果设置了排序字段则用排序的字段进行排序,否则就默认排序 + if (orderField != null) { + for (String field : orderField) { + sourceBuilder.sort(new FieldSortBuilder(field).order(order2)); + } + } else { + sourceBuilder.sort(new ScoreSortBuilder().order(order2)); + } + } + String[] includeFields = esQueryCondition.getIncludeFields(); + String[] excludeFields = esQueryCondition.getExcludeFields(); + if (includeFields != null && includeFields.length > 0 && excludeFields != null && excludeFields.length > 0) { + sourceBuilder.fetchSource(includeFields, excludeFields); + } + sourceBuilder.fetchSource(esQueryCondition.isCloseSource()); + } + //设置条件 + if (queryBuilders != null) { + for (QueryBuilder queryBuilder : queryBuilders) { + sourceBuilder.query(queryBuilder); + } + } + + searchRequest.source(sourceBuilder); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + + if(queryBuilders != null|| (esQueryCondition != null && esQueryCondition.isQueryData())){ + // 结果 + searchResponse.getHits().forEach(hit -> { + Map map = hit.getSourceAsMap(); + list.add(map); + }); + } + + if(esQueryCondition != null && esQueryCondition.isNeedTotal()){ + Map mapTotal = new HashMap<>(); + mapTotal.put("total", searchResponse.getHits().getTotalHits()); + list.add(mapTotal); + } + + } finally { + if (isAutoClose) { + close(); + } + } + return list; + } + + + /** + * @return boolean + * @Author pancm + * @Description 根据条件更新 + * @Date 2019/3/21 + * @Param [] + **/ + public static Map updateByQuery(String index, String type, QueryBuilder... queryBuilders) throws IOException { + if (index == null || type == null) { + return null; + } + Map map = new HashMap<>(); + try { + UpdateByQueryRequest request = new UpdateByQueryRequest(); + request.indices(index); + request.setDocTypes(type); + + if (queryBuilders != null) { + for (QueryBuilder queryBuilder : queryBuilders) { + request.setQuery(queryBuilder); + } + } + // 同步执行 + BulkByScrollResponse bulkResponse = client.updateByQuery(request, RequestOptions.DEFAULT); + + // 响应结果处理 + map.put("time", bulkResponse.getTook().getMillis()); + map.put("total", bulkResponse.getTotal()); + + } finally { + if (isAutoClose) { + close(); + } + } + return map; + } + + /** + * @return Map + * @Author pancm + * @Description //根据条件删除数据 + * @Date 2019/3/21 + * @Param [] + **/ + public static Map deleteByQuery(String index, String type, QueryBuilder[] queryBuilders) throws IOException { + if (index == null || type == null || queryBuilders == null) { + return null; + } + Map map = new HashMap<>(); + try { + DeleteByQueryRequest request = new DeleteByQueryRequest(index, type); + if (queryBuilders != null) { + for (QueryBuilder queryBuilder : queryBuilders) { + request.setQuery(queryBuilder); + } + } + // 同步执行 + BulkByScrollResponse bulkResponse = client.deleteByQuery(request, RequestOptions.DEFAULT); + // 响应结果处理 + map.put("time", bulkResponse.getTook().getMillis()); + map.put("total", bulkResponse.getTotal()); + + } finally { + if (isAutoClose) { + close(); + } + } + return map; + } + + + /** + * @return Map + * @Author pancm + * @Description //重索引 + * @Date 2019/3/21 + * @Param [] + **/ + public static Map reindexByQuery(String index, String destIndex, QueryBuilder[] queryBuilders) throws IOException { + if (index == null || destIndex == null) { + return null; + } + Map map = new HashMap<>(); + try { + // 创建索引复制请求并进行索引复制 + ReindexRequest request = new ReindexRequest(); + // 需要复制的索引 + request.setSourceIndices(index); + /* 复制的目标索引 */ + request.setDestIndex(destIndex); + if (queryBuilders != null) { + for (QueryBuilder queryBuilder : queryBuilders) { + request.setSourceQuery(queryBuilder); + } + } + // 表示如果在复制索引的时候有缺失的文档的话会进行创建,默认是index + request.setDestOpType("create"); + // 如果在复制的过程中发现版本冲突,那么会继续进行复制 + request.setConflicts("proceed"); + + + // 设置复制文档的数量 + // request.setSize(10); + // 设置一次批量处理的条数,默认是1000 + // request.setSourceBatchSize(10000); + + //设置超时时间 + request.setTimeout(TimeValue.timeValueMinutes(2)); + // 同步执行 + BulkByScrollResponse bulkResponse = client.reindex(request, RequestOptions.DEFAULT); + + // 响应结果处理 + map.put("time", bulkResponse.getTook().getMillis()); + map.put("total", bulkResponse.getTotal()); + map.put("createdDocs", bulkResponse.getCreated()); + map.put("updatedDocs", bulkResponse.getUpdated()); + + } finally { + if (isAutoClose) { + close(); + } + } + return map; + } + + + /* + * 初始化服务 + */ + private static void init() { + if (client == null) { + synchronized (EsUtil.class) { + if (client == null) { + RestClientBuilder restClientBuilder = RestClient.builder(httpHosts); + client = new RestHighLevelClient(restClientBuilder); + } + } + } + + } + + /* + * 关闭服务 + */ + private static void close() throws IOException { + if (client != null) { + try { + client.close(); + setIsAutoClose(true); + } catch (IOException e) { + throw e; + } + } + } + + public static boolean isIsAutoClose() { + return isAutoClose; + } + + public static void setIsAutoClose(boolean isAutoClose) { + EsUtil.isAutoClose = isAutoClose; + } + + + private static String[] elasticIps; + private static int elasticPort; + private static HttpHost[] httpHosts; + private static volatile RestHighLevelClient client = null; + /** + * 是否自动关闭连接 + */ + private static boolean isAutoClose = true; + + private static final String COMMA_SIGN = ","; + + +} + +/** + * @Author pancm + * @Description 查询条件的类 + * @Date 2019/6/19 + * @Param + * @return + **/ +class EsQueryCondition implements Serializable { + private static final long serialVersionUID = 1L; + + /** + * 下标和条数 都为null或小于0表示不分页 + */ + private Integer index; + private Integer pagesize; + + /** + * 排序规则 asc:升序,desc:降序,为空表示不排序 + */ + private String order; + /** + * 排序字段 + */ + private String[] orderField; + + /** + * 路由 为空表示不设置 + */ + private String routing; + + /** + * 返回的字段 + */ + private String[] includeFields; + /** + * 排除的字段 + */ + private String[] excludeFields; + + /** + * 是否关闭source查询 + */ + private boolean isCloseSource; + + /** 是否需要查询数据 */ + private boolean isQueryData = true; + + /** 是否需要 返回总数 */ + private boolean isNeedTotal = true; + + + public boolean isQueryData() { + return isQueryData; + } + + public void setQueryData(boolean queryData) { + isQueryData = queryData; + } + + public boolean isNeedTotal() { + return isNeedTotal; + } + + public void setNeedTotal(boolean needTotal) { + isNeedTotal = needTotal; + } + + public boolean isCloseSource() { + return isCloseSource; + } + + public void setCloseSource(boolean closeSource) { + isCloseSource = closeSource; + } + + public String[] getIncludeFields() { + return includeFields; + } + + public void setIncludeFields(String[] includeFields) { + this.includeFields = includeFields; + } + + public String[] getExcludeFields() { + return excludeFields; + } + + public void setExcludeFields(String[] excludeFields) { + this.excludeFields = excludeFields; + } + + public Integer getIndex() { + return index; + } + + public void setIndex(Integer index) { + this.index = index; + } + + public Integer getPagesize() { + return pagesize; + } + + public void setPagesize(Integer pagesize) { + this.pagesize = pagesize; + } + + public String getOrder() { + return order; + } + + public void setOrder(String order) { + this.order = order; + } + + public String getRouting() { + return routing; + } + + public void setRouting(String routing) { + this.routing = routing; + } + + public String[] getOrderField() { + return orderField; + } + + public void setOrderField(String[] orderField) { + this.orderField = orderField; + } +} + + +/* + * ES的mapping创建的基础类 + */ +class EsBasicModelConfig implements Serializable { + private static final long serialVersionUID = 1L; + /*** 索引库 ***/ + private String index; + private String type; + private Map settings; + private String mappings; + private String alias; + + public EsBasicModelConfig() { + } + + public EsBasicModelConfig(String index, String type) { + this.index = index; + this.type = type; + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Map getSettings() { + return settings; + } + + public void setSettings(Map settings) { + this.settings = settings; + } + + public void setSettings(SettingEntity settings) { + this.settings = Objects.requireNonNull(settings, "setting can not null").toDSL(); + } + + public String getMappings() { + return mappings; + } + + public void setMappings(String mappings) { + this.mappings = mappings; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + @Override + public String toString() { + return "EsBasicModelConfig [index=" + index + ", type=" + type + ", settings=" + settings + ", mappings=" + + mappings + "]"; + } + +} + +/* + * setting 实体类的配置 + */ +class SettingEntity implements Serializable { + /** + * @Fields serialVersionUID : TODO + */ + private static final long serialVersionUID = 1L; + // 默认分片数 + private int numberOfShards = 5; + // 分片路由数 + private int number_of_routing_shards = 30; + // 副本数 + private int numberOfReplicas = 1; + /***** 刷新频率 单位:秒 *********/ + private int refreshInterval = 5; + /** + * 查询最大返回的时间 + */ + private int maxResultWindow = 10000; + + public SettingEntity(int numberOfShards, int numberOfReplicas, int refreshInterval) { + this.numberOfShards = numberOfShards; + this.numberOfReplicas = numberOfReplicas; + this.refreshInterval = refreshInterval; + } + + public SettingEntity(int numberOfShards, int numberOfReplicas, int refreshInterval, int number_of_routing_shards, + int maxResultWindow, String alias) { + this.numberOfShards = numberOfShards; + this.numberOfReplicas = numberOfReplicas; + this.refreshInterval = refreshInterval; + this.number_of_routing_shards = number_of_routing_shards; + this.maxResultWindow = maxResultWindow; + } + + public SettingEntity() { + + } + + public int getNumberOfShards() { + return numberOfShards; + } + + /** + * 分片数 + * + * @param numberOfShards 默认5 + */ + public void setNumberOfShards(int numberOfShards) { + this.numberOfShards = numberOfShards; + } + + public int getNumberOfReplicas() { + return numberOfReplicas; + } + + /** + * 副本数 + * + * @param numberOfReplicas 默认1 + */ + public void setNumberOfReplicas(int numberOfReplicas) { + this.numberOfReplicas = numberOfReplicas; + } + + public int getRefreshInterval() { + return refreshInterval; + } + + public int getNumber_of_routing_shards() { + return number_of_routing_shards; + } + + public void setNumber_of_routing_shards(int number_of_routing_shards) { + this.number_of_routing_shards = number_of_routing_shards; + } + + public int getMaxResultWindow() { + return maxResultWindow; + } + + public void setMaxResultWindow(int maxResultWindow) { + this.maxResultWindow = maxResultWindow; + } + + /** + * 刷新频率 单位:秒 + * + * @param refreshInterval 默认5秒 设置为-1为无限刷新 + */ + public void setRefreshInterval(int refreshInterval) { + if (refreshInterval < -1) { + refreshInterval = -1; + } + this.refreshInterval = refreshInterval; + } + + public Map toDSL() { + Map json = new HashMap<>(); + json.put("number_of_shards", numberOfShards); + json.put("number_of_routing_shards", number_of_routing_shards); + json.put("number_of_replicas", numberOfReplicas); + json.put("refresh_interval", refreshInterval + "s"); + json.put("max_result_window", maxResultWindow); + return json; + } + + @Override + public String toString() { + return "SettingEntity [numberOfShards=" + numberOfShards + ", numberOfReplicas=" + numberOfReplicas + + ", refreshInterval=" + refreshInterval + ", maxResultWindow=" + maxResultWindow + "]"; + } + +} + +class IpHandler { + + private String ip; + private Integer port; + private static Pattern p = Pattern.compile("(?<=//|)((\\w)+\\.)+\\w+(:\\d{0,5})?"); + + /** + * 冒号 + */ + private static final String COMMA_COLON = ":"; + + /** + * 从url中分析出hostIP:PORT
+ * + * @param url + */ + public void IpPortFromUrl(String url) { + + String host = ""; + + Matcher matcher = p.matcher(url); + if (matcher.find()) { + host = matcher.group(); + } + // 如果 + if (host.contains(COMMA_COLON) == false) { + this.ip = host; + this.port = 80; + } else { + String[] ipPortArr = host.split(COMMA_COLON); + this.ip = ipPortArr[0]; + this.port = Integer.valueOf(ipPortArr[1].trim()); + } + } + + public String getIp() { + return ip; + } + + public Integer getPort() { + return port; + } +} diff --git a/src/main/java/com/pancm/elasticsearch/JestTest.java b/src/main/java/com/pancm/elasticsearch/JestTest.java new file mode 100644 index 0000000..553677b --- /dev/null +++ b/src/main/java/com/pancm/elasticsearch/JestTest.java @@ -0,0 +1,370 @@ +package com.pancm.sql.easticsearch; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import io.searchbox.client.JestClient; +import io.searchbox.client.JestClientFactory; +import io.searchbox.client.JestResult; +import io.searchbox.client.config.HttpClientConfig; +import io.searchbox.core.Bulk; +import io.searchbox.core.BulkResult; +import io.searchbox.core.Delete; +import io.searchbox.core.DocumentResult; +import io.searchbox.core.Index; +import io.searchbox.core.Search; +import io.searchbox.indices.CreateIndex; +import io.searchbox.indices.DeleteIndex; +import io.searchbox.indices.mapping.GetMapping; +import io.searchbox.indices.mapping.PutMapping; + +/** + * +* @Title: JestTest +* @Description: es 的 Jest 测试类 +* @Version:1.0.0 +* @author pancm +* @date 2019年2月28日 + */ +public class JestTest { + private static JestClient jestClient; + private static String indexName = "userindex"; + private static String typeName = "user"; + private static String elasticIps="http://127.0.0.1:9200"; + + + public static void main(String[] args) throws Exception { + jestClient = getJestClient(); + insertBatch(); + serach1(); + serach2(); + serach3(); + jestClient.close(); + + } + + private static JestClient getJestClient() { + JestClientFactory factory = new JestClientFactory(); + factory.setHttpClientConfig(new HttpClientConfig.Builder(elasticIps).connTimeout(60000).readTimeout(60000).multiThreaded(true).build()); + return factory.getObject(); + } + + public static void insertBatch() { + List objs = new ArrayList(); + objs.add(new User(1L, "张三", 20, "张三是个Java开发工程师","2018-4-25 11:07:42")); + objs.add(new User(2L, "李四", 24, "李四是个测试工程师","1980-2-15 19:01:32")); + objs.add(new User(3L, "王五", 25, "王五是个运维工程师","2016-8-21 06:11:32")); + boolean result = false; + try { + result = insertBatch(jestClient,indexName, typeName,objs); + } catch (Exception e) { + e.printStackTrace(); + } + System.out.println("批量新增:"+result); + } + + + /** + * 全文搜索 + */ + public static void serach1() { + String query ="工程师"; + try { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.queryStringQuery(query)); + //分页设置 + searchSourceBuilder.from(0).size(2); + System.out.println("全文搜索查询语句:"+searchSourceBuilder.toString()); + System.out.println("全文搜索返回结果:"+search(jestClient,indexName, typeName, searchSourceBuilder.toString())); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 精确搜索 + */ + public static void serach2() { + try { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.termQuery("age", 24)); + System.out.println("精确搜索查询语句:"+searchSourceBuilder.toString()); + System.out.println("精确搜索返回结果:"+search(jestClient,indexName, typeName, searchSourceBuilder.toString())); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + /** + * 区间搜索 + */ + public static void serach3() { + String createtm="createtm"; + String from="2016-8-21 06:11:32"; + String to="2018-8-21 06:11:32"; + + try { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.rangeQuery(createtm).gte(from).lte(to)); + System.out.println("区间搜索语句:"+searchSourceBuilder.toString()); + System.out.println("区间搜索返回结果:"+search(jestClient,indexName, typeName, searchSourceBuilder.toString())); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + /** + * 创建索引 + * @param indexName + * @return + * @throws Exception + */ + public boolean createIndex(JestClient jestClient,String indexName) throws Exception { + JestResult jr = jestClient.execute(new CreateIndex.Builder(indexName).build()); + return jr.isSucceeded(); + } + + /** + * 新增数据 + * @param indexName + * @param typeName + * @param source + * @return + * @throws Exception + */ + public boolean insert(JestClient jestClient,String indexName, String typeName, String source) throws Exception { + PutMapping putMapping = new PutMapping.Builder(indexName, typeName, source).build(); + JestResult jr = jestClient.execute(putMapping); + return jr.isSucceeded(); + } + + + /** + * 查询数据 + * @param indexName + * @param typeName + * @return + * @throws Exception + */ + public static String getIndexMapping(JestClient jestClient,String indexName, String typeName) throws Exception { + GetMapping getMapping = new GetMapping.Builder().addIndex(indexName).addType(typeName).build(); + JestResult jr =jestClient.execute(getMapping); + return jr.getJsonString(); + } + + + + /** + * 批量新增数据 + * @param indexName + * @param typeName + * @param objs + * @return + * @throws Exception + */ + public static boolean insertBatch(JestClient jestClient,String indexName, String typeName, List objs) throws Exception { + Bulk.Builder bulk = new Bulk.Builder().defaultIndex(indexName).defaultType(typeName); + for (Object obj : objs) { + Index index = new Index.Builder(obj).build(); + bulk.addAction(index); + } + BulkResult br = jestClient.execute(bulk.build()); + return br.isSucceeded(); + } + + /** + * 全文搜索 + * @param indexName + * @param typeName + * @param query + * @return + * @throws Exception + */ + public static String search(JestClient jestClient,String indexName, String typeName, String query) throws Exception { + Search search = new Search.Builder(query) + .addIndex(indexName) + .addType(typeName) + .build(); + JestResult jr = jestClient.execute(search); +// System.out.println("--"+jr.getJsonString()); +// System.out.println("--"+jr.getSourceAsObject(User.class)); + return jr.getSourceAsString(); + } + + + + + + /** + * 删除索引 + * @param indexName + * @return + * @throws Exception + */ + public boolean delete(JestClient jestClient,String indexName) throws Exception { + JestResult jr = jestClient.execute(new DeleteIndex.Builder(indexName).build()); + return jr.isSucceeded(); + } + + /** + * 删除数据 + * @param indexName + * @param typeName + * @param id + * @return + * @throws Exception + */ + public boolean delete(JestClient jestClient,String indexName, String typeName, String id) throws Exception { + DocumentResult dr = jestClient.execute(new Delete.Builder(id).index(indexName).type(typeName).build()); + return dr.isSucceeded(); + } + +} + + + class User implements Serializable{ + /** + * + */ + private static final long serialVersionUID = 1L; + /** 编号 */ + private Long id; + /** 姓名 */ + private String name; + + /** 年龄 */ + private Integer age; + + /** 描述 */ + private String description; + + /** 创建时间 */ + private String createtm; + + + public User(){ + } + + + + + public User(Long id, String name, Integer age, String description, String createtm) { + super(); + this.id = id; + this.name = name; + this.age = age; + this.description = description; + this.createtm = createtm; + } + + + + + /** + * 获取编号 + * @return id + */ + public Long getId() { + return id; + } + + + /** + * 设置编号 + * @param Long id + */ + public void setId(Long id) { + this.id = id; + } + + + /** + * 获取姓名 + * @return name + */ + public String getName() { + return name; + } + + + /** + * 设置姓名 + * @param String name + */ + public void setName(String name) { + this.name = name; + } + + + /** + * 获取年龄 + * @return age + */ + public Integer getAge() { + return age; + } + + + /** + * 设置年龄 + * @param Integer age + */ + public void setAge(Integer age) { + this.age = age; + } + + + /** + * 获取描述 + * @return description + */ + public String getDescription() { + return description; + } + + + /** + * 设置描述 + * @param String description + */ + public void setDescription(String description) { + this.description = description; + } + + + /** + * 获取创建时间 + * @return createtm + */ + public String getCreatetm() { + return createtm; + } + + + /** + * 设置创建时间 + * @param String createtm + */ + public void setCreatetm(String createtm) { + this.createtm = createtm; + } + + + + + @Override + public String toString() { + return "User [id=" + id + ", name=" + name + ", age=" + age + ", description=" + description + ", createtm=" + + createtm + "]"; + } + + + +} + diff --git a/src/main/java/com/pancm/elasticsearch/package-info.java b/src/main/java/com/pancm/elasticsearch/package-info.java new file mode 100644 index 0000000..841cffa --- /dev/null +++ b/src/main/java/com/pancm/elasticsearch/package-info.java @@ -0,0 +1,8 @@ +/** +* @Title: package-info +* @Description: es 相关测试类 +* @Version:1.0.0 +* @author pancm +* @date 2019年2月28日 +*/ +package com.pancm.elasticsearch; \ No newline at end of file diff --git a/src/main/java/com/pancm/excel/DemoData.java b/src/main/java/com/pancm/excel/DemoData.java new file mode 100644 index 0000000..ecae316 --- /dev/null +++ b/src/main/java/com/pancm/excel/DemoData.java @@ -0,0 +1,22 @@ +package com.pancm.excel; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.Date; + +/** + * @author pancm + * @Title: pancm_project + * @Description: + * @Version:1.0.0 + * @Since:jdk1.8 + * @date 2023/3/23 + */ +@Data +@EqualsAndHashCode +public class DemoData { + private String string; + private Date date; + private Double doubleData; +} diff --git a/src/main/java/com/pancm/excel/EasyExcelMergeTest.java b/src/main/java/com/pancm/excel/EasyExcelMergeTest.java new file mode 100644 index 0000000..46da597 --- /dev/null +++ b/src/main/java/com/pancm/excel/EasyExcelMergeTest.java @@ -0,0 +1,206 @@ +package com.pancm.excel; + +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.ExcelWriter; +import com.alibaba.excel.metadata.Head; +import com.alibaba.excel.support.ExcelTypeEnum; +import com.alibaba.excel.util.CollectionUtils; +import com.alibaba.excel.write.merge.AbstractMergeStrategy; +import com.alibaba.excel.write.metadata.WriteSheet; +import com.alibaba.excel.write.metadata.WriteTable; +import com.google.common.collect.Lists; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.util.CellRangeAddress; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author pancm + * @Title: pancm_project + * @Description: 多行合并 + * 合并单元格 + * @Version:1.0.0 + * @Since:jdk1.8 + * @date 2023/3/23 + */ +public class EasyExcelMergeTest { + + public static void main(String[] args) { + writeExcel(); + writeExcel01(); + writeExcel02(); + writeExcel03(); + } + + + private static String getPath(String s) { + return System.getProperty("user.dir") + "/" + s+"_"+System.currentTimeMillis() + ".xlsx"; + } + + private static List data1() { + List list = Lists.newArrayList(); + for (int i = 0; i < 3; i++) { + DemoData data = new DemoData(); + data.setString("字符串" + 1); + data.setDate(new Date()); + data.setDoubleData(0.56); + list.add(data); + } + for (int i = 0; i < 3; i++) { + DemoData data = new DemoData(); + data.setString("字符串" + 2); + data.setDate(new Date()); + data.setDoubleData(0.56); + list.add(data); + } + for (int i = 0; i < 4; i++) { + DemoData data = new DemoData(); + data.setString("字符串" + 3); + data.setDate(new Date()); + data.setDoubleData(0.57); + list.add(data); + } + return list; + } + + + + // 自定义合并策略 该类继承了AbstractMergeStrategy抽象合并策略,需要重写merge()方法 + public static class CustomMergeStrategy extends AbstractMergeStrategy { + + /** + * 分组,每几行合并一次 + */ + private List exportFieldGroupCountList; + + /** + * 目标合并列index + */ + private Integer targetColumnIndex; + + // 需要开始合并单元格的首行index + private Integer rowIndex; + + // exportDataList为待合并目标列的值 + public CustomMergeStrategy(List exportDataList, Integer targetColumnIndex) { + this.exportFieldGroupCountList = getGroupCountList(exportDataList); + this.targetColumnIndex = targetColumnIndex; + } + + + @Override + protected void merge(Sheet sheet, Cell cell, Head head, Integer relativeRowIndex) { + + if (null == rowIndex) { + rowIndex = cell.getRowIndex(); + } + // 仅从首行以及目标列的单元格开始合并,忽略其他 + if (cell.getRowIndex() == rowIndex && cell.getColumnIndex() == targetColumnIndex) { + mergeGroupColumn(sheet); + } + } + + private void mergeGroupColumn(Sheet sheet) { + int rowCount = rowIndex; + for (Integer count : exportFieldGroupCountList) { + if (count == 1) { + rowCount += count; + continue; + } + // 合并单元格 + CellRangeAddress cellRangeAddress = new CellRangeAddress(rowCount, rowCount + count - 1, targetColumnIndex, targetColumnIndex); + sheet.addMergedRegionUnsafe(cellRangeAddress); + rowCount += count; + } + } + + // 该方法将目标列根据值是否相同连续可合并,存储可合并的行数 + private List getGroupCountList(List exportDataList) { + if (CollectionUtils.isEmpty(exportDataList)) { + return new ArrayList<>(); + } + List groupCountList = new ArrayList<>(); + int count = 1; + for (int i = 1; i < exportDataList.size(); i++) { + if (exportDataList.get(i).equals(exportDataList.get(i - 1))) { + count++; + } else { + groupCountList.add(count); + count = 1; + } + } + // 处理完最后一条后 + groupCountList.add(count); + return groupCountList; + } + } + + // 单列多行合并 + public static void writeExcel() { + String fileName = getPath("单列多行"); + ExcelWriter excelWriter = EasyExcel.write(fileName).excelType(ExcelTypeEnum.XLSX).build(); + + List demoDataList = data1(); + // 写sheet的时候注册相应的自定义合并单元格策略 + WriteSheet writeSheet = EasyExcel.writerSheet("模板1").head(DemoData.class) + .registerWriteHandler(new CustomMergeStrategy(demoDataList.stream().map(DemoData::getString).collect(Collectors.toList()), 0)) + .build(); + excelWriter.write(demoDataList, writeSheet); + excelWriter.finish(); + } + + //多列多行合并 + public static void writeExcel01() { + String fileName = getPath("多行多列"); + ExcelWriter excelWriter = EasyExcel.write(fileName).excelType(ExcelTypeEnum.XLSX).build(); + + List demoDataList = data1(); + WriteSheet writeSheet = EasyExcel.writerSheet("模板1").head(DemoData.class) + .registerWriteHandler(new CustomMergeStrategy(demoDataList.stream().map(DemoData::getString).collect(Collectors.toList()), 0)) + .registerWriteHandler(new CustomMergeStrategy(demoDataList.stream().map(o -> o.getDoubleData().toString()).collect(Collectors.toList()), 2)) + .build(); + excelWriter.write(demoDataList, writeSheet); + excelWriter.finish(); + } + //多sheet + public static void writeExcel02() { + String fileName = getPath("多sheet"); + ExcelWriter excelWriter = EasyExcel.write(fileName).excelType(ExcelTypeEnum.XLSX).build(); + + List demoDataList = data1(); + WriteSheet writeSheet = EasyExcel.writerSheet("模板1").head(DemoData.class) + .registerWriteHandler(new CustomMergeStrategy(demoDataList.stream().map(DemoData::getString).collect(Collectors.toList()), 0)) + .registerWriteHandler(new CustomMergeStrategy(demoDataList.stream().map(o -> o.getDoubleData().toString()).collect(Collectors.toList()), 2)) + .build(); + excelWriter.write(demoDataList, writeSheet); + + WriteSheet writeSheet1 = EasyExcel.writerSheet("模板2").head(DemoData.class).build(); + excelWriter.write(data1(), writeSheet1); + excelWriter.finish(); + } + + //多表 + public static void writeExcel03() { + String fileName = getPath("多表"); + ExcelWriter excelWriter = EasyExcel.write(fileName).excelType(ExcelTypeEnum.XLSX).build(); + WriteSheet writeSheet = EasyExcel.writerSheet("模板").needHead(Boolean.FALSE).build(); + + List demoDataList = data1(); + // 需要表头设置为true,WriteTable一些属性会继承自WriteSheet + WriteTable writeTable = EasyExcel.writerTable(1).head(DemoData.class).needHead(Boolean.TRUE) + .registerWriteHandler(new CustomMergeStrategy(demoDataList.stream().map(DemoData::getString).collect(Collectors.toList()), 0)) + .registerWriteHandler(new CustomMergeStrategy(demoDataList.stream().map(o -> o.getDoubleData().toString()).collect(Collectors.toList()), 2)) + .build(); + excelWriter.write(demoDataList, writeSheet, writeTable); + + WriteTable writeTable1 = EasyExcel.writerTable(2).head(DemoData.class).needHead(Boolean.TRUE).build(); + excelWriter.write(data1(), writeSheet, writeTable1); + excelWriter.finish(); + } + +} + diff --git a/src/main/java/com/pancm/excel/EasyExcelTest.java b/src/main/java/com/pancm/excel/EasyExcelTest.java new file mode 100644 index 0000000..709c684 --- /dev/null +++ b/src/main/java/com/pancm/excel/EasyExcelTest.java @@ -0,0 +1,288 @@ +package com.pancm.excel; + +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.ExcelReader; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.metadata.CellData; +import com.alibaba.excel.metadata.CellExtra; +import com.alibaba.excel.read.listener.ReadListener; +import com.alibaba.excel.read.metadata.ReadSheet; +import com.alibaba.fastjson.JSON; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; + +import java.io.File; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; + +/** + * @author pancm + * @Title: pancm_project + * @Description: 参考: https://www.yuque.com/easyexcel/doc + * @Version:1.0.0 + * @Since:jdk1.8 + * @date 2021/1/26 + */ +@Slf4j +public class EasyExcelTest { + + + /** + * 最简单的读 + *

+ * 1. 创建excel对应的实体对象 参照{@link DemoData} + *

+ * 2. 由于默认一行行的读取excel,所以需要创建excel一行一行的回调监听器,参照{@link DemoDataListener} + *

+ * 3. 直接读即可 + */ + @Test + public void simpleRead() { + // 写法1:JDK8+ ,不用额外写一个DemoDataListener + // since: 3.0.0-beta1 + String fileName = "/home" + "demo" + File.separator + "demo.xlsx"; + // 这里 需要指定读用哪个class去读,然后读取第一个sheet 文件流会自动关闭 + // 这里每次会读取3000条数据 然后返回过来 直接调用使用数据就行 +// EasyExcel.read(fileName, DemoData.class, new PageReadListener(dataList -> { +// for (DemoData demoData : dataList) { +// log.info("读取到一条数据{}", JSON.toJSONString(demoData)); +// } +// })).sheet().doRead(); + + // 写法2: + // 匿名内部类 不用额外写一个DemoDataListener + fileName = "/home" + "demo" + File.separator + "demo.xlsx"; + // 这里 需要指定读用哪个class去读,然后读取第一个sheet 文件流会自动关闭 + EasyExcel.read(fileName, DemoData.class, new ReadListener() { + /** + * 单次缓存的数据量 + */ + public static final int BATCH_COUNT = 100; + /** + *临时存储 + */ + private List cachedDataList = new ArrayList<>(); + + /** + * All listeners receive this method when any one Listener does an error report. If an exception is thrown here, the + * entire read will terminate. + * + * @param exception + * @param context + * @throws Exception + */ + @Override + public void onException(Exception exception, AnalysisContext context) throws Exception { + + } + + /** + * When analysis one head row trigger invoke function. + * + * @param headMap + * @param context + */ + @Override + public void invokeHead(Map headMap, AnalysisContext context) { + + } + + @Override + public void invoke(DemoData data, AnalysisContext context) { + cachedDataList.add(data); + if (cachedDataList.size() >= BATCH_COUNT) { + saveData(); + // 存储完成清理 list + cachedDataList = new ArrayList<>(); + } + } + + /** + * The current method is called when extra information is returned + * + * @param extra extra information + * @param context + */ + @Override + public void extra(CellExtra extra, AnalysisContext context) { + + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + saveData(); + } + + /** + * Verify that there is another piece of data.You can stop the read by returning false + * + * @param context + * @return + */ + @Override + public boolean hasNext(AnalysisContext context) { + return false; + } + + /** + * 加上存储数据库 + */ + private void saveData() { + log.info("{}条数据,开始存储数据库!", cachedDataList.size()); + log.info("存储数据库成功!"); + } + }).sheet().doRead(); + + // 有个很重要的点 DemoDataListener 不能被spring管理,要每次读取excel都要new,然后里面用到spring可以构造方法传进去 + // 写法3: + fileName = "/home" + "demo" + File.separator + "demo.xlsx"; + // 这里 需要指定读用哪个class去读,然后读取第一个sheet 文件流会自动关闭 + EasyExcel.read(fileName, DemoData.class, new DemoDataListener()).sheet().doRead(); + + // 写法4: + fileName = "/home" + "demo" + File.separator + "demo.xlsx"; + // 一个文件一个reader + ExcelReader excelReader = null; + try { + excelReader = EasyExcel.read(fileName, DemoData.class, new DemoDataListener()).build(); + // 构建一个sheet 这里可以指定名字或者no + ReadSheet readSheet = EasyExcel.readSheet(0).build(); + // 读取一个sheet + excelReader.read(readSheet); + } finally { + if (excelReader != null) { + // 这里千万别忘记关闭,读的时候会创建临时文件,到时磁盘会崩的 + excelReader.finish(); + } + } + } + + + @Data + @EqualsAndHashCode + class DemoData { + private String string; + private Date date; + private Double doubleData; + } + + + class DemoDataListener implements ReadListener { + + /** + * 每隔5条存储数据库,实际使用中可以100条,然后清理list ,方便内存回收 + */ + private static final int BATCH_COUNT = 100; + /** + * 缓存的数据 + */ + private List cachedDataList = new ArrayList<>(); + /** + * 假设这个是一个DAO,当然有业务逻辑这个也可以是一个service。当然如果不用存储这个对象没用。 + */ +// private DemoDAO demoDAO; +// +// public DemoDataListener() { +// // 这里是demo,所以随便new一个。实际使用如果到了spring,请使用下面的有参构造函数 +// demoDAO = new DemoDAO(); +// } + + /** + * 如果使用了spring,请使用这个构造方法。每次创建Listener的时候需要把spring管理的类传进来 + * + * @param demoDAO + */ +// public DemoDataListener(DemoDAO demoDAO) { +// this.demoDAO = demoDAO; +// } + + /** + * All listeners receive this method when any one Listener does an error report. If an exception is thrown here, the + * entire read will terminate. + * + * @param exception + * @param context + * @throws Exception + */ + @Override + public void onException(Exception exception, AnalysisContext context) throws Exception { + + } + + /** + * When analysis one head row trigger invoke function. + * + * @param headMap + * @param context + */ + @Override + public void invokeHead(Map headMap, AnalysisContext context) { + + } + + /** + * 这个每一条数据解析都会来调用 + * + * @param data one row value. Is is same as {@link AnalysisContext#readRowHolder()} + * @param context + */ + @Override + public void invoke(DemoData data, AnalysisContext context) { + log.info("解析到一条数据:{}", JSON.toJSONString(data)); + cachedDataList.add(data); + // 达到BATCH_COUNT了,需要去存储一次数据库,防止数据几万条数据在内存,容易OOM + if (cachedDataList.size() >= BATCH_COUNT) { + saveData(); + // 存储完成清理 list +// cachedDataList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT); + } + } + + /** + * The current method is called when extra information is returned + * + * @param extra extra information + * @param context + */ + @Override + public void extra(CellExtra extra, AnalysisContext context) { + + } + + /** + * 所有数据解析完成了 都会来调用 + * + * @param context + */ + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // 这里也要保存数据,确保最后遗留的数据也存储到数据库 + saveData(); + log.info("所有数据解析完成!"); + } + + /** + * Verify that there is another piece of data.You can stop the read by returning false + * + * @param context + * @return + */ + @Override + public boolean hasNext(AnalysisContext context) { + return false; + } + + /** + * 加上存储数据库 + */ + private void saveData() { + log.info("{}条数据,开始存储数据库!", cachedDataList.size()); +// demoDAO.save(cachedDataList); + log.info("存储数据库成功!"); + } + } +} diff --git a/src/main/java/com/pancm/excel/EasyExcelUtils.java b/src/main/java/com/pancm/excel/EasyExcelUtils.java new file mode 100644 index 0000000..281ac49 --- /dev/null +++ b/src/main/java/com/pancm/excel/EasyExcelUtils.java @@ -0,0 +1,110 @@ +package com.pancm.excel; + +import com.alibaba.excel.EasyExcel; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import lombok.Data; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.Serializable; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class EasyExcelUtils { + + public static void main(String[] args) { + String[] headMap = { "项目名称", "楼栋名称", "单元名称", "楼层名称", "房间名称", "业主/租户姓名", "房间状态", "房间功能","认证人数","测试" }; + String[] dataStrMap={"hName","bName","uName","fName","pName","cName","pState","pFunction","pNum"}; + NoModelWriteData d = new NoModelWriteData(); + d.setFileName("认证统计"); + d.setHeadMap(headMap); + d.setDataStrMap(dataStrMap); + List listDatas = new ArrayList<>(); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("hName","项目1"); + jsonObject.put("bName","二楼"); + jsonObject.put("aa","测试"); + d.setDataList(listDatas); + EasyExcelUtils easyExcelUtils = new EasyExcelUtils(); +// easyExcelUtils.jsonWrite(d, response); + } + + //不创建对象的导出 + public void jsonWrite(NoModelWriteData data, HttpServletResponse response) throws IOException { + // 这里注意 有同学反应使用swagger 会导致各种问题,请直接用浏览器或者用postman + try { +// response.setContentType("application/vnd.ms-excel"); + response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + response.setCharacterEncoding("utf-8"); + // 这里URLEncoder.encode可以防止中文乱码 当然和easyexcel没有关系 + String fileName = URLEncoder.encode(data.getFileName(), "UTF-8"); + response.setHeader("Content-disposition", "attachment;filename=" + fileName + ".xlsx"); + // 这里需要设置不关闭流 + EasyExcel.write(response.getOutputStream()).head(head(data.getHeadMap())).sheet(data.getFileName()).doWrite(dataList(data.getDataList(), data.getDataStrMap())); + } catch (Exception e) { + // 重置response + response.reset(); + response.setContentType("application/json"); + response.setCharacterEncoding("utf-8"); + Map map = new HashMap(); + map.put("status", "failure"); + map.put("message", "下载文件失败" + e.getMessage()); + response.getWriter().println(JSON.toJSONString(map)); + } + } + + //创建对象的导出 + public void simpleWrite(SimpleWriteData data,Class clazz, HttpServletResponse response) throws IOException { + // 这里注意 有同学反应使用swagger 会导致各种问题,请直接用浏览器或者用postman +// response.setContentType("application/vnd.ms-excel"); + response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + response.setCharacterEncoding("utf-8"); + // 这里URLEncoder.encode可以防止中文乱码 当然和easyexcel没有关系 + String fileName = URLEncoder.encode(data.getFileName(), "UTF-8"); + response.setHeader("Content-disposition", "attachment;filename=" + fileName + ".xlsx"); + EasyExcel.write(response.getOutputStream(), clazz).sheet(data.getFileName()).doWrite(data.getDataList()); + } + + //设置表头 + private List> head(String[] headMap) { + List> list = new ArrayList>(); + + for (String head : headMap) { + List headList = new ArrayList(); + headList.add(head); + list.add(headList); + } + return list; + } + + //设置导出的数据内容 + private List> dataList(List dataList, String[] dataStrMap) { + List> list = new ArrayList>(); + for (JSONObject map : dataList) { + List data = new ArrayList(); + for (int i = 0; i < dataStrMap.length; i++) { + data.add(map.get(dataStrMap[i])); + } + list.add(data); + } + return list; + } +} + +@Data +class NoModelWriteData implements Serializable { + private String fileName;//文件名 + private String[] headMap;//表头数组 + private String[] dataStrMap;//对应数据字段数组 + private List dataList;//数据集合 +} + +@Data +class SimpleWriteData implements Serializable { + private String fileName;//文件名 + private List dataList;//数据列表 +} diff --git a/src/main/java/com/pancm/excel/EasyExcelWriteTest.java b/src/main/java/com/pancm/excel/EasyExcelWriteTest.java new file mode 100644 index 0000000..abaa8bd --- /dev/null +++ b/src/main/java/com/pancm/excel/EasyExcelWriteTest.java @@ -0,0 +1,115 @@ +package com.pancm.excel; + +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.ExcelWriter; +import com.alibaba.excel.annotation.ExcelIgnore; +import com.alibaba.excel.annotation.ExcelProperty; +import com.alibaba.excel.write.metadata.WriteSheet; +import com.alibaba.fastjson.JSONObject; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/** + * @author pancm + * @Title: pancm_project + * @Description: excel写模块测试 + * @Version:1.0.0 + * @Since:jdk1.8 + * @date 2022/3/2 + */ +@Slf4j +public class EasyExcelWriteTest { + + + public static void main(String[] args) { + List jsonObjectList = new ArrayList<>(); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("t1",1); + jsonObject.put("t2","2"); + JSONObject jsonObject2 = new JSONObject(); + jsonObject2.put("t1",11); + jsonObject2.put("t2","22"); + jsonObjectList.add(jsonObject); + jsonObjectList.add(jsonObject2); + + String fileName = "simpleWrite" + System.currentTimeMillis() + ".xlsx"; + // 这里 需要指定写用哪个class去写,然后写到第一个sheet,名字为模板 然后文件流会自动关闭 + // 如果这里想使用03 则 传入excelType参数即可 + EasyExcel.write(fileName).sheet("模板").doWrite(jsonObjectList); + + } + + + public void excludeOrIncludeWrite() { + // 注意 simpleWrite在数据量不大的情况下可以使用(5000以内,具体也要看实际情况),数据量大参照 重复多次写入 + + // 写法1 JDK8+ + // since: 3.0.0-beta1 + String fileName = "/home"+ "simpleWrite" + System.currentTimeMillis() + ".xlsx"; + // 这里 需要指定写用哪个class去写,然后写到第一个sheet,名字为模板 然后文件流会自动关闭 + // 如果这里想使用03 则 传入excelType参数即可 +// EasyExcel.write(fileName, DemoData.class) +// .sheet("模板") +// .doWrite( +// () -> { +// // 分页查询数据 +// return data(); +// }); + + // 写法2 + fileName = "/home"+ "simpleWrite" + System.currentTimeMillis() + ".xlsx"; + // 这里 需要指定写用哪个class去写,然后写到第一个sheet,名字为模板 然后文件流会自动关闭 + // 如果这里想使用03 则 传入excelType参数即可 + EasyExcel.write(fileName, DemoData.class).sheet("模板").doWrite(data()); + + // 写法3 + fileName = "/home"+ "/simpleWrite" + System.currentTimeMillis() + ".xlsx"; + // 这里 需要指定写用哪个class去写 + ExcelWriter excelWriter = null; + try { + excelWriter = EasyExcel.write(fileName, DemoData.class).build(); + WriteSheet writeSheet = EasyExcel.writerSheet("模板").build(); + excelWriter.write(data(), writeSheet); + } finally { + // 千万别忘记finish 会帮忙关闭流 + if (excelWriter != null) { + excelWriter.finish(); + } + } + } + + + private List data() { + List list = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + DemoData data = new DemoData(); + data.setString("字符串" + i); + data.setDate(new Date()); + data.setDoubleData(0.56); + list.add(data); + } + return list; + } + + @Data + @EqualsAndHashCode + class DemoData { + @ExcelProperty("字符串标题") + private String string; + @ExcelProperty("日期标题") + private Date date; + @ExcelProperty("数字标题") + private Double doubleData; + /** + * 忽略这个字段 + */ + @ExcelIgnore + private String ignore; + } +} + diff --git a/src/main/java/com/pancm/excel/package-info.java b/src/main/java/com/pancm/excel/package-info.java new file mode 100644 index 0000000..fb1e93b --- /dev/null +++ b/src/main/java/com/pancm/excel/package-info.java @@ -0,0 +1,9 @@ +/** + * @Title: pancm_project + * @Description: excel的工具类 + * @Version:1.0.0 + * @Since:jdk1.8 + * @author pancm + * @date 2021/1/26 + */ +package com.pancm.excel; \ No newline at end of file diff --git a/src/main/java/com/pancm/ffmpeg/FFmpegTest.java b/src/main/java/com/pancm/ffmpeg/FFmpegTest.java new file mode 100644 index 0000000..75d8674 --- /dev/null +++ b/src/main/java/com/pancm/ffmpeg/FFmpegTest.java @@ -0,0 +1,66 @@ +package com.pancm.ffmpeg; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +/** + * @author pancm + * @Title: FFmpegTest + * @Description: FFmpeg获取视频图片 + * 调用FFmpeg的命令获取ts视频的图片 + * @Version:1.0.0 + * @Since:jdk1.8 + * @Date 2021/4/14 + **/ +public class FFmpegTest { + + + public static void main(String[] args) { + + String ffmpegExePath = "C:\\ffmpeg\\bin\\ffmpeg.exe"; + + String inputFilePath = "D:\\video\\ts\\25-16_940.ts"; + + String outputFilePath = "D:\\video\\ts\\t2.jpg"; + + List command = new ArrayList(); + command.add(ffmpegExePath); + command.add("-i"); + command.add(inputFilePath); + command.add("-f"); + command.add("image2"); + command.add("-ss"); + command.add("1"); + command.add("-t"); + command.add("0.001"); + command.add("-s"); + command.add("320*240"); + command.add(outputFilePath); + ProcessBuilder builder = new ProcessBuilder(); + builder.command(command); + //正常信息和错误信息合并输出 + builder.redirectErrorStream(true); + try { + //开始执行命令 + Process process = builder.start(); + //如果你想获取到执行完后的信息,那么下面的代码也是需要的 + StringBuffer sbf = new StringBuffer(); + String line = null; + BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream())); + while ((line = br.readLine()) != null) { + sbf.append(line); + sbf.append(" "); + } + String resultInfo = sbf.toString(); + System.out.println(resultInfo); + } catch (IOException e) { + e.printStackTrace(); + } + + } + + +} diff --git a/src/main/java/com/pancm/ffmpeg/FFmpegUtil.java b/src/main/java/com/pancm/ffmpeg/FFmpegUtil.java new file mode 100644 index 0000000..af038d2 --- /dev/null +++ b/src/main/java/com/pancm/ffmpeg/FFmpegUtil.java @@ -0,0 +1,69 @@ +package com.pancm.ffmpeg; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +/** + * @author pancm + * @Title: gb28181_platform + * @Description: FFmpeg相关的工具类 + * @Version:1.0.0 + * @Since:jdk1.8 + * @date 2021/4/15 + */ +public class FFmpegUtil { + + + /** + * 执行ffmpeg的项目命令 + * @param cmdList + * @throws IOException + */ + public static void exec(List cmdList) throws IOException { + BufferedReader br = null; + try { + ProcessBuilder builder = new ProcessBuilder(); + builder.command(cmdList); + //正常信息和错误信息合并输出 + builder.redirectErrorStream(true); + //开始执行命令 + Process process = builder.start(); + StringBuffer sbf = new StringBuffer(); + String line = null; + br = new BufferedReader(new InputStreamReader(process.getInputStream())); + while ((line = br.readLine()) != null) { + sbf.append(line); + sbf.append(" "); + } + String resultInfo = sbf.toString(); + System.out.println(resultInfo); + } finally { + if (br != null) { + br.close(); + } + } + } + + public static void main(String[] args) throws IOException { + String ffmpegExePath = "C:\\ffmpeg\\bin\\ffmpeg.exe"; + String inputFilePath = "D:\\video\\ts\\25-16_940.ts"; + String outputFilePath = "D:\\video\\ts\\t3.jpg"; + List command = new ArrayList(); + command.add(ffmpegExePath); + command.add("-i"); + command.add(inputFilePath); + command.add("-f"); + command.add("image2"); + command.add("-ss"); + command.add("1"); + command.add("-t"); + command.add("0.001"); + command.add("-s"); + command.add("640*480"); + command.add(outputFilePath); + exec(command); + } +} diff --git a/src/main/java/com/pancm/ffmpeg/package-info.java b/src/main/java/com/pancm/ffmpeg/package-info.java new file mode 100644 index 0000000..c181f44 --- /dev/null +++ b/src/main/java/com/pancm/ffmpeg/package-info.java @@ -0,0 +1,9 @@ +/** + * @Title: pancm_project + * @Description: ffmpeg的工具类 + * @Version:1.0.0 + * @Since:jdk1.8 + * @author pancm + * @date 2021/1/26 + */ +package com.pancm.ffmpeg; \ No newline at end of file diff --git a/src/main/java/com/pancm/ftp/FtpHelper.java b/src/main/java/com/pancm/ftp/FtpHelper.java new file mode 100644 index 0000000..1b36067 --- /dev/null +++ b/src/main/java/com/pancm/ftp/FtpHelper.java @@ -0,0 +1,96 @@ +package com.pancm.ftp; + +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.net.ftp.FTPFile; +import org.apache.commons.net.ftp.FTPReply; + +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author pancm + * @Title: leakproof-server + * @Description: ftp帮助类 + * @Version:1.0.0 + * @Since:jdk1.8 + * @date 2021/8/18 + */ +public class FtpHelper { + + /** + * 获取文件列表文件的属性 + * + * @param ip + * @param port + * @param user + * @param pwd + * @param url + * @return + */ + public static List> getListFiles(String ip, int port, String user, String pwd, String url) throws IOException { + List> mapList = new ArrayList<>(); + FTPClient ftpClient = new FTPClient(); + ftpClient.connect(ip, port); + ftpClient.login(user, pwd); + FTPFile[] ftpFiles = ftpClient.listFiles(url); + if(ftpFiles!=null && ftpFiles.length>0) { + for (FTPFile ftpFile : ftpFiles) { + Map map = new HashMap<>(); + map.put("fileName",ftpFile.getName()); + map.put("fileSize",getSize(ftpFile.getSize())); + map.put("fileTime",ftpFile.getTimestamp().getTime().toString()); + mapList.add(map); + } + } + return mapList; + } + + + + private static boolean testFtp(String ip, int port, String user, String pwd) throws IOException { + FTPClient ftpClient = new FTPClient(); + ftpClient.connect(ip, port);//连接ftp + ftpClient.login(user, pwd);//登陆ftp + return FTPReply.isPositiveCompletion(ftpClient.getReplyCode()); + } + + + + + public static String getSize(long size) { + //获取到的size为:1705230 + long GB = 1024 * 1024 * 1024;//定义GB的计算常量 + long MB = 1024 * 1024;//定义MB的计算常量 + long KB = 1024;//定义KB的计算常量 + DecimalFormat df = new DecimalFormat("0.00");//格式化小数 + String resultSize = ""; + if (size / GB >= 1) { + //如果当前Byte的值大于等于1GB + resultSize = df.format(size / (float) GB) + "GB"; + } else if (size / MB >= 1) { + //如果当前Byte的值大于等于1MB + resultSize = df.format(size / (float) MB) + "MB"; + } else if (size / KB >= 1) { + //如果当前Byte的值大于等于1KB + resultSize = df.format(size / (float) KB) + "KB"; + } else { + resultSize = size + "B"; + } + return resultSize; + } + + + public static void main(String[] args) throws Exception { + String ip = "192.168.10.90"; + int port = 21; + String user = "root"; + String pwd = "lgwy@2020"; + String url = "/home/userfile/admin"; + System.out.println(testFtp(ip,port,user,pwd)); + System.out.println(getListFiles(ip,port,user,pwd,url)); + } +} diff --git a/src/main/java/com/pancm/ftp/SmartSshUtils.java b/src/main/java/com/pancm/ftp/SmartSshUtils.java new file mode 100644 index 0000000..2b4d124 --- /dev/null +++ b/src/main/java/com/pancm/ftp/SmartSshUtils.java @@ -0,0 +1,88 @@ +//package com.pancm.ftp; +// +//import lombok.extern.slf4j.Slf4j; +//import net.schmizz.sshj.SSHClient; +//import net.schmizz.sshj.sftp.SFTPClient; +//import net.schmizz.sshj.transport.verification.PromiscuousVerifier; +// +//import java.io.IOException; +// +// +//@Slf4j +//public final class SmartSshUtils { +// +// public static boolean testSFTP(String hostName, +// String username, +// String password){ +// SSHClient ssh = new SSHClient(); +// SFTPClient sftpClient = null; +// try { +// //ssh.loadKnownHosts(); to skip host verification +// ssh.addHostKeyVerifier(new PromiscuousVerifier()); +// ssh.connect(hostName); +// ssh.authPassword(username, password); +// sftpClient = ssh.newSFTPClient(); +// return true; +// }catch (IOException e) { +// e.printStackTrace(); +// } +// +// return false; +// } +// +// +// public static void downLoadFileBySsh(String hostName, +// String username, +// String password, +// String srcFilePath, +// String targetFilePath +// ) { +// SSHClient ssh = new SSHClient(); +// SFTPClient sftpClient = null; +// try { +// //ssh.loadKnownHosts(); to skip host verification +// ssh.addHostKeyVerifier(new PromiscuousVerifier()); +// ssh.connect(hostName); +// ssh.authPassword(username, password); +// sftpClient = ssh.newSFTPClient(); +// sftpClient.get(srcFilePath, targetFilePath); +// //create a folder +//// sftpClient.mkdir("/opt/app/testFolder"); +// //sftpClient.mkdirs("");创建多级文件夹 +// //sftpClient.rmdir("");重命名文件夹 +// //sftpClient.ls(""); //列出当前目录 +// } catch (IOException e) { +// log.error(e.getMessage(), e); +// } finally { +// if (null != sftpClient) { +// try { +// sftpClient.close(); +// } catch (IOException e) { +// log.error(e.getMessage(), e); +// } +// } +// try { +// ssh.disconnect(); +// } catch (IOException e) { +// log.error(e.getMessage(), e); +// } +// } +// } +// +// /** +// * 静态工具类应该禁用构造方法 +// */ +// private SmartSshUtils(){} +// +// +// public static void main(String[] args) { +// String hostName="192.168.9.80"; +// String username="root"; +// String password="Admin#12$34!"; +// String srcFilePath="/home/release/file"; +// String targetFilePath="D:\\d1"; +// +// SmartSshUtils.downLoadFileBySsh(hostName,username,password,srcFilePath,targetFilePath); +// +// } +//} diff --git a/src/main/java/com/pancm/ftp/package-info.java b/src/main/java/com/pancm/ftp/package-info.java new file mode 100644 index 0000000..4a57335 --- /dev/null +++ b/src/main/java/com/pancm/ftp/package-info.java @@ -0,0 +1,9 @@ +/** + * @Title: pancm_project + * @Description: ftp的工具类 + * @Version:1.0.0 + * @Since:jdk1.8 + * @author pancm + * @date 2021/1/26 + */ +package com.pancm.ftp; \ No newline at end of file diff --git a/src/main/java/com/pancm/sql/redis/RedisTest.java b/src/main/java/com/pancm/redis/RedisTest.java similarity index 97% rename from src/main/java/com/pancm/sql/redis/RedisTest.java rename to src/main/java/com/pancm/redis/RedisTest.java index 4d80304..0b27397 100644 --- a/src/main/java/com/pancm/sql/redis/RedisTest.java +++ b/src/main/java/com/pancm/redis/RedisTest.java @@ -1,4 +1,4 @@ -package com.pancm.sql.redis; +package com.pancm.redis; import java.util.Iterator; import java.util.List; @@ -25,7 +25,6 @@ public static void main(String[] args) { System.out.println("连接成功"); // 查看服务是否运行 System.out.println("服务正在运行: " + jedis.ping()); - // 存储到列表中 jedis.lpush("list", "redis"); jedis.lpush("list", "java"); @@ -35,7 +34,6 @@ public static void main(String[] args) { for (int i = 0, j = list.size(); i < j; i++) { System.out.println("list的输出结果:" + list.get(i)); } - // 设置 redis 字符串数据 jedis.set("rst", "redisStringTest"); // 获取存储的数据并输出 diff --git a/src/main/java/com/pancm/sql/redis/package-info.java b/src/main/java/com/pancm/redis/package-info.java similarity index 81% rename from src/main/java/com/pancm/sql/redis/package-info.java rename to src/main/java/com/pancm/redis/package-info.java index 8289742..f36f17a 100644 --- a/src/main/java/com/pancm/sql/redis/package-info.java +++ b/src/main/java/com/pancm/redis/package-info.java @@ -5,4 +5,4 @@ * @author pancm * @date 2018年9月21日 */ -package com.pancm.sql.redis; \ No newline at end of file +package com.pancm.redis; \ No newline at end of file diff --git a/src/main/java/com/pancm/sql/package-info.java b/src/main/java/com/pancm/sql/package-info.java index 77a75cc..090097b 100644 --- a/src/main/java/com/pancm/sql/package-info.java +++ b/src/main/java/com/pancm/sql/package-info.java @@ -1,8 +1,8 @@ -/** -* @Title: package-info -* @Description: 数据库相关的代码 -* @Version:1.0.0 -* @author pancm -* @date 2018年9月21日 -*/ +/** +* @Title: package-info +* @Description: 数据库相关的代码 +* @Version:1.0.0 +* @author pancm +* @date 2018年9月21日 +*/ package com.pancm.sql; \ No newline at end of file diff --git a/src/main/java/com/pancm/thread/test/NotifyTest.java b/src/main/java/com/pancm/thread/test/NotifyTest.java index 6453d5e..460c056 100644 --- a/src/main/java/com/pancm/thread/test/NotifyTest.java +++ b/src/main/java/com/pancm/thread/test/NotifyTest.java @@ -1,53 +1,53 @@ -package com.pancm.thread.test; - -import java.util.Random; - -/** - * @Title: NotifyTest - * @Description: - * wait 和 notify测试 - * @Version:1.0.0 - * @author pancm - * @date 2018年5月22日 - */ -public class NotifyTest { - - public static void main(String[] args) { - Test4 t1 = new Test4("张三"); - Test4 t2 = new Test4("李四"); - t1.start(); - t2.start(); - } -} - -class Test4 extends Thread { - private String name; - public Test4(String name) { - super(name); - this.name=name; - } - - @Override - public void run() { - System.out.println(this.getName() + " 线程运行开始!"); - - for (int i = 0; i < 5; i++) { - System.out.println("子线程" + this.getName() + "运行 : " + i); - try { - sleep(new Random().nextInt(100)); - } catch (InterruptedException e) { - e.printStackTrace(); - } - try { - if("李四".equals(this.getName())){ - this.getName().wait(); - } - } catch (InterruptedException e) { - e.printStackTrace(); - } - this.getName().notify(); - - } - System.out.println(this.getName() + " 线程运行结束!"); - } -} +package com.pancm.thread.test; + +import java.util.Random; + +/** + * @Title: NotifyTest + * @Description: + * wait 和 notify测试 + * @Version:1.0.0 + * @author pancm + * @date 2018年5月22日 + */ +public class NotifyTest { + + public static void main(String[] args) { + Test4 t1 = new Test4("张三"); + Test4 t2 = new Test4("李四"); + t1.start(); + t2.start(); + } +} + +class Test4 extends Thread { + private String name; + public Test4(String name) { + super(name); + this.name=name; + } + + @Override + public void run() { + System.out.println(this.getName() + " 线程运行开始!"); + + for (int i = 0; i < 5; i++) { + System.out.println("子线程" + this.getName() + "运行 : " + i); + try { + sleep(new Random().nextInt(100)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + try { + if("李四".equals(this.getName())){ + this.getName().wait(); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + this.getName().notify(); + + } + System.out.println(this.getName() + " 线程运行结束!"); + } +}