From bb97b1c8212cddeb9d55d7f816f29e1883e9dc8f Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Thu, 21 Feb 2019 18:00:06 +0800 Subject: [PATCH 01/53] =?UTF-8?q?=E6=96=B0=E5=A2=9EList=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=A4=BA=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/pancm/basics/ListTest.java | 587 ++++++++++--------- 1 file changed, 295 insertions(+), 292 deletions(-) diff --git a/src/main/java/com/pancm/basics/ListTest.java b/src/main/java/com/pancm/basics/ListTest.java index 41640dd..b863c96 100644 --- a/src/main/java/com/pancm/basics/ListTest.java +++ b/src/main/java/com/pancm/basics/ListTest.java @@ -1,292 +1,295 @@ -package com.pancm.basics; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Vector; -import java.util.stream.Collectors; - -/** - * -* @Title: ListTest -* @Description:关于list测试 -* @Version:1.0.0 -* @author pancm -* @date 2017年10月13日 - */ -public class ListTest { - - private final static int count=50000; - - private static ArrayList arrayList = new ArrayList<>(); - private static LinkedList linkedList = new LinkedList<>(); - private static Vector vector = new Vector<>(); - - - public static void main(String[] args) { -// test1(); -// test2(); -// test3(); - test4(); -// test5(); - - Vector v=new Vector<>(); - } - - - /** - * 遍历方法 - */ - private static void test1() { - List list=new ArrayList(); - list.add("a"); - list.add("b"); - list.add("c"); - - System.out.println("list截取:"+list.subList(0, 1)); - System.out.println("list截取:"+list.subList(1, 3)); - - - //第一种使用 - for(int i=0;i iterator=list.iterator(); - while(iterator.hasNext())//判断下一个元素之后有值 - { - System.out.println(iterator.next()); - } - } - - /** - * 数组变化遍历 - */ - private static void test2() { - - List list1 = new ArrayList(); - list1.add("1"); - list1.add("2"); - System.out.println("list1遍历之前:"+list1); - Iterator iterator = list1.iterator(); - while (iterator.hasNext()) { - String item = iterator.next(); - if ("2".equals(item)) { - iterator.remove(); - } - } - System.out.println("list1遍历之后:"+list1); - - List list = new ArrayList(); - list.add("1"); - list.add("2"); - System.out.println("list遍历之前:"+list); - for (String item : list) { - if ("2".equals(item)) { - list.remove(item); - //如果这里不适用break的话，会直接报错的 - break; - } - } - System.out.println("list遍历之后:"+list); - - - } - - - /** - * 常用方法 - */ - private static void test3() { - List list=initData(100); - System.out.println("list:"+list); - System.out.println("removeList:"+removeList(list,10,20)); - list.subList(10, 20).clear(); - System.out.println("subList:"+list); - - List ls1=new ArrayList(); - List ls2=new ArrayList(); - ls1.add("a"); - ls1.add("b"); - ls1.add("c"); - ls2.add("a"); - ls2.add("d"); - ls2.add("e"); -// System.out.println("合集:"+addAll(ls1,ls2)); - System.out.println("交集 :"+retainAll(ls1,ls2)); - System.out.println("差集 :"+removeAll(ls1,ls2)); - System.out.println("并集 :"+andAll(ls1,ls2)); - - } - - private static void test4() { - LinkedList list=new LinkedList(); - list.add(3); - list.add(5); - list.add(4); - System.out.println(list); - list.addFirst(2); - list.addLast(4); - System.out.println(list); - - LinkedList list2=new LinkedList(); - list2.add(1); - list2.add(2); - list2.add(4); - list2.add(4); - list2.add(6); - list2.add(6); - list2.add(5); - System.out.println("去重之前:"+list2); - //jdk1.8去重 - List newList = list2.stream().distinct().collect(Collectors.toList()); - System.out.println("去重之后:"+newList); - - } - - private static void test5() { - insertList(arrayList); - insertList(linkedList); - insertList(vector); - - System.out.println("--------------------"); - - readList(arrayList); - readList(linkedList); - readList(vector); - - System.out.println("--------------------"); - - delList(arrayList); - delList(linkedList); - delList(vector); - - } - - - private static void insertList(List list){ - long start=System.currentTimeMillis(); - Object o = new Object(); - for(int i=0;i removeList(List list,int s,int d){ - for(int i=0,j=list.size();i=s&&i initData(int j){ - List list=new ArrayList(); - for(int i=1;i<=j;i++){ - list.add(i); - } - return list; - } - - /** - * 数组合集 - * @param ls1 - * @param ls2 - * @return - */ - private static List addAll(List ls1,Listls2){ - ls1.addAll(ls2); - return ls1; - } - - /** - * 数组交集 （retainAll 会删除 ls1在ls2中没有的元素） - * @param ls1 - * @param ls2 - * @return - */ - private static List retainAll(List ls1,Listls2){ - System.out.println("ls1:"+ls1+";ls2:"+ls2); - ls1.retainAll(ls2); - return ls1; - } - - /** - * 差集 (删除ls2中没有ls1中的元素) - * @param ls1 - * @param ls2 - * @return - */ - private static List removeAll(List ls1,Listls2){ - ls1.removeAll(ls2); - System.out.println(ls1+";ls2:"+ls2); - return ls1; - } - - /** - * 无重复的并集 (ls1和ls2中并集，并无重复) - * @param ls1 - * @param ls2 - * @return - */ - private static List andAll(List ls1,Listls2){ - //删除在ls1中出现的元素 - ls2.removeAll(ls1); - //将剩余的ls2中的元素添加到ls1中 - ls1.addAll(ls2); - System.out.println(ls1+";ls2:"+ls2); - return ls1; - } -} +package com.pancm.basics; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Vector; +import java.util.stream.Collectors; + +/** + * +* @Title: ListTest +* @Description:关于list测试 +* @Version:1.0.0 +* @author pancm +* @date 2017年10月13日 + */ +public class ListTest { + + private final static int count=50000; + + private static ArrayList arrayList = new ArrayList<>(); + private static LinkedList linkedList = new LinkedList<>(); + private static Vector vector = new Vector<>(); + + + public static void main(String[] args) { +// test1(); +// test2(); + test3(); +// test4(); +// test5(); + + + Vector v=new Vector<>(); + } + + + /** + * 遍历方法 + */ + private static void test1() { + List list=new ArrayList(); + list.add("a"); + list.add("b"); + list.add("c"); + + System.out.println("list截取:"+list.subList(0, 1)); + System.out.println("list截取:"+list.subList(1, 3)); + + + //第一种使用 + for(int i=0;i iterator=list.iterator(); + while(iterator.hasNext())//判断下一个元素之后有值 + { + System.out.println(iterator.next()); + } + } + + /** + * 数组变化遍历 + */ + private static void test2() { + + List list1 = new ArrayList(); + list1.add("1"); + list1.add("2"); + System.out.println("list1遍历之前:"+list1); + Iterator iterator = list1.iterator(); + while (iterator.hasNext()) { + String item = iterator.next(); + if ("2".equals(item)) { + iterator.remove(); + } + } + System.out.println("list1遍历之后:"+list1); + + List list = new ArrayList(); + list.add("1"); + list.add("2"); + System.out.println("list遍历之前:"+list); + for (String item : list) { + if ("2".equals(item)) { + list.remove(item); + //如果这里不适用break的话，会直接报错的 + break; + } + } + System.out.println("list遍历之后:"+list); + + + } + + + /** + * 常用方法 + */ + private static void test3() { + List list=initData(100); + System.out.println("list:"+list); + System.out.println("removeList:"+removeList(list,10,20)); + list.subList(10, 20).clear(); + System.out.println("subList:"+list); + + List ls1=new ArrayList(); + List ls2=new ArrayList(); + ls1.add("a"); + ls1.add("b"); + ls1.add("c"); + ls2.add("a"); + ls2.add("b"); + ls2.add("c"); + ls2.add("e"); +// System.out.println("合集:"+addAll(ls1,ls2)); +// System.out.println("交集 :"+retainAll(ls1,ls2)); + System.out.println("差集 :"+removeAll(ls2,ls1)); +// System.out.println("并集 :"+andAll(ls1,ls2)); + + } + + private static void test4() { + LinkedList list=new LinkedList(); + list.add(3); + list.add(5); + list.add(4); + System.out.println(list); + list.addFirst(2); + list.addLast(4); + System.out.println(list); + + LinkedList list2=new LinkedList(); + list2.add(1); + list2.add(2); + list2.add(4); + list2.add(4); + list2.add(6); + list2.add(6); + list2.add(5); + System.out.println("去重之前:"+list2); + //jdk1.8去重 + List newList = list2.stream().distinct().collect(Collectors.toList()); + System.out.println("去重之后:"+newList); + + } + + private static void test5() { + insertList(arrayList); + insertList(linkedList); + insertList(vector); + + System.out.println("--------------------"); + + readList(arrayList); + readList(linkedList); + readList(vector); + + System.out.println("--------------------"); + + delList(arrayList); + delList(linkedList); + delList(vector); + + } + + + private static void insertList(List list){ + long start=System.currentTimeMillis(); + Object o = new Object(); + for(int i=0;i removeList(List list,int s,int d){ + for(int i=0,j=list.size();i=s&&i initData(int j){ + List list=new ArrayList(); + for(int i=1;i<=j;i++){ + list.add(i); + } + return list; + } + + /** + * 数组合集 + * @param ls1 + * @param ls2 + * @return + */ + private static List addAll(List ls1,Listls2){ + ls1.addAll(ls2); + return ls1; + } + + /** + * 数组交集 （retainAll 会删除 ls1在ls2中没有的元素） + * @param ls1 + * @param ls2 + * @return + */ + private static List retainAll(List ls1,Listls2){ + System.out.println("ls1:"+ls1+";ls2:"+ls2); + ls1.retainAll(ls2); + return ls1; + } + + /** + * 差集 (删除ls2中没有ls1中的元素) + * @param ls1 + * @param ls2 + * @return + */ + private static List removeAll(List ls1,Listls2){ + System.out.println("ls1:"+ls1+";ls2:"+ls2); + ls1.removeAll(ls2); + return ls1; + } + + /** + * 无重复的并集 (ls1和ls2中并集，并无重复) + * @param ls1 + * @param ls2 + * @return + */ + private static List andAll(List ls1,Listls2){ + //删除在ls1中出现的元素 + ls2.removeAll(ls1); + //将剩余的ls2中的元素添加到ls1中 + ls1.addAll(ls2); + System.out.println(ls1+";ls2:"+ls2); + return ls1; + } +} From 6ebe0c351f98615def935d341ee7f85dd29d1a2e Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Sun, 24 Feb 2019 12:53:54 +0800 Subject: [PATCH 02/53] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 146 ++++++++++++++++++++++++++---------------------------- 1 file changed, 71 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index e6d95f9..41bdf8e 100644 --- a/README.md +++ b/README.md @@ -1,75 +1,71 @@ -## java-study - -**介绍** -[java-study](https://github.com/xuwujing/java-study) 是本人学习Java过程中记录的一些代码！从Java基础的数据类型、jdk1.8的使用、IO、集合、线程等等技术以及一些常用框架，netty、mina、springboot、kafka、storm、zookeeper、redis、hbase、hive等等。 - -**使用** - -下载： - - git clone https://github.com/xuwujing/java-study - -然后使用maven方式导入IDE中运行main方法即可。 - - -## 项目结构 - - - com.pancm.arithmetic - 一些算法相关类 - com.pancm.basics - 一些Java基础相关类 主要是三大特性、修饰符、io、集合、反射、克隆等等相关代码 - com.pancm.bigdata - 大数据相关的类 主要是hbase、storm、zookeeper等等相关的代码 - com.pancm.commons - 一些第三方工具类的测试用例 主要是apache commons、apache lang、google common、google guava、joda等等一些工具包测试使用代码 - com.pancm.design - 设计模式相关的示例类 包含常用的23种设计模式 - com.pancm.jdk8 - jdk1.8相关的类 主要是lambda、stream以及LocalDateTime等等测试代码 - com.pancm.mq - 一些消息中间件的类，主要包含kafka、rabbitmq相关的测试代码 - com.pancm.nio - 一些nio框架，主要是netty和mina - com.pancm.others - 一些不知道怎么定义的测试类，Jsoup(爬虫)、logback、lombok等等测试代码 - com.pancm.pojo - 实体相关类 - com.pancm.question - 一些面试可能会问的问题的类 - com.pancm.sql - 一些数据库相关的类，包括非关系型数据库 - com.pancm.thread - 一些线程相关的类 从基本的使用到各种并发的测试类 - com.pancm.utils - 一些常用的工具类 主要是Json数据转换，日期转换，二维码图片生成工具类，常用的AES、MD5、BASE64等等编码解码工具类，redis、kafka、zookeeper等等工具类 - - - - - -## 相关文章 -这里介绍的文章主要是本人写的一些博客。博客主要发布在[个人博客](http://www.panchengming.com)、[CSDN](https://blog.csdn.net/qazwsxpcm)、[博客园](https://www.cnblogs.com/xuwujing/)等，但是由于个人博客在github上，访问可能较慢，CSDN目前观感体验不好，所以以下链接主要就在博客园中了。 - -**Java基础相关:** - -- [基本数据类型](https://www.cnblogs.com/xuwujing/p/8597557.html) -- [修饰符和String](https://www.cnblogs.com/xuwujing/p/8638329.html) -- [封装、继承和多态](https://www.cnblogs.com/xuwujing/p/8681123.html) -- [集合List、Map和Set](https://www.cnblogs.com/xuwujing/p/8886821.html) -- [多线程](https://www.cnblogs.com/xuwujing/p/9102870.html) -- [IO流](https://www.cnblogs.com/xuwujing/p/9191546.html) -- [总结篇](https://www.cnblogs.com/xuwujing/p/9236376.html) - - -**设计模式:** - -- [单例模式](https://www.cnblogs.com/xuwujing/p/9277266.html) -- [工厂方法和抽象工厂模式](https://www.cnblogs.com/xuwujing/p/9363142.html) -- [建造者模式和原型模式](https://www.cnblogs.com/xuwujing/p/9496346.html) -- [适配器模式和桥接模式](https://www.cnblogs.com/xuwujing/p/9520851.html) -- [外观模式和装饰器模式](https://www.cnblogs.com/xuwujing/p/9545272.html) -- [组合模式和过滤器模式](https://www.cnblogs.com/xuwujing/p/9630850.html) -- [享元模式和代理模式](https://www.cnblogs.com/xuwujing/p/9704228.html) -- [责任链模式和命令模式](https://www.cnblogs.com/xuwujing/p/9794886.html) -- [解释器模式和迭代器模式](https://www.cnblogs.com/xuwujing/p/9873514.html) -- [访问者模式和中介者模式](https://www.cnblogs.com/xuwujing/p/9911997.html) -- [策略模式和模板方法模式](https://www.cnblogs.com/xuwujing/p/9954263.html) -- [观察者模式和空对象模式](https://www.cnblogs.com/xuwujing/p/10036204.html) -- [总结篇](https://www.cnblogs.com/xuwujing/p/10134494.html) - - - - - -## 其他 - - -在这些代码中，虽然大部分都是自己写的，但是也有不少是在学习过程中从网上或书上直接摘抄的，当时有些并未标明出处，现在由于忘了出处，有些代码并未标明，若有冒犯，忘请见谅！ - +## java-study + +**介绍** +[java-study](https://github.com/xuwujing/java-study) 是本人学习Java过程中记录的一些代码！从Java基础的数据类型、jdk1.8的使用、IO、集合、线程等等技术以及一些常用框架，netty、mina、springboot、kafka、storm、zookeeper、redis、hbase、hive等等。 + +**使用** + +下载： + + git clone https://github.com/xuwujing/java-study + +然后使用maven方式导入IDE中运行main方法即可。 + +## 项目结构 + + com.pancm.arithmetic - 一些算法相关类 + com.pancm.basics - 一些Java基础相关类 主要是三大特性、修饰符、io、集合、反射、克隆等等相关代码 + com.pancm.bigdata - 大数据相关的类 主要是hbase、storm、zookeeper等等相关的代码 + com.pancm.commons - 一些第三方工具类的测试用例 主要是apache commons、apache lang、google common、google guava、joda等等一些工具包测试使用代码 + com.pancm.design - 设计模式相关的示例类 包含常用的23种设计模式 + com.pancm.jdk8 - jdk1.8相关的类 主要是lambda、stream以及LocalDateTime等等测试代码 + com.pancm.mq - 一些消息中间件的类，主要包含kafka、rabbitmq相关的测试代码 + com.pancm.nio - 一些nio框架，主要是netty和mina + com.pancm.others - 一些不知道怎么定义的测试类，Jsoup(爬虫)、logback、lombok等等测试代码 + com.pancm.pojo - 实体相关类 + com.pancm.question - 一些面试可能会问的问题的类 + com.pancm.sql - 一些数据库相关的类，包括非关系型数据库 + com.pancm.thread - 一些线程相关的类 从基本的使用到各种并发的测试类 + com.pancm.utils - 一些常用的工具类 主要是Json数据转换，日期转换，二维码图片生成工具类，常用的AES、MD5、BASE64等等编码解码工具类，redis、kafka、zookeeper等等工具类 + +## 相关文章 + +这里介绍的文章主要是本人写的一些博客。博客主要发布在[个人博客](http://www.panchengming.com)、[CSDN](https://blog.csdn.net/qazwsxpcm)、[博客园](https://www.cnblogs.com/xuwujing/)等，但是由于个人博客在github上，访问可能较慢，CSDN目前观感体验不好，所以以下链接主要就在博客园中了。 + +**Java基础相关:** + +- [基本数据类型](https://www.cnblogs.com/xuwujing/p/8597557.html) +- [修饰符和String](https://www.cnblogs.com/xuwujing/p/8638329.html) +- [封装、继承和多态](https://www.cnblogs.com/xuwujing/p/8681123.html) +- [集合List、Map和Set](https://www.cnblogs.com/xuwujing/p/8886821.html) +- [多线程](https://www.cnblogs.com/xuwujing/p/9102870.html) +- [IO流](https://www.cnblogs.com/xuwujing/p/9191546.html) +- [总结篇](https://www.cnblogs.com/xuwujing/p/9236376.html) + +**设计模式:** + +- [单例模式](https://www.cnblogs.com/xuwujing/p/9277266.html) +- [工厂方法和抽象工厂模式](https://www.cnblogs.com/xuwujing/p/9363142.html) +- [建造者模式和原型模式](https://www.cnblogs.com/xuwujing/p/9496346.html) +- [适配器模式和桥接模式](https://www.cnblogs.com/xuwujing/p/9520851.html) +- [外观模式和装饰器模式](https://www.cnblogs.com/xuwujing/p/9545272.html) +- [组合模式和过滤器模式](https://www.cnblogs.com/xuwujing/p/9630850.html) +- [享元模式和代理模式](https://www.cnblogs.com/xuwujing/p/9704228.html) +- [责任链模式和命令模式](https://www.cnblogs.com/xuwujing/p/9794886.html) +- [解释器模式和迭代器模式](https://www.cnblogs.com/xuwujing/p/9873514.html) +- [访问者模式和中介者模式](https://www.cnblogs.com/xuwujing/p/9911997.html) +- [策略模式和模板方法模式](https://www.cnblogs.com/xuwujing/p/9954263.html) +- [观察者模式和空对象模式](https://www.cnblogs.com/xuwujing/p/10036204.html) +- [总结篇](https://www.cnblogs.com/xuwujing/p/10134494.html) + +**进阶相关:** + +- [JDK1.8的Lambda、Stream和日期的使用详解](https://www.cnblogs.com/xuwujing/p/10145691.html) + +**其他:** + +- [个人收集的资源分享](https://www.cnblogs.com/xuwujing/p/10393111.html) + +## 其他 + +在这些代码中，虽然大部分都是自己写的，但是也有不少是在学习过程中从网上或书上直接摘抄的，当时有些并未标明出处，现在由于忘了出处，有些代码并未标明，若有冒犯，忘请见谅！ From 0a96efc289280741130e0d4afcb6d44ed953a76c Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Thu, 28 Feb 2019 18:28:07 +0800 Subject: [PATCH 03/53] =?UTF-8?q?1.=E6=8F=90=E4=BA=A4ES=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 60 ++- .../com/pancm/sql/easticsearch/JestTest.java | 370 ++++++++++++++++++ 2 files changed, 410 insertions(+), 20 deletions(-) create mode 100644 src/main/java/com/pancm/sql/easticsearch/JestTest.java diff --git a/pom.xml b/pom.xml index a78a143..e4d103a 100644 --- a/pom.xml +++ b/pom.xml @@ -1,14 +1,15 @@ - 4.0.0 1.0.0 - java_study + java_study 0.0.1-SNAPSHOT jar - + java_study - http://maven.apache.org + http://maven.apache.org UTF-8 @@ -28,15 +29,15 @@ junit 4.12 - - + + - jdk.tools - jdk.tools - 1.8 - system - ${JAVA_HOME}/lib/tools.jar - + jdk.tools + jdk.tools + 1.8 + system + ${JAVA_HOME}/lib/tools.jar + @@ -102,8 +103,8 @@ commons-lang3 3.7 - - + + org.apache.commons commons-compress @@ -319,8 +320,8 @@ zkclient 0.10 - - + + org.apache.hadoop @@ -369,15 +370,15 @@ hbase-common 1.4.8 - - + + org.apache.hive hive-jdbc 3.1.1 - + org.apache.storm @@ -391,7 +392,7 @@ 1.2.2 - + org.apache.spark @@ -420,7 +421,26 @@ 2.4.0 provided + + + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 6.6.1 + + + + + io.searchbox + jest + 6.3.1 + + + diff --git a/src/main/java/com/pancm/sql/easticsearch/JestTest.java b/src/main/java/com/pancm/sql/easticsearch/JestTest.java new file mode 100644 index 0000000..13c4127 --- /dev/null +++ b/src/main/java/com/pancm/sql/easticsearch/JestTest.java @@ -0,0 +1,370 @@ +package com.pancm.sql.easticsearch; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import io.searchbox.client.JestClient; +import io.searchbox.client.JestClientFactory; +import io.searchbox.client.JestResult; +import io.searchbox.client.config.HttpClientConfig; +import io.searchbox.core.Bulk; +import io.searchbox.core.BulkResult; +import io.searchbox.core.Delete; +import io.searchbox.core.DocumentResult; +import io.searchbox.core.Index; +import io.searchbox.core.Search; +import io.searchbox.indices.CreateIndex; +import io.searchbox.indices.DeleteIndex; +import io.searchbox.indices.mapping.GetMapping; +import io.searchbox.indices.mapping.PutMapping; + +/** + * +* @Title: JestTest +* @Description: es 的 Jest 测试类 +* @Version:1.0.0 +* @author pancm +* @date 2019年2月28日 + */ +public class JestTest { + private static JestClient jestClient; + private static String indexName = "userindex"; + private static String typeName = "user"; + private static String elasticIps="http://127.0.0.1:9200"; + + + public static void main(String[] args) throws Exception { + jestClient = getJestClient(); + insertBatch(); + serach1(); + serach2(); + serach3(); + jestClient.close(); + + } + + private static JestClient getJestClient() { + JestClientFactory factory = new JestClientFactory(); + factory.setHttpClientConfig(new HttpClientConfig.Builder(elasticIps).connTimeout(60000).readTimeout(60000).multiThreaded(true).build()); + return factory.getObject(); + } + + public static void insertBatch() { + List objs = new ArrayList(); + objs.add(new User(1L, "张三", 20, "张三是个Java开发工程师","2018-4-25 11:07:42")); + objs.add(new User(2L, "李四", 24, "李四是个测试工程师","1980-2-15 19:01:32")); + objs.add(new User(3L, "王五", 25, "王五是个运维工程师","2016-8-21 06:11:32")); + boolean result = false; + try { + result = insertBatch(jestClient,indexName, typeName,objs); + } catch (Exception e) { + e.printStackTrace(); + } + System.out.println("批量新增:"+result); + } + + + /** + * 全文搜索 + */ + public static void serach1() { + String query ="工程师"; + try { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.queryStringQuery(query)); + //分页设置 + searchSourceBuilder.from(0).size(2); + System.out.println("全文搜索查询语句:"+searchSourceBuilder.toString()); + System.out.println("全文搜索返回结果:"+search(jestClient,indexName, typeName, searchSourceBuilder.toString())); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 精确搜索 + */ + public static void serach2() { + try { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.termQuery("age", 24)); + System.out.println("精确搜索查询语句:"+searchSourceBuilder.toString()); + System.out.println("精确搜索返回结果:"+search(jestClient,indexName, typeName, searchSourceBuilder.toString())); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + /** + * 区间搜索 + */ + public static void serach3() { + String createtm="createtm"; + String from="2016-8-21 06:11:32"; + String to="2018-8-21 06:11:32"; + + try { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.rangeQuery(createtm).gte(from).lte(to)); + System.out.println("区间搜索语句:"+searchSourceBuilder.toString()); + System.out.println("区间搜索返回结果:"+search(jestClient,indexName, typeName, searchSourceBuilder.toString())); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + /** + * 创建索引 + * @param indexName + * @return + * @throws Exception + */ + public boolean createIndex(JestClient jestClient,String indexName) throws Exception { + JestResult jr = jestClient.execute(new CreateIndex.Builder(indexName).build()); + return jr.isSucceeded(); + } + + /** + * 新增数据 + * @param indexName + * @param typeName + * @param source + * @return + * @throws Exception + */ + public boolean insert(JestClient jestClient,String indexName, String typeName, String source) throws Exception { + PutMapping putMapping = new PutMapping.Builder(indexName, typeName, source).build(); + JestResult jr = jestClient.execute(putMapping); + return jr.isSucceeded(); + } + + + /** + * 查询数据 + * @param indexName + * @param typeName + * @return + * @throws Exception + */ + public static String getIndexMapping(JestClient jestClient,String indexName, String typeName) throws Exception { + GetMapping getMapping = new GetMapping.Builder().addIndex(indexName).addType(typeName).build(); + JestResult jr =jestClient.execute(getMapping); + return jr.getJsonString(); + } + + + + /** + * 批量新增数据 + * @param indexName + * @param typeName + * @param objs + * @return + * @throws Exception + */ + public static boolean insertBatch(JestClient jestClient,String indexName, String typeName, List objs) throws Exception { + Bulk.Builder bulk = new Bulk.Builder().defaultIndex(indexName).defaultType(typeName); + for (Object obj : objs) { + Index index = new Index.Builder(obj).build(); + bulk.addAction(index); + } + BulkResult br = jestClient.execute(bulk.build()); + return br.isSucceeded(); + } + + /** + * 全文搜索 + * @param indexName + * @param typeName + * @param query + * @return + * @throws Exception + */ + public static String search(JestClient jestClient,String indexName, String typeName, String query) throws Exception { + Search search = new Search.Builder(query) + .addIndex(indexName) + .addType(typeName) + .build(); + JestResult jr = jestClient.execute(search); +// System.out.println("--"+jr.getJsonString()); +// System.out.println("--"+jr.getSourceAsObject(User.class)); + return jr.getSourceAsString(); + } + + + + + + /** + * 删除索引 + * @param indexName + * @return + * @throws Exception + */ + public boolean delete(JestClient jestClient,String indexName) throws Exception { + JestResult jr = jestClient.execute(new DeleteIndex.Builder(indexName).build()); + return jr.isSucceeded(); + } + + /** + * 删除数据 + * @param indexName + * @param typeName + * @param id + * @return + * @throws Exception + */ + public boolean delete(JestClient jestClient,String indexName, String typeName, String id) throws Exception { + DocumentResult dr = jestClient.execute(new Delete.Builder(id).index(indexName).type(typeName).build()); + return dr.isSucceeded(); + } + +} + + + class User implements Serializable{ + /** + * + */ + private static final long serialVersionUID = 1L; + /** 编号 */ + private Long id; + /** 姓名 */ + private String name; + + /** 年龄 */ + private Integer age; + + /** 描述 */ + private String description; + + /** 创建时间 */ + private String createtm; + + + public User(){ + } + + + + + public User(Long id, String name, Integer age, String description, String createtm) { + super(); + this.id = id; + this.name = name; + this.age = age; + this.description = description; + this.createtm = createtm; + } + + + + + /** + * 获取编号 + * @return id + */ + public Long getId() { + return id; + } + + + /** + * 设置编号 + * @param Long id + */ + public void setId(Long id) { + this.id = id; + } + + + /** + * 获取姓名 + * @return name + */ + public String getName() { + return name; + } + + + /** + * 设置姓名 + * @param String name + */ + public void setName(String name) { + this.name = name; + } + + + /** + * 获取年龄 + * @return age + */ + public Integer getAge() { + return age; + } + + + /** + * 设置年龄 + * @param Integer age + */ + public void setAge(Integer age) { + this.age = age; + } + + + /** + * 获取描述 + * @return description + */ + public String getDescription() { + return description; + } + + + /** + * 设置描述 + * @param String description + */ + public void setDescription(String description) { + this.description = description; + } + + + /** + * 获取创建时间 + * @return createtm + */ + public String getCreatetm() { + return createtm; + } + + + /** + * 设置创建时间 + * @param String createtm + */ + public void setCreatetm(String createtm) { + this.createtm = createtm; + } + + + + + @Override + public String toString() { + return "User [id=" + id + ", name=" + name + ", age=" + age + ", description=" + description + ", createtm=" + + createtm + "]"; + } + + + +} + From 6e92f15da420b0742c1b74c218b036571c4ffc11 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Fri, 1 Mar 2019 11:03:56 +0800 Subject: [PATCH 04/53] =?UTF-8?q?=E6=8F=90=E4=BA=A4ES=E7=9A=84JEST?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pancm/sql/easticsearch/package-info.java | 8 ++ src/main/java/com/pancm/sql/package-info.java | 14 +- .../java/com/pancm/sql/redis/RedisTest.java | 120 +++++++++--------- .../com/pancm/sql/redis/package-info.java | 14 +- 4 files changed, 82 insertions(+), 74 deletions(-) create mode 100644 src/main/java/com/pancm/sql/easticsearch/package-info.java diff --git a/src/main/java/com/pancm/sql/easticsearch/package-info.java b/src/main/java/com/pancm/sql/easticsearch/package-info.java new file mode 100644 index 0000000..ef3a9e9 --- /dev/null +++ b/src/main/java/com/pancm/sql/easticsearch/package-info.java @@ -0,0 +1,8 @@ +/** +* @Title: package-info +* @Description: es 相关测试类 +* @Version:1.0.0 +* @author pancm +* @date 2019年2月28日 +*/ +package com.pancm.sql.easticsearch; \ No newline at end of file diff --git a/src/main/java/com/pancm/sql/package-info.java b/src/main/java/com/pancm/sql/package-info.java index 77a75cc..090097b 100644 --- a/src/main/java/com/pancm/sql/package-info.java +++ b/src/main/java/com/pancm/sql/package-info.java @@ -1,8 +1,8 @@ -/** -* @Title: package-info -* @Description: 数据库相关的代码 -* @Version:1.0.0 -* @author pancm -* @date 2018年9月21日 -*/ +/** +* @Title: package-info +* @Description: 数据库相关的代码 +* @Version:1.0.0 +* @author pancm +* @date 2018年9月21日 +*/ package com.pancm.sql; \ No newline at end of file diff --git a/src/main/java/com/pancm/sql/redis/RedisTest.java b/src/main/java/com/pancm/sql/redis/RedisTest.java index 4d80304..65530a6 100644 --- a/src/main/java/com/pancm/sql/redis/RedisTest.java +++ b/src/main/java/com/pancm/sql/redis/RedisTest.java @@ -1,60 +1,60 @@ -package com.pancm.sql.redis; - -import java.util.Iterator; -import java.util.List; -import java.util.Set; - -import redis.clients.jedis.Jedis; - -/** - * - * @Title: RedisTest - * @Description: redis测试代码 - * @Version:1.0.0 - * @author pancm - * @date 2017-8-19 - */ -public class RedisTest { - - /** - * @param args - */ - public static void main(String[] args) { - // 连接到本地的 redis服务 - Jedis jedis = new Jedis("127.0.0.1"); - System.out.println("连接成功"); - // 查看服务是否运行 - System.out.println("服务正在运行: " + jedis.ping()); - - // 存储到列表中 - jedis.lpush("list", "redis"); - jedis.lpush("list", "java"); - jedis.lpush("list", "mysql"); - // 获取存储的数据并输出 - List list = jedis.lrange("list", 0, 2); - for (int i = 0, j = list.size(); i < j; i++) { - System.out.println("list的输出结果:" + list.get(i)); - } - - // 设置 redis 字符串数据 - jedis.set("rst", "redisStringTest"); - // 获取存储的数据并输出 - System.out.println("redis 存储的字符串为: " + jedis.get("rst")); - - // 存储数据 - jedis.sadd("setTest1", "abc"); - jedis.sadd("setTest1", "abcd"); - jedis.sadd("setTest1", "abcde"); - // 获取数据并输出 - Set keys = jedis.keys("*"); - // Set keys=jedis.smembers("setTest1"); - // 定义迭代器输出 - Iterator it = keys.iterator(); - while (it.hasNext()) { - String key = it.next(); - System.out.println(key); - } - - } - -} +package com.pancm.sql.redis; + +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import redis.clients.jedis.Jedis; + +/** + * + * @Title: RedisTest + * @Description: redis测试代码 + * @Version:1.0.0 + * @author pancm + * @date 2017-8-19 + */ +public class RedisTest { + + /** + * @param args + */ + public static void main(String[] args) { + // 连接到本地的 redis服务 + Jedis jedis = new Jedis("127.0.0.1"); + System.out.println("连接成功"); + // 查看服务是否运行 + System.out.println("服务正在运行: " + jedis.ping()); + + // 存储到列表中 + jedis.lpush("list", "redis"); + jedis.lpush("list", "java"); + jedis.lpush("list", "mysql"); + // 获取存储的数据并输出 + List list = jedis.lrange("list", 0, 2); + for (int i = 0, j = list.size(); i < j; i++) { + System.out.println("list的输出结果:" + list.get(i)); + } + + // 设置 redis 字符串数据 + jedis.set("rst", "redisStringTest"); + // 获取存储的数据并输出 + System.out.println("redis 存储的字符串为: " + jedis.get("rst")); + + // 存储数据 + jedis.sadd("setTest1", "abc"); + jedis.sadd("setTest1", "abcd"); + jedis.sadd("setTest1", "abcde"); + // 获取数据并输出 + Set keys = jedis.keys("*"); + // Set keys=jedis.smembers("setTest1"); + // 定义迭代器输出 + Iterator it = keys.iterator(); + while (it.hasNext()) { + String key = it.next(); + System.out.println(key); + } + + } + +} diff --git a/src/main/java/com/pancm/sql/redis/package-info.java b/src/main/java/com/pancm/sql/redis/package-info.java index 8289742..c72cd24 100644 --- a/src/main/java/com/pancm/sql/redis/package-info.java +++ b/src/main/java/com/pancm/sql/redis/package-info.java @@ -1,8 +1,8 @@ -/** -* @Title: package-info -* @Description: redis 相关代码 -* @Version:1.0.0 -* @author pancm -* @date 2018年9月21日 -*/ +/** +* @Title: package-info +* @Description: redis 相关代码 +* @Version:1.0.0 +* @author pancm +* @date 2018年9月21日 +*/ package com.pancm.sql.redis; \ No newline at end of file From bf960cc54eebf52abfeff071fa39134403317e71 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Mon, 11 Mar 2019 16:21:24 +0800 Subject: [PATCH 05/53] =?UTF-8?q?=E5=A2=9E=E5=8A=A0ES=E9=AB=98=E7=BA=A7?= =?UTF-8?q?=E5=AE=A2=E6=88=B7=E7=AB=AF=E5=9F=BA=E6=9C=AC=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E7=A4=BA=E4=BE=8B=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../easticsearch/EsHighLevelRestTest1.java | 471 ++++++++++++++++++ .../com/pancm/thread/test/NotifyTest.java | 106 ++-- 2 files changed, 524 insertions(+), 53 deletions(-) create mode 100644 src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java diff --git a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java b/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java new file mode 100644 index 0000000..3d46fa3 --- /dev/null +++ b/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java @@ -0,0 +1,471 @@ +package com.pancm.sql.easticsearch; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; + +import org.apache.http.HttpHost; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.bulk.BackoffPolicy; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkProcessor; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.support.replication.ReplicationResponse; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @Title: EsHighLevelRestTest1 + * @Description: Java High Level REST Client Es高级客户端使用教程一 (基本使用) + * @since jdk 1.8 + * @Version:1.0.0 + * @author pancm + * @date 2019年3月5日 + */ +public class EsHighLevelRestTest1 { + + private static String elasticIp = "192.169.0.23"; + private static int elasticPort = 9200; + + private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestTest1.class); + + private static RestHighLevelClient client = null; + + public static void main(String[] args) { + try { + init(); + careatindex(); + get(); + exists(); + update(); + delete(); + close(); + } catch (IOException e) { + e.printStackTrace(); + } + + } + + /* + * 初始化服务 + */ + private static void init() { + client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort, "http"))); + + } + + /* + * 关闭服务 + */ + private static void close() { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + /** + * 创建索引 + * + * @throws IOException + */ + private static void careatindex() throws IOException { + String index = "user"; + String type = "userindex"; + // 唯一编号 + String id = "1"; + + IndexRequest request = new IndexRequest(index, type, id); + + /* + * 第一种方式，通过jsonString进行创建 + */ + // json + String jsonString = "{" + "\"user\":\"pancm\"," + "\"postDate\":\"2019-03-08\"," + + "\"message\":\"study Elasticsearch\"" + "}"; + + request.source(jsonString, XContentType.JSON); + + /* + * 第二种方式，通过map创建,，会自动转换成json的数据 + */ + + Map jsonMap = new HashMap<>(); + jsonMap.put("user", "pancm"); + jsonMap.put("postDate", "2019-03-08"); + jsonMap.put("message", "study Elasticsearch"); + + request.source(jsonMap); + + /* + * 第三种方式 : 通过XContentBuilder对象进行创建 + */ + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.field("user", "pancm"); + builder.timeField("postDate", "2019-03-08"); + builder.field("message", "study Elasticsearch"); + } + builder.endObject(); + request.source(builder); + + IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT); + + + + //对响应结果进行处理 + + String index1 = indexResponse.getIndex(); + String type1 = indexResponse.getType(); + String id1 = indexResponse.getId(); + long version = indexResponse.getVersion(); + // 如果是新增/修改的话的话 + if (indexResponse.getResult() == DocWriteResponse.Result.CREATED) { + + } else if (indexResponse.getResult() == DocWriteResponse.Result.UPDATED) { + + } + ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo(); + if (shardInfo.getTotal() != shardInfo.getSuccessful()) { + + } + if (shardInfo.getFailed() > 0) { + for (ReplicationResponse.ShardInfo.Failure failure : shardInfo.getFailures()) { + String reason = failure.reason(); + } + } + + System.out.println("创建成功！"); + } + + /** + * 查询数据 + * + * @throws IOException + */ + private static void get() { + String index = "user"; + String type = "userindex"; + // 唯一编号 + String id = "1"; + // 创建查询请求 + GetRequest getRequest = new GetRequest(index, type, id); + + GetResponse getResponse = null; + try { + getResponse = client.get(getRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ElasticsearchException e) { + // 如果是索引不存在 + if (e.status() == RestStatus.NOT_FOUND) { + System.out.println("该索引库不存在！"+index); + } + + } + // 如果存在该数据则返回对应的结果 + if (getResponse.isExists()) { + long version = getResponse.getVersion(); + String sourceAsString = getResponse.getSourceAsString(); + Map sourceAsMap = getResponse.getSourceAsMap(); + byte[] sourceAsBytes = getResponse.getSourceAsBytes(); + System.out.println("查询返回结果String:"+sourceAsString); + System.out.println("查询返回结果Map:"+sourceAsMap); + } else { + System.out.println("没有找到该数据！"); + } + } + + + /** + * 是否存在 + * @throws IOException + */ + private static void exists() throws IOException { + String index = "user"; + String type = "userindex"; + // 唯一编号 + String id = "1"; + // 创建查询请求 + GetRequest getRequest = new GetRequest(index, type, id); + + boolean exists = client.exists(getRequest, RequestOptions.DEFAULT); + + + ActionListener listener = new ActionListener() { + @Override + public void onResponse(Boolean exists) { + System.out.println("=="+exists); + } + + @Override + public void onFailure(Exception e) { + System.out.println("失败的原因："+e.getMessage()); + } + }; + //进行异步监听 +// client.existsAsync(getRequest, RequestOptions.DEFAULT, listener); + + System.out.println("是否存在："+exists); + } + + + /** + * 更新操作 + * @throws IOException + */ + private static void update() throws IOException { + String index = "user"; + String type = "userindex"; + // 唯一编号 + String id = "1"; + UpdateRequest upateRequest=new UpdateRequest(); + upateRequest.id(id); + upateRequest.index(index); + upateRequest.type(type); + + //依旧可以使用Map这种集合作为更新条件 + Map jsonMap = new HashMap<>(); + jsonMap.put("user", "xuwujing"); + jsonMap.put("postDate", "2019-03-11"); + + upateRequest.doc(jsonMap); + + // + upateRequest.docAsUpsert(true); + // upsert 方法表示如果数据不存在，那么就新增一条 + upateRequest.upsert(jsonMap); + + client.update(upateRequest, RequestOptions.DEFAULT); + System.out.println("更新成功！"); + + } + + /** + * 删除 + * @throws IOException + * + */ + private static void delete() throws IOException { + + String index = "user"; + String type = "userindex"; + // 唯一编号 + String id = "1"; + DeleteRequest deleteRequest=new DeleteRequest(); + deleteRequest.id(id); + deleteRequest.index(index); + deleteRequest.type(type); + + //设置超时时间 + deleteRequest.timeout(TimeValue.timeValueMinutes(2)); + //设置刷新策略"wait_for" + //保持此请求打开，直到刷新使此请求的内容可以搜索为止。此刷新策略与高索引和搜索吞吐量兼容，但它会导致请求等待响应，直到发生刷新 + deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); + + //同步删除 + DeleteResponse deleteResponse = client.delete(deleteRequest, RequestOptions.DEFAULT); + + /* + * 异步删除操作 + */ + + //进行监听 + ActionListener listener = new ActionListener() { + @Override + public void onResponse(DeleteResponse deleteResponse) { + System.out.println("响应:"+deleteResponse); + } + + @Override + public void onFailure(Exception e) { + System.out.println("删除监听异常:"+e.getMessage()); + } + }; + + //异步删除 +// client.deleteAsync(deleteRequest, RequestOptions.DEFAULT, listener); + + ReplicationResponse.ShardInfo shardInfo = deleteResponse.getShardInfo(); + //如果处理成功碎片的数量少于总碎片的情况,说明还在处理或者处理发生异常 + if (shardInfo.getTotal() != shardInfo.getSuccessful()) { + System.out.println("需要处理的碎片总量:"+shardInfo.getTotal()); + System.out.println("处理成功的碎片总量:"+shardInfo.getSuccessful()); + } + + if (shardInfo.getFailed() > 0) { + for (ReplicationResponse.ShardInfo.Failure failure : + shardInfo.getFailures()) { + String reason = failure.reason(); + } + } + System.out.println("删除成功!"); + } + + + /** + * 批量操作示例 + * @throws InterruptedException + */ + private static void bulk() throws IOException, InterruptedException { + String index = "estest"; + String type = "estest"; + + BulkRequest request = new BulkRequest(); + //批量新增 + request.add(new IndexRequest(index, type, "1") + .source(XContentType.JSON,"field", "foo")); + request.add(new IndexRequest("posts", type, "2") + .source(XContentType.JSON,"field", "bar")); + request.add(new IndexRequest("posts", type, "3") + .source(XContentType.JSON,"field", "baz")); + + //可以进行修改/删除/新增 操作 + request.add(new UpdateRequest("posts", "doc", "2") + .doc(XContentType.JSON,"other", "test")); + request.add(new DeleteRequest("posts", "doc", "3")); + request.add(new IndexRequest("posts", "doc", "4") + .source(XContentType.JSON,"field", "baz")); + + + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + + //可以快速检查一个或多个操作是否失败 true是有至少一个失败！ + if (bulkResponse.hasFailures()) { + System.out.println("有一个操作失败!"); + } + + //对处理结果进行遍历操作并根据不同的操作进行处理 + for (BulkItemResponse bulkItemResponse : bulkResponse) { + DocWriteResponse itemResponse = bulkItemResponse.getResponse(); + + //操作失败的进行处理 + if (bulkItemResponse.isFailed()) { + BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); + + } + + if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.INDEX + || bulkItemResponse.getOpType() == DocWriteRequest.OpType.CREATE) { + IndexResponse indexResponse = (IndexResponse) itemResponse; + + } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.UPDATE) { + UpdateResponse updateResponse = (UpdateResponse) itemResponse; + + } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.DELETE) { + DeleteResponse deleteResponse = (DeleteResponse) itemResponse; + } + } + + System.out.println("批量执行成功！"); + + + + /* + * 批量执行处理器相关示例代码 + */ + + + //批量处理器的监听器设置 + + BulkProcessor.Listener listener = new BulkProcessor.Listener() { + + //在执行BulkRequest的每次执行之前调用，这个方法允许知道将要在BulkRequest中执行的操作的数量 + @Override + public void beforeBulk(long executionId, BulkRequest request) { + int numberOfActions = request.numberOfActions(); + logger.debug("Executing bulk [{}] with {} requests", + executionId, numberOfActions); + } + + //在每次执行BulkRequest之后调用，这个方法允许知道BulkResponse是否包含错误 + @Override + public void afterBulk(long executionId, BulkRequest request, + BulkResponse response) { + if (response.hasFailures()) { + logger.warn("Bulk [{}] executed with failures", executionId); + } else { + logger.debug("Bulk [{}] completed in {} milliseconds", + executionId, response.getTook().getMillis()); + } + } + + //如果BulkRequest失败，则调用该方法，该方法允许知道失败 + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + logger.error("Failed to execute bulk", failure); + } + }; + + BiConsumer> bulkConsumer = + (request2, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener); + //创建一个批量执行的处理器 + BulkProcessor bulkProcessor = BulkProcessor.builder(bulkConsumer, listener).build(); + BulkProcessor.Builder builder = BulkProcessor.builder(bulkConsumer, listener); + //根据当前添加的操作数量设置刷新新批量请求的时间(默认为1000，使用-1禁用它) + builder.setBulkActions(500); + //根据当前添加的操作大小设置刷新新批量请求的时间(默认为5Mb，使用-1禁用) + builder.setBulkSize(new ByteSizeValue(1L, ByteSizeUnit.MB)); + //设置允许执行的并发请求数量(默认为1，使用0只允许执行单个请求) + builder.setConcurrentRequests(0); + //设置刷新间隔如果间隔通过，则刷新任何挂起的BulkRequest(默认为未设置) + builder.setFlushInterval(TimeValue.timeValueSeconds(10L)); + //设置一个常量后退策略，该策略最初等待1秒并重试最多3次。 + builder.setBackoffPolicy(BackoffPolicy + .constantBackoff(TimeValue.timeValueSeconds(1L), 3)); + + + IndexRequest one = new IndexRequest("posts", "doc", "1"). + source(XContentType.JSON, "title", + "In which order are my Elasticsearch queries executed?"); + IndexRequest two = new IndexRequest("posts", "doc", "2") + .source(XContentType.JSON, "title", + "Current status and upcoming changes in Elasticsearch"); + IndexRequest three = new IndexRequest("posts", "doc", "3") + .source(XContentType.JSON, "title", + "The Future of Federated Search in Elasticsearch"); + bulkProcessor.add(one); + bulkProcessor.add(two); + bulkProcessor.add(three); + + + //如果所有大容量请求都已完成，则该方法返回true;如果在所有大容量请求完成之前的等待时间已经过去，则返回false + boolean terminated = bulkProcessor.awaitClose(30L, TimeUnit.SECONDS); + + System.out.println("请求的响应结果:"+terminated); + + + } + +} diff --git a/src/main/java/com/pancm/thread/test/NotifyTest.java b/src/main/java/com/pancm/thread/test/NotifyTest.java index 6453d5e..460c056 100644 --- a/src/main/java/com/pancm/thread/test/NotifyTest.java +++ b/src/main/java/com/pancm/thread/test/NotifyTest.java @@ -1,53 +1,53 @@ -package com.pancm.thread.test; - -import java.util.Random; - -/** - * @Title: NotifyTest - * @Description: - * wait 和 notify测试 - * @Version:1.0.0 - * @author pancm - * @date 2018年5月22日 - */ -public class NotifyTest { - - public static void main(String[] args) { - Test4 t1 = new Test4("张三"); - Test4 t2 = new Test4("李四"); - t1.start(); - t2.start(); - } -} - -class Test4 extends Thread { - private String name; - public Test4(String name) { - super(name); - this.name=name; - } - - @Override - public void run() { - System.out.println(this.getName() + " 线程运行开始!"); - - for (int i = 0; i < 5; i++) { - System.out.println("子线程" + this.getName() + "运行 : " + i); - try { - sleep(new Random().nextInt(100)); - } catch (InterruptedException e) { - e.printStackTrace(); - } - try { - if("李四".equals(this.getName())){ - this.getName().wait(); - } - } catch (InterruptedException e) { - e.printStackTrace(); - } - this.getName().notify(); - - } - System.out.println(this.getName() + " 线程运行结束!"); - } -} +package com.pancm.thread.test; + +import java.util.Random; + +/** + * @Title: NotifyTest + * @Description: + * wait 和 notify测试 + * @Version:1.0.0 + * @author pancm + * @date 2018年5月22日 + */ +public class NotifyTest { + + public static void main(String[] args) { + Test4 t1 = new Test4("张三"); + Test4 t2 = new Test4("李四"); + t1.start(); + t2.start(); + } +} + +class Test4 extends Thread { + private String name; + public Test4(String name) { + super(name); + this.name=name; + } + + @Override + public void run() { + System.out.println(this.getName() + " 线程运行开始!"); + + for (int i = 0; i < 5; i++) { + System.out.println("子线程" + this.getName() + "运行 : " + i); + try { + sleep(new Random().nextInt(100)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + try { + if("李四".equals(this.getName())){ + this.getName().wait(); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + this.getName().notify(); + + } + System.out.println(this.getName() + " 线程运行结束!"); + } +} From 698b86c8978592c984738e277e5c82791b2a369b Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Tue, 12 Mar 2019 17:37:34 +0800 Subject: [PATCH 06/53] =?UTF-8?q?ES=E9=AB=98=E7=BA=A7=E5=AE=A2=E6=88=B7?= =?UTF-8?q?=E7=AB=AF=E4=BD=BF=E7=94=A8=E7=A4=BA=E4=BE=8B=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../easticsearch/EsHighLevelRestTest1.java | 22 +- .../easticsearch/EsHighLevelRestTest2.java | 343 ++++++++++++++++++ 2 files changed, 355 insertions(+), 10 deletions(-) create mode 100644 src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest2.java diff --git a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java b/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java index 3d46fa3..51a376e 100644 --- a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java +++ b/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java @@ -41,7 +41,8 @@ /** * @Title: EsHighLevelRestTest1 - * @Description: Java High Level REST Client Es高级客户端使用教程一 (基本使用) + * @Description: Java High Level REST Client Es高级客户端使用教程一 (基本CRUD使用) + 官方文档地址: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high.html * @since jdk 1.8 * @Version:1.0.0 * @author pancm @@ -64,8 +65,9 @@ public static void main(String[] args) { exists(); update(); delete(); + bulk(); close(); - } catch (IOException e) { + } catch (Exception e) { e.printStackTrace(); } @@ -346,16 +348,16 @@ private static void bulk() throws IOException, InterruptedException { //批量新增 request.add(new IndexRequest(index, type, "1") .source(XContentType.JSON,"field", "foo")); - request.add(new IndexRequest("posts", type, "2") + request.add(new IndexRequest(index, type, "2") .source(XContentType.JSON,"field", "bar")); - request.add(new IndexRequest("posts", type, "3") + request.add(new IndexRequest(index, type, "3") .source(XContentType.JSON,"field", "baz")); //可以进行修改/删除/新增 操作 - request.add(new UpdateRequest("posts", "doc", "2") + request.add(new UpdateRequest(index, type, "2") .doc(XContentType.JSON,"other", "test")); - request.add(new DeleteRequest("posts", "doc", "3")); - request.add(new IndexRequest("posts", "doc", "4") + request.add(new DeleteRequest(index, type, "3")); + request.add(new IndexRequest(index, type, "4") .source(XContentType.JSON,"field", "baz")); @@ -446,13 +448,13 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) .constantBackoff(TimeValue.timeValueSeconds(1L), 3)); - IndexRequest one = new IndexRequest("posts", "doc", "1"). + IndexRequest one = new IndexRequest(index, type, "1"). source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?"); - IndexRequest two = new IndexRequest("posts", "doc", "2") + IndexRequest two = new IndexRequest(index, type, "2") .source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch"); - IndexRequest three = new IndexRequest("posts", "doc", "3") + IndexRequest three = new IndexRequest(index, type, "3") .source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch"); bulkProcessor.add(one); diff --git a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest2.java b/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest2.java new file mode 100644 index 0000000..88dcae3 --- /dev/null +++ b/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest2.java @@ -0,0 +1,343 @@ +package com.pancm.sql.easticsearch; + +import static org.junit.Assert.assertNull; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.http.HttpHost; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetItemResponse; +import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.RethrottleRequest; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.index.reindex.ReindexRequest; +import org.elasticsearch.index.reindex.ScrollableHitSource; +import org.elasticsearch.index.reindex.UpdateByQueryRequest; +import org.elasticsearch.tasks.TaskId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @Title: EsHighLevelRestTest2 + * @Description: Java High Level REST Client Es高级客户端使用教程二 (关于组合使用) + * @since jdk 1.8 + * @Version:1.0.0 + * @author pancm + * @date 2019年3月11日 + */ +public class EsHighLevelRestTest2 { + + private static String elasticIp = "192.169.0.23"; + private static int elasticPort = 9200; + + private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestTest2.class); + + private static RestHighLevelClient client = null; + + /** + * @param args + */ + public static void main(String[] args) { + try { + init(); + multiGet(); + reindex(); + updataByQuery(); + deleteByQuery(); + rethrottleByQuery(); + close(); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + /* + * 初始化服务 + */ + private static void init() { + client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort, "http"))); + + } + + /* + * 关闭服务 + */ + private static void close() { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + /** + * 多查询使用 + * + * @throws IOException + */ + private static void multiGet() throws IOException { + + MultiGetRequest request = new MultiGetRequest(); + request.add(new MultiGetRequest.Item("estest", "estest", "1")); + request.add(new MultiGetRequest.Item("user", "userindex", "2")); + // 禁用源检索，默认启用 +// request.add(new MultiGetRequest.Item("user", "userindex", "2").fetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE)); + + // 同步构建 + MultiGetResponse response = client.mget(request, RequestOptions.DEFAULT); + + // 异步构建 +// MultiGetResponse response2 = client.mgetAsync(request, RequestOptions.DEFAULT, listener); + + /* + * 返回的MultiGetResponse包含在' getResponses中的MultiGetItemResponse的列表，其顺序与请求它们的顺序相同。 + * 如果成功，MultiGetItemResponse包含GetResponse或MultiGetResponse。如果失败了就失败。 + * 成功看起来就像一个正常的GetResponse + */ + + for (MultiGetItemResponse item : response.getResponses()) { + assertNull(item.getFailure()); + GetResponse get = item.getResponse(); + String index = item.getIndex(); + String type = item.getType(); + String id = item.getId(); + // 如果请求存在 + if (get.isExists()) { + long version = get.getVersion(); + String sourceAsString = get.getSourceAsString(); + Map sourceAsMap = get.getSourceAsMap(); + byte[] sourceAsBytes = get.getSourceAsBytes(); + System.out.println("查询的结果:" + sourceAsMap); + } else { + System.out.println("没有找到该文档!"); + } + + } + + } + + /** + * 索引复制 + * + * @throws IOException + */ + private static void reindex() throws IOException { + // 创建索引复制请求并进行索引复制 + ReindexRequest request = new ReindexRequest(); + // 需要复制的索引 + request.setSourceIndices("user"); + // 复制的目标索引 + request.setDestIndex("dest_test"); + + // 表示如果在复制索引的时候有缺失的文档的话会进行创建,默认是index + request.setDestOpType("create"); + // 如果在复制的过程中发现版本冲突，那么会继续进行复制 + request.setConflicts("proceed"); + + // 只复制文档类型为 userindex 的数据 + request.setSourceDocTypes("userindex"); + // 只复制 pancm 用户的数据 + request.setSourceQuery(new TermQueryBuilder("user", "pancm")); + // 设置复制文档的数量 + request.setSize(10); + // 设置一次批量处理的条数，默认是1000 + request.setSourceBatchSize(100); + + // 进行排序 +// request.addSortField("postDate", SortOrder.DESC); + + // 指定切片大小 + request.setSlices(2); + + //设置超时时间 + request.setTimeout(TimeValue.timeValueMinutes(2)); + //允许刷新 + request.setRefresh(true); + + // 同步执行 + BulkByScrollResponse bulkResponse = client.reindex(request, RequestOptions.DEFAULT); + + // 异步执行 +// client.reindexAsync(request, RequestOptions.DEFAULT, listener); + + // 响应结果处理 + + TimeValue timeTaken = bulkResponse.getTook(); + boolean timedOut = bulkResponse.isTimedOut(); + long totalDocs = bulkResponse.getTotal(); + long updatedDocs = bulkResponse.getUpdated(); + long createdDocs = bulkResponse.getCreated(); + long deletedDocs = bulkResponse.getDeleted(); + long batches = bulkResponse.getBatches(); + long noops = bulkResponse.getNoops(); + long versionConflicts = bulkResponse.getVersionConflicts(); + long bulkRetries = bulkResponse.getBulkRetries(); + long searchRetries = bulkResponse.getSearchRetries(); + TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); + TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); + List searchFailures = bulkResponse.getSearchFailures(); + List bulkFailures = bulkResponse.getBulkFailures(); + + System.out.println("索引复制总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",创建数:" + createdDocs + + ",更新数:" + updatedDocs); + } + + + + /** + * 根据查询条件更新 + * + * @throws IOException + */ + private static void updataByQuery() throws IOException { + // + UpdateByQueryRequest request = new UpdateByQueryRequest("user"); + + // 设置查询条件 + request.setQuery(new TermQueryBuilder("user", "pancm")); + + // 设置复制文档的数量 + request.setSize(10); + // 设置一次批量处理的条数，默认是1000 + request.setBatchSize(100); + //设置路由 + request.setRouting("=cat"); + //设置超时时间 + request.setTimeout(TimeValue.timeValueMinutes(2)); + //允许刷新 + request.setRefresh(true); + //索引选项 + request.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); + + // 同步执行 + BulkByScrollResponse bulkResponse = client.updateByQuery(request, RequestOptions.DEFAULT); + + // 异步执行 +// client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener); + + // 返回结果 + TimeValue timeTaken = bulkResponse.getTook(); + boolean timedOut = bulkResponse.isTimedOut(); + long totalDocs = bulkResponse.getTotal(); + long updatedDocs = bulkResponse.getUpdated(); + long deletedDocs = bulkResponse.getDeleted(); + long batches = bulkResponse.getBatches(); + long noops = bulkResponse.getNoops(); + long versionConflicts = bulkResponse.getVersionConflicts(); + long bulkRetries = bulkResponse.getBulkRetries(); + long searchRetries = bulkResponse.getSearchRetries(); + TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); + TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); + List searchFailures = bulkResponse.getSearchFailures(); + List bulkFailures = bulkResponse.getBulkFailures(); + System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",更新数:" + updatedDocs); + + } + + + /** + * 根据查询条件删除 + * @throws IOException + */ + private static void deleteByQuery() throws IOException { + // + DeleteByQueryRequest request = new DeleteByQueryRequest("user"); + + // 设置查询条件 + request.setQuery(new TermQueryBuilder("user", "pancm")); + + // 设置复制文档的数量 + request.setSize(10); + // 设置一次批量处理的条数，默认是1000 + request.setBatchSize(100); + //设置路由 + request.setRouting("=cat"); + //设置超时时间 + request.setTimeout(TimeValue.timeValueMinutes(2)); + //允许刷新 + request.setRefresh(true); + //索引选项 + request.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); + + // 同步执行 + BulkByScrollResponse bulkResponse = client.deleteByQuery(request, RequestOptions.DEFAULT); + + // 异步执行 +// client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener); + + // 返回结果 + TimeValue timeTaken = bulkResponse.getTook(); + boolean timedOut = bulkResponse.isTimedOut(); + long totalDocs = bulkResponse.getTotal(); + long deletedDocs = bulkResponse.getDeleted(); + long batches = bulkResponse.getBatches(); + long noops = bulkResponse.getNoops(); + long versionConflicts = bulkResponse.getVersionConflicts(); + long bulkRetries = bulkResponse.getBulkRetries(); + long searchRetries = bulkResponse.getSearchRetries(); + TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); + TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); + List searchFailures = bulkResponse.getSearchFailures(); + List bulkFailures = bulkResponse.getBulkFailures(); + System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",删除数:" + deletedDocs); + + } + + + /** + * 用于更改正在运行的重索引、逐查询更新或逐查询删除任务的当前节流，或完全禁用任务的节流。 + * @throws IOException + */ + private static void rethrottleByQuery() throws IOException { + TaskId taskId=new TaskId("1"); + //用于更改正在运行的重索引、逐查询更新或逐查询删除任务的当前节流，或完全禁用任务的节流。 + //并且将请求将任务的节流更改为每秒100个请求 + RethrottleRequest request = new RethrottleRequest(taskId,100.0f); + + // 同步设置需要更改的流 +// client.reindexRethrottle(request, RequestOptions.DEFAULT); +// client.updateByQueryRethrottle(request, RequestOptions.DEFAULT); +// client.deleteByQueryRethrottle(request, RequestOptions.DEFAULT); + + + ActionListener listener = new ActionListener() { + @Override + public void onResponse(ListTasksResponse response) { + System.out.println("===="+response.getTasks().toString()); + } + + @Override + public void onFailure(Exception e) { + System.out.println("====---"+e.getMessage()); + } + }; + + // 异步设置要更改的流 + client.reindexRethrottleAsync(request, RequestOptions.DEFAULT, listener); + client.updateByQueryRethrottleAsync(request, RequestOptions.DEFAULT, listener); + client.deleteByQueryRethrottleAsync(request, RequestOptions.DEFAULT, listener); + + System.out.println("已成功设置!"); + + } + + + + +} From 3444f3bf97fae18ace648ecbd1947b13823f2b83 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Wed, 13 Mar 2019 17:50:13 +0800 Subject: [PATCH 07/53] =?UTF-8?q?ES=E9=AB=98=E7=BA=A7=E5=AE=A2=E6=88=B7?= =?UTF-8?q?=E7=AB=AFSearch=E7=9B=B8=E5=85=B3=E7=A4=BA=E4=BE=8B=E4=BB=A3?= =?UTF-8?q?=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../EsHighLevelRestSearchTest.java | 344 ++++++++++++++++++ .../easticsearch/EsHighLevelRestTest1.java | 6 +- 2 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestSearchTest.java diff --git a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestSearchTest.java b/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestSearchTest.java new file mode 100644 index 0000000..9342051 --- /dev/null +++ b/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestSearchTest.java @@ -0,0 +1,344 @@ +package com.pancm.sql.easticsearch; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.http.HttpHost; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.suggest.SuggestBuilders; +import org.elasticsearch.search.suggest.SuggestionBuilder; +import org.elasticsearch.search.suggest.term.TermSuggestion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * @Title: EsHighLevelRestSearchTest + * @Description: Java High Level REST Client Es高级客户端查询使用使用教程 (Search查询使用教程) + * 官方文档地址: + * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high.html + * @Version:1.0.0 + * @author pancm + * @date 2019年3月12日 + */ +public class EsHighLevelRestSearchTest { + + private static String elasticIp = "192.169.0.23"; + private static int elasticPort = 9200; + + private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestSearchTest.class); + + private static RestHighLevelClient client = null; + + /** + * @param args + */ + public static void main(String[] args) { + + try { + init(); + search(); + + + } catch (Exception e) { + e.printStackTrace(); + }finally { + close(); + } + + } + + /* + * 初始化服务 + */ + private static void init() { + client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort, "http"))); + + } + + /* + * 关闭服务 + */ + private static void close() { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + /** + * search查询使用示例 + * + * @throws IOException + */ + private static void search() throws IOException { + + /* + * 查询集群所有的索引 + * + */ + SearchRequest searchRequestAll = new SearchRequest(); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchRequestAll.source(searchSourceBuilder); + + // 同步查询 + SearchResponse searchResponseAll = client.search(searchRequestAll, RequestOptions.DEFAULT); + + System.out.println("所有查询总数:" + searchResponseAll.getHits().getTotalHits()); + + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest("user"); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + // 设置查询条件 + sourceBuilder.query(QueryBuilders.termQuery("user", "pancm")); + // 设置起止和结束 + sourceBuilder.from(0); + sourceBuilder.size(5); + sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS)); + // 设置路由 +// searchRequest.routing("routing"); + // 设置索引库表达式 + searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); + // 查询选择本地分片，默认是集群分片 + searchRequest.preference("_local"); + + // 排序 + // 根据默认值进行降序排序 +// sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); + // 根据字段进行升序排序 +// sourceBuilder.sort(new FieldSortBuilder("id").order(SortOrder.ASC)); + + // 关闭suorce查询 +// sourceBuilder.fetchSource(false); + + String[] includeFields = new String[] { "title", "user", "innerObject.*" }; + String[] excludeFields = new String[] { "_type" }; + // 包含或排除字段 +// sourceBuilder.fetchSource(includeFields, excludeFields); + + searchRequest.source(sourceBuilder); + + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + + // HTTP状态代码、执行时间或请求是否提前终止或超时 + RestStatus status = searchResponse.status(); + TimeValue took = searchResponse.getTook(); + Boolean terminatedEarly = searchResponse.isTerminatedEarly(); + boolean timedOut = searchResponse.isTimedOut(); + + // 供关于受搜索影响的切分总数的统计信息，以及成功和失败的切分 + int totalShards = searchResponse.getTotalShards(); + int successfulShards = searchResponse.getSuccessfulShards(); + int failedShards = searchResponse.getFailedShards(); + // 失败的原因 + for (ShardSearchFailure failure : searchResponse.getShardFailures()) { + // failures should be handled here + } + + // 结果 + searchResponse.getHits().forEach(hit -> { + Map map = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("普通查询的Map结果:" + map); + System.out.println("普通查询的String结果:" + string); + }); + + + System.out.println("\n=================\n"); + + /* + * 全文查询使用示例 + */ + + // 搜索字段user为pancm的数据 + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("user", "pancm"); + + // 设置模糊查询 + matchQueryBuilder.fuzziness(Fuzziness.AUTO); + // 设置前缀长度 + matchQueryBuilder.prefixLength(3); + // 设置最大扩展选项来控制查询的模糊过程 + matchQueryBuilder.maxExpansions(10); + + /* + * QueryBuilder也可以 + */ + +// QueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("user", "kimchy") +// .fuzziness(Fuzziness.AUTO) +// .prefixLength(3) +// .maxExpansions(10); + + SearchSourceBuilder searchSourceBuilder2 = new SearchSourceBuilder(); + searchSourceBuilder2.query(matchQueryBuilder); + + SearchRequest searchRequest2 = new SearchRequest(); + searchRequest2.source(searchSourceBuilder2); + // 同步查询 + SearchResponse searchResponse2 = client.search(searchRequest, RequestOptions.DEFAULT); + + SearchHits hits = searchResponse.getHits(); + //总条数和分值 + long totalHits = hits.getTotalHits(); + float maxScore = hits.getMaxScore(); + + + + hits.forEach(hit -> { + + String index = hit.getIndex(); + String type = hit.getType(); + String id = hit.getId(); + float score = hit.getScore(); + + + + Map sourceAsMap = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("Match查询的Map结果:" + sourceAsMap); + System.out.println("Match查询的String结果:" + string); + + String documentTitle = (String) sourceAsMap.get("title"); +// List users = (List) sourceAsMap.get("user"); + Map innerObject = + (Map) sourceAsMap.get("innerObject"); + }); + + + + + System.out.println("\n=================\n"); + + /* + * 高亮查询 + */ + + SearchSourceBuilder searchSourceBuilder3 = new SearchSourceBuilder(); + HighlightBuilder highlightBuilder = new HighlightBuilder(); + HighlightBuilder.Field highlightTitle = new HighlightBuilder.Field("title"); + // 设置字段高亮字体 + highlightTitle.highlighterType("user"); + highlightBuilder.field(highlightTitle); + HighlightBuilder.Field highlightUser = new HighlightBuilder.Field("user"); + highlightBuilder.field(highlightUser); + searchSourceBuilder3.highlighter(highlightBuilder); + + SearchRequest searchRequest3 = new SearchRequest(); + searchRequest3.source(searchSourceBuilder3); + + // 同步查询 + SearchResponse searchResponse3 = client.search(searchRequest3, RequestOptions.DEFAULT); + + searchResponse3.getHits().forEach(hit -> { + + Map map = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("Highlight查询的Map结果:" + map); + System.out.println("Highlight查询的String结果:" + string); + }); + + System.out.println("\n=================\n"); + + /** + * 聚合查询 + */ + + SearchSourceBuilder searchSourceBuilder4 = new SearchSourceBuilder(); + + //terms 就是分组统计 根据user进行分组并创建一个新的聚合user_ + TermsAggregationBuilder aggregation = AggregationBuilders.terms("user_").field("user"); + aggregation.subAggregation(AggregationBuilders.avg("average_age").field("age")); + searchSourceBuilder4.aggregation(aggregation); + + SearchRequest searchRequest4 = new SearchRequest(); + searchRequest4.source(searchSourceBuilder4); + + // 同步查询 + SearchResponse searchResponse4 = client.search(searchRequest4, RequestOptions.DEFAULT); + //聚合查询返回条件 + Aggregations aggregations = searchResponse4.getAggregations(); + List aggregationList = aggregations.asList(); + + for (Aggregation agg : aggregations) { + String type = agg.getType(); + if (type.equals(TermsAggregationBuilder.NAME)) { + Bucket elasticBucket = ((Terms) agg).getBucketByKey("Elastic"); + long numberOfDocs = elasticBucket.getDocCount(); + System.out.println("条数:"+numberOfDocs); + } + } + + +// Terms byCompanyAggregation = aggregations.get("user_"); +// Bucket elasticBucket = byCompanyAggregation.getBucketByKey("Elastic"); +// Avg averageAge = elasticBucket.getAggregations().get("average_age"); +// double avg = averageAge.getValue(); +// +// System.out.println("聚合查询返回的结果:"+avg); + + + /* + * 建议查询 + */ + SearchSourceBuilder searchSourceBuilder5 = new SearchSourceBuilder(); + SuggestionBuilder termSuggestionBuilder = SuggestBuilders.termSuggestion("user").text("pancm"); + SuggestBuilder suggestBuilder = new SuggestBuilder(); + suggestBuilder.addSuggestion("suggest_user", termSuggestionBuilder); + searchSourceBuilder5.suggest(suggestBuilder); + + SearchRequest searchRequest5 = new SearchRequest(); + searchRequest5.source(searchSourceBuilder5); + + // 同步查询 + SearchResponse searchResponse5= client.search(searchRequest5, RequestOptions.DEFAULT); + + Suggest suggest = searchResponse5.getSuggest(); + TermSuggestion termSuggestion = suggest.getSuggestion("suggest_user"); + for (TermSuggestion.Entry entry : termSuggestion.getEntries()) { + for (TermSuggestion.Entry.Option option : entry) { + String suggestText = option.getText().string(); + System.out.println("返回结果:"+suggestText); + } + } + + + + + + + + + + } + +} diff --git a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java b/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java index 51a376e..74bf214 100644 --- a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java +++ b/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java @@ -111,7 +111,7 @@ private static void careatindex() throws IOException { * 第一种方式，通过jsonString进行创建 */ // json - String jsonString = "{" + "\"user\":\"pancm\"," + "\"postDate\":\"2019-03-08\"," + String jsonString = "{" + "\"user\":\"pancm\"," + "\"postDate\":\"2019-03-08\","+ "\"age\":\"18\"," + "\"message\":\"study Elasticsearch\"" + "}"; request.source(jsonString, XContentType.JSON); @@ -123,6 +123,7 @@ private static void careatindex() throws IOException { Map jsonMap = new HashMap<>(); jsonMap.put("user", "pancm"); jsonMap.put("postDate", "2019-03-08"); + jsonMap.put("age", "18"); jsonMap.put("message", "study Elasticsearch"); request.source(jsonMap); @@ -136,6 +137,7 @@ private static void careatindex() throws IOException { { builder.field("user", "pancm"); builder.timeField("postDate", "2019-03-08"); + builder.field("age", "18"); builder.field("message", "study Elasticsearch"); } builder.endObject(); @@ -355,7 +357,7 @@ private static void bulk() throws IOException, InterruptedException { //可以进行修改/删除/新增 操作 request.add(new UpdateRequest(index, type, "2") - .doc(XContentType.JSON,"other", "test")); + .doc(XContentType.JSON,"field", "test")); request.add(new DeleteRequest(index, type, "3")); request.add(new IndexRequest(index, type, "4") .source(XContentType.JSON,"field", "baz")); From 3f26e9165d9db8b4eadfb090a28de72e2b80cfc4 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Thu, 14 Mar 2019 21:16:15 +0800 Subject: [PATCH 08/53] =?UTF-8?q?pom=E5=A2=9E=E5=8A=A0=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e4d103a..b28295b 100644 --- a/pom.xml +++ b/pom.xml @@ -438,11 +438,19 @@ io.searchbox jest 6.3.1 - + + + + + + spring-milestone + http://repo.spring.io/libs-release + + From a4660c8c10a5fd860d9ea7829454692f18d8c4d6 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Sat, 30 Mar 2019 19:01:51 +0800 Subject: [PATCH 09/53] =?UTF-8?q?es=20=E6=B5=8B=E8=AF=95=E7=B1=BB=E6=8F=90?= =?UTF-8?q?=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../EsHighLevelRestSearchTest.java | 31 +- .../easticsearch/EsHighLevelRestTest1.java | 449 ++++++++++-------- .../com/pancm/sql/easticsearch/JestTest.java | 47 +- 3 files changed, 294 insertions(+), 233 deletions(-) diff --git a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestSearchTest.java b/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestSearchTest.java index 9342051..6f215bf 100644 --- a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestSearchTest.java +++ b/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestSearchTest.java @@ -1,9 +1,4 @@ -package com.pancm.sql.easticsearch; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; +package com.pancm.test.esTest; import org.apache.http.HttpHost; import org.elasticsearch.action.search.SearchRequest; @@ -12,6 +7,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.unit.TimeValue; @@ -35,6 +31,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + /** * @Title: EsHighLevelRestSearchTest @@ -49,7 +50,6 @@ public class EsHighLevelRestSearchTest { private static String elasticIp = "192.169.0.23"; private static int elasticPort = 9200; - private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestSearchTest.class); private static RestHighLevelClient client = null; @@ -76,7 +76,8 @@ public static void main(String[] args) { * 初始化服务 */ private static void init() { - client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort, "http"))); + RestClientBuilder restClientBuilder =RestClient.builder(new HttpHost(elasticIp, elasticPort)); + client = new RestHighLevelClient(restClientBuilder); } @@ -89,6 +90,8 @@ private static void close() { client.close(); } catch (IOException e) { e.printStackTrace(); + }finally{ + client=null; } } } @@ -331,14 +334,10 @@ private static void search() throws IOException { } } - - - - - - - - } + + + + } diff --git a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java b/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java index 74bf214..98f4f97 100644 --- a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java +++ b/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java @@ -1,4 +1,4 @@ -package com.pancm.sql.easticsearch; +package com.pancm.test.esTest; import java.io.IOException; import java.util.HashMap; @@ -11,6 +11,10 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkProcessor; @@ -41,8 +45,8 @@ /** * @Title: EsHighLevelRestTest1 - * @Description: Java High Level REST Client Es高级客户端使用教程一 (基本CRUD使用) - 官方文档地址: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high.html + * @Description: Java High Level REST Client Es高级客户端使用教程一 (基本CRUD使用) 官方文档地址: + * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high.html * @since jdk 1.8 * @Version:1.0.0 * @author pancm @@ -54,7 +58,7 @@ public class EsHighLevelRestTest1 { private static int elasticPort = 9200; private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestTest1.class); - + private static RestHighLevelClient client = null; public static void main(String[] args) { @@ -77,7 +81,8 @@ public static void main(String[] args) { * 初始化服务 */ private static void init() { - client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort, "http"))); + + client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort))); } @@ -111,7 +116,7 @@ private static void careatindex() throws IOException { * 第一种方式，通过jsonString进行创建 */ // json - String jsonString = "{" + "\"user\":\"pancm\"," + "\"postDate\":\"2019-03-08\","+ "\"age\":\"18\"," + String jsonString = "{" + "\"user\":\"pancm\"," + "\"postDate\":\"2019-03-08\"," + "\"age\":\"18\"," + "\"message\":\"study Elasticsearch\"" + "}"; request.source(jsonString, XContentType.JSON); @@ -144,11 +149,9 @@ private static void careatindex() throws IOException { request.source(builder); IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT); - - - - //对响应结果进行处理 - + + // 对响应结果进行处理 + String index1 = indexResponse.getIndex(); String type1 = indexResponse.getType(); String id1 = indexResponse.getId(); @@ -169,7 +172,82 @@ private static void careatindex() throws IOException { } } - System.out.println("创建成功！"); + System.out.println("创建索引库成功！"); + + // 类型 + String type2 = "student"; + String index2 = "student"; + + // setting 的值 + Map setmapping = new HashMap<>(); + + // 分区数、路由分片数、副本数、缓存刷新时间 + setmapping.put("number_of_shards", 12); + setmapping.put("number_of_routing_shards", 24); + setmapping.put("number_of_replicas", 1); + setmapping.put("refresh_interval", "5s"); + + + + // mapping 的值 +// Map mapping = new HashMap<>(); +// +// mapping.put("id", "long"); +// mapping.put("name", "keyword"); + + + Map jsonMap2 = new HashMap<>(); + Map message = new HashMap<>(); + //设置类型 + message.put("type", "text"); + Map properties = new HashMap<>(); + //设置字段message信息 + properties.put("message", message); + Map mapping = new HashMap<>(); + mapping.put("properties", properties); + jsonMap2.put(type2, mapping); + + + GetIndexRequest getRequest2 = new GetIndexRequest(); + getRequest2.indices(index); + getRequest2.local(false); + getRequest2.humanReadable(true); + boolean exists2 = client.indices().exists(getRequest2, RequestOptions.DEFAULT); + //如果存在就不创建了 + if(exists2) { + System.out.println(type2+"索引库已经存在!"); + return; + } + + // 开始创建库 + CreateIndexRequest request2 = new CreateIndexRequest(index2); + try { + // 加载数据类型 + request2.settings(setmapping); + //第二种方式 +// request2.settings(Settings.builder() +// .put("index.number_of_shards", 3) +// .put("index.number_of_replicas", 1)); + + + //设置mapping参数 + request2.mapping(type2, jsonMap2); + + //设置别名 + request2.alias(new Alias("user_alias")); + + + CreateIndexResponse createIndexResponse = client.indices().create(request2, RequestOptions.DEFAULT); + boolean falg = createIndexResponse.isAcknowledged(); + logger.info("创建索引库" + index + ",状态为:" + falg); + } catch (IOException e) { + logger.error("创建INDEX报错", e); + } catch (NullPointerException e) { + logger.error("模板文件中的mappings或settings不能为空", e); + } + + + } /** @@ -194,7 +272,7 @@ private static void get() { } catch (ElasticsearchException e) { // 如果是索引不存在 if (e.status() == RestStatus.NOT_FOUND) { - System.out.println("该索引库不存在！"+index); + System.out.println("该索引库不存在！" + index); } } @@ -204,17 +282,17 @@ private static void get() { String sourceAsString = getResponse.getSourceAsString(); Map sourceAsMap = getResponse.getSourceAsMap(); byte[] sourceAsBytes = getResponse.getSourceAsBytes(); - System.out.println("查询返回结果String:"+sourceAsString); - System.out.println("查询返回结果Map:"+sourceAsMap); + System.out.println("查询返回结果String:" + sourceAsString); + System.out.println("查询返回结果Map:" + sourceAsMap); } else { System.out.println("没有找到该数据！"); } } - - + /** - * 是否存在 - * @throws IOException + * 是否存在 + * + * @throws IOException */ private static void exists() throws IOException { String index = "user"; @@ -223,30 +301,30 @@ private static void exists() throws IOException { String id = "1"; // 创建查询请求 GetRequest getRequest = new GetRequest(index, type, id); + boolean exists = client.exists(getRequest, RequestOptions.DEFAULT); - ActionListener listener = new ActionListener() { - @Override - public void onResponse(Boolean exists) { - System.out.println("=="+exists); - } - - @Override - public void onFailure(Exception e) { - System.out.println("失败的原因："+e.getMessage()); - } + @Override + public void onResponse(Boolean exists) { + System.out.println("==" + exists); + } + + @Override + public void onFailure(Exception e) { + System.out.println("失败的原因：" + e.getMessage()); + } }; - //进行异步监听 + // 进行异步监听 // client.existsAsync(getRequest, RequestOptions.DEFAULT, listener); - - System.out.println("是否存在："+exists); + + System.out.println("数据是否存在：" + exists); } - - + /** - * 更新操作 + * 更新操作 + * * @throws IOException */ private static void update() throws IOException { @@ -254,31 +332,32 @@ private static void update() throws IOException { String type = "userindex"; // 唯一编号 String id = "1"; - UpdateRequest upateRequest=new UpdateRequest(); + UpdateRequest upateRequest = new UpdateRequest(); upateRequest.id(id); upateRequest.index(index); upateRequest.type(type); - - //依旧可以使用Map这种集合作为更新条件 + + // 依旧可以使用Map这种集合作为更新条件 Map jsonMap = new HashMap<>(); jsonMap.put("user", "xuwujing"); jsonMap.put("postDate", "2019-03-11"); - + upateRequest.doc(jsonMap); - + // upateRequest.docAsUpsert(true); // upsert 方法表示如果数据不存在，那么就新增一条 upateRequest.upsert(jsonMap); - + client.update(upateRequest, RequestOptions.DEFAULT); System.out.println("更新成功！"); - + } - + /** * 删除 - * @throws IOException + * + * @throws IOException * */ private static void delete() throws IOException { @@ -287,189 +366,169 @@ private static void delete() throws IOException { String type = "userindex"; // 唯一编号 String id = "1"; - DeleteRequest deleteRequest=new DeleteRequest(); + DeleteRequest deleteRequest = new DeleteRequest(); deleteRequest.id(id); deleteRequest.index(index); deleteRequest.type(type); - - //设置超时时间 - deleteRequest.timeout(TimeValue.timeValueMinutes(2)); - //设置刷新策略"wait_for" - //保持此请求打开，直到刷新使此请求的内容可以搜索为止。此刷新策略与高索引和搜索吞吐量兼容，但它会导致请求等待响应，直到发生刷新 + + // 设置超时时间 + deleteRequest.timeout(TimeValue.timeValueMinutes(2)); + // 设置刷新策略"wait_for" + // 保持此请求打开，直到刷新使此请求的内容可以搜索为止。此刷新策略与高索引和搜索吞吐量兼容，但它会导致请求等待响应，直到发生刷新 deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); - - //同步删除 + + // 同步删除 DeleteResponse deleteResponse = client.delete(deleteRequest, RequestOptions.DEFAULT); - + /* - * 异步删除操作 + * 异步删除操作 */ - - //进行监听 + + // 进行监听 ActionListener listener = new ActionListener() { - @Override - public void onResponse(DeleteResponse deleteResponse) { - System.out.println("响应:"+deleteResponse); - } - - @Override - public void onFailure(Exception e) { - System.out.println("删除监听异常:"+e.getMessage()); - } + @Override + public void onResponse(DeleteResponse deleteResponse) { + System.out.println("响应:" + deleteResponse); + } + + @Override + public void onFailure(Exception e) { + System.out.println("删除监听异常:" + e.getMessage()); + } }; - - //异步删除 + + // 异步删除 // client.deleteAsync(deleteRequest, RequestOptions.DEFAULT, listener); - + ReplicationResponse.ShardInfo shardInfo = deleteResponse.getShardInfo(); - //如果处理成功碎片的数量少于总碎片的情况,说明还在处理或者处理发生异常 + // 如果处理成功碎片的数量少于总碎片的情况,说明还在处理或者处理发生异常 if (shardInfo.getTotal() != shardInfo.getSuccessful()) { - System.out.println("需要处理的碎片总量:"+shardInfo.getTotal()); - System.out.println("处理成功的碎片总量:"+shardInfo.getSuccessful()); + System.out.println("需要处理的碎片总量:" + shardInfo.getTotal()); + System.out.println("处理成功的碎片总量:" + shardInfo.getSuccessful()); } - + if (shardInfo.getFailed() > 0) { - for (ReplicationResponse.ShardInfo.Failure failure : - shardInfo.getFailures()) { - String reason = failure.reason(); - } + for (ReplicationResponse.ShardInfo.Failure failure : shardInfo.getFailures()) { + String reason = failure.reason(); + } } System.out.println("删除成功!"); } - - + /** - * 批量操作示例 - * @throws InterruptedException + * 批量操作示例 + * + * @throws InterruptedException */ - private static void bulk() throws IOException, InterruptedException { + private static void bulk() throws IOException, InterruptedException { String index = "estest"; String type = "estest"; - - BulkRequest request = new BulkRequest(); - //批量新增 - request.add(new IndexRequest(index, type, "1") - .source(XContentType.JSON,"field", "foo")); - request.add(new IndexRequest(index, type, "2") - .source(XContentType.JSON,"field", "bar")); - request.add(new IndexRequest(index, type, "3") - .source(XContentType.JSON,"field", "baz")); - - //可以进行修改/删除/新增 操作 - request.add(new UpdateRequest(index, type, "2") - .doc(XContentType.JSON,"field", "test")); - request.add(new DeleteRequest(index, type, "3")); - request.add(new IndexRequest(index, type, "4") - .source(XContentType.JSON,"field", "baz")); - - + + BulkRequest request = new BulkRequest(); + // 批量新增 + request.add(new IndexRequest(index, type, "1").source(XContentType.JSON, "field", "foo")); + request.add(new IndexRequest(index, type, "2").source(XContentType.JSON, "field", "bar")); + request.add(new IndexRequest(index, type, "3").source(XContentType.JSON, "field", "baz")); + + // 可以进行修改/删除/新增 操作 + request.add(new UpdateRequest(index, type, "2").doc(XContentType.JSON, "field", "test")); + request.add(new DeleteRequest(index, type, "3")); + request.add(new IndexRequest(index, type, "4").source(XContentType.JSON, "field", "baz")); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); - - //可以快速检查一个或多个操作是否失败 true是有至少一个失败！ - if (bulkResponse.hasFailures()) { + + // 可以快速检查一个或多个操作是否失败 true是有至少一个失败！ + if (bulkResponse.hasFailures()) { System.out.println("有一个操作失败!"); } - - //对处理结果进行遍历操作并根据不同的操作进行处理 - for (BulkItemResponse bulkItemResponse : bulkResponse) { - DocWriteResponse itemResponse = bulkItemResponse.getResponse(); - - //操作失败的进行处理 - if (bulkItemResponse.isFailed()) { - BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); - - } - - if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.INDEX - || bulkItemResponse.getOpType() == DocWriteRequest.OpType.CREATE) { - IndexResponse indexResponse = (IndexResponse) itemResponse; - - } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.UPDATE) { - UpdateResponse updateResponse = (UpdateResponse) itemResponse; - - } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.DELETE) { - DeleteResponse deleteResponse = (DeleteResponse) itemResponse; - } - } - + + // 对处理结果进行遍历操作并根据不同的操作进行处理 + for (BulkItemResponse bulkItemResponse : bulkResponse) { + DocWriteResponse itemResponse = bulkItemResponse.getResponse(); + + // 操作失败的进行处理 + if (bulkItemResponse.isFailed()) { + BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); + + } + + if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.INDEX + || bulkItemResponse.getOpType() == DocWriteRequest.OpType.CREATE) { + IndexResponse indexResponse = (IndexResponse) itemResponse; + + } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.UPDATE) { + UpdateResponse updateResponse = (UpdateResponse) itemResponse; + + } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.DELETE) { + DeleteResponse deleteResponse = (DeleteResponse) itemResponse; + } + } + System.out.println("批量执行成功！"); - - - + /* - * 批量执行处理器相关示例代码 + * 批量执行处理器相关示例代码 */ - - - //批量处理器的监听器设置 - + + // 批量处理器的监听器设置 + BulkProcessor.Listener listener = new BulkProcessor.Listener() { - - //在执行BulkRequest的每次执行之前调用，这个方法允许知道将要在BulkRequest中执行的操作的数量 - @Override - public void beforeBulk(long executionId, BulkRequest request) { - int numberOfActions = request.numberOfActions(); - logger.debug("Executing bulk [{}] with {} requests", - executionId, numberOfActions); - } - - //在每次执行BulkRequest之后调用，这个方法允许知道BulkResponse是否包含错误 - @Override - public void afterBulk(long executionId, BulkRequest request, - BulkResponse response) { - if (response.hasFailures()) { - logger.warn("Bulk [{}] executed with failures", executionId); - } else { - logger.debug("Bulk [{}] completed in {} milliseconds", - executionId, response.getTook().getMillis()); - } - } - - //如果BulkRequest失败，则调用该方法，该方法允许知道失败 - @Override - public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - logger.error("Failed to execute bulk", failure); - } + + // 在执行BulkRequest的每次执行之前调用，这个方法允许知道将要在BulkRequest中执行的操作的数量 + @Override + public void beforeBulk(long executionId, BulkRequest request) { + int numberOfActions = request.numberOfActions(); + logger.debug("Executing bulk [{}] with {} requests", executionId, numberOfActions); + } + + // 在每次执行BulkRequest之后调用，这个方法允许知道BulkResponse是否包含错误 + @Override + public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + if (response.hasFailures()) { + logger.warn("Bulk [{}] executed with failures", executionId); + } else { + logger.debug("Bulk [{}] completed in {} milliseconds", executionId, response.getTook().getMillis()); + } + } + + // 如果BulkRequest失败，则调用该方法，该方法允许知道失败 + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + logger.error("Failed to execute bulk", failure); + } }; - - BiConsumer> bulkConsumer = - (request2, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener); - //创建一个批量执行的处理器 - BulkProcessor bulkProcessor = BulkProcessor.builder(bulkConsumer, listener).build(); + + BiConsumer> bulkConsumer = (request2, bulkListener) -> client + .bulkAsync(request, RequestOptions.DEFAULT, bulkListener); + // 创建一个批量执行的处理器 + BulkProcessor bulkProcessor = BulkProcessor.builder(bulkConsumer, listener).build(); BulkProcessor.Builder builder = BulkProcessor.builder(bulkConsumer, listener); - //根据当前添加的操作数量设置刷新新批量请求的时间(默认为1000，使用-1禁用它) - builder.setBulkActions(500); - //根据当前添加的操作大小设置刷新新批量请求的时间(默认为5Mb，使用-1禁用) - builder.setBulkSize(new ByteSizeValue(1L, ByteSizeUnit.MB)); - //设置允许执行的并发请求数量(默认为1，使用0只允许执行单个请求) - builder.setConcurrentRequests(0); - //设置刷新间隔如果间隔通过，则刷新任何挂起的BulkRequest(默认为未设置) - builder.setFlushInterval(TimeValue.timeValueSeconds(10L)); - //设置一个常量后退策略，该策略最初等待1秒并重试最多3次。 - builder.setBackoffPolicy(BackoffPolicy - .constantBackoff(TimeValue.timeValueSeconds(1L), 3)); - - - IndexRequest one = new IndexRequest(index, type, "1"). - source(XContentType.JSON, "title", - "In which order are my Elasticsearch queries executed?"); - IndexRequest two = new IndexRequest(index, type, "2") - .source(XContentType.JSON, "title", - "Current status and upcoming changes in Elasticsearch"); - IndexRequest three = new IndexRequest(index, type, "3") - .source(XContentType.JSON, "title", - "The Future of Federated Search in Elasticsearch"); + // 根据当前添加的操作数量设置刷新新批量请求的时间(默认为1000，使用-1禁用它) + builder.setBulkActions(500); + // 根据当前添加的操作大小设置刷新新批量请求的时间(默认为5Mb，使用-1禁用) + builder.setBulkSize(new ByteSizeValue(1L, ByteSizeUnit.MB)); + // 设置允许执行的并发请求数量(默认为1，使用0只允许执行单个请求) + builder.setConcurrentRequests(0); + // 设置刷新间隔如果间隔通过，则刷新任何挂起的BulkRequest(默认为未设置) + builder.setFlushInterval(TimeValue.timeValueSeconds(10L)); + // 设置一个常量后退策略，该策略最初等待1秒并重试最多3次。 + builder.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3)); + + IndexRequest one = new IndexRequest(index, type, "1").source(XContentType.JSON, "title", + "In which order are my Elasticsearch queries executed?"); + IndexRequest two = new IndexRequest(index, type, "2").source(XContentType.JSON, "title", + "Current status and upcoming changes in Elasticsearch"); + IndexRequest three = new IndexRequest(index, type, "3").source(XContentType.JSON, "title", + "The Future of Federated Search in Elasticsearch"); bulkProcessor.add(one); bulkProcessor.add(two); bulkProcessor.add(three); - - - //如果所有大容量请求都已完成，则该方法返回true;如果在所有大容量请求完成之前的等待时间已经过去，则返回false - boolean terminated = bulkProcessor.awaitClose(30L, TimeUnit.SECONDS); - - System.out.println("请求的响应结果:"+terminated); - - + + // 如果所有大容量请求都已完成，则该方法返回true;如果在所有大容量请求完成之前的等待时间已经过去，则返回false + boolean terminated = bulkProcessor.awaitClose(30L, TimeUnit.SECONDS); + + System.out.println("请求的响应结果:" + terminated); + } - + } diff --git a/src/main/java/com/pancm/sql/easticsearch/JestTest.java b/src/main/java/com/pancm/sql/easticsearch/JestTest.java index 13c4127..ce41f99 100644 --- a/src/main/java/com/pancm/sql/easticsearch/JestTest.java +++ b/src/main/java/com/pancm/sql/easticsearch/JestTest.java @@ -1,26 +1,20 @@ -package com.pancm.sql.easticsearch; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; +package com.pancm.test.esTest; import io.searchbox.client.JestClient; import io.searchbox.client.JestClientFactory; import io.searchbox.client.JestResult; import io.searchbox.client.config.HttpClientConfig; -import io.searchbox.core.Bulk; -import io.searchbox.core.BulkResult; -import io.searchbox.core.Delete; -import io.searchbox.core.DocumentResult; -import io.searchbox.core.Index; -import io.searchbox.core.Search; +import io.searchbox.core.*; import io.searchbox.indices.CreateIndex; import io.searchbox.indices.DeleteIndex; import io.searchbox.indices.mapping.GetMapping; import io.searchbox.indices.mapping.PutMapping; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; /** * @@ -34,7 +28,7 @@ public class JestTest { private static JestClient jestClient; private static String indexName = "userindex"; private static String typeName = "user"; - private static String elasticIps="http://127.0.0.1:9200"; + private static String elasticIps="http://192.169.0.23:9200,http://192.169.0.24:9200"; public static void main(String[] args) throws Exception { @@ -48,8 +42,13 @@ public static void main(String[] args) throws Exception { } private static JestClient getJestClient() { - JestClientFactory factory = new JestClientFactory(); - factory.setHttpClientConfig(new HttpClientConfig.Builder(elasticIps).connTimeout(60000).readTimeout(60000).multiThreaded(true).build()); + JestClientFactory factory = new JestClientFactory(); + //指定集群地址，设置超时时间，开启多线程，并发连接最多两个，最大连接数 + factory.setHttpClientConfig(new HttpClientConfig.Builder(elasticIps) + .connTimeout(60000).readTimeout(60000).multiThreaded(true) + .defaultMaxTotalConnectionPerRoute(2) + .maxTotalConnection(10) + .build()); return factory.getObject(); } @@ -174,7 +173,11 @@ public static boolean insertBatch(JestClient jestClient,String indexName, String Index index = new Index.Builder(obj).build(); bulk.addAction(index); } + BulkResult br = jestClient.execute(bulk.build()); + System.out.println("DSL语句:"+br.toString()); + System.out.println("DSL语句:"+br.getJsonString()); + System.out.println("DSL语句:"+br.getJsonObject()); return br.isSucceeded(); } @@ -275,9 +278,9 @@ public Long getId() { } - /** - * 设置编号 - * @param Long id + /** + * 设置编号 + * @param Long id */ public void setId(Long id) { this.id = id; @@ -295,7 +298,7 @@ public String getName() { /** * 设置姓名 - * @param String name + * @param String name */ public void setName(String name) { this.name = name; @@ -313,7 +316,7 @@ public Integer getAge() { /** * 设置年龄 - * @param Integer age + * @param age */ public void setAge(Integer age) { this.age = age; From 01a841009d887287f1d76692e7290dc3514b719c Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Mon, 1 Apr 2019 17:51:32 +0800 Subject: [PATCH 10/53] =?UTF-8?q?1.=E5=8D=87=E7=BA=A7pom=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E4=B8=ADjar=E7=89=88=E6=9C=AC=202.=E5=A2=9E=E5=8A=A0elasticsea?= =?UTF-8?q?rch=E3=80=81redis=E7=9A=84=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 145 +-- pom.xml | 856 ++++++++-------- .../EsHighLevelRestSearchTest.java | 688 ++++++------- .../easticsearch/EsHighLevelRestTest1.java | 950 +++++++++--------- .../easticsearch/EsHighLevelRestTest2.java | 686 ++++++------- .../{sql => }/easticsearch/JestTest.java | 740 +++++++------- .../{sql => }/easticsearch/package-info.java | 16 +- .../com/pancm/{sql => }/redis/RedisTest.java | 120 +-- .../pancm/{sql => }/redis/package-info.java | 16 +- 9 files changed, 2082 insertions(+), 2135 deletions(-) rename src/main/java/com/pancm/{sql => }/easticsearch/EsHighLevelRestSearchTest.java (96%) rename src/main/java/com/pancm/{sql => }/easticsearch/EsHighLevelRestTest1.java (96%) rename src/main/java/com/pancm/{sql => }/easticsearch/EsHighLevelRestTest2.java (96%) rename src/main/java/com/pancm/{sql => }/easticsearch/JestTest.java (95%) rename src/main/java/com/pancm/{sql => }/easticsearch/package-info.java (74%) rename src/main/java/com/pancm/{sql => }/redis/RedisTest.java (94%) rename src/main/java/com/pancm/{sql => }/redis/package-info.java (78%) diff --git a/README.md b/README.md index 41bdf8e..6dd1725 100644 --- a/README.md +++ b/README.md @@ -1,71 +1,74 @@ -## java-study - -**介绍** -[java-study](https://github.com/xuwujing/java-study) 是本人学习Java过程中记录的一些代码！从Java基础的数据类型、jdk1.8的使用、IO、集合、线程等等技术以及一些常用框架，netty、mina、springboot、kafka、storm、zookeeper、redis、hbase、hive等等。 - -**使用** - -下载： - - git clone https://github.com/xuwujing/java-study - -然后使用maven方式导入IDE中运行main方法即可。 - -## 项目结构 - - com.pancm.arithmetic - 一些算法相关类 - com.pancm.basics - 一些Java基础相关类 主要是三大特性、修饰符、io、集合、反射、克隆等等相关代码 - com.pancm.bigdata - 大数据相关的类 主要是hbase、storm、zookeeper等等相关的代码 - com.pancm.commons - 一些第三方工具类的测试用例 主要是apache commons、apache lang、google common、google guava、joda等等一些工具包测试使用代码 - com.pancm.design - 设计模式相关的示例类 包含常用的23种设计模式 - com.pancm.jdk8 - jdk1.8相关的类 主要是lambda、stream以及LocalDateTime等等测试代码 - com.pancm.mq - 一些消息中间件的类，主要包含kafka、rabbitmq相关的测试代码 - com.pancm.nio - 一些nio框架，主要是netty和mina - com.pancm.others - 一些不知道怎么定义的测试类，Jsoup(爬虫)、logback、lombok等等测试代码 - com.pancm.pojo - 实体相关类 - com.pancm.question - 一些面试可能会问的问题的类 - com.pancm.sql - 一些数据库相关的类，包括非关系型数据库 - com.pancm.thread - 一些线程相关的类 从基本的使用到各种并发的测试类 - com.pancm.utils - 一些常用的工具类 主要是Json数据转换，日期转换，二维码图片生成工具类，常用的AES、MD5、BASE64等等编码解码工具类，redis、kafka、zookeeper等等工具类 - -## 相关文章 - -这里介绍的文章主要是本人写的一些博客。博客主要发布在[个人博客](http://www.panchengming.com)、[CSDN](https://blog.csdn.net/qazwsxpcm)、[博客园](https://www.cnblogs.com/xuwujing/)等，但是由于个人博客在github上，访问可能较慢，CSDN目前观感体验不好，所以以下链接主要就在博客园中了。 - -**Java基础相关:** - -- [基本数据类型](https://www.cnblogs.com/xuwujing/p/8597557.html) -- [修饰符和String](https://www.cnblogs.com/xuwujing/p/8638329.html) -- [封装、继承和多态](https://www.cnblogs.com/xuwujing/p/8681123.html) -- [集合List、Map和Set](https://www.cnblogs.com/xuwujing/p/8886821.html) -- [多线程](https://www.cnblogs.com/xuwujing/p/9102870.html) -- [IO流](https://www.cnblogs.com/xuwujing/p/9191546.html) -- [总结篇](https://www.cnblogs.com/xuwujing/p/9236376.html) - -**设计模式:** - -- [单例模式](https://www.cnblogs.com/xuwujing/p/9277266.html) -- [工厂方法和抽象工厂模式](https://www.cnblogs.com/xuwujing/p/9363142.html) -- [建造者模式和原型模式](https://www.cnblogs.com/xuwujing/p/9496346.html) -- [适配器模式和桥接模式](https://www.cnblogs.com/xuwujing/p/9520851.html) -- [外观模式和装饰器模式](https://www.cnblogs.com/xuwujing/p/9545272.html) -- [组合模式和过滤器模式](https://www.cnblogs.com/xuwujing/p/9630850.html) -- [享元模式和代理模式](https://www.cnblogs.com/xuwujing/p/9704228.html) -- [责任链模式和命令模式](https://www.cnblogs.com/xuwujing/p/9794886.html) -- [解释器模式和迭代器模式](https://www.cnblogs.com/xuwujing/p/9873514.html) -- [访问者模式和中介者模式](https://www.cnblogs.com/xuwujing/p/9911997.html) -- [策略模式和模板方法模式](https://www.cnblogs.com/xuwujing/p/9954263.html) -- [观察者模式和空对象模式](https://www.cnblogs.com/xuwujing/p/10036204.html) -- [总结篇](https://www.cnblogs.com/xuwujing/p/10134494.html) - -**进阶相关:** - -- [JDK1.8的Lambda、Stream和日期的使用详解](https://www.cnblogs.com/xuwujing/p/10145691.html) - -**其他:** - -- [个人收集的资源分享](https://www.cnblogs.com/xuwujing/p/10393111.html) - -## 其他 - -在这些代码中，虽然大部分都是自己写的，但是也有不少是在学习过程中从网上或书上直接摘抄的，当时有些并未标明出处，现在由于忘了出处，有些代码并未标明，若有冒犯，忘请见谅！ +## java-study + +**介绍** + +[java-study](https://github.com/xuwujing/java-study) 是本人学习Java过程中记录的一些代码！从Java基础的数据类型、jdk1.8的使用、IO、集合、线程等等技术以及一些常用框架，netty、mina、springboot、kafka、storm、zookeeper、es、redis、hbase、hive等等。 + +**使用** + +下载： + + git clone https://github.com/xuwujing/java-study + +然后使用maven方式导入IDE中运行main方法即可。 + +## 项目结构 + + com.pancm.arithmetic - 一些算法相关类 + com.pancm.basics - 一些Java基础相关类 主要是三大特性、修饰符、io、集合、反射、克隆等等相关代码 + com.pancm.bigdata - 大数据相关的类 主要是hbase、storm、zookeeper等等相关的代码 + com.pancm.commons - 一些第三方工具类的测试用例 主要是apache commons、apache lang、google common、google guava、joda等等一些工具包测试使用代码 + com.pancm.design - 设计模式相关的示例类 包含常用的23种设计模式 + com.pancm.elasticsearch - elasticsearch相关使用的测试用例，包括索引mapping的创建、全文检索、聚合查询等等 + com.pancm.jdk8 - jdk1.8相关的类 主要是lambda、stream以及LocalDateTime等等测试代码 + com.pancm.mq - 一些消息中间件的类，主要包含kafka、rabbitmq相关的测试代码 + com.pancm.nio - 一些nio框架，主要是netty和mina + com.pancm.others - 一些不知道怎么定义的测试类，Jsoup(爬虫)、logback、lombok等等测试代码 + com.pancm.pojo - 实体相关类 + com.pancm.question - 一些面试可能会问的问题的类 + com.pancm.redis - redis相关使用的类 + com.pancm.sql - 一些数据库相关的类 + com.pancm.thread - 一些线程相关的类 从基本的使用到各种并发的测试类 + com.pancm.utils - 一些常用的工具类 主要是Json数据转换，日期转换，二维码图片生成工具类，常用的AES、MD5、BASE64等等编码解码工具类，redis、kafka、zookeeper等等工具类 + +## 相关文章 + +这里介绍的文章主要是本人写的一些博客。博客主要发布在[个人博客](http://www.panchengming.com)、[CSDN](https://blog.csdn.net/qazwsxpcm)、[博客园](https://www.cnblogs.com/xuwujing/)等，但是由于个人博客在github上，访问可能较慢，CSDN目前观感体验不好，所以以下链接主要就在博客园中了。 + +**Java基础相关:** + +- [基本数据类型](https://www.cnblogs.com/xuwujing/p/8597557.html) +- [修饰符和String](https://www.cnblogs.com/xuwujing/p/8638329.html) +- [封装、继承和多态](https://www.cnblogs.com/xuwujing/p/8681123.html) +- [集合List、Map和Set](https://www.cnblogs.com/xuwujing/p/8886821.html) +- [多线程](https://www.cnblogs.com/xuwujing/p/9102870.html) +- [IO流](https://www.cnblogs.com/xuwujing/p/9191546.html) +- [总结篇](https://www.cnblogs.com/xuwujing/p/9236376.html) + +**设计模式:** + +- [单例模式](https://www.cnblogs.com/xuwujing/p/9277266.html) +- [工厂方法和抽象工厂模式](https://www.cnblogs.com/xuwujing/p/9363142.html) +- [建造者模式和原型模式](https://www.cnblogs.com/xuwujing/p/9496346.html) +- [适配器模式和桥接模式](https://www.cnblogs.com/xuwujing/p/9520851.html) +- [外观模式和装饰器模式](https://www.cnblogs.com/xuwujing/p/9545272.html) +- [组合模式和过滤器模式](https://www.cnblogs.com/xuwujing/p/9630850.html) +- [享元模式和代理模式](https://www.cnblogs.com/xuwujing/p/9704228.html) +- [责任链模式和命令模式](https://www.cnblogs.com/xuwujing/p/9794886.html) +- [解释器模式和迭代器模式](https://www.cnblogs.com/xuwujing/p/9873514.html) +- [访问者模式和中介者模式](https://www.cnblogs.com/xuwujing/p/9911997.html) +- [策略模式和模板方法模式](https://www.cnblogs.com/xuwujing/p/9954263.html) +- [观察者模式和空对象模式](https://www.cnblogs.com/xuwujing/p/10036204.html) +- [总结篇](https://www.cnblogs.com/xuwujing/p/10134494.html) + +**进阶相关:** + +- [JDK1.8的Lambda、Stream和日期的使用详解](https://www.cnblogs.com/xuwujing/p/10145691.html) + +**其他:** + +- [个人收集的资源分享](https://www.cnblogs.com/xuwujing/p/10393111.html) + +## 其他 + +在这些代码中，虽然大部分都是自己写的，但是也有不少是在学习过程中从网上或书上直接摘抄的，当时有些并未标明出处，现在由于忘了出处，有些代码并未标明，若有冒犯，请见谅！ diff --git a/pom.xml b/pom.xml index b28295b..c0e565f 100644 --- a/pom.xml +++ b/pom.xml @@ -1,456 +1,400 @@ - - 4.0.0 - - 1.0.0 - java_study - 0.0.1-SNAPSHOT - jar - - java_study - http://maven.apache.org - - - UTF-8 - UTF-8 - 1.8 - 1.8 - 1.8 - - - - - - - - - junit - junit - 4.12 - - - - - jdk.tools - jdk.tools - 1.8 - system - ${JAVA_HOME}/lib/tools.jar - - - - - - org.slf4j - slf4j-api - 1.7.25 - - - - - ch.qos.logback - logback-classic - 1.2.3 - - - - ch.qos.logback - logback-core - 1.2.3 - - - - - - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - 2.9.6 - - - - - com.google.code.gson - gson - 2.8.5 - - - - - com.alibaba - fastjson - 1.2.49 - - - - commons-codec - commons-codec - 1.11 - - - - - - commons-lang - commons-lang - 2.6 - - - - org.apache.commons - commons-lang3 - 3.7 - - - - - org.apache.commons - commons-compress - 1.18 - - - - - - - org.apache.poi - poi - 3.9 - - - - org.apache.poi - poi-ooxml - 3.17 - - - - - com.google.zxing - core - 3.0.0 - - - com.google.zxing - javase - 3.0.0 - - - - - - com.github.pagehelper - pagehelper - 4.1.0 - - - - - org.projectlombok - lombok - 1.16.18 - - - - - com.geccocrawler - gecco - 1.2.8 - - - - - - com.google.protobuf - protobuf-java - 3.5.1 - - - - - - org.quartz-scheduler - quartz - 2.3.0 - - - - org.quartz-scheduler - quartz-jobs - 2.3.0 - - - - - - - - - - - redis.clients - jedis - 2.9.0 - - - - - mysql - mysql-connector-java - 5.1.41 - - - - - org.xerial - sqlite-jdbc - 3.20.1 - - - - - com.microsoft.sqlserver - sqljdbc4 - 4.0 - - - - - - - - - - - io.netty - netty-all - 4.1.17.Final - - - - - org.apache.mina - mina-core - 2.0.16 - - - - - com.github.kevinsawicki - http-request - 6.0 - - - - - - - - - - - - - com.rabbitmq - amqp-client - 4.3.0 - - - - - org.apache.kafka - kafka_2.12 - 1.0.0 - - - org.apache.zookeeper - zookeeper - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - - - - - org.apache.kafka - kafka-clients - 1.0.0 - - - - org.apache.kafka - kafka-streams - 1.0.0 - - - - - - - - - - org.apache.zookeeper - zookeeper - 3.4.10 - - - org.slf4j - slf4j-log4j12 - - - log4j - log4j - - - - - - - com.101tec - zkclient - 0.10 - - - - - - org.apache.hadoop - hadoop-common - 2.8.2 - - - org.apache.hadoop - hadoop-client - 2.8.2 - - - org.apache.hadoop - hadoop-hdfs - 2.8.2 - - - org.apache.hadoop - hadoop-mapreduce-client-core - 2.8.2 - - - org.apache.hadoop - hadoop-yarn-common - 2.8.2 - - - - - org.apache.hbase - hbase-hadoop-compat - 1.3.1 - - - org.apache.hbase - hbase-server - 1.4.8 - - - org.apache.hbase - hbase-client - 1.4.8 - - - org.apache.hbase - hbase-common - 1.4.8 - - - - - - org.apache.hive - hive-jdbc - 3.1.1 - - - - - org.apache.storm - storm-core - 1.2.2 - - - - org.apache.storm - storm-kafka - 1.2.2 - - - - - - org.apache.spark - spark-core_2.11 - 2.4.0 - - - org.apache.spark - spark-sql_2.11 - 2.4.0 - - - org.apache.spark - spark-streaming_2.11 - 2.4.0 - provided - - - com.sparkjava - spark-core - 2.8.0 - - - org.apache.spark - spark-mllib_2.11 - 2.4.0 - provided - - - - - - - - - org.elasticsearch.client - elasticsearch-rest-high-level-client - 6.6.1 - - - - - io.searchbox - jest - 6.3.1 - - - - - - - - - - - - spring-milestone - http://repo.spring.io/libs-release - - - + + 4.0.0 + + 1.0.0 + java_study + 0.0.1-SNAPSHOT + jar + + java_study + http://maven.apache.org + + + UTF-8 + UTF-8 + 1.8 + 1.8 + 1.8 + + + + + + + + + junit + junit + 4.12 + + + + + + + + + org.slf4j + slf4j-api + 1.7.25 + + + + + ch.qos.logback + logback-classic + 1.2.3 + + + + ch.qos.logback + logback-core + 1.2.3 + + + + + + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider + 2.9.6 + + + + + com.google.code.gson + gson + 2.8.5 + + + + + com.alibaba + fastjson + 1.2.49 + + + + commons-codec + commons-codec + 1.11 + + + + + + commons-lang + commons-lang + 2.6 + + + + org.apache.commons + commons-lang3 + 3.7 + + + + + org.apache.commons + commons-compress + 1.18 + + + + + + + org.apache.poi + poi + 3.9 + + + + org.apache.poi + poi-ooxml + 3.17 + + + + + com.google.zxing + core + 3.0.0 + + + com.google.zxing + javase + 3.0.0 + + + + + + com.github.pagehelper + pagehelper + 4.1.0 + + + + + org.projectlombok + lombok + 1.16.18 + + + + + com.geccocrawler + gecco + 1.2.8 + + + + + + com.google.protobuf + protobuf-java + 3.5.1 + + + + + + org.quartz-scheduler + quartz + 2.3.0 + + + + org.quartz-scheduler + quartz-jobs + 2.3.0 + + + + + + + + + + + redis.clients + jedis + 2.9.0 + + + + + mysql + mysql-connector-java + 5.1.41 + + + + + org.xerial + sqlite-jdbc + 3.20.1 + + + + + com.microsoft.sqlserver + sqljdbc4 + 4.0 + + + + + + + + + + + io.netty + netty-all + 4.1.17.Final + + + + + org.apache.mina + mina-core + 2.0.16 + + + + + com.github.kevinsawicki + http-request + 6.0 + + + + + + + + + + + + + com.rabbitmq + amqp-client + 4.3.0 + + + + + org.apache.kafka + kafka_2.12 + 1.0.0 + + + org.apache.zookeeper + zookeeper + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + + + + + org.apache.kafka + kafka-clients + 1.0.0 + + + + org.apache.kafka + kafka-streams + 1.0.0 + + + + + + + + + + org.apache.zookeeper + zookeeper + 3.4.10 + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + + + + + com.101tec + zkclient + 0.10 + + + + + org.apache.hbase + hbase-client + 2.1.4 + + + org.apache.hbase + hbase-common + 2.1.4 + + + + org.apache.hadoop + hadoop-client + 3.2.0 + + + + org.apache.spark + spark-core_2.12 + 2.4.0 + + + + org.apache.spark + spark-sql_2.12 + 2.4.0 + + + + + org.apache.storm + storm-core + 1.2.2 + provided + + + + org.apache.storm + storm-kafka + 1.2.2 + provided + + + + + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + 6.6.1 + + + + + io.searchbox + jest + 6.3.1 + + + + + + + + + + + + spring-milestone + http://repo.spring.io/libs-release + + + diff --git a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestSearchTest.java b/src/main/java/com/pancm/easticsearch/EsHighLevelRestSearchTest.java similarity index 96% rename from src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestSearchTest.java rename to src/main/java/com/pancm/easticsearch/EsHighLevelRestSearchTest.java index 9342051..5aa24ea 100644 --- a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestSearchTest.java +++ b/src/main/java/com/pancm/easticsearch/EsHighLevelRestSearchTest.java @@ -1,344 +1,344 @@ -package com.pancm.sql.easticsearch; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import org.apache.http.HttpHost; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.ShardSearchFailure; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.Aggregations; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; -import org.elasticsearch.search.suggest.Suggest; -import org.elasticsearch.search.suggest.SuggestBuilder; -import org.elasticsearch.search.suggest.SuggestBuilders; -import org.elasticsearch.search.suggest.SuggestionBuilder; -import org.elasticsearch.search.suggest.term.TermSuggestion; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -/** - * @Title: EsHighLevelRestSearchTest - * @Description: Java High Level REST Client Es高级客户端查询使用使用教程 (Search查询使用教程) - * 官方文档地址: - * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high.html - * @Version:1.0.0 - * @author pancm - * @date 2019年3月12日 - */ -public class EsHighLevelRestSearchTest { - - private static String elasticIp = "192.169.0.23"; - private static int elasticPort = 9200; - - private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestSearchTest.class); - - private static RestHighLevelClient client = null; - - /** - * @param args - */ - public static void main(String[] args) { - - try { - init(); - search(); - - - } catch (Exception e) { - e.printStackTrace(); - }finally { - close(); - } - - } - - /* - * 初始化服务 - */ - private static void init() { - client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort, "http"))); - - } - - /* - * 关闭服务 - */ - private static void close() { - if (client != null) { - try { - client.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - /** - * search查询使用示例 - * - * @throws IOException - */ - private static void search() throws IOException { - - /* - * 查询集群所有的索引 - * - */ - SearchRequest searchRequestAll = new SearchRequest(); - - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.matchAllQuery()); - searchRequestAll.source(searchSourceBuilder); - - // 同步查询 - SearchResponse searchResponseAll = client.search(searchRequestAll, RequestOptions.DEFAULT); - - System.out.println("所有查询总数:" + searchResponseAll.getHits().getTotalHits()); - - // 查询指定的索引库 - SearchRequest searchRequest = new SearchRequest("user"); - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - // 设置查询条件 - sourceBuilder.query(QueryBuilders.termQuery("user", "pancm")); - // 设置起止和结束 - sourceBuilder.from(0); - sourceBuilder.size(5); - sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS)); - // 设置路由 -// searchRequest.routing("routing"); - // 设置索引库表达式 - searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); - // 查询选择本地分片，默认是集群分片 - searchRequest.preference("_local"); - - // 排序 - // 根据默认值进行降序排序 -// sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); - // 根据字段进行升序排序 -// sourceBuilder.sort(new FieldSortBuilder("id").order(SortOrder.ASC)); - - // 关闭suorce查询 -// sourceBuilder.fetchSource(false); - - String[] includeFields = new String[] { "title", "user", "innerObject.*" }; - String[] excludeFields = new String[] { "_type" }; - // 包含或排除字段 -// sourceBuilder.fetchSource(includeFields, excludeFields); - - searchRequest.source(sourceBuilder); - - // 同步查询 - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - - // HTTP状态代码、执行时间或请求是否提前终止或超时 - RestStatus status = searchResponse.status(); - TimeValue took = searchResponse.getTook(); - Boolean terminatedEarly = searchResponse.isTerminatedEarly(); - boolean timedOut = searchResponse.isTimedOut(); - - // 供关于受搜索影响的切分总数的统计信息，以及成功和失败的切分 - int totalShards = searchResponse.getTotalShards(); - int successfulShards = searchResponse.getSuccessfulShards(); - int failedShards = searchResponse.getFailedShards(); - // 失败的原因 - for (ShardSearchFailure failure : searchResponse.getShardFailures()) { - // failures should be handled here - } - - // 结果 - searchResponse.getHits().forEach(hit -> { - Map map = hit.getSourceAsMap(); - String string = hit.getSourceAsString(); - System.out.println("普通查询的Map结果:" + map); - System.out.println("普通查询的String结果:" + string); - }); - - - System.out.println("\n=================\n"); - - /* - * 全文查询使用示例 - */ - - // 搜索字段user为pancm的数据 - MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("user", "pancm"); - - // 设置模糊查询 - matchQueryBuilder.fuzziness(Fuzziness.AUTO); - // 设置前缀长度 - matchQueryBuilder.prefixLength(3); - // 设置最大扩展选项来控制查询的模糊过程 - matchQueryBuilder.maxExpansions(10); - - /* - * QueryBuilder也可以 - */ - -// QueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("user", "kimchy") -// .fuzziness(Fuzziness.AUTO) -// .prefixLength(3) -// .maxExpansions(10); - - SearchSourceBuilder searchSourceBuilder2 = new SearchSourceBuilder(); - searchSourceBuilder2.query(matchQueryBuilder); - - SearchRequest searchRequest2 = new SearchRequest(); - searchRequest2.source(searchSourceBuilder2); - // 同步查询 - SearchResponse searchResponse2 = client.search(searchRequest, RequestOptions.DEFAULT); - - SearchHits hits = searchResponse.getHits(); - //总条数和分值 - long totalHits = hits.getTotalHits(); - float maxScore = hits.getMaxScore(); - - - - hits.forEach(hit -> { - - String index = hit.getIndex(); - String type = hit.getType(); - String id = hit.getId(); - float score = hit.getScore(); - - - - Map sourceAsMap = hit.getSourceAsMap(); - String string = hit.getSourceAsString(); - System.out.println("Match查询的Map结果:" + sourceAsMap); - System.out.println("Match查询的String结果:" + string); - - String documentTitle = (String) sourceAsMap.get("title"); -// List users = (List) sourceAsMap.get("user"); - Map innerObject = - (Map) sourceAsMap.get("innerObject"); - }); - - - - - System.out.println("\n=================\n"); - - /* - * 高亮查询 - */ - - SearchSourceBuilder searchSourceBuilder3 = new SearchSourceBuilder(); - HighlightBuilder highlightBuilder = new HighlightBuilder(); - HighlightBuilder.Field highlightTitle = new HighlightBuilder.Field("title"); - // 设置字段高亮字体 - highlightTitle.highlighterType("user"); - highlightBuilder.field(highlightTitle); - HighlightBuilder.Field highlightUser = new HighlightBuilder.Field("user"); - highlightBuilder.field(highlightUser); - searchSourceBuilder3.highlighter(highlightBuilder); - - SearchRequest searchRequest3 = new SearchRequest(); - searchRequest3.source(searchSourceBuilder3); - - // 同步查询 - SearchResponse searchResponse3 = client.search(searchRequest3, RequestOptions.DEFAULT); - - searchResponse3.getHits().forEach(hit -> { - - Map map = hit.getSourceAsMap(); - String string = hit.getSourceAsString(); - System.out.println("Highlight查询的Map结果:" + map); - System.out.println("Highlight查询的String结果:" + string); - }); - - System.out.println("\n=================\n"); - - /** - * 聚合查询 - */ - - SearchSourceBuilder searchSourceBuilder4 = new SearchSourceBuilder(); - - //terms 就是分组统计 根据user进行分组并创建一个新的聚合user_ - TermsAggregationBuilder aggregation = AggregationBuilders.terms("user_").field("user"); - aggregation.subAggregation(AggregationBuilders.avg("average_age").field("age")); - searchSourceBuilder4.aggregation(aggregation); - - SearchRequest searchRequest4 = new SearchRequest(); - searchRequest4.source(searchSourceBuilder4); - - // 同步查询 - SearchResponse searchResponse4 = client.search(searchRequest4, RequestOptions.DEFAULT); - //聚合查询返回条件 - Aggregations aggregations = searchResponse4.getAggregations(); - List aggregationList = aggregations.asList(); - - for (Aggregation agg : aggregations) { - String type = agg.getType(); - if (type.equals(TermsAggregationBuilder.NAME)) { - Bucket elasticBucket = ((Terms) agg).getBucketByKey("Elastic"); - long numberOfDocs = elasticBucket.getDocCount(); - System.out.println("条数:"+numberOfDocs); - } - } - - -// Terms byCompanyAggregation = aggregations.get("user_"); -// Bucket elasticBucket = byCompanyAggregation.getBucketByKey("Elastic"); -// Avg averageAge = elasticBucket.getAggregations().get("average_age"); -// double avg = averageAge.getValue(); -// -// System.out.println("聚合查询返回的结果:"+avg); - - - /* - * 建议查询 - */ - SearchSourceBuilder searchSourceBuilder5 = new SearchSourceBuilder(); - SuggestionBuilder termSuggestionBuilder = SuggestBuilders.termSuggestion("user").text("pancm"); - SuggestBuilder suggestBuilder = new SuggestBuilder(); - suggestBuilder.addSuggestion("suggest_user", termSuggestionBuilder); - searchSourceBuilder5.suggest(suggestBuilder); - - SearchRequest searchRequest5 = new SearchRequest(); - searchRequest5.source(searchSourceBuilder5); - - // 同步查询 - SearchResponse searchResponse5= client.search(searchRequest5, RequestOptions.DEFAULT); - - Suggest suggest = searchResponse5.getSuggest(); - TermSuggestion termSuggestion = suggest.getSuggestion("suggest_user"); - for (TermSuggestion.Entry entry : termSuggestion.getEntries()) { - for (TermSuggestion.Entry.Option option : entry) { - String suggestText = option.getText().string(); - System.out.println("返回结果:"+suggestText); - } - } - - - - - - - - - - } - -} +package com.pancm.easticsearch; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.http.HttpHost; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.search.suggest.SuggestBuilder; +import org.elasticsearch.search.suggest.SuggestBuilders; +import org.elasticsearch.search.suggest.SuggestionBuilder; +import org.elasticsearch.search.suggest.term.TermSuggestion; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * @Title: EsHighLevelRestSearchTest + * @Description: Java High Level REST Client Es高级客户端查询使用使用教程 (Search查询使用教程) + * 官方文档地址: + * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high.html + * @Version:1.0.0 + * @author pancm + * @date 2019年3月12日 + */ +public class EsHighLevelRestSearchTest { + + private static String elasticIp = "192.169.0.23"; + private static int elasticPort = 9200; + + private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestSearchTest.class); + + private static RestHighLevelClient client = null; + + /** + * @param args + */ + public static void main(String[] args) { + + try { + init(); + search(); + + + } catch (Exception e) { + e.printStackTrace(); + }finally { + close(); + } + + } + + /* + * 初始化服务 + */ + private static void init() { + client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort, "http"))); + + } + + /* + * 关闭服务 + */ + private static void close() { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + /** + * search查询使用示例 + * + * @throws IOException + */ + private static void search() throws IOException { + + /* + * 查询集群所有的索引 + * + */ + SearchRequest searchRequestAll = new SearchRequest(); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchRequestAll.source(searchSourceBuilder); + + // 同步查询 + SearchResponse searchResponseAll = client.search(searchRequestAll, RequestOptions.DEFAULT); + + System.out.println("所有查询总数:" + searchResponseAll.getHits().getTotalHits()); + + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest("user"); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + // 设置查询条件 + sourceBuilder.query(QueryBuilders.termQuery("user", "pancm")); + // 设置起止和结束 + sourceBuilder.from(0); + sourceBuilder.size(5); + sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS)); + // 设置路由 +// searchRequest.routing("routing"); + // 设置索引库表达式 + searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); + // 查询选择本地分片，默认是集群分片 + searchRequest.preference("_local"); + + // 排序 + // 根据默认值进行降序排序 +// sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); + // 根据字段进行升序排序 +// sourceBuilder.sort(new FieldSortBuilder("id").order(SortOrder.ASC)); + + // 关闭suorce查询 +// sourceBuilder.fetchSource(false); + + String[] includeFields = new String[] { "title", "user", "innerObject.*" }; + String[] excludeFields = new String[] { "_type" }; + // 包含或排除字段 +// sourceBuilder.fetchSource(includeFields, excludeFields); + + searchRequest.source(sourceBuilder); + + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + + // HTTP状态代码、执行时间或请求是否提前终止或超时 + RestStatus status = searchResponse.status(); + TimeValue took = searchResponse.getTook(); + Boolean terminatedEarly = searchResponse.isTerminatedEarly(); + boolean timedOut = searchResponse.isTimedOut(); + + // 供关于受搜索影响的切分总数的统计信息，以及成功和失败的切分 + int totalShards = searchResponse.getTotalShards(); + int successfulShards = searchResponse.getSuccessfulShards(); + int failedShards = searchResponse.getFailedShards(); + // 失败的原因 + for (ShardSearchFailure failure : searchResponse.getShardFailures()) { + // failures should be handled here + } + + // 结果 + searchResponse.getHits().forEach(hit -> { + Map map = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("普通查询的Map结果:" + map); + System.out.println("普通查询的String结果:" + string); + }); + + + System.out.println("\n=================\n"); + + /* + * 全文查询使用示例 + */ + + // 搜索字段user为pancm的数据 + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("user", "pancm"); + + // 设置模糊查询 + matchQueryBuilder.fuzziness(Fuzziness.AUTO); + // 设置前缀长度 + matchQueryBuilder.prefixLength(3); + // 设置最大扩展选项来控制查询的模糊过程 + matchQueryBuilder.maxExpansions(10); + + /* + * QueryBuilder也可以 + */ + +// QueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("user", "kimchy") +// .fuzziness(Fuzziness.AUTO) +// .prefixLength(3) +// .maxExpansions(10); + + SearchSourceBuilder searchSourceBuilder2 = new SearchSourceBuilder(); + searchSourceBuilder2.query(matchQueryBuilder); + + SearchRequest searchRequest2 = new SearchRequest(); + searchRequest2.source(searchSourceBuilder2); + // 同步查询 + SearchResponse searchResponse2 = client.search(searchRequest, RequestOptions.DEFAULT); + + SearchHits hits = searchResponse.getHits(); + //总条数和分值 + long totalHits = hits.getTotalHits(); + float maxScore = hits.getMaxScore(); + + + + hits.forEach(hit -> { + + String index = hit.getIndex(); + String type = hit.getType(); + String id = hit.getId(); + float score = hit.getScore(); + + + + Map sourceAsMap = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("Match查询的Map结果:" + sourceAsMap); + System.out.println("Match查询的String结果:" + string); + + String documentTitle = (String) sourceAsMap.get("title"); +// List users = (List) sourceAsMap.get("user"); + Map innerObject = + (Map) sourceAsMap.get("innerObject"); + }); + + + + + System.out.println("\n=================\n"); + + /* + * 高亮查询 + */ + + SearchSourceBuilder searchSourceBuilder3 = new SearchSourceBuilder(); + HighlightBuilder highlightBuilder = new HighlightBuilder(); + HighlightBuilder.Field highlightTitle = new HighlightBuilder.Field("title"); + // 设置字段高亮字体 + highlightTitle.highlighterType("user"); + highlightBuilder.field(highlightTitle); + HighlightBuilder.Field highlightUser = new HighlightBuilder.Field("user"); + highlightBuilder.field(highlightUser); + searchSourceBuilder3.highlighter(highlightBuilder); + + SearchRequest searchRequest3 = new SearchRequest(); + searchRequest3.source(searchSourceBuilder3); + + // 同步查询 + SearchResponse searchResponse3 = client.search(searchRequest3, RequestOptions.DEFAULT); + + searchResponse3.getHits().forEach(hit -> { + + Map map = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("Highlight查询的Map结果:" + map); + System.out.println("Highlight查询的String结果:" + string); + }); + + System.out.println("\n=================\n"); + + /** + * 聚合查询 + */ + + SearchSourceBuilder searchSourceBuilder4 = new SearchSourceBuilder(); + + //terms 就是分组统计 根据user进行分组并创建一个新的聚合user_ + TermsAggregationBuilder aggregation = AggregationBuilders.terms("user_").field("user"); + aggregation.subAggregation(AggregationBuilders.avg("average_age").field("age")); + searchSourceBuilder4.aggregation(aggregation); + + SearchRequest searchRequest4 = new SearchRequest(); + searchRequest4.source(searchSourceBuilder4); + + // 同步查询 + SearchResponse searchResponse4 = client.search(searchRequest4, RequestOptions.DEFAULT); + //聚合查询返回条件 + Aggregations aggregations = searchResponse4.getAggregations(); + List aggregationList = aggregations.asList(); + + for (Aggregation agg : aggregations) { + String type = agg.getType(); + if (type.equals(TermsAggregationBuilder.NAME)) { + Bucket elasticBucket = ((Terms) agg).getBucketByKey("Elastic"); + long numberOfDocs = elasticBucket.getDocCount(); + System.out.println("条数:"+numberOfDocs); + } + } + + +// Terms byCompanyAggregation = aggregations.get("user_"); +// Bucket elasticBucket = byCompanyAggregation.getBucketByKey("Elastic"); +// Avg averageAge = elasticBucket.getAggregations().get("average_age"); +// double avg = averageAge.getValue(); +// +// System.out.println("聚合查询返回的结果:"+avg); + + + /* + * 建议查询 + */ + SearchSourceBuilder searchSourceBuilder5 = new SearchSourceBuilder(); + SuggestionBuilder termSuggestionBuilder = SuggestBuilders.termSuggestion("user").text("pancm"); + SuggestBuilder suggestBuilder = new SuggestBuilder(); + suggestBuilder.addSuggestion("suggest_user", termSuggestionBuilder); + searchSourceBuilder5.suggest(suggestBuilder); + + SearchRequest searchRequest5 = new SearchRequest(); + searchRequest5.source(searchSourceBuilder5); + + // 同步查询 + SearchResponse searchResponse5= client.search(searchRequest5, RequestOptions.DEFAULT); + + Suggest suggest = searchResponse5.getSuggest(); + TermSuggestion termSuggestion = suggest.getSuggestion("suggest_user"); + for (TermSuggestion.Entry entry : termSuggestion.getEntries()) { + for (TermSuggestion.Entry.Option option : entry) { + String suggestText = option.getText().string(); + System.out.println("返回结果:"+suggestText); + } + } + + + + + + + + + + } + +} diff --git a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java b/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest1.java similarity index 96% rename from src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java rename to src/main/java/com/pancm/easticsearch/EsHighLevelRestTest1.java index 74bf214..b1726f8 100644 --- a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest1.java +++ b/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest1.java @@ -1,475 +1,475 @@ -package com.pancm.sql.easticsearch; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.function.BiConsumer; - -import org.apache.http.HttpHost; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.bulk.BackoffPolicy; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkProcessor; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.delete.DeleteResponse; -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.action.support.replication.ReplicationResponse; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.rest.RestStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @Title: EsHighLevelRestTest1 - * @Description: Java High Level REST Client Es高级客户端使用教程一 (基本CRUD使用) - 官方文档地址: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high.html - * @since jdk 1.8 - * @Version:1.0.0 - * @author pancm - * @date 2019年3月5日 - */ -public class EsHighLevelRestTest1 { - - private static String elasticIp = "192.169.0.23"; - private static int elasticPort = 9200; - - private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestTest1.class); - - private static RestHighLevelClient client = null; - - public static void main(String[] args) { - try { - init(); - careatindex(); - get(); - exists(); - update(); - delete(); - bulk(); - close(); - } catch (Exception e) { - e.printStackTrace(); - } - - } - - /* - * 初始化服务 - */ - private static void init() { - client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort, "http"))); - - } - - /* - * 关闭服务 - */ - private static void close() { - if (client != null) { - try { - client.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - /** - * 创建索引 - * - * @throws IOException - */ - private static void careatindex() throws IOException { - String index = "user"; - String type = "userindex"; - // 唯一编号 - String id = "1"; - - IndexRequest request = new IndexRequest(index, type, id); - - /* - * 第一种方式，通过jsonString进行创建 - */ - // json - String jsonString = "{" + "\"user\":\"pancm\"," + "\"postDate\":\"2019-03-08\","+ "\"age\":\"18\"," - + "\"message\":\"study Elasticsearch\"" + "}"; - - request.source(jsonString, XContentType.JSON); - - /* - * 第二种方式，通过map创建,，会自动转换成json的数据 - */ - - Map jsonMap = new HashMap<>(); - jsonMap.put("user", "pancm"); - jsonMap.put("postDate", "2019-03-08"); - jsonMap.put("age", "18"); - jsonMap.put("message", "study Elasticsearch"); - - request.source(jsonMap); - - /* - * 第三种方式 : 通过XContentBuilder对象进行创建 - */ - - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - { - builder.field("user", "pancm"); - builder.timeField("postDate", "2019-03-08"); - builder.field("age", "18"); - builder.field("message", "study Elasticsearch"); - } - builder.endObject(); - request.source(builder); - - IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT); - - - - //对响应结果进行处理 - - String index1 = indexResponse.getIndex(); - String type1 = indexResponse.getType(); - String id1 = indexResponse.getId(); - long version = indexResponse.getVersion(); - // 如果是新增/修改的话的话 - if (indexResponse.getResult() == DocWriteResponse.Result.CREATED) { - - } else if (indexResponse.getResult() == DocWriteResponse.Result.UPDATED) { - - } - ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo(); - if (shardInfo.getTotal() != shardInfo.getSuccessful()) { - - } - if (shardInfo.getFailed() > 0) { - for (ReplicationResponse.ShardInfo.Failure failure : shardInfo.getFailures()) { - String reason = failure.reason(); - } - } - - System.out.println("创建成功！"); - } - - /** - * 查询数据 - * - * @throws IOException - */ - private static void get() { - String index = "user"; - String type = "userindex"; - // 唯一编号 - String id = "1"; - // 创建查询请求 - GetRequest getRequest = new GetRequest(index, type, id); - - GetResponse getResponse = null; - try { - getResponse = client.get(getRequest, RequestOptions.DEFAULT); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (ElasticsearchException e) { - // 如果是索引不存在 - if (e.status() == RestStatus.NOT_FOUND) { - System.out.println("该索引库不存在！"+index); - } - - } - // 如果存在该数据则返回对应的结果 - if (getResponse.isExists()) { - long version = getResponse.getVersion(); - String sourceAsString = getResponse.getSourceAsString(); - Map sourceAsMap = getResponse.getSourceAsMap(); - byte[] sourceAsBytes = getResponse.getSourceAsBytes(); - System.out.println("查询返回结果String:"+sourceAsString); - System.out.println("查询返回结果Map:"+sourceAsMap); - } else { - System.out.println("没有找到该数据！"); - } - } - - - /** - * 是否存在 - * @throws IOException - */ - private static void exists() throws IOException { - String index = "user"; - String type = "userindex"; - // 唯一编号 - String id = "1"; - // 创建查询请求 - GetRequest getRequest = new GetRequest(index, type, id); - - boolean exists = client.exists(getRequest, RequestOptions.DEFAULT); - - - ActionListener listener = new ActionListener() { - @Override - public void onResponse(Boolean exists) { - System.out.println("=="+exists); - } - - @Override - public void onFailure(Exception e) { - System.out.println("失败的原因："+e.getMessage()); - } - }; - //进行异步监听 -// client.existsAsync(getRequest, RequestOptions.DEFAULT, listener); - - System.out.println("是否存在："+exists); - } - - - /** - * 更新操作 - * @throws IOException - */ - private static void update() throws IOException { - String index = "user"; - String type = "userindex"; - // 唯一编号 - String id = "1"; - UpdateRequest upateRequest=new UpdateRequest(); - upateRequest.id(id); - upateRequest.index(index); - upateRequest.type(type); - - //依旧可以使用Map这种集合作为更新条件 - Map jsonMap = new HashMap<>(); - jsonMap.put("user", "xuwujing"); - jsonMap.put("postDate", "2019-03-11"); - - upateRequest.doc(jsonMap); - - // - upateRequest.docAsUpsert(true); - // upsert 方法表示如果数据不存在，那么就新增一条 - upateRequest.upsert(jsonMap); - - client.update(upateRequest, RequestOptions.DEFAULT); - System.out.println("更新成功！"); - - } - - /** - * 删除 - * @throws IOException - * - */ - private static void delete() throws IOException { - - String index = "user"; - String type = "userindex"; - // 唯一编号 - String id = "1"; - DeleteRequest deleteRequest=new DeleteRequest(); - deleteRequest.id(id); - deleteRequest.index(index); - deleteRequest.type(type); - - //设置超时时间 - deleteRequest.timeout(TimeValue.timeValueMinutes(2)); - //设置刷新策略"wait_for" - //保持此请求打开，直到刷新使此请求的内容可以搜索为止。此刷新策略与高索引和搜索吞吐量兼容，但它会导致请求等待响应，直到发生刷新 - deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); - - //同步删除 - DeleteResponse deleteResponse = client.delete(deleteRequest, RequestOptions.DEFAULT); - - /* - * 异步删除操作 - */ - - //进行监听 - ActionListener listener = new ActionListener() { - @Override - public void onResponse(DeleteResponse deleteResponse) { - System.out.println("响应:"+deleteResponse); - } - - @Override - public void onFailure(Exception e) { - System.out.println("删除监听异常:"+e.getMessage()); - } - }; - - //异步删除 -// client.deleteAsync(deleteRequest, RequestOptions.DEFAULT, listener); - - ReplicationResponse.ShardInfo shardInfo = deleteResponse.getShardInfo(); - //如果处理成功碎片的数量少于总碎片的情况,说明还在处理或者处理发生异常 - if (shardInfo.getTotal() != shardInfo.getSuccessful()) { - System.out.println("需要处理的碎片总量:"+shardInfo.getTotal()); - System.out.println("处理成功的碎片总量:"+shardInfo.getSuccessful()); - } - - if (shardInfo.getFailed() > 0) { - for (ReplicationResponse.ShardInfo.Failure failure : - shardInfo.getFailures()) { - String reason = failure.reason(); - } - } - System.out.println("删除成功!"); - } - - - /** - * 批量操作示例 - * @throws InterruptedException - */ - private static void bulk() throws IOException, InterruptedException { - String index = "estest"; - String type = "estest"; - - BulkRequest request = new BulkRequest(); - //批量新增 - request.add(new IndexRequest(index, type, "1") - .source(XContentType.JSON,"field", "foo")); - request.add(new IndexRequest(index, type, "2") - .source(XContentType.JSON,"field", "bar")); - request.add(new IndexRequest(index, type, "3") - .source(XContentType.JSON,"field", "baz")); - - //可以进行修改/删除/新增 操作 - request.add(new UpdateRequest(index, type, "2") - .doc(XContentType.JSON,"field", "test")); - request.add(new DeleteRequest(index, type, "3")); - request.add(new IndexRequest(index, type, "4") - .source(XContentType.JSON,"field", "baz")); - - - BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); - - //可以快速检查一个或多个操作是否失败 true是有至少一个失败！ - if (bulkResponse.hasFailures()) { - System.out.println("有一个操作失败!"); - } - - //对处理结果进行遍历操作并根据不同的操作进行处理 - for (BulkItemResponse bulkItemResponse : bulkResponse) { - DocWriteResponse itemResponse = bulkItemResponse.getResponse(); - - //操作失败的进行处理 - if (bulkItemResponse.isFailed()) { - BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); - - } - - if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.INDEX - || bulkItemResponse.getOpType() == DocWriteRequest.OpType.CREATE) { - IndexResponse indexResponse = (IndexResponse) itemResponse; - - } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.UPDATE) { - UpdateResponse updateResponse = (UpdateResponse) itemResponse; - - } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.DELETE) { - DeleteResponse deleteResponse = (DeleteResponse) itemResponse; - } - } - - System.out.println("批量执行成功！"); - - - - /* - * 批量执行处理器相关示例代码 - */ - - - //批量处理器的监听器设置 - - BulkProcessor.Listener listener = new BulkProcessor.Listener() { - - //在执行BulkRequest的每次执行之前调用，这个方法允许知道将要在BulkRequest中执行的操作的数量 - @Override - public void beforeBulk(long executionId, BulkRequest request) { - int numberOfActions = request.numberOfActions(); - logger.debug("Executing bulk [{}] with {} requests", - executionId, numberOfActions); - } - - //在每次执行BulkRequest之后调用，这个方法允许知道BulkResponse是否包含错误 - @Override - public void afterBulk(long executionId, BulkRequest request, - BulkResponse response) { - if (response.hasFailures()) { - logger.warn("Bulk [{}] executed with failures", executionId); - } else { - logger.debug("Bulk [{}] completed in {} milliseconds", - executionId, response.getTook().getMillis()); - } - } - - //如果BulkRequest失败，则调用该方法，该方法允许知道失败 - @Override - public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - logger.error("Failed to execute bulk", failure); - } - }; - - BiConsumer> bulkConsumer = - (request2, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener); - //创建一个批量执行的处理器 - BulkProcessor bulkProcessor = BulkProcessor.builder(bulkConsumer, listener).build(); - BulkProcessor.Builder builder = BulkProcessor.builder(bulkConsumer, listener); - //根据当前添加的操作数量设置刷新新批量请求的时间(默认为1000，使用-1禁用它) - builder.setBulkActions(500); - //根据当前添加的操作大小设置刷新新批量请求的时间(默认为5Mb，使用-1禁用) - builder.setBulkSize(new ByteSizeValue(1L, ByteSizeUnit.MB)); - //设置允许执行的并发请求数量(默认为1，使用0只允许执行单个请求) - builder.setConcurrentRequests(0); - //设置刷新间隔如果间隔通过，则刷新任何挂起的BulkRequest(默认为未设置) - builder.setFlushInterval(TimeValue.timeValueSeconds(10L)); - //设置一个常量后退策略，该策略最初等待1秒并重试最多3次。 - builder.setBackoffPolicy(BackoffPolicy - .constantBackoff(TimeValue.timeValueSeconds(1L), 3)); - - - IndexRequest one = new IndexRequest(index, type, "1"). - source(XContentType.JSON, "title", - "In which order are my Elasticsearch queries executed?"); - IndexRequest two = new IndexRequest(index, type, "2") - .source(XContentType.JSON, "title", - "Current status and upcoming changes in Elasticsearch"); - IndexRequest three = new IndexRequest(index, type, "3") - .source(XContentType.JSON, "title", - "The Future of Federated Search in Elasticsearch"); - bulkProcessor.add(one); - bulkProcessor.add(two); - bulkProcessor.add(three); - - - //如果所有大容量请求都已完成，则该方法返回true;如果在所有大容量请求完成之前的等待时间已经过去，则返回false - boolean terminated = bulkProcessor.awaitClose(30L, TimeUnit.SECONDS); - - System.out.println("请求的响应结果:"+terminated); - - - } - -} +package com.pancm.easticsearch; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; + +import org.apache.http.HttpHost; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.bulk.BackoffPolicy; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkProcessor; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.support.replication.ReplicationResponse; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @Title: EsHighLevelRestTest1 + * @Description: Java High Level REST Client Es高级客户端使用教程一 (基本CRUD使用) + 官方文档地址: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high.html + * @since jdk 1.8 + * @Version:1.0.0 + * @author pancm + * @date 2019年3月5日 + */ +public class EsHighLevelRestTest1 { + + private static String elasticIp = "192.169.0.23"; + private static int elasticPort = 9200; + + private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestTest1.class); + + private static RestHighLevelClient client = null; + + public static void main(String[] args) { + try { + init(); + careatindex(); + get(); + exists(); + update(); + delete(); + bulk(); + close(); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + /* + * 初始化服务 + */ + private static void init() { + client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort, "http"))); + + } + + /* + * 关闭服务 + */ + private static void close() { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + /** + * 创建索引 + * + * @throws IOException + */ + private static void careatindex() throws IOException { + String index = "user"; + String type = "userindex"; + // 唯一编号 + String id = "1"; + + IndexRequest request = new IndexRequest(index, type, id); + + /* + * 第一种方式，通过jsonString进行创建 + */ + // json + String jsonString = "{" + "\"user\":\"pancm\"," + "\"postDate\":\"2019-03-08\","+ "\"age\":\"18\"," + + "\"message\":\"study Elasticsearch\"" + "}"; + + request.source(jsonString, XContentType.JSON); + + /* + * 第二种方式，通过map创建,，会自动转换成json的数据 + */ + + Map jsonMap = new HashMap<>(); + jsonMap.put("user", "pancm"); + jsonMap.put("postDate", "2019-03-08"); + jsonMap.put("age", "18"); + jsonMap.put("message", "study Elasticsearch"); + + request.source(jsonMap); + + /* + * 第三种方式 : 通过XContentBuilder对象进行创建 + */ + + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.field("user", "pancm"); + builder.timeField("postDate", "2019-03-08"); + builder.field("age", "18"); + builder.field("message", "study Elasticsearch"); + } + builder.endObject(); + request.source(builder); + + IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT); + + + + //对响应结果进行处理 + + String index1 = indexResponse.getIndex(); + String type1 = indexResponse.getType(); + String id1 = indexResponse.getId(); + long version = indexResponse.getVersion(); + // 如果是新增/修改的话的话 + if (indexResponse.getResult() == DocWriteResponse.Result.CREATED) { + + } else if (indexResponse.getResult() == DocWriteResponse.Result.UPDATED) { + + } + ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo(); + if (shardInfo.getTotal() != shardInfo.getSuccessful()) { + + } + if (shardInfo.getFailed() > 0) { + for (ReplicationResponse.ShardInfo.Failure failure : shardInfo.getFailures()) { + String reason = failure.reason(); + } + } + + System.out.println("创建成功！"); + } + + /** + * 查询数据 + * + * @throws IOException + */ + private static void get() { + String index = "user"; + String type = "userindex"; + // 唯一编号 + String id = "1"; + // 创建查询请求 + GetRequest getRequest = new GetRequest(index, type, id); + + GetResponse getResponse = null; + try { + getResponse = client.get(getRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ElasticsearchException e) { + // 如果是索引不存在 + if (e.status() == RestStatus.NOT_FOUND) { + System.out.println("该索引库不存在！"+index); + } + + } + // 如果存在该数据则返回对应的结果 + if (getResponse.isExists()) { + long version = getResponse.getVersion(); + String sourceAsString = getResponse.getSourceAsString(); + Map sourceAsMap = getResponse.getSourceAsMap(); + byte[] sourceAsBytes = getResponse.getSourceAsBytes(); + System.out.println("查询返回结果String:"+sourceAsString); + System.out.println("查询返回结果Map:"+sourceAsMap); + } else { + System.out.println("没有找到该数据！"); + } + } + + + /** + * 是否存在 + * @throws IOException + */ + private static void exists() throws IOException { + String index = "user"; + String type = "userindex"; + // 唯一编号 + String id = "1"; + // 创建查询请求 + GetRequest getRequest = new GetRequest(index, type, id); + + boolean exists = client.exists(getRequest, RequestOptions.DEFAULT); + + + ActionListener listener = new ActionListener() { + @Override + public void onResponse(Boolean exists) { + System.out.println("=="+exists); + } + + @Override + public void onFailure(Exception e) { + System.out.println("失败的原因："+e.getMessage()); + } + }; + //进行异步监听 +// client.existsAsync(getRequest, RequestOptions.DEFAULT, listener); + + System.out.println("是否存在："+exists); + } + + + /** + * 更新操作 + * @throws IOException + */ + private static void update() throws IOException { + String index = "user"; + String type = "userindex"; + // 唯一编号 + String id = "1"; + UpdateRequest upateRequest=new UpdateRequest(); + upateRequest.id(id); + upateRequest.index(index); + upateRequest.type(type); + + //依旧可以使用Map这种集合作为更新条件 + Map jsonMap = new HashMap<>(); + jsonMap.put("user", "xuwujing"); + jsonMap.put("postDate", "2019-03-11"); + + upateRequest.doc(jsonMap); + + // + upateRequest.docAsUpsert(true); + // upsert 方法表示如果数据不存在，那么就新增一条 + upateRequest.upsert(jsonMap); + + client.update(upateRequest, RequestOptions.DEFAULT); + System.out.println("更新成功！"); + + } + + /** + * 删除 + * @throws IOException + * + */ + private static void delete() throws IOException { + + String index = "user"; + String type = "userindex"; + // 唯一编号 + String id = "1"; + DeleteRequest deleteRequest=new DeleteRequest(); + deleteRequest.id(id); + deleteRequest.index(index); + deleteRequest.type(type); + + //设置超时时间 + deleteRequest.timeout(TimeValue.timeValueMinutes(2)); + //设置刷新策略"wait_for" + //保持此请求打开，直到刷新使此请求的内容可以搜索为止。此刷新策略与高索引和搜索吞吐量兼容，但它会导致请求等待响应，直到发生刷新 + deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); + + //同步删除 + DeleteResponse deleteResponse = client.delete(deleteRequest, RequestOptions.DEFAULT); + + /* + * 异步删除操作 + */ + + //进行监听 + ActionListener listener = new ActionListener() { + @Override + public void onResponse(DeleteResponse deleteResponse) { + System.out.println("响应:"+deleteResponse); + } + + @Override + public void onFailure(Exception e) { + System.out.println("删除监听异常:"+e.getMessage()); + } + }; + + //异步删除 +// client.deleteAsync(deleteRequest, RequestOptions.DEFAULT, listener); + + ReplicationResponse.ShardInfo shardInfo = deleteResponse.getShardInfo(); + //如果处理成功碎片的数量少于总碎片的情况,说明还在处理或者处理发生异常 + if (shardInfo.getTotal() != shardInfo.getSuccessful()) { + System.out.println("需要处理的碎片总量:"+shardInfo.getTotal()); + System.out.println("处理成功的碎片总量:"+shardInfo.getSuccessful()); + } + + if (shardInfo.getFailed() > 0) { + for (ReplicationResponse.ShardInfo.Failure failure : + shardInfo.getFailures()) { + String reason = failure.reason(); + } + } + System.out.println("删除成功!"); + } + + + /** + * 批量操作示例 + * @throws InterruptedException + */ + private static void bulk() throws IOException, InterruptedException { + String index = "estest"; + String type = "estest"; + + BulkRequest request = new BulkRequest(); + //批量新增 + request.add(new IndexRequest(index, type, "1") + .source(XContentType.JSON,"field", "foo")); + request.add(new IndexRequest(index, type, "2") + .source(XContentType.JSON,"field", "bar")); + request.add(new IndexRequest(index, type, "3") + .source(XContentType.JSON,"field", "baz")); + + //可以进行修改/删除/新增 操作 + request.add(new UpdateRequest(index, type, "2") + .doc(XContentType.JSON,"field", "test")); + request.add(new DeleteRequest(index, type, "3")); + request.add(new IndexRequest(index, type, "4") + .source(XContentType.JSON,"field", "baz")); + + + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + + //可以快速检查一个或多个操作是否失败 true是有至少一个失败！ + if (bulkResponse.hasFailures()) { + System.out.println("有一个操作失败!"); + } + + //对处理结果进行遍历操作并根据不同的操作进行处理 + for (BulkItemResponse bulkItemResponse : bulkResponse) { + DocWriteResponse itemResponse = bulkItemResponse.getResponse(); + + //操作失败的进行处理 + if (bulkItemResponse.isFailed()) { + BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); + + } + + if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.INDEX + || bulkItemResponse.getOpType() == DocWriteRequest.OpType.CREATE) { + IndexResponse indexResponse = (IndexResponse) itemResponse; + + } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.UPDATE) { + UpdateResponse updateResponse = (UpdateResponse) itemResponse; + + } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.DELETE) { + DeleteResponse deleteResponse = (DeleteResponse) itemResponse; + } + } + + System.out.println("批量执行成功！"); + + + + /* + * 批量执行处理器相关示例代码 + */ + + + //批量处理器的监听器设置 + + BulkProcessor.Listener listener = new BulkProcessor.Listener() { + + //在执行BulkRequest的每次执行之前调用，这个方法允许知道将要在BulkRequest中执行的操作的数量 + @Override + public void beforeBulk(long executionId, BulkRequest request) { + int numberOfActions = request.numberOfActions(); + logger.debug("Executing bulk [{}] with {} requests", + executionId, numberOfActions); + } + + //在每次执行BulkRequest之后调用，这个方法允许知道BulkResponse是否包含错误 + @Override + public void afterBulk(long executionId, BulkRequest request, + BulkResponse response) { + if (response.hasFailures()) { + logger.warn("Bulk [{}] executed with failures", executionId); + } else { + logger.debug("Bulk [{}] completed in {} milliseconds", + executionId, response.getTook().getMillis()); + } + } + + //如果BulkRequest失败，则调用该方法，该方法允许知道失败 + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + logger.error("Failed to execute bulk", failure); + } + }; + + BiConsumer> bulkConsumer = + (request2, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener); + //创建一个批量执行的处理器 + BulkProcessor bulkProcessor = BulkProcessor.builder(bulkConsumer, listener).build(); + BulkProcessor.Builder builder = BulkProcessor.builder(bulkConsumer, listener); + //根据当前添加的操作数量设置刷新新批量请求的时间(默认为1000，使用-1禁用它) + builder.setBulkActions(500); + //根据当前添加的操作大小设置刷新新批量请求的时间(默认为5Mb，使用-1禁用) + builder.setBulkSize(new ByteSizeValue(1L, ByteSizeUnit.MB)); + //设置允许执行的并发请求数量(默认为1，使用0只允许执行单个请求) + builder.setConcurrentRequests(0); + //设置刷新间隔如果间隔通过，则刷新任何挂起的BulkRequest(默认为未设置) + builder.setFlushInterval(TimeValue.timeValueSeconds(10L)); + //设置一个常量后退策略，该策略最初等待1秒并重试最多3次。 + builder.setBackoffPolicy(BackoffPolicy + .constantBackoff(TimeValue.timeValueSeconds(1L), 3)); + + + IndexRequest one = new IndexRequest(index, type, "1"). + source(XContentType.JSON, "title", + "In which order are my Elasticsearch queries executed?"); + IndexRequest two = new IndexRequest(index, type, "2") + .source(XContentType.JSON, "title", + "Current status and upcoming changes in Elasticsearch"); + IndexRequest three = new IndexRequest(index, type, "3") + .source(XContentType.JSON, "title", + "The Future of Federated Search in Elasticsearch"); + bulkProcessor.add(one); + bulkProcessor.add(two); + bulkProcessor.add(three); + + + //如果所有大容量请求都已完成，则该方法返回true;如果在所有大容量请求完成之前的等待时间已经过去，则返回false + boolean terminated = bulkProcessor.awaitClose(30L, TimeUnit.SECONDS); + + System.out.println("请求的响应结果:"+terminated); + + + } + +} diff --git a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest2.java b/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest2.java similarity index 96% rename from src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest2.java rename to src/main/java/com/pancm/easticsearch/EsHighLevelRestTest2.java index 88dcae3..595249a 100644 --- a/src/main/java/com/pancm/sql/easticsearch/EsHighLevelRestTest2.java +++ b/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest2.java @@ -1,343 +1,343 @@ -package com.pancm.sql.easticsearch; - -import static org.junit.Assert.assertNull; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import org.apache.http.HttpHost; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.get.MultiGetItemResponse; -import org.elasticsearch.action.get.MultiGetRequest; -import org.elasticsearch.action.get.MultiGetResponse; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.RethrottleRequest; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.TermQueryBuilder; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.elasticsearch.index.reindex.DeleteByQueryRequest; -import org.elasticsearch.index.reindex.ReindexRequest; -import org.elasticsearch.index.reindex.ScrollableHitSource; -import org.elasticsearch.index.reindex.UpdateByQueryRequest; -import org.elasticsearch.tasks.TaskId; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @Title: EsHighLevelRestTest2 - * @Description: Java High Level REST Client Es高级客户端使用教程二 (关于组合使用) - * @since jdk 1.8 - * @Version:1.0.0 - * @author pancm - * @date 2019年3月11日 - */ -public class EsHighLevelRestTest2 { - - private static String elasticIp = "192.169.0.23"; - private static int elasticPort = 9200; - - private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestTest2.class); - - private static RestHighLevelClient client = null; - - /** - * @param args - */ - public static void main(String[] args) { - try { - init(); - multiGet(); - reindex(); - updataByQuery(); - deleteByQuery(); - rethrottleByQuery(); - close(); - } catch (Exception e) { - e.printStackTrace(); - } - - } - - /* - * 初始化服务 - */ - private static void init() { - client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort, "http"))); - - } - - /* - * 关闭服务 - */ - private static void close() { - if (client != null) { - try { - client.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - /** - * 多查询使用 - * - * @throws IOException - */ - private static void multiGet() throws IOException { - - MultiGetRequest request = new MultiGetRequest(); - request.add(new MultiGetRequest.Item("estest", "estest", "1")); - request.add(new MultiGetRequest.Item("user", "userindex", "2")); - // 禁用源检索，默认启用 -// request.add(new MultiGetRequest.Item("user", "userindex", "2").fetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE)); - - // 同步构建 - MultiGetResponse response = client.mget(request, RequestOptions.DEFAULT); - - // 异步构建 -// MultiGetResponse response2 = client.mgetAsync(request, RequestOptions.DEFAULT, listener); - - /* - * 返回的MultiGetResponse包含在' getResponses中的MultiGetItemResponse的列表，其顺序与请求它们的顺序相同。 - * 如果成功，MultiGetItemResponse包含GetResponse或MultiGetResponse。如果失败了就失败。 - * 成功看起来就像一个正常的GetResponse - */ - - for (MultiGetItemResponse item : response.getResponses()) { - assertNull(item.getFailure()); - GetResponse get = item.getResponse(); - String index = item.getIndex(); - String type = item.getType(); - String id = item.getId(); - // 如果请求存在 - if (get.isExists()) { - long version = get.getVersion(); - String sourceAsString = get.getSourceAsString(); - Map sourceAsMap = get.getSourceAsMap(); - byte[] sourceAsBytes = get.getSourceAsBytes(); - System.out.println("查询的结果:" + sourceAsMap); - } else { - System.out.println("没有找到该文档!"); - } - - } - - } - - /** - * 索引复制 - * - * @throws IOException - */ - private static void reindex() throws IOException { - // 创建索引复制请求并进行索引复制 - ReindexRequest request = new ReindexRequest(); - // 需要复制的索引 - request.setSourceIndices("user"); - // 复制的目标索引 - request.setDestIndex("dest_test"); - - // 表示如果在复制索引的时候有缺失的文档的话会进行创建,默认是index - request.setDestOpType("create"); - // 如果在复制的过程中发现版本冲突，那么会继续进行复制 - request.setConflicts("proceed"); - - // 只复制文档类型为 userindex 的数据 - request.setSourceDocTypes("userindex"); - // 只复制 pancm 用户的数据 - request.setSourceQuery(new TermQueryBuilder("user", "pancm")); - // 设置复制文档的数量 - request.setSize(10); - // 设置一次批量处理的条数，默认是1000 - request.setSourceBatchSize(100); - - // 进行排序 -// request.addSortField("postDate", SortOrder.DESC); - - // 指定切片大小 - request.setSlices(2); - - //设置超时时间 - request.setTimeout(TimeValue.timeValueMinutes(2)); - //允许刷新 - request.setRefresh(true); - - // 同步执行 - BulkByScrollResponse bulkResponse = client.reindex(request, RequestOptions.DEFAULT); - - // 异步执行 -// client.reindexAsync(request, RequestOptions.DEFAULT, listener); - - // 响应结果处理 - - TimeValue timeTaken = bulkResponse.getTook(); - boolean timedOut = bulkResponse.isTimedOut(); - long totalDocs = bulkResponse.getTotal(); - long updatedDocs = bulkResponse.getUpdated(); - long createdDocs = bulkResponse.getCreated(); - long deletedDocs = bulkResponse.getDeleted(); - long batches = bulkResponse.getBatches(); - long noops = bulkResponse.getNoops(); - long versionConflicts = bulkResponse.getVersionConflicts(); - long bulkRetries = bulkResponse.getBulkRetries(); - long searchRetries = bulkResponse.getSearchRetries(); - TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); - TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); - List searchFailures = bulkResponse.getSearchFailures(); - List bulkFailures = bulkResponse.getBulkFailures(); - - System.out.println("索引复制总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",创建数:" + createdDocs - + ",更新数:" + updatedDocs); - } - - - - /** - * 根据查询条件更新 - * - * @throws IOException - */ - private static void updataByQuery() throws IOException { - // - UpdateByQueryRequest request = new UpdateByQueryRequest("user"); - - // 设置查询条件 - request.setQuery(new TermQueryBuilder("user", "pancm")); - - // 设置复制文档的数量 - request.setSize(10); - // 设置一次批量处理的条数，默认是1000 - request.setBatchSize(100); - //设置路由 - request.setRouting("=cat"); - //设置超时时间 - request.setTimeout(TimeValue.timeValueMinutes(2)); - //允许刷新 - request.setRefresh(true); - //索引选项 - request.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); - - // 同步执行 - BulkByScrollResponse bulkResponse = client.updateByQuery(request, RequestOptions.DEFAULT); - - // 异步执行 -// client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener); - - // 返回结果 - TimeValue timeTaken = bulkResponse.getTook(); - boolean timedOut = bulkResponse.isTimedOut(); - long totalDocs = bulkResponse.getTotal(); - long updatedDocs = bulkResponse.getUpdated(); - long deletedDocs = bulkResponse.getDeleted(); - long batches = bulkResponse.getBatches(); - long noops = bulkResponse.getNoops(); - long versionConflicts = bulkResponse.getVersionConflicts(); - long bulkRetries = bulkResponse.getBulkRetries(); - long searchRetries = bulkResponse.getSearchRetries(); - TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); - TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); - List searchFailures = bulkResponse.getSearchFailures(); - List bulkFailures = bulkResponse.getBulkFailures(); - System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",更新数:" + updatedDocs); - - } - - - /** - * 根据查询条件删除 - * @throws IOException - */ - private static void deleteByQuery() throws IOException { - // - DeleteByQueryRequest request = new DeleteByQueryRequest("user"); - - // 设置查询条件 - request.setQuery(new TermQueryBuilder("user", "pancm")); - - // 设置复制文档的数量 - request.setSize(10); - // 设置一次批量处理的条数，默认是1000 - request.setBatchSize(100); - //设置路由 - request.setRouting("=cat"); - //设置超时时间 - request.setTimeout(TimeValue.timeValueMinutes(2)); - //允许刷新 - request.setRefresh(true); - //索引选项 - request.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); - - // 同步执行 - BulkByScrollResponse bulkResponse = client.deleteByQuery(request, RequestOptions.DEFAULT); - - // 异步执行 -// client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener); - - // 返回结果 - TimeValue timeTaken = bulkResponse.getTook(); - boolean timedOut = bulkResponse.isTimedOut(); - long totalDocs = bulkResponse.getTotal(); - long deletedDocs = bulkResponse.getDeleted(); - long batches = bulkResponse.getBatches(); - long noops = bulkResponse.getNoops(); - long versionConflicts = bulkResponse.getVersionConflicts(); - long bulkRetries = bulkResponse.getBulkRetries(); - long searchRetries = bulkResponse.getSearchRetries(); - TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); - TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); - List searchFailures = bulkResponse.getSearchFailures(); - List bulkFailures = bulkResponse.getBulkFailures(); - System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",删除数:" + deletedDocs); - - } - - - /** - * 用于更改正在运行的重索引、逐查询更新或逐查询删除任务的当前节流，或完全禁用任务的节流。 - * @throws IOException - */ - private static void rethrottleByQuery() throws IOException { - TaskId taskId=new TaskId("1"); - //用于更改正在运行的重索引、逐查询更新或逐查询删除任务的当前节流，或完全禁用任务的节流。 - //并且将请求将任务的节流更改为每秒100个请求 - RethrottleRequest request = new RethrottleRequest(taskId,100.0f); - - // 同步设置需要更改的流 -// client.reindexRethrottle(request, RequestOptions.DEFAULT); -// client.updateByQueryRethrottle(request, RequestOptions.DEFAULT); -// client.deleteByQueryRethrottle(request, RequestOptions.DEFAULT); - - - ActionListener listener = new ActionListener() { - @Override - public void onResponse(ListTasksResponse response) { - System.out.println("===="+response.getTasks().toString()); - } - - @Override - public void onFailure(Exception e) { - System.out.println("====---"+e.getMessage()); - } - }; - - // 异步设置要更改的流 - client.reindexRethrottleAsync(request, RequestOptions.DEFAULT, listener); - client.updateByQueryRethrottleAsync(request, RequestOptions.DEFAULT, listener); - client.deleteByQueryRethrottleAsync(request, RequestOptions.DEFAULT, listener); - - System.out.println("已成功设置!"); - - } - - - - -} +package com.pancm.easticsearch; + +import static org.junit.Assert.assertNull; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.http.HttpHost; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetItemResponse; +import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.RethrottleRequest; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.index.reindex.ReindexRequest; +import org.elasticsearch.index.reindex.ScrollableHitSource; +import org.elasticsearch.index.reindex.UpdateByQueryRequest; +import org.elasticsearch.tasks.TaskId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @Title: EsHighLevelRestTest2 + * @Description: Java High Level REST Client Es高级客户端使用教程二 (关于组合使用) + * @since jdk 1.8 + * @Version:1.0.0 + * @author pancm + * @date 2019年3月11日 + */ +public class EsHighLevelRestTest2 { + + private static String elasticIp = "192.169.0.23"; + private static int elasticPort = 9200; + + private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestTest2.class); + + private static RestHighLevelClient client = null; + + /** + * @param args + */ + public static void main(String[] args) { + try { + init(); + multiGet(); + reindex(); + updataByQuery(); + deleteByQuery(); + rethrottleByQuery(); + close(); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + /* + * 初始化服务 + */ + private static void init() { + client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort, "http"))); + + } + + /* + * 关闭服务 + */ + private static void close() { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + /** + * 多查询使用 + * + * @throws IOException + */ + private static void multiGet() throws IOException { + + MultiGetRequest request = new MultiGetRequest(); + request.add(new MultiGetRequest.Item("estest", "estest", "1")); + request.add(new MultiGetRequest.Item("user", "userindex", "2")); + // 禁用源检索，默认启用 +// request.add(new MultiGetRequest.Item("user", "userindex", "2").fetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE)); + + // 同步构建 + MultiGetResponse response = client.mget(request, RequestOptions.DEFAULT); + + // 异步构建 +// MultiGetResponse response2 = client.mgetAsync(request, RequestOptions.DEFAULT, listener); + + /* + * 返回的MultiGetResponse包含在' getResponses中的MultiGetItemResponse的列表，其顺序与请求它们的顺序相同。 + * 如果成功，MultiGetItemResponse包含GetResponse或MultiGetResponse。如果失败了就失败。 + * 成功看起来就像一个正常的GetResponse + */ + + for (MultiGetItemResponse item : response.getResponses()) { + assertNull(item.getFailure()); + GetResponse get = item.getResponse(); + String index = item.getIndex(); + String type = item.getType(); + String id = item.getId(); + // 如果请求存在 + if (get.isExists()) { + long version = get.getVersion(); + String sourceAsString = get.getSourceAsString(); + Map sourceAsMap = get.getSourceAsMap(); + byte[] sourceAsBytes = get.getSourceAsBytes(); + System.out.println("查询的结果:" + sourceAsMap); + } else { + System.out.println("没有找到该文档!"); + } + + } + + } + + /** + * 索引复制 + * + * @throws IOException + */ + private static void reindex() throws IOException { + // 创建索引复制请求并进行索引复制 + ReindexRequest request = new ReindexRequest(); + // 需要复制的索引 + request.setSourceIndices("user"); + // 复制的目标索引 + request.setDestIndex("dest_test"); + + // 表示如果在复制索引的时候有缺失的文档的话会进行创建,默认是index + request.setDestOpType("create"); + // 如果在复制的过程中发现版本冲突，那么会继续进行复制 + request.setConflicts("proceed"); + + // 只复制文档类型为 userindex 的数据 + request.setSourceDocTypes("userindex"); + // 只复制 pancm 用户的数据 + request.setSourceQuery(new TermQueryBuilder("user", "pancm")); + // 设置复制文档的数量 + request.setSize(10); + // 设置一次批量处理的条数，默认是1000 + request.setSourceBatchSize(100); + + // 进行排序 +// request.addSortField("postDate", SortOrder.DESC); + + // 指定切片大小 + request.setSlices(2); + + //设置超时时间 + request.setTimeout(TimeValue.timeValueMinutes(2)); + //允许刷新 + request.setRefresh(true); + + // 同步执行 + BulkByScrollResponse bulkResponse = client.reindex(request, RequestOptions.DEFAULT); + + // 异步执行 +// client.reindexAsync(request, RequestOptions.DEFAULT, listener); + + // 响应结果处理 + + TimeValue timeTaken = bulkResponse.getTook(); + boolean timedOut = bulkResponse.isTimedOut(); + long totalDocs = bulkResponse.getTotal(); + long updatedDocs = bulkResponse.getUpdated(); + long createdDocs = bulkResponse.getCreated(); + long deletedDocs = bulkResponse.getDeleted(); + long batches = bulkResponse.getBatches(); + long noops = bulkResponse.getNoops(); + long versionConflicts = bulkResponse.getVersionConflicts(); + long bulkRetries = bulkResponse.getBulkRetries(); + long searchRetries = bulkResponse.getSearchRetries(); + TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); + TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); + List searchFailures = bulkResponse.getSearchFailures(); + List bulkFailures = bulkResponse.getBulkFailures(); + + System.out.println("索引复制总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",创建数:" + createdDocs + + ",更新数:" + updatedDocs); + } + + + + /** + * 根据查询条件更新 + * + * @throws IOException + */ + private static void updataByQuery() throws IOException { + // + UpdateByQueryRequest request = new UpdateByQueryRequest("user"); + + // 设置查询条件 + request.setQuery(new TermQueryBuilder("user", "pancm")); + + // 设置复制文档的数量 + request.setSize(10); + // 设置一次批量处理的条数，默认是1000 + request.setBatchSize(100); + //设置路由 + request.setRouting("=cat"); + //设置超时时间 + request.setTimeout(TimeValue.timeValueMinutes(2)); + //允许刷新 + request.setRefresh(true); + //索引选项 + request.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); + + // 同步执行 + BulkByScrollResponse bulkResponse = client.updateByQuery(request, RequestOptions.DEFAULT); + + // 异步执行 +// client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener); + + // 返回结果 + TimeValue timeTaken = bulkResponse.getTook(); + boolean timedOut = bulkResponse.isTimedOut(); + long totalDocs = bulkResponse.getTotal(); + long updatedDocs = bulkResponse.getUpdated(); + long deletedDocs = bulkResponse.getDeleted(); + long batches = bulkResponse.getBatches(); + long noops = bulkResponse.getNoops(); + long versionConflicts = bulkResponse.getVersionConflicts(); + long bulkRetries = bulkResponse.getBulkRetries(); + long searchRetries = bulkResponse.getSearchRetries(); + TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); + TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); + List searchFailures = bulkResponse.getSearchFailures(); + List bulkFailures = bulkResponse.getBulkFailures(); + System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",更新数:" + updatedDocs); + + } + + + /** + * 根据查询条件删除 + * @throws IOException + */ + private static void deleteByQuery() throws IOException { + // + DeleteByQueryRequest request = new DeleteByQueryRequest("user"); + + // 设置查询条件 + request.setQuery(new TermQueryBuilder("user", "pancm")); + + // 设置复制文档的数量 + request.setSize(10); + // 设置一次批量处理的条数，默认是1000 + request.setBatchSize(100); + //设置路由 + request.setRouting("=cat"); + //设置超时时间 + request.setTimeout(TimeValue.timeValueMinutes(2)); + //允许刷新 + request.setRefresh(true); + //索引选项 + request.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); + + // 同步执行 + BulkByScrollResponse bulkResponse = client.deleteByQuery(request, RequestOptions.DEFAULT); + + // 异步执行 +// client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener); + + // 返回结果 + TimeValue timeTaken = bulkResponse.getTook(); + boolean timedOut = bulkResponse.isTimedOut(); + long totalDocs = bulkResponse.getTotal(); + long deletedDocs = bulkResponse.getDeleted(); + long batches = bulkResponse.getBatches(); + long noops = bulkResponse.getNoops(); + long versionConflicts = bulkResponse.getVersionConflicts(); + long bulkRetries = bulkResponse.getBulkRetries(); + long searchRetries = bulkResponse.getSearchRetries(); + TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); + TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); + List searchFailures = bulkResponse.getSearchFailures(); + List bulkFailures = bulkResponse.getBulkFailures(); + System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",删除数:" + deletedDocs); + + } + + + /** + * 用于更改正在运行的重索引、逐查询更新或逐查询删除任务的当前节流，或完全禁用任务的节流。 + * @throws IOException + */ + private static void rethrottleByQuery() throws IOException { + TaskId taskId=new TaskId("1"); + //用于更改正在运行的重索引、逐查询更新或逐查询删除任务的当前节流，或完全禁用任务的节流。 + //并且将请求将任务的节流更改为每秒100个请求 + RethrottleRequest request = new RethrottleRequest(taskId,100.0f); + + // 同步设置需要更改的流 +// client.reindexRethrottle(request, RequestOptions.DEFAULT); +// client.updateByQueryRethrottle(request, RequestOptions.DEFAULT); +// client.deleteByQueryRethrottle(request, RequestOptions.DEFAULT); + + + ActionListener listener = new ActionListener() { + @Override + public void onResponse(ListTasksResponse response) { + System.out.println("===="+response.getTasks().toString()); + } + + @Override + public void onFailure(Exception e) { + System.out.println("====---"+e.getMessage()); + } + }; + + // 异步设置要更改的流 + client.reindexRethrottleAsync(request, RequestOptions.DEFAULT, listener); + client.updateByQueryRethrottleAsync(request, RequestOptions.DEFAULT, listener); + client.deleteByQueryRethrottleAsync(request, RequestOptions.DEFAULT, listener); + + System.out.println("已成功设置!"); + + } + + + + +} diff --git a/src/main/java/com/pancm/sql/easticsearch/JestTest.java b/src/main/java/com/pancm/easticsearch/JestTest.java similarity index 95% rename from src/main/java/com/pancm/sql/easticsearch/JestTest.java rename to src/main/java/com/pancm/easticsearch/JestTest.java index 13c4127..267d659 100644 --- a/src/main/java/com/pancm/sql/easticsearch/JestTest.java +++ b/src/main/java/com/pancm/easticsearch/JestTest.java @@ -1,370 +1,370 @@ -package com.pancm.sql.easticsearch; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; - -import io.searchbox.client.JestClient; -import io.searchbox.client.JestClientFactory; -import io.searchbox.client.JestResult; -import io.searchbox.client.config.HttpClientConfig; -import io.searchbox.core.Bulk; -import io.searchbox.core.BulkResult; -import io.searchbox.core.Delete; -import io.searchbox.core.DocumentResult; -import io.searchbox.core.Index; -import io.searchbox.core.Search; -import io.searchbox.indices.CreateIndex; -import io.searchbox.indices.DeleteIndex; -import io.searchbox.indices.mapping.GetMapping; -import io.searchbox.indices.mapping.PutMapping; - -/** - * -* @Title: JestTest -* @Description: es 的 Jest 测试类 -* @Version:1.0.0 -* @author pancm -* @date 2019年2月28日 - */ -public class JestTest { - private static JestClient jestClient; - private static String indexName = "userindex"; - private static String typeName = "user"; - private static String elasticIps="http://127.0.0.1:9200"; - - - public static void main(String[] args) throws Exception { - jestClient = getJestClient(); - insertBatch(); - serach1(); - serach2(); - serach3(); - jestClient.close(); - - } - - private static JestClient getJestClient() { - JestClientFactory factory = new JestClientFactory(); - factory.setHttpClientConfig(new HttpClientConfig.Builder(elasticIps).connTimeout(60000).readTimeout(60000).multiThreaded(true).build()); - return factory.getObject(); - } - - public static void insertBatch() { - List objs = new ArrayList(); - objs.add(new User(1L, "张三", 20, "张三是个Java开发工程师","2018-4-25 11:07:42")); - objs.add(new User(2L, "李四", 24, "李四是个测试工程师","1980-2-15 19:01:32")); - objs.add(new User(3L, "王五", 25, "王五是个运维工程师","2016-8-21 06:11:32")); - boolean result = false; - try { - result = insertBatch(jestClient,indexName, typeName,objs); - } catch (Exception e) { - e.printStackTrace(); - } - System.out.println("批量新增:"+result); - } - - - /** - * 全文搜索 - */ - public static void serach1() { - String query ="工程师"; - try { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.queryStringQuery(query)); - //分页设置 - searchSourceBuilder.from(0).size(2); - System.out.println("全文搜索查询语句:"+searchSourceBuilder.toString()); - System.out.println("全文搜索返回结果:"+search(jestClient,indexName, typeName, searchSourceBuilder.toString())); - } catch (Exception e) { - e.printStackTrace(); - } - } - - /** - * 精确搜索 - */ - public static void serach2() { - try { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.termQuery("age", 24)); - System.out.println("精确搜索查询语句:"+searchSourceBuilder.toString()); - System.out.println("精确搜索返回结果:"+search(jestClient,indexName, typeName, searchSourceBuilder.toString())); - } catch (Exception e) { - e.printStackTrace(); - } - } - - - /** - * 区间搜索 - */ - public static void serach3() { - String createtm="createtm"; - String from="2016-8-21 06:11:32"; - String to="2018-8-21 06:11:32"; - - try { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.rangeQuery(createtm).gte(from).lte(to)); - System.out.println("区间搜索语句:"+searchSourceBuilder.toString()); - System.out.println("区间搜索返回结果:"+search(jestClient,indexName, typeName, searchSourceBuilder.toString())); - } catch (Exception e) { - e.printStackTrace(); - } - } - - - /** - * 创建索引 - * @param indexName - * @return - * @throws Exception - */ - public boolean createIndex(JestClient jestClient,String indexName) throws Exception { - JestResult jr = jestClient.execute(new CreateIndex.Builder(indexName).build()); - return jr.isSucceeded(); - } - - /** - * 新增数据 - * @param indexName - * @param typeName - * @param source - * @return - * @throws Exception - */ - public boolean insert(JestClient jestClient,String indexName, String typeName, String source) throws Exception { - PutMapping putMapping = new PutMapping.Builder(indexName, typeName, source).build(); - JestResult jr = jestClient.execute(putMapping); - return jr.isSucceeded(); - } - - - /** - * 查询数据 - * @param indexName - * @param typeName - * @return - * @throws Exception - */ - public static String getIndexMapping(JestClient jestClient,String indexName, String typeName) throws Exception { - GetMapping getMapping = new GetMapping.Builder().addIndex(indexName).addType(typeName).build(); - JestResult jr =jestClient.execute(getMapping); - return jr.getJsonString(); - } - - - - /** - * 批量新增数据 - * @param indexName - * @param typeName - * @param objs - * @return - * @throws Exception - */ - public static boolean insertBatch(JestClient jestClient,String indexName, String typeName, List objs) throws Exception { - Bulk.Builder bulk = new Bulk.Builder().defaultIndex(indexName).defaultType(typeName); - for (Object obj : objs) { - Index index = new Index.Builder(obj).build(); - bulk.addAction(index); - } - BulkResult br = jestClient.execute(bulk.build()); - return br.isSucceeded(); - } - - /** - * 全文搜索 - * @param indexName - * @param typeName - * @param query - * @return - * @throws Exception - */ - public static String search(JestClient jestClient,String indexName, String typeName, String query) throws Exception { - Search search = new Search.Builder(query) - .addIndex(indexName) - .addType(typeName) - .build(); - JestResult jr = jestClient.execute(search); -// System.out.println("--"+jr.getJsonString()); -// System.out.println("--"+jr.getSourceAsObject(User.class)); - return jr.getSourceAsString(); - } - - - - - - /** - * 删除索引 - * @param indexName - * @return - * @throws Exception - */ - public boolean delete(JestClient jestClient,String indexName) throws Exception { - JestResult jr = jestClient.execute(new DeleteIndex.Builder(indexName).build()); - return jr.isSucceeded(); - } - - /** - * 删除数据 - * @param indexName - * @param typeName - * @param id - * @return - * @throws Exception - */ - public boolean delete(JestClient jestClient,String indexName, String typeName, String id) throws Exception { - DocumentResult dr = jestClient.execute(new Delete.Builder(id).index(indexName).type(typeName).build()); - return dr.isSucceeded(); - } - -} - - - class User implements Serializable{ - /** - * - */ - private static final long serialVersionUID = 1L; - /** 编号 */ - private Long id; - /** 姓名 */ - private String name; - - /** 年龄 */ - private Integer age; - - /** 描述 */ - private String description; - - /** 创建时间 */ - private String createtm; - - - public User(){ - } - - - - - public User(Long id, String name, Integer age, String description, String createtm) { - super(); - this.id = id; - this.name = name; - this.age = age; - this.description = description; - this.createtm = createtm; - } - - - - - /** - * 获取编号 - * @return id - */ - public Long getId() { - return id; - } - - - /** - * 设置编号 - * @param Long id - */ - public void setId(Long id) { - this.id = id; - } - - - /** - * 获取姓名 - * @return name - */ - public String getName() { - return name; - } - - - /** - * 设置姓名 - * @param String name - */ - public void setName(String name) { - this.name = name; - } - - - /** - * 获取年龄 - * @return age - */ - public Integer getAge() { - return age; - } - - - /** - * 设置年龄 - * @param Integer age - */ - public void setAge(Integer age) { - this.age = age; - } - - - /** - * 获取描述 - * @return description - */ - public String getDescription() { - return description; - } - - - /** - * 设置描述 - * @param String description - */ - public void setDescription(String description) { - this.description = description; - } - - - /** - * 获取创建时间 - * @return createtm - */ - public String getCreatetm() { - return createtm; - } - - - /** - * 设置创建时间 - * @param String createtm - */ - public void setCreatetm(String createtm) { - this.createtm = createtm; - } - - - - - @Override - public String toString() { - return "User [id=" + id + ", name=" + name + ", age=" + age + ", description=" + description + ", createtm=" - + createtm + "]"; - } - - - -} - +package com.pancm.easticsearch; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import io.searchbox.client.JestClient; +import io.searchbox.client.JestClientFactory; +import io.searchbox.client.JestResult; +import io.searchbox.client.config.HttpClientConfig; +import io.searchbox.core.Bulk; +import io.searchbox.core.BulkResult; +import io.searchbox.core.Delete; +import io.searchbox.core.DocumentResult; +import io.searchbox.core.Index; +import io.searchbox.core.Search; +import io.searchbox.indices.CreateIndex; +import io.searchbox.indices.DeleteIndex; +import io.searchbox.indices.mapping.GetMapping; +import io.searchbox.indices.mapping.PutMapping; + +/** + * +* @Title: JestTest +* @Description: es 的 Jest 测试类 +* @Version:1.0.0 +* @author pancm +* @date 2019年2月28日 + */ +public class JestTest { + private static JestClient jestClient; + private static String indexName = "userindex"; + private static String typeName = "user"; + private static String elasticIps="http://127.0.0.1:9200"; + + + public static void main(String[] args) throws Exception { + jestClient = getJestClient(); + insertBatch(); + serach1(); + serach2(); + serach3(); + jestClient.close(); + + } + + private static JestClient getJestClient() { + JestClientFactory factory = new JestClientFactory(); + factory.setHttpClientConfig(new HttpClientConfig.Builder(elasticIps).connTimeout(60000).readTimeout(60000).multiThreaded(true).build()); + return factory.getObject(); + } + + public static void insertBatch() { + List objs = new ArrayList(); + objs.add(new User(1L, "张三", 20, "张三是个Java开发工程师","2018-4-25 11:07:42")); + objs.add(new User(2L, "李四", 24, "李四是个测试工程师","1980-2-15 19:01:32")); + objs.add(new User(3L, "王五", 25, "王五是个运维工程师","2016-8-21 06:11:32")); + boolean result = false; + try { + result = insertBatch(jestClient,indexName, typeName,objs); + } catch (Exception e) { + e.printStackTrace(); + } + System.out.println("批量新增:"+result); + } + + + /** + * 全文搜索 + */ + public static void serach1() { + String query ="工程师"; + try { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.queryStringQuery(query)); + //分页设置 + searchSourceBuilder.from(0).size(2); + System.out.println("全文搜索查询语句:"+searchSourceBuilder.toString()); + System.out.println("全文搜索返回结果:"+search(jestClient,indexName, typeName, searchSourceBuilder.toString())); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 精确搜索 + */ + public static void serach2() { + try { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.termQuery("age", 24)); + System.out.println("精确搜索查询语句:"+searchSourceBuilder.toString()); + System.out.println("精确搜索返回结果:"+search(jestClient,indexName, typeName, searchSourceBuilder.toString())); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + /** + * 区间搜索 + */ + public static void serach3() { + String createtm="createtm"; + String from="2016-8-21 06:11:32"; + String to="2018-8-21 06:11:32"; + + try { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.rangeQuery(createtm).gte(from).lte(to)); + System.out.println("区间搜索语句:"+searchSourceBuilder.toString()); + System.out.println("区间搜索返回结果:"+search(jestClient,indexName, typeName, searchSourceBuilder.toString())); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + /** + * 创建索引 + * @param indexName + * @return + * @throws Exception + */ + public boolean createIndex(JestClient jestClient,String indexName) throws Exception { + JestResult jr = jestClient.execute(new CreateIndex.Builder(indexName).build()); + return jr.isSucceeded(); + } + + /** + * 新增数据 + * @param indexName + * @param typeName + * @param source + * @return + * @throws Exception + */ + public boolean insert(JestClient jestClient,String indexName, String typeName, String source) throws Exception { + PutMapping putMapping = new PutMapping.Builder(indexName, typeName, source).build(); + JestResult jr = jestClient.execute(putMapping); + return jr.isSucceeded(); + } + + + /** + * 查询数据 + * @param indexName + * @param typeName + * @return + * @throws Exception + */ + public static String getIndexMapping(JestClient jestClient,String indexName, String typeName) throws Exception { + GetMapping getMapping = new GetMapping.Builder().addIndex(indexName).addType(typeName).build(); + JestResult jr =jestClient.execute(getMapping); + return jr.getJsonString(); + } + + + + /** + * 批量新增数据 + * @param indexName + * @param typeName + * @param objs + * @return + * @throws Exception + */ + public static boolean insertBatch(JestClient jestClient,String indexName, String typeName, List objs) throws Exception { + Bulk.Builder bulk = new Bulk.Builder().defaultIndex(indexName).defaultType(typeName); + for (Object obj : objs) { + Index index = new Index.Builder(obj).build(); + bulk.addAction(index); + } + BulkResult br = jestClient.execute(bulk.build()); + return br.isSucceeded(); + } + + /** + * 全文搜索 + * @param indexName + * @param typeName + * @param query + * @return + * @throws Exception + */ + public static String search(JestClient jestClient,String indexName, String typeName, String query) throws Exception { + Search search = new Search.Builder(query) + .addIndex(indexName) + .addType(typeName) + .build(); + JestResult jr = jestClient.execute(search); +// System.out.println("--"+jr.getJsonString()); +// System.out.println("--"+jr.getSourceAsObject(User.class)); + return jr.getSourceAsString(); + } + + + + + + /** + * 删除索引 + * @param indexName + * @return + * @throws Exception + */ + public boolean delete(JestClient jestClient,String indexName) throws Exception { + JestResult jr = jestClient.execute(new DeleteIndex.Builder(indexName).build()); + return jr.isSucceeded(); + } + + /** + * 删除数据 + * @param indexName + * @param typeName + * @param id + * @return + * @throws Exception + */ + public boolean delete(JestClient jestClient,String indexName, String typeName, String id) throws Exception { + DocumentResult dr = jestClient.execute(new Delete.Builder(id).index(indexName).type(typeName).build()); + return dr.isSucceeded(); + } + +} + + + class User implements Serializable{ + /** + * + */ + private static final long serialVersionUID = 1L; + /** 编号 */ + private Long id; + /** 姓名 */ + private String name; + + /** 年龄 */ + private Integer age; + + /** 描述 */ + private String description; + + /** 创建时间 */ + private String createtm; + + + public User(){ + } + + + + + public User(Long id, String name, Integer age, String description, String createtm) { + super(); + this.id = id; + this.name = name; + this.age = age; + this.description = description; + this.createtm = createtm; + } + + + + + /** + * 获取编号 + * @return id + */ + public Long getId() { + return id; + } + + + /** + * 设置编号 + * @param Long id + */ + public void setId(Long id) { + this.id = id; + } + + + /** + * 获取姓名 + * @return name + */ + public String getName() { + return name; + } + + + /** + * 设置姓名 + * @param String name + */ + public void setName(String name) { + this.name = name; + } + + + /** + * 获取年龄 + * @return age + */ + public Integer getAge() { + return age; + } + + + /** + * 设置年龄 + * @param Integer age + */ + public void setAge(Integer age) { + this.age = age; + } + + + /** + * 获取描述 + * @return description + */ + public String getDescription() { + return description; + } + + + /** + * 设置描述 + * @param String description + */ + public void setDescription(String description) { + this.description = description; + } + + + /** + * 获取创建时间 + * @return createtm + */ + public String getCreatetm() { + return createtm; + } + + + /** + * 设置创建时间 + * @param String createtm + */ + public void setCreatetm(String createtm) { + this.createtm = createtm; + } + + + + + @Override + public String toString() { + return "User [id=" + id + ", name=" + name + ", age=" + age + ", description=" + description + ", createtm=" + + createtm + "]"; + } + + + +} + diff --git a/src/main/java/com/pancm/sql/easticsearch/package-info.java b/src/main/java/com/pancm/easticsearch/package-info.java similarity index 74% rename from src/main/java/com/pancm/sql/easticsearch/package-info.java rename to src/main/java/com/pancm/easticsearch/package-info.java index ef3a9e9..9a4d2a5 100644 --- a/src/main/java/com/pancm/sql/easticsearch/package-info.java +++ b/src/main/java/com/pancm/easticsearch/package-info.java @@ -1,8 +1,8 @@ -/** -* @Title: package-info -* @Description: es 相关测试类 -* @Version:1.0.0 -* @author pancm -* @date 2019年2月28日 -*/ -package com.pancm.sql.easticsearch; \ No newline at end of file +/** +* @Title: package-info +* @Description: es 相关测试类 +* @Version:1.0.0 +* @author pancm +* @date 2019年2月28日 +*/ +package com.pancm.easticsearch; \ No newline at end of file diff --git a/src/main/java/com/pancm/sql/redis/RedisTest.java b/src/main/java/com/pancm/redis/RedisTest.java similarity index 94% rename from src/main/java/com/pancm/sql/redis/RedisTest.java rename to src/main/java/com/pancm/redis/RedisTest.java index 65530a6..10eed50 100644 --- a/src/main/java/com/pancm/sql/redis/RedisTest.java +++ b/src/main/java/com/pancm/redis/RedisTest.java @@ -1,60 +1,60 @@ -package com.pancm.sql.redis; - -import java.util.Iterator; -import java.util.List; -import java.util.Set; - -import redis.clients.jedis.Jedis; - -/** - * - * @Title: RedisTest - * @Description: redis测试代码 - * @Version:1.0.0 - * @author pancm - * @date 2017-8-19 - */ -public class RedisTest { - - /** - * @param args - */ - public static void main(String[] args) { - // 连接到本地的 redis服务 - Jedis jedis = new Jedis("127.0.0.1"); - System.out.println("连接成功"); - // 查看服务是否运行 - System.out.println("服务正在运行: " + jedis.ping()); - - // 存储到列表中 - jedis.lpush("list", "redis"); - jedis.lpush("list", "java"); - jedis.lpush("list", "mysql"); - // 获取存储的数据并输出 - List list = jedis.lrange("list", 0, 2); - for (int i = 0, j = list.size(); i < j; i++) { - System.out.println("list的输出结果:" + list.get(i)); - } - - // 设置 redis 字符串数据 - jedis.set("rst", "redisStringTest"); - // 获取存储的数据并输出 - System.out.println("redis 存储的字符串为: " + jedis.get("rst")); - - // 存储数据 - jedis.sadd("setTest1", "abc"); - jedis.sadd("setTest1", "abcd"); - jedis.sadd("setTest1", "abcde"); - // 获取数据并输出 - Set keys = jedis.keys("*"); - // Set keys=jedis.smembers("setTest1"); - // 定义迭代器输出 - Iterator it = keys.iterator(); - while (it.hasNext()) { - String key = it.next(); - System.out.println(key); - } - - } - -} +package com.pancm.redis; + +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import redis.clients.jedis.Jedis; + +/** + * + * @Title: RedisTest + * @Description: redis测试代码 + * @Version:1.0.0 + * @author pancm + * @date 2017-8-19 + */ +public class RedisTest { + + /** + * @param args + */ + public static void main(String[] args) { + // 连接到本地的 redis服务 + Jedis jedis = new Jedis("127.0.0.1"); + System.out.println("连接成功"); + // 查看服务是否运行 + System.out.println("服务正在运行: " + jedis.ping()); + + // 存储到列表中 + jedis.lpush("list", "redis"); + jedis.lpush("list", "java"); + jedis.lpush("list", "mysql"); + // 获取存储的数据并输出 + List list = jedis.lrange("list", 0, 2); + for (int i = 0, j = list.size(); i < j; i++) { + System.out.println("list的输出结果:" + list.get(i)); + } + + // 设置 redis 字符串数据 + jedis.set("rst", "redisStringTest"); + // 获取存储的数据并输出 + System.out.println("redis 存储的字符串为: " + jedis.get("rst")); + + // 存储数据 + jedis.sadd("setTest1", "abc"); + jedis.sadd("setTest1", "abcd"); + jedis.sadd("setTest1", "abcde"); + // 获取数据并输出 + Set keys = jedis.keys("*"); + // Set keys=jedis.smembers("setTest1"); + // 定义迭代器输出 + Iterator it = keys.iterator(); + while (it.hasNext()) { + String key = it.next(); + System.out.println(key); + } + + } + +} diff --git a/src/main/java/com/pancm/sql/redis/package-info.java b/src/main/java/com/pancm/redis/package-info.java similarity index 78% rename from src/main/java/com/pancm/sql/redis/package-info.java rename to src/main/java/com/pancm/redis/package-info.java index c72cd24..f36f17a 100644 --- a/src/main/java/com/pancm/sql/redis/package-info.java +++ b/src/main/java/com/pancm/redis/package-info.java @@ -1,8 +1,8 @@ -/** -* @Title: package-info -* @Description: redis 相关代码 -* @Version:1.0.0 -* @author pancm -* @date 2018年9月21日 -*/ -package com.pancm.sql.redis; \ No newline at end of file +/** +* @Title: package-info +* @Description: redis 相关代码 +* @Version:1.0.0 +* @author pancm +* @date 2018年9月21日 +*/ +package com.pancm.redis; \ No newline at end of file From c80ca2aabd7ea55e190a88f251af10003f60ae9c Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Tue, 2 Apr 2019 19:25:25 +0800 Subject: [PATCH 11/53] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0elasticsearch?= =?UTF-8?q?=E8=81=9A=E5=90=88=E4=BD=BF=E7=94=A8=E7=9A=84=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../easticsearch/EsAggregationSearchTest.java | 277 ++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 src/main/java/com/pancm/easticsearch/EsAggregationSearchTest.java diff --git a/src/main/java/com/pancm/easticsearch/EsAggregationSearchTest.java b/src/main/java/com/pancm/easticsearch/EsAggregationSearchTest.java new file mode 100644 index 0000000..f9d1905 --- /dev/null +++ b/src/main/java/com/pancm/easticsearch/EsAggregationSearchTest.java @@ -0,0 +1,277 @@ +package com.pancm.easticsearch; + +import org.apache.http.HttpHost; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.avg.Avg; +import org.elasticsearch.search.aggregations.metrics.max.Max; +import org.elasticsearch.search.aggregations.metrics.min.Min; +import org.elasticsearch.search.aggregations.metrics.sum.Sum; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +/** + * @author pancm + * @Title: pancm_project + * @Description:ES聚合查询测试用例 + * @Version:1.0.0 + * @Since:jdk1.8 + * @date 2019/4/2 + */ +public class EsAggregationSearchTest { + + + + private static String elasticIp = "192.169.0.23"; + private static int elasticPort = 9200; + private static Logger logger = LoggerFactory.getLogger(com.pancm.sql.easticsearch.EsHighLevelRestSearchTest.class); + + private static RestHighLevelClient client = null; + + /** + * @param args + */ + public static void main(String[] args) { + + try { + init(); + avgSearch(); + maxSearch(); + sumSearch(); + avgGroupSearch(); + maxGroupSearch(); + sumGroupSearch(); + + } catch (Exception e) { + e.printStackTrace(); + }finally { + close(); + } + + } + + /* + * 初始化服务 + */ + private static void init() { + RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(elasticIp, elasticPort)); + client = new RestHighLevelClient(restClientBuilder); + + } + + /* + * 关闭服务 + */ + private static void close() { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + e.printStackTrace(); + }finally{ + client=null; + } + } + } + + /** + * @Author pancm + * @Description 平均聚合查询测试用例 + * @Date 2019/4/1 + * @Param [] + * @return void + **/ + private static void avgSearch() throws IOException { + + String buk="t_grade_avg"; + //直接求平均数 + AggregationBuilder aggregation = AggregationBuilders.avg(buk).field("grade"); + logger.info("求班级的平均分数:"); + agg(aggregation,buk); + + } + + private static void maxSearch() throws IOException{ + String buk="t_grade"; + AggregationBuilder aggregation = AggregationBuilders.max(buk).field("grade"); + logger.info("求班级的最分数:"); + agg(aggregation,buk); + } + + private static void sumSearch() throws IOException{ + String buk="t_grade"; + AggregationBuilder aggregation = AggregationBuilders.sum(buk).field("grade"); + logger.info("求班级的总分数:"); + agg(aggregation,buk); + } + + /** + * @Author pancm + * @Description 平均聚合查询测试用例 + * @Date 2019/4/1 + * @Param [] + * @return void + **/ + private static void avgGroupSearch() throws IOException { + + + String agg="t_class_avg"; + String buk="t_grade"; + //terms 就是分组统计 根据student的grade成绩进行分组并创建一个新的聚合 + TermsAggregationBuilder aggregation = AggregationBuilders.terms(agg).field("class"); + aggregation.subAggregation(AggregationBuilders.avg(buk).field("grade")); + + logger.info("根据班级求平均分数:"); + agg(aggregation,agg,buk); + + } + + + private static void maxGroupSearch() throws IOException{ + + String agg="t_class_max"; + String buk="t_grade"; + //terms 就是分组统计 根据student的grade成绩进行分组并创建一个新的聚合 + TermsAggregationBuilder aggregation = AggregationBuilders.terms(agg).field("class"); + aggregation.subAggregation(AggregationBuilders.max(buk).field("grade")); + logger.info("根据班级求最大分数:"); + agg(aggregation,agg,buk); + } + + + private static void sumGroupSearch() throws IOException{ + String agg="t_class_sum"; + String buk="t_grade"; + //terms 就是分组统计 根据student的grade成绩进行分组并创建一个新的聚合 + TermsAggregationBuilder aggregation = AggregationBuilders.terms(agg).field("class"); + aggregation.subAggregation(AggregationBuilders.sum(buk).field("grade")); + + logger.info("根据班级求总分:"); + agg(aggregation,agg,buk); + } + + + protected static void agg(AggregationBuilder aggregation, String buk) throws IOException{ + SearchResponse searchResponse = search(aggregation); + if(RestStatus.OK.equals(searchResponse.status())) { + // 获取聚合结果 + Aggregations aggregations = searchResponse.getAggregations(); + + if(buk.contains("avg")){ + //取子聚合 + Avg ba = aggregations.get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + }else if(buk.contains("max")){ + //取子聚合 + Max ba = aggregations.get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + + }else if(buk.contains("min")){ + //取子聚合 + Min ba = aggregations.get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + }else if(buk.contains("sum")){ + //取子聚合 + Sum ba = aggregations.get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + } + + + } + } + + private static SearchResponse search(AggregationBuilder aggregation) throws IOException { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices("student"); + searchRequest.types("_doc"); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + //不需要解释 + searchSourceBuilder.explain(false); + //不需要原始数据 + searchSourceBuilder.fetchSource(false); + //不需要版本号 + searchSourceBuilder.version(false); + searchSourceBuilder.aggregation(aggregation); + logger.info("查询的语句:"+searchSourceBuilder.toString()); + searchRequest.source(searchSourceBuilder); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + return searchResponse; + } + + /** + * @Author pancm + * @Description 进行聚合 + * @Date 2019/4/2 + * @Param [] + * @return void + **/ + protected static void agg(AggregationBuilder aggregation, String agg, String buk) throws IOException{ + + // 同步查询 + SearchResponse searchResponse = search(aggregation); + + //4、处理响应 + //搜索结果状态信息 + if(RestStatus.OK.equals(searchResponse.status())) { + // 获取聚合结果 + Aggregations aggregations = searchResponse.getAggregations(); + + + //分组 + Terms byAgeAggregation = aggregations.get(agg); + logger.info(agg+" 结果"); + logger.info("name: " + byAgeAggregation.getName()); + logger.info("type: " + byAgeAggregation.getType()); + logger.info("sumOfOtherDocCounts: " + byAgeAggregation.getSumOfOtherDocCounts()); + + logger.info("------------------------------------"); + for(Terms.Bucket buck : byAgeAggregation.getBuckets()) { + logger.info("key: " + buck.getKeyAsNumber()); + logger.info("docCount: " + buck.getDocCount()); + logger.info("docCountError: " + buck.getDocCountError()); + + + if(agg.contains("avg")){ + //取子聚合 + Avg ba = buck.getAggregations().get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + }else if(agg.contains("max")){ + //取子聚合 + Max ba = buck.getAggregations().get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + + }else if(agg.contains("min")){ + //取子聚合 + Min ba = buck.getAggregations().get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + }else if(agg.contains("sum")){ + //取子聚合 + Sum ba = buck.getAggregations().get(buk); + logger.info(buk+":" + ba.getValue()); + logger.info("------------------------------------"); + } + } + } + } +} From 75c95792a82353f2470c7700d2c234154c968194 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Thu, 4 Apr 2019 17:41:23 +0800 Subject: [PATCH 12/53] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0elasticsearch?= =?UTF-8?q?=E8=81=9A=E5=90=88=E4=BD=BF=E7=94=A8=E7=9A=84=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../easticsearch/EsAggregationSearchTest.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/pancm/easticsearch/EsAggregationSearchTest.java b/src/main/java/com/pancm/easticsearch/EsAggregationSearchTest.java index f9d1905..df86c74 100644 --- a/src/main/java/com/pancm/easticsearch/EsAggregationSearchTest.java +++ b/src/main/java/com/pancm/easticsearch/EsAggregationSearchTest.java @@ -17,6 +17,7 @@ import org.elasticsearch.search.aggregations.metrics.max.Max; import org.elasticsearch.search.aggregations.metrics.min.Min; import org.elasticsearch.search.aggregations.metrics.sum.Sum; +import org.elasticsearch.search.aggregations.metrics.tophits.TopHits; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,7 +55,7 @@ public static void main(String[] args) { avgGroupSearch(); maxGroupSearch(); sumGroupSearch(); - + topSearch(); } catch (Exception e) { e.printStackTrace(); }finally { @@ -191,6 +192,11 @@ protected static void agg(AggregationBuilder aggregation, String buk) throws Sum ba = aggregations.get(buk); logger.info(buk+":" + ba.getValue()); logger.info("------------------------------------"); + }else{ + //取子聚合 + TopHits ba = aggregations.get(buk); + logger.info(buk+":" + ba.getHits().totalHits); + logger.info("------------------------------------"); } @@ -274,4 +280,11 @@ protected static void agg(AggregationBuilder aggregation, String agg, String b } } } + + private static void topSearch() throws IOException{ + + + } + + } From 183982df80c274c2e5e84442834b50318cf07360 Mon Sep 17 00:00:00 2001 From: dark369 <784675658@qq.com> Date: Sun, 7 Apr 2019 20:28:32 +0800 Subject: [PATCH 13/53] =?UTF-8?q?=E5=A2=9E=E5=8A=A0es=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/pancm/easticsearch/EsUtil.java | 482 ++++++++++++++++++ 1 file changed, 482 insertions(+) create mode 100644 src/main/java/com/pancm/easticsearch/EsUtil.java diff --git a/src/main/java/com/pancm/easticsearch/EsUtil.java b/src/main/java/com/pancm/easticsearch/EsUtil.java new file mode 100644 index 0000000..c28f20a --- /dev/null +++ b/src/main/java/com/pancm/easticsearch/EsUtil.java @@ -0,0 +1,482 @@ +package com.pancm.test.esTest; + + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.http.HttpHost; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * @Title: EsUtil + * @Description: ES工具类 + * @Version:1.0.0 + * @author pancm + * @date 2019年3月19日 + */ +public final class EsUtil { + + private EsUtil() { + + } + + + + + /** + * 创建链接 + * + * @param nodes + * @return + */ + public static boolean build( String... nodes) throws IOException { + boolean falg = false; + Objects.requireNonNull(nodes, "hosts can not null"); + ArrayList ahosts = new ArrayList(); + for (String host : nodes) { + IpHandler addr = new IpHandler(); + addr.IpPortFromUrl(host); + ahosts.add(new HttpHost(addr.getIp(),addr.getPort())); + } + httpHosts = ahosts.toArray(new HttpHost[0]); + try { + init(); + falg = true; + } catch (IOException e) { + throw e; + } + + return falg; + + } + + + /** + * @Author pancm + * @Description //创建索引库(指定Mpping类型) + * @Date 2019/3/21 + * @Param [esBasicModelConfig] + * @return boolean + **/ + + public static boolean creatIndex(EsBasicModelConfig esBasicModelConfig) throws IOException { + boolean falg = true; + Objects.requireNonNull(esBasicModelConfig, "esBasicModelConfig is not null"); + String type = Objects.requireNonNull(esBasicModelConfig.getType(), "type is not null"); + String index = Objects.requireNonNull(esBasicModelConfig.getIndex(), "index is not null"); + if (exitsIndex(index)) { + logger.warn("索引库{}已经存在!无需在进行创建!", index); + return true; + } + String mapping = esBasicModelConfig.getMappings(); + Map setting = esBasicModelConfig.getSettings(); + String alias = esBasicModelConfig.getAlias(); + // 开始创建库 + CreateIndexRequest request = new CreateIndexRequest(index); + try { + if (Objects.nonNull(mapping)) { + // 加载数据类型 + request.mapping(type, mapping); + } + if (Objects.nonNull(setting)) { + // 分片数 + request.settings(setting); + } + if (Objects.nonNull(alias)) { + // 别名 + request.alias(new Alias(alias)); + } + + CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); + falg = createIndexResponse.isAcknowledged(); + } catch (IOException e) { + throw e; + } + return falg; + + } + + /** + * 判断索引库是否存在 + * + * @param index + * @return + * @throws IOException + */ + public static boolean exitsIndex(String index) throws IOException { + GetIndexRequest getRequest2 = new GetIndexRequest(); + getRequest2.indices(index); + getRequest2.local(false); + getRequest2.humanReadable(true); + boolean exists2 = client.indices().exists(getRequest2, RequestOptions.DEFAULT); + return exists2; + } + + + /** + * @Author pancm + * @Description //新增/更新数据 + * @Date 2019/3/21 + * @Param [] + * @return boolean + **/ + public static boolean insert() throws IOException{ + + return false; + } + + + /** + * @Author pancm + * @Description //删除数据 + * @Date 2019/3/21 + * @Param [] + * @return boolean + **/ + public static boolean delete() throws IOException{ + + return false; + } + + + + + + /* + * 初始化服务 + */ + private static void init() throws IOException { + if (client == null) { + RestClientBuilder restClientBuilder = RestClient.builder(httpHosts); + client = new RestHighLevelClient(restClientBuilder); + } + } + + /* + * 关闭服务 + */ + private static void close() throws IOException { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + throw e; + } + } + } + + private static String[] elasticIps; + private static int elasticPort; + private static HttpHost[] httpHosts; + private static RestHighLevelClient client = null; + + private static final String COMMA_SIGN = ","; + + + + private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestSearchTest.class); + + /** + * @param args + * @throws IOException + */ + public static void main(String[] args) { + + try { + + + EsUtil.build("192.169.0.23:9200"); + + System.out.println("ES连接初始化成功!"); + + // setting 的值 + Map setmapping = new HashMap<>(); + + // 分区数、路由分片数、副本数、缓存刷新时间 + setmapping.put("number_of_shards", 12); + setmapping.put("number_of_routing_shards", 24); + setmapping.put("number_of_replicas", 1); + setmapping.put("refresh_interval", "5s"); + + String index = "test5"; + String type = "test5"; + String alias = "test"; + + Map jsonMap2 = new HashMap<>(); + Map message = new HashMap<>(); + // 设置类型 + message.put("type", "text"); + Map properties = new HashMap<>(); + // 设置字段message信息 + properties.put("message", message); + Map mapping = new HashMap<>(); + mapping.put("properties", properties); + jsonMap2.put(type, mapping); + + String mappings = jsonMap2.toString(); + + EsBasicModelConfig esBasicModelConfig = new EsBasicModelConfig(); + esBasicModelConfig.setIndex(index); + esBasicModelConfig.setType(type); + esBasicModelConfig.setMappings(mappings); + esBasicModelConfig.setSettings(setmapping); + esBasicModelConfig.setAlias(alias); + + EsUtil.creatIndex(esBasicModelConfig); + }catch(IOException e) { + e.printStackTrace(); + + } finally { + // TODO: handle finally clause + try { + close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + +} + +/* + * ES的mapping创建的基础类 + */ +class EsBasicModelConfig implements Serializable { + private static final long serialVersionUID = 1L; + /*** 索引库 ***/ + private String index; + private String type; + private Map settings; + private String mappings; + private String alias; + + public EsBasicModelConfig() { + } + + public EsBasicModelConfig(String index, String type) { + this.index = index; + this.type = type; + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Map getSettings() { + return settings; + } + + public void setSettings(Map settings) { + this.settings = settings; + } + + public void setSettings(SettingEntity settings) { + this.settings = Objects.requireNonNull(settings, "setting can not null").toDSL(); + } + + public String getMappings() { + return mappings; + } + + public void setMappings(String mappings) { + this.mappings = mappings; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + @Override + public String toString() { + return "EsBasicModelConfig [index=" + index + ", type=" + type + ", settings=" + settings + ", mappings=" + + mappings + "]"; + } + +} + +/* + * setting 实体类的配置 + */ +class SettingEntity implements Serializable { + /** + * @Fields serialVersionUID : TODO + */ + private static final long serialVersionUID = 1L; + // 默认分片数 + private int numberOfShards = 5; + // 分片路由数 + private int number_of_routing_shards = 30; + // 副本数 + private int numberOfReplicas = 1; + /***** 刷新频率 单位:秒 *********/ + private int refreshInterval = 5; + /** 查询最大返回的时间 */ + private int maxResultWindow = 10000; + + public SettingEntity(int numberOfShards, int numberOfReplicas, int refreshInterval) { + this.numberOfShards = numberOfShards; + this.numberOfReplicas = numberOfReplicas; + this.refreshInterval = refreshInterval; + } + + public SettingEntity(int numberOfShards, int numberOfReplicas, int refreshInterval, int number_of_routing_shards, + int maxResultWindow, String alias) { + this.numberOfShards = numberOfShards; + this.numberOfReplicas = numberOfReplicas; + this.refreshInterval = refreshInterval; + this.number_of_routing_shards = number_of_routing_shards; + this.maxResultWindow = maxResultWindow; + } + + public SettingEntity() { + + } + + public int getNumberOfShards() { + return numberOfShards; + } + + /** + * 分片数 + * + * @param numberOfShards 默认5 + */ + public void setNumberOfShards(int numberOfShards) { + this.numberOfShards = numberOfShards; + } + + public int getNumberOfReplicas() { + return numberOfReplicas; + } + + /** + * 副本数 + * + * @param numberOfReplicas 默认1 + */ + public void setNumberOfReplicas(int numberOfReplicas) { + this.numberOfReplicas = numberOfReplicas; + } + + public int getRefreshInterval() { + return refreshInterval; + } + + public int getNumber_of_routing_shards() { + return number_of_routing_shards; + } + + public void setNumber_of_routing_shards(int number_of_routing_shards) { + this.number_of_routing_shards = number_of_routing_shards; + } + + public int getMaxResultWindow() { + return maxResultWindow; + } + + public void setMaxResultWindow(int maxResultWindow) { + this.maxResultWindow = maxResultWindow; + } + + /** + * 刷新频率 单位:秒 + * + * @param refreshInterval 默认5秒 设置为-1为无限刷新 + */ + public void setRefreshInterval(int refreshInterval) { + if (refreshInterval < -1) { + refreshInterval = -1; + } + this.refreshInterval = refreshInterval; + } + + public Map toDSL() { + Map json = new HashMap<>(); + json.put("number_of_shards", numberOfShards); + json.put("number_of_routing_shards", number_of_routing_shards); + json.put("number_of_replicas", numberOfReplicas); + json.put("refresh_interval", refreshInterval + "s"); + json.put("max_result_window", maxResultWindow); + return json; + } + + @Override + public String toString() { + return "SettingEntity [numberOfShards=" + numberOfShards + ", numberOfReplicas=" + numberOfReplicas + + ", refreshInterval=" + refreshInterval + ", maxResultWindow=" + maxResultWindow + "]"; + } + +} + +class IpHandler { + + private String ip; + private Integer port; + private static Pattern p = Pattern.compile("(?<=//|)((\\w)+\\.)+\\w+(:\\d{0,5})?"); + + /** 冒号 */ + private static final String COMMA_COLON = ":"; + /** + * 从url中分析出hostIP:PORT
+ * @param url + * */ + public void IpPortFromUrl(String url) { + + String host = ""; + + Matcher matcher = p.matcher(url); + if (matcher.find()) { + host = matcher.group() ; + } + // 如果 + if(host.contains(COMMA_COLON) == false){ + this.ip=host; + this.port=80; + }else{ + String[] ipPortArr = host.split(COMMA_COLON); + this.ip=ipPortArr[0]; + this.port=Integer.valueOf(ipPortArr[1].trim()); + } + } + + public String getIp() { + return ip; + } + + public Integer getPort() { + return port; + } +} From 8326bfde3f7d50a0b797034bdb8fa4835914dc29 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Mon, 8 Apr 2019 11:58:13 +0800 Subject: [PATCH 14/53] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0es=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/pancm/easticsearch/EsUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/pancm/easticsearch/EsUtil.java b/src/main/java/com/pancm/easticsearch/EsUtil.java index c28f20a..31d01f7 100644 --- a/src/main/java/com/pancm/easticsearch/EsUtil.java +++ b/src/main/java/com/pancm/easticsearch/EsUtil.java @@ -1,4 +1,4 @@ -package com.pancm.test.esTest; +package com.pancm.easticsearch; import java.io.IOException; @@ -190,7 +190,7 @@ private static void close() throws IOException { - private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestSearchTest.class); + private static Logger logger = LoggerFactory.getLogger(EsUtil.class); /** * @param args From 68626ffa8bc6c8d693f5be1ba743ce25e4a411a5 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Mon, 8 Apr 2019 13:58:46 +0800 Subject: [PATCH 15/53] =?UTF-8?q?1.=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/pancm/easticsearch/EsUtil.java | 482 ------------------ 1 file changed, 482 deletions(-) delete mode 100644 src/main/java/com/pancm/easticsearch/EsUtil.java diff --git a/src/main/java/com/pancm/easticsearch/EsUtil.java b/src/main/java/com/pancm/easticsearch/EsUtil.java deleted file mode 100644 index 31d01f7..0000000 --- a/src/main/java/com/pancm/easticsearch/EsUtil.java +++ /dev/null @@ -1,482 +0,0 @@ -package com.pancm.easticsearch; - - -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.http.HttpHost; -import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; -import org.elasticsearch.client.RestHighLevelClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -/** - * @Title: EsUtil - * @Description: ES工具类 - * @Version:1.0.0 - * @author pancm - * @date 2019年3月19日 - */ -public final class EsUtil { - - private EsUtil() { - - } - - - - - /** - * 创建链接 - * - * @param nodes - * @return - */ - public static boolean build( String... nodes) throws IOException { - boolean falg = false; - Objects.requireNonNull(nodes, "hosts can not null"); - ArrayList ahosts = new ArrayList(); - for (String host : nodes) { - IpHandler addr = new IpHandler(); - addr.IpPortFromUrl(host); - ahosts.add(new HttpHost(addr.getIp(),addr.getPort())); - } - httpHosts = ahosts.toArray(new HttpHost[0]); - try { - init(); - falg = true; - } catch (IOException e) { - throw e; - } - - return falg; - - } - - - /** - * @Author pancm - * @Description //创建索引库(指定Mpping类型) - * @Date 2019/3/21 - * @Param [esBasicModelConfig] - * @return boolean - **/ - - public static boolean creatIndex(EsBasicModelConfig esBasicModelConfig) throws IOException { - boolean falg = true; - Objects.requireNonNull(esBasicModelConfig, "esBasicModelConfig is not null"); - String type = Objects.requireNonNull(esBasicModelConfig.getType(), "type is not null"); - String index = Objects.requireNonNull(esBasicModelConfig.getIndex(), "index is not null"); - if (exitsIndex(index)) { - logger.warn("索引库{}已经存在!无需在进行创建!", index); - return true; - } - String mapping = esBasicModelConfig.getMappings(); - Map setting = esBasicModelConfig.getSettings(); - String alias = esBasicModelConfig.getAlias(); - // 开始创建库 - CreateIndexRequest request = new CreateIndexRequest(index); - try { - if (Objects.nonNull(mapping)) { - // 加载数据类型 - request.mapping(type, mapping); - } - if (Objects.nonNull(setting)) { - // 分片数 - request.settings(setting); - } - if (Objects.nonNull(alias)) { - // 别名 - request.alias(new Alias(alias)); - } - - CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); - falg = createIndexResponse.isAcknowledged(); - } catch (IOException e) { - throw e; - } - return falg; - - } - - /** - * 判断索引库是否存在 - * - * @param index - * @return - * @throws IOException - */ - public static boolean exitsIndex(String index) throws IOException { - GetIndexRequest getRequest2 = new GetIndexRequest(); - getRequest2.indices(index); - getRequest2.local(false); - getRequest2.humanReadable(true); - boolean exists2 = client.indices().exists(getRequest2, RequestOptions.DEFAULT); - return exists2; - } - - - /** - * @Author pancm - * @Description //新增/更新数据 - * @Date 2019/3/21 - * @Param [] - * @return boolean - **/ - public static boolean insert() throws IOException{ - - return false; - } - - - /** - * @Author pancm - * @Description //删除数据 - * @Date 2019/3/21 - * @Param [] - * @return boolean - **/ - public static boolean delete() throws IOException{ - - return false; - } - - - - - - /* - * 初始化服务 - */ - private static void init() throws IOException { - if (client == null) { - RestClientBuilder restClientBuilder = RestClient.builder(httpHosts); - client = new RestHighLevelClient(restClientBuilder); - } - } - - /* - * 关闭服务 - */ - private static void close() throws IOException { - if (client != null) { - try { - client.close(); - } catch (IOException e) { - throw e; - } - } - } - - private static String[] elasticIps; - private static int elasticPort; - private static HttpHost[] httpHosts; - private static RestHighLevelClient client = null; - - private static final String COMMA_SIGN = ","; - - - - private static Logger logger = LoggerFactory.getLogger(EsUtil.class); - - /** - * @param args - * @throws IOException - */ - public static void main(String[] args) { - - try { - - - EsUtil.build("192.169.0.23:9200"); - - System.out.println("ES连接初始化成功!"); - - // setting 的值 - Map setmapping = new HashMap<>(); - - // 分区数、路由分片数、副本数、缓存刷新时间 - setmapping.put("number_of_shards", 12); - setmapping.put("number_of_routing_shards", 24); - setmapping.put("number_of_replicas", 1); - setmapping.put("refresh_interval", "5s"); - - String index = "test5"; - String type = "test5"; - String alias = "test"; - - Map jsonMap2 = new HashMap<>(); - Map message = new HashMap<>(); - // 设置类型 - message.put("type", "text"); - Map properties = new HashMap<>(); - // 设置字段message信息 - properties.put("message", message); - Map mapping = new HashMap<>(); - mapping.put("properties", properties); - jsonMap2.put(type, mapping); - - String mappings = jsonMap2.toString(); - - EsBasicModelConfig esBasicModelConfig = new EsBasicModelConfig(); - esBasicModelConfig.setIndex(index); - esBasicModelConfig.setType(type); - esBasicModelConfig.setMappings(mappings); - esBasicModelConfig.setSettings(setmapping); - esBasicModelConfig.setAlias(alias); - - EsUtil.creatIndex(esBasicModelConfig); - }catch(IOException e) { - e.printStackTrace(); - - } finally { - // TODO: handle finally clause - try { - close(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - } - -} - -/* - * ES的mapping创建的基础类 - */ -class EsBasicModelConfig implements Serializable { - private static final long serialVersionUID = 1L; - /*** 索引库 ***/ - private String index; - private String type; - private Map settings; - private String mappings; - private String alias; - - public EsBasicModelConfig() { - } - - public EsBasicModelConfig(String index, String type) { - this.index = index; - this.type = type; - } - - public String getIndex() { - return index; - } - - public void setIndex(String index) { - this.index = index; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public Map getSettings() { - return settings; - } - - public void setSettings(Map settings) { - this.settings = settings; - } - - public void setSettings(SettingEntity settings) { - this.settings = Objects.requireNonNull(settings, "setting can not null").toDSL(); - } - - public String getMappings() { - return mappings; - } - - public void setMappings(String mappings) { - this.mappings = mappings; - } - - public String getAlias() { - return alias; - } - - public void setAlias(String alias) { - this.alias = alias; - } - - @Override - public String toString() { - return "EsBasicModelConfig [index=" + index + ", type=" + type + ", settings=" + settings + ", mappings=" - + mappings + "]"; - } - -} - -/* - * setting 实体类的配置 - */ -class SettingEntity implements Serializable { - /** - * @Fields serialVersionUID : TODO - */ - private static final long serialVersionUID = 1L; - // 默认分片数 - private int numberOfShards = 5; - // 分片路由数 - private int number_of_routing_shards = 30; - // 副本数 - private int numberOfReplicas = 1; - /***** 刷新频率 单位:秒 *********/ - private int refreshInterval = 5; - /** 查询最大返回的时间 */ - private int maxResultWindow = 10000; - - public SettingEntity(int numberOfShards, int numberOfReplicas, int refreshInterval) { - this.numberOfShards = numberOfShards; - this.numberOfReplicas = numberOfReplicas; - this.refreshInterval = refreshInterval; - } - - public SettingEntity(int numberOfShards, int numberOfReplicas, int refreshInterval, int number_of_routing_shards, - int maxResultWindow, String alias) { - this.numberOfShards = numberOfShards; - this.numberOfReplicas = numberOfReplicas; - this.refreshInterval = refreshInterval; - this.number_of_routing_shards = number_of_routing_shards; - this.maxResultWindow = maxResultWindow; - } - - public SettingEntity() { - - } - - public int getNumberOfShards() { - return numberOfShards; - } - - /** - * 分片数 - * - * @param numberOfShards 默认5 - */ - public void setNumberOfShards(int numberOfShards) { - this.numberOfShards = numberOfShards; - } - - public int getNumberOfReplicas() { - return numberOfReplicas; - } - - /** - * 副本数 - * - * @param numberOfReplicas 默认1 - */ - public void setNumberOfReplicas(int numberOfReplicas) { - this.numberOfReplicas = numberOfReplicas; - } - - public int getRefreshInterval() { - return refreshInterval; - } - - public int getNumber_of_routing_shards() { - return number_of_routing_shards; - } - - public void setNumber_of_routing_shards(int number_of_routing_shards) { - this.number_of_routing_shards = number_of_routing_shards; - } - - public int getMaxResultWindow() { - return maxResultWindow; - } - - public void setMaxResultWindow(int maxResultWindow) { - this.maxResultWindow = maxResultWindow; - } - - /** - * 刷新频率 单位:秒 - * - * @param refreshInterval 默认5秒 设置为-1为无限刷新 - */ - public void setRefreshInterval(int refreshInterval) { - if (refreshInterval < -1) { - refreshInterval = -1; - } - this.refreshInterval = refreshInterval; - } - - public Map toDSL() { - Map json = new HashMap<>(); - json.put("number_of_shards", numberOfShards); - json.put("number_of_routing_shards", number_of_routing_shards); - json.put("number_of_replicas", numberOfReplicas); - json.put("refresh_interval", refreshInterval + "s"); - json.put("max_result_window", maxResultWindow); - return json; - } - - @Override - public String toString() { - return "SettingEntity [numberOfShards=" + numberOfShards + ", numberOfReplicas=" + numberOfReplicas - + ", refreshInterval=" + refreshInterval + ", maxResultWindow=" + maxResultWindow + "]"; - } - -} - -class IpHandler { - - private String ip; - private Integer port; - private static Pattern p = Pattern.compile("(?<=//|)((\\w)+\\.)+\\w+(:\\d{0,5})?"); - - /** 冒号 */ - private static final String COMMA_COLON = ":"; - /** - * 从url中分析出hostIP:PORT
- * @param url - * */ - public void IpPortFromUrl(String url) { - - String host = ""; - - Matcher matcher = p.matcher(url); - if (matcher.find()) { - host = matcher.group() ; - } - // 如果 - if(host.contains(COMMA_COLON) == false){ - this.ip=host; - this.port=80; - }else{ - String[] ipPortArr = host.split(COMMA_COLON); - this.ip=ipPortArr[0]; - this.port=Integer.valueOf(ipPortArr[1].trim()); - } - } - - public String getIp() { - return ip; - } - - public Integer getPort() { - return port; - } -} From 4e3b7d0be3164dc2dbab31928b1aba0eb0cf683e Mon Sep 17 00:00:00 2001 From: Zhoust Date: Fri, 26 Apr 2019 16:03:25 +0800 Subject: [PATCH 16/53] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20gitignore=20?= =?UTF-8?q?=E6=96=87=E4=BB=B6=EF=BC=8C=E4=BF=AE=E6=94=B9=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 +++- pom.xml | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 3cd2d05..a99dd7a 100644 --- a/.gitignore +++ b/.gitignore @@ -27,11 +27,13 @@ *.tar *.zip *.via +*.iml *.tmp *.err *.log # OS generated files # -.DS_Store +/.idea +.DS_Store .DS_Store? ._* .Spotlight-V100 diff --git a/pom.xml b/pom.xml index c0e565f..6542c35 100644 --- a/pom.xml +++ b/pom.xml @@ -4,11 +4,11 @@ 4.0.0 1.0.0 - java_study + java-study 0.0.1-SNAPSHOT jar - java_study + java-study http://maven.apache.org From 881a7fe121c510ee1fc1af39cebedea477b13b8b Mon Sep 17 00:00:00 2001 From: Zhoust Date: Wed, 1 May 2019 12:10:29 +0800 Subject: [PATCH 17/53] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=89=91=E6=8C=87=20Of?= =?UTF-8?q?fer=20=E9=A2=98=E7=9B=AE[=E6=95=B0=E7=BB=84=E4=B8=AD=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E6=95=B0=E5=AD=97]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jzoffer/DuplicateNumberInArray.java | 147 ++++++++++++++++++ .../arithmetic/jzoffer/package-info.java | 7 + 2 files changed, 154 insertions(+) create mode 100644 src/main/java/com/pancm/arithmetic/jzoffer/DuplicateNumberInArray.java create mode 100644 src/main/java/com/pancm/arithmetic/jzoffer/package-info.java diff --git a/src/main/java/com/pancm/arithmetic/jzoffer/DuplicateNumberInArray.java b/src/main/java/com/pancm/arithmetic/jzoffer/DuplicateNumberInArray.java new file mode 100644 index 0000000..8f7734b --- /dev/null +++ b/src/main/java/com/pancm/arithmetic/jzoffer/DuplicateNumberInArray.java @@ -0,0 +1,147 @@ +package com.pancm.arithmetic.jzoffer; + +/** + * https://www.nowcoder.com/practice/623a5ac0ea5b4e5f95552655361ae0a8?tpId=13&tqId=11203&tPage=1&rp=1&ru=/ta/coding-interviews&qru=/ta/coding-interviews/question-ranking + * + * @description: + * 3.数组中的重复数字 + * 在一个长度为 n 的数组里的所有数字都在 0 到 n-1 的范围内。 + * 数组中某些数字是重复的，但不知道有几个数字是重复的，也不知道每个数字重复几次。 + * 请找出数组中任意一个重复的数字。 + * + * @author: Zhoust + * @date: 2019/04/28 10:58 + * @version: V1.0 + */ +public class DuplicateNumberInArray { + + /** + * 一种简单的实现思路，借助外部数组保存 numbers 中每个值的出现次数，即 numbers 中的值作为数组 array 的下标。 + * 题目中说了长度为 n 的数组中每个值都在 0 到 n-1 范围内，因此遍历数组 numbers 存入对应的 array 不会出现数组下标越界的问题 + * @param numbers + * @param length numbers 数组的长度 + * @param duplication (Output) the duplicated number in the array number,length of duplication array is 1,so using duplication[0] = ? in implementation; + * 这里要特别注意~返回任意重复的一个，赋值duplication[0] + * @return true if the input is valid, and there are some duplications in the array number, otherwise false + */ + public boolean duplicate(int[] numbers, int length, int[] duplication) { + if (numbers == null) { + return false; + } + boolean result = false; + int[] array = new int[length]; + for (int i = 0; i < length; i++) { + if (++array[numbers[i]] > 1 && !result) { + result = true; + duplication[0] = numbers[i]; + } + } + return result; + } + + /** + * 先排序，然后找到排序后数组中第一个重复的数字（主要就是练习一下快排） + * @param numbers + * @param length + * @param duplication + * @return + */ + public boolean sortThenFindFirstDuplicateNumber(int[] numbers, int length, int[] duplication) { + if (numbers == null) { + return false; + } + quickSort(numbers, 0, length-1); + int temp = numbers[0]; + boolean result = false; + + for (int i = 1; i < length; i++) { + if (numbers[i] == temp) { + result = true; + duplication[0] = temp; + break; + } + temp = numbers[i]; + } + return result; + } + + /** + * 将值为 i 的元素调整到下标为 i 的位置，如果该位置上的数已经等于 i 了，就说明 i 已经重复了 + * @param numbers + * @param length + * @param duplication + * @return + */ + public static boolean jz(int[] numbers, int length, int[] duplication) { + for (int i = 0; i < length; i++) { + //第 i 个位置上的数 numbers[i] 若不与下标 i 相等，就把 numbers[i] 放到下标为 numbers[i] 的位置 + while (numbers[i] != i) { + //在交换位置之前还要检查 numbers[i] 位置上的数字是否已经是 numbers[i]，如果是就已经找到了重复的数字 + if (numbers[numbers[i]] == numbers[i]) { + duplication[0] = numbers[i]; + //注意这里不是 break + return true; + } + + //交换下标为 i、numbers[i] 两个位置上的数字（保证下标为 numbers[i] 的位置，放置的数字是 numbers[i]） + swap(numbers, i, numbers[i]); + } + } + return false; + } + + /** + * 交换下标为 i、j 的两个数 + * @param numbers + * @param i + * @param j + */ + public static void swap(int[] numbers, int i, int j) { + int swap = numbers[i]; + numbers[i] = numbers[j]; + numbers[j] = swap; + } + + /** + * 快排 + * @param array + * @param length + */ + @SuppressWarnings("all") + public static void quickSort(int[] array, int low, int high) { + //这里必须首先限制 low temp && h > l) { + h--; + } + array[l] = array[h]; + while (array[l] <= temp && l < h) { + l++; + } + array[h] = array[l]; + } + array[l] = temp; + quickSort(array, low, h-1); + quickSort(array, l+1, high); + } + } + + public static void main(String[] args) { + int[] result = new int[1]; + jz(new int[]{2,4,2,1,4}, 5, result); + System.out.println(result[0]); +// int[] d = new int[1]; +// jz(new int[]{2,1,3,0,4}, 5, d); +// int[] array = new int[]{1,4,4,4,12,78,8,45,67,39,20,90,65,34,34,6,7,88,9,1,-3,-88,-2}; + int[] array = new int[]{2,4,3,1,4}; + quickSort(array, 0, array.length-1); + for (int i = 0; i < array.length; i++) { + System.out.println(array[i]); + } + + } + +} \ No newline at end of file diff --git a/src/main/java/com/pancm/arithmetic/jzoffer/package-info.java b/src/main/java/com/pancm/arithmetic/jzoffer/package-info.java new file mode 100644 index 0000000..5a7d751 --- /dev/null +++ b/src/main/java/com/pancm/arithmetic/jzoffer/package-info.java @@ -0,0 +1,7 @@ +/** + * @description: 剑指 Offer 中的题目 + * @author: Zhoust + * @date: 2019/04/28 10:57 + * @version: V1.0 + */ +package com.pancm.arithmetic.jzoffer; \ No newline at end of file From b0bfa61226baa6912ab050c849baabb9a32b3245 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Thu, 9 May 2019 19:45:08 +0800 Subject: [PATCH 18/53] Create CONTRIBUTING.md --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1 @@ + From f5c7eff9839a6e9fe4c15c2e56a40ade9bd853e2 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Sun, 2 Jun 2019 18:40:49 +0800 Subject: [PATCH 19/53] =?UTF-8?q?pom=E6=96=87=E4=BB=B6=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E8=B0=B7=E6=AD=8C=E5=B7=A5=E5=85=B7=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pom.xml b/pom.xml index 6542c35..bde2feb 100644 --- a/pom.xml +++ b/pom.xml @@ -182,6 +182,13 @@ 2.3.0 + + + com.google.guava + guava + 27.1-jre + + From bbe7b92cdb3ea934b248eeddb4b09e57827dd9a2 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Mon, 24 Jun 2019 15:15:51 +0800 Subject: [PATCH 20/53] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0es=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/pancm/easticsearch/EsUtil.java | 1043 +++++++++++++++++ 1 file changed, 1043 insertions(+) create mode 100644 src/main/java/com/pancm/easticsearch/EsUtil.java diff --git a/src/main/java/com/pancm/easticsearch/EsUtil.java b/src/main/java/com/pancm/easticsearch/EsUtil.java new file mode 100644 index 0000000..5e84527 --- /dev/null +++ b/src/main/java/com/pancm/easticsearch/EsUtil.java @@ -0,0 +1,1043 @@ +package com.pancm.easticsearch; + + +import org.apache.http.HttpHost; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.index.reindex.ReindexRequest; +import org.elasticsearch.index.reindex.UpdateByQueryRequest; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.SortOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.Serializable; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/** + * @author pancm + * @Title: EsUtil + * @Description: ES的工具类 + * @Version:1.0.0 + * @date 2019年3月19日 + */ +public final class EsUtil { + private static Logger logger = LoggerFactory.getLogger(com.pancm.sql.easticsearch.EsHighLevelRestSearchTest.class); + + private EsUtil() { + + } + + + /** + * @param args + * @throws IOException + */ + public static void main(String[] args) { + + try { + + EsUtil.build("192.169.0.23:9200"); + System.out.println("ES连接初始化成功!"); +// createIndexTest(); +// System.out.println("ES索引库创建成功！"); + String index = "student"; + String type = "_doc"; + List> list = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + Map map = new HashMap<>(); + map.put("id", i); + map.put("name", "张三" + i); + map.put("age", 10 + i); + list.add(map); + } + EsUtil.setIsAutoClose(false); + saveBulk(list, index, type, "id"); + System.out.println("批量写入成功!"); + System.out.println("查询的结果1:" + queryById(index, type, "1")); + QueryBuilder queryBuilder = new TermQueryBuilder("name", "xuwujing"); + System.out.println("更新的结果:" + updateByQuery(index, type, queryBuilder)); + System.out.println("查询的结果2:" + queryById(index, type, "1")); + QueryBuilder queryBuilder3 = QueryBuilders.matchAllQuery(); + System.out.println("查询的结果3:" + query(index, type, queryBuilder3)); + QueryBuilder queryBuilder4 = QueryBuilders.rangeQuery("age").from(15); + QueryBuilder queryBuilder5 = QueryBuilders.rangeQuery("id").from(5); + System.out.println("查询的结果4:" + query(index, type, queryBuilder4,queryBuilder5)); + EsQueryCondition esQueryCondition = new EsQueryCondition(); + esQueryCondition.setCloseSource(true); + esQueryCondition.setIndex(1); + esQueryCondition.setPagesize(3); + esQueryCondition.setOrder("desc"); + esQueryCondition.setOrderField(new String[]{"age"}); + String [] incStrings = new String[]{"age","name"}; + esQueryCondition.setIncludeFields(incStrings); + esQueryCondition.setExcludeFields(new String[]{"id"}); + System.out.println("查询的结果5:" + query(index, type,esQueryCondition, queryBuilder4)); + + + // TODO: + + } catch (IOException e) { + e.printStackTrace(); + + } finally { + // TODO: handle finally clause + try { + close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + + private static void createIndexTest() throws IOException { + // setting 的值 + Map setmapping = new HashMap<>(); + + // 分区数、路由分片数、副本数、缓存刷新时间 + setmapping.put("number_of_shards", 12); + setmapping.put("number_of_routing_shards", 24); + setmapping.put("number_of_replicas", 1); + setmapping.put("refresh_interval", "5s"); + + String index = "test5"; + String type = "test5"; + String alias = "test"; + + Map jsonMap2 = new HashMap<>(); + Map message = new HashMap<>(); + // 设置类型 + message.put("type", "text"); + Map properties = new HashMap<>(); + // 设置字段message信息 + properties.put("msg", message); + Map mapping = new HashMap<>(); + mapping.put("properties", properties); + jsonMap2.put(type, mapping); + + String mappings = jsonMap2.toString(); + + EsBasicModelConfig esBasicModelConfig = new EsBasicModelConfig(); + esBasicModelConfig.setIndex(index); + esBasicModelConfig.setType(type); + esBasicModelConfig.setMappings(mappings); + esBasicModelConfig.setSettings(setmapping); + esBasicModelConfig.setAlias(alias); + + EsUtil.creatIndex(esBasicModelConfig); + } + + + /** + * 创建链接 + * + * @param nodes + * @return + */ + public static void build(String... nodes) { + Objects.requireNonNull(nodes, "hosts can not null"); + ArrayList ahosts = new ArrayList(); + for (String host : nodes) { + IpHandler addr = new IpHandler(); + addr.IpPortFromUrl(host); + ahosts.add(new HttpHost(addr.getIp(), addr.getPort())); + } + httpHosts = ahosts.toArray(new HttpHost[0]); + init(); + } + + + /** + * @return boolean + * @Author pancm + * @Description //创建索引库(指定Mpping类型) + * @Date 2019/3/21 + * @Param [esBasicModelConfig] + **/ + public static boolean creatIndex(EsBasicModelConfig esBasicModelConfig) throws IOException { + boolean falg = true; + Objects.requireNonNull(esBasicModelConfig, "esBasicModelConfig is not null"); + String type = Objects.requireNonNull(esBasicModelConfig.getType(), "type is not null"); + String index = Objects.requireNonNull(esBasicModelConfig.getIndex(), "index is not null"); + if (exitsIndex(index)) { + logger.warn("索引库{}已经存在!无需在进行创建!", index); + return true; + } + String mapping = esBasicModelConfig.getMappings(); + Map setting = esBasicModelConfig.getSettings(); + String alias = esBasicModelConfig.getAlias(); + // 开始创建库 + CreateIndexRequest request = new CreateIndexRequest(index); + try { + if (Objects.nonNull(mapping)) { + // 加载数据类型 + request.mapping(type, mapping); + } + if (Objects.nonNull(setting)) { + // 分片数 + request.settings(setting); + } + if (Objects.nonNull(alias)) { + // 别名 + request.alias(new Alias(alias)); + } + + CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); + falg = createIndexResponse.isAcknowledged(); + } catch (IOException e) { + throw e; + } finally { + if (isAutoClose) { + close(); + } + } + return falg; + + } + + + /** + * 判断索引库是否存在 + * + * @param index + * @return + * @throws IOException + */ + public static boolean exitsIndex(String index) throws IOException { + try { + GetIndexRequest getRequest = new GetIndexRequest(); + getRequest.indices(index); + getRequest.local(false); + getRequest.humanReadable(true); + return client.indices().exists(getRequest, RequestOptions.DEFAULT); + } finally { + if (isAutoClose) { + close(); + } + } + } + + + /** + * @return boolean + * @Author pancm + * @Description 单条新增/更新数据 + * @Date 2019/6/5 + * @Param [mapList, index, type] + **/ + public static boolean save(Map map, String index, String type) throws IOException { + List> mapList = new ArrayList<>(); + mapList.add(map); + return saveBulk(mapList, index, type, null); + } + + /** + * @return boolean + * @Author pancm + * @Description 批量新增/更新数据 + * @Date 2019/6/5 + * @Param [mapList, index, type] + **/ + public static boolean saveBulk(List> mapList, String index, String type) throws IOException { + return saveBulk(mapList, index, type, null); + } + + /** + * @return boolean + * @Author pancm + * @Description 批量新增/更新数据 + * @Date 2019/3/21 + * @Param [mapList:存储参数, index:索引库名, type:索引库类型,key:存储的主键，为空表示使用ES主键] + **/ + public static boolean saveBulk(List> mapList, String index, String type, String key) throws IOException { + + if (mapList == null || mapList.size() == 0) { + return true; + } + if (index == null || index.trim().length() == 0 || type == null || type.trim().length() == 0) { + return false; + } + try { + BulkRequest request = new BulkRequest(); + mapList.forEach(map -> { + if (key != null) { + String id = map.get(key) + ""; + if (id == null || id.trim().length() == 0) { + request.add(new IndexRequest(index, type).source(map, XContentType.JSON)); + } else { + request.add(new IndexRequest(index, type, id).source(map, XContentType.JSON)); + } + } else { + request.add(new IndexRequest(index, type).source(map, XContentType.JSON)); + } + }); + + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + //说明至少有一个失败了，这里就直接返回false + if (bulkResponse.hasFailures()) { + return false; + } + + return true; + } finally { + if (isAutoClose) { + close(); + } + } + } + + + /** + * @return boolean + * @Author pancm + * @Description //删除数据 + * 根据ID进行单条删除 + * @Date 2019/3/21 + * @Param [] + **/ + public static boolean deleteById(String index, String type, String id) throws IOException { + if (index == null || type == null || id == null) { + return true; + } + try { + DeleteRequest deleteRequest = new DeleteRequest(); + deleteRequest.id(id); + deleteRequest.index(index); + deleteRequest.type(type); + // 同步删除 + client.delete(deleteRequest, RequestOptions.DEFAULT); + } finally { + if (isAutoClose) { + close(); + } + } + return true; + } + + + /** + * @return boolean + * @Author pancm + * @Description //批量删除数据 + * 根据ID进行批量删除 + * @Date 2019/3/21 + * @Param [] + **/ + public static boolean deleteByIds(String index, String type, Set ids) throws IOException { + if (index == null || type == null || ids == null) { + return true; + } + try { + BulkRequest requestBulk = new BulkRequest(); + ids.forEach(id -> { + DeleteRequest deleteRequest = new DeleteRequest(index, type, id); + requestBulk.add(deleteRequest); + }); + // 同步删除 + client.bulk(requestBulk, RequestOptions.DEFAULT); + } finally { + if (isAutoClose) { + close(); + } + } + return false; + } + + /** + * @return boolean + * @Author pancm + * @Description 根据id查询 + * @Date 2019/3/21 + * @Param [] + **/ + public static Map queryById(String index, String type, String id) throws IOException { + if (index == null || type == null) { + return null; + } + Map map = new HashMap<>(); + try { + GetRequest request = new GetRequest(); + request.index(index); + request.type(type); + request.id(id); + GetResponse getResponse = client.get(request, RequestOptions.DEFAULT); + // 如果存在该数据则返回对应的结果 + if (getResponse.isExists()) { + map = getResponse.getSourceAsMap(); + } + } finally { + if (isAutoClose) { + close(); + } + } + return map; + } + + + public static List> query(String index, String type, QueryBuilder... queryBuilders) throws IOException { + return query(index, type, null , queryBuilders); + } + + /** + * @return boolean + * @Author pancm + * @Description 根据条件查询 + * @Date 2019/3/21 + * @Param [] + **/ + public static List> query(String index, String type, EsQueryCondition esQueryCondition, QueryBuilder... queryBuilders) throws IOException { + if (index == null || type == null) { + return null; + } + List> list = new ArrayList<>(); + try { + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.types(type); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + + if (esQueryCondition != null) { + Integer form = esQueryCondition.getIndex(); + Integer pagesize = esQueryCondition.getPagesize(); + if (form != null && form > 0 && pagesize != null && pagesize > 0) { + form = (form - 1) * pagesize; + pagesize = form + pagesize; + // 设置起止和结束 + sourceBuilder.from(form); + sourceBuilder.size(pagesize); + } + String routing = esQueryCondition.getRouting(); + if (routing != null && routing.length() > 0) { + // 设置路由 + searchRequest.routing(routing); + } + + // 设置索引库表达式 + searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); + + //设置排序 + String order = esQueryCondition.getOrder(); + if (order != null) { + String[] orderField = esQueryCondition.getOrderField(); + SortOrder order2 = order.equals(SortOrder.DESC) ? SortOrder.DESC : SortOrder.ASC; + //如果设置了排序字段则用排序的字段进行排序，否则就默认排序 + if (orderField != null) { + for (String field : orderField) { + sourceBuilder.sort(new FieldSortBuilder(field).order(order2)); + } + } else { + sourceBuilder.sort(new ScoreSortBuilder().order(order2)); + } + } + String[] includeFields = esQueryCondition.getIncludeFields(); + String[] excludeFields = esQueryCondition.getExcludeFields(); + if (includeFields != null && includeFields.length > 0 && excludeFields != null && excludeFields.length > 0) { + sourceBuilder.fetchSource(includeFields, excludeFields); + } + sourceBuilder.fetchSource(esQueryCondition.isCloseSource()); + } + //设置条件 + if (queryBuilders != null) { + for (QueryBuilder queryBuilder : queryBuilders) { + sourceBuilder.query(queryBuilder); + } + } + + searchRequest.source(sourceBuilder); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + + if(queryBuilders != null|| (esQueryCondition != null && esQueryCondition.isQueryData())){ + // 结果 + searchResponse.getHits().forEach(hit -> { + Map map = hit.getSourceAsMap(); + list.add(map); + }); + } + + if(esQueryCondition != null && esQueryCondition.isNeedTotal()){ + Map mapTotal = new HashMap<>(); + mapTotal.put("total", searchResponse.getHits().getTotalHits()); + list.add(mapTotal); + } + + } finally { + if (isAutoClose) { + close(); + } + } + return list; + } + + + /** + * @return boolean + * @Author pancm + * @Description 根据条件更新 + * @Date 2019/3/21 + * @Param [] + **/ + public static Map updateByQuery(String index, String type, QueryBuilder... queryBuilders) throws IOException { + if (index == null || type == null) { + return null; + } + Map map = new HashMap<>(); + try { + UpdateByQueryRequest request = new UpdateByQueryRequest(); + request.indices(index); + request.setDocTypes(type); + + if (queryBuilders != null) { + for (QueryBuilder queryBuilder : queryBuilders) { + request.setQuery(queryBuilder); + } + } + // 同步执行 + BulkByScrollResponse bulkResponse = client.updateByQuery(request, RequestOptions.DEFAULT); + + // 响应结果处理 + map.put("time", bulkResponse.getTook().getMillis()); + map.put("total", bulkResponse.getTotal()); + + } finally { + if (isAutoClose) { + close(); + } + } + return map; + } + + /** + * @return Map + * @Author pancm + * @Description //根据条件删除数据 + * @Date 2019/3/21 + * @Param [] + **/ + public static Map deleteByQuery(String index, String type, QueryBuilder[] queryBuilders) throws IOException { + if (index == null || type == null || queryBuilders == null) { + return null; + } + Map map = new HashMap<>(); + try { + DeleteByQueryRequest request = new DeleteByQueryRequest(index, type); + if (queryBuilders != null) { + for (QueryBuilder queryBuilder : queryBuilders) { + request.setQuery(queryBuilder); + } + } + // 同步执行 + BulkByScrollResponse bulkResponse = client.deleteByQuery(request, RequestOptions.DEFAULT); + // 响应结果处理 + map.put("time", bulkResponse.getTook().getMillis()); + map.put("total", bulkResponse.getTotal()); + + } finally { + if (isAutoClose) { + close(); + } + } + return map; + } + + + /** + * @return Map + * @Author pancm + * @Description //重索引 + * @Date 2019/3/21 + * @Param [] + **/ + public static Map reindexByQuery(String index, String destIndex, QueryBuilder[] queryBuilders) throws IOException { + if (index == null || destIndex == null) { + return null; + } + Map map = new HashMap<>(); + try { + // 创建索引复制请求并进行索引复制 + ReindexRequest request = new ReindexRequest(); + // 需要复制的索引 + request.setSourceIndices(index); + /* 复制的目标索引 */ + request.setDestIndex(destIndex); + if (queryBuilders != null) { + for (QueryBuilder queryBuilder : queryBuilders) { + request.setSourceQuery(queryBuilder); + } + } + // 表示如果在复制索引的时候有缺失的文档的话会进行创建,默认是index + request.setDestOpType("create"); + // 如果在复制的过程中发现版本冲突，那么会继续进行复制 + request.setConflicts("proceed"); + + + // 设置复制文档的数量 + // request.setSize(10); + // 设置一次批量处理的条数，默认是1000 + // request.setSourceBatchSize(10000); + + //设置超时时间 + request.setTimeout(TimeValue.timeValueMinutes(2)); + // 同步执行 + BulkByScrollResponse bulkResponse = client.reindex(request, RequestOptions.DEFAULT); + + // 响应结果处理 + map.put("time", bulkResponse.getTook().getMillis()); + map.put("total", bulkResponse.getTotal()); + map.put("createdDocs", bulkResponse.getCreated()); + map.put("updatedDocs", bulkResponse.getUpdated()); + + } finally { + if (isAutoClose) { + close(); + } + } + return map; + } + + + /* + * 初始化服务 + */ + private static void init() { + if (client == null) { + synchronized (EsUtil.class) { + if (client == null) { + RestClientBuilder restClientBuilder = RestClient.builder(httpHosts); + client = new RestHighLevelClient(restClientBuilder); + } + } + } + + } + + /* + * 关闭服务 + */ + private static void close() throws IOException { + if (client != null) { + try { + client.close(); + setIsAutoClose(true); + } catch (IOException e) { + throw e; + } + } + } + + public static boolean isIsAutoClose() { + return isAutoClose; + } + + public static void setIsAutoClose(boolean isAutoClose) { + EsUtil.isAutoClose = isAutoClose; + } + + + private static String[] elasticIps; + private static int elasticPort; + private static HttpHost[] httpHosts; + private static volatile RestHighLevelClient client = null; + /** + * 是否自动关闭连接 + */ + private static boolean isAutoClose = true; + + private static final String COMMA_SIGN = ","; + + +} + +/** + * @Author pancm + * @Description 查询条件的类 + * @Date 2019/6/19 + * @Param + * @return + **/ +class EsQueryCondition implements Serializable { + private static final long serialVersionUID = 1L; + + /** + * 下标和条数 都为null或小于0表示不分页 + */ + private Integer index; + private Integer pagesize; + + /** + * 排序规则 asc:升序，desc:降序，为空表示不排序 + */ + private String order; + /** + * 排序字段 + */ + private String[] orderField; + + /** + * 路由 为空表示不设置 + */ + private String routing; + + /** + * 返回的字段 + */ + private String[] includeFields; + /** + * 排除的字段 + */ + private String[] excludeFields; + + /** + * 是否关闭source查询 + */ + private boolean isCloseSource; + + /** 是否需要查询数据 */ + private boolean isQueryData = true; + + /** 是否需要 返回总数 */ + private boolean isNeedTotal = true; + + + public boolean isQueryData() { + return isQueryData; + } + + public void setQueryData(boolean queryData) { + isQueryData = queryData; + } + + public boolean isNeedTotal() { + return isNeedTotal; + } + + public void setNeedTotal(boolean needTotal) { + isNeedTotal = needTotal; + } + + public boolean isCloseSource() { + return isCloseSource; + } + + public void setCloseSource(boolean closeSource) { + isCloseSource = closeSource; + } + + public String[] getIncludeFields() { + return includeFields; + } + + public void setIncludeFields(String[] includeFields) { + this.includeFields = includeFields; + } + + public String[] getExcludeFields() { + return excludeFields; + } + + public void setExcludeFields(String[] excludeFields) { + this.excludeFields = excludeFields; + } + + public Integer getIndex() { + return index; + } + + public void setIndex(Integer index) { + this.index = index; + } + + public Integer getPagesize() { + return pagesize; + } + + public void setPagesize(Integer pagesize) { + this.pagesize = pagesize; + } + + public String getOrder() { + return order; + } + + public void setOrder(String order) { + this.order = order; + } + + public String getRouting() { + return routing; + } + + public void setRouting(String routing) { + this.routing = routing; + } + + public String[] getOrderField() { + return orderField; + } + + public void setOrderField(String[] orderField) { + this.orderField = orderField; + } +} + + +/* + * ES的mapping创建的基础类 + */ +class EsBasicModelConfig implements Serializable { + private static final long serialVersionUID = 1L; + /*** 索引库 ***/ + private String index; + private String type; + private Map settings; + private String mappings; + private String alias; + + public EsBasicModelConfig() { + } + + public EsBasicModelConfig(String index, String type) { + this.index = index; + this.type = type; + } + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Map getSettings() { + return settings; + } + + public void setSettings(Map settings) { + this.settings = settings; + } + + public void setSettings(SettingEntity settings) { + this.settings = Objects.requireNonNull(settings, "setting can not null").toDSL(); + } + + public String getMappings() { + return mappings; + } + + public void setMappings(String mappings) { + this.mappings = mappings; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + @Override + public String toString() { + return "EsBasicModelConfig [index=" + index + ", type=" + type + ", settings=" + settings + ", mappings=" + + mappings + "]"; + } + +} + +/* + * setting 实体类的配置 + */ +class SettingEntity implements Serializable { + /** + * @Fields serialVersionUID : TODO + */ + private static final long serialVersionUID = 1L; + // 默认分片数 + private int numberOfShards = 5; + // 分片路由数 + private int number_of_routing_shards = 30; + // 副本数 + private int numberOfReplicas = 1; + /***** 刷新频率 单位:秒 *********/ + private int refreshInterval = 5; + /** + * 查询最大返回的时间 + */ + private int maxResultWindow = 10000; + + public SettingEntity(int numberOfShards, int numberOfReplicas, int refreshInterval) { + this.numberOfShards = numberOfShards; + this.numberOfReplicas = numberOfReplicas; + this.refreshInterval = refreshInterval; + } + + public SettingEntity(int numberOfShards, int numberOfReplicas, int refreshInterval, int number_of_routing_shards, + int maxResultWindow, String alias) { + this.numberOfShards = numberOfShards; + this.numberOfReplicas = numberOfReplicas; + this.refreshInterval = refreshInterval; + this.number_of_routing_shards = number_of_routing_shards; + this.maxResultWindow = maxResultWindow; + } + + public SettingEntity() { + + } + + public int getNumberOfShards() { + return numberOfShards; + } + + /** + * 分片数 + * + * @param numberOfShards 默认5 + */ + public void setNumberOfShards(int numberOfShards) { + this.numberOfShards = numberOfShards; + } + + public int getNumberOfReplicas() { + return numberOfReplicas; + } + + /** + * 副本数 + * + * @param numberOfReplicas 默认1 + */ + public void setNumberOfReplicas(int numberOfReplicas) { + this.numberOfReplicas = numberOfReplicas; + } + + public int getRefreshInterval() { + return refreshInterval; + } + + public int getNumber_of_routing_shards() { + return number_of_routing_shards; + } + + public void setNumber_of_routing_shards(int number_of_routing_shards) { + this.number_of_routing_shards = number_of_routing_shards; + } + + public int getMaxResultWindow() { + return maxResultWindow; + } + + public void setMaxResultWindow(int maxResultWindow) { + this.maxResultWindow = maxResultWindow; + } + + /** + * 刷新频率 单位:秒 + * + * @param refreshInterval 默认5秒 设置为-1为无限刷新 + */ + public void setRefreshInterval(int refreshInterval) { + if (refreshInterval < -1) { + refreshInterval = -1; + } + this.refreshInterval = refreshInterval; + } + + public Map toDSL() { + Map json = new HashMap<>(); + json.put("number_of_shards", numberOfShards); + json.put("number_of_routing_shards", number_of_routing_shards); + json.put("number_of_replicas", numberOfReplicas); + json.put("refresh_interval", refreshInterval + "s"); + json.put("max_result_window", maxResultWindow); + return json; + } + + @Override + public String toString() { + return "SettingEntity [numberOfShards=" + numberOfShards + ", numberOfReplicas=" + numberOfReplicas + + ", refreshInterval=" + refreshInterval + ", maxResultWindow=" + maxResultWindow + "]"; + } + +} + +class IpHandler { + + private String ip; + private Integer port; + private static Pattern p = Pattern.compile("(?<=//|)((\\w)+\\.)+\\w+(:\\d{0,5})?"); + + /** + * 冒号 + */ + private static final String COMMA_COLON = ":"; + + /** + * 从url中分析出hostIP:PORT
+ * + * @param url + */ + public void IpPortFromUrl(String url) { + + String host = ""; + + Matcher matcher = p.matcher(url); + if (matcher.find()) { + host = matcher.group(); + } + // 如果 + if (host.contains(COMMA_COLON) == false) { + this.ip = host; + this.port = 80; + } else { + String[] ipPortArr = host.split(COMMA_COLON); + this.ip = ipPortArr[0]; + this.port = Integer.valueOf(ipPortArr[1].trim()); + } + } + + public String getIp() { + return ip; + } + + public Integer getPort() { + return port; + } +} From b572c5ac6b89ff51a9bb4158ee5018f9b03a5e2f Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Sat, 31 Aug 2019 16:54:54 +0800 Subject: [PATCH 21/53] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=B8=AD=E4=BB=8B?= =?UTF-8?q?=E8=80=85=E6=A8=A1=E5=BC=8F=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/com/pancm/design/mediator/MediatorTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/pancm/design/mediator/MediatorTest.java b/src/main/java/com/pancm/design/mediator/MediatorTest.java index 02c80c4..e582d55 100644 --- a/src/main/java/com/pancm/design/mediator/MediatorTest.java +++ b/src/main/java/com/pancm/design/mediator/MediatorTest.java @@ -29,9 +29,8 @@ public static void main(String[] args) { XuWuJing xwj = new XuWuJing("xuwujing", jq); jq.setZs(zs); jq.setXwj(xwj); - - zs.talk("大家好！我是张三!");; - xwj.talk("欢迎你！张三！"); + zs.exchange("大家好！我是张三!"); + xwj.exchange("欢迎你！张三！"); /* From 0e28dfae734dcc3de443f490df8887b20141ab4b Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Mon, 30 Sep 2019 17:59:19 +0800 Subject: [PATCH 22/53] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0es=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E7=A4=BA=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../EsHighLevelRestSearchTest.java | 788 +++++++++++------- 1 file changed, 498 insertions(+), 290 deletions(-) diff --git a/src/main/java/com/pancm/easticsearch/EsHighLevelRestSearchTest.java b/src/main/java/com/pancm/easticsearch/EsHighLevelRestSearchTest.java index 280e434..9287e2b 100644 --- a/src/main/java/com/pancm/easticsearch/EsHighLevelRestSearchTest.java +++ b/src/main/java/com/pancm/easticsearch/EsHighLevelRestSearchTest.java @@ -1,9 +1,4 @@ -package com.pancm.sql.easticsearch; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; +package com.pancm.easticsearch; import org.apache.http.HttpHost; import org.elasticsearch.action.search.SearchRequest; @@ -12,9 +7,11 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; @@ -25,6 +22,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.avg.Avg; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.suggest.Suggest; @@ -35,310 +33,520 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; + /** + * @author pancm * @Title: EsHighLevelRestSearchTest * @Description: Java High Level REST Client Es高级客户端查询使用使用教程 (Search查询使用教程) - * 官方文档地址: - * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high.html + * 官方文档地址: + * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high.html * @Version:1.0.0 - * @author pancm * @date 2019年3月12日 */ public class EsHighLevelRestSearchTest { - private static String elasticIp = "192.169.0.23"; - private static int elasticPort = 9200; - - private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestSearchTest.class); - - private static RestHighLevelClient client = null; - - /** - * @param args - */ - public static void main(String[] args) { - - try { - init(); - search(); - - - } catch (Exception e) { - e.printStackTrace(); - }finally { - close(); - } - - } - - /* - * 初始化服务 - */ - private static void init() { - client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort, "http"))); - - } - - /* - * 关闭服务 - */ - private static void close() { - if (client != null) { - try { - client.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - /** - * search查询使用示例 - * - * @throws IOException - */ - private static void search() throws IOException { - - /* - * 查询集群所有的索引 - * - */ - SearchRequest searchRequestAll = new SearchRequest(); - - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.matchAllQuery()); - searchRequestAll.source(searchSourceBuilder); - - // 同步查询 - SearchResponse searchResponseAll = client.search(searchRequestAll, RequestOptions.DEFAULT); - - System.out.println("所有查询总数:" + searchResponseAll.getHits().getTotalHits()); - - // 查询指定的索引库 - SearchRequest searchRequest = new SearchRequest("user"); - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - // 设置查询条件 - sourceBuilder.query(QueryBuilders.termQuery("user", "pancm")); - // 设置起止和结束 - sourceBuilder.from(0); - sourceBuilder.size(5); - sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS)); - // 设置路由 + private static String elasticIp = "192.169.2.98"; + private static int elasticPort = 9200; + private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestSearchTest.class); + + private static RestHighLevelClient client = null; + + /** + * @param args + */ + public static void main(String[] args) { + + try { + init(); + allSearch(); + //普通查询 + genSearch(); + orSearch(); + likeSearch(); + inSearch(); + existSearch(); + rangeSearch(); +// search(); +// search2(); + + } catch (Exception e) { + e.printStackTrace(); + } finally { + close(); + } + + /** + * p_test + *{ + * "settings" : { + * "number_of_shards" : 10, + * "refresh_interval" : "1s" + * }, + * "mappings" : { + * "_doc" : { + * "properties" : { + * "uid" : { "type" : "long" }, + * "phone" : { "type" : "long" }, + * "userid" : { "type" : "keyword" }, + * "sendday" : { "type" : "long" }, + * "message" : { "type" : "keyword" }, + * "msgcode" : { "type" : "long" }, + * "price" : { "type" : "double","index": "false" }, + * "sendtime" : { + * "type" : "date", + * "format" : "yyyy-MM-dd HH:mm:ss.SSS" + * }, + * "sendtime2" : { + * "type" : "date", + * "format" : "yyyy-MM-dd HH:mm:ss.SSS" + * }, + * "sendtm" : { "type" : "long" }, + * "sendtm2" : { "type" : "long" } + * } + * } + * } + * } + */ + + } + + /** + * @Author pancm + * @Description 范围查询 + * @Date 2019/9/30 + * @Param [] + * @return void + **/ + private static void rangeSearch() throws IOException{ + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest("user"); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + + // 设置查询条件 + sourceBuilder.query(QueryBuilders.rangeQuery("sendtime").gte("2019-01-01 00:00:00.000").lte("2019-12-31 23:59:59.999")); + searchRequest.source(sourceBuilder); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + // 结果 + searchResponse.getHits().forEach(hit -> { + Map map = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("范围查询的Map结果:" + map); + System.out.println("范围查询的String结果:" + string); + }); + + + System.out.println("\n=================\n"); + } + + /** + * @return void + * @Author pancm + * @Description exist查询 + * @Date 2019/9/17 + * @Param [] + **/ + private static void existSearch() throws IOException { + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest("user"); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + + // 设置查询条件 + sourceBuilder.query(QueryBuilders.existsQuery("uid")); + searchRequest.source(sourceBuilder); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + // 结果 + searchResponse.getHits().forEach(hit -> { + Map map = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("存在查询的Map结果:" + map); + System.out.println("存在查询的String结果:" + string); + }); + + + System.out.println("\n=================\n"); + + } + + /** + * @return void + * @Author pancm + * @Description in查询 + * @Date 2019/9/16 + * @Param [] + **/ + private static void inSearch() throws IOException { + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest("user"); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + /** + * SELECT * FROM p_test where uid in (1,2) + * */ + // 设置查询条件 + sourceBuilder.query(QueryBuilders.termsQuery("uid", 1, 2)); + searchRequest.source(sourceBuilder); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + // 结果 + searchResponse.getHits().forEach(hit -> { + Map map = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("in查询的Map结果:" + map); + System.out.println("in查询的String结果:" + string); + }); + + + System.out.println("\n=================\n"); + } + + /** + * @return void + * @Author pancm + * @Description 模糊查询 + * @Date 2019/9/12 + * @Param [] + **/ + private static void likeSearch() throws IOException { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices("p_test"); + searchRequest.types("_doc"); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + /** + * SELECT * FROM p_test where message like '%pan%'; + * */ + boolQueryBuilder.must(QueryBuilders.wildcardQuery("message", "*pan*")); + searchSourceBuilder.query(boolQueryBuilder); + + System.out.println("模糊查询语句:" + searchSourceBuilder.toString()); + searchRequest.source(searchSourceBuilder); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + searchResponse.getHits().forEach(documentFields -> { + System.out.println("查询结果:" + documentFields.getSourceAsMap()); + }); + + } + + + /** + * @return void + * @Author pancm + * @Description 普通查询 + * @Date 2019/9/12 + * @Param [] + **/ + private static void genSearch() throws IOException { + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest("user"); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + // 设置查询条件 + sourceBuilder.query(QueryBuilders.termQuery("user", "pancm")); + // 设置起止和结束 + sourceBuilder.from(0); + sourceBuilder.size(5); + sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS)); + // 设置路由 // searchRequest.routing("routing"); - // 设置索引库表达式 - searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); - // 查询选择本地分片，默认是集群分片 - searchRequest.preference("_local"); - - // 排序 - // 根据默认值进行降序排序 -// sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); - // 根据字段进行升序排序 -// sourceBuilder.sort(new FieldSortBuilder("id").order(SortOrder.ASC)); - - // 关闭suorce查询 + // 设置索引库表达式 + searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); + // 查询选择本地分片，默认是集群分片 + searchRequest.preference("_local"); + + // 排序 + // 根据默认值进行降序排序 +// sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); + // 根据字段进行升序排序 +// sourceBuilder.sort(new FieldSortBuilder("id").order(SortOrder.ASC)); + + // 关闭suorce查询 // sourceBuilder.fetchSource(false); - String[] includeFields = new String[] { "title", "user", "innerObject.*" }; - String[] excludeFields = new String[] { "_type" }; - // 包含或排除字段 + String[] includeFields = new String[]{"title", "user", "innerObject.*"}; + String[] excludeFields = new String[]{"_type"}; + // 包含或排除字段 // sourceBuilder.fetchSource(includeFields, excludeFields); - searchRequest.source(sourceBuilder); - - // 同步查询 - SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - - // HTTP状态代码、执行时间或请求是否提前终止或超时 - RestStatus status = searchResponse.status(); - TimeValue took = searchResponse.getTook(); - Boolean terminatedEarly = searchResponse.isTerminatedEarly(); - boolean timedOut = searchResponse.isTimedOut(); - - // 供关于受搜索影响的切分总数的统计信息，以及成功和失败的切分 - int totalShards = searchResponse.getTotalShards(); - int successfulShards = searchResponse.getSuccessfulShards(); - int failedShards = searchResponse.getFailedShards(); - // 失败的原因 - for (ShardSearchFailure failure : searchResponse.getShardFailures()) { - // failures should be handled here - } - - // 结果 - searchResponse.getHits().forEach(hit -> { - Map map = hit.getSourceAsMap(); - String string = hit.getSourceAsString(); - System.out.println("普通查询的Map结果:" + map); - System.out.println("普通查询的String结果:" + string); - }); - - - System.out.println("\n=================\n"); - - /* - * 全文查询使用示例 - */ - - // 搜索字段user为pancm的数据 - MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("user", "pancm"); - - // 设置模糊查询 - matchQueryBuilder.fuzziness(Fuzziness.AUTO); - // 设置前缀长度 - matchQueryBuilder.prefixLength(3); - // 设置最大扩展选项来控制查询的模糊过程 - matchQueryBuilder.maxExpansions(10); - - /* - * QueryBuilder也可以 - */ + searchRequest.source(sourceBuilder); + + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + + // HTTP状态代码、执行时间或请求是否提前终止或超时 + RestStatus status = searchResponse.status(); + TimeValue took = searchResponse.getTook(); + Boolean terminatedEarly = searchResponse.isTerminatedEarly(); + boolean timedOut = searchResponse.isTimedOut(); + + // 供关于受搜索影响的切分总数的统计信息，以及成功和失败的切分 + int totalShards = searchResponse.getTotalShards(); + int successfulShards = searchResponse.getSuccessfulShards(); + int failedShards = searchResponse.getFailedShards(); + // 失败的原因 + for (ShardSearchFailure failure : searchResponse.getShardFailures()) { + // failures should be handled here + } + + // 结果 + searchResponse.getHits().forEach(hit -> { + Map map = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("普通查询的Map结果:" + map); + System.out.println("普通查询的String结果:" + string); + }); + + + System.out.println("\n=================\n"); + + } + + private static void allSearch() throws IOException { + /* + * 查询集群所有的索引 + * + */ + SearchRequest searchRequestAll = new SearchRequest(); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchRequestAll.source(searchSourceBuilder); + // 同步查询 + SearchResponse searchResponseAll = client.search(searchRequestAll, RequestOptions.DEFAULT); + + System.out.println("所有查询总数:" + searchResponseAll.getHits().getTotalHits()); + } + + /** + * @return void + * @Author pancm + * @Description 或查询 + * @Date 2019/9/6 + * @Param [] + **/ + private static void orSearch() throws IOException { + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices("p_test"); + searchRequest.types("_doc"); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + BoolQueryBuilder boolQueryBuilder2 = new BoolQueryBuilder(); + /** + * SELECT * FROM p_test where (uid = 1 or uid =2) and phone = 12345678919 + * */ + boolQueryBuilder2.should(QueryBuilders.termQuery("uid", 1)); + boolQueryBuilder2.should(QueryBuilders.termQuery("uid", 2)); + boolQueryBuilder.must(boolQueryBuilder2); + boolQueryBuilder.must(QueryBuilders.termQuery("phone", "12345678919")); + searchSourceBuilder.query(boolQueryBuilder); + + System.out.println("或查询语句:" + searchSourceBuilder.toString()); + searchRequest.source(searchSourceBuilder); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + + searchResponse.getHits().forEach(documentFields -> { + + System.out.println("查询结果:" + documentFields.getSourceAsMap()); + }); + + } + + /* + * 初始化服务 + */ + private static void init() { + RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(elasticIp, elasticPort)); + client = new RestHighLevelClient(restClientBuilder); + + } + + /* + * 关闭服务 + */ + private static void close() { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + client = null; + } + } + } + + + /** + * search查询使用示例 + * + * @throws IOException + */ + private static void search() throws IOException { + + + + /* + * 全文查询使用示例 + */ + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest("user"); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + // 搜索字段user为pancm的数据 + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("user", "pancm"); + + // 设置模糊查询 + matchQueryBuilder.fuzziness(Fuzziness.AUTO); + // 设置前缀长度 + matchQueryBuilder.prefixLength(3); + // 设置最大扩展选项来控制查询的模糊过程 + matchQueryBuilder.maxExpansions(10); + + /* + * QueryBuilder也可以 + */ // QueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("user", "kimchy") // .fuzziness(Fuzziness.AUTO) // .prefixLength(3) // .maxExpansions(10); - SearchSourceBuilder searchSourceBuilder2 = new SearchSourceBuilder(); - searchSourceBuilder2.query(matchQueryBuilder); - - SearchRequest searchRequest2 = new SearchRequest(); - searchRequest2.source(searchSourceBuilder2); - // 同步查询 - SearchResponse searchResponse2 = client.search(searchRequest, RequestOptions.DEFAULT); - - SearchHits hits = searchResponse.getHits(); - //总条数和分值 - long totalHits = hits.getTotalHits(); - float maxScore = hits.getMaxScore(); - - - - hits.forEach(hit -> { - - String index = hit.getIndex(); - String type = hit.getType(); - String id = hit.getId(); - float score = hit.getScore(); - - - - Map sourceAsMap = hit.getSourceAsMap(); - String string = hit.getSourceAsString(); - System.out.println("Match查询的Map结果:" + sourceAsMap); - System.out.println("Match查询的String结果:" + string); - - String documentTitle = (String) sourceAsMap.get("title"); + SearchSourceBuilder searchSourceBuilder2 = new SearchSourceBuilder(); + searchSourceBuilder2.query(matchQueryBuilder); + + SearchRequest searchRequest2 = new SearchRequest(); + searchRequest2.source(searchSourceBuilder2); + // 同步查询 + SearchResponse searchResponse2 = client.search(searchRequest, RequestOptions.DEFAULT); + + SearchHits hits = searchResponse2.getHits(); + //总条数和分值 + long totalHits = hits.getTotalHits(); + float maxScore = hits.getMaxScore(); + + + hits.forEach(hit -> { + + String index = hit.getIndex(); + String type = hit.getType(); + String id = hit.getId(); + float score = hit.getScore(); + + + Map sourceAsMap = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("Match查询的Map结果:" + sourceAsMap); + System.out.println("Match查询的String结果:" + string); + + String documentTitle = (String) sourceAsMap.get("title"); // List users = (List) sourceAsMap.get("user"); - Map innerObject = - (Map) sourceAsMap.get("innerObject"); - }); - - - - - System.out.println("\n=================\n"); - - /* - * 高亮查询 - */ - - SearchSourceBuilder searchSourceBuilder3 = new SearchSourceBuilder(); - HighlightBuilder highlightBuilder = new HighlightBuilder(); - HighlightBuilder.Field highlightTitle = new HighlightBuilder.Field("title"); - // 设置字段高亮字体 - highlightTitle.highlighterType("user"); - highlightBuilder.field(highlightTitle); - HighlightBuilder.Field highlightUser = new HighlightBuilder.Field("user"); - highlightBuilder.field(highlightUser); - searchSourceBuilder3.highlighter(highlightBuilder); - - SearchRequest searchRequest3 = new SearchRequest(); - searchRequest3.source(searchSourceBuilder3); - - // 同步查询 - SearchResponse searchResponse3 = client.search(searchRequest3, RequestOptions.DEFAULT); - - searchResponse3.getHits().forEach(hit -> { - - Map map = hit.getSourceAsMap(); - String string = hit.getSourceAsString(); - System.out.println("Highlight查询的Map结果:" + map); - System.out.println("Highlight查询的String结果:" + string); - }); - - System.out.println("\n=================\n"); - - /** - * 聚合查询 - */ - - SearchSourceBuilder searchSourceBuilder4 = new SearchSourceBuilder(); - - //terms 就是分组统计 根据user进行分组并创建一个新的聚合user_ - TermsAggregationBuilder aggregation = AggregationBuilders.terms("user_").field("user"); - aggregation.subAggregation(AggregationBuilders.avg("average_age").field("age")); - searchSourceBuilder4.aggregation(aggregation); - - SearchRequest searchRequest4 = new SearchRequest(); - searchRequest4.source(searchSourceBuilder4); - - // 同步查询 - SearchResponse searchResponse4 = client.search(searchRequest4, RequestOptions.DEFAULT); - //聚合查询返回条件 - Aggregations aggregations = searchResponse4.getAggregations(); - List aggregationList = aggregations.asList(); - - for (Aggregation agg : aggregations) { - String type = agg.getType(); - if (type.equals(TermsAggregationBuilder.NAME)) { - Bucket elasticBucket = ((Terms) agg).getBucketByKey("Elastic"); - long numberOfDocs = elasticBucket.getDocCount(); - System.out.println("条数:"+numberOfDocs); - } - } - - -// Terms byCompanyAggregation = aggregations.get("user_"); -// Bucket elasticBucket = byCompanyAggregation.getBucketByKey("Elastic"); -// Avg averageAge = elasticBucket.getAggregations().get("average_age"); -// double avg = averageAge.getValue(); -// -// System.out.println("聚合查询返回的结果:"+avg); - - - /* - * 建议查询 - */ - SearchSourceBuilder searchSourceBuilder5 = new SearchSourceBuilder(); - SuggestionBuilder termSuggestionBuilder = SuggestBuilders.termSuggestion("user").text("pancm"); - SuggestBuilder suggestBuilder = new SuggestBuilder(); - suggestBuilder.addSuggestion("suggest_user", termSuggestionBuilder); - searchSourceBuilder5.suggest(suggestBuilder); - - SearchRequest searchRequest5 = new SearchRequest(); - searchRequest5.source(searchSourceBuilder5); - - // 同步查询 - SearchResponse searchResponse5= client.search(searchRequest5, RequestOptions.DEFAULT); - - Suggest suggest = searchResponse5.getSuggest(); - TermSuggestion termSuggestion = suggest.getSuggestion("suggest_user"); - for (TermSuggestion.Entry entry : termSuggestion.getEntries()) { - for (TermSuggestion.Entry.Option option : entry) { - String suggestText = option.getText().string(); - System.out.println("返回结果:"+suggestText); - } - } - - - - - - - - - - } + Map innerObject = + (Map) sourceAsMap.get("innerObject"); + }); + + + System.out.println("\n=================\n"); + + /* + * 高亮查询 + */ + + SearchSourceBuilder searchSourceBuilder3 = new SearchSourceBuilder(); + HighlightBuilder highlightBuilder = new HighlightBuilder(); + HighlightBuilder.Field highlightTitle = new HighlightBuilder.Field("title"); + // 设置字段高亮字体 + highlightTitle.highlighterType("user"); + highlightBuilder.field(highlightTitle); + HighlightBuilder.Field highlightUser = new HighlightBuilder.Field("user"); + highlightBuilder.field(highlightUser); + searchSourceBuilder3.highlighter(highlightBuilder); + + SearchRequest searchRequest3 = new SearchRequest(); + searchRequest3.source(searchSourceBuilder3); + + // 同步查询 + SearchResponse searchResponse3 = client.search(searchRequest3, RequestOptions.DEFAULT); + + searchResponse3.getHits().forEach(hit -> { + + Map map = hit.getSourceAsMap(); + String string = hit.getSourceAsString(); + System.out.println("Highlight查询的Map结果:" + map); + System.out.println("Highlight查询的String结果:" + string); + }); + + System.out.println("\n=================\n"); + + /** + * 聚合查询 + */ + + SearchSourceBuilder searchSourceBuilder4 = new SearchSourceBuilder(); + + //terms 就是分组统计 根据user进行分组并创建一个新的聚合user_ + TermsAggregationBuilder aggregation = AggregationBuilders.terms("user_").field("user"); + aggregation.subAggregation(AggregationBuilders.avg("average_age").field("age")); + searchSourceBuilder4.aggregation(aggregation); + + SearchRequest searchRequest4 = new SearchRequest(); + searchRequest4.source(searchSourceBuilder4); + + // 同步查询 + SearchResponse searchResponse4 = client.search(searchRequest4, RequestOptions.DEFAULT); + //聚合查询返回条件 + Aggregations aggregations = searchResponse4.getAggregations(); + System.out.println("聚合查询"); + for (Aggregation agg : aggregations) { + String type = agg.getType(); + String name = agg.getName(); + + Terms terms = (Terms) aggregations.get(name); + for (Bucket bucket : terms.getBuckets()) { + System.out.println("条数:" + bucket.getDocCount()); + System.out.println("key:" + bucket.getKey()); + System.out.println("num:" + bucket.getKeyAsNumber()); + Avg avg = bucket.getAggregations().get("average_age"); + System.out.println("value:" + avg.getValue()); + } + if (type.equals(TermsAggregationBuilder.NAME)) { + Bucket elasticBucket = ((Terms) agg).getBucketByKey("average_age"); + long numberOfDocs = elasticBucket.getDocCount(); + System.out.println("条数:" + numberOfDocs); + } + } + + + + + + /* + * 建议查询 + */ + SearchSourceBuilder searchSourceBuilder5 = new SearchSourceBuilder(); + SuggestionBuilder termSuggestionBuilder = SuggestBuilders.termSuggestion("user").text("pancm"); + SuggestBuilder suggestBuilder = new SuggestBuilder(); + suggestBuilder.addSuggestion("suggest_user", termSuggestionBuilder); + searchSourceBuilder5.suggest(suggestBuilder); + + SearchRequest searchRequest5 = new SearchRequest(); + searchRequest5.source(searchSourceBuilder5); + + // 同步查询 + SearchResponse searchResponse5 = client.search(searchRequest5, RequestOptions.DEFAULT); + + Suggest suggest = searchResponse5.getSuggest(); + TermSuggestion termSuggestion = suggest.getSuggestion("suggest_user"); + for (TermSuggestion.Entry entry : termSuggestion.getEntries()) { + for (TermSuggestion.Entry.Option option : entry) { + String suggestText = option.getText().string(); + System.out.println("返回结果:" + suggestText); + } + } + + } + } From 297ed8b53a4dd12052e01f4d5b8c3eabdbe8857d Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Mon, 30 Sep 2019 18:01:59 +0800 Subject: [PATCH 23/53] =?UTF-8?q?1.=E6=9B=B4=E6=96=B0=E8=AF=B4=E6=98=8E?= =?UTF-8?q?=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 6dd1725..258cea9 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,11 @@ - [JDK1.8的Lambda、Stream和日期的使用详解](https://www.cnblogs.com/xuwujing/p/10145691.html) +**ElasticSearch相关:** +- [ElasticSearch实战系列一: ElasticSearch集群+Kinaba安装教程](https://www.cnblogs.com/xuwujing/p/11385255.html) +- [ElasticSearch实战系列二: ElasticSearch的DSL语句使用教程---图文详解](https://www.cnblogs.com/xuwujing/p/11567053.html) + + **其他:** - [个人收集的资源分享](https://www.cnblogs.com/xuwujing/p/10393111.html) From d415a280e12ed28575c6ffc12d6ccf2f5365f325 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Wed, 9 Oct 2019 14:34:55 +0800 Subject: [PATCH 24/53] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0es=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E7=A4=BA=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../EsHighLevelRestSearchTest.java | 153 ++-- .../easticsearch/EsHighLevelRestTest1.java | 707 +++++++++++------- .../easticsearch/EsHighLevelRestTest2.java | 67 +- 3 files changed, 556 insertions(+), 371 deletions(-) diff --git a/src/main/java/com/pancm/easticsearch/EsHighLevelRestSearchTest.java b/src/main/java/com/pancm/easticsearch/EsHighLevelRestSearchTest.java index 9287e2b..532cdd3 100644 --- a/src/main/java/com/pancm/easticsearch/EsHighLevelRestSearchTest.java +++ b/src/main/java/com/pancm/easticsearch/EsHighLevelRestSearchTest.java @@ -49,7 +49,7 @@ */ public class EsHighLevelRestSearchTest { - private static String elasticIp = "192.169.2.98"; + private static String elasticIp = "192.169.0.23"; private static int elasticPort = 9200; private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestSearchTest.class); @@ -67,9 +67,11 @@ public static void main(String[] args) { genSearch(); orSearch(); likeSearch(); - inSearch(); +// inSearch(); existSearch(); rangeSearch(); + regexpSearch(); + boolSearch(); // search(); // search2(); @@ -80,7 +82,7 @@ public static void main(String[] args) { } /** - * p_test + * test1 *{ * "settings" : { * "number_of_shards" : 10, @@ -92,7 +94,7 @@ public static void main(String[] args) { * "uid" : { "type" : "long" }, * "phone" : { "type" : "long" }, * "userid" : { "type" : "keyword" }, - * "sendday" : { "type" : "long" }, + * * "message" : { "type" : "keyword" }, * "msgcode" : { "type" : "long" }, * "price" : { "type" : "double","index": "false" }, @@ -114,32 +116,94 @@ public static void main(String[] args) { } + /** * @Author pancm - * @Description 范围查询 + * @Description 组合查询 * @Date 2019/9/30 * @Param [] * @return void **/ - private static void rangeSearch() throws IOException{ + private static void boolSearch() throws IOException{ + String type = "_doc"; + String index = "test1"; // 查询指定的索引库 - SearchRequest searchRequest = new SearchRequest("user"); + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.types(type); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.must(QueryBuilders.termQuery("uid",12345)); + boolQueryBuilder.must(QueryBuilders.termQuery("msgcode",1)); + // 设置查询条件 + sourceBuilder.query(boolQueryBuilder); + searchRequest.source(sourceBuilder); + System.out.println("组合查询的DSL语句:"+sourceBuilder.toString()); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + // 结果 + searchResponse.getHits().forEach(hit -> { + String string = hit.getSourceAsString(); + System.out.println("组合查询的String结果:" + string); + }); + System.out.println("\n=================\n"); + } + /** + * @Author pancm + * @Description 正则查询 + * @Date 2019/9/30 + * @Param [] + * @return void + **/ + private static void regexpSearch() throws IOException{ + String type = "_doc"; + String index = "test1"; + // 查询指定的索引库 + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.types(type); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); // 设置查询条件 - sourceBuilder.query(QueryBuilders.rangeQuery("sendtime").gte("2019-01-01 00:00:00.000").lte("2019-12-31 23:59:59.999")); + sourceBuilder.query(QueryBuilders.regexpQuery("message","xu[0-9]")); searchRequest.source(sourceBuilder); + System.out.println("正则查询的DSL语句:"+sourceBuilder.toString()); // 同步查询 SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); // 结果 searchResponse.getHits().forEach(hit -> { Map map = hit.getSourceAsMap(); String string = hit.getSourceAsString(); - System.out.println("范围查询的Map结果:" + map); - System.out.println("范围查询的String结果:" + string); + System.out.println("正则查询的Map结果:" + map); + System.out.println("正则查询的String结果:" + string); }); + System.out.println("\n=================\n"); + } + /** + * @Author pancm + * @Description 范围查询 + * @Date 2019/9/30 + * @Param [] + * @return void + **/ + private static void rangeSearch() throws IOException{ + String type = "_doc"; + String index = "test1"; + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.types(type); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + + // 设置查询条件 + sourceBuilder.query(QueryBuilders.rangeQuery("sendtime").gte("2019-01-01 00:00:00").lte("2019-12-31 23:59:59")); + searchRequest.source(sourceBuilder); + System.out.println("范围查询的DSL语句:"+sourceBuilder.toString()); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + // 结果 + searchResponse.getHits().forEach(hit -> { + String string = hit.getSourceAsString(); + System.out.println("范围查询的String结果:" + string); + }); System.out.println("\n=================\n"); } @@ -151,13 +215,17 @@ private static void rangeSearch() throws IOException{ * @Param [] **/ private static void existSearch() throws IOException { + String type = "_doc"; + String index = "test1"; // 查询指定的索引库 - SearchRequest searchRequest = new SearchRequest("user"); + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.types(type); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); // 设置查询条件 - sourceBuilder.query(QueryBuilders.existsQuery("uid")); + sourceBuilder.query(QueryBuilders.existsQuery("msgcode")); searchRequest.source(sourceBuilder); + System.out.println("存在查询的DSL语句:"+sourceBuilder.toString()); // 同步查询 SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); // 结果 @@ -167,10 +235,7 @@ private static void existSearch() throws IOException { System.out.println("存在查询的Map结果:" + map); System.out.println("存在查询的String结果:" + string); }); - - System.out.println("\n=================\n"); - } /** @@ -181,15 +246,18 @@ private static void existSearch() throws IOException { * @Param [] **/ private static void inSearch() throws IOException { + String type = "_doc"; + String index = "test1"; // 查询指定的索引库 - SearchRequest searchRequest = new SearchRequest("user"); + SearchRequest searchRequest = new SearchRequest(index,type); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); /** * SELECT * FROM p_test where uid in (1,2) * */ // 设置查询条件 - sourceBuilder.query(QueryBuilders.termsQuery("uid", 1, 2)); +// sourceBuilder.query(QueryBuilders.termsQuery("uid", 1, 2)); searchRequest.source(sourceBuilder); + System.out.println("in查询的DSL语句:"+sourceBuilder.toString()); // 同步查询 SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); // 结果 @@ -200,7 +268,6 @@ private static void inSearch() throws IOException { System.out.println("in查询的String结果:" + string); }); - System.out.println("\n=================\n"); } @@ -212,25 +279,26 @@ private static void inSearch() throws IOException { * @Param [] **/ private static void likeSearch() throws IOException { + String type = "_doc"; + String index = "test1"; SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("p_test"); - searchRequest.types("_doc"); + searchRequest.indices(index); + searchRequest.types(type); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); /** - * SELECT * FROM p_test where message like '%pan%'; + * SELECT * FROM p_test where message like '%xu%'; * */ - boolQueryBuilder.must(QueryBuilders.wildcardQuery("message", "*pan*")); + boolQueryBuilder.must(QueryBuilders.wildcardQuery("message", "*xu*")); searchSourceBuilder.query(boolQueryBuilder); - System.out.println("模糊查询语句:" + searchSourceBuilder.toString()); searchRequest.source(searchSourceBuilder); // 同步查询 SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); searchResponse.getHits().forEach(documentFields -> { - System.out.println("查询结果:" + documentFields.getSourceAsMap()); + System.out.println("模糊查询结果:" + documentFields.getSourceAsMap()); }); - + System.out.println("\n=================\n"); } @@ -242,11 +310,14 @@ private static void likeSearch() throws IOException { * @Param [] **/ private static void genSearch() throws IOException { + String type = "_doc"; + String index = "test1"; // 查询指定的索引库 - SearchRequest searchRequest = new SearchRequest("user"); + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.types(type); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); // 设置查询条件 - sourceBuilder.query(QueryBuilders.termQuery("user", "pancm")); + sourceBuilder.query(QueryBuilders.termQuery("uid", "1234")); // 设置起止和结束 sourceBuilder.from(0); sourceBuilder.size(5); @@ -273,7 +344,7 @@ private static void genSearch() throws IOException { // sourceBuilder.fetchSource(includeFields, excludeFields); searchRequest.source(sourceBuilder); - + System.out.println("普通查询的DSL语句:"+sourceBuilder.toString()); // 同步查询 SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); @@ -291,18 +362,12 @@ private static void genSearch() throws IOException { for (ShardSearchFailure failure : searchResponse.getShardFailures()) { // failures should be handled here } - // 结果 searchResponse.getHits().forEach(hit -> { Map map = hit.getSourceAsMap(); - String string = hit.getSourceAsString(); - System.out.println("普通查询的Map结果:" + map); - System.out.println("普通查询的String结果:" + string); + System.out.println("普通查询的结果:" + map); }); - - System.out.println("\n=================\n"); - } private static void allSearch() throws IOException { @@ -316,7 +381,6 @@ private static void allSearch() throws IOException { searchRequestAll.source(searchSourceBuilder); // 同步查询 SearchResponse searchResponseAll = client.search(searchRequestAll, RequestOptions.DEFAULT); - System.out.println("所有查询总数:" + searchResponseAll.getHits().getTotalHits()); } @@ -329,30 +393,27 @@ private static void allSearch() throws IOException { **/ private static void orSearch() throws IOException { SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("p_test"); + searchRequest.indices("test1"); searchRequest.types("_doc"); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); BoolQueryBuilder boolQueryBuilder2 = new BoolQueryBuilder(); /** - * SELECT * FROM p_test where (uid = 1 or uid =2) and phone = 12345678919 + * SELECT * FROM test1 where (uid = 1234 or uid =12345) and phone = 12345678909 * */ - boolQueryBuilder2.should(QueryBuilders.termQuery("uid", 1)); - boolQueryBuilder2.should(QueryBuilders.termQuery("uid", 2)); + boolQueryBuilder2.should(QueryBuilders.termQuery("uid", 1234)); + boolQueryBuilder2.should(QueryBuilders.termQuery("uid", 12345)); boolQueryBuilder.must(boolQueryBuilder2); - boolQueryBuilder.must(QueryBuilders.termQuery("phone", "12345678919")); + boolQueryBuilder.must(QueryBuilders.termQuery("phone", "12345678909")); searchSourceBuilder.query(boolQueryBuilder); - System.out.println("或查询语句:" + searchSourceBuilder.toString()); searchRequest.source(searchSourceBuilder); // 同步查询 SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); - searchResponse.getHits().forEach(documentFields -> { - - System.out.println("查询结果:" + documentFields.getSourceAsMap()); + System.out.println("或查询结果:" + documentFields.getSourceAsMap()); }); - + System.out.println("\n=================\n"); } /* diff --git a/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest1.java b/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest1.java index ce6a4ac..ce36442 100644 --- a/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest1.java +++ b/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest1.java @@ -1,27 +1,23 @@ -package com.pancm.sql.easticsearch; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.function.BiConsumer; +package com.pancm.easticsearch; import org.apache.http.HttpHost; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.bulk.BackoffPolicy; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkProcessor; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.bulk.*; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.action.update.UpdateRequest; @@ -35,14 +31,28 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.index.reindex.ScrollableHitSource; +import org.elasticsearch.index.reindex.UpdateByQueryRequest; import org.elasticsearch.rest.RestStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; + /** * @Title: EsHighLevelRestTest1 - * @Description: Java High Level REST Client Es高级客户端使用教程一 (基本CRUD使用) - 官方文档地址: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high.html + * @Description: Java High Level REST Client Es高级客户端使用教程一 (基本CRUD使用) 官方文档地址: + * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high.html * @since jdk 1.8 * @Version:1.0.0 * @author pancm @@ -54,18 +64,21 @@ public class EsHighLevelRestTest1 { private static int elasticPort = 9200; private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestTest1.class); - + private static RestHighLevelClient client = null; public static void main(String[] args) { try { init(); - careatindex(); - get(); + createIndex(); + insert(); + queryById(); exists(); update(); - delete(); - bulk(); +// deleteByQuery(); +// deleteIndex(); +// delete(); +// bulk(); close(); } catch (Exception e) { e.printStackTrace(); @@ -73,59 +86,29 @@ public static void main(String[] args) { } - /* - * 初始化服务 - */ - private static void init() { - client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort, "http"))); - - } - - /* - * 关闭服务 - */ - private static void close() { - if (client != null) { - try { - client.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - /** - * 创建索引 - * - * @throws IOException - */ - private static void careatindex() throws IOException { - String index = "user"; - String type = "userindex"; + private static void insert() throws IOException { + String index = "test1"; + String type = "_doc"; // 唯一编号 String id = "1"; - IndexRequest request = new IndexRequest(index, type, id); - /* * 第一种方式，通过jsonString进行创建 */ // json - String jsonString = "{" + "\"user\":\"pancm\"," + "\"postDate\":\"2019-03-08\","+ "\"age\":\"18\"," - + "\"message\":\"study Elasticsearch\"" + "}"; - + String jsonString = "{" + "\"uid\":\"1234\","+ "\"phone\":\"12345678909\","+ "\"msgcode\":\"1\"," + "\"sendtime\":\"2019-03-14 01:57:04\"," + + "\"message\":\"xuwujing study Elasticsearch\"" + "}"; request.source(jsonString, XContentType.JSON); /* * 第二种方式，通过map创建,，会自动转换成json的数据 */ - Map jsonMap = new HashMap<>(); - jsonMap.put("user", "pancm"); - jsonMap.put("postDate", "2019-03-08"); - jsonMap.put("age", "18"); - jsonMap.put("message", "study Elasticsearch"); - + jsonMap.put("uid", 1234); + jsonMap.put("phone", 12345678909L); + jsonMap.put("msgcode", 1); + jsonMap.put("sendtime", "2019-03-14 01:57:04"); + jsonMap.put("message", "xuwujing study Elasticsearch"); request.source(jsonMap); /* @@ -135,20 +118,23 @@ private static void careatindex() throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); { - builder.field("user", "pancm"); - builder.timeField("postDate", "2019-03-08"); - builder.field("age", "18"); - builder.field("message", "study Elasticsearch"); + builder.field("uid", 1234); + builder.field("phone", 12345678909L); + builder.field("msgcode", 1); + builder.timeField("sendtime", "2019-03-14 01:57:04"); + builder.field("message", "xuwujing study Elasticsearch"); } builder.endObject(); request.source(builder); IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT); - - - - //对响应结果进行处理 - + + //如果是200则表示成功，否则就是失败 + if(200 == indexResponse.status().getStatus()){ + + } + + // 对响应结果进行处理 String index1 = indexResponse.getIndex(); String type1 = indexResponse.getType(); String id1 = indexResponse.getId(); @@ -168,8 +154,113 @@ private static void careatindex() throws IOException { String reason = failure.reason(); } } + } + + /* + * 初始化服务 + */ + private static void init() { + + client = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticIp, elasticPort))); + + } + + /* + * 关闭服务 + */ + private static void close() { + if (client != null) { + try { + client.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + /** + * 创建索引 + * + * @throws IOException + */ + private static void createIndex() throws IOException { + + // 类型 + String type = "_doc"; + String index = "test1"; + // setting 的值 + Map setmapping = new HashMap<>(); + // 分区数、副本数、缓存刷新时间 + setmapping.put("number_of_shards", 10); + setmapping.put("number_of_replicas", 1); + setmapping.put("refresh_interval", "5s"); + Map keyword = new HashMap<>(); + //设置类型 + keyword.put("type", "keyword"); + Map lon = new HashMap<>(); + //设置类型 + lon.put("type", "long"); + Map date = new HashMap<>(); + //设置类型 + date.put("type", "date"); + date.put("format", "yyyy-MM-dd HH:mm:ss"); + + Map jsonMap2 = new HashMap<>(); + Map properties = new HashMap<>(); + //设置字段message信息 + properties.put("uid", lon); + properties.put("phone", lon); + properties.put("msgcode", lon); + properties.put("message", keyword); + properties.put("sendtime", date); + Map mapping = new HashMap<>(); + mapping.put("properties", properties); + jsonMap2.put(type, mapping); + + GetIndexRequest getRequest = new GetIndexRequest(); + getRequest.indices(index); + getRequest.types(type); + getRequest.local(false); + getRequest.humanReadable(true); + boolean exists2 = client.indices().exists(getRequest, RequestOptions.DEFAULT); + //如果存在就不创建了 + if(exists2) { + System.out.println(index+"索引库已经存在!"); + return; + } + // 开始创建库 + CreateIndexRequest request = new CreateIndexRequest(index); + try { + // 加载数据类型 + request.settings(setmapping); + //设置mapping参数 + request.mapping(type, jsonMap2); + //设置别名 + request.alias(new Alias("pancm_alias")); + CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); + boolean falg = createIndexResponse.isAcknowledged(); + if(falg){ + System.out.println("创建索引库:"+index+"成功！" ); + } + } catch (IOException e) { + e.printStackTrace(); + } + + } + + + /** + * 删除索引 + * + * @throws IOException + */ + private static void deleteIndex() throws IOException { + String index = "userindex"; + DeleteIndexRequest request = new DeleteIndexRequest(index); + // 同步删除 + client.indices().delete(request,RequestOptions.DEFAULT); + System.out.println("删除索引库成功！"+index); - System.out.println("创建成功！"); } /** @@ -177,9 +268,9 @@ private static void careatindex() throws IOException { * * @throws IOException */ - private static void get() { - String index = "user"; - String type = "userindex"; + private static void queryById() { + String type = "_doc"; + String index = "test1"; // 唯一编号 String id = "1"; // 创建查询请求 @@ -194,7 +285,7 @@ private static void get() { } catch (ElasticsearchException e) { // 如果是索引不存在 if (e.status() == RestStatus.NOT_FOUND) { - System.out.println("该索引库不存在！"+index); + System.out.println("该索引库不存在！" + index); } } @@ -204,272 +295,356 @@ private static void get() { String sourceAsString = getResponse.getSourceAsString(); Map sourceAsMap = getResponse.getSourceAsMap(); byte[] sourceAsBytes = getResponse.getSourceAsBytes(); - System.out.println("查询返回结果String:"+sourceAsString); - System.out.println("查询返回结果Map:"+sourceAsMap); + System.out.println("查询返回结果String:" + sourceAsString); + System.out.println("查询返回结果Map:" + sourceAsMap); } else { System.out.println("没有找到该数据！"); } } - - + /** - * 是否存在 - * @throws IOException + * 是否存在 + * + * @throws IOException */ private static void exists() throws IOException { - String index = "user"; - String type = "userindex"; + String type = "_doc"; + String index = "test1"; // 唯一编号 String id = "1"; // 创建查询请求 GetRequest getRequest = new GetRequest(index, type, id); + boolean exists = client.exists(getRequest, RequestOptions.DEFAULT); - ActionListener listener = new ActionListener() { - @Override - public void onResponse(Boolean exists) { - System.out.println("=="+exists); - } - - @Override - public void onFailure(Exception e) { - System.out.println("失败的原因："+e.getMessage()); - } + @Override + public void onResponse(Boolean exists) { + System.out.println("==" + exists); + } + + @Override + public void onFailure(Exception e) { + System.out.println("失败的原因：" + e.getMessage()); + } }; - //进行异步监听 + // 进行异步监听 // client.existsAsync(getRequest, RequestOptions.DEFAULT, listener); - - System.out.println("是否存在："+exists); + + System.out.println("数据是否存在：" + exists); } - - + /** - * 更新操作 + * 更新操作 + * * @throws IOException */ private static void update() throws IOException { - String index = "user"; - String type = "userindex"; + String type = "_doc"; + String index = "test1"; // 唯一编号 String id = "1"; - UpdateRequest upateRequest=new UpdateRequest(); + UpdateRequest upateRequest = new UpdateRequest(); upateRequest.id(id); upateRequest.index(index); upateRequest.type(type); - - //依旧可以使用Map这种集合作为更新条件 + + // 依旧可以使用Map这种集合作为更新条件 Map jsonMap = new HashMap<>(); - jsonMap.put("user", "xuwujing"); - jsonMap.put("postDate", "2019-03-11"); - + jsonMap.put("uid", 12345); + jsonMap.put("phone", 123456789019L); + jsonMap.put("msgcode", 2); + jsonMap.put("sendtime", "2019-03-14 01:57:04"); + jsonMap.put("message", "xuwujing study Elasticsearch"); upateRequest.doc(jsonMap); - - // - upateRequest.docAsUpsert(true); // upsert 方法表示如果数据不存在，那么就新增一条 - upateRequest.upsert(jsonMap); - + upateRequest.docAsUpsert(true); client.update(upateRequest, RequestOptions.DEFAULT); System.out.println("更新成功！"); - + } - + + + + /** + * 根据查询条件更新 + * + * @throws IOException + */ + private static void updateByQuery() throws IOException { + String type = "_doc"; + String index = "test1"; + // + UpdateByQueryRequest request = new UpdateByQueryRequest(index,type); + // 设置查询条件 + request.setQuery(new TermQueryBuilder("user", "pancm")); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + + // 设置复制文档的数量 + request.setSize(10); + // 设置一次批量处理的条数，默认是1000 + request.setBatchSize(100); + //设置超时时间 + request.setTimeout(TimeValue.timeValueMinutes(2)); + //索引选项 + request.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); + + // 同步执行 + BulkByScrollResponse bulkResponse = client.updateByQuery(request, RequestOptions.DEFAULT); + + // 异步执行 +// client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener); + + // 返回结果 + TimeValue timeTaken = bulkResponse.getTook(); + boolean timedOut = bulkResponse.isTimedOut(); + long totalDocs = bulkResponse.getTotal(); + long updatedDocs = bulkResponse.getUpdated(); + long deletedDocs = bulkResponse.getDeleted(); + long batches = bulkResponse.getBatches(); + long noops = bulkResponse.getNoops(); + long versionConflicts = bulkResponse.getVersionConflicts(); + long bulkRetries = bulkResponse.getBulkRetries(); + long searchRetries = bulkResponse.getSearchRetries(); + TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); + TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); + List searchFailures = bulkResponse.getSearchFailures(); + List bulkFailures = bulkResponse.getBulkFailures(); + System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",更新数:" + updatedDocs); + + } + /** * 删除 - * @throws IOException + * + * @throws IOException * */ private static void delete() throws IOException { - String index = "user"; - String type = "userindex"; + String type = "_doc"; + String index = "test1"; // 唯一编号 String id = "1"; - DeleteRequest deleteRequest=new DeleteRequest(); + DeleteRequest deleteRequest = new DeleteRequest(); deleteRequest.id(id); deleteRequest.index(index); deleteRequest.type(type); - - //设置超时时间 - deleteRequest.timeout(TimeValue.timeValueMinutes(2)); - //设置刷新策略"wait_for" - //保持此请求打开，直到刷新使此请求的内容可以搜索为止。此刷新策略与高索引和搜索吞吐量兼容，但它会导致请求等待响应，直到发生刷新 + // 设置超时时间 + deleteRequest.timeout(TimeValue.timeValueMinutes(2)); + // 设置刷新策略"wait_for" + // 保持此请求打开，直到刷新使此请求的内容可以搜索为止。此刷新策略与高索引和搜索吞吐量兼容，但它会导致请求等待响应，直到发生刷新 deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); - - //同步删除 + // 同步删除 DeleteResponse deleteResponse = client.delete(deleteRequest, RequestOptions.DEFAULT); - + /* - * 异步删除操作 + * 异步删除操作 */ - - //进行监听 + + // 进行监听 ActionListener listener = new ActionListener() { - @Override - public void onResponse(DeleteResponse deleteResponse) { - System.out.println("响应:"+deleteResponse); - } - - @Override - public void onFailure(Exception e) { - System.out.println("删除监听异常:"+e.getMessage()); - } + @Override + public void onResponse(DeleteResponse deleteResponse) { + System.out.println("响应:" + deleteResponse); + } + + @Override + public void onFailure(Exception e) { + System.out.println("删除监听异常:" + e.getMessage()); + } }; - - //异步删除 + + // 异步删除 // client.deleteAsync(deleteRequest, RequestOptions.DEFAULT, listener); - + ReplicationResponse.ShardInfo shardInfo = deleteResponse.getShardInfo(); - //如果处理成功碎片的数量少于总碎片的情况,说明还在处理或者处理发生异常 + // 如果处理成功碎片的数量少于总碎片的情况,说明还在处理或者处理发生异常 if (shardInfo.getTotal() != shardInfo.getSuccessful()) { - System.out.println("需要处理的碎片总量:"+shardInfo.getTotal()); - System.out.println("处理成功的碎片总量:"+shardInfo.getSuccessful()); + System.out.println("需要处理的碎片总量:" + shardInfo.getTotal()); + System.out.println("处理成功的碎片总量:" + shardInfo.getSuccessful()); } - + if (shardInfo.getFailed() > 0) { - for (ReplicationResponse.ShardInfo.Failure failure : - shardInfo.getFailures()) { - String reason = failure.reason(); - } + for (ReplicationResponse.ShardInfo.Failure failure : shardInfo.getFailures()) { + String reason = failure.reason(); + } } System.out.println("删除成功!"); } - - + + /** - * 批量操作示例 - * @throws InterruptedException + * 根据查询条件删除 + * + * @throws IOException */ - private static void bulk() throws IOException, InterruptedException { + private static void deleteByQuery() throws IOException { + String type = "_doc"; + String index = "test1"; + DeleteByQueryRequest request = new DeleteByQueryRequest(index,type); + // 设置查询条件 + request.setQuery(QueryBuilders.termQuery("uid",1234)); + // 同步执行 + BulkByScrollResponse bulkResponse = client.deleteByQuery(request, RequestOptions.DEFAULT); + + // 异步执行 +// client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener); + + // 返回结果 + TimeValue timeTaken = bulkResponse.getTook(); + boolean timedOut = bulkResponse.isTimedOut(); + long totalDocs = bulkResponse.getTotal(); + long updatedDocs = bulkResponse.getUpdated(); + long deletedDocs = bulkResponse.getDeleted(); + long batches = bulkResponse.getBatches(); + long noops = bulkResponse.getNoops(); + long versionConflicts = bulkResponse.getVersionConflicts(); + long bulkRetries = bulkResponse.getBulkRetries(); + long searchRetries = bulkResponse.getSearchRetries(); + TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); + TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); + List searchFailures = bulkResponse.getSearchFailures(); + List bulkFailures = bulkResponse.getBulkFailures(); + System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",更新数:" + updatedDocs); + + } + + /** + * 批量操作示例 + * + * @throws InterruptedException + */ + private static void bulk() throws IOException, InterruptedException { String index = "estest"; String type = "estest"; - - BulkRequest request = new BulkRequest(); - //批量新增 - request.add(new IndexRequest(index, type, "1") - .source(XContentType.JSON,"field", "foo")); - request.add(new IndexRequest(index, type, "2") - .source(XContentType.JSON,"field", "bar")); - request.add(new IndexRequest(index, type, "3") - .source(XContentType.JSON,"field", "baz")); - - //可以进行修改/删除/新增 操作 - request.add(new UpdateRequest(index, type, "2") - .doc(XContentType.JSON,"field", "test")); - request.add(new DeleteRequest(index, type, "3")); - request.add(new IndexRequest(index, type, "4") - .source(XContentType.JSON,"field", "baz")); - - + + BulkRequest request = new BulkRequest(); + // 批量新增,存在会直接覆盖 + request.add(new IndexRequest(index, type, "1").source(XContentType.JSON, "field", "foo")); + request.add(new IndexRequest(index, type, "2").source(XContentType.JSON, "field", "bar")); + request.add(new IndexRequest(index, type, "3").source(XContentType.JSON, "field", "baz")); + + // 可以进行修改/删除/新增 操作 + //docAsUpsert 为true表示存在更新，不存在插入，为false表示不存在就是不做更新 + request.add(new UpdateRequest(index, type, "2").doc(XContentType.JSON, "field", "test").docAsUpsert(true)); + request.add(new DeleteRequest(index, type, "3")); + request.add(new IndexRequest(index, type, "4").source(XContentType.JSON, "field", "baz")); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); - - //可以快速检查一个或多个操作是否失败 true是有至少一个失败！ - if (bulkResponse.hasFailures()) { + + + ActionListener listener3 = new ActionListener() { + @Override + public void onResponse(BulkResponse response) { + System.out.println("===="+response.buildFailureMessage()); + } + + @Override + public void onFailure(Exception e) { + System.out.println("====---"+e.getMessage()); + } + }; + + client.bulkAsync(request, RequestOptions.DEFAULT,listener3); + + // 可以快速检查一个或多个操作是否失败 true是有至少一个失败！ + if (bulkResponse.hasFailures()) { System.out.println("有一个操作失败!"); } - - //对处理结果进行遍历操作并根据不同的操作进行处理 - for (BulkItemResponse bulkItemResponse : bulkResponse) { - DocWriteResponse itemResponse = bulkItemResponse.getResponse(); - - //操作失败的进行处理 - if (bulkItemResponse.isFailed()) { - BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); - - } - - if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.INDEX - || bulkItemResponse.getOpType() == DocWriteRequest.OpType.CREATE) { - IndexResponse indexResponse = (IndexResponse) itemResponse; - - } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.UPDATE) { - UpdateResponse updateResponse = (UpdateResponse) itemResponse; - - } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.DELETE) { - DeleteResponse deleteResponse = (DeleteResponse) itemResponse; - } - } - + + // 对处理结果进行遍历操作并根据不同的操作进行处理 + for (BulkItemResponse bulkItemResponse : bulkResponse) { + DocWriteResponse itemResponse = bulkItemResponse.getResponse(); + + // 操作失败的进行处理 + if (bulkItemResponse.isFailed()) { + BulkItemResponse.Failure failure = bulkItemResponse.getFailure(); + + } + + if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.INDEX + || bulkItemResponse.getOpType() == DocWriteRequest.OpType.CREATE) { + IndexResponse indexResponse = (IndexResponse) itemResponse; + System.out.println("新增失败!"+indexResponse.toString()); + + } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.UPDATE) { + UpdateResponse updateResponse = (UpdateResponse) itemResponse; + System.out.println("更新失败!"+updateResponse.toString()); + + } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.DELETE) { + DeleteResponse deleteResponse = (DeleteResponse) itemResponse; + System.out.println("删除失败!"+deleteResponse.toString()); + + } + } + System.out.println("批量执行成功！"); - - - + /* - * 批量执行处理器相关示例代码 + * 批量执行处理器相关示例代码 */ - - - //批量处理器的监听器设置 - + + // 批量处理器的监听器设置 + BulkProcessor.Listener listener = new BulkProcessor.Listener() { - - //在执行BulkRequest的每次执行之前调用，这个方法允许知道将要在BulkRequest中执行的操作的数量 - @Override - public void beforeBulk(long executionId, BulkRequest request) { - int numberOfActions = request.numberOfActions(); - logger.debug("Executing bulk [{}] with {} requests", - executionId, numberOfActions); - } - - //在每次执行BulkRequest之后调用，这个方法允许知道BulkResponse是否包含错误 - @Override - public void afterBulk(long executionId, BulkRequest request, - BulkResponse response) { - if (response.hasFailures()) { - logger.warn("Bulk [{}] executed with failures", executionId); - } else { - logger.debug("Bulk [{}] completed in {} milliseconds", - executionId, response.getTook().getMillis()); - } - } - - //如果BulkRequest失败，则调用该方法，该方法允许知道失败 - @Override - public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - logger.error("Failed to execute bulk", failure); - } + + // 在执行BulkRequest的每次执行之前调用，这个方法允许知道将要在BulkRequest中执行的操作的数量 + @Override + public void beforeBulk(long executionId, BulkRequest request) { + int numberOfActions = request.numberOfActions(); + logger.debug("Executing bulk [{}] with {} requests", executionId, numberOfActions); + } + + // 在每次执行BulkRequest之后调用，这个方法允许知道BulkResponse是否包含错误 + @Override + public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + if (response.hasFailures()) { + logger.warn("Bulk [{}] executed with failures", executionId); + } else { + logger.debug("Bulk [{}] completed in {} milliseconds", executionId, response.getTook().getMillis()); + } + } + + // 如果BulkRequest失败，则调用该方法，该方法允许知道失败 + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + logger.error("Failed to execute bulk", failure); + } }; - - BiConsumer> bulkConsumer = - (request2, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener); - //创建一个批量执行的处理器 - BulkProcessor bulkProcessor = BulkProcessor.builder(bulkConsumer, listener).build(); + + BiConsumer> bulkConsumer = (request2, bulkListener) -> client + .bulkAsync(request, RequestOptions.DEFAULT, bulkListener); + // 创建一个批量执行的处理器 + BulkProcessor bulkProcessor = BulkProcessor.builder(bulkConsumer, listener).build(); BulkProcessor.Builder builder = BulkProcessor.builder(bulkConsumer, listener); - //根据当前添加的操作数量设置刷新新批量请求的时间(默认为1000，使用-1禁用它) - builder.setBulkActions(500); - //根据当前添加的操作大小设置刷新新批量请求的时间(默认为5Mb，使用-1禁用) - builder.setBulkSize(new ByteSizeValue(1L, ByteSizeUnit.MB)); - //设置允许执行的并发请求数量(默认为1，使用0只允许执行单个请求) - builder.setConcurrentRequests(0); - //设置刷新间隔如果间隔通过，则刷新任何挂起的BulkRequest(默认为未设置) - builder.setFlushInterval(TimeValue.timeValueSeconds(10L)); - //设置一个常量后退策略，该策略最初等待1秒并重试最多3次。 - builder.setBackoffPolicy(BackoffPolicy - .constantBackoff(TimeValue.timeValueSeconds(1L), 3)); - - - IndexRequest one = new IndexRequest(index, type, "1"). - source(XContentType.JSON, "title", - "In which order are my Elasticsearch queries executed?"); - IndexRequest two = new IndexRequest(index, type, "2") - .source(XContentType.JSON, "title", - "Current status and upcoming changes in Elasticsearch"); - IndexRequest three = new IndexRequest(index, type, "3") - .source(XContentType.JSON, "title", - "The Future of Federated Search in Elasticsearch"); + // 根据当前添加的操作数量设置刷新新批量请求的时间(默认为1000，使用-1禁用它) + builder.setBulkActions(500); + // 根据当前添加的操作大小设置刷新新批量请求的时间(默认为5Mb，使用-1禁用) + builder.setBulkSize(new ByteSizeValue(1L, ByteSizeUnit.MB)); + // 设置允许执行的并发请求数量(默认为1，使用0只允许执行单个请求) + builder.setConcurrentRequests(0); + // 设置刷新间隔如果间隔通过，则刷新任何挂起的BulkRequest(默认为未设置) + builder.setFlushInterval(TimeValue.timeValueSeconds(10L)); + // 设置一个常量后退策略，该策略最初等待1秒并重试最多3次。 + builder.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3)); + + IndexRequest one = new IndexRequest(index, type, "1").source(XContentType.JSON, "title", + "In which order are my Elasticsearch queries executed?"); + IndexRequest two = new IndexRequest(index, type, "2").source(XContentType.JSON, "title", + "Current status and upcoming changes in Elasticsearch"); + IndexRequest three = new IndexRequest(index, type, "3").source(XContentType.JSON, "title", + "The Future of Federated Search in Elasticsearch"); bulkProcessor.add(one); bulkProcessor.add(two); bulkProcessor.add(three); - - - //如果所有大容量请求都已完成，则该方法返回true;如果在所有大容量请求完成之前的等待时间已经过去，则返回false - boolean terminated = bulkProcessor.awaitClose(30L, TimeUnit.SECONDS); - - System.out.println("请求的响应结果:"+terminated); - - + + // 如果所有大容量请求都已完成，则该方法返回true;如果在所有大容量请求完成之前的等待时间已经过去，则返回false + boolean terminated = bulkProcessor.awaitClose(30L, TimeUnit.SECONDS); + + System.out.println("请求的响应结果:" + terminated); + } - + } diff --git a/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest2.java b/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest2.java index 595249a..d76fbad 100644 --- a/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest2.java +++ b/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest2.java @@ -1,11 +1,5 @@ package com.pancm.easticsearch; -import static org.junit.Assert.assertNull; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - import org.apache.http.HttpHost; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; @@ -25,11 +19,16 @@ import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.index.reindex.ReindexRequest; import org.elasticsearch.index.reindex.ScrollableHitSource; -import org.elasticsearch.index.reindex.UpdateByQueryRequest; import org.elasticsearch.tasks.TaskId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertNull; + /** * @Title: EsHighLevelRestTest2 * @Description: Java High Level REST Client Es高级客户端使用教程二 (关于组合使用) @@ -55,7 +54,6 @@ public static void main(String[] args) { init(); multiGet(); reindex(); - updataByQuery(); deleteByQuery(); rethrottleByQuery(); close(); @@ -174,7 +172,7 @@ private static void reindex() throws IOException { BulkByScrollResponse bulkResponse = client.reindex(request, RequestOptions.DEFAULT); // 异步执行 -// client.reindexAsync(request, RequestOptions.DEFAULT, listener); +// client.reindexAsync(request, RequestOptions.DEFAULT, listener); // 响应结果处理 @@ -199,56 +197,7 @@ private static void reindex() throws IOException { } - - /** - * 根据查询条件更新 - * - * @throws IOException - */ - private static void updataByQuery() throws IOException { - // - UpdateByQueryRequest request = new UpdateByQueryRequest("user"); - - // 设置查询条件 - request.setQuery(new TermQueryBuilder("user", "pancm")); - - // 设置复制文档的数量 - request.setSize(10); - // 设置一次批量处理的条数，默认是1000 - request.setBatchSize(100); - //设置路由 - request.setRouting("=cat"); - //设置超时时间 - request.setTimeout(TimeValue.timeValueMinutes(2)); - //允许刷新 - request.setRefresh(true); - //索引选项 - request.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); - - // 同步执行 - BulkByScrollResponse bulkResponse = client.updateByQuery(request, RequestOptions.DEFAULT); - // 异步执行 -// client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener); - - // 返回结果 - TimeValue timeTaken = bulkResponse.getTook(); - boolean timedOut = bulkResponse.isTimedOut(); - long totalDocs = bulkResponse.getTotal(); - long updatedDocs = bulkResponse.getUpdated(); - long deletedDocs = bulkResponse.getDeleted(); - long batches = bulkResponse.getBatches(); - long noops = bulkResponse.getNoops(); - long versionConflicts = bulkResponse.getVersionConflicts(); - long bulkRetries = bulkResponse.getBulkRetries(); - long searchRetries = bulkResponse.getSearchRetries(); - TimeValue throttledMillis = bulkResponse.getStatus().getThrottled(); - TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil(); - List searchFailures = bulkResponse.getSearchFailures(); - List bulkFailures = bulkResponse.getBulkFailures(); - System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",更新数:" + updatedDocs); - - } /** @@ -332,7 +281,7 @@ public void onFailure(Exception e) { client.reindexRethrottleAsync(request, RequestOptions.DEFAULT, listener); client.updateByQueryRethrottleAsync(request, RequestOptions.DEFAULT, listener); client.deleteByQueryRethrottleAsync(request, RequestOptions.DEFAULT, listener); - + System.out.println("已成功设置!"); } From da5da2a9cba96841249e2ba2948286acea496cc9 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Fri, 11 Oct 2019 16:01:28 +0800 Subject: [PATCH 25/53] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0=E5=8D=9A=E6=96=87?= =?UTF-8?q?=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 258cea9..90b7ef7 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,8 @@ **ElasticSearch相关:** - [ElasticSearch实战系列一: ElasticSearch集群+Kinaba安装教程](https://www.cnblogs.com/xuwujing/p/11385255.html) - [ElasticSearch实战系列二: ElasticSearch的DSL语句使用教程---图文详解](https://www.cnblogs.com/xuwujing/p/11567053.html) +- [ElasticSearch实战系列三: ElasticSearch的JAVA API使用教程](https://www.cnblogs.com/xuwujing/p/11645630.html) + **其他:** From 6f9dc96c03709e92d857f46436563393f38bb7ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2019 14:54:57 +0000 Subject: [PATCH 26/53] Bump netty-all from 4.1.17.Final to 4.1.42.Final Bumps [netty-all](https://github.com/netty/netty) from 4.1.17.Final to 4.1.42.Final. - [Release notes](https://github.com/netty/netty/releases) - [Commits](https://github.com/netty/netty/compare/netty-4.1.17.Final...netty-4.1.42.Final) Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bde2feb..3d598b2 100644 --- a/pom.xml +++ b/pom.xml @@ -233,7 +233,7 @@ io.netty netty-all - 4.1.17.Final + 4.1.42.Final From 89e43ad0cdd4f85d41aae33a18b8bb63f14311d7 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Thu, 24 Oct 2019 17:47:05 +0800 Subject: [PATCH 27/53] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0=E5=8D=9A=E6=96=87?= =?UTF-8?q?=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 90b7ef7..856dce7 100644 --- a/README.md +++ b/README.md @@ -61,10 +61,18 @@ - [观察者模式和空对象模式](https://www.cnblogs.com/xuwujing/p/10036204.html) - [总结篇](https://www.cnblogs.com/xuwujing/p/10134494.html) -**进阶相关:** +**JAVA进阶相关:** - [JDK1.8的Lambda、Stream和日期的使用详解](https://www.cnblogs.com/xuwujing/p/10145691.html) + +**大数据相关:** +- [大数据学习系列之三 ----- HBase Java Api 图文详解](https://www.cnblogs.com/xuwujing/p/8039175.html) +- [Kafka 使用Java实现数据的生产和消费demo](https://www.cnblogs.com/xuwujing/p/8371127.html) +- [关于Kafka 的 consumer 消费者手动提交详解](https://www.cnblogs.com/xuwujing/p/8432984.html) +- [Storm 入门的Demo教程](https://www.cnblogs.com/xuwujing/p/8584684.html) + + **ElasticSearch相关:** - [ElasticSearch实战系列一: ElasticSearch集群+Kinaba安装教程](https://www.cnblogs.com/xuwujing/p/11385255.html) - [ElasticSearch实战系列二: ElasticSearch的DSL语句使用教程---图文详解](https://www.cnblogs.com/xuwujing/p/11567053.html) @@ -72,8 +80,13 @@ -**其他:** +**其他博客:** +- [两年JAVA程序员的面试总结](https://www.cnblogs.com/xuwujing/p/7613084.html) +- [一个两年java程序猿的2017个人总结](https://www.cnblogs.com/xuwujing/p/8158716.html) +- [写了一年的博客，我收获了什么](https://www.cnblogs.com/xuwujing/p/8747769.html) +- [给刚工作不久的程序猿同学的一封信](https://www.cnblogs.com/xuwujing/p/9665966.html) +- [一个平凡但不平庸的程序猿2018个人总结](https://www.cnblogs.com/xuwujing/p/9665966.html) - [个人收集的资源分享](https://www.cnblogs.com/xuwujing/p/10393111.html) ## 其他 From 38ab3d6248c735bd4c5cfdec653b18de9a0ddd26 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Mon, 4 Nov 2019 08:53:08 +0800 Subject: [PATCH 28/53] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0=E5=8D=9A=E6=96=87?= =?UTF-8?q?=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 856dce7..f675f50 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,7 @@ - [给刚工作不久的程序猿同学的一封信](https://www.cnblogs.com/xuwujing/p/9665966.html) - [一个平凡但不平庸的程序猿2018个人总结](https://www.cnblogs.com/xuwujing/p/9665966.html) - [个人收集的资源分享](https://www.cnblogs.com/xuwujing/p/10393111.html) +- [一个毕业三年的程序猿对于提升自我的一些建议](https://www.cnblogs.com/xuwujing/p/11735726.html) ## 其他 From e587757535d368b8e70d624b18731640d899e666 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Mon, 4 Nov 2019 08:58:47 +0800 Subject: [PATCH 29/53] =?UTF-8?q?1.=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../EsAggregationSearchTest.java | 4 ++-- .../EsHighLevelRestSearchTest.java | 2 +- .../{easticsearch => elasticsearch}/EsHighLevelRestTest1.java | 2 +- .../{easticsearch => elasticsearch}/EsHighLevelRestTest2.java | 2 +- .../com/pancm/{easticsearch => elasticsearch}/EsUtil.java | 4 ++-- .../com/pancm/{easticsearch => elasticsearch}/JestTest.java | 0 .../pancm/{easticsearch => elasticsearch}/package-info.java | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) rename src/main/java/com/pancm/{easticsearch => elasticsearch}/EsAggregationSearchTest.java (98%) rename src/main/java/com/pancm/{easticsearch => elasticsearch}/EsHighLevelRestSearchTest.java (99%) rename src/main/java/com/pancm/{easticsearch => elasticsearch}/EsHighLevelRestTest1.java (99%) rename src/main/java/com/pancm/{easticsearch => elasticsearch}/EsHighLevelRestTest2.java (99%) rename src/main/java/com/pancm/{easticsearch => elasticsearch}/EsUtil.java (99%) rename src/main/java/com/pancm/{easticsearch => elasticsearch}/JestTest.java (100%) rename src/main/java/com/pancm/{easticsearch => elasticsearch}/package-info.java (79%) diff --git a/src/main/java/com/pancm/easticsearch/EsAggregationSearchTest.java b/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java similarity index 98% rename from src/main/java/com/pancm/easticsearch/EsAggregationSearchTest.java rename to src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java index df86c74..440764a 100644 --- a/src/main/java/com/pancm/easticsearch/EsAggregationSearchTest.java +++ b/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java @@ -1,4 +1,4 @@ -package com.pancm.easticsearch; +package com.pancm.elasticsearch; import org.apache.http.HttpHost; import org.elasticsearch.action.search.SearchRequest; @@ -38,7 +38,7 @@ public class EsAggregationSearchTest { private static String elasticIp = "192.169.0.23"; private static int elasticPort = 9200; - private static Logger logger = LoggerFactory.getLogger(com.pancm.sql.easticsearch.EsHighLevelRestSearchTest.class); + private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestSearchTest.class); private static RestHighLevelClient client = null; diff --git a/src/main/java/com/pancm/easticsearch/EsHighLevelRestSearchTest.java b/src/main/java/com/pancm/elasticsearch/EsHighLevelRestSearchTest.java similarity index 99% rename from src/main/java/com/pancm/easticsearch/EsHighLevelRestSearchTest.java rename to src/main/java/com/pancm/elasticsearch/EsHighLevelRestSearchTest.java index 532cdd3..0ef3d42 100644 --- a/src/main/java/com/pancm/easticsearch/EsHighLevelRestSearchTest.java +++ b/src/main/java/com/pancm/elasticsearch/EsHighLevelRestSearchTest.java @@ -1,4 +1,4 @@ -package com.pancm.easticsearch; +package com.pancm.elasticsearch; import org.apache.http.HttpHost; import org.elasticsearch.action.search.SearchRequest; diff --git a/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest1.java b/src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest1.java similarity index 99% rename from src/main/java/com/pancm/easticsearch/EsHighLevelRestTest1.java rename to src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest1.java index ce36442..0238621 100644 --- a/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest1.java +++ b/src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest1.java @@ -1,4 +1,4 @@ -package com.pancm.easticsearch; +package com.pancm.elasticsearch; import org.apache.http.HttpHost; import org.elasticsearch.ElasticsearchException; diff --git a/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest2.java b/src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest2.java similarity index 99% rename from src/main/java/com/pancm/easticsearch/EsHighLevelRestTest2.java rename to src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest2.java index d76fbad..68a339a 100644 --- a/src/main/java/com/pancm/easticsearch/EsHighLevelRestTest2.java +++ b/src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest2.java @@ -1,4 +1,4 @@ -package com.pancm.easticsearch; +package com.pancm.elasticsearch; import org.apache.http.HttpHost; import org.elasticsearch.action.ActionListener; diff --git a/src/main/java/com/pancm/easticsearch/EsUtil.java b/src/main/java/com/pancm/elasticsearch/EsUtil.java similarity index 99% rename from src/main/java/com/pancm/easticsearch/EsUtil.java rename to src/main/java/com/pancm/elasticsearch/EsUtil.java index 5e84527..31b2dcc 100644 --- a/src/main/java/com/pancm/easticsearch/EsUtil.java +++ b/src/main/java/com/pancm/elasticsearch/EsUtil.java @@ -1,4 +1,4 @@ -package com.pancm.easticsearch; +package com.pancm.elasticsearch; import org.apache.http.HttpHost; @@ -50,7 +50,7 @@ * @date 2019年3月19日 */ public final class EsUtil { - private static Logger logger = LoggerFactory.getLogger(com.pancm.sql.easticsearch.EsHighLevelRestSearchTest.class); + private static Logger logger = LoggerFactory.getLogger(EsHighLevelRestSearchTest.class); private EsUtil() { diff --git a/src/main/java/com/pancm/easticsearch/JestTest.java b/src/main/java/com/pancm/elasticsearch/JestTest.java similarity index 100% rename from src/main/java/com/pancm/easticsearch/JestTest.java rename to src/main/java/com/pancm/elasticsearch/JestTest.java diff --git a/src/main/java/com/pancm/easticsearch/package-info.java b/src/main/java/com/pancm/elasticsearch/package-info.java similarity index 79% rename from src/main/java/com/pancm/easticsearch/package-info.java rename to src/main/java/com/pancm/elasticsearch/package-info.java index 9a4d2a5..841cffa 100644 --- a/src/main/java/com/pancm/easticsearch/package-info.java +++ b/src/main/java/com/pancm/elasticsearch/package-info.java @@ -5,4 +5,4 @@ * @author pancm * @date 2019年2月28日 */ -package com.pancm.easticsearch; \ No newline at end of file +package com.pancm.elasticsearch; \ No newline at end of file From 3f10814ac280c525a313eaaf1cf73db905050a17 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Thu, 7 Nov 2019 20:31:05 +0800 Subject: [PATCH 30/53] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0Spring=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index bde2feb..02b28cd 100644 --- a/pom.xml +++ b/pom.xml @@ -182,6 +182,12 @@ 2.3.0 + + org.springframework + spring-beans + 5.2.1.RELEASE + + com.google.guava From 7e79fdcaad458bea29c2923849a86fa1637b956e Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Thu, 7 Nov 2019 20:31:26 +0800 Subject: [PATCH 31/53] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0es=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E7=A4=BA=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../EsAggregationSearchTest.java | 200 +++++++++++++++++- 1 file changed, 196 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java b/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java index 440764a..8559dd4 100644 --- a/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java +++ b/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java @@ -1,16 +1,25 @@ package com.pancm.elasticsearch; import org.apache.http.HttpHost; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.avg.Avg; @@ -21,8 +30,15 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * @author pancm @@ -49,6 +65,9 @@ public static void main(String[] args) { try { init(); + createIndex(); + bulk(); + groupbySearch(); avgSearch(); maxSearch(); sumSearch(); @@ -64,13 +83,13 @@ public static void main(String[] args) { } + /* * 初始化服务 */ private static void init() { RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(elasticIp, elasticPort)); client = new RestHighLevelClient(restClientBuilder); - } /* @@ -88,6 +107,149 @@ private static void close() { } } + /** + * 创建索引 + * + * @throws IOException + */ + private static void createIndex() throws IOException { + + // 类型 + String type = "_doc"; + String index = "student"; + // setting 的值 + Map setmapping = new HashMap<>(); + // 分区数、副本数、缓存刷新时间 + setmapping.put("number_of_shards", 10); + setmapping.put("number_of_replicas", 1); + setmapping.put("refresh_interval", "5s"); + Map keyword = new HashMap<>(); + //设置类型 + keyword.put("type", "keyword"); + Map lon = new HashMap<>(); + //设置类型 + lon.put("type", "long"); + Map date = new HashMap<>(); + //设置类型 + date.put("type", "date"); + date.put("format", "yyyy-MM-dd"); + + Map date2 = new HashMap<>(); + //设置类型 + date2.put("type", "date"); + date2.put("format", "yyyy-MM-dd HH:mm:ss.SSS"); + Map jsonMap2 = new HashMap<>(); + Map properties = new HashMap<>(); + //设置字段message信息 + properties.put("uid", lon); + properties.put("grade", lon); + properties.put("class", lon); + properties.put("age", lon); + properties.put("name", keyword); + properties.put("createtm", date); + properties.put("updatetm", date2); + Map mapping = new HashMap<>(); + mapping.put("properties", properties); + jsonMap2.put(type, mapping); + + GetIndexRequest getRequest = new GetIndexRequest(); + getRequest.indices(index); + getRequest.types(type); + getRequest.local(false); + getRequest.humanReadable(true); + boolean exists2 = client.indices().exists(getRequest, RequestOptions.DEFAULT); + //如果存在就不创建了 + if(exists2) { + System.out.println(index+"索引库已经存在!"); + return; + } + // 开始创建库 + CreateIndexRequest request = new CreateIndexRequest(index); + try { + // 加载数据类型 + request.settings(setmapping); + //设置mapping参数 + request.mapping(type, jsonMap2); + //设置别名 + request.alias(new Alias("pancm_alias")); + CreateIndexResponse createIndexResponse = client.indices().create(request, RequestOptions.DEFAULT); + boolean falg = createIndexResponse.isAcknowledged(); + if(falg){ + System.out.println("创建索引库:"+index+"成功！" ); + } + } catch (IOException e) { + e.printStackTrace(); + } + + } + + + + /** + * 批量操作示例 + * + * @throws InterruptedException + */ + private static void bulk() throws IOException{ + // 类型 + String type = "_doc"; + String index = "student"; + + BulkRequest request = new BulkRequest(); + int k =10; + List> mapList = new ArrayList<>(); + LocalDateTime ldt = LocalDateTime.now(); + for (int i = 1; i <=k ; i++) { + Map map = new HashMap<>(); + map.put("uid",i); + map.put("age",i); + map.put("name","虚无境"+(i%3)); + map.put("class",i%10); + map.put("grade",400+i); + map.put("createtm",ldt.plusDays(i).format(DateTimeFormatter.ofPattern("yyyy-MM-dd"))); + map.put("updatetm",ldt.plusDays(i).format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS"))); + if(i==5){ + map.put("updatetm","2019-11-31 21:04:55.268"); + } + mapList.add(map); + } + + + for (int i = 0; i map = mapList.get(i); + String id = map.get("uid").toString(); + // 可以进行修改/删除/新增 操作 + //docAsUpsert 为true表示存在更新，不存在插入，为false表示不存在就是不做更新 + request.add(new UpdateRequest(index, type, id).doc(map, XContentType.JSON).docAsUpsert(true).retryOnConflict(5)); + } + + client.bulk(request, RequestOptions.DEFAULT); + System.out.println("批量执行成功！"); + } + + /** + * @Author pancm + * @Description 多个聚合条件测试 + * SQL: select age, name, count(*) as count1 from student group by age, name; + * @Date 2019/7/3 + * @Param [] + * @return void + **/ + private static void groupbySearch() throws IOException{ + String buk="group"; + AggregationBuilder aggregation = AggregationBuilders.terms("age").field("age"); + AggregationBuilder aggregation2 = AggregationBuilders.terms("name").field("name"); + //根据创建时间按天分组 + AggregationBuilder aggregation3 = AggregationBuilders.dateHistogram("createtm") + .field("createtm") + .format("yyyy-MM-dd") + .dateHistogramInterval(DateHistogramInterval.DAY); + + aggregation2.subAggregation(aggregation3); + aggregation.subAggregation(aggregation2); + agg(aggregation,buk); + } + /** * @Author pancm * @Description 平均聚合查询测试用例 @@ -192,17 +354,47 @@ protected static void agg(AggregationBuilder aggregation, String buk) throws Sum ba = aggregations.get(buk); logger.info(buk+":" + ba.getValue()); logger.info("------------------------------------"); - }else{ - //取子聚合 + }else if(buk.contains("top")){ + //取子聚合TopHits TopHits ba = aggregations.get(buk); logger.info(buk+":" + ba.getHits().totalHits); logger.info("------------------------------------"); + }else if (buk.contains("group")){ + Map map = new HashMap<>(); + List> list = new ArrayList<>(); + agg(map,list,aggregations); + logger.info("聚合查询结果:"+list); + logger.info("------------------------------------"); } - } } + private static void agg(Map map, List> list, Aggregations aggregations) { + aggregations.forEach(aggregation -> { + String name = aggregation.getName(); + Terms genders = aggregations.get(name); + for (Terms.Bucket entry : genders.getBuckets()) { + String key = entry.getKey().toString(); + long t = entry.getDocCount(); + map.put(name,key); + map.put(name+"_"+"count",t); + + //判断里面是否还有嵌套的数据 + List list2 = entry.getAggregations().asList(); + if (list2.isEmpty()) { + Map map2 = new HashMap<>(); + BeanUtils.copyProperties(map,map2); + list.add(map2); + }else{ + agg(map, list, entry.getAggregations()); + } + } + }); + } + + + private static SearchResponse search(AggregationBuilder aggregation) throws IOException { SearchRequest searchRequest = new SearchRequest(); searchRequest.indices("student"); From 10bcec916b5a6094c750d2e297108f09f40d4e0b Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Wed, 20 Nov 2019 15:26:42 +0800 Subject: [PATCH 32/53] =?UTF-8?q?=E5=A2=9E=E5=8A=A0License?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index f675f50..beab3e6 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,10 @@ +
+ +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](http://github.com/hhyo/archery/blob/master/LICENSE) +[![996.icu](https://img.shields.io/badge/link-996.icu-red.svg)](https://996.icu) + +
+ ## java-study **介绍** From c0216cdd858761af27a8f7088c7c2ecb06983ea6 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Tue, 3 Dec 2019 16:09:13 +0800 Subject: [PATCH 33/53] Create maven.yml --- .github/workflows/maven.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/maven.yml diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml new file mode 100644 index 0000000..91106d3 --- /dev/null +++ b/.github/workflows/maven.yml @@ -0,0 +1,17 @@ +name: Java CI + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + - name: Build with Maven + run: mvn -B package --file pom.xml From 6b8ce73420047bf0ba6f092288e6be712b59c029 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Sat, 28 Dec 2019 14:34:08 +0800 Subject: [PATCH 34/53] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0=E5=8D=9A=E6=96=87?= =?UTF-8?q?=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index beab3e6..eaf274e 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ - [ElasticSearch实战系列一: ElasticSearch集群+Kinaba安装教程](https://www.cnblogs.com/xuwujing/p/11385255.html) - [ElasticSearch实战系列二: ElasticSearch的DSL语句使用教程---图文详解](https://www.cnblogs.com/xuwujing/p/11567053.html) - [ElasticSearch实战系列三: ElasticSearch的JAVA API使用教程](https://www.cnblogs.com/xuwujing/p/11645630.html) +- [ElasticSearch实战系列四: ElasticSearch理论知识介绍](https://www.cnblogs.com/xuwujing/p/12093933.html) From afac736f8f0b85c64841b42265983b4b18ad755d Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Fri, 10 Jan 2020 17:58:34 +0800 Subject: [PATCH 35/53] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0=E5=8D=9A=E6=96=87?= =?UTF-8?q?=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index eaf274e..381cc41 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ - [一个平凡但不平庸的程序猿2018个人总结](https://www.cnblogs.com/xuwujing/p/9665966.html) - [个人收集的资源分享](https://www.cnblogs.com/xuwujing/p/10393111.html) - [一个毕业三年的程序猿对于提升自我的一些建议](https://www.cnblogs.com/xuwujing/p/11735726.html) +- [认清自我，不在迷茫！2019个人年终总结！](https://www.cnblogs.com/xuwujing/p/12174112.html) ## 其他 From 4399d27ee11bf295512d04c9e6db9a18f398d637 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Thu, 20 Feb 2020 18:03:07 +0800 Subject: [PATCH 36/53] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 381cc41..99b2c83 100644 --- a/README.md +++ b/README.md @@ -102,3 +102,5 @@ ## 其他 在这些代码中，虽然大部分都是自己写的，但是也有不少是在学习过程中从网上或书上直接摘抄的，当时有些并未标明出处，现在由于忘了出处，有些代码并未标明，若有冒犯，请见谅！ + +Thanks to [Jetbrains](https://www.jetbrains.com/?from=java-study) for IDE support! \ No newline at end of file From 035656604dca505e1d2930150d60fcc3a7f69bca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2020 10:06:42 +0000 Subject: [PATCH 37/53] Bump storm-kafka from 1.2.2 to 1.2.3 Bumps storm-kafka from 1.2.2 to 1.2.3. Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5835509..89993c9 100644 --- a/pom.xml +++ b/pom.xml @@ -375,7 +375,7 @@ org.apache.storm storm-kafka - 1.2.2 + 1.2.3 provided From 09a38484f622b4f8a8fd5c7ee42ad17fce08155b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2020 08:56:20 +0000 Subject: [PATCH 38/53] Bump junit from 4.12 to 4.13.1 Bumps [junit](https://github.com/junit-team/junit4) from 4.12 to 4.13.1. - [Release notes](https://github.com/junit-team/junit4/releases) - [Changelog](https://github.com/junit-team/junit4/blob/main/doc/ReleaseNotes4.12.md) - [Commits](https://github.com/junit-team/junit4/compare/r4.12...r4.13.1) Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5835509..f524471 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,7 @@ junit junit - 4.12 + 4.13.1 From 1fc0ef47a38046cc7b0410afe25c996f3536d15e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Jan 2021 19:55:45 +0000 Subject: [PATCH 39/53] Bump poi from 3.9 to 3.17 Bumps poi from 3.9 to 3.17. Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5835509..9504bc1 100644 --- a/pom.xml +++ b/pom.xml @@ -117,7 +117,7 @@ org.apache.poi poi - 3.9 + 3.17 From 9800dabe89508a9c9997cd41305afbfdbe1fdce7 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Sat, 6 Mar 2021 14:20:40 +0800 Subject: [PATCH 40/53] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 99b2c83..03d3120 100644 --- a/README.md +++ b/README.md @@ -81,11 +81,15 @@ **ElasticSearch相关:** -- [ElasticSearch实战系列一: ElasticSearch集群+Kinaba安装教程](https://www.cnblogs.com/xuwujing/p/11385255.html) +- [ElasticSearch实战系列一: ElasticSearch集群+Kibana安装教程](https://www.cnblogs.com/xuwujing/p/11385255.html) - [ElasticSearch实战系列二: ElasticSearch的DSL语句使用教程---图文详解](https://www.cnblogs.com/xuwujing/p/11567053.html) - [ElasticSearch实战系列三: ElasticSearch的JAVA API使用教程](https://www.cnblogs.com/xuwujing/p/11645630.html) - [ElasticSearch实战系列四: ElasticSearch理论知识介绍](https://www.cnblogs.com/xuwujing/p/12093933.html) - +- [ElasticSearch实战系列五: ElasticSearch的聚合查询基础使用教程之度量(Metric)聚合](https://www.cnblogs.com/xuwujing/p/12385903.html) +- [ElasticSearch实战系列六: Logstash快速入门](https://www.cnblogs.com/xuwujing/p/13412108.html) +- [ElasticSearch实战系列七: Logstash实战使用-图文讲解](https://www.cnblogs.com/xuwujing/p/13520666.html) +- [ElasticSearch实战系列八: Filebeat快速入门和使用---图文详解](https://www.cnblogs.com/xuwujing/p/13532125.html) +- [ElasticSearch实战系列九: ELK日志系统介绍和安装](https://www.cnblogs.com/xuwujing/p/13870806.html) **其他博客:** @@ -98,6 +102,7 @@ - [个人收集的资源分享](https://www.cnblogs.com/xuwujing/p/10393111.html) - [一个毕业三年的程序猿对于提升自我的一些建议](https://www.cnblogs.com/xuwujing/p/11735726.html) - [认清自我，不在迷茫！2019个人年终总结！](https://www.cnblogs.com/xuwujing/p/12174112.html) +- [纵然前路坎坷，也要毅然前行！2020年终总结！](https://www.cnblogs.com/xuwujing/p/14233270.html) ## 其他 From f697826997904081b63541b542471f791f8cf7b0 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Sat, 6 Mar 2021 14:22:51 +0800 Subject: [PATCH 41/53] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0ElasticSearch?= =?UTF-8?q?=E8=81=9A=E5=90=88=E4=BD=BF=E7=94=A8=E7=A4=BA=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/pancm/elasticsearch/EsAggregationSearchTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java b/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java index 8559dd4..7c02ce6 100644 --- a/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java +++ b/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java @@ -270,7 +270,7 @@ private static void avgSearch() throws IOException { private static void maxSearch() throws IOException{ String buk="t_grade"; AggregationBuilder aggregation = AggregationBuilders.max(buk).field("grade"); - logger.info("求班级的最分数:"); + logger.info("求班级的最高分数:"); agg(aggregation,buk); } From a3a5104f01bc7f8099419faa2c382018caa6c6ca Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Sat, 6 Mar 2021 14:32:33 +0800 Subject: [PATCH 42/53] =?UTF-8?q?1.=E5=A2=9E=E5=8A=A0ElasticSearch?= =?UTF-8?q?=E8=81=9A=E5=90=88=E4=BD=BF=E7=94=A8=E7=A4=BA=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 87 +++++++------ .../EsAggregationSearchTest.java | 122 ++++++++++++++++++ 2 files changed, 166 insertions(+), 43 deletions(-) diff --git a/.gitignore b/.gitignore index a99dd7a..409684a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,43 +1,44 @@ -/target/ -/classes/ -/log/ -/logs/ -.classpath -.project -.settings -.myeclipse - ##filter databfilesln file## -*.mdb -*.ldb -*.sln -##class file## -*.com -*.class -*.dll -*.exe -*.o -*.so -# compression file -*.7z -*.dmg -*.gz -*.iso -*.jar -*.rar -*.tar -*.zip -*.via -*.iml -*.tmp -*.err -*.log -# OS generated files # -/.idea -.DS_Store -.DS_Store? -._* -.Spotlight-V100 -.Trashes -Icon? -ehthumbs.db -Thumbs.db \ No newline at end of file +/target/ +/classes/ +/log/ +/logs/ +.classpath +.project +.settings +.myeclipse + ##filter databfile��sln file## +*.mdb +*.ldb +*.sln +##class file## +*.com +*.class +*.dll +*.exe +*.o +*.so +# compression file +*.7z +*.dmg +*.gz +*.iso +*.jar +*.rar +*.tar +*.zip +*.via +*.iml +*.tmp +*.err +*.log +# OS generated files # +/.idea +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +Icon? +ehthumbs.db +Thumbs.db +/.github/ diff --git a/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java b/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java index 7c02ce6..22b3049 100644 --- a/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java +++ b/src/main/java/com/pancm/elasticsearch/EsAggregationSearchTest.java @@ -8,13 +8,17 @@ import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.Script; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; @@ -23,10 +27,13 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.avg.Avg; +import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.max.Max; import org.elasticsearch.search.aggregations.metrics.min.Min; import org.elasticsearch.search.aggregations.metrics.sum.Sum; import org.elasticsearch.search.aggregations.metrics.tophits.TopHits; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders; +import org.elasticsearch.search.aggregations.pipeline.bucketselector.BucketSelectorPipelineAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -393,6 +400,28 @@ private static void agg(Map map, List> list, A }); } + private static void agg(List> list, Aggregations aggregations) { + aggregations.forEach(aggregation -> { + String name = aggregation.getName(); + Terms genders = aggregations.get(name); + for (Terms.Bucket entry : genders.getBuckets()) { + String key = entry.getKey().toString(); + long t = entry.getDocCount(); + Map map =new HashMap<>(); + map.put(name,key); + map.put(name+"_"+"count",t); + //判断里面是否还有嵌套的数据 + List list2 = entry.getAggregations().asList(); + if (list2.isEmpty()) { + list.add(map); + }else{ + agg(list, entry.getAggregations()); + } + } + }); + System.out.println(list); + } + private static SearchResponse search(AggregationBuilder aggregation) throws IOException { @@ -473,6 +502,99 @@ protected static void agg(AggregationBuilder aggregation, String agg, String b } } + /** + * @Author pancm + * @Description having + * @Date 2020/8/21 + * @Param [] + * @return void + **/ + private static void havingSearch() throws IOException{ + String index=""; + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.indices(index); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); + String alias_name = "nas_ip_address_group"; + String group_name = "nas_ip_address"; + String query_name = "acct_start_time"; + String query_type = "gte,lte"; + String query_name_value="2020-08-05 13:25:55,2020-08-20 13:26:55"; + String[] query_types= query_type.split(","); + String[] query_name_values= query_name_value.split(","); + + for (int i = 0; i < query_types.length; i++) { + if("gte".equals(query_types[i])){ + boolQueryBuilder.must(QueryBuilders.rangeQuery(query_name).gte(query_name_values[i])); + } + if("lte".equals(query_types[i])){ + boolQueryBuilder.must(QueryBuilders.rangeQuery(query_name).lte(query_name_values[i])); + } + } + + AggregationBuilder aggregationBuilder = AggregationBuilders.terms(alias_name).field(group_name).size(Integer.MAX_VALUE); + + //声明BucketPath，用于后面的bucket筛选 + Map bucketsPathsMap = new HashMap<>(8); + bucketsPathsMap.put("groupCount", "_count"); + //设置脚本 + Script script = new Script("params.groupCount >= 1000"); + //构建bucket选择器 + BucketSelectorPipelineAggregationBuilder bs = + PipelineAggregatorBuilders.bucketSelector("having", bucketsPathsMap, script); + aggregationBuilder.subAggregation(bs); + sourceBuilder.aggregation(aggregationBuilder); + //不需要解释 + sourceBuilder.explain(false); + //不需要原始数据 + sourceBuilder.fetchSource(false); + //不需要版本号 + sourceBuilder.version(false); + sourceBuilder.query(boolQueryBuilder); + searchRequest.source(sourceBuilder); + + + System.out.println(sourceBuilder); + // 同步查询 + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + // 查询条数 + long count = searchResponse.getHits().getHits().length; + Aggregations aggregations = searchResponse.getAggregations(); +// agg(aggregations); + Map map =new HashMap<>(); + List> list =new ArrayList<>(); + agg(list,aggregations); +// System.out.println(map); + System.out.println(list); + } + + + /** + * @Author pancm + * @Description 去重 + * @Date 2020/8/26 + * @Param [] + * @return void + **/ + private static void distinctSearch() throws IOException{ + String buk="group"; + String distinctName="name"; + AggregationBuilder aggregation = AggregationBuilders.terms("age").field("age"); + CardinalityAggregationBuilder cardinalityBuilder = AggregationBuilders.cardinality(distinctName).field(distinctName); + //根据创建时间按天分组 +// AggregationBuilder aggregation3 = AggregationBuilders.dateHistogram("createtm") +// .field("createtm") +// .format("yyyy-MM-dd") +// .dateHistogramInterval(DateHistogramInterval.DAY); +// +// aggregation2.subAggregation(aggregation3); + aggregation.subAggregation(cardinalityBuilder); + agg(aggregation,buk); + } + + + private static void topSearch() throws IOException{ From 29b1a0a78386c6213017e29391c06940fccf2754 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Sun, 7 Mar 2021 17:42:34 +0800 Subject: [PATCH 43/53] insignificance update --- pom.xml | 9 +++- .../responsibility/ResponsibilityTest.java | 48 +++++++++---------- .../java/com/pancm/excel/EasyExcelTest.java | 15 ++++++ .../java/com/pancm/excel/package-info.java | 8 ++++ 4 files changed, 54 insertions(+), 26 deletions(-) create mode 100644 src/main/java/com/pancm/excel/EasyExcelTest.java create mode 100644 src/main/java/com/pancm/excel/package-info.java diff --git a/pom.xml b/pom.xml index 5835509..a44c5ef 100644 --- a/pom.xml +++ b/pom.xml @@ -395,9 +395,14 @@ io.searchbox jest 6.3.1 - - + + + + com.alibaba + easyexcel + 2.2.7 + diff --git a/src/main/java/com/pancm/design/responsibility/ResponsibilityTest.java b/src/main/java/com/pancm/design/responsibility/ResponsibilityTest.java index b810e58..fa1b4a3 100644 --- a/src/main/java/com/pancm/design/responsibility/ResponsibilityTest.java +++ b/src/main/java/com/pancm/design/responsibility/ResponsibilityTest.java @@ -22,19 +22,19 @@ public static void main(String[] args) { String name = "xuwujing"; String something = "去聚餐"; String something2 = "去旅游"; - Learder learder1 =new Supervisor(name, something); - Learder learder2 =new BranchManager(name, something); - Learder learder3 =new GeneralManager(name, something); - learder1.setLearder(learder2); - learder2.setLearder(learder3); - learder1.handler(1); + Leader leader1 =new Supervisor(name, something); + Leader leader2 =new BranchManager(name, something); + Leader leader3 =new GeneralManager(name, something); + leader1.setLeader(leader2); + leader2.setLeader(leader3); + leader1.handler(1); - Learder learder4 =new Supervisor(name, something2); - Learder learder5 =new BranchManager(name, something2); - Learder learder6 =new GeneralManager(name, something2); - learder4.setLearder(learder5); - learder5.setLearder(learder6); - learder4.handler(0); + Leader leader4 =new Supervisor(name, something2); + Leader leader5 =new BranchManager(name, something2); + Leader leader6 =new GeneralManager(name, something2); + leader4.setLeader(leader5); + leader5.setLeader(leader6); + leader4.handler(0); @@ -76,24 +76,24 @@ public void handleRequest() { } -abstract class Learder{ +abstract class Leader { - protected Learder learder; + protected Leader leader; - protected void setLearder(Learder learder){ - this.learder=learder; + protected void setLeader(Leader leader){ + this.leader = leader; } - protected Learder getLearder(){ - return learder; + protected Leader getLeader(){ + return leader; } abstract void handler(int level); } //主管 -class Supervisor extends Learder{ +class Supervisor extends Leader { private String name; private String something; public Supervisor(String name,String something) { @@ -108,13 +108,13 @@ void handler(int level) { System.out.println("主管处理了 "+name+"所述的<"+something+">事情!"); }else{ System.out.println("主管未能处理 "+name+"所述的<"+something+">事情!转交给上级!"); - getLearder().handler(level); + getLeader().handler(level); } } } //部门经理 -class BranchManager extends Learder{ +class BranchManager extends Leader { private String name; private String something; public BranchManager(String name,String something) { @@ -129,13 +129,13 @@ void handler(int level) { System.out.println("部门经理处理了 "+name+"所述的<"+something+">事情!"); }else{ System.out.println("部门经理未能处理 "+name+"所述的<"+something+">事情!转交给上级!"); - getLearder().handler(level); + getLeader().handler(level); } } } //总经理 -class GeneralManager extends Learder{ +class GeneralManager extends Leader { private String name; private String something; public GeneralManager(String name,String something) { @@ -150,7 +150,7 @@ void handler(int level) { System.out.println("总经理处理了 "+name+"所述的<"+something+">事情!"); }else{ System.out.println("总经理未能处理 "+name+"所述的<"+something+">事情!转交给上级!"); - getLearder().handler(level); + getLeader().handler(level); } } } diff --git a/src/main/java/com/pancm/excel/EasyExcelTest.java b/src/main/java/com/pancm/excel/EasyExcelTest.java new file mode 100644 index 0000000..e5e0f23 --- /dev/null +++ b/src/main/java/com/pancm/excel/EasyExcelTest.java @@ -0,0 +1,15 @@ +package com.pancm.excel; + +/** + * @author pancm + * @Title: pancm_project + * @Description: + * 参考: https://www.yuque.com/easyexcel/doc + * @Version:1.0.0 + * @Since:jdk1.8 + * @date 2021/1/26 + */ +public class EasyExcelTest { + + +} diff --git a/src/main/java/com/pancm/excel/package-info.java b/src/main/java/com/pancm/excel/package-info.java new file mode 100644 index 0000000..5becb8e --- /dev/null +++ b/src/main/java/com/pancm/excel/package-info.java @@ -0,0 +1,8 @@ +/** +* @Title: package-info +* @Description: excel相关包 +* @Version:1.0.0 +* @author pancm +* @date 2019年2月28日 +*/ +package com.pancm.excel; \ No newline at end of file From 3fba28f56e41535b1b46f09eca5df0aed3a7eb99 Mon Sep 17 00:00:00 2001 From: pancm <1060589146@qq.com> Date: Wed, 7 Jun 2023 17:42:39 +0800 Subject: [PATCH 44/53] =?UTF-8?q?readme=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 03d3120..a8dff8b 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,18 @@ - [ElasticSearch实战系列七: Logstash实战使用-图文讲解](https://www.cnblogs.com/xuwujing/p/13520666.html) - [ElasticSearch实战系列八: Filebeat快速入门和使用---图文详解](https://www.cnblogs.com/xuwujing/p/13532125.html) - [ElasticSearch实战系列九: ELK日志系统介绍和安装](https://www.cnblogs.com/xuwujing/p/13870806.html) +- [ElasticSearch实战系列十: ElasticSearch冷热分离架构](https://www.cnblogs.com/xuwujing/p/14599290.html) +- [ElasticSearch实战系列十一: ElasticSearch错误问题解决方案](https://www.cnblogs.com/xuwujing/p/14806392.html) + +**手记系列:** + +[手记系列](https://www.cnblogs.com/xuwujing/category/2236708.html) + +- [手记系列之一 ----- 关于微信公众号和小程序的开发流程](https://www.cnblogs.com/xuwujing/p/16841577.html) +- [手记系列之二 ----- 关于IDEA的一些使用方法经验](https://www.cnblogs.com/xuwujing/p/16862451.html) +- [手记系列之三 ----- 关于使用Nginx的一些使用方法和经验](https://www.cnblogs.com/xuwujing/p/16885964.html) +- [手记系列之四 ----- 关于使用MySql的经验](https://www.cnblogs.com/xuwujing/p/17356379.html) + **其他博客:** @@ -103,6 +115,8 @@ - [一个毕业三年的程序猿对于提升自我的一些建议](https://www.cnblogs.com/xuwujing/p/11735726.html) - [认清自我，不在迷茫！2019个人年终总结！](https://www.cnblogs.com/xuwujing/p/12174112.html) - [纵然前路坎坷，也要毅然前行！2020年终总结！](https://www.cnblogs.com/xuwujing/p/14233270.html) +- [有一点思考的2021年终总结！](https://www.cnblogs.com/xuwujing/p/15746791.html) +- [一个想活得简单的程序猿的2022年终总结！](https://www.cnblogs.com/xuwujing/p/17060965.html) ## 其他 From 62c4c0b957df59aa60c2eae6b1d756da60e66b43 Mon Sep 17 00:00:00 2001 From: pancm <1060589146@qq.com> Date: Wed, 7 Jun 2023 17:43:36 +0800 Subject: [PATCH 45/53] =?UTF-8?q?readme=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index a8dff8b..3deef01 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,6 @@ **手记系列:** -[手记系列](https://www.cnblogs.com/xuwujing/category/2236708.html) - [手记系列之一 ----- 关于微信公众号和小程序的开发流程](https://www.cnblogs.com/xuwujing/p/16841577.html) - [手记系列之二 ----- 关于IDEA的一些使用方法经验](https://www.cnblogs.com/xuwujing/p/16862451.html) From f02d2cc6f41e7df478019f6ecd9f313f0cac964f Mon Sep 17 00:00:00 2001 From: pancm <1060589146@qq.com> Date: Thu, 8 Jun 2023 11:14:41 +0800 Subject: [PATCH 46/53] add easyExcel --- src/main/java/com/pancm/excel/DemoData.java | 22 ++ .../com/pancm/excel/EasyExcelMergeTest.java | 208 +++++++++++++ .../java/com/pancm/excel/EasyExcelTest.java | 277 +++++++++++++++++- .../java/com/pancm/excel/EasyExcelUtils.java | 110 +++++++ .../com/pancm/excel/EasyExcelWriteTest.java | 115 ++++++++ .../java/com/pancm/excel/package-info.java | 13 +- 6 files changed, 737 insertions(+), 8 deletions(-) create mode 100644 src/main/java/com/pancm/excel/DemoData.java create mode 100644 src/main/java/com/pancm/excel/EasyExcelMergeTest.java create mode 100644 src/main/java/com/pancm/excel/EasyExcelUtils.java create mode 100644 src/main/java/com/pancm/excel/EasyExcelWriteTest.java diff --git a/src/main/java/com/pancm/excel/DemoData.java b/src/main/java/com/pancm/excel/DemoData.java new file mode 100644 index 0000000..ecae316 --- /dev/null +++ b/src/main/java/com/pancm/excel/DemoData.java @@ -0,0 +1,22 @@ +package com.pancm.excel; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.Date; + +/** + * @author pancm + * @Title: pancm_project + * @Description: + * @Version:1.0.0 + * @Since:jdk1.8 + * @date 2023/3/23 + */ +@Data +@EqualsAndHashCode +public class DemoData { + private String string; + private Date date; + private Double doubleData; +} diff --git a/src/main/java/com/pancm/excel/EasyExcelMergeTest.java b/src/main/java/com/pancm/excel/EasyExcelMergeTest.java new file mode 100644 index 0000000..ea1556b --- /dev/null +++ b/src/main/java/com/pancm/excel/EasyExcelMergeTest.java @@ -0,0 +1,208 @@ +package com.pancm.excel; + +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.ExcelWriter; +import com.alibaba.excel.metadata.Head; +import com.alibaba.excel.support.ExcelTypeEnum; +import com.alibaba.excel.util.CollectionUtils; +import com.alibaba.excel.write.merge.AbstractMergeStrategy; +import com.alibaba.excel.write.metadata.WriteSheet; +import com.alibaba.excel.write.metadata.WriteTable; +import com.google.common.collect.Lists; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.util.CellRangeAddress; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author pancm + * @Title: pancm_project + * @Description: 多行合并 + * 合并单元格 + * @Version:1.0.0 + * @Since:jdk1.8 + * @date 2023/3/23 + */ +public class EasyExcelMergeTest { + + public static void main(String[] args) { + writeExcel(); + writeExcel01(); + writeExcel02(); + writeExcel03(); + } + + + private static String getPath(String s) { + return System.getProperty("user.dir") + "/" + s+"_"+System.currentTimeMillis() + ".xlsx"; + } + + private static List data1() { + List list = Lists.newArrayList(); + for (int i = 0; i < 3; i++) { + DemoData data = new DemoData(); + data.setString("字符串" + 1); + data.setDate(new Date()); + data.setDoubleData(0.56); + list.add(data); + } + for (int i = 0; i < 3; i++) { + DemoData data = new DemoData(); + data.setString("字符串" + 2); + data.setDate(new Date()); + data.setDoubleData(0.56); + list.add(data); + } + for (int i = 0; i < 4; i++) { + DemoData data = new DemoData(); + data.setString("字符串" + 3); + data.setDate(new Date()); + data.setDoubleData(0.57); + list.add(data); + } + return list; + } + + + + // 自定义合并策略 该类继承了AbstractMergeStrategy抽象合并策略，需要重写merge()方法 + public static class CustomMergeStrategy extends AbstractMergeStrategy { + + /** + * 分组，每几行合并一次 + */ + private List exportFieldGroupCountList; + + /** + * 目标合并列index + */ + private Integer targetColumnIndex; + + // 需要开始合并单元格的首行index + private Integer rowIndex; + + // exportDataList为待合并目标列的值 + public CustomMergeStrategy(List exportDataList, Integer targetColumnIndex) { + this.exportFieldGroupCountList = getGroupCountList(exportDataList); + this.targetColumnIndex = targetColumnIndex; + } + + + @Override + protected void merge(Sheet sheet, Cell cell, Head head, Integer relativeRowIndex) { + + if (null == rowIndex) { + rowIndex = cell.getRowIndex(); + } + // 仅从首行以及目标列的单元格开始合并，忽略其他 + if (cell.getRowIndex() == rowIndex && cell.getColumnIndex() == targetColumnIndex) { + mergeGroupColumn(sheet); + } + } + + private void mergeGroupColumn(Sheet sheet) { + int rowCount = rowIndex; + for (Integer count : exportFieldGroupCountList) { + if (count == 1) { + rowCount += count; + continue; + } + // 合并单元格 + CellRangeAddress cellRangeAddress = new CellRangeAddress(rowCount, rowCount + count - 1, targetColumnIndex, targetColumnIndex); + sheet.addMergedRegionUnsafe(cellRangeAddress); + rowCount += count; + } + } + + // 该方法将目标列根据值是否相同连续可合并，存储可合并的行数 + private List getGroupCountList(List exportDataList) { + if (CollectionUtils.isEmpty(exportDataList)) { + return new ArrayList<>(); + } + + List groupCountList = new ArrayList<>(); + int count = 1; + + for (int i = 1; i < exportDataList.size(); i++) { + if (exportDataList.get(i).equals(exportDataList.get(i - 1))) { + count++; + } else { + groupCountList.add(count); + count = 1; + } + } + // 处理完最后一条后 + groupCountList.add(count); + return groupCountList; + } + } + + // 单列多行合并 + public static void writeExcel() { + String fileName = getPath("单列多行"); + ExcelWriter excelWriter = EasyExcel.write(fileName).excelType(ExcelTypeEnum.XLSX).build(); + + List demoDataList = data1(); + // 写sheet的时候注册相应的自定义合并单元格策略 + WriteSheet writeSheet = EasyExcel.writerSheet("模板1").head(DemoData.class) + .registerWriteHandler(new CustomMergeStrategy(demoDataList.stream().map(DemoData::getString).collect(Collectors.toList()), 0)) + .build(); + excelWriter.write(demoDataList, writeSheet); + excelWriter.finish(); + } + + //多列多行合并 + public static void writeExcel01() { + String fileName = getPath("多行多列"); + ExcelWriter excelWriter = EasyExcel.write(fileName).excelType(ExcelTypeEnum.XLSX).build(); + + List demoDataList = data1(); + WriteSheet writeSheet = EasyExcel.writerSheet("模板1").head(DemoData.class) + .registerWriteHandler(new CustomMergeStrategy(demoDataList.stream().map(DemoData::getString).collect(Collectors.toList()), 0)) + .registerWriteHandler(new CustomMergeStrategy(demoDataList.stream().map(o -> o.getDoubleData().toString()).collect(Collectors.toList()), 2)) + .build(); + excelWriter.write(demoDataList, writeSheet); + excelWriter.finish(); + } + //多sheet + public static void writeExcel02() { + String fileName = getPath("多sheet"); + ExcelWriter excelWriter = EasyExcel.write(fileName).excelType(ExcelTypeEnum.XLSX).build(); + + List demoDataList = data1(); + WriteSheet writeSheet = EasyExcel.writerSheet("模板1").head(DemoData.class) + .registerWriteHandler(new CustomMergeStrategy(demoDataList.stream().map(DemoData::getString).collect(Collectors.toList()), 0)) + .registerWriteHandler(new CustomMergeStrategy(demoDataList.stream().map(o -> o.getDoubleData().toString()).collect(Collectors.toList()), 2)) + .build(); + excelWriter.write(demoDataList, writeSheet); + + WriteSheet writeSheet1 = EasyExcel.writerSheet("模板2").head(DemoData.class).build(); + excelWriter.write(data1(), writeSheet1); + excelWriter.finish(); + } + + //多表 + public static void writeExcel03() { + String fileName = getPath("多表"); + ExcelWriter excelWriter = EasyExcel.write(fileName).excelType(ExcelTypeEnum.XLSX).build(); + WriteSheet writeSheet = EasyExcel.writerSheet("模板").needHead(Boolean.FALSE).build(); + + List demoDataList = data1(); + // 需要表头设置为true，WriteTable一些属性会继承自WriteSheet + WriteTable writeTable = EasyExcel.writerTable(1).head(DemoData.class).needHead(Boolean.TRUE) + .registerWriteHandler(new CustomMergeStrategy(demoDataList.stream().map(DemoData::getString).collect(Collectors.toList()), 0)) + .registerWriteHandler(new CustomMergeStrategy(demoDataList.stream().map(o -> o.getDoubleData().toString()).collect(Collectors.toList()), 2)) + .build(); + excelWriter.write(demoDataList, writeSheet, writeTable); + + WriteTable writeTable1 = EasyExcel.writerTable(2).head(DemoData.class).needHead(Boolean.TRUE).build(); + excelWriter.write(data1(), writeSheet, writeTable1); + excelWriter.finish(); + } + +} + diff --git a/src/main/java/com/pancm/excel/EasyExcelTest.java b/src/main/java/com/pancm/excel/EasyExcelTest.java index e5e0f23..709c684 100644 --- a/src/main/java/com/pancm/excel/EasyExcelTest.java +++ b/src/main/java/com/pancm/excel/EasyExcelTest.java @@ -1,15 +1,288 @@ package com.pancm.excel; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.ExcelReader; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.metadata.CellData; +import com.alibaba.excel.metadata.CellExtra; +import com.alibaba.excel.read.listener.ReadListener; +import com.alibaba.excel.read.metadata.ReadSheet; +import com.alibaba.fastjson.JSON; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; + +import java.io.File; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; + /** * @author pancm * @Title: pancm_project - * @Description: - * 参考: https://www.yuque.com/easyexcel/doc + * @Description: 参考: https://www.yuque.com/easyexcel/doc * @Version:1.0.0 * @Since:jdk1.8 * @date 2021/1/26 */ +@Slf4j public class EasyExcelTest { + /** + * 最简单的读 + *

+ * 1. 创建excel对应的实体对象 参照{@link DemoData} + *

+ * 2. 由于默认一行行的读取excel，所以需要创建excel一行一行的回调监听器，参照{@link DemoDataListener} + *

+ * 3. 直接读即可 + */ + @Test + public void simpleRead() { + // 写法1：JDK8+ ,不用额外写一个DemoDataListener + // since: 3.0.0-beta1 + String fileName = "/home" + "demo" + File.separator + "demo.xlsx"; + // 这里 需要指定读用哪个class去读，然后读取第一个sheet 文件流会自动关闭 + // 这里每次会读取3000条数据 然后返回过来 直接调用使用数据就行 +// EasyExcel.read(fileName, DemoData.class, new PageReadListener(dataList -> { +// for (DemoData demoData : dataList) { +// log.info("读取到一条数据{}", JSON.toJSONString(demoData)); +// } +// })).sheet().doRead(); + + // 写法2： + // 匿名内部类 不用额外写一个DemoDataListener + fileName = "/home" + "demo" + File.separator + "demo.xlsx"; + // 这里 需要指定读用哪个class去读，然后读取第一个sheet 文件流会自动关闭 + EasyExcel.read(fileName, DemoData.class, new ReadListener() { + /** + * 单次缓存的数据量 + */ + public static final int BATCH_COUNT = 100; + /** + *临时存储 + */ + private List cachedDataList = new ArrayList<>(); + + /** + * All listeners receive this method when any one Listener does an error report. If an exception is thrown here, the + * entire read will terminate. + * + * @param exception + * @param context + * @throws Exception + */ + @Override + public void onException(Exception exception, AnalysisContext context) throws Exception { + + } + + /** + * When analysis one head row trigger invoke function. + * + * @param headMap + * @param context + */ + @Override + public void invokeHead(Map headMap, AnalysisContext context) { + + } + + @Override + public void invoke(DemoData data, AnalysisContext context) { + cachedDataList.add(data); + if (cachedDataList.size() >= BATCH_COUNT) { + saveData(); + // 存储完成清理 list + cachedDataList = new ArrayList<>(); + } + } + + /** + * The current method is called when extra information is returned + * + * @param extra extra information + * @param context + */ + @Override + public void extra(CellExtra extra, AnalysisContext context) { + + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + saveData(); + } + + /** + * Verify that there is another piece of data.You can stop the read by returning false + * + * @param context + * @return + */ + @Override + public boolean hasNext(AnalysisContext context) { + return false; + } + + /** + * 加上存储数据库 + */ + private void saveData() { + log.info("{}条数据，开始存储数据库！", cachedDataList.size()); + log.info("存储数据库成功！"); + } + }).sheet().doRead(); + + // 有个很重要的点 DemoDataListener 不能被spring管理，要每次读取excel都要new,然后里面用到spring可以构造方法传进去 + // 写法3： + fileName = "/home" + "demo" + File.separator + "demo.xlsx"; + // 这里 需要指定读用哪个class去读，然后读取第一个sheet 文件流会自动关闭 + EasyExcel.read(fileName, DemoData.class, new DemoDataListener()).sheet().doRead(); + + // 写法4： + fileName = "/home" + "demo" + File.separator + "demo.xlsx"; + // 一个文件一个reader + ExcelReader excelReader = null; + try { + excelReader = EasyExcel.read(fileName, DemoData.class, new DemoDataListener()).build(); + // 构建一个sheet 这里可以指定名字或者no + ReadSheet readSheet = EasyExcel.readSheet(0).build(); + // 读取一个sheet + excelReader.read(readSheet); + } finally { + if (excelReader != null) { + // 这里千万别忘记关闭，读的时候会创建临时文件，到时磁盘会崩的 + excelReader.finish(); + } + } + } + + + @Data + @EqualsAndHashCode + class DemoData { + private String string; + private Date date; + private Double doubleData; + } + + + class DemoDataListener implements ReadListener { + + /** + * 每隔5条存储数据库，实际使用中可以100条，然后清理list ，方便内存回收 + */ + private static final int BATCH_COUNT = 100; + /** + * 缓存的数据 + */ + private List cachedDataList = new ArrayList<>(); + /** + * 假设这个是一个DAO，当然有业务逻辑这个也可以是一个service。当然如果不用存储这个对象没用。 + */ +// private DemoDAO demoDAO; +// +// public DemoDataListener() { +// // 这里是demo，所以随便new一个。实际使用如果到了spring,请使用下面的有参构造函数 +// demoDAO = new DemoDAO(); +// } + + /** + * 如果使用了spring,请使用这个构造方法。每次创建Listener的时候需要把spring管理的类传进来 + * + * @param demoDAO + */ +// public DemoDataListener(DemoDAO demoDAO) { +// this.demoDAO = demoDAO; +// } + + /** + * All listeners receive this method when any one Listener does an error report. If an exception is thrown here, the + * entire read will terminate. + * + * @param exception + * @param context + * @throws Exception + */ + @Override + public void onException(Exception exception, AnalysisContext context) throws Exception { + + } + + /** + * When analysis one head row trigger invoke function. + * + * @param headMap + * @param context + */ + @Override + public void invokeHead(Map headMap, AnalysisContext context) { + + } + + /** + * 这个每一条数据解析都会来调用 + * + * @param data one row value. Is is same as {@link AnalysisContext#readRowHolder()} + * @param context + */ + @Override + public void invoke(DemoData data, AnalysisContext context) { + log.info("解析到一条数据:{}", JSON.toJSONString(data)); + cachedDataList.add(data); + // 达到BATCH_COUNT了，需要去存储一次数据库，防止数据几万条数据在内存，容易OOM + if (cachedDataList.size() >= BATCH_COUNT) { + saveData(); + // 存储完成清理 list +// cachedDataList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT); + } + } + + /** + * The current method is called when extra information is returned + * + * @param extra extra information + * @param context + */ + @Override + public void extra(CellExtra extra, AnalysisContext context) { + + } + + /** + * 所有数据解析完成了 都会来调用 + * + * @param context + */ + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // 这里也要保存数据，确保最后遗留的数据也存储到数据库 + saveData(); + log.info("所有数据解析完成！"); + } + + /** + * Verify that there is another piece of data.You can stop the read by returning false + * + * @param context + * @return + */ + @Override + public boolean hasNext(AnalysisContext context) { + return false; + } + + /** + * 加上存储数据库 + */ + private void saveData() { + log.info("{}条数据，开始存储数据库！", cachedDataList.size()); +// demoDAO.save(cachedDataList); + log.info("存储数据库成功！"); + } + } } diff --git a/src/main/java/com/pancm/excel/EasyExcelUtils.java b/src/main/java/com/pancm/excel/EasyExcelUtils.java new file mode 100644 index 0000000..281ac49 --- /dev/null +++ b/src/main/java/com/pancm/excel/EasyExcelUtils.java @@ -0,0 +1,110 @@ +package com.pancm.excel; + +import com.alibaba.excel.EasyExcel; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import lombok.Data; + +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.Serializable; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class EasyExcelUtils { + + public static void main(String[] args) { + String[] headMap = { "项目名称", "楼栋名称", "单元名称", "楼层名称", "房间名称", "业主/租户姓名", "房间状态", "房间功能","认证人数","测试" }; + String[] dataStrMap={"hName","bName","uName","fName","pName","cName","pState","pFunction","pNum"}; + NoModelWriteData d = new NoModelWriteData(); + d.setFileName("认证统计"); + d.setHeadMap(headMap); + d.setDataStrMap(dataStrMap); + List listDatas = new ArrayList<>(); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("hName","项目1"); + jsonObject.put("bName","二楼"); + jsonObject.put("aa","测试"); + d.setDataList(listDatas); + EasyExcelUtils easyExcelUtils = new EasyExcelUtils(); +// easyExcelUtils.jsonWrite(d, response); + } + + //不创建对象的导出 + public void jsonWrite(NoModelWriteData data, HttpServletResponse response) throws IOException { + // 这里注意 有同学反应使用swagger 会导致各种问题，请直接用浏览器或者用postman + try { +// response.setContentType("application/vnd.ms-excel"); + response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + response.setCharacterEncoding("utf-8"); + // 这里URLEncoder.encode可以防止中文乱码 当然和easyexcel没有关系 + String fileName = URLEncoder.encode(data.getFileName(), "UTF-8"); + response.setHeader("Content-disposition", "attachment;filename=" + fileName + ".xlsx"); + // 这里需要设置不关闭流 + EasyExcel.write(response.getOutputStream()).head(head(data.getHeadMap())).sheet(data.getFileName()).doWrite(dataList(data.getDataList(), data.getDataStrMap())); + } catch (Exception e) { + // 重置response + response.reset(); + response.setContentType("application/json"); + response.setCharacterEncoding("utf-8"); + Map map = new HashMap(); + map.put("status", "failure"); + map.put("message", "下载文件失败" + e.getMessage()); + response.getWriter().println(JSON.toJSONString(map)); + } + } + + //创建对象的导出 + public void simpleWrite(SimpleWriteData data,Class clazz, HttpServletResponse response) throws IOException { + // 这里注意 有同学反应使用swagger 会导致各种问题，请直接用浏览器或者用postman +// response.setContentType("application/vnd.ms-excel"); + response.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + response.setCharacterEncoding("utf-8"); + // 这里URLEncoder.encode可以防止中文乱码 当然和easyexcel没有关系 + String fileName = URLEncoder.encode(data.getFileName(), "UTF-8"); + response.setHeader("Content-disposition", "attachment;filename=" + fileName + ".xlsx"); + EasyExcel.write(response.getOutputStream(), clazz).sheet(data.getFileName()).doWrite(data.getDataList()); + } + + //设置表头 + private List> head(String[] headMap) { + List> list = new ArrayList>(); + + for (String head : headMap) { + List headList = new ArrayList(); + headList.add(head); + list.add(headList); + } + return list; + } + + //设置导出的数据内容 + private List> dataList(List dataList, String[] dataStrMap) { + List> list = new ArrayList>(); + for (JSONObject map : dataList) { + List data = new ArrayList(); + for (int i = 0; i < dataStrMap.length; i++) { + data.add(map.get(dataStrMap[i])); + } + list.add(data); + } + return list; + } +} + +@Data +class NoModelWriteData implements Serializable { + private String fileName;//文件名 + private String[] headMap;//表头数组 + private String[] dataStrMap;//对应数据字段数组 + private List dataList;//数据集合 +} + +@Data +class SimpleWriteData implements Serializable { + private String fileName;//文件名 + private List dataList;//数据列表 +} diff --git a/src/main/java/com/pancm/excel/EasyExcelWriteTest.java b/src/main/java/com/pancm/excel/EasyExcelWriteTest.java new file mode 100644 index 0000000..abaa8bd --- /dev/null +++ b/src/main/java/com/pancm/excel/EasyExcelWriteTest.java @@ -0,0 +1,115 @@ +package com.pancm.excel; + +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.ExcelWriter; +import com.alibaba.excel.annotation.ExcelIgnore; +import com.alibaba.excel.annotation.ExcelProperty; +import com.alibaba.excel.write.metadata.WriteSheet; +import com.alibaba.fastjson.JSONObject; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/** + * @author pancm + * @Title: pancm_project + * @Description: excel写模块测试 + * @Version:1.0.0 + * @Since:jdk1.8 + * @date 2022/3/2 + */ +@Slf4j +public class EasyExcelWriteTest { + + + public static void main(String[] args) { + List jsonObjectList = new ArrayList<>(); + JSONObject jsonObject = new JSONObject(); + jsonObject.put("t1",1); + jsonObject.put("t2","2"); + JSONObject jsonObject2 = new JSONObject(); + jsonObject2.put("t1",11); + jsonObject2.put("t2","22"); + jsonObjectList.add(jsonObject); + jsonObjectList.add(jsonObject2); + + String fileName = "simpleWrite" + System.currentTimeMillis() + ".xlsx"; + // 这里 需要指定写用哪个class去写，然后写到第一个sheet，名字为模板 然后文件流会自动关闭 + // 如果这里想使用03 则 传入excelType参数即可 + EasyExcel.write(fileName).sheet("模板").doWrite(jsonObjectList); + + } + + + public void excludeOrIncludeWrite() { + // 注意 simpleWrite在数据量不大的情况下可以使用（5000以内，具体也要看实际情况），数据量大参照 重复多次写入 + + // 写法1 JDK8+ + // since: 3.0.0-beta1 + String fileName = "/home"+ "simpleWrite" + System.currentTimeMillis() + ".xlsx"; + // 这里 需要指定写用哪个class去写，然后写到第一个sheet，名字为模板 然后文件流会自动关闭 + // 如果这里想使用03 则 传入excelType参数即可 +// EasyExcel.write(fileName, DemoData.class) +// .sheet("模板") +// .doWrite( +// () -> { +// // 分页查询数据 +// return data(); +// }); + + // 写法2 + fileName = "/home"+ "simpleWrite" + System.currentTimeMillis() + ".xlsx"; + // 这里 需要指定写用哪个class去写，然后写到第一个sheet，名字为模板 然后文件流会自动关闭 + // 如果这里想使用03 则 传入excelType参数即可 + EasyExcel.write(fileName, DemoData.class).sheet("模板").doWrite(data()); + + // 写法3 + fileName = "/home"+ "/simpleWrite" + System.currentTimeMillis() + ".xlsx"; + // 这里 需要指定写用哪个class去写 + ExcelWriter excelWriter = null; + try { + excelWriter = EasyExcel.write(fileName, DemoData.class).build(); + WriteSheet writeSheet = EasyExcel.writerSheet("模板").build(); + excelWriter.write(data(), writeSheet); + } finally { + // 千万别忘记finish 会帮忙关闭流 + if (excelWriter != null) { + excelWriter.finish(); + } + } + } + + + private List data() { + List list = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + DemoData data = new DemoData(); + data.setString("字符串" + i); + data.setDate(new Date()); + data.setDoubleData(0.56); + list.add(data); + } + return list; + } + + @Data + @EqualsAndHashCode + class DemoData { + @ExcelProperty("字符串标题") + private String string; + @ExcelProperty("日期标题") + private Date date; + @ExcelProperty("数字标题") + private Double doubleData; + /** + * 忽略这个字段 + */ + @ExcelIgnore + private String ignore; + } +} + diff --git a/src/main/java/com/pancm/excel/package-info.java b/src/main/java/com/pancm/excel/package-info.java index 5becb8e..fb1e93b 100644 --- a/src/main/java/com/pancm/excel/package-info.java +++ b/src/main/java/com/pancm/excel/package-info.java @@ -1,8 +1,9 @@ /** -* @Title: package-info -* @Description: excel相关包 -* @Version:1.0.0 -* @author pancm -* @date 2019年2月28日 -*/ + * @Title: pancm_project + * @Description: excel的工具类 + * @Version:1.0.0 + * @Since:jdk1.8 + * @author pancm + * @date 2021/1/26 + */ package com.pancm.excel; \ No newline at end of file From b6783b702577eb9812efd46c91a12a0951838df3 Mon Sep 17 00:00:00 2001 From: pancm <1060589146@qq.com> Date: Thu, 8 Jun 2023 11:24:10 +0800 Subject: [PATCH 47/53] add ffmpeg --- .../java/com/pancm/ffmpeg/FFmpegTest.java | 66 ++++++++++++++++++ .../java/com/pancm/ffmpeg/FFmpegUtil.java | 69 +++++++++++++++++++ .../java/com/pancm/ffmpeg/package-info.java | 9 +++ 3 files changed, 144 insertions(+) create mode 100644 src/main/java/com/pancm/ffmpeg/FFmpegTest.java create mode 100644 src/main/java/com/pancm/ffmpeg/FFmpegUtil.java create mode 100644 src/main/java/com/pancm/ffmpeg/package-info.java diff --git a/src/main/java/com/pancm/ffmpeg/FFmpegTest.java b/src/main/java/com/pancm/ffmpeg/FFmpegTest.java new file mode 100644 index 0000000..75d8674 --- /dev/null +++ b/src/main/java/com/pancm/ffmpeg/FFmpegTest.java @@ -0,0 +1,66 @@ +package com.pancm.ffmpeg; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +/** + * @author pancm + * @Title: FFmpegTest + * @Description: FFmpeg获取视频图片 + * 调用FFmpeg的命令获取ts视频的图片 + * @Version:1.0.0 + * @Since:jdk1.8 + * @Date 2021/4/14 + **/ +public class FFmpegTest { + + + public static void main(String[] args) { + + String ffmpegExePath = "C:\\ffmpeg\\bin\\ffmpeg.exe"; + + String inputFilePath = "D:\\video\\ts\\25-16_940.ts"; + + String outputFilePath = "D:\\video\\ts\\t2.jpg"; + + List command = new ArrayList(); + command.add(ffmpegExePath); + command.add("-i"); + command.add(inputFilePath); + command.add("-f"); + command.add("image2"); + command.add("-ss"); + command.add("1"); + command.add("-t"); + command.add("0.001"); + command.add("-s"); + command.add("320*240"); + command.add(outputFilePath); + ProcessBuilder builder = new ProcessBuilder(); + builder.command(command); + //正常信息和错误信息合并输出 + builder.redirectErrorStream(true); + try { + //开始执行命令 + Process process = builder.start(); + //如果你想获取到执行完后的信息，那么下面的代码也是需要的 + StringBuffer sbf = new StringBuffer(); + String line = null; + BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream())); + while ((line = br.readLine()) != null) { + sbf.append(line); + sbf.append(" "); + } + String resultInfo = sbf.toString(); + System.out.println(resultInfo); + } catch (IOException e) { + e.printStackTrace(); + } + + } + + +} diff --git a/src/main/java/com/pancm/ffmpeg/FFmpegUtil.java b/src/main/java/com/pancm/ffmpeg/FFmpegUtil.java new file mode 100644 index 0000000..af038d2 --- /dev/null +++ b/src/main/java/com/pancm/ffmpeg/FFmpegUtil.java @@ -0,0 +1,69 @@ +package com.pancm.ffmpeg; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +/** + * @author pancm + * @Title: gb28181_platform + * @Description: FFmpeg相关的工具类 + * @Version:1.0.0 + * @Since:jdk1.8 + * @date 2021/4/15 + */ +public class FFmpegUtil { + + + /** + * 执行ffmpeg的项目命令 + * @param cmdList + * @throws IOException + */ + public static void exec(List cmdList) throws IOException { + BufferedReader br = null; + try { + ProcessBuilder builder = new ProcessBuilder(); + builder.command(cmdList); + //正常信息和错误信息合并输出 + builder.redirectErrorStream(true); + //开始执行命令 + Process process = builder.start(); + StringBuffer sbf = new StringBuffer(); + String line = null; + br = new BufferedReader(new InputStreamReader(process.getInputStream())); + while ((line = br.readLine()) != null) { + sbf.append(line); + sbf.append(" "); + } + String resultInfo = sbf.toString(); + System.out.println(resultInfo); + } finally { + if (br != null) { + br.close(); + } + } + } + + public static void main(String[] args) throws IOException { + String ffmpegExePath = "C:\\ffmpeg\\bin\\ffmpeg.exe"; + String inputFilePath = "D:\\video\\ts\\25-16_940.ts"; + String outputFilePath = "D:\\video\\ts\\t3.jpg"; + List command = new ArrayList(); + command.add(ffmpegExePath); + command.add("-i"); + command.add(inputFilePath); + command.add("-f"); + command.add("image2"); + command.add("-ss"); + command.add("1"); + command.add("-t"); + command.add("0.001"); + command.add("-s"); + command.add("640*480"); + command.add(outputFilePath); + exec(command); + } +} diff --git a/src/main/java/com/pancm/ffmpeg/package-info.java b/src/main/java/com/pancm/ffmpeg/package-info.java new file mode 100644 index 0000000..c181f44 --- /dev/null +++ b/src/main/java/com/pancm/ffmpeg/package-info.java @@ -0,0 +1,9 @@ +/** + * @Title: pancm_project + * @Description: ffmpeg的工具类 + * @Version:1.0.0 + * @Since:jdk1.8 + * @author pancm + * @date 2021/1/26 + */ +package com.pancm.ffmpeg; \ No newline at end of file From ef5dc3deb8c653229f0d2c3c87ceb25c88148d1e Mon Sep 17 00:00:00 2001 From: pancm <1060589146@qq.com> Date: Thu, 8 Jun 2023 11:24:25 +0800 Subject: [PATCH 48/53] add ftp --- src/main/java/com/pancm/ftp/FtpHelper.java | 96 +++++++++++++++++++ .../java/com/pancm/ftp/SmartSshUtils.java | 88 +++++++++++++++++ src/main/java/com/pancm/ftp/package-info.java | 9 ++ 3 files changed, 193 insertions(+) create mode 100644 src/main/java/com/pancm/ftp/FtpHelper.java create mode 100644 src/main/java/com/pancm/ftp/SmartSshUtils.java create mode 100644 src/main/java/com/pancm/ftp/package-info.java diff --git a/src/main/java/com/pancm/ftp/FtpHelper.java b/src/main/java/com/pancm/ftp/FtpHelper.java new file mode 100644 index 0000000..1b36067 --- /dev/null +++ b/src/main/java/com/pancm/ftp/FtpHelper.java @@ -0,0 +1,96 @@ +package com.pancm.ftp; + +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.net.ftp.FTPFile; +import org.apache.commons.net.ftp.FTPReply; + +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author pancm + * @Title: leakproof-server + * @Description: ftp帮助类 + * @Version:1.0.0 + * @Since:jdk1.8 + * @date 2021/8/18 + */ +public class FtpHelper { + + /** + * 获取文件列表文件的属性 + * + * @param ip + * @param port + * @param user + * @param pwd + * @param url + * @return + */ + public static List> getListFiles(String ip, int port, String user, String pwd, String url) throws IOException { + List> mapList = new ArrayList<>(); + FTPClient ftpClient = new FTPClient(); + ftpClient.connect(ip, port); + ftpClient.login(user, pwd); + FTPFile[] ftpFiles = ftpClient.listFiles(url); + if(ftpFiles!=null && ftpFiles.length>0) { + for (FTPFile ftpFile : ftpFiles) { + Map map = new HashMap<>(); + map.put("fileName",ftpFile.getName()); + map.put("fileSize",getSize(ftpFile.getSize())); + map.put("fileTime",ftpFile.getTimestamp().getTime().toString()); + mapList.add(map); + } + } + return mapList; + } + + + + private static boolean testFtp(String ip, int port, String user, String pwd) throws IOException { + FTPClient ftpClient = new FTPClient(); + ftpClient.connect(ip, port);//连接ftp + ftpClient.login(user, pwd);//登陆ftp + return FTPReply.isPositiveCompletion(ftpClient.getReplyCode()); + } + + + + + public static String getSize(long size) { + //获取到的size为：1705230 + long GB = 1024 * 1024 * 1024;//定义GB的计算常量 + long MB = 1024 * 1024;//定义MB的计算常量 + long KB = 1024;//定义KB的计算常量 + DecimalFormat df = new DecimalFormat("0.00");//格式化小数 + String resultSize = ""; + if (size / GB >= 1) { + //如果当前Byte的值大于等于1GB + resultSize = df.format(size / (float) GB) + "GB"; + } else if (size / MB >= 1) { + //如果当前Byte的值大于等于1MB + resultSize = df.format(size / (float) MB) + "MB"; + } else if (size / KB >= 1) { + //如果当前Byte的值大于等于1KB + resultSize = df.format(size / (float) KB) + "KB"; + } else { + resultSize = size + "B"; + } + return resultSize; + } + + + public static void main(String[] args) throws Exception { + String ip = "192.168.10.90"; + int port = 21; + String user = "root"; + String pwd = "lgwy@2020"; + String url = "/home/userfile/admin"; + System.out.println(testFtp(ip,port,user,pwd)); + System.out.println(getListFiles(ip,port,user,pwd,url)); + } +} diff --git a/src/main/java/com/pancm/ftp/SmartSshUtils.java b/src/main/java/com/pancm/ftp/SmartSshUtils.java new file mode 100644 index 0000000..68ef080 --- /dev/null +++ b/src/main/java/com/pancm/ftp/SmartSshUtils.java @@ -0,0 +1,88 @@ +package com.pancm.ftp; + +import lombok.extern.slf4j.Slf4j; +import net.schmizz.sshj.SSHClient; +import net.schmizz.sshj.sftp.SFTPClient; +import net.schmizz.sshj.transport.verification.PromiscuousVerifier; + +import java.io.IOException; + + +@Slf4j +public final class SmartSshUtils { + + public static boolean testSFTP(String hostName, + String username, + String password){ + SSHClient ssh = new SSHClient(); + SFTPClient sftpClient = null; + try { + //ssh.loadKnownHosts(); to skip host verification + ssh.addHostKeyVerifier(new PromiscuousVerifier()); + ssh.connect(hostName); + ssh.authPassword(username, password); + sftpClient = ssh.newSFTPClient(); + return true; + }catch (IOException e) { + e.printStackTrace(); + } + + return false; + } + + + public static void downLoadFileBySsh(String hostName, + String username, + String password, + String srcFilePath, + String targetFilePath + ) { + SSHClient ssh = new SSHClient(); + SFTPClient sftpClient = null; + try { + //ssh.loadKnownHosts(); to skip host verification + ssh.addHostKeyVerifier(new PromiscuousVerifier()); + ssh.connect(hostName); + ssh.authPassword(username, password); + sftpClient = ssh.newSFTPClient(); + sftpClient.get(srcFilePath, targetFilePath); + //create a folder +// sftpClient.mkdir("/opt/app/testFolder"); + //sftpClient.mkdirs("");创建多级文件夹 + //sftpClient.rmdir("");重命名文件夹 + //sftpClient.ls(""); //列出当前目录 + } catch (IOException e) { + log.error(e.getMessage(), e); + } finally { + if (null != sftpClient) { + try { + sftpClient.close(); + } catch (IOException e) { + log.error(e.getMessage(), e); + } + } + try { + ssh.disconnect(); + } catch (IOException e) { + log.error(e.getMessage(), e); + } + } + } + + /** + * 静态工具类应该禁用构造方法 + */ + private SmartSshUtils(){} + + + public static void main(String[] args) { + String hostName="192.168.9.80"; + String username="root"; + String password="Admin#12$34!"; + String srcFilePath="/home/release/file"; + String targetFilePath="D:\\d1"; + + SmartSshUtils.downLoadFileBySsh(hostName,username,password,srcFilePath,targetFilePath); + + } +} diff --git a/src/main/java/com/pancm/ftp/package-info.java b/src/main/java/com/pancm/ftp/package-info.java new file mode 100644 index 0000000..4a57335 --- /dev/null +++ b/src/main/java/com/pancm/ftp/package-info.java @@ -0,0 +1,9 @@ +/** + * @Title: pancm_project + * @Description: ftp的工具类 + * @Version:1.0.0 + * @Since:jdk1.8 + * @author pancm + * @date 2021/1/26 + */ +package com.pancm.ftp; \ No newline at end of file From a0336482755e0dab693c1752ed3f61d9b6d8c111 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Tue, 26 Dec 2023 16:48:13 +0800 Subject: [PATCH 49/53] =?UTF-8?q?1.=E6=9B=B4=E6=96=B0readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 3deef01..ae50e7e 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,9 @@ - [手记系列之二 ----- 关于IDEA的一些使用方法经验](https://www.cnblogs.com/xuwujing/p/16862451.html) - [手记系列之三 ----- 关于使用Nginx的一些使用方法和经验](https://www.cnblogs.com/xuwujing/p/16885964.html) - [手记系列之四 ----- 关于使用MySql的经验](https://www.cnblogs.com/xuwujing/p/17356379.html) +- [手记系列之五 ----- SQL使用经验分享](https://www.cnblogs.com/xuwujing/p/17444266.html) +- [手记系列之六 ----- 分享个人使用kafka经验](https://www.cnblogs.com/xuwujing/p/17466519.html) +- [手记系列之七 ----- 分享Linux使用经验](https://www.cnblogs.com/xuwujing/p/17807802.html) @@ -116,6 +119,7 @@ - [纵然前路坎坷，也要毅然前行！2020年终总结！](https://www.cnblogs.com/xuwujing/p/14233270.html) - [有一点思考的2021年终总结！](https://www.cnblogs.com/xuwujing/p/15746791.html) - [一个想活得简单的程序猿的2022年终总结！](https://www.cnblogs.com/xuwujing/p/17060965.html) +- [写给步入三十的自己，2023年终总结!](https://www.cnblogs.com/xuwujing/p/17868627.html) ## 其他 From d26489d68f9438d94604c2cfb2641400a13c9c47 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Fri, 29 Dec 2023 14:47:39 +0800 Subject: [PATCH 50/53] Changes --- pom.xml | 9 +- .../java/com/pancm/ftp/SmartSshUtils.java | 176 +++++++++--------- 2 files changed, 96 insertions(+), 89 deletions(-) diff --git a/pom.xml b/pom.xml index b0792c2..2029aef 100644 --- a/pom.xml +++ b/pom.xml @@ -223,12 +223,19 @@ - + + + com.microsoft.sqlserver + mssql-jdbc + 6.2.0.jre8 + test + diff --git a/src/main/java/com/pancm/ftp/SmartSshUtils.java b/src/main/java/com/pancm/ftp/SmartSshUtils.java index 68ef080..2b4d124 100644 --- a/src/main/java/com/pancm/ftp/SmartSshUtils.java +++ b/src/main/java/com/pancm/ftp/SmartSshUtils.java @@ -1,88 +1,88 @@ -package com.pancm.ftp; - -import lombok.extern.slf4j.Slf4j; -import net.schmizz.sshj.SSHClient; -import net.schmizz.sshj.sftp.SFTPClient; -import net.schmizz.sshj.transport.verification.PromiscuousVerifier; - -import java.io.IOException; - - -@Slf4j -public final class SmartSshUtils { - - public static boolean testSFTP(String hostName, - String username, - String password){ - SSHClient ssh = new SSHClient(); - SFTPClient sftpClient = null; - try { - //ssh.loadKnownHosts(); to skip host verification - ssh.addHostKeyVerifier(new PromiscuousVerifier()); - ssh.connect(hostName); - ssh.authPassword(username, password); - sftpClient = ssh.newSFTPClient(); - return true; - }catch (IOException e) { - e.printStackTrace(); - } - - return false; - } - - - public static void downLoadFileBySsh(String hostName, - String username, - String password, - String srcFilePath, - String targetFilePath - ) { - SSHClient ssh = new SSHClient(); - SFTPClient sftpClient = null; - try { - //ssh.loadKnownHosts(); to skip host verification - ssh.addHostKeyVerifier(new PromiscuousVerifier()); - ssh.connect(hostName); - ssh.authPassword(username, password); - sftpClient = ssh.newSFTPClient(); - sftpClient.get(srcFilePath, targetFilePath); - //create a folder -// sftpClient.mkdir("/opt/app/testFolder"); - //sftpClient.mkdirs("");创建多级文件夹 - //sftpClient.rmdir("");重命名文件夹 - //sftpClient.ls(""); //列出当前目录 - } catch (IOException e) { - log.error(e.getMessage(), e); - } finally { - if (null != sftpClient) { - try { - sftpClient.close(); - } catch (IOException e) { - log.error(e.getMessage(), e); - } - } - try { - ssh.disconnect(); - } catch (IOException e) { - log.error(e.getMessage(), e); - } - } - } - - /** - * 静态工具类应该禁用构造方法 - */ - private SmartSshUtils(){} - - - public static void main(String[] args) { - String hostName="192.168.9.80"; - String username="root"; - String password="Admin#12$34!"; - String srcFilePath="/home/release/file"; - String targetFilePath="D:\\d1"; - - SmartSshUtils.downLoadFileBySsh(hostName,username,password,srcFilePath,targetFilePath); - - } -} +//package com.pancm.ftp; +// +//import lombok.extern.slf4j.Slf4j; +//import net.schmizz.sshj.SSHClient; +//import net.schmizz.sshj.sftp.SFTPClient; +//import net.schmizz.sshj.transport.verification.PromiscuousVerifier; +// +//import java.io.IOException; +// +// +//@Slf4j +//public final class SmartSshUtils { +// +// public static boolean testSFTP(String hostName, +// String username, +// String password){ +// SSHClient ssh = new SSHClient(); +// SFTPClient sftpClient = null; +// try { +// //ssh.loadKnownHosts(); to skip host verification +// ssh.addHostKeyVerifier(new PromiscuousVerifier()); +// ssh.connect(hostName); +// ssh.authPassword(username, password); +// sftpClient = ssh.newSFTPClient(); +// return true; +// }catch (IOException e) { +// e.printStackTrace(); +// } +// +// return false; +// } +// +// +// public static void downLoadFileBySsh(String hostName, +// String username, +// String password, +// String srcFilePath, +// String targetFilePath +// ) { +// SSHClient ssh = new SSHClient(); +// SFTPClient sftpClient = null; +// try { +// //ssh.loadKnownHosts(); to skip host verification +// ssh.addHostKeyVerifier(new PromiscuousVerifier()); +// ssh.connect(hostName); +// ssh.authPassword(username, password); +// sftpClient = ssh.newSFTPClient(); +// sftpClient.get(srcFilePath, targetFilePath); +// //create a folder +//// sftpClient.mkdir("/opt/app/testFolder"); +// //sftpClient.mkdirs("");创建多级文件夹 +// //sftpClient.rmdir("");重命名文件夹 +// //sftpClient.ls(""); //列出当前目录 +// } catch (IOException e) { +// log.error(e.getMessage(), e); +// } finally { +// if (null != sftpClient) { +// try { +// sftpClient.close(); +// } catch (IOException e) { +// log.error(e.getMessage(), e); +// } +// } +// try { +// ssh.disconnect(); +// } catch (IOException e) { +// log.error(e.getMessage(), e); +// } +// } +// } +// +// /** +// * 静态工具类应该禁用构造方法 +// */ +// private SmartSshUtils(){} +// +// +// public static void main(String[] args) { +// String hostName="192.168.9.80"; +// String username="root"; +// String password="Admin#12$34!"; +// String srcFilePath="/home/release/file"; +// String targetFilePath="D:\\d1"; +// +// SmartSshUtils.downLoadFileBySsh(hostName,username,password,srcFilePath,targetFilePath); +// +// } +//} From dd422e13aa8cf72e1b305db847d3b95f3f64389c Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Wed, 17 Jan 2024 14:49:19 +0800 Subject: [PATCH 51/53] Changes --- src/main/java/com/pancm/redis/RedisTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/pancm/redis/RedisTest.java b/src/main/java/com/pancm/redis/RedisTest.java index 10eed50..0b27397 100644 --- a/src/main/java/com/pancm/redis/RedisTest.java +++ b/src/main/java/com/pancm/redis/RedisTest.java @@ -25,7 +25,6 @@ public static void main(String[] args) { System.out.println("连接成功"); // 查看服务是否运行 System.out.println("服务正在运行: " + jedis.ping()); - // 存储到列表中 jedis.lpush("list", "redis"); jedis.lpush("list", "java"); @@ -35,7 +34,6 @@ public static void main(String[] args) { for (int i = 0, j = list.size(); i < j; i++) { System.out.println("list的输出结果:" + list.get(i)); } - // 设置 redis 字符串数据 jedis.set("rst", "redisStringTest"); // 获取存储的数据并输出 From 1ce0ec06f58c95443470e9cd0adc51a74c483186 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Thu, 18 Jan 2024 15:46:40 +0800 Subject: [PATCH 52/53] Changes --- src/main/java/com/pancm/excel/EasyExcelMergeTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/pancm/excel/EasyExcelMergeTest.java b/src/main/java/com/pancm/excel/EasyExcelMergeTest.java index ea1556b..46da597 100644 --- a/src/main/java/com/pancm/excel/EasyExcelMergeTest.java +++ b/src/main/java/com/pancm/excel/EasyExcelMergeTest.java @@ -123,10 +123,8 @@ private List getGroupCountList(List exportDataList) { if (CollectionUtils.isEmpty(exportDataList)) { return new ArrayList<>(); } - List groupCountList = new ArrayList<>(); int count = 1; - for (int i = 1; i < exportDataList.size(); i++) { if (exportDataList.get(i).equals(exportDataList.get(i - 1))) { count++; From b807ffea96132c54200a7c1414d9db1748aa5c12 Mon Sep 17 00:00:00 2001 From: xuwujing <1060589146@qq.com> Date: Wed, 24 Jan 2024 16:28:09 +0800 Subject: [PATCH 53/53] Changes --- .../java/com/pancm/elasticsearch/EsHighLevelRestTest2.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest2.java b/src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest2.java index 68a339a..a42595b 100644 --- a/src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest2.java +++ b/src/main/java/com/pancm/elasticsearch/EsHighLevelRestTest2.java @@ -96,13 +96,10 @@ private static void multiGet() throws IOException { request.add(new MultiGetRequest.Item("user", "userindex", "2")); // 禁用源检索，默认启用 // request.add(new MultiGetRequest.Item("user", "userindex", "2").fetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE)); - // 同步构建 MultiGetResponse response = client.mget(request, RequestOptions.DEFAULT); - // 异步构建 // MultiGetResponse response2 = client.mgetAsync(request, RequestOptions.DEFAULT, listener); - /* * 返回的MultiGetResponse包含在' getResponses中的MultiGetItemResponse的列表，其顺序与请求它们的顺序相同。 * 如果成功，MultiGetItemResponse包含GetResponse或MultiGetResponse。如果失败了就失败。