springboot整合elasticsearch2.0+logstach-干货满满实战篇(上)

一 准备

1 环境搭建

目的: 使用es实现大批量数据的中文搜索，使用logstach进行大批量数据同步到es。
JDK1.8
Tomcat8
elasticsearch +kibana5.2.2 +logstach (环境请自行搭建)

2 在elastchsearch创建索引

首先理解mysql和es各个关键词的关系，如下图

3 es创建索引

索引名称为testIndex，表Type为app (mysql数据库请自行创建)
字段有: id、name、namePinyin、source、type、updateTime
注意: 此脚本需要在kibana上执行、创建索引(每次tomcat启动也会自行创建-如果此所以不存在)

PUT testIndexl
{
  "testIndex": {
    "aliases": {},
    "mappings": {
      "app": {
        "properties": {
          "id": {
            "type": "long"
          },
          "name": {
            "type": "text",
            "store": true,
            "analyzer": "ik_max_word"
          },
          "namePinyin": {
            "type": "text",
            "store": true,
            "analyzer": "pinyin"
          },
          "source": {
            "type": "long"
          },
          "type": {
            "type": "text",
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "updateTime": {
            "type": "date"
          }  
       }
      }
    },
    "settings": {
      "index": {
        "refresh_interval": "1s",
        "number_of_shards": "5",
        "provided_name": "testIndex",
        "creation_date": "1546568298797",
        "store": {
          "type": "fs",
          "type": "fs"
        },
        "number_of_replicas": "1",
        "uuid": "21321",
        "version": {
          "created": "5020299"
        }
      }
    }
  }
}

二 代码

1 elasticsearch依赖

         <!--elasticsearch依赖 -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
            <version>2.0.6.RELEASE</version>
        </dependency>

        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>transport</artifactId>
            <version>5.6.10</version>
        </dependency>

        <!--需要引入transport-netty3-client，否则会启动报错-->
        <dependency>
            <groupId>org.elasticsearch.plugin</groupId>
            <artifactId>transport-netty3-client</artifactId>
            <version>5.6.10</version>
        </dependency>

2 初始化执行配置-ElasticsearchConfig

项目启动初始化、防止报错

@Configuration
public class ElasticsearchConfig {
    /**
     * 防止netty的bug
     *  * java.lang.IllegalStateEateException: availableProcessors is already set to [4], rejecting [4]
     *  System.setProperty("es.set.netty.runtime.available.processors", "false");   // 解决:Error creating bean with name 'elasticsearchClient'
     */
    @PostConstruct  //初始化前执行
    void init() {
        System.setProperty("es.set.netty.runtime.available.processors", "false");
    }
}

3 启动类-Application

@SpringBootApplication
@EnableAspectJAutoProxy(proxyTargetClass = true, exposeProxy = true)
@MapperScan(basePackages = "")
@EnableSwagger2     //启动swagger
public class Application extends SpringBootServletInitializer {

    @Override
    protected SpringApplicationBuilder configure(SpringApplicationBuilder builder) {
        return builder.sources(Application.class);
    }

    public static void main(String[] args) {

        System.setProperty("es.set.netty.runtime.available.processors", "false");   // 解决:Error creating bean with name 'elasticsearchClient'
        SpringApplication.run(Application.class, args);
    }
}

4 实体类-App

从mysql中的app导入数据到es的app、根据自己的需求做映射(mysql<->es)

import lombok.Data;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;

import java.io.Serializable;
import java.math.BigDecimal;
import java.util.Date;

@Data
@Document(indexName = "testIndexl", type = "app" )
public class App implements Serializable {
    @Id
    private Long id;

    private String appThirdId;

    @Field(type = FieldType.Text, index = true, store = true, fielddata = false, searchAnalyzer = "ik_max_word", analyzer = "ik_max_word", includeInParent = false)
    private String name;

    @Field(type = FieldType.Text, index = true, store = true, fielddata = false, searchAnalyzer = "pinyin", analyzer = "pinyin", includeInParent = false)
    private String namePinyin;
}

5 实体操作类-AppRepository

import com.konka.appstoreserver.model.po.App;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
import org.springframework.stereotype.Repository;

/**
 * Created by zhiji on 2018/11/26.
 */
@Repository()
public interface AppRepository extends ElasticsearchRepository<App, Long> {
}

6 es搜索-EsServiceImpl

@Slf4j
@Service
public class EsServiceImpl {
	

    /**
     * app表的name属性
     */
    private static final String APP_FIELD_NAME = "name";

    /**
     * app表的name属性
     */
    private static final String APP_FIELD_NAMEPINYIN = "namePinyin";

    /**
     * 是否删除
     */
    private static final String APP_FIELD_IS_DELETED = "isDeleted";

    /**
     * 是否删除
     */
    private static final String APP_FIELD_ONLINE_STATE = "onlineState";

    /**
     * category表的name属性
     */
    private static final String CATEGORY_FIELD_NAME = "name";

    @Autowired
    private AppRepository appRepository;
	
 public  Iterable<App> queryApp(AppQueryAppInDTO appQueryAppInDTO) {

        //分页参数
        PageRequest pageRequest = new PageRequest(appQueryAppInDTO.getFrom(), appQueryAppInDTO.getSize());

        //1 验参
        String name = appQueryAppInDTO.getName();
        Boolean containChinese = JavaStringUtils.isContainChinese(name);

        // 2 查询
        // 2.1 中文查询
        BoolQueryBuilder boolQueryBuilder = null;
        if (containChinese) {
            boolQueryBuilder = QueryBuilders.boolQuery()
                    .must(QueryBuilders.termQuery(APP_FIELD_IS_DELETED, 0))   //未删除
                    .should(QueryBuilders.matchQuery(APP_FIELD_NAME, name));   // 先分词再查询
        }
        // 2.2 拼音查询
        else {
            boolQueryBuilder = QueryBuilders.boolQuery()
                    .mustNot(QueryBuilders.termQuery(APP_FIELD_IS_DELETED, 1))
                    .should(QueryBuilders.fuzzyQuery(APP_FIELD_NAMEPINYIN, name))   // 模糊匹配
                    .should(QueryBuilders.matchQuery(APP_FIELD_NAMEPINYIN, name));  //分词匹配

//                    .should(QueryBuilders.multiMatchQuery("namePinyin", name))    // 完全匹配
//                    .should(QueryBuilders.wildcardQuery("namePinyin", "" + name + "*"));  //通配符匹配
        }
        // 排序 - start
        ArrayList<SortBuilder> sortBuilders = new ArrayList<>();
        ScoreSortBuilder order_score = SortBuilders.scoreSort().order(SortOrder.DESC);  //先按分数排序
//        FieldSortBuilder order_id = SortBuilders.fieldSort("id").order(SortOrder.DESC);   //再按名称排序
        FieldSortBuilder download_times = SortBuilders.fieldSort("downloadTimes").order(SortOrder.DESC);   //再按名称排序

        sortBuilders.add(order_score);  //优先级最高
//        sortBuilders.add(order_id);
        sortBuilders.add(download_times);
        NativeSearchQuery nativeSearchQuery = new NativeSearchQuery(boolQueryBuilder, null, sortBuilders);
        QueryBuilder query = nativeSearchQuery.getQuery();
        // 排序 end

        Iterable<App> search = appRepository.search(query, pageRequest);

        return search;
    }
}

三 下篇引入

如上已经将es的搜索环境搭建好了，下篇讲述使用logstash将mysql大批量数据导入es中。
地址: springboot整合elasticsearch2.0+logstach-干货满满实战篇-下