分词统计（一）使用讯飞语言云进行统计

本文链接：https://blog.csdn.net/iromkoear/article/details/60600190

最近想取一个网名，想起中国文化博大精深，如果用古代的唐诗宋词组合一定有一个意向不到的名字。组合首先要分词，想起锤子手机有一个很火的功能叫BigBang，它用的是讯飞的语言云，是免费提供的，所以这次使用讯飞的语言云进行分词，然后随机组合。另外，还可以进行有趣的数据统计，看看古代的高频诗词是哪些？

大概的步骤如下：

1、获取讯飞语言云ApiKey

2、搭建网络请求

3、选择诗词的txt文件

4、显示分词结果

5、保存结果到sqlite数据库

6、进行分词随机组合

7、利用分词进行数据统计

使用到的技术:

ButterKnife

Retrofit

android-filepicker 一个支持单选对选，可以选择文件或者目录的文件选择对话框，非常流畅

1 获取讯飞语言云ApiKey

首先去讯飞开发者平台申请语言云的ApiKey

2 搭建网络框架

在工程的build文件添加

  dependencies {
        classpath 'com.android.tools.build:gradle:2.3.0'
        classpath 'com.neenbedankt.gradle.plugins:android-apt:1.8'
    }

在项目的build文件添加

apply plugin: 'com.android.application'
apply plugin: 'com.neenbedankt.android-apt'
android {
    compileSdkVersion 25
    buildToolsVersion "25.0.0"
    defaultConfig {
        applicationId "com.mwf.analyze"
        minSdkVersion 15
        targetSdkVersion 19
        versionCode 1
        versionName "1.0"
        testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
    }
    buildTypes {
        release {
            minifyEnabled false
            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
        }
    }
}

dependencies {
    compile fileTree(dir: 'libs', include: ['*.jar'])
    compile 'com.android.support:appcompat-v7:25.0.0'
    testCompile 'junit:junit:4.12'
    compile 'com.android.support:design:25.0.0'
    compile 'com.android.support:appcompat-v7:25.0.0'
    compile 'com.squareup.okhttp3:okhttp:3.3.1'
    compile 'com.squareup.retrofit2:retrofit:2.1.0'
    compile 'com.squareup.retrofit2:converter-gson:2.1.0'
    compile 'com.squareup.retrofit2:adapter-rxjava:2.1.0'
    compile 'io.reactivex:rxandroid:1.1.0'
    compile 'io.reactivex:rxjava:1.1.0'
    compile 'com.jakewharton:butterknife:8.2.1'
    apt 'com.jakewharton:butterknife-compiler:8.2.1'
    compile 'com.squareup.okhttp3:logging-interceptor:3.3.1'
    compile 'com.github.angads25:filepicker:1.0.9'
}

添加一个常量文件Constant.java

package com.mwf.analyze;

/**
 *常量
 */
public class Constant {
    /**
     * 服务器地址
     */
    public  static String BASEURL="http://ltpapi.voicecloud.cn/";

    /**
     * 讯飞语言云的ApiKey
     */
    public  static String APIKEY="写上你申请的ApiKey";

}

添加一个网络请求的主体文件RetrofitWrapper.java

package com.mwf.analyze;

import android.content.Context;
import android.util.Log;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.mwf.analyze.utils.ToStringConverterFactory;

import okhttp3.OkHttpClient;
import okhttp3.logging.HttpLoggingInterceptor;
import retrofit2.Retrofit;

/**
 * 网络请求主体
 */
public class RetrofitWrapper {
    private static RetrofitWrapper instance;
    private Context mContext;
    private Retrofit mRetrofit;

    public RetrofitWrapper(String url) {
        OkHttpClient.Builder builder = new OkHttpClient.Builder();
      
        HttpLoggingInterceptor logging = new HttpLoggingInterceptor(new HttpLoggingInterceptor.Logger() {
            @Override
            public void log(String message) {
                // 打印Log
                Log.i("OkHttp", message);
            }
        });
        logging.setLevel(HttpLoggingInterceptor.Level.BODY);
        OkHttpClient client = builder.addInterceptor(logging)
                .build();

        Gson gson = new GsonBuilder()
                .setLenient()
                .create();

        mRetrofit = new Retrofit.Builder().baseUrl(url)
//                .addConverterFactory(GsonConverterFactory.create(gson))
                  .addConverterFactory(new ToStringConverterFactory())
//                .addCallAdapterFactory(RxJavaCallAdapterFactory.create())
                .client(client)
                .build();
    }

    public  static RetrofitWrapper getInstance(String url){

        if(null == instance){
            synchronized (RetrofitWrapper.class){
                instance = new RetrofitWrapper(url);
            }
        }
        return instance;
    }

    public <T> T create(final Class<T> service) {
        return mRetrofit.create(service);
    }
}

因为请求的结果是Sting文件，所以addConverterFactory为自定义的一个工具类ToStringConverterFactory.java

package com.mwf.analyze.utils;

import java.io.IOException;
import java.lang.annotation.Annotation;
import java.lang.reflect.Type;

import okhttp3.MediaType;
import okhttp3.RequestBody;
import okhttp3.ResponseBody;
import retrofit2.Converter;
import retrofit2.Retrofit;

/**
 * 此类将结果转为String
 */
public class ToStringConverterFactory extends Converter.Factory {
    private static final MediaType MEDIA_TYPE = MediaType.parse("text/plain");


    @Override
    public Converter<ResponseBody, ?> responseBodyConverter(Type type, Annotation[] annotations, Retrofit retrofit) {
        if (String.class.equals(type)) {
            return new Converter<ResponseBody, String>() {
                @Override
                public String convert(ResponseBody value) throws IOException {
                    return value.string();
                }
            };
        }
        return null;
    }

    @Override public Converter<?, RequestBody> requestBodyConverter(Type type, Annotation[] parameterAnnotations,
                                                                    Annotation[] methodAnnotations, Retrofit retrofit) {
        if (String.class.equals(type)) {
            return new Converter<String, RequestBody>() {
                @Override
                public RequestBody convert(String value) throws IOException {
                    return RequestBody.create(MEDIA_TYPE, value);
                }
            };
        }
        return null;
    }
}

网络请求单例：

package com.mwf.analyze.model;

import android.content.Context;

import com.mwf.analyze.Constant;
import com.mwf.analyze.RetrofitWrapper;
import com.mwf.analyze.bean.FamousInfoReq;
import com.mwf.analyze.intf.IFamousInfo;

import retrofit2.Call;

/**
 * 网络请求单例
 */
public class FamousInfoModel {
    private static FamousInfoModel famousInfoModel;
    private IFamousInfo mIFamousInfo;

    public FamousInfoModel(Context context) {
        mIFamousInfo = RetrofitWrapper.getInstance(Constant.BASEURL).create(IFamousInfo.class);
    }

    public static FamousInfoModel getInstance(Context context){
        if(famousInfoModel == null) {
            famousInfoModel = new FamousInfoModel(context);
        }
        return famousInfoModel;
    }

    public Call<String> queryLookUp(FamousInfoReq famousInfoReq) {
        Call<String > infoCall = mIFamousInfo.getFamousResult(famousInfoReq.api_key, famousInfoReq.text, famousInfoReq.pattern, famousInfoReq.format);
        return infoCall;
    }
}

请求参数接口

package com.mwf.analyze.intf;


import retrofit2.Call;
import retrofit2.http.GET;
import retrofit2.http.Query;

/**
 * 请求参数接口
 */
public interface IFamousInfo {

    @GET("/analysis")
    Call<String> getFamousResult(@Query("api_key") String api_key,
                                       @Query("text") String text,
                                       @Query("pattern") String pattern,
                                       @Query("format") String format);
}

请求的参数实体

package com.mwf.analyze.bean;

/**
 * 请求的参数实体
 */
public class FamousInfoReq {
    /**
     * 	用户注册语音云服务后获得的认证标识
     */
    public String api_key;

    /**
     * 待分析的文本
     */
    public String text;

    /**
     * 用以指定分析模式，可选值包括ws(分词)，pos(词性标注)，ner(命名实体识别)，
     * dp(依存句法分析)，srl(语义角色标注),all(全部任务)
     */
    public String pattern;  //

    /**
     * 用以指定结果格式类型，可选值包括xml(XML格式)，json(JSON格式)，conll(CONLL格式)，plain(简洁文本格式)
     */
    public String format;   //

    public String getApi_key() {
        return api_key;
    }

    public void setApi_key(String api_key) {
        this.api_key = api_key;
    }

    public String getText() {
        return text;
    }

    public void setText(String text) {
        this.text = text;
    }

    public String getPattern() {
        return pattern;
    }

    public void setPattern(String pattern) {
        this.pattern = pattern;
    }

    public String getFormat() {
        return format;
    }

    public void setFormat(String format) {
        this.format = format;
    }
}

主界面的布局

<?xml version="1.0" encoding="utf-8"?>
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
                android:layout_width="match_parent"
                android:layout_height="match_parent"
>

    <RelativeLayout
        android:id="@+id/search_layout"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        android:layout_margin="10dp"
        android:gravity="center_vertical"
        android:orientation="horizontal">

        <EditText
            android:id="@+id/edit_keyword"
            android:layout_width="match_parent"
            android:layout_height="120dp"
            android:layout_weight="1"
            android:hint="请输入文字"
            android:paddingLeft="10dp"
        />


    </RelativeLayout>
    <Button
        android:id="@+id/button_search"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_below="@id/search_layout"
        android:layout_centerHorizontal="true"
        android:layout_centerVertical="true"
        android:layout_marginLeft="5dp"
        android:layout_marginRight="15dp"
        android:text="输入框解析"/>
    <Button
        android:id="@+id/button_file"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_below="@id/button_search"
        android:layout_centerHorizontal="true"
        android:layout_centerVertical="true"
        android:layout_marginLeft="5dp"
        android:layout_marginRight="15dp"
        android:text="文件解析"/>
    <ScrollView android:layout_width="match_parent"
                android:layout_height="match_parent"
                android:layout_below="@id/button_file"
                android:layout_margin="10dp">
        <LinearLayout

            android:layout_width="match_parent"
            android:layout_height="wrap_content"
            android:orientation="vertical">
            <TextView
                android:layout_width="match_parent"
                android:layout_height="wrap_content"
                android:text="结果："
                android:textSize="20dp"/>
            <TextView
                android:id="@+id/txt_content"
                android:layout_width="match_parent"
                android:layout_height="wrap_content"
                android:textColor="#000000"/>

        </LinearLayout>
    </ScrollView>
</RelativeLayout>

主界面的代码

package com.mwf.analyze.activity;

import android.os.Bundle;
import android.support.v7.app.AppCompatActivity;
import android.text.TextUtils;
import android.util.Log;
import android.view.View;
import android.widget.Button;
import android.widget.EditText;
import android.widget.TextView;

import com.github.angads25.filepicker.controller.DialogSelectionListener;
import com.github.angads25.filepicker.model.DialogConfigs;
import com.github.angads25.filepicker.model.DialogProperties;
import com.github.angads25.filepicker.view.FilePickerDialog;
import com.mwf.analyze.Constant;
import com.mwf.analyze.R;
import com.mwf.analyze.bean.CloudResultPlainParse;
import com.mwf.analyze.bean.FamousInfoReq;
import com.mwf.analyze.model.FamousInfoModel;
import com.mwf.analyze.utils.FileUtils;

import java.io.File;
import java.util.ArrayList;

import butterknife.BindView;
import butterknife.ButterKnife;
import butterknife.OnClick;
import retrofit2.Call;
import retrofit2.Callback;
import retrofit2.Response;

public class MainActivity extends AppCompatActivity implements View.OnClickListener {

    public final String TAG = this.getClass().getName();
    private FamousInfoModel famousInfoModel;

    @BindView(R.id.edit_keyword)
    EditText mEditKeyWord;
    @BindView(R.id.button_search)
    Button mSerachBtn;
    @BindView(R.id.button_file)
    Button button_file;
    @BindView(R.id.txt_content)
    TextView mTxtContent;

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);
        ButterKnife.bind(this);
        famousInfoModel = FamousInfoModel.getInstance(this);
        getSupportActionBar().hide();
    }

    @Override
    @OnClick({R.id.button_search, R.id.button_file})
    public void onClick(View view) {
        if (view.getId() == R.id.button_search) {
            parseEditext();
        } else if (view.getId() == R.id.button_file) {
            parseFile();
        }
    }

    /**
     * 初始化请求参数
     * @param text
     * @return
     */
    private FamousInfoReq initParams(String text) {
        FamousInfoReq mFamousInfoReq = null;
        mFamousInfoReq = new FamousInfoReq();
        mFamousInfoReq.api_key = Constant.APIKEY;
        mFamousInfoReq.text = text;
        mFamousInfoReq.pattern = "ws";
        mFamousInfoReq.format = "plain";
        return mFamousInfoReq;
    }

    /**
     * 解析输入框内容
     */
    private void parseEditext() {
        String text = mEditKeyWord.getText().toString();

        famousInfoModel.queryLookUp(initParams(text)).enqueue(new Callback<String>() {
            @Override
            public void onResponse(Call<String> call, Response<String> response) {
                String result = response.body().trim();
                CloudResultPlainParse parse = new CloudResultPlainParse();
                ArrayList<String> list = parse.parse(result);
//                    Log.e(TAG, "result====" + result);
                String string = "";
                for (String tmp : list) {
                    string += tmp + "\n";
                    Log.e(TAG, tmp.toString());
                }
                mTxtContent.setText(string);
            }

            @Override
            public void onFailure(Call<String> call, Throwable t) {

            }
        });
    }


    /**
     * 选择文件解析
     */
    private void parseFile() {
        DialogProperties properties = new DialogProperties();
        properties.selection_mode = DialogConfigs.SINGLE_MODE;
        properties.selection_type = DialogConfigs.FILE_SELECT;
        properties.root = new File(DialogConfigs.DEFAULT_DIR);
        properties.error_dir = new File(DialogConfigs.DEFAULT_DIR);
        properties.offset = new File(DialogConfigs.DEFAULT_DIR);
        properties.extensions = null;

        FilePickerDialog dialog = new FilePickerDialog(MainActivity.this, properties);
        dialog.setTitle("Select a File");
        dialog.setDialogSelectionListener(new DialogSelectionListener() {
            @Override
            public void onSelectedFilePaths(String[] files) {
                String string = FileUtils.readTxtFile(files[0]);
                if (!TextUtils.isEmpty(string)) {
                    Log.e(TAG, string);
                    mEditKeyWord.setText(string);
                    parseF(string);
                }

            }
        });
        dialog.show();
    }

    /**
     * 解析文件的网络请求
     * @param text
     */
    private void parseF(String text) {
        famousInfoModel.queryLookUp(initParams(text)).enqueue(new Callback<String>() {
            @Override
            public void onResponse(Call<String> call, Response<String> response) {
                String result = response.body().trim();
                CloudResultPlainParse parse = new CloudResultPlainParse();
                ArrayList<String> list = parse.parse(result);
//                    Log.e(TAG, "result====" + result);
                String string = "";
                for (String tmp : list) {
                    string += tmp + "\n";
                    Log.e(TAG, tmp.toString());
                }
                mTxtContent.setText(string);
            }

            @Override
            public void onFailure(Call<String> call, Throwable t) {

            }
        });
    }
}

文件操作工具类

package com.mwf.analyze.utils;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;

/**
 * 文件管理工具
 */
public class FileUtils {
    /**
     * 功能：Java读取txt文件的内容
     * 步骤：1：先获得文件句柄
     * 2：获得文件句柄当做是输入一个字节码流，需要对这个输入流进行读取
     * 3：读取到输入流后，需要读取生成字节流
     * 4：一行一行的输出。readline()。
     * 备注：需要考虑的是异常情况
     * @param filePath
     */
    public static String readTxtFile(String filePath){
        try {
//            String encoding="GBK";
            String encoding="UTF-8";
            File file=new File(filePath);
            if(file.isFile() && file.exists()){ //判断文件是否存在
                InputStreamReader read = new InputStreamReader(
                        new FileInputStream(file),encoding);//考虑到编码格式
                BufferedReader bufferedReader = new BufferedReader(read);
                String lineTxt = null;
                String result="";
                while((lineTxt = bufferedReader.readLine()) != null){
                    System.out.println(lineTxt);
                    result+=lineTxt;
                }
                read.close();
                return  result;
            }else{
                System.out.println("找不到指定的文件");
                return  null;
            }
        } catch (Exception e) {
            System.out.println("读取文件内容出错");
            e.printStackTrace();
            return  null;
        }

    }
}

项目地址: 玩转数据统计