《理财市场情绪监测系统》代码实现【1】之行业词库

最新推荐文章于 2024-04-30 23:32:15 发布
最新推荐文章于 2024-04-30 23:32:15 发布
阅读量108 收藏 2
点赞数
文章标签： 大数据 数据库 java

系统的目标有两个：

一是行业分类；

二是每个行业的情感值计算；

①因为是基金推荐项目，分析了国民经济分类和同花顺，最终采用同花顺的66个行业分类为标准。

②行业分类依据词，词库采用从百度词库、搜狗细胞词库手工分析下载而来，下载而来的是未解析的格式，

我这里以‘深蓝词库转换’这个工具进行转换成对应的66个行业的66个txt，然后再用scala代码解析到mysql中，在MySQL中以words为索引建好表。

建表语句如下：

--  以名词关联查看结果
SELECT  w.`innersessionId`,w.`words`,c.`username`,c.`category` FROM  `words2` w JOIN category  c
ON w.`words`=c.`username`
  WHERE w.`sensi`='n'



--  创建category  初步结果存储表

CREATE TABLE `category` (
`id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,
`editdate` char(20) NOT NULL DEFAULT "20170622", 
`number`  CHAR(100) NOT NULL DEFAULT "spell", 
`category` CHAR(100) NOT NULL DEFAULT "综合", 
`words` CHAR(100) NOT NULL DEFAULT "name",
 PRIMARY KEY (`id`) 
) DEFAULT CHARSET=utf8;
ALTER TABLE `category` ADD INDEX(`words`)；
--  创建category2 带双索引的表作为分类结果存储



CREATE TABLE `category2` (
`id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,
`editdate` char(20) NOT NULL DEFAULT "20170622", 
`number`  CHAR(100) NOT NULL DEFAULT "spell", 
`category` CHAR(100) NOT NULL DEFAULT "综合", 
`words` CHAR(100) NOT NULL DEFAULT "name",
 PRIMARY KEY (`id`) 
) DEFAULT CHARSET=utf8;
ALTER TABLE `category` ADD INDEX(`words`)；


--- 填充到分类结果表
INSERT INTO category2(editdate,number,category,words) 
SELECT  DISTINCT DATE,number,category,username FROM category；




--- 创建源数据存储表


CREATE TABLE  title(
id INT(12) UNSIGNED NOT NULL AUTO_INCREMENT,
bizNo VARCHAR(100),
bizType VARCHAR(100),
bizTypeEnum VARCHAR(100),
channelEnum VARCHAR(100),
channelId VARCHAR(100),
channelType VARCHAR(100),
collectTime VARCHAR(100),
hostIp VARCHAR(100),
hostName VARCHAR(100),
innerSessionId VARCHAR(100),
logNo VARCHAR(100),
merchantNo VARCHAR(100),
modelNo VARCHAR(100),
operateType VARCHAR(100),
operateTypeEnum VARCHAR(100),
PROCESS VARCHAR(100),
productNo VARCHAR(100),
protocolVersion VARCHAR(100),
receiveTime VARCHAR(100),
sessionId VARCHAR(100),
sourcetitle VARCHAR(100),
systemCode VARCHAR(100),
taskType VARCHAR(100),
TIME VARCHAR(100),
title VARCHAR(200),
 PRIMARY KEY (`id`) 
) DEFAULT CHARSET=utf8;



val pp = value.map(p => {
        val v0 = p.getString(0).toInt
        val v1 = p.getInt(1).toString
        val v2 = p.getString(2)
        val v3 = p.getString(3)
        (v0,v1,v2,v3)
    })


---  创建解析title
CREATE TABLE titlesplit(id INT(12) UNSIGNED NOT NULL  AUTO_INCREMENT,
innserSessionid VARCHAR(50),
times VARCHAR(50),
channelType VARCHAR(50),
sourcetitle VARCHAR(500),
title VARCHAR(500),
words VARCHAR(500),
characters VARCHAR(150),
refer VARCHAR(150),
role VARCHAR(150),
PRIMARY KEY(id)
)  DEFAULT CHARSET=utf8;

ALTER TABLE `titlesplit` ADD INDEX(`words`)


---情感词-极值表

CREATE table  layer(id INT(12) UNSIGNED NOT NULL  AUTO_INCREMENT,
words VARCHAR(500),
value  decimal(10,7),
PRIMARY KEY(id)
)  DEFAULT CHARSET=utf8;


ALTER TABLE `layer` ADD INDEX(`words`)



--- 查询关键词对应的情感值

select  l.id,l.`words`,t.`innserSessionid`,t.`times`,t.`id`,t.`words`,t.`refer`,t.`role`,l.`VALUE` from layer l join  titlesplit  t on l.`words`=t.`words`


---  创建结果表

CREATE TABLE `connect` (
`id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,
`innserSessionid` VARCHAR(50), 
`sourcetitle`  varCHAR(500), 
`times` varCHAR(50) , 
`words` varCHAR(500) ,
`characters` varchar(150), 
`refer` varchar(150) , 
`role`  varCHAR(150), 
`category` CHAR(100), 
`number` CHAR(100),
`VALUE`  decimal(10,7),
 PRIMARY KEY (`id`) 
) DEFAULT CHARSET=utf8;
ALTER TABLE `connect` ADD INDEX(`words`);



SELECT  titlesplit.`innserSessionid`,titlesplit.`sourcetitle`,titlesplit.`times`,titlesplit.`words`,titlesplit.`refer`,titlesplit.`role`,category.`category`,category.`number`,layer.`VALUE` FROM titlesplit   LEFT JOIN category  ON titlesplit.`words`=category.`words`
LEFT  JOIN layer   ON titlesplit.`words`=layer.`words`;



CREATE TABLE `CsenticNetlayer2` (
`id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,
`words` VARCHAR(250), 
`attitude`  varCHAR(150), 
`expression` varCHAR(150) , 
`extreme` varCHAR(500) ,
`VALUE`  decimal(10,7),
 PRIMARY KEY (`id`) 
) DEFAULT CHARSET=utf8;
ALTER TABLE `connect` ADD INDEX(`words`);



--  划分主题逻辑
1、极值情感词不能成为主题词
 SELECT   t.`innserSessionid`,t.`words`,l.`words`,l2.`words`,c.`category`,t.`characters`  FROM titlesplit  t  JOIN category c  ON  t.`words`=c.`words` 
 LEFT  JOIN layer  l  ON t.`words`=l.`words` 
 LEFT  JOIN  CsenticNetlayer2  l2 ON t.`words`=l2.`words`
 WHERE l.`words` IS  NULL  AND l2.`words` IS NULL AND  t.`characters`='n' 


 2、去除地名
  SELECT t.`words`,t.`characters`,t.`refer`,t.`role`  FROM `titlesplit` t   WHERE t.`characters`='ns' --  地名


3、词性分类
-- b j n nh ni nz  ws
-- 25325  387
SELECT DISTINCT  t.`sourcetitle`,t.`characters`,t.`words` FROM  titlesplit t  WHERE t.`characters`='b' OR t.`characters`='j'
OR  t.`characters`='nh' OR t.`characters`='ni' OR t.`characters`='nz' OR t.`characters`='ws';


-- b j n nh ni nz  ws
-- 25325  387
SELECT MIN(c.`id`),t.`sourcetitle`,t.`characters`,t.`words`,c.`category` FROM  titlesplit t  
JOIN category c ON t.`words`=c.`words`
WHERE t.`characters`='b' OR t.`characters`='j'
OR  t.`characters`='nh' OR t.`characters`='ni' OR t.`characters`='nz' OR t.`characters`='ws'
AND LENGTH(t.`sourcetitle`)>16
GROUP  BY t.`sourcetitle`,t.`characters`,t.`words`


4、筛选主语和相关词性

SELECT  t.`innserSessionid`,t.`times`,t.`sourcetitle`,t.words,t.`characters`,t.`refer`,t.`role`  FROM `titlesplit`  t 
WHERE 
 (t.`characters`='n'   OR t.`characters`='ni')  AND 
role='SBV'

5

SELECT  t.`innserSessionid`,t.`times`,t.`sourcetitle`,t.words,t.`characters`,t.`refer`,t.`role`,c.`id`,c.`category`  FROM `titlesplit`  t  JOIN category c
ON t.`words`=c.`words`
WHERE (t.`characters`='n'   OR t.`characters`='ni')  AND  role='SBV';




--  行业划分
 -- select  t.`innserSessionid`,t.`id`,c.`category`  from 
 -- categorys  c  join titlesplit  t  on c.`words`=t.`words`  where c.`pos` Like "n%" 

-- layer词性划分  1613
SELECT  t.`innserSessionid`,t.`sourcetitle`,t.sm FROM 
(SELECT   t.`innserSessionid`,t.`sourcetitle`,SUM(l.`VALUE`) sm FROM `titlesplit` t LEFT  JOIN layer l   
ON t.`words`=l.words   GROUP  BY t.`innserSessionid`,t.`sourcetitle`)t
 WHERE t.sm IS NOT NULL
 
 -- layer词性划分  1820
SELECT  t.`innserSessionid`,t.`sourcetitle`,t.sm FROM 
(SELECT   t.`innserSessionid`,t.`sourcetitle`,SUM(l.`VALUE`) sm FROM `titlesplit` t LEFT  JOIN layer l   
ON t.`words`=l.words   GROUP  BY t.`innserSessionid`,t.`sourcetitle`)t

-- `CsenticNetlayer2`词性划分  1170
SELECT  t.`innserSessionid`,t.`sourcetitle`,t.sm FROM 
(SELECT   t.`innserSessionid`,t.`sourcetitle`,SUM(l.`VALUE`) sm FROM `titlesplit` t LEFT  JOIN CsenticNetlayer2 l   
ON t.`words`=l.words   GROUP  BY t.`innserSessionid`,t.`sourcetitle`)t
 WHERE t.sm IS NOT NULL
 
 -- `CsenticNetlayer2`词性划分  1820
SELECT  t.`innserSessionid`,t.`sourcetitle`,t.sm FROM 
(SELECT   t.`innserSessionid`,t.`sourcetitle`,SUM(l.`VALUE`) sm FROM `titlesplit` t LEFT  JOIN CsenticNetlayer2 l   
ON t.`words`=l.words   GROUP  BY t.`innserSessionid`,t.`sourcetitle`)t

-- 关联
SELECT  t.`innserSessionid`,t.`sourcetitle`,t.sm1,t.sm2 FROM 
(SELECT   t.`innserSessionid`,t.`sourcetitle`,SUM(l.`VALUE`) sm1,SUM(l2.`VALUE`) sm2 FROM `titlesplit` t 
LEFT  JOIN CsenticNetlayer2 l   
ON t.`words`=l.words  
LEFT  JOIN layer l2  
ON t.`words`=l2.`words`
 GROUP  BY t.`innserSessionid`,t.`sourcetitle`)t





SELECT  DISTINCT t.`innserSessionid`,t.`words`,s.`classification`,s.`value`  FROM   `titlesplit`  t  JOIN  `sentiment`  s  ON s.`words`=t.`words`;

SELECT  DISTINCT  t.`innserSessionid`  FROM  `titlesplit`  t;
SELECT   *  FROM `sentiment`   s   WHERE s.`words`='坦诚';
SELECT  * FROM   `titlesplit`  t  WHERE t.`innserSessionid`='6D238824ECD84D1C9BF42B2920E3FE9C'

 

将66个行业词库导入到MySQL中的scala代码如下

import java.sql.{DriverManager, ResultSet}
import org.apache.spark.SparkContext
import org.apache.spark.SparkConf
object category {
  val rl= "jdbc:mysql://192.168.0.37:3306/emotional?user=root&password=123456&useUnicode=true&characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false"
  classOf[com.mysql.jdbc.Driver]
  val conn = DriverManager.getConnection(rl)

  def main(args: Array[String]) {


    val conf = new SparkConf().setMaster("local").setAppName("test")
    val sc = new SparkContext(conf)
    val sqlContext = new org.apache.spark.sql.SQLContext(sc)
    // val role = "jdbc:mysql://192.168.0.37:3306/emotional?user=root&password=123456&useUnicode=true&characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false"
    val log881101 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881101.种植业与林业.txt")
    val format = new java.text.SimpleDateFormat("yyyyMMdd")
    val date = format.format(new java.util.Date().getTime()).toString
    val rowRDD881101= log881101.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val s881101 = rowRDD881101.map(line => (date, "881101", "种植业与林业", line._4))
    import sqlContext.implicits._
    val s2881101= s881101.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881101= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881101= value881101.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })

    p881101.foreach(p => {
      val v0=p._1
      val v1=p._2
      val v2=p._3
      val v3=p._4
      insert(v0,v1,v2,v3)

    })


    val log881102 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881102.养殖业.txt")
    val  rowRDDlog881102=log881102.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881102=rowRDDlog881102.map(line=>(date,"881102","养殖业",line._4))
    import sqlContext.implicits._
    val  s2881102= slog881102.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881102= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881102= value881102.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881102.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)


    })



    val log881103 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881103.农产品加工.txt")
    val  rowRDDlog881103=log881103.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881103=rowRDDlog881103.map(line=>(date,"881103","农产品加工",line._4))
    import sqlContext.implicits._
    val  s2881103= slog881103.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881103= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881103= value881103.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881103.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)


    })

    val log881104 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881104.农业服务.txt")
    val  rowRDDlog881104=log881104.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881104=rowRDDlog881104.map(line=>(date,"881104","农业服务",line._4))
    import sqlContext.implicits._
    val  s2881104= slog881104.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881104= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881104= value881104.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881104.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)


    })

    val log881105 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881105.煤炭开采.txt")
    val  rowRDDlog881105=log881105.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881105=rowRDDlog881105.map(line=>(date,"881105","煤炭开采",line._4))
    import sqlContext.implicits._
    val  s2881105= slog881105.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881105= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881105= value881105.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881105.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)


    })

    val log881106 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881106.石油矿业开采.txt")
    val  rowRDDlog881106=log881106.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881106=rowRDDlog881106.map(line=>(date,"881106","石油矿业开采",line._4))
    import sqlContext.implicits._
    val  s2881106= slog881106.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881106= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881106= value881106.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)

    })
    p881106.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)


    })

    val log881107 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881107.采掘服务.txt")
    val  rowRDDlog881107=log881107.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881107=rowRDDlog881107.map(line=>(date,"881107","采掘服务",line._4))
    import sqlContext.implicits._
    val  s2881107= slog881107.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881107= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881107= value881107.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881107.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)


    })

    val log881108 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881108.基础化学.txt")
    val  rowRDDlog881108=log881108.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881108=rowRDDlog881108.map(line=>(date,"881108","基础化学",line._4))
    import sqlContext.implicits._
    val  s2881108= slog881108.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881108= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881108= value881108.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881108.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)


    })

    val log881109 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881109.化学制品.txt")
    val  rowRDDlog881109=log881109.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881109=rowRDDlog881109.map(line=>(date,"881109","化学制品",line._4))
    import sqlContext.implicits._
    val  s2881109= slog881109.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881109= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881109= value881109.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881109.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)


    })

    val log881110 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881110.化工合成材料.txt")
    val  rowRDDlog881110=log881110.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881110=rowRDDlog881110.map(line=>(date,"881110","化工合成材料",line._4))
    import sqlContext.implicits._
    val  s2881110= slog881110.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881110= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881110= value881110.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881110.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)


    })

    val log881111 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881111.化工新材料.txt")
    val  rowRDDlog881111=log881111.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881111=rowRDDlog881111.map(line=>(date,"881111","化工新材料",line._4))
    import sqlContext.implicits._
    val  s2881111= slog881111.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881111= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881111= value881111.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881111.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)


    })


    val log881112 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881112.钢铁.txt")
    val  rowRDDlog881112=log881112.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881112=rowRDDlog881112.map(line=>(date,"881112","钢铁",line._4))
    import sqlContext.implicits._
    val  s2881112= slog881112.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881112= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881112= value881112.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881112.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)


    })

    val log881113 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881113.有色冶炼加工.txt")
    val  rowRDDlog881113=log881113.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881113=rowRDDlog881113.map(line=>(date,"881113","有色冶炼加工",line._4))
    import sqlContext.implicits._
    val  s2881113= slog881113.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881113= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881113= value881113.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881113.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)


    })

    val log881114 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881114.新材料.txt")
    val  rowRDDlog881114=log881114.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881114=rowRDDlog881114.map(line=>(date,"881114","新材料",line._4))
    import sqlContext.implicits._
    val  s2881114= slog881114.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881114= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881114= value881114.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881114.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)
    })

    val log881115 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881115.建筑材料.txt")
    val  rowRDDlog881115=log881115.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881115=rowRDDlog881115.map(line=>(date,"881115","建筑材料",line._4))
    import sqlContext.implicits._
    val  s2881115= slog881115.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881115= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881115= value881115.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881115.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })

    val log881116 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881116.建筑装饰.txt")
    val  rowRDDlog881116=log881116.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881116=rowRDDlog881116.map(line=>(date,"881116","建筑材料",line._4))
    import sqlContext.implicits._
    val  s2881116= slog881116.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881116= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881116= value881116.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881116.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })

    val log881117 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881117.通用设备.txt")
    val  rowRDDlog881117=log881117.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881117=rowRDDlog881117.map(line=>(date,"881117","通用设备",line._4))
    import sqlContext.implicits._
    val  s2881117= slog881117.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881117= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881117= value881117.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881117.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })

    val log881118 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881118.专用设备.txt")
    val  rowRDDlog881118=log881118.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881118=rowRDDlog881118.map(line=>(date,"881118","专用设备",line._4))
    import sqlContext.implicits._
    val  s2881118= slog881118.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881118= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881118= value881118.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881118.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })

    val log881119 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881119.仪器仪表.txt")
    val  rowRDDlog881119=log881119.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881119=rowRDDlog881119.map(line=>(date,"881119","仪器仪表",line._4))
    import sqlContext.implicits._
    val  s2881119= slog881119.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881119= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881119= value881119.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881119.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })

    val log881120 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881120.电气设备.txt")
    val  rowRDDlog881120=log881120.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881120=rowRDDlog881120.map(line=>(date,"881120","电气设备",line._4))
    import sqlContext.implicits._
    val  s2881120= slog881120.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881120= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881120= value881120.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881120.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })

    val log881121 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881121.半导体及元件.txt")
    val  rowRDDlog881121=log881121.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881121=rowRDDlog881121.map(line=>(date,"881121","半导体及元件",line._4))
    import sqlContext.implicits._
    val  s2881121= slog881121.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881121= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881121= value881121.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881121.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })


    val log881122 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881122.光学光电子.txt")
    val  rowRDDlog881122=log881122.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881122=rowRDDlog881122.map(line=>(date,"881122","光学光电子",line._4))
    import sqlContext.implicits._
    val  s2881122= slog881122.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881122= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881122= value881122.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881122.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })


    val log881123 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881123.其他电子.txt")
    val  rowRDDlog881123=log881123.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881123=rowRDDlog881123.map(line=>(date,"881123","其他电子",line._4))
    import sqlContext.implicits._
    val  s2881123= slog881123.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881123= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881123= value881123.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881123.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })

    val log881124 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881124.电子制造.txt")
    val  rowRDDlog881124=log881124.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881124=rowRDDlog881124.map(line=>(date,"881124","电子制造",line._4))
    import sqlContext.implicits._
    val  s2881124= slog881124.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881124= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881124= value881124.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881124.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })


    val log881125 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881125.汽车整车.txt")
    val  rowRDDlog881125=log881125.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881125=rowRDDlog881125.map(line=>(date,"881125","汽车整车",line._4))
    import sqlContext.implicits._
    val  s2881125= slog881125.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881125= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881125= value881125.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881125.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })



    val log881126 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881126.汽车零部件.txt")
    val  rowRDDlog881126=log881126.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881126=rowRDDlog881126.map(line=>(date,"881126","汽车零部件",line._4))
    import sqlContext.implicits._
    val  s2881126= slog881126.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881126= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881126= value881126.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881126.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })


    val log881127 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881127.非汽车交运.txt")
    val  rowRDDlog881127=log881127.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881127=rowRDDlog881127.map(line=>(date,"881127","非汽车交运",line._4))
    import sqlContext.implicits._
    val  s2881127= slog881127.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881127= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881127= value881127.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881127.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })

    val log881128 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881128.交运设备服务.txt")
    val  rowRDDlog881128=log881128.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881128=rowRDDlog881128.map(line=>(date,"881128","交运设备服务",line._4))
    import sqlContext.implicits._
    val  s2881128= slog881128.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881128= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881128= value881128.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881128.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })

    val log881129 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881129.通信设备.txt")
    val  rowRDDlog881129=log881129.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881129=rowRDDlog881129.map(line=>(date,"881129","通信设备",line._4))
    import sqlContext.implicits._
    val  s2881129= slog881129.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881129= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881129= value881129.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881129.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })

    val log881130 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881130.计算机设备.txt")
    val  rowRDDlog881130=log881130.map(line=>(line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").head.trim(),line.split(" ").last.trim))
    val slog881130=rowRDDlog881130.map(line=>(date,"881130","计算机设备",line._4))
    import sqlContext.implicits._
    val  s2881130= slog881130.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val  value881130= sqlContext.sql("select distinct editdate,number,category,words  from category")
    val  p881130= value881130.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0,v1,v2,v3)
    })
    p881130.foreach(p => {
      val v0=p._1
      val v1=p._2.toString
      val v2=p._3.toString
      val v3=p._4.toString
      insert(v0,v1,v2,v3)

    })


    val log881131 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881131.白色家电.txt")
    val rowRDDlog881131 = log881131.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881131 = rowRDDlog881131.map(line => (date, "881131", "白色家电", line._4))
    import sqlContext.implicits._
    val s2881131 = slog881131.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881131 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881131 = value881131.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881131.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)


    })



    val log881132 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881132.视听器材.txt")
    val rowRDDlog881132 = log881132.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881132 = rowRDDlog881132.map(line => (date, "881132", "视听器材", line._4))
    import sqlContext.implicits._
    val s2881132 = slog881132.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881132 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881132 = value881132.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881132.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)


    })

    val log881133 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881133.饮料制造.txt")
    val rowRDDlog881133 = log881133.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881133 = rowRDDlog881133.map(line => (date, "881133", "饮料制造", line._4))
    import sqlContext.implicits._
    val s2881133 = slog881133.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881133 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881133 = value881133.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881133.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)



    })

    val log881134 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881134.食品加工制造.txt")
    val rowRDDlog881134 = log881134.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881134 = rowRDDlog881134.map(line => (date, "881134", "食品加工制造", line._4))
    import sqlContext.implicits._
    val s2881134 = slog881134.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881134 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881134 = value881134.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881134.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)



    })

    val log881135 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881135.纺织制造.txt")
    val rowRDDlog881135 = log881135.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881135 = rowRDDlog881135.map(line => (date, "881135", "纺织制造", line._4))
    import sqlContext.implicits._
    val s2881135 = slog881135.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881135 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881135 = value881135.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881135.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)



    })

    val log881136 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881136.服装家纺.txt")
    val rowRDDlog881136 = log881136.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881136 = rowRDDlog881136.map(line => (date, "881136", "服装家纺", line._4))
    import sqlContext.implicits._
    val s2881136 = slog881136.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881136 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881136 = value881136.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881136.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)


    })

    val log881137 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881137.造纸.txt")
    val rowRDDlog881137 = log881137.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881137 = rowRDDlog881137.map(line => (date, "881137", "造纸", line._4))
    import sqlContext.implicits._
    val s2881137 = slog881137.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881137 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881137 = value881137.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881137.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })
    val log881138 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881138.包装印刷.txt")
    val rowRDDlog881138 = log881138.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881138 = rowRDDlog881138.map(line => (date, "881138", "包装印刷", line._4))
    import sqlContext.implicits._
    val s2881138 = slog881138.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881138 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881138 = value881138.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881138.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })

    val log881139 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881139.家用轻工.txt")
    val rowRDDlog881139 = log881139.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881139 = rowRDDlog881139.map(line => (date, "881139", "家用轻工", line._4))
    import sqlContext.implicits._
    val s2881139 = slog881139.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881139 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881139 = value881139.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881139.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })
    val log881140 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881140.化学制药.txt")
    val rowRDDlog881140 = log881140.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881140 = rowRDDlog881140.map(line => (date, "881140", "化学制药", line._4))
    import sqlContext.implicits._
    val s2881140 = slog881140.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881140 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881140 = value881140.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881140.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })
    val log881141 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881141.中药.txt")
    val rowRDDlog881141 = log881141.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881141 = rowRDDlog881141.map(line => (date, "881141", "中药", line._4))
    import sqlContext.implicits._
    val s2881141 = slog881141.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881141 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881141 = value881141.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881141.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })

    val log881142 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881142.生物制品.txt")
    val rowRDDlog881142 = log881142.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881142 = rowRDDlog881142.map(line => (date, "881142", "生物制品", line._4))
    import sqlContext.implicits._
    val s2881142 = slog881142.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881142 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881142 = value881142.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881142.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })
    val log881143 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881143.医药商业.txt")
    val rowRDDlog881143 = log881143.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881143 = rowRDDlog881143.map(line => (date, "881143", "医药商业", line._4))
    import sqlContext.implicits._
    val s2881143 = slog881143.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881143 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881143 = value881143.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881143.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })
    val log881144 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881144.医疗器械服务.txt")
    val rowRDDlog881144 = log881144.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881144 = rowRDDlog881144.map(line => (date, "881144", "医疗器械服务", line._4))
    import sqlContext.implicits._
    val s2881144 = slog881144.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881144 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881144 = value881144.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881144.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })
    val log881145 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881145.电力.txt")
    val rowRDDlog881145 = log881145.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881145 = rowRDDlog881145.map(line => (date, "881145", "电力", line._4))
    import sqlContext.implicits._
    val s2881145 = slog881145.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881145 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881145 = value881145.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881145.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })

    val log881146 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881146.燃气水务.txt")
    val rowRDDlog881146 = log881146.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881146 = rowRDDlog881146.map(line => (date, "881146", "燃气水务", line._4))
    import sqlContext.implicits._
    val s2881146 = slog881146.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881146 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881146 = value881146.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881146.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })


    val log881147 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881147.环保工程.txt")
    val rowRDDlog881147 = log881147.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881147 = rowRDDlog881147.map(line => (date, "881147", "环保工程", line._4))
    import sqlContext.implicits._
    val s2881147 = slog881147.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881147 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881147 = value881147.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881147.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)


    })


    val log881148 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881148.港口航运.txt")
    val rowRDDlog881148 = log881148.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881148 = rowRDDlog881148.map(line => (date, "881148", "港口航运", line._4))
    import sqlContext.implicits._
    val s2881148 = slog881148.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881148 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881148 = value881148.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881148.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })

    val log881149 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881149.公路铁路运输.txt")
    val rowRDDlog881149 = log881149.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881149 = rowRDDlog881149.map(line => (date, "881149", "公路铁路运输", line._4))
    import sqlContext.implicits._
    val s2881149 = slog881149.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881149 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881149 = value881149.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881149.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })

    val log881150 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881150.公交.txt")
    val rowRDDlog881150 = log881150.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881150 = rowRDDlog881150.map(line => (date, "881150", "公交", line._4))
    import sqlContext.implicits._
    val s2881150 = slog881150.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881150 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881150 = value881150.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881150.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })


    val log881151 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881151.机场航运.txt")
    val rowRDDlog881151 = log881151.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881151 = rowRDDlog881151.map(line => (date, "881151", "机场航运", line._4))
    import sqlContext.implicits._
    val s2881151 = slog881151.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881151 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881151 = value881151.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881151.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })

    val log881152 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881152.物流.txt")
    val rowRDDlog881152 = log881152.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881152 = rowRDDlog881152.map(line => (date, "881152", "物流", line._4))
    import sqlContext.implicits._
    val s2881152 = slog881152.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881152 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881152 = value881152.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881152.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })

    val log881153 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881153.房地产开发.txt")
    val rowRDDlog881153 = log881153.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881153 = rowRDDlog881153.map(line => (date, "881153", "房地产开发", line._4))
    import sqlContext.implicits._
    val s2881153 = slog881153.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881153 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881153 = value881153.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881153.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })
    val log881154 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881154.园区开发.txt")
    val rowRDDlog881154 = log881154.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881154 = rowRDDlog881154.map(line => (date, "881154", "园区开发", line._4))
    import sqlContext.implicits._
    val s2881154 = slog881154.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881154 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881154 = value881154.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881154.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })

    val log881155 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881155.银行.txt")
    val rowRDDlog881155 = log881155.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881155 = rowRDDlog881155.map(line => (date, "881155", "银行", line._4))
    import sqlContext.implicits._
    val s2881155 = slog881155.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881155 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881155 = value881155.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881155.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })

    val log881156 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881156.保险及其他.txt")
    val rowRDDlog881156 = log881156.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881156 = rowRDDlog881156.map(line => (date, "881156", "保险及其他", line._4))
    import sqlContext.implicits._
    val s2881156 = slog881156.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881156 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881156 = value881156.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881156.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })
    val log881157 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881157.证券.txt")
    val rowRDDlog881157 = log881157.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881157 = rowRDDlog881157.map(line => (date, "881157", "证券", line._4))
    import sqlContext.implicits._
    val s2881157 = slog881157.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881157 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881157 = value881157.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881157.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })
    val log881158 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881158.零售.txt")
    val rowRDDlog881158 = log881158.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881158 = rowRDDlog881158.map(line => (date, "881158", "零售", line._4))
    import sqlContext.implicits._
    val s2881158 = slog881158.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881158 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881158 = value881158.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881158.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })
    val log881159 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881159.贸易.txt")
    val rowRDDlog881159 = log881159.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881159 = rowRDDlog881159.map(line => (date, "881159", "贸易", line._4))
    val s2881159 = slog881159.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881159 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881159 = value881159.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881159.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })

    val log881160 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881160.景点及旅游.txt")
    val rowRDDlog881160 = log881160.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881160 = rowRDDlog881160.map(line => (date, "881160", "景点及旅游", line._4))
    import sqlContext.implicits._
    val s2881160 = slog881160.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881160 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881160 = value881160.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881160.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })

    val log881161 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881161.酒店及餐饮.txt")
    val rowRDDlog881161 = log881161.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881161 = rowRDDlog881161.map(line => (date, "881161", "酒店及餐饮", line._4))
    import sqlContext.implicits._
    val s2881161 = slog881161.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881161 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881161 = value881161.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881161.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })
    val log881162 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881162.通信服务.txt")
    val rowRDDlog881162 = log881162.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881162 = rowRDDlog881162.map(line => (date, "881162", "通信服务", line._4))
    import sqlContext.implicits._
    val s2881162 = slog881162.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881162 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881162 = value881162.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881162.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })

    val log881163 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881163.计算机应用.txt")
    val rowRDDlog881163 = log881163.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881163 = rowRDDlog881163.map(line => (date, "881163", "计算机应用", line._4))
    import sqlContext.implicits._
    val s2881163 = slog881163.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881163 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881163 = value881163.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881163.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })
    val log881164 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881164.传媒.txt")
    val rowRDDlog881164 = log881164.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881164 = rowRDDlog881164.map(line => (date, "881164", "传媒", line._4))
    import sqlContext.implicits._
    val s2881164 = slog881164.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881164 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881164 = value881164.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881164.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })
    val log881165 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881165.综合.txt")
    val rowRDDlog881165 = log881165.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881165 = rowRDDlog881165.map(line => (date, "881165", "综合", line._4))
    import sqlContext.implicits._
    val s2881165 = slog881165.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881165 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881165 = value881165.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881165.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)

    })

    val log881166 = sc.textFile("hdfs://192.168.0.211:9000/user/hadoop/emotion/881166.国防军工.txt")
    val rowRDDlog881166 = log881166.map(line => (line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").head.trim(), line.split(" ").last.trim))
    val slog881166 = rowRDDlog881166.map(line => (date, "881166", "国防军工", line._4))
    import sqlContext.implicits._
    val s2881166 = slog881166.toDF("editdate", "number", "category", "words").registerTempTable("category")
    val value881166 = sqlContext.sql("select distinct editdate,number,category,words  from category")
    val p881166 = value881166.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p881166.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert(v0, v1, v2, v3)
    })


    // 解析结果去重后插入category2
    val job = sqlContext.jdbc("jdbc:mysql://192.168.0.37:3306/emotional?user=root&password=123456", "category")
    val jo = job.toDF().registerTempTable("job")
    val value = sqlContext.sql("select distinct  editdate,number,category,words  from job")

    val p = value.map(p => {
      val v0 = p.getString(0)
      val v1 = p.getString(1).toString
      val v2 = p.getString(2).toString
      val v3 = p.getString(3).toString
      (v0, v1, v2, v3)
    })
    p.foreach(p => {
      val v0 = p._1
      val v1 = p._2.toString
      val v2 = p._3.toString
      val v3 = p._4.toString
      insert2(v0, v1, v2, v3)
    })
    conn.close()
  }

  def insert(value0: String, value1: String, value2: String, value3: String): Unit = {

    //val statement = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_UPDATABLE)
    // CREATE TABLE words2(innersessionId VARCHAR(100),words VARCHAR(100), VARCHAR(100),posit VARCHAR(100),va VARCHAR(100))
    try {
      val prep = conn.prepareStatement("INSERT INTO category(editdate,number,category,words) VALUES (?,?,?,?) ")
      prep.setString(1, value0)
      prep.setString(2, value1)
      prep.setString(3, value2)
      prep.setString(4, value3)
      prep.executeUpdate
    } catch {
      case e: Exception => e.printStackTrace
    }

  }


  def insert2(value0: String, value1: String, value2: String, value3: String): Unit = {

    //val statement = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_UPDATABLE)
    // CREATE TABLE words2(innersessionId VARCHAR(100),words VARCHAR(100), VARCHAR(100),posit VARCHAR(100),va VARCHAR(100))
    try {
      val prep = conn.prepareStatement("INSERT INTO category2(editdate,number,category,words) VALUES (?,?,?,?) ")
      prep.setString(1, value0)
      prep.setString(2, value1)
      prep.setString(3, value2)
      prep.setString(4, value3)
      prep.executeUpdate
    } catch {
      case e: Exception => e.printStackTrace
    }
  }
}

 

weixin_34050427
关注
  • 0
    点赞
  • 踩
  • 2
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
百万级词库--分类词库
07-05
因工作需要，最近在做自然语言处理方面的研究，第一步需要实现中文分词，市面上有许多开源的分词组件，本人使用的是word分词，但这些开源组件中的词库都比较小，因此这段时间主要是在网上爬了各种词库，并通过XX词典校验，挑选在XX词典中收录的（耗费相当多的精力）。此份分类词库，包括财经、汽车、IT、数学、农业、动物、植物、成语、诗词、机构、地址、食物等
搜狗分类词库（2018）
09-01
2016年上传过搜狗12个分类词库，两年没更新了，今天用程序重新跑了下数据获取到2018最新的数据，导出为txt格式，方便大家做分词用，我不生成词，我只是词的搬运工，版权归搜狗公司所有。
行业类型字典涵盖大多数行业
02-07
行业类型字典,涵盖大多数行业,比较全面
百度的常见分类词词库！（seo）
05-26
seo或者网站站长用于研究百度的分词词库！大家可以下载看一下！
行业类别数据字典，数据库
04-11
按照 国民经济行业分类(GB/T 4754-2011)，整理行业树形结构! 包含Excel源文件，数据库sql语句
android安卓源码海量项目合集打包-1
热门推荐
chenhao0568的专栏
06-11 13万+
下载地址 最后更新共计113个分类5177套源码29.2 GB。 卷 新加卷 的文件夹 PATH 列表 卷序列号为 00000200 5E7A:7F30 F:. ├─前台界面 │ ├─3D标签云卡片热门 │ │ Android TagCloudView云标签的灵活运用.rar │ │ Android 实现 标签 拖动 改变位置.rar │ │ android 流式布局和热门标签.zip │ │ ...
5118理财行业词库，38个公开高频词数据
09-17
5118理财行业词库下的38个公开的高频词数据，并非完整高频词数据。 同时由于所有等级的VIP账号单个高频词最高下载量均为50W关键词。
5118保险行业词库，38个公开高频词数据
09-17
5118保险行业词库下的38个公开的高频词数据，并非完整高频词数据。 同时由于所有等级的VIP账号单个高频词最高下载量均为50W关键词。
5118融资行业词库，38个公开高频词数据
09-23
5118融资行业词库下的38个公开的高频词数据，并非完整高频词数据。 同时由于所有等级的VIP账号单个高频词最高下载量均为50W关键词。
5118卡盟行业词库，38个公开高频词数据
09-24
5118卡盟行业词库下的38个公开的高频词数据，并非完整高频词数据。 同时由于所有等级的VIP账号单个高频词最高下载量均为50W关键词。
5118母婴行业词库，38个公开高频词数据
09-16
5118母婴行业词库下的38个公开的高频词数据，并非完整高频词数据。 同时由于所有等级的VIP账号单个高频词最高下载量均为50W关键词。
对文本分类词库的考虑
青青陵上柏，磊磊涧中石
05-27 3478
老板喊看看能不能做个分类词库的东西，以后我们分类、聚类就根据这个库来做了。所谓分类词库，我是这样理解的：每个类别都有N个词，当需聚类文本分词之后，统计其词的分部情况，看看词落在哪个类比较多，从而组成特征向量用于聚类。查了些资料，发现他的思路已经被N多人做过，而且过程比较复杂：对文本分词、去停用词、特征选择、生成向量空间、聚类算法……总而言之，是采用词做为最小的义原，这样做的弊端是步骤太多，如何
ES集群分布式查询原理
最新发布
weixin_53676834的博客
04-30 214
当新增文档时，应该保存到不同分片，保证数据均衡，那么coordinating node如何确定数据该存储到哪个分片呢？2）对id做hash运算，假如得到的是2，则应该存储到shard-2。3）shard-2的主分片在node3节点，将数据路由到node3。5）同步给shard-2的副本replica-2，在node2节点。6）返回结果给coordinating-node节点。1）新增一个id=1的文档。
云计算知识点-02
lv785的博客
04-27 1062
DataNode是HDFS文件系统的工作节点，DataNode会按照客户端或者是NameNode的调度来存储和检索数据，并定期向NameNode发送它所存储的数据块列表，DataNode是文件系统中真正存储数据的地方。通过这种虚拟化可以减少服务器的数量，提高服务器的使用效率，可以在一定程度上摆脱物理上的空间限制，实现随时随地随需的自由掌控。Glance(镜像服务组件)，其作用是：提供虚拟机镜像的存储，查询和检索功能，为nova进行服务，依赖于存储服务（存储镜像本身）和数据库服务（存储镜像相关的数据）。
Impala系统架构理解
weixin_64881847的博客
04-26 488
这些执行任务的Impalad实例会根据查询的执行计划，直接在HDFS上读取数据并进行计算。用于跟踪Impala集群中所有Impalad实例的健康状态和位置信息，确保Impala能够了解集群中各个Impalad的状态，从而能够选择合适的Impalad来执行查询任务。这也就意味着一个Impalad进程可能不仅作为协调者分发任务，还可能作为接受者接收并执行来自其他Impalad的任务，这取决于集群的负载和任务的分布。，以便选择最靠近数据所在DataNode的那个节点上的Impalad来执行任务。
哪些情况下左联接的数据量比左表大？
Java/Python/大数据干货分享
04-26 357
本文主要探讨SQL和Pandas左连接数据量比左表大的问题。在某些情况下，执行左连接时，SQL和Pandas输出的结果会比左表数据量大。这是为什么呢？左连接是关系型数据库中最常见的连接之一。它的作用是从左表中取出所有的行，以及与右表匹配的所有行。如果某个行在右表中没有匹配到，则会在输出结果中以NULL的形式呈现1）左表与右表为一对多或多对多2）左表与右表关联键都存在空值NULL1）视情景删除或聚合右表中重复关联键的数据2）过滤表中关联键值为NULL的行。
word2vec关键词词库扩充代码实现
12-20
根据提供的引用内容，以下是一个使用gensim库中的Word2Vec模型来扩充关键词词库的代码示例： ```python from gensim.models import Word2Vec # 假设已有的关键词列表 keywords = [['apple'], ['banana'], ['orange...

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值