案例-中国法律法规doc文档爬取

中企集成：http://www.jincao.com

中国法律分类检索：http://www.jincao.com/t1.htm

setwd("F:/...")
library("RCurl")
library("XML")
library("stringr")
library("dplyr")
#library("plyr")
library("data.table")
library("downloader")
url<-"http://www.jincao.com/t1.htm"
url_yuan<-"http://www.jincao.com/"
rd<-getURL(url,.encoding="gb2312")
#利用xml包函数整理网页树结构
rdhtml<-htmlParse(enc2utf8(rd),encoding="utf-8") 
#获取根目录
tian<-getNodeSet(rdhtml,"/html/body/div[3]/table/tr/td[2]/div[2]/center/table/tr/td/p//a")
#获取网址特征向量list,标签-属性值
wangzhi<-lapply(tian,xmlAttrs)
rm(rd)
rm(rdhtml)

#构造函数提取href
href_function=function(x){
return(x["href"])
}
wangzhi_url<-lapply(wangzhi,href_function)
wangzhi_url<-as.character(wangzhi_url)
names(wangzhi_url)<-NULL
wangzhi_url<-paste0(url_yuan,wangzhi_url,paste="")
rm(wangzhi)
#提取属性所对应的值
shuxing<-lapply(tian,xmlValue)
shuxing<-as.character(shuxing)
url_fangwen<-as.data.frame(cbind(wangzhi_url,shuxing),stringsAsFactors=F)
law_class_url<-as.data.table(url_fangwen)
rm(url_fangwen)
rm(shuxing)
rm(wangzhi_url)
rm(url)
rm(url_yuan)
rm(tian)


#进一步访问获取文本数据
for (i in 1:nrow(law_class_url)) {
url<-law_class_url[i,wangzhi_url]
raw_name<-gsub("[[:space:]]","",law_class_url[i,shuxing])
rd<-getURL(url,.encoding="gb2312")
rdhtml<-htmlParse(enc2utf8(rd),encoding="utf-8") 
rm(rd)
rd<-getNodeSet(rdhtml,"/html/body/div[3]/center/table/tr/td[2]/div/table/tr/td/p//font//a")
#提取href
wangzhi<-lapply(rd,xmlAttrs)
wangzhi_url<-lapply(wangzhi,href_function)
wangzhi_url<-as.character(wangzhi_url)
names(wangzhi_url)<-NULL
rm(wangzhi)
#截取字符串到指定位置
gg<-gregexpr("/",url)
url_sub<-substr(url,1,gg[[1]][5])
rm(gg)
wangzhi_url<-paste0(url_sub,wangzhi_url,paste="")
shuxing<-as.character(lapply(rd,xmlValue))  
url_fangwen_xiazhuan<-as.data.frame(cbind(wangzhi_url,shuxing),stringsAsFactors = F)

  rm(rd)
  rm(rdhtml)
  rm(shuxing)
  rm(url)
  rm(url_sub)
  rm(wangzhi_url)

for (j in 1:nrow(url_fangwen_xiazhuan)) {
#一些特殊字符的处理
name_detail<-gsub("[[:space:]]","",url_fangwen_xiazhuan[j,2])
name_detail<-gsub("（GB/T4754-94）","",name_detail)
name_detail<-gsub("[<|>]","",name_detail)
name_detail<-gsub("[/]","_",name_detail)
if (nchar(name_detail)<=3) next
download(url_fangwen_xiazhuan[j,1],paste0(raw_name,"_",name_detail,".doc"), mode="wb")
    rm(name_detail)
}
}
rm(j)
rm(raw_name)

得到DOC文档后，需要处理成txt格式，参考：xpdf用法、antiword用法、python-docx用法