WebDriver driver = SeleniumUtil.getNormalDriver();
ExecutorService pool = Executors.newCachedThreadPool();
pool = Executors.newFixedThreadPool(5);
for(String type:"xuanhuan jingdian dushi lishi wuxiao".split(" ")) {
try {
for (int i = 1; i < 1000; i++) {
String subfix="";
if(i>1) {
subfix="index_"+i+".html";
}
String page=SpiderUtil.getPageContent("https://www.downtxts.com/article/"+type+"/"+subfix);
for(String a:HtmlUtil.find(page, "div[class=item-content]>h4>a[getPropertyValue=href]")) {
try {
a=SpiderUtil.getPageContent(a);
for(String as:HtmlUtil.find(a, "a")) {
if("下载地址".equals(HtmlUtil.getValue(as))){
a=SpiderUtil.getPageContent(HtmlUtil.propertyValue(as, "href"));
for(String a2:HtmlUtil.find(a, "a")) {
if("点击进入网盘下载一".equals(HtmlUtil.getValue(a2))) {
a2=HtmlUtil.propertyValue(a2, "href");
if(a2.contains("?http")) a2=a2.substring(a2.indexOf("?http")+1);
driver.get(a2);
SeleniumUtil.waitAppear(driver, 5, By.linkText("普通下载"));
// Thread.sleep(1000);
a2=driver.getPageSource();
String title=HtmlUtil.find(a2, "title[value]").get(0).replace("txt免费下载 - 118网盘", "");
int end=a2.indexOf(">普通下载</span>");
a2=a2.substring(a2.lastIndexOf("<a",end),end);
a2=a2.substring(0,a2.indexOf(">")+1);
String url=HtmlUtil.propertyValue(a2, "href");
if(!FileUtil.isFileExists("f:/novel/"+title+".txt")) {
pool.execute(new Runnable() {
public void run() {
try {
SpiderUtil.download(url, "f:/novel/"+title+".txt");
} catch (IOException e) {
e.printStackTrace();
}
}
});
}
}
}
}
}
}catch(Exception e) {
e.printStackTrace();
}
}
}
}catch(Exception ee) {
ee.printStackTrace();
}
}
pool.shutdown();
while(true){
if(pool.isTerminated()){
break;
}
Thread.sleep(1000);
}
08-13
222
06-29
3329