java多线程爬论文

最新推荐文章于 2021-03-08 07:19:26 发布
最新推荐文章于 2021-03-08 07:19:26 发布
阅读量6.6w 收藏
点赞数
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/qq_39898191/article/details/106957276
版权

学习笔记仅供参考

1.xiancheng.class

package com.example.util;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.net.*;

import java.util.ArrayList;
import java.util.Random;

public class xiancheng implements Runnable {

    static ArrayList<String> ips = new ArrayList<String>();
    public ArrayList<LW> lws = new ArrayList<>();
    static xiancheng xian = new xiancheng();
    static String name="";
    int q = 0;
    public xiancheng(){

    }
    public xiancheng(ArrayList<String> ipss, int pages) {
        ips = ipss;
        q = pages;
    }

    public ArrayList<LW>getlws(){
        return lws;
    }
     public String getip() throws InterruptedException {
         System.getProperties().setProperty("http.proxuHost","113.140.84.97");
         System.getProperties().setProperty("http.proxyPort","80");
        String ip = new pachong().ips1().get(0);
         System.out.println("IP是："+ip);
        return ip;
    }
    @Override
    public void run() {


            Random x = new Random();
            String path = "";

//        String [] ips = {"120.79.64.147","175.148.74.121","139.9.195.202","60.167.132.223"};
//        String [] ports = {"8118","1133","8118","808"};
//        System.getProperties().setProperty("http.proxyHost", ips[q%5]);
//        System.getProperties().setProperty("http.proxyPort", ports[q%5]);
//        System.err.println(ips[q%5]+":"+ports[q%5]);
            try {
                path = "http://qikan.chaoxing.com/searchjour?sw=" + URLEncoder.encode("西北农林科技大学", "utf-8") + "&stryear=17&nosim=1&size=50&x=0_900&pages=" + String.valueOf(q);
            } catch (UnsupportedEncodingException e) {
                e.printStackTrace();
            }
            q++;
            Random r = new Random();
            String[] ua = {"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36 OPR/37.0.2178.32",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
                    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586",
                    "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
                    "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
                    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)",
                    "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0)",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 BIDUBrowser/8.3 Safari/537.36",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36 Core/1.47.277.400 QQBrowser/9.4.7658.400",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 UBrowser/5.6.12150.8 Safari/537.36",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36 TheWorld 7",
                    "Mozilla/5.0 (Windows NT 6.1; W…) Gecko/20100101 Firefox/60.0"};
            int k = r.nextInt(14);
            String ip = null;

            Document doc = null;

            try {
                doc = Jsoup.connect(path)
                        .timeout(10000)
                        .ignoreHttpErrors(true)
                        .userAgent(ua[k])
//                    .cookie("msign_dsr", "1592578548328")
//                    .cookie("search_uuid", "59d60d20%2d888c%2d4339%2d893c%2d3f2df0b416a9")
//                    .cookie("UM_distinctid", "172cd13faab18a-0cc175a1ccab1-3c3f5a0c-1fa400-172cd13faac61e")
//                    .cookie("__dxca", "7959aeaa-2cb3-44b4-ac6d-5a19f2ac1261")
//                    .cookie("lv", "0")
//                    .cookie("chaoxinguser", "1")
//                    .cookie("uname", "")
//                    .cookie("_uid", "143680086")
//                    .cookie("uf", "f9866f9a46b70622b0dee5ad98478a8391831cfec1654dd57a4fb73fc6375849f7893fe5a7d754cdff17dba0ec4416549b0594e13f4b452fbdd6b93a431584911471850d8bf7e34c24848f3d3f307228570bdddc4e9885c5")
//                    .cookie("_d", "1592928430294")
//                    .cookie("UID", "143680086")
//                    .cookie("vc", "A5267B905B437CE795E472D324AD65E0")
//                    .cookie("vc2", "C473E4E0C10364B7C0A31DE46070C411")
//                    .cookie("vc3", "C9JC%2Ff8LUHJ%2FbURcxo1vggUNgTmarTCVJFGMM3kP7MoczjjCxUPi89UGJjjLU4sQ6WNHQgZOUN1mj6sDYiMWNCcwVw%2BMLRe1D3y5d%2FofpoF2AMh3LToyTKUsC9ykkTfsWnWiDPL32C%2Bziy%2BnJFuaW5KB9%2FlDulddw%2B9thByI%2Bvg%3Daaeb61274732c450c6a62c27f9001cce")
//                    .cookie("xxtenc", "3fa0ee7c7d4f9d548e4bc1d7336c838c")
//                    .cookie("duxiu", "userName%5fdsr%2c%3dxbnlkjdx%2c%21userid%5fdsr%2c%3d1406%2c%21char%5fdsr%2c%3d%u78cb%2c%21metaType%2c%3d353%2c%21dsr%5ffrom%2c%3d1%2c%21logo%5fdsr%2c%3dlogo0408%2ejpg%2c%21logosmall%5fdsr%2c%3dsmall0408%2ejpg%2c%21title%5fdsr%2c%3d%u897f%u5317%u519c%u6797%u79d1%u6280%u5927%u5b66%2c%21url%5fdsr%2c%3debook%2c%21compcode%5fdsr%2c%3d1176%2c%21province%5fdsr%2c%3d%u9655%u897f%2c%21readDom%2c%3d0%2c%21isdomain%2c%3d61707%2c%21showcol%2c%3d0%2c%21hu%2c%3d0%2c%21uscol%2c%3d0%2c%21isfirst%2c%3d0%2c%21istest%2c%3d0%2c%21cdb%2c%3d0%2c%21og%2c%3d1%2c%21ogvalue%2c%3d7%2c%21testornot%2c%3d1%2c%21remind%2c%3d0%2c%21datecount%2c%3d3478%2c%21userIPType%2c%3d2%2c%21lt%2c%3d0%2c%21ttt%2c%3dfxlogin%2echaoxing%2c%21enc%5fdsr%2c%3dD5E78CB6EBD2A70090256E4C463A3566")
                        .get();
            } catch (IOException e) {
                return;
            }

            Elements subjects = doc.select("#liebiaoDivId tr");
            Elements authors = doc.select("#liebiaoDivId tr");
            Elements downloadnum = doc.select("#liebiaoDivId tr");
            Elements quotenum = doc.select("#liebiaoDivId tr");

            String[] pathes = new String[50];
            String[] quotenums = new String[50];
            String[] downloadnums = new String[50];
            String[] periodicals = new String[50];
            String[] publishtimes = new String[50];
            String[] funds = new String[50];
            String[][] keys = new String[50][8];
            String[] units = new String[50];
            String[] abstracts = new String[50];
            for (int i = 0; i < 50; i++) {
                pathes[i] = "http://qikan.chaoxing.com" + subjects.get(i + 1).child(1).child(0).attr("href").trim();
                quotenums[i] = quotenum.get(i + 1).child(6).child(0).text();
                downloadnums[i] = downloadnum.get(i + 1).child(7).child(0).text();
                periodicals[i] = downloadnum.get(i + 1).child(3).child(0).text();

                if (downloadnum.get(i + 1).child(5).child(0).text().length() == 8) {
                    publishtimes[i] = downloadnum.get(i + 1).child(5).child(0).text().substring(0, 4) + "-0" + String.valueOf(Integer.parseInt(String.valueOf(downloadnum.get(i + 1).child(5).child(0).text().charAt(6))));
                } else if (downloadnum.get(i + 1).child(5).child(0).text().length() > 8 && (Integer.valueOf(downloadnum.get(i + 1).child(5).child(0).text().charAt(6)) <= 9) && (Integer.valueOf(downloadnum.get(i + 1).child(5).child(0).text().charAt(6)) >= 0)) {

                    publishtimes[i] = downloadnum.get(i + 1).child(5).child(0).text().substring(0, 4) + "-" + String.valueOf(Integer.valueOf(downloadnum.get(i + 1).child(5).child(0).text().substring(6, 8)) / 2);
                } else {
                    publishtimes[i] = "1";
                }
                //System.out.println(publishtimes[i]);
                //System.out.println(periodicals[i]);
                //System.out.println(downloadnums[i]);
                //System.out.println(quotenums[i]);
                //System.out.println(pathes[i]);
            }

            for (int i = 0; i < 50; i++) {


                if (i % 10 == 0) {
                    try {
                        ip = xian.getip();
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }


                    String[] r1 = ip.split(":");
                    System.out.println(ip);
                    System.getProperties().setProperty("http.proxyHost", r1[0]);
                    System.getProperties().setProperty("http.proxyPort", r1[1]);
                    System.err.println(r1[0] + ":" + r1[1]);
                }
                Document doc1 = null;


                try {
                    doc1 = Jsoup.connect(pathes[i])
                            .timeout(10000)
                            .ignoreHttpErrors(true)
                            .userAgent(ua[k])
//                        .cookie("msign_dsr", "1592578548328")
//                        .cookie("search_uuid", "59d60d20%2d888c%2d4339%2d893c%2d3f2df0b416a9")
//                        .cookie("UM_distinctid", "172cd13faab18a-0cc175a1ccab1-3c3f5a0c-1fa400-172cd13faac61e")
//                        .cookie("__dxca", "7959aeaa-2cb3-44b4-ac6d-5a19f2ac1261")
//                        .cookie("lv", "0")
//                        .cookie("chaoxinguser", "1")
//                        .cookie("uname", "")
//                        .cookie("_uid", "143680086")
//                        .cookie("uf", "f9866f9a46b70622b0dee5ad98478a8391831cfec1654dd57a4fb73fc6375849f7893fe5a7d754cdff17dba0ec4416549b0594e13f4b452fbdd6b93a431584911471850d8bf7e34c24848f3d3f307228570bdddc4e9885c5")
//                        .cookie("_d", "1592928430294")
//                        .cookie("UID", "143680086")
//                        .cookie("vc", "A5267B905B437CE795E472D324AD65E0")
//                        .cookie("vc2", "C473E4E0C10364B7C0A31DE46070C411")
//                        .cookie("vc3", "C9JC%2Ff8LUHJ%2FbURcxo1vggUNgTmarTCVJFGMM3kP7MoczjjCxUPi89UGJjjLU4sQ6WNHQgZOUN1mj6sDYiMWNCcwVw%2BMLRe1D3y5d%2FofpoF2AMh3LToyTKUsC9ykkTfsWnWiDPL32C%2Bziy%2BnJFuaW5KB9%2FlDulddw%2B9thByI%2Bvg%3Daaeb61274732c450c6a62c27f9001cce")
//                        .cookie("xxtenc", "3fa0ee7c7d4f9d548e4bc1d7336c838c")
//                        .cookie("duxiu", "userName%5fdsr%2c%3dxbnlkjdx%2c%21userid%5fdsr%2c%3d1406%2c%21char%5fdsr%2c%3d%u78cb%2c%21metaType%2c%3d353%2c%21dsr%5ffrom%2c%3d1%2c%21logo%5fdsr%2c%3dlogo0408%2ejpg%2c%21logosmall%5fdsr%2c%3dsmall0408%2ejpg%2c%21title%5fdsr%2c%3d%u897f%u5317%u519c%u6797%u79d1%u6280%u5927%u5b66%2c%21url%5fdsr%2c%3debook%2c%21compcode%5fdsr%2c%3d1176%2c%21province%5fdsr%2c%3d%u9655%u897f%2c%21readDom%2c%3d0%2c%21isdomain%2c%3d61707%2c%21showcol%2c%3d0%2c%21hu%2c%3d0%2c%21uscol%2c%3d0%2c%21isfirst%2c%3d0%2c%21istest%2c%3d0%2c%21cdb%2c%3d0%2c%21og%2c%3d1%2c%21ogvalue%2c%3d7%2c%21testornot%2c%3d1%2c%21remind%2c%3d0%2c%21datecount%2c%3d3478%2c%21userIPType%2c%3d2%2c%21lt%2c%3d0%2c%21ttt%2c%3dfxlogin%2echaoxing%2c%21enc%5fdsr%2c%3dD5E78CB6EBD2A70090256E4C463A3566")
                            .get();
                } catch (IOException e) {
                    return;
                }
                Elements key = doc1.select(".Fmian1 table tbody tr");
                System.out.println(i + 1 + ": " + pathes[i]);

                //【作者机构】
                //【来    源】
                //【分 类 号】
                //【分类导航】
                //【关 键 词】
                //【基    金】
                //【摘    要】
                //【统计数据】
                //【全文获取】
                for (Element e : key) {
                    if (e.child(0).text().equals("【关 键 词】")) {
                        for (int z = 0; z < e.child(1).children().size() && z < 8; z++) {
                            keys[i][z] = e.child(1).children().get(z).text();
                            //System.out.println(keys[i][z]);
                        }
                    }
                    if (e.child(0).text().equals("【基    金】")) {
                        funds[i] = e.child(1).text();
                        //System.out.println(funds[i]);
                    }
                    if (e.child(0).text().equals("【作者机构】")) {
                        if (e.child(1).children().size() == 0) {
                            units[i] = "西北农林科技大学黄土高原土壤侵蚀与旱地农业国家重点实验室";
                        } else {
                            units[i] = e.child(1).child(0).text();
                        }

                        //System.out.println(units[i]);
                    }
                    if (e.child(0).text().equals("【摘    要】")) {
                        abstracts[i] = e.child(1).text();
                        //System.out.println(abstracts[i]);
                    }
                    //System.out.println(e.child(0).text());
                }
                LW lw = new LW();
                lw.setSubject(subjects.get(i + 1).child(1).text());
                lw.setAuthor(authors.get(i + 1).child(2).text());
                lw.setType("cssci");
                if (quotenums[i] != null) lw.setQuote_num(quotenums[i]);
                else lw.setQuote_num("");
                if (downloadnums[i] != null) lw.setDownload_num(downloadnums[i]);
                else lw.setDownload_num("");
                lw.setPeriodical(periodicals[i]);
                lw.setPublish_time(publishtimes[i]);
                if (keys[i][0] != null) lw.setKey(keys[i][0]);
                else lw.setKey("");
                if (keys[i][1] != null) lw.setKey1(keys[i][1]);
                else lw.setKey1("");
                if (keys[i][2] != null) lw.setKey2(keys[i][2]);
                else lw.setKey2("");
                if (keys[i][3] != null) lw.setKey3(keys[i][3]);
                else lw.setKey3("");
                if (keys[i][4] != null) lw.setKey4(keys[i][4]);
                else lw.setKey4("");
                if (keys[i][5] != null) lw.setKey5(keys[i][5]);
                else lw.setKey5("");
                if (keys[i][6] != null) lw.setKey6(keys[i][6]);
                else lw.setKey6("");
                if (keys[i][7] != null) lw.setKey7(keys[i][7]);
                else lw.setKey7("");
                if (funds[i] != null) lw.setFund(funds[i]);
                else lw.setFund("");
                if (units[i] != null) lw.setUnit(units[i]);
                else lw.setUnit("");
                lw.setAbstract1(abstracts[i]);

                lws.add(lw);
                LW.lws.add(lw);
//            funds[i] = key.get(5).child(1).text();
//            System.out.println(funds[i]);


            }


            for (int i = 0; i < 50; i++) {

            }
            return;
        }
    }

2.pachong.class

package com.example.util;


import com.example.mapper.bookmapper;
import com.example.service.bookservice;
import com.example.service.serviceimpl.bookserviceimpl;
import org.apache.xmlbeans.impl.xb.xsdschema.Public;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Controller;
import org.springframework.stereotype.Service;
import org.springframework.test.context.junit4.SpringRunner;

import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.io.*;
import java.net.*;
import java.security.PublicKey;
import java.util.ArrayList;
import java.util.Random;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@Service
public class pachong{

    @Autowired
    bookmapper bm;

    public static void main(String[] args) throws IOException, InterruptedException {


    }

    public ArrayList<LW> palunwen(Integer q){
        String path="";
        ArrayList<LW>lws = new ArrayList<>();
//        String [] ips = {"120.79.64.147","175.148.74.121","139.9.195.202","60.167.132.223"};
//        String [] ports = {"8118","1133","8118","808"};
//        System.getProperties().setProperty("http.proxyHost", ips[q%5]);
//        System.getProperties().setProperty("http.proxyPort", ports[q%5]);
//        System.err.println(ips[q%5]+":"+ports[q%5]);
        try {
            path = "http://qikan.chaoxing.com/searchjour?sw=" + URLEncoder.encode("西北农林科技大学", "utf-8") + "&stryear=13&nosim=1&size=50&x=0_900&pages="+String.valueOf(q);
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        }
        Random r = new Random();
        String[] ua = {"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36 OPR/37.0.2178.32",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586",
                "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
                "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
                "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)",
                "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0)",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 BIDUBrowser/8.3 Safari/537.36",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36 Core/1.47.277.400 QQBrowser/9.4.7658.400",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 UBrowser/5.6.12150.8 Safari/537.36",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0",
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36 TheWorld 7",
                "Mozilla/5.0 (Windows NT 6.1; W…) Gecko/20100101 Firefox/60.0"};
        int k = r.nextInt(14);
        String ip = null;
        ip = ips1().get(0);

        Document doc = null;

        try {
            doc = Jsoup.connect(path)
                    .timeout(10000)
                    .ignoreHttpErrors(true)
                    .userAgent(ua[k])
//                    .cookie("msign_dsr", "1592578548328")
//                    .cookie("search_uuid", "59d60d20%2d888c%2d4339%2d893c%2d3f2df0b416a9")
//                    .cookie("UM_distinctid", "172cd13faab18a-0cc175a1ccab1-3c3f5a0c-1fa400-172cd13faac61e")
//                    .cookie("__dxca", "7959aeaa-2cb3-44b4-ac6d-5a19f2ac1261")
//                    .cookie("lv", "0")
//                    .cookie("chaoxinguser", "1")
//                    .cookie("uname", "")
//                    .cookie("_uid", "143680086")
//                    .cookie("uf", "f9866f9a46b70622b0dee5ad98478a8391831cfec1654dd57a4fb73fc6375849f7893fe5a7d754cdff17dba0ec4416549b0594e13f4b452fbdd6b93a431584911471850d8bf7e34c24848f3d3f307228570bdddc4e9885c5")
//                    .cookie("_d", "1592928430294")
//                    .cookie("UID", "143680086")
//                    .cookie("vc", "A5267B905B437CE795E472D324AD65E0")
//                    .cookie("vc2", "C473E4E0C10364B7C0A31DE46070C411")
//                    .cookie("vc3", "C9JC%2Ff8LUHJ%2FbURcxo1vggUNgTmarTCVJFGMM3kP7MoczjjCxUPi89UGJjjLU4sQ6WNHQgZOUN1mj6sDYiMWNCcwVw%2BMLRe1D3y5d%2FofpoF2AMh3LToyTKUsC9ykkTfsWnWiDPL32C%2Bziy%2BnJFuaW5KB9%2FlDulddw%2B9thByI%2Bvg%3Daaeb61274732c450c6a62c27f9001cce")
//                    .cookie("xxtenc", "3fa0ee7c7d4f9d548e4bc1d7336c838c")
//                    .cookie("duxiu", "userName%5fdsr%2c%3dxbnlkjdx%2c%21userid%5fdsr%2c%3d1406%2c%21char%5fdsr%2c%3d%u78cb%2c%21metaType%2c%3d353%2c%21dsr%5ffrom%2c%3d1%2c%21logo%5fdsr%2c%3dlogo0408%2ejpg%2c%21logosmall%5fdsr%2c%3dsmall0408%2ejpg%2c%21title%5fdsr%2c%3d%u897f%u5317%u519c%u6797%u79d1%u6280%u5927%u5b66%2c%21url%5fdsr%2c%3debook%2c%21compcode%5fdsr%2c%3d1176%2c%21province%5fdsr%2c%3d%u9655%u897f%2c%21readDom%2c%3d0%2c%21isdomain%2c%3d61707%2c%21showcol%2c%3d0%2c%21hu%2c%3d0%2c%21uscol%2c%3d0%2c%21isfirst%2c%3d0%2c%21istest%2c%3d0%2c%21cdb%2c%3d0%2c%21og%2c%3d1%2c%21ogvalue%2c%3d7%2c%21testornot%2c%3d1%2c%21remind%2c%3d0%2c%21datecount%2c%3d3478%2c%21userIPType%2c%3d2%2c%21lt%2c%3d0%2c%21ttt%2c%3dfxlogin%2echaoxing%2c%21enc%5fdsr%2c%3dD5E78CB6EBD2A70090256E4C463A3566")
                    .get();
        } catch (IOException e) {
            return lws;
        }

        Elements subjects = doc.select("#liebiaoDivId tr");
        Elements authors = doc.select("#liebiaoDivId tr");
        Elements downloadnum = doc.select("#liebiaoDivId tr");
        Elements quotenum = doc.select("#liebiaoDivId tr");

        String[] pathes = new String[50];
        String[] quotenums = new String[50];
        String[] downloadnums = new String[50];
        String[] periodicals = new String[50];
        String[] publishtimes = new String[50];
        String[] funds = new String[50];
        String[][] keys = new String[50][8];
        String[] units = new String[50];
        String[] abstracts = new String[50];
        for (int i = 0; i < 50; i++) {
            pathes[i] = "http://qikan.chaoxing.com" + subjects.get(i + 1).child(1).child(0).attr("href").trim();
            quotenums[i] = quotenum.get(i + 1).child(6).child(0).text();
            downloadnums[i] = downloadnum.get(i + 1).child(7).child(0).text();
            periodicals[i] = downloadnum.get(i + 1).child(3).child(0).text();

            if (downloadnum.get(i + 1).child(5).child(0).text().length() == 8) {
                publishtimes[i] = downloadnum.get(i + 1).child(5).child(0).text().substring(0, 4) + "-0" + String.valueOf(Integer.parseInt(String.valueOf(downloadnum.get(i + 1).child(5).child(0).text().charAt(6))));
            } else if (downloadnum.get(i + 1).child(5).child(0).text().length() > 8&&(Integer.valueOf(downloadnum.get(i + 1).child(5).child(0).text().charAt(6))<=9)&&(Integer.valueOf(downloadnum.get(i + 1).child(5).child(0).text().charAt(6))>=0)) {

                publishtimes[i] = downloadnum.get(i + 1).child(5).child(0).text().substring(0, 4) + "-" + String.valueOf(Integer.valueOf(downloadnum.get(i + 1).child(5).child(0).text().substring(6, 8)) / 2);
            }
            else{
                publishtimes[i] = "1";
            }
            //System.out.println(publishtimes[i]);
            //System.out.println(periodicals[i]);
            //System.out.println(downloadnums[i]);
            //System.out.println(quotenums[i]);
            //System.out.println(pathes[i]);
        }
        int num=0;
        ArrayList<String> ipa = ips1();
        for (int i = 0; i < 50; i++) {
            if(num>195){
                ipa = ips1();
            }
            if(i%10==0){
                ip = ipa.get(num);
                num++;
                String[] r1 = ip.split(":");
                System.out.println(ip);
                System.getProperties().setProperty("http.proxyHost", r1[0]);
                System.getProperties().setProperty("http.proxyPort", r1[1]);
                System.err.println(r1[0]+":"+r1[1]);
            }
            Document doc1 = null;


            try {
                doc1 = Jsoup.connect(pathes[i])
                        .timeout(10000)
                        .ignoreHttpErrors(true)
                        .userAgent(ua[k])
//                        .cookie("msign_dsr", "1592578548328")
//                        .cookie("search_uuid", "59d60d20%2d888c%2d4339%2d893c%2d3f2df0b416a9")
//                        .cookie("UM_distinctid", "172cd13faab18a-0cc175a1ccab1-3c3f5a0c-1fa400-172cd13faac61e")
//                        .cookie("__dxca", "7959aeaa-2cb3-44b4-ac6d-5a19f2ac1261")
//                        .cookie("lv", "0")
//                        .cookie("chaoxinguser", "1")
//                        .cookie("uname", "")
//                        .cookie("_uid", "143680086")
//                        .cookie("uf", "f9866f9a46b70622b0dee5ad98478a8391831cfec1654dd57a4fb73fc6375849f7893fe5a7d754cdff17dba0ec4416549b0594e13f4b452fbdd6b93a431584911471850d8bf7e34c24848f3d3f307228570bdddc4e9885c5")
//                        .cookie("_d", "1592928430294")
//                        .cookie("UID", "143680086")
//                        .cookie("vc", "A5267B905B437CE795E472D324AD65E0")
//                        .cookie("vc2", "C473E4E0C10364B7C0A31DE46070C411")
//                        .cookie("vc3", "C9JC%2Ff8LUHJ%2FbURcxo1vggUNgTmarTCVJFGMM3kP7MoczjjCxUPi89UGJjjLU4sQ6WNHQgZOUN1mj6sDYiMWNCcwVw%2BMLRe1D3y5d%2FofpoF2AMh3LToyTKUsC9ykkTfsWnWiDPL32C%2Bziy%2BnJFuaW5KB9%2FlDulddw%2B9thByI%2Bvg%3Daaeb61274732c450c6a62c27f9001cce")
//                        .cookie("xxtenc", "3fa0ee7c7d4f9d548e4bc1d7336c838c")
//                        .cookie("duxiu", "userName%5fdsr%2c%3dxbnlkjdx%2c%21userid%5fdsr%2c%3d1406%2c%21char%5fdsr%2c%3d%u78cb%2c%21metaType%2c%3d353%2c%21dsr%5ffrom%2c%3d1%2c%21logo%5fdsr%2c%3dlogo0408%2ejpg%2c%21logosmall%5fdsr%2c%3dsmall0408%2ejpg%2c%21title%5fdsr%2c%3d%u897f%u5317%u519c%u6797%u79d1%u6280%u5927%u5b66%2c%21url%5fdsr%2c%3debook%2c%21compcode%5fdsr%2c%3d1176%2c%21province%5fdsr%2c%3d%u9655%u897f%2c%21readDom%2c%3d0%2c%21isdomain%2c%3d61707%2c%21showcol%2c%3d0%2c%21hu%2c%3d0%2c%21uscol%2c%3d0%2c%21isfirst%2c%3d0%2c%21istest%2c%3d0%2c%21cdb%2c%3d0%2c%21og%2c%3d1%2c%21ogvalue%2c%3d7%2c%21testornot%2c%3d1%2c%21remind%2c%3d0%2c%21datecount%2c%3d3478%2c%21userIPType%2c%3d2%2c%21lt%2c%3d0%2c%21ttt%2c%3dfxlogin%2echaoxing%2c%21enc%5fdsr%2c%3dD5E78CB6EBD2A70090256E4C463A3566")
                        .get();
            } catch (IOException e) {
                return lws;
            }
            Elements key = doc1.select(".Fmian1 table tbody tr");
            System.out.println(i+1+": "+pathes[i]);

            //【作者机构】
            //【来    源】
            //【分 类 号】
            //【分类导航】
            //【关 键 词】
            //【基    金】
            //【摘    要】
            //【统计数据】
            //【全文获取】
            for(Element e:key){
                if(e.child(0).text().equals("【关 键 词】")){
                    for (int z = 0; z < e.child(1).children().size() && z < 8; z++) {
                        keys[i][z] = e.child(1).children().get(z).text();
                        //System.out.println(keys[i][z]);
                    }
                }
                if(e.child(0).text().equals("【基    金】")){
                    funds[i] = e.child(1).text();
                    //System.out.println(funds[i]);
                }
                if(e.child(0).text().equals("【作者机构】")){
                    if(e.child(1).children().size()==0){
                        units[i] = "西北农林科技大学黄土高原土壤侵蚀与旱地农业国家重点实验室";
                    }
                    else{
                        units[i] = e.child(1).child(0).text();
                    }

                    //System.out.println(units[i]);
                }
                if(e.child(0).text().equals("【摘    要】")){
                    abstracts[i] = e.child(1).text();
                    //System.out.println(abstracts[i]);
                }
                //System.out.println(e.child(0).text());
            }
//            funds[i] = key.get(5).child(1).text();
//            System.out.println(funds[i]);


        }


        for (int i = 0; i < 50; i++) {
            LW lw = new LW();
            lw.setSubject(subjects.get(i + 1).child(1).text());
            lw.setAuthor(authors.get(i + 1).child(2).text());
            lw.setType("cssci");
            if (quotenums[i] != null) lw.setQuote_num(quotenums[i]);
            else lw.setQuote_num("");
            if (downloadnums[i] != null) lw.setDownload_num(downloadnums[i]);
            else lw.setDownload_num("");
            lw.setPeriodical(periodicals[i]);
            lw.setPublish_time(publishtimes[i]);
            if (keys[i][0] != null) lw.setKey(keys[i][0]);
            else lw.setKey("");
            if (keys[i][1] != null) lw.setKey1(keys[i][1]);
            else lw.setKey1("");
            if (keys[i][2] != null) lw.setKey2(keys[i][2]);
            else lw.setKey2("");
            if (keys[i][3] != null) lw.setKey3(keys[i][3]);
            else lw.setKey3("");
            if (keys[i][4] != null) lw.setKey4(keys[i][4]);
            else lw.setKey4("");
            if (keys[i][5] != null) lw.setKey5(keys[i][5]);
            else lw.setKey5("");
            if (keys[i][6] != null) lw.setKey6(keys[i][6]);
            else lw.setKey6("");
            if (keys[i][7] != null) lw.setKey7(keys[i][7]);
            else lw.setKey7("");
            if (funds[i] != null) lw.setFund(funds[i]);
            else lw.setFund("");
            if (units[i] != null) lw.setUnit(units[i]);
            else lw.setUnit("");
            lw.setAbstract1(abstracts[i]);

            lws.add(lw);

        }
        return lws;
    }
    public ArrayList<String> ips() throws IOException {
        String path = "http://api.xiequ.cn/VAD/GetIp.aspx?act=get&num=100&time=30&plat=1&re=1&type=0&so=1&ow=1&spl=1&addr=&db=1";// 要获得html页面内容的地址

        URL url = new URL(path);// 创建url对象

        HttpURLConnection conn = (HttpURLConnection) url.openConnection();// 打开连接
        conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
        conn.setRequestProperty("contentType", "GBK"); // 设置url中文参数编码

        conn.setConnectTimeout(5 * 1000);// 请求的时间

        conn.setRequestMethod("GET");// 请求方式

        InputStream inStream = conn.getInputStream();
        // readLesoSysXML(inStream);

        BufferedReader in = new BufferedReader(new InputStreamReader(inStream, "GBK"));
        StringBuffer buffer = new StringBuffer();
        ArrayList<String> ipp = new ArrayList<String>();
        String line = "";
        // 读取获取到内容的最后一行,写入
        while ((line = in.readLine()) != null) {
            buffer.append(line);
            ipp.add(line);
            System.out.println(line);
        }
        String str = buffer.toString();
//    JSONObject json1 = JSONObject.parseObject(str);
//    JSONArray jsons =  JSONArray.parseArray(json1.get("data").toString());

//    for(Object json:jsons){
//        JSONObject ips = JSONObject.parseObject(json.toString());
//        String ip = ips.get("IP").toString();
//        System.out.println(ip);
//        ipp.add(ip);
//    }
        return ipp;

    }

    public ArrayList<String> ips1(){
        String path = "http://119.45.8.232/Api/?k=NRAX9VIN451I35QJP36SEM&num=1&type=1&f=1&repeat=1&respone=0&ptn=1";// 要获得html页面内容的地址

        URL url = null;// 创建url对象
        try {
            url = new URL(path);
        } catch (MalformedURLException e) {
            ArrayList<String>s = new ArrayList<>();
            s.add("1.1.1.1:80");
            return s;
        }

        HttpURLConnection conn = null;// 打开连接
        try {
            conn = (HttpURLConnection) url.openConnection();
        } catch (IOException e) {
            ArrayList<String>s = new ArrayList<>();
            s.add("58.218.200.248:22069");
            return s;
        }
        conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
        conn.setRequestProperty("contentType", "utf-8"); // 设置url中文参数编码

        conn.setConnectTimeout(5 * 1000);// 请求的时间

        try {
            conn.setRequestMethod("GET");// 请求方式
        } catch (ProtocolException e) {
            ArrayList<String>s = new ArrayList<>();
            s.add("1.1.1.1:80");
            return s;
        }

        InputStream inStream = null;
        try {
            inStream = conn.getInputStream();
        } catch (IOException e) {
            ArrayList<String>s = new ArrayList<>();
            s.add("1.1.1.1:80");
            return s;
        }
        // readLesoSysXML(inStream);

        BufferedReader in = null;
        try {
            in = new BufferedReader(new InputStreamReader(inStream, "utf-8"));
        } catch (UnsupportedEncodingException e) {
            ArrayList<String>s = new ArrayList<>();
            s.add("1.1.1.1:80");
            return s;
        }
        StringBuffer buffer = new StringBuffer();
        ArrayList<String> ipp = new ArrayList<String>();
        String line = "";
        // 读取获取到内容的最后一行,写入
        while (true) {
            try {
                if (!((line = in.readLine()) != null)) break;
            } catch (IOException e) {
                ArrayList<String>s = new ArrayList<>();
                s.add("1.1.1.1:80");
                return s;
            }
            buffer.append(line);
            ipp.add(line);
            System.out.println(line);
        }
        String str = buffer.toString();
//    JSONObject json1 = JSONObject.parseObject(str);
//    JSONArray jsons =  JSONArray.parseArray(json1.get("data").toString());

//    for(Object json:jsons){
//        JSONObject ips = JSONObject.parseObject(json.toString());
//        String ip = ips.get("IP").toString();
//        System.out.println(ip);
//        ipp.add(ip);
//    }
        return ipp;

    }

    public void xiancheng() throws IOException, InterruptedException {
        ExecutorService pool = Executors.newCachedThreadPool();

//        ArrayList<String>ips = new ArrayList<>();
//        try {
//            File file = new File("C:/IMAGES/ip.txt");
//            BufferedReader bufferedReader = new BufferedReader(new FileReader(file));
//            String strLine = null;
//            int lineCount = 1;
//            while(null != (strLine = bufferedReader.readLine())){
//                System.err.println(strLine);
//                ips.add(strLine);
//                lineCount++;
//            }
//        }catch(Exception e){
//            e.printStackTrace();
//        }

        ArrayList ips = ips();
        for (int i = 0; i < 1500; i++) {
            xiancheng xian = new xiancheng(ips, i);
            xian.run();
            //pool.execute(xian);
            for(LW lw:xian.lws) {
                if (!bm.searchlw(lw.getSubject())) {
                    if (bm.lwcount() == null) {
                        lw.setId(String.valueOf("1"));
                    } else {
                        lw.setId(String.valueOf(Integer.valueOf(bm.lwcount()) + 1));
                    }
                }
                bm.insertlw(lw);
                System.out.println(lw.toString());
            }

        }

    }
}
旭旭1998
关注
java多线程技术论文
02-21
java多线程技术论文—毕业设计论文说明书.
Java总结篇系列：Java多线程（三）
weixin_34258782的博客
07-06 112
本文主要接着前面多线程的两篇文章总结Java多线程中的线程安全问题。 一.一个典型的Java线程安全例子 1 public class ThreadTest { 2 3 public static void main(String[] args) { 4 Account account = new Account("123456", 1000); ...
java多线程论文_Java5 多线程之入门篇-论文
weixin_36235398的博客
03-08 211
Java5多线程之入门篇Java5 多线程之入门篇首先回顾一下JDK1.5之前的线程相关的知识:1 线程的入门.什么是线程,线程就是程序执行的线索,Java是面向对象的语言什么类来表示这样一个东西呢?Thread.通过start()方法启动它,线程所要执行的任务放在run()方法里面,下面可以看一下run()方法里面的源码创建线程的两种传统方式(注: Runnable类并不是一个线程,它只是线程...
多线程详解
qq_38788128的博客
06-25 281
转：https://blog.csdn.net/c10wtiybq1ye3/article/details/78212045进程与线程进程是程序的一次动态执行过程，它需要经历从代码加载，代码执行到执行完毕的一个完整的过程，这个过程也是进程本身从产生，发展到最终消亡的过程。多进程操作系统能同时达运行多个进程（程序），由于 CPU 具备分时机制，所以每个进程都能循环获得自己的CPU 时间片。由于 CP...
java线程技术论文_多线程技术在数据通信中的应用论文
weixin_34064233的博客
03-01 281
多线程技术在数据通信中的应用论文数据通信系统得益于现代数据技术的大力发展，其结构和功能也在不断完善。但由于信息数据在传输中受到现有材料科学的制约，在某些方面还存在一定的弊端，多线程技术的有效应用解决了这一难题，它的优点在于其可以提高系统的安全性与数据的可靠性。多线程技术的广泛应用对整个数据通信领域产生了重大的积极作用。1多线程技术及其适用场合假定在一个系统中，整个系统在结构和功能上是一个同步的整体...
论文研究-Java多线程测试策略及测试方法探讨.pdf
07-22
在分析Java 多线程特性的基础上, 探讨了Java 多线程的测试策略及测试方法, 提出Java 多线程测试由类测试、集成模块测试和系统测试三个层次组成, 并讨论了多线程的继承测试、同步测试以及效率测试。
JAVA多线程与线程安全实践-基于Http协议的断点续传【毕业论文】.zip
最新发布
05-21
1. **Java多线程**：Java中的多线程允许程序同时执行多个任务，提高系统资源利用率。通过创建Thread类的实例或者实现Runnable接口来创建线程。线程间通信可以通过共享内存（如共享变量）或使用线程安全的数据结构...
Java多线程-多功能演示系统.zip
02-02
Java多线程-多功能演示系统，连接MySQL数据库利用多线程实现信息交流，为教师提供教学服务，便于教师整理教学资料，整合所需知识内容，更好的为学生提供生动形象的理解方式，加强教师与学生之间的互动沟通。
java多线程下载工具 论文
05-24
一个简单多线程下载工具的设计与实现，论文+下载工具 程序中用到jdk1.6的GroupLayout布局。 论文格式规范，详细。非常适合作参考
多线程文章
weixin_46942139的博客
05-14 218
关于多线程文章 1.基本概念: 多线程指的是这个程序（一个进程）运行时产生了不止一个线程 并行： 多个cpu实例或者多台机器同时执行一段处理逻辑，是真正的同时。 并发： 通过cpu调度算法，让用户看上去同时执行，实际上从cpu操作层面不是真正的同时。并发往往在场景中有公用的资源，那么针对这个公用的资源往往 产生瓶颈，我们会用TPS或者QPS来反应这个系统的处理能力。 线程安全 指在并发的情况之下，该代码经过多线程使用，线程的调度顺序不影响任何结果。这个时候使用...
多线程的系列文章
u013452337的博客
01-30 425
Java多线程学习（一）Java多线程入门 Java多线程学习（二）synchronized关键字（1）   Java多线程学习（二）synchronized关键字（2） Java多线程学习（三）volatile关键字 Java多线程学习（四）等待/通知（wait/notify）机制 Java多线程学习（五）线程间通信知识点补充 Java多线程学习（六）Lock锁的使用 Java多...
多线程下载CVPR ICCV IJCAI论文
Zjhao666的博客
09-18 410
前提： 科学上网 谷歌浏览器 python3 特点： 自定义多线程下载数目threads_num 自定义下载路径 下载的文件均以论文的title命名 原始title中的非法字符（win10不允许作为文件名的字符）已被处理为下划线或空格 代码如下： # coding:utf-8 import re import urllib.request import os import thr...
多线程设计和实现
tornmy的博客
08-31 2706
站在老罗的肩膀上：https://blog.csdn.net/Luoshengyang/article/details/46855395 多线程模型则是为了解决网页的卡顿问题。为了达到这个目的，Chromium的多线程模型是基于异步通信的。 一个典型的异步通信过程如图1所示： 图1  线程异步通信过程        Task-1被分解成三个子任务Task-1(1)、Task-1(2)和...
UE4影视特效学习资源整理
咬咬牙 再来啊
01-05 2482
houdini官方文档 https://www.sidefx.com/docs/unreal/index.html UE官方大佬 AsherTech Artist @ Epic Games 个人网站http://asher.gg/ 推特https://twitter.com/Vuthric/ 知乎https://www.zhihu.com/people/vuth B站https://space.bilibili.com/101491 Houdini Niagara插件与UE4的特效交互Houdini to
JAVA多线程————一篇文章让你彻底征服多线程开发(一)
热门推荐
姚佳伟
06-29 3万+
线程生命周期 线程的调度与控制 synchronized 死锁 守护线程 Timerschedule线程生命周期线程是一个进程中的执行场景,一个进程可以启动多个线程 多线程不是为了提高执行速度,而是提高应用程序的使用率.线程和线程共享”堆内存和方法区内存”.栈内存是独立的,一个线程一个栈.可以给现实世界中的人类一种错觉 : 感觉多线程在同时并发执行. 很多人都对其中的一些概念不够明确，如同步、
Java多线程使用浅析
WilsonSong1024的博客
05-29 3110
多线程是在同一个程序内部并行执行，因此会对相同的内存空间进行并发读写操作。有一个容易混淆的概念叫做进程。进程：一个计算机程序的运行实例，包含了需要执行的指令；有自己的独立地址空间，包含程序内容和数据；不同进程的地址空间是互相隔离的；进程拥有各种资源和状态信息，包括打开的文件、子进程和信号处理。线程：表示程序的执行流程，是CPU调度执行的基本单位；线程有自己的程序计数器、寄存器、堆栈和帧。同一进程中...
ssr的pac中加入学校图书馆数据库访问pac
Fxxk CPP
03-25 2881
在家的时候, 需要使用代理, 同时要使用图书馆的数据库, 两个代理有点冲突, 自己研究了一下 ssr 的规则, 发现比较简单. 可以页面后贴出的代码进行研究 pac.txt pac 的内部完全就是 js 的语法逻辑, 不是十分熟悉, 写的有点硬, 可以自己考虑优化 执行从 FindProxyForURL 开始执行, 然后根据具体内容进行代理访问 按照学校代理的判断方式, 判断一个 url 是否符...
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值