po一个爬虫项目

最新推荐文章于 2022-03-04 09:55:46 发布
最新推荐文章于 2022-03-04 09:55:46 发布
阅读量291 收藏
点赞数
分类专栏： java爬虫 java解析 java
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/qq_42863233/article/details/107482665
版权
java 同时被 3 个专栏收录
13 篇文章 0 订阅
订阅专栏
java解析
10 篇文章 0 订阅
订阅专栏
java爬虫
5 篇文章 0 订阅
订阅专栏 
package com.baosight.utils;

import org.quartz.JobBuilder;
import org.quartz.JobDetail;
import org.quartz.Scheduler;
import org.quartz.SchedulerFactory;
import org.quartz.SimpleScheduleBuilder;
import org.quartz.Trigger;
import org.quartz.TriggerBuilder;
import org.quartz.impl.StdSchedulerFactory;

public class QuartzUtil {

    public static void handleSimpleTrigger(String jobName, String jobGroupName,
                                           String triggerName, String triggerGroupName, Class jobClass,
                                           int time) {
        // 通过schedulerFactory获取一个调度器
        SchedulerFactory schedulerfactory = new StdSchedulerFactory();
        Scheduler scheduler = null;
        try {
            // 通过schedulerFactory获取一个调度器
            scheduler = schedulerfactory.getScheduler();
            // 创建jobDetail实例，绑定Job实现类
            // 指明job的名称，所在组的名称，以及绑定job类
            JobDetail job = JobBuilder.newJob(jobClass)
                    .withIdentity(jobName, jobGroupName).build();
            // 定义调度触发规则
            //使用simpleTrigger规则
            Trigger trigger=TriggerBuilder.newTrigger().withIdentity(triggerName,
                    triggerGroupName)
                    .withSchedule(SimpleScheduleBuilder.repeatSecondlyForever(time)).startNow().build();
            // 把作业和触发器注册到任务调度中
            scheduler.scheduleJob(job, trigger);
            // 启动调度
            scheduler.start();
        } catch (Exception e) {
            System.out.println("错了拉");
        }
    }

}


package com.baosight.utils;

import com.baosight.po.DriverPo;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.openqa.selenium.By;
import org.openqa.selenium.Dimension;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.phantomjs.PhantomJSDriver;
import org.openqa.selenium.phantomjs.PhantomJSDriverService;
import org.openqa.selenium.remote.DesiredCapabilities;

import java.io.File;
import java.io.InputStream;
import java.io.OutputStream;

public class PhontomJsUtil {

    public static DriverPo getHtml(String loginUrl) {

        DriverPo dp = new DriverPo();
        String htmlStr = null;
        WebDriver driver = null;
        try {
            driver = getPhantomJSDriver();
            driver.manage().window().setSize(new Dimension(1920, 1080));
            driver.get(loginUrl);
            Thread.sleep(1000);
            driver.findElement(By.xpath("//*[@id=\"hp-login-user\"]")).sendKeys("3paradm");
            Thread.sleep(1000);
            System.out.println("用户");
            driver.findElement(By.xpath("//*[@id=\"hp-login-password\"]")).sendKeys("3pardata");
            Thread.sleep(1000);
            System.out.println("登密码");
            driver.findElement(By.xpath("//*[@id=\"hp-login-button\"]")).click();
            System.out.println("登陆成功");
            Thread.sleep(5000);
            // 关闭弹窗
            driver.navigate().refresh();
            Thread.sleep(7000);
            dp.setDriver(driver);
            htmlStr = driver.getPageSource();
            dp.setHtmlStr(htmlStr);
        } catch (InterruptedException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } /*
         * finally { driver.quit(); }
         */

        return dp;

    }

    // 驱动加载
    public static WebDriver getPhantomJSDriver() throws Exception {
        DesiredCapabilities dcaps = new DesiredCapabilities();

        //InputStream in = PhontomJsUtil.class.getClassLoader().getResourceAsStream("phantomjs-2.1.1-windows/bin/phantomjs.exe");
        dcaps.setCapability(PhantomJSDriverService.PHANTOMJS_EXECUTABLE_PATH_PROPERTY,loadPhantomJS());
        //PhontomJsUtil.class.getClassLoader().getResource("");
        dcaps.setCapability(PhantomJSDriverService.PHANTOMJS_CLI_ARGS,
                new String[] { "--web-security=false", "--ignore-ssl-errors=true" });

        dcaps.setCapability(PhantomJSDriverService.PHANTOMJS_PAGE_SETTINGS_PREFIX + "webSecurityEnabled", false);
        dcaps.setCapability(PhantomJSDriverService.PHANTOMJS_PAGE_SETTINGS_PREFIX + "XSSAuditingEnabled", false);

        WebDriver driver = new PhantomJSDriver(dcaps);
        return driver;
    }

    private static String loadPhantomJS() {
        String phantomJs = "phantomjs.exe";
        try {
            InputStream in = PhontomJsUtil.class.getClassLoader().getResourceAsStream("phantomjs-2.1.1-windows/bin/" + phantomJs);
            File fileOut = new File("D:/sunbeamApps/" + phantomJs);
            OutputStream out = FileUtils.openOutputStream(fileOut);
            IOUtils.copy(in, out);
            in.close();
            out.close();
            return fileOut.getAbsolutePath();
        } catch (Exception e) {
            return "";
        }
    }

}

package com.baosight.utils;

import java.io.*;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;

import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.io.SAXReader;


public class XmlUtil {

    /**
     * 通过文件的路径获取xml的document对象
     *
     * @param path    文件的路径
     * @return		返回文档对象
     */
    public static Document getXMLByFilePath(InputStream path) {
        if (null == path) {
            return null;
        }
        Document document = null;
        try {
            SAXReader reader = new SAXReader();
            document = reader.read(path);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return document;
    }

    /**
     * 获取保存信息
     *
     * @param path
     */

    public static Map<String, String> getRequest(InputStream path){
        Map<String, String> mapRequest = new HashMap<>();

        Document document = XmlUtil.getXMLByFilePath(path);
        Element rootElm = document.getRootElement();
        for(Iterator iterator = rootElm.element("T_RJJH_DIA_13_42_LOG-netDevice-10.240.13.421").elementIterator();iterator.hasNext();){
            Element element = (Element) iterator.next();
            Iterator provinces = element.elementIterator();
            while (provinces.hasNext()) {
                Element enEle = (Element) provinces.next();
                mapRequest.put(enEle.getName(), enEle.getText());
            }
        }

        return mapRequest;
    }


    /**
     * 获取页面信息
     *
     * @param path
     */

    public static Map<String, String> getPage(InputStream path){
        Map<String, String> page = new HashMap<>();

        Document document = XmlUtil.getXMLByFilePath(path);
        Element rootElm = document.getRootElement();
        for (Iterator iterator = rootElm.element("T_RJJH_DIA_13_42_LOG-netDevice-10.240.13.421")
                .elementIterator(); iterator.hasNext();) {
            Element element = (Element) iterator.next();
            Iterator provinces = element.elementIterator();
            while (provinces.hasNext()) {
                Element enEle = (Element) provinces.next();
                if (enEle.getName().equals("request")) {
                    Iterator lastIter = enEle.elementIterator();
                    while (lastIter.hasNext()) {
                        Element lastEle = (Element) lastIter.next();
                        page.put(lastEle.getName(), lastEle.getText());
                    }
                }
            }
        }

        return page;
    }



    /**
     * 获取element的单个属性
     * @param node		需要获取属性的节点对象
     * @param attr		需要获取的属性值
     * @return			返回属性的值
     */
    public static String getAttribute(Element node,String attr){
        if(null == node||attr==null||"".equals(attr)){
            return "";
        }
        return node.attributeValue(attr);
    }

    /**
     * 去掉声明头的(即<?xml...?>去掉)
     *
     * @param document
     * @param charset
     * @return
     */
    public static String documentToStringNoDeclaredHeader(Document document, String charset) {
        String xml = documentToStringNoDeclaredHeader(document, charset);
        return xml.replaceFirst("\\s*<[^<>]+>\\s*", "");
    }

}


package com.baosight.utils;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Map;

public class JDBCUtil {

    public static Connection getConnection(Connection conn,Map<String, String> map) throws ClassNotFoundException {
        Class.forName(map.get("mysqlDriver"));
        if (conn == null) {
            try {
                conn = DriverManager.getConnection(map.get("mysqlUrl"), map.get("mysqlAdmin"),
                        map.get("mysqlPassword"));
            } catch (SQLException e) {
                throw new RuntimeException(e);
            }
        }
        return conn;
    }

    public static void close(Connection conn, PreparedStatement prep, ResultSet rs) {
        if (rs != null) {
            try {
                rs.close();
            } catch (SQLException e) {
                e.printStackTrace();
            } finally {
                rs = null;
            }
        }
        if (prep != null) {
            try {
                prep.close();
            } catch (SQLException e) {
                e.printStackTrace();
            } finally {
                prep = null;
            }
        }
        if (conn != null) {
            try {
                conn.close();
            } catch (SQLException e) {
                e.printStackTrace();
            } finally {
                conn = null;
            }
        }
    }

}

package com.baosight.utils;

import java.security.MessageDigest;

public class MD5Util {

    private static String byteArrayToHexString(byte b[]) {
        StringBuffer resultSb = new StringBuffer();
        for (int i = 0; i < b.length; i++)
            resultSb.append(byteToHexString(b[i]));

        return resultSb.toString();
    }

    private static String byteToHexString(byte b) {
        int n = b;
        if (n < 0)
            n += 256;
        int d1 = n / 16;
        int d2 = n % 16;
        return hexDigits[d1] + hexDigits[d2];
    }

    /**
     * 返回MD5
     *
     * @return
     */
    public static String MD5Encode(String origin, String charsetname) {
        String resultString = null;
        try {
            resultString = new String(origin);
            MessageDigest md = MessageDigest.getInstance("MD5");
            if (charsetname == null || "".equals(charsetname))
                resultString = byteArrayToHexString(md.digest(resultString.getBytes()));
            else
                resultString = byteArrayToHexString(md.digest(resultString.getBytes(charsetname)));
        } catch (Exception exception) {
            System.out.println();
        }
        return resultString;
    }

    private static final String hexDigits[] = { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d",
            "e", "f" };

}


package com.baosight.run;

import java.io.InputStream;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Map;

import com.baosight.po.DriverPo;
import com.baosight.utils.*;
import org.jsoup.Jsoup;
import org.openqa.selenium.WebDriver;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;

public class App implements Job {
    static DriverPo dp = null;
    static PreparedStatement pstmt = null;
    static InputStream is = App.class.getClassLoader().getResourceAsStream("appConfig.xml");
    static Map<String, String> map = XmlUtil.getRequest(is);
    static InputStream in = App.class.getClassLoader().getResourceAsStream("appConfig.xml");
    static Map<String, String> mapPage = XmlUtil.getPage(in);

    public static void main(String[] args) {
        Connection conn = null;

        try {
            conn = JDBCUtil.getConnection(conn, map);
            for (Map.Entry<String, String> entry : map.entrySet()) {
                String mapKey = entry.getKey();
                String mapValue = entry.getValue();
                if (mapKey.equals("mysqlAdmin") || mapKey.equals("mysqlPassword")) {
                    pstmt = conn.prepareStatement("INSERT into cvs (attrKey,attrValue) VALUES (?,?)");
                    String mapSetValue = MD5Util.MD5Encode(mapValue, "utf-8");
                    pstmt.setString(1, mapKey);
                    pstmt.setString(2, mapSetValue);
                    int result = pstmt.executeUpdate();

                    if (result == 1) {
                        System.out.println("数据插入成功!");
                    } else {
                        System.out.println("数据插入失败!");
                    }
                } else if (mapKey.equals("url")) {
                    StringBuffer sql = new StringBuffer();
                    sql.append("INSERT into cvs (attrKey,attrValue) VALUES ");
                    dp = PhontomJsUtil.getHtml(mapValue);
                    String htmlStr = dp.getHtmlStr();
                    org.jsoup.nodes.Document documentHtml = Jsoup.parse(htmlStr);
                    //documentHtml.
                    org.jsoup.nodes.Element valueEle = documentHtml.selectFirst(mapPage.get("equipmentState"));
                    sql.append("('" + "equipmentState" + "','" + valueEle.text() + "'),");
                    org.jsoup.nodes.Element allocValue = documentHtml.selectFirst(mapPage.get("diskArraryUsed"));
                    sql.append("('" + "diskArraryUsed" + "','" + allocValue.text() + "'),");
                    org.jsoup.nodes.Element freeValue = documentHtml.selectFirst(mapPage.get("diskArraryFree"));
                    sql.append("('" + "diskArraryFree" + "','" + freeValue.text() + "'),");
                    org.jsoup.nodes.Element totalValue = documentHtml.selectFirst(mapPage.get("diskArraryTotal"));
                    sql.append("('" + "diskArraryTotal" + "','" + totalValue.text() + "'),");
                    String a = allocValue.text();
                    String b = totalValue.text();
                    String q = a.substring(0, a.indexOf(" ")).substring(0, a.indexOf(","))+a.substring(0, a.indexOf(" ")).substring(a.indexOf(",")+1);
                    String w = b.substring(0, b.indexOf(" ")).substring(0, b.indexOf(","))+b.substring(0, b.indexOf(" ")).substring(b.indexOf(",")+1);
                    int c = (int) (Double.parseDouble(q)/Double.parseDouble(w)*100);

                    sql.append("('" + "diskArrayUtilization" + "','" + c + "%')");
                    pstmt = conn.prepareStatement(sql.toString());
                    int result = pstmt.executeUpdate();
                    if (result > 0) {
                        System.out.println("数据插入成功!");
                    } else {
                        System.out.println("数据插入失败!");
                    }

                }else{
                    pstmt = conn.prepareStatement("INSERT into cvs (attrKey,attrValue) VALUES (?,?)");
                    pstmt.setString(1, mapKey);
                    pstmt.setString(2, mapValue);
                    int result = pstmt.executeUpdate();
                    if (result >0) {
                        System.out.println("数据插入成功!");
                    } else {
                        System.out.println("数据插入失败!");
                    }
                }
            }
            JDBCUtil.close(conn, pstmt, null);
            QuartzUtil.handleSimpleTrigger("44033", "3333", "44033", "3333", App.class, 240);
        } catch (SQLException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (ClassNotFoundException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally {
            JDBCUtil.close(conn, pstmt, null);
            // dp.getDriver().quit();
        }

    }

    /**
     *
     */
    @Override
    public void execute(JobExecutionContext context) throws JobExecutionException {
        Connection conn = null;
        WebDriver driverJob = null;
        try {
            conn = JDBCUtil.getConnection(conn, map);
            conn.setAutoCommit(false);
            driverJob = dp.getDriver();
            driverJob.navigate().refresh();
            Thread.sleep(6000);
            String htmlStri = driverJob.getPageSource();
            org.jsoup.nodes.Document documentHtml = Jsoup.parse(htmlStri);
            org.jsoup.nodes.Element valueEle = documentHtml.selectFirst(mapPage.get("equipmentState"));
            pstmt = conn.prepareStatement(
                    "update cvs set attrValue='" + valueEle.text() + "' where attrKey='" + "equipmentState" + "'");
            pstmt.addBatch();
            org.jsoup.nodes.Element allocValue = documentHtml.selectFirst(mapPage.get("diskArraryUsed"));
            pstmt = conn.prepareStatement(
                    "update cvs set attrValue='" + allocValue.text() + "' where attrKey='" + "diskArraryUsed" + "'");
            System.out.println(allocValue.text());
            pstmt.addBatch();
            org.jsoup.nodes.Element freeValue = documentHtml.selectFirst(mapPage.get("diskArraryFree"));
            pstmt = conn.prepareStatement(
                    "update cvs set attrValue='" + freeValue.text() + "' where attrKey='" + "diskArraryFree" + "'");
            pstmt.addBatch();
            org.jsoup.nodes.Element totalValue = documentHtml.selectFirst(mapPage.get("diskArraryTotal"));
            pstmt = conn.prepareStatement(
                    "update cvs set attrValue='" + totalValue.text() + "' where attrKey='" + "diskArraryTotal" + "'");
            pstmt.addBatch();
            String a = allocValue.text();
            String b = totalValue.text();
            String q = a.substring(0, a.indexOf(" ")).substring(0, a.indexOf(","))+a.substring(0, a.indexOf(" ")).substring(a.indexOf(",")+1);
            String w = b.substring(0, b.indexOf(" ")).substring(0, b.indexOf(","))+b.substring(0, b.indexOf(" ")).substring(b.indexOf(",")+1);
            int c = (int) (Double.parseDouble(q)/Double.parseDouble(w)*100);
            pstmt = conn.prepareStatement(
                    "update cvs set attrValue='" +c+"%"+ "' where attrKey='" + "diskArrayUtilization" + "'");
            pstmt.addBatch();
            pstmt.executeBatch();
            conn.commit();
            System.out.println("修改成功");
            conn.setAutoCommit(true);
        } catch (SQLException | InterruptedException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (ClassNotFoundException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } finally {
            JDBCUtil.close(conn, pstmt, null);
            System.out.println("数据库连接关闭");
        }

    }
}


package com.baosight.po;

import org.openqa.selenium.WebDriver;

public class DriverPo {

    WebDriver driver;

    String htmlStr;

    public WebDriver getDriver() {
        return driver;
    }

    public void setDriver(WebDriver driver) {
        this.driver = driver;
    }

    public String getHtmlStr() {
        return htmlStr;
    }

    public void setHtmlStr(String htmlStr) {
        this.htmlStr = htmlStr;
    }

}

maven依赖

<dependencies>
		<dependency>
			<groupId>net.sf.cssbox</groupId>
			<artifactId>cssbox</artifactId>
			<version>4.14</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.quartz-scheduler/quartz -->
		<dependency>
			<groupId>org.quartz-scheduler</groupId>
			<artifactId>quartz</artifactId>
			<version>2.2.3</version>
		</dependency>

		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>3.8.1</version>
			<scope>test</scope>
		</dependency>
		<dependency>
			<groupId>us.codecraft</groupId>
			<artifactId>webmagic-selenium</artifactId>
			<version>0.5.2</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
		<dependency>
			<groupId>org.jsoup</groupId>
			<artifactId>jsoup</artifactId>
			<version>1.11.3</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/dom4j/dom4j -->
		<dependency>
			<groupId>dom4j</groupId>
			<artifactId>dom4j</artifactId>
			<version>1.6.1</version>
		</dependency>
		<dependency>
    		<groupId>org.slf4j</groupId>
   	 			<artifactId>slf4j-log4j12</artifactId>
    		<version>1.5.6</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/log4j/log4j -->
		<dependency>
			<groupId>log4j</groupId>
			<artifactId>log4j</artifactId>
			<version>1.2.12</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->
		<dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>
			<version>5.1.38</version>
		</dependency>
		<dependency>
			<groupId>com.codeborne</groupId>
			<artifactId>phantomjsdriver</artifactId>
			<version>1.2.1</version>
		</dependency>

	</dependencies>
Sunbeam___
关注
专栏目录
python爬虫项目（六十一）：爬取社交平台用户互动数据并分析社交圈层关系
sybh的博客
10-04 1296
通过本次项目，我们成功地爬取了社交平台的用户互动数据，构建了用户的社交网络，并利用社区发现算法和机器学习模型对用户的社交圈层进行了深入分析。社交网络的研究为理解信息传播、用户行为预测等提供了强有力的工具，未来可以进一步结合自然语言处理技术，对用户生成内容进行情感分析和话题建模，挖掘更深层次的社交关系。
一个1000元的爬虫外包项目，三种爬虫模式给你轻松做出来
LLand520的博客
01-06 1849
前言 本文的文字及图片来源于网络,仅供学习、交流使用,不具有任何商业用途,如有问题请及时联系我们以作处理。 对于这个外包给予的网站信息，那么我们就爬取选择第一个吧<猎聘网>，也就简单实现，并不会进行打包处理，以及输入关键字爬取。 本篇文章就使用三种爬虫模式爬取相关数据 1、常规爬取数据 2、多线程爬取数据 3、scrapy框架爬取数据 基本开发环境 Python 3.6 Pycharm 相关模块的使用 常规爬取数据 import requests import parsel import csv
Python网络爬虫实战18：爬取热门网络小说
简时刻的博客
07-23 1608
1. 网页源码 2. 代码实例 # coding:utf-8 # 第一步：导入相关库 import requests # 用于请求网页获取网页数据 import re # 使用正则表达式来匹配相关字符 import time # 使用time库中的.sleep()方法让程序暂停 # 第二步：用户代理设置，伪装成浏览器进行访问 headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WO...
java前台接口调用_定义接口，前台调用
weixin_39699070的博客
02-27 716
新建一个ServiceAA04.java文件新建的时候自动生成的，问我我也不懂是干嘛的package com.baosight.bgcloud.aa.service;//这里放要导入的类public class ServiceAA04 {// 这里面写一些接口}接下来导入需要的类import com.baosight.bgcloud.utils.RestfulManager;import com....
XINBEI-监控管理-历史视频检索=添加图选下载路径功能
weixin_30501857的博客
08-07 58
①在ViewAllPlat项目中包com.baosight.viewall.common下建立类FileTool,继承Applet，类中添加如下方法: /** * @author DongMingWang * @time 20150807 * @return filePath */ public String ...
springboot整合mybatis
工学院的博客
05-15 438
springboot整合mybatis pom.xml，我有的是聚合工程 &amp;amp;amp;lt;project xmlns=&amp;amp;quot;http://maven.apache.org/POM/4.0.0&amp;amp;quot; xmlns:xsi=&amp;amp;quot;http://www.w3.org/2001/XMLSchema-instance&amp;amp;quot; xsi:schemaLocation=&amp
Android中ContextMenu的使用
游戏盒子
08-24 242
package com.baosight.menu;   import android.app.Activity; import android.graphics.Color; import android.os.Bundle; import android.view.ContextMenu; import android.view.Menu; import android.v...
elem_data.rar_elem.data_python中elem_trick5po_简单的scrapy爬虫类编写_饿了么
09-20
在本项目中，我们主要关注的是使用Python的Scrapy框架编写一个简单的爬虫来抓取饿了么（Ele.me）网站的首页信息。Scrapy是一个强大的、开源的爬虫框架，用于高效地处理网络数据。以下是对这个项目涉及的知识点的详细...
python网络爬虫二：Requests库网络爬虫实战项目
qq_34964418的博客
04-13 674
Requests库网络爬虫实战 目录：Requests库网络爬虫实战1京东商品页面爬取2 当当网商品页面爬取3 百度360搜索引擎关键词提交4 网络图片爬取和存储5 ip地址归属地查询6 有道翻译翻译表单提交 1京东商品页面爬取 目标页面地址：https://item.jd.com/5089267.html import requests url = 'https://item.jd.com/50...
Python网络爬虫简单入门
weixin_45820912的博客
01-23 2065
预备知识 学习者需要预先掌握Python的数字类型、字符串类型、分支、循环、函数、列表类型、字典类型、文件和第三方库使用等概念和编程方法。 Python爬虫基本流程 a. 发送请求 使用http库向目标站点发起请求，即发送一个Request，Request包含：请求头、请求体等。 Request模块缺陷：不能执行JS 和CSS 代码。 b. 获取响应内容 如果requests的内容存在于目标服务器上，那么服务器会返回请求内容。 Response包含：html、Json字符串、图片，视频等。 c..
java 上传图片与图片读取
热门推荐
dangerYe的专栏
12-04 3万+
上传： //获取参数文件流,并将文件流转化成二进制数组，适应JSP与Servlet DiskFileItemFactory factory = new DiskFileItemFactory(); ServletFileUpload sfu = new ServletFileUpload(factory); sfu.setFileSizeMax(102400000);  sfu.setSi
Could not transfer artifact -boot-starter:pom:6.2.2 from/to maven-Blocked mirror for repositories:
赚钱养家
03-04 1274
拉取git代码直接报错 报错信息：Could not transfer artifact com.baosight.iplat4j:iplat4j-boot-starter:pom:6.2.2 from/to maven-default-http-blocker (http://0.0.0.0/): Blocked mirror for repositories: [baocloud-maven (http://nexus.baocloud.cn/content/groups/public/, defau
使用javabean实现用户登录
u011526599的专栏
05-22 2万+
关于使用javabean实现用户登录初探
记录老项目整合Excel
u011220266的博客
11-07 102
1.前台页面 前台页面请求发送方式：超链接A标签 、form表单提交、Ajax请求 超链接A标签：适合于只有资源路径，少量请求参数，适用于图片下载 form表单提交：使用原有的form标签提交请求，或者自定义form表单提交请求，多参数 Ajax请求：返回数据类型只有：json,text,html,xml，不能处理流，无法用于图片下载 <!-- 方式一：少量参数：超链接 ​--> <a href=...
com.baosight.riasso-crx插件
04-04
语言:中文 (简体) @2019 宝信软件南京分公司系统服务部，统一认证插件。 调用本地客户端程序，作为统一认证系统登录的插件使用。
爬虫回响521_爬虫遇到521错误怎么办
weixin_35645701的博客
02-05 1313
本人最近用python开发着爬虫相关的项目，在上个月就已经把爬虫的相关代码写得差不多了，因为春节，项目停滞了十多天。最近，当我再次运行之前的爬虫项目的时候，What？怎么不行了？？本来好好的，报了521错误。我只是一个爬虫菜鸟，还是第一次接触到这个错误。然后我就去百度了。百度一圈后，发现这是一个js设置cookie的反爬方式。于是乎我将521返回的内容写到了txt文件里，显示出来是这样的结果。太乱...
爬虫项目
Janson_Lin
08-22 498
WechatSogou[1]-微信公众号爬虫。基于搜狗微信搜索的微信公众号爬虫接口，可以扩展成基于搜狗搜索的爬虫，返回结果是列表，每一项均是公众号具体信息字典。 DouBanSpider[2]- 豆瓣读书爬虫。可以爬下豆瓣读书标签下的所有图书，按评分排名依次存储，存储到Excel中，可方便大家筛选搜罗，比如筛选评价人数>1000的高分书籍；可依据不同的主题存储到Excel不同的She...
用java实现des加密和解密
aaaaatiger--寻路
01-10 1027
个用DES来加密、解密的类 http://www.javanb.com/java/1/17816.html import java.security.*; import javax.crypto.Cipher; import javax.crypto.SecretKey; import javax.crypto.SecretKeyFactory; import javax.crypto.spec.
小爬虫demo——爬取“妹子”等网站链接____使用requests库
weixin_30649859的博客
05-26 377
# 文章分为三个部分：1. po代码 ；2. 提示运行过程中的一些细节 和需要改进的地方； 3. 常规送福利# 文章将持续完善和补充# 欢迎拍砖、交流、指正第一部分：Po 代码：import re import requests key = input("请输入爬虫key：") page = int(input("请输入爬取页数：")) def ...
SQL报错:No value specified for parameter 1
Sunbeam的博客
09-16 3023
ResultSet rs = pstmt.executeQuery(); pstmt.setInt(1, resourceId); 在数据还没传进去就executeQuery()了 修改: 先pstmt.set 再executeQuery(); pstmt.setInt(1, resourceId); ResultSet rs = pstmt.executeQuery(); ...
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值