JAVA爬虫获取气象数据

1.打开网页http://www.weather.com.cn

2.点击今天可查看详细天气信息

3.按F12查看网络接口，发现这个接口就是我们需要的信息了

http://d1.weather.com.cn/dingzhi/101230101.html?_=1655689910056

4.可以看到响应结果包含了天气数据

 

 5.这个101230101是城市编码，可在city.js中查看各个城市的编码

 

 6.通过postman或者apifox生成java代码（具体过程请参考原文）

      <dependency>
            <groupId>com.squareup.okhttp3</groupId>
            <artifactId>okhttp</artifactId>
            <version>3.14.2</version>
      </dependency>

OkHttpClient client = new OkHttpClient().newBuilder()
  .build();
Request request = new Request.Builder()
  .url("http://d1.weather.com.cn/sk_2d/101230101.html")
  .method("GET", null)
  .addHeader("Referer", "http://www.weather.com.cn/")
  .build();
Response response = client.newCall(request).execute();

7.创建数据库表

CREATE TABLE `weather` (
  `id` varchar(36) NOT NULL,
  `city_name` varchar(32) DEFAULT NULL COMMENT '城市名称',
  `city` varchar(32) DEFAULT NULL COMMENT '城市编码',
  `temp` int(11) DEFAULT NULL COMMENT '温度',
  `tempf` int(11) DEFAULT NULL COMMENT '华氏度',
  `wd` varchar(32) DEFAULT NULL COMMENT '风向',
  `ws` varchar(32) DEFAULT NULL COMMENT '风力',
  `wse` varchar(32) DEFAULT NULL COMMENT '风速',
  `sd` varchar(32) DEFAULT NULL COMMENT '湿度',
  `date_time` datetime DEFAULT NULL COMMENT '日期时间',
  `weather` varchar(32) DEFAULT NULL COMMENT '天气',
  `qy` varchar(32) DEFAULT NULL COMMENT '气压',
  `njd` varchar(32) DEFAULT NULL COMMENT '能见度',
  `rain` decimal(10,2) DEFAULT NULL COMMENT '降雨量',
  `rain24h` decimal(10,2) DEFAULT NULL COMMENT '降雨量24小时',
  `aqi` int(11) DEFAULT NULL COMMENT '空气质量',
  `limitnumber` varchar(32) DEFAULT NULL COMMENT '限号',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

8.定时抓取（根据需求更改）

    /**
     * 获取数据业务代码
     */
    @SneakyThrows
    @Scheduled(cron = "0 0/3 * * * ? ")//每3分钟执行一次
    public void getWeather(){
        OkHttpClient client = new OkHttpClient().newBuilder()
                .build();
        Request request = new Request.Builder()
                .url("http://d1.weather.com.cn/sk_2d/101110101.html")
                .method("GET", null)
                .addHeader("Referer", "http://www.weather.com.cn/")
                .build();
        Response response = client.newCall(request).execute();
        String body = response.body().string();
        //去除多余字符串
        body = body.substring(body.indexOf('{'));
        JSONObject jsonObject = JSONObject.parseObject(body);
        //创建实体类
        Weather weather = new Weather();
        weather.setCityName(jsonObject.getString("cityname"));
        weather.setCity(jsonObject.getString("city"));
        weather.setTemp(jsonObject.getInteger("temp"));
        weather.setTempf(jsonObject.getInteger("tempf"));
        weather.setWd(jsonObject.getString("WD"));
        weather.setWs(jsonObject.getString("WS"));
        weather.setWse(jsonObject.getString("wse"));
        weather.setSd(jsonObject.getString("SD"));
        weather.setQy(jsonObject.getString("qy"));  
        weather.setNjd(jsonObject.getString("njd"));
        weather.setDateTime(dateFormat(jsonObject.getString("date"),jsonObject.getString("time")));
        weather.setRain(jsonObject.getBigDecimal("rain"));
        weather.setRain24h(jsonObject.getBigDecimal("rain24h"));
        weather.setAqi(jsonObject.getInteger("aqi"));
        weather.setWeather(jsonObject.getString("weather"));
        weather.setLimitnumber(jsonObject.getString("limitnumber"));
 
        //查看库中是否有重复数据
        QueryWrapper<Weather> queryWrapper = new QueryWrapper<>();
        queryWrapper.eq("city",weather.getCity());
        queryWrapper.eq("date_time",weather.getDateTime());
        List<Weather> weathers = baseMapper.selectList(queryWrapper);
        if(weathers.size()==0){
            baseMapper.insert(weather);
        }
 
    }
 
    /**
     * 日期格式转换
     * [url=home.php?mod=space&uid=952169]@Param[/url] dateStr 日期
     * @param timeStr 时间
     * @return
     */
    @SneakyThrows
    private Date dateFormat(String dateStr, String timeStr){
        Calendar cal = Calendar.getInstance();
        int year = cal.get(Calendar.YEAR);
        String dateString = year+"年"+dateStr.substring(0,dateStr.indexOf("("))+timeStr;
        DateFormat format = new SimpleDateFormat("yyyy年MM月dd日HH:mm");
        Date date;
        date = format.parse(dateString);
        return date;
    }