java抓取豆瓣电影数据，分析电影评分，生成统计图表 ---servlet

2019独角兽企业重金招聘Python工程师标准>>>

    最近花时间学习了一下使用Java获取网站数据的方法，自己也亲自动手实践一下；共获取3000+数据，去除重复的数据剩余2000+，使用JFreeChart根据电影评分做出几张简单的统计图。

电影评分统计图：

    JFreeChart生成图片

  

    使用jsoup获取该网站的电影数据信息，此网站动态加载数据，如果直接查看网页源代码是看不到数据的。可以通过js文件，获取相应的数据：

部分代码如下：

movieServlet.java

    主要的功能为：获取网站的电影数据

    首先获取每一个电影分类的链接：

        HashMap<String, String> urlandnames = new HashMap<String, String>();
		MovieService movieService = new MovieService();
		// 排行榜页面
		String url = "http://movie.douban.com/chart";
		// 获取分类的所有相对链接和分类名称
		try {
			Document kinds = Jsoup.connect(url)
							  .userAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36")
							  .timeout(10000)
							  .get();
			Elements elements = kinds.select("#content .types a");
			for(Element element : elements){
				String kindurl = element.attr("href");	// 链接地址
				String name = element.text();			// 类别
				urlandnames.put(kindurl,name);
			}
		} catch (IOException e) {
			e.printStackTrace();
			System.out.println("获取urlandname出现错误！！");
		}
		//获取所有的key
		Set<String> keySet = urlandnames.keySet();	
		//迭代key值
		Iterator<String> iterator = keySet.iterator();
		List<Movie> allMovies = new ArrayList<Movie>();
		while(iterator.hasNext()){
			// 获取到key值,即url
			String next = iterator.next();
			// 根据某一个类别的链接，获取行对应的电影数据
			List<Movie> listMovie = getMovieInfo(next);
			allMovies.addAll(listMovie);
		}

 根据对应的链接获取相应的数据，保存至数据库：

	/**
	 * 获取种类电影信息，保存到数据库
	 * @param url 某一个种类的链接地址
	 */
	private List<Movie> getMovieInfo(String url){
		String[] tempurl = url.split("&");
		String finalurl = "http://movie.douban.com/j/chart/top_list_count?"+tempurl[1]+"&"+tempurl[2];
		// finalurl ---------http://movie.douban.com/j/chart/top_list_count?type=18&interval_id=100:90
		String document = null;
		try {
			//获取该类别影片的数量total、可在线观看数量playable_count
			document = Jsoup.connect(finalurl).timeout(10000).ignoreContentType(true).execute().body();	
			// document------{"playable_count":18,"total":32,"unwatched_count":32}可在线观看18部，共32部，未观看32部
		} catch (IOException e) {
			e.printStackTrace();
		}

		//json解析器
		JsonParser parser = new JsonParser();
		//获取json对象
		JsonObject jsonObject = (JsonObject) parser.parse(document);
		//将json数据转为int型数据
		int movienum = jsonObject.get("total").getAsInt();
		System.out.println(movienum);//该类型的数量
		String nameurl = "http://movie.douban.com/j/chart/top_list?"+tempurl[1]+"&"+tempurl[2]+"&action=&start=0&limit="+movienum;
		// nameurl-------------http://movie.douban.com/j/chart/top_list?type=18&interval_id=100:90&action=&start=0&limit=32
		FileWriter fw = null;
		String doc = null;
		try {
			//获取该类别的所有影片的信息
			doc = Jsoup.connect(nameurl).timeout(10000).ignoreContentType(true).execute().body();
		} catch (Exception e) {
			e.printStackTrace();
		}
		//将json的一个对象数组解析成JsonElement对象
		JsonElement element = null;
		try {
			//通过JsonParser对象可以把json格式的字符串解析成一个JsonElement对象
			element = parser.parse(doc);
		} catch (NullPointerException e) {
			e.printStackTrace();
		}
		
		JsonArray jsonArray = null;
		if(element.isJsonArray()){
			//JsonElement对象如果是一个数组的话转化成jsonArray
			jsonArray = element.getAsJsonArray();
		}
		
		//遍历json的对象数组
		Iterator it = jsonArray.iterator();
		List<Movie> listMovie = new ArrayList<Movie>();
		while (it.hasNext()) {
			JsonObject e = (JsonObject)it.next();
			//电影名称
			String name = e.get("title").getAsString();
			//豆瓣评分
			float score = e.get("score").getAsFloat();
			//发布时间
			String release_date = e.get("release_date").getAsString();
			//类型
			JsonArray jsonArray2 = e.get("types").getAsJsonArray();
			String types = jsonArray2.toString();
			//链接地址
			String movieUrl = e.get("url").getAsString();
			//是否可以在线播放
			String is_playable = e.get("is_playable").getAsString();
			
			String substring = movieUrl.substring(0, movieUrl.lastIndexOf("/"));
			String keyID = substring.substring(substring.lastIndexOf("/"), substring.length());
			
			if(cache.get(keyID) != null){
				String value = (String) cache.get(keyID).getObjectValue();
				if(!name.equals(value)){
					net.sf.ehcache.Element element2 = new net.sf.ehcache.Element(keyID,name);
					cache.put(element2);
				}else {
//					System.out.println("重复的 movie Info");
					continue;
				}
			}else {
				net.sf.ehcache.Element element2 = new net.sf.ehcache.Element(keyID,name);
				cache.put(element2);
			}
			
			Movie movie = new Movie();
			
			movie.setName(name);
			movie.setTypes(types);
			movie.setRelease_date(release_date);
			movie.setScore(score);
			movie.setMovieUrl(movieUrl);
			movie.setIs_playable(is_playable);

			//在控制台输出
//			System.out.println(movie.toString());
//			System.out.println("正在获取数据ing...");
			
			listMovie.add(movie);
		}
		return listMovie;
	}

ScoreServlet.java    主要是生成图表

    生成柱状图：

	protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
		String method = request.getParameter("method");
		System.out.println(method+"===================method");
		MovieService movieService = new MovieService();
		
		Map<String, Integer> map = movieService.Count();
		Integer one = map.get("one");
		Integer two = map.get("two");
		Integer three = map.get("three");
		Integer four = map.get("four");
		Integer five = map.get("five");
		
		if(method.equals("barChart")){
			double [][]data = new double[][]{{one},{two},{three},{four},{five}};
			String []rowKeys = {">=9",">=8.5",">=8",">=7.5","<7.5"}; 
			String []columnKeys = {"评分"};
			
			CategoryDataset dataset = DatasetUtilities.createCategoryDataset(rowKeys, columnKeys, data);
			
	        JFreeChart chart = ChartFactory.createBarChart3D(
	        		"电影评分柱状图", // 图表标题
	                "电影", // 目录轴的显示标签
	                "数量", // 数值轴的显示标签
	                 dataset, // 数据集
	                 PlotOrientation.VERTICAL, // 图表方向：水平、垂直
	                 true,  // 是否显示图例(对于简单的柱状图必须是 false)
	                 false, // 是否创建工具提示 (tooltip) 
	                 false  // 是否生成 URL 链接
	                 ); 
	        
	        CategoryPlot plot = chart.getCategoryPlot();
	        // 设置网格背景颜色
	 		plot.setBackgroundPaint(Color.white);
	 		// 设置网格竖线颜色
	 		plot.setDomainGridlinePaint(Color.pink);
	 		// 设置网格横线颜色
	 		plot.setRangeGridlinePaint(Color.pink);
	 		
	 		// 显示每个柱的数值，并修改该数值的字体属性
	 		BarRenderer3D renderer=new BarRenderer3D();
	 		renderer.setBaseItemLabelGenerator(new StandardCategoryItemLabelGenerator());
	 		renderer.setBaseItemLabelsVisible(true);
	 		
	 		renderer.setBasePositiveItemLabelPosition(new ItemLabelPosition(ItemLabelAnchor.OUTSIDE12, TextAnchor.BASELINE_LEFT));
	 		renderer.setItemLabelAnchorOffset(10D);  
	 		
	 		// 设置平行柱的之间距离
	 		renderer.setItemMargin(0.4);
	 		plot.setRenderer(renderer);
	        
	        FileOutputStream fos_jpg = null; 
	        try { 
	        	//将图片保存至Tomcat服务器WebRoot下的img目录中
	            fos_jpg = new FileOutputStream(request.getSession().getServletContext().getRealPath("/")+"barChart.jpg");
	            ChartUtilities.writeChartAsJPEG(fos_jpg,1,chart,700,500,null); 
	        } catch (Exception e) {
	        	System.out.println("error");
			} finally { 
	            try { 
	                fos_jpg.close(); 
	            } catch (Exception e) {
	            	System.out.println("error2");
	            } 
	        }
	        request.setAttribute("barChart", "barChart.jpg");
			
		}

生成饼状图：

        MovieService movieService = new MovieService();
		
		Map<String, Integer> map = movieService.Count();
		Integer one = map.get("one");
		Integer two = map.get("two");
		Integer three = map.get("three");
		Integer four = map.get("four");
		Integer five = map.get("five");
		
        if (method.equals("pieChart")) {
			
			DefaultPieDataset data = new DefaultPieDataset();
			data.setValue(">=9",one); 
			data.setValue(">=8.5",two); 
			data.setValue(">=8",three); 
			data.setValue(">=7.5",four); 
			data.setValue("<7.5",five); 
	        
	        JFreeChart chart = ChartFactory.createPieChart3D(
	        		"评分饼状图",  		// 图表标题
			        data, 
			        true, 			// 是否显示图例
			        false, 			// 是否创建工具提示 (tooltip) 
	                false  			// 是否生成 URL 链接
			        ); 
	        
			//显示百分比
			PiePlot pieplot = (PiePlot)chart.getPlot();
	        pieplot.setLabelFont(new Font("宋体", 0, 12));
	        pieplot.setNoDataMessage("无数据");
	        pieplot.setCircular(true);
	        pieplot.setLabelGap(0.02D);
	        pieplot.setLabelGenerator(new StandardPieSectionLabelGenerator("{0} {2}",NumberFormat.getNumberInstance(),new DecimalFormat("0.00%")));
	        
	        PiePlot3D pieplot3d = (PiePlot3D)chart.getPlot(); 
			//设置开始角度  
			pieplot3d.setStartAngle(120D);  
			//设置方向为”顺时针方向“  
			pieplot3d.setDirection(Rotation.CLOCKWISE);  
			//设置透明度，0.5F为半透明，1为不透明，0为全透明  
			pieplot3d.setForegroundAlpha(0.7F); 
	        
	        FileOutputStream fos_jpg = null; 
	        try { 
	        	//将图片保存至Tomcat服务器WebRoot目录下
	            fos_jpg = new FileOutputStream(request.getSession().getServletContext().getRealPath("/")+"pieChart.jpg");
	            ChartUtilities.writeChartAsJPEG(fos_jpg,1,chart,700,500,null); 
	        } catch (Exception e) {
	        	System.out.println("error");
			} finally { 
	            try { 
	                fos_jpg.close(); 
	            } catch (Exception e) {
	            	System.out.println("error2");
	            } 
	        }
	        request.setAttribute("pieChart", "pieChart.jpg");
			
		}

生成折线图

       if (method.equals("lineChart")) {
			XYSeriesCollection collection = new XYSeriesCollection();
			XYSeries series = new XYSeries("折线");
			
			Map<String, Integer> map2 = movieService.lineChart();
			int number = 99;
			for(int i=0; i<map2.size(); i++){
				String s= number+"";
				String score = s.charAt(0)+"."+s.charAt(1);
				series.add(Double.parseDouble(score),map2.get(score));
//				System.out.println(Double.parseDouble(score)+"--"+map2.get(score));
				number--;
			}
			collection.addSeries(series);
			
			JFreeChart chart = ChartFactory.createXYLineChart(
				        "评分折线图",
				        "评分",
				        "数量",				
				        collection,
				        PlotOrientation.VERTICAL,
				        true, 
				        true, 
				        false);
			
			XYPlot plot = (XYPlot) chart.getPlot(); 
			//设置曲线是否显示数据点
			XYLineAndShapeRenderer xylinerenderer = (XYLineAndShapeRenderer)plot.getRenderer();
			xylinerenderer.setBaseShapesVisible(true); 
			
			//设置曲线显示各数据点的值
			XYItemRenderer xyitem = plot.getRenderer(); 
			xyitem.setBaseItemLabelsVisible(true);
			xyitem.setBasePositiveItemLabelPosition(new ItemLabelPosition(ItemLabelAnchor.OUTSIDE12, TextAnchor.BASELINE_CENTER)); 
			xyitem.setBaseItemLabelGenerator(new StandardXYItemLabelGenerator());
			xyitem.setBaseItemLabelFont(new Font("Dialog", 1, 10)); 
			plot.setRenderer(xyitem);
			
			FileOutputStream fos_jpg = null; 
	        try { 
	        	//将图片保存至Tomcat服务器WebRoot目录下
	            fos_jpg = new FileOutputStream(request.getSession().getServletContext().getRealPath("/")+"lineChart.jpg");
	            ChartUtilities.writeChartAsJPEG(fos_jpg,1,chart,700,500,null); 
	        } catch (Exception e) {
	        	System.out.println("error");
			} finally { 
	            try { 
	                fos_jpg.close(); 
	            } catch (Exception e) {
	            	System.out.println("error2");
	            } 
	        }
	        request.setAttribute("lineChart", "lineChart.jpg");
		}

MovieDao.java

把数据插入到数据库

public class MovieDao {
	
	/**
	 * 把获取的数据，一次性插入
	 * @param listMovie
	 */
	public void save(List<Movie> listMovie){
		Connection connection = null;
		PreparedStatement statement = null;
		
		connection = JdbcUtils.getConnection();

		try {
			int i = 1;
			for(Movie movie : listMovie){
				System.out.println("正在插入第"+(i++)+"条数据到数据库ing...");
				String sql = " INSERT INTO movie(NAME,TYPES,release_date,score,movieUrl,is_playable) VALUE( ?,?,?,?,?,? ) ";
				
				statement = connection.prepareStatement(sql);
				
				statement.setString(1, movie.getName());
				statement.setString(2, movie.getTypes());
				statement.setString(3, movie.getRelease_date());
				statement.setFloat(4, movie.getScore());
				statement.setString(5, movie.getMovieUrl());
				statement.setString(6, movie.getIs_playable());
				
				statement.execute();
			}
			System.out.println("保存数据完成");
		} catch (SQLException e) {
			System.out.println("保存数据出现错误 MovieDao error");
			e.printStackTrace();
			throw new RuntimeException(e);
		} finally {
			try {
				connection.close();
				statement.close();
			} catch (SQLException e) {
				e.printStackTrace();
				throw new RuntimeException(e);
			}
		}
	}
	

 查询所有数据

   /**
	 * 查询所有数据
	 * @return
	 */
	public List<Movie> findAll(){
		
		Connection connection = null;
		PreparedStatement statement = null;
		ResultSet resultSet = null;
		
		try {
			connection = JdbcUtils.getConnection();
			
			String sql = " select * from movie ";
			
			statement = connection.prepareStatement(sql);
			
			resultSet = statement.executeQuery();

			List<Movie> list = new ArrayList<Movie>();
			while (resultSet.next()) {
				Movie movie = new Movie();
				
				movie.setId(resultSet.getInt("id"));
				movie.setName(resultSet.getString("name"));
				movie.setTypes(resultSet.getString("types"));
				movie.setRelease_date(resultSet.getString("release_date"));
				movie.setScore(resultSet.getFloat("score"));
				movie.setMovieUrl(resultSet.getString("movieUrl"));
				movie.setIs_playable(resultSet.getString("is_playable"));
				
				list.add(movie);
			}
			
			return list;
		} catch (SQLException e) {
			e.printStackTrace();
			throw new RuntimeException(e);
		} finally {
			try {
				connection.close();
				statement.close();
			} catch (SQLException e) {
				e.printStackTrace();
				throw new RuntimeException(e);
			}
		}
	}

获取不同分数等级的电影数量

   /**
	 * 统计不同分数级别的电影数量
	 * @return
	 */
	public Map<String,Integer> Count(){
		
		Connection conn = null;
		PreparedStatement stmt = null;
		ResultSet resultSet = null;
		
		Map<String,Integer> mapCount = new HashMap<String, Integer>();
		
		conn = JdbcUtils.getConnection();
		String sql = null;
		String key = null;
		int i=0;
		while (i<5) {
			switch (i) {
				case 0:
					sql = "SELECT COUNT(1) FROM movie WHERE score>=9 ";
					key = "one";
					break;
				case 1:
					sql = "SELECT COUNT(1) FROM movie WHERE score>=8.5 && score<9 ";
					key = "two";
					break;
				case 2:
					sql = "SELECT COUNT(1) FROM movie WHERE score>=8 && score<8.5 ";
					key = "three";
					break;
				case 3:
					sql = "SELECT COUNT(1) FROM movie WHERE score>=7.5 && score<8 ";
					key = "four";
					break;
				case 4:
					sql = "SELECT COUNT(1) FROM movie WHERE score<7.5 ";
					key = "five";
					break;
			}
			try {
				stmt = conn.prepareStatement(sql);
				resultSet = stmt.executeQuery();
				while (resultSet.next()) {
					mapCount.put(key, resultSet.getInt(1));
				}
			} catch (SQLException e) {
				e.printStackTrace();
			}
			i++;
		}
		return mapCount;
	}
	

获取每个电影评分的电影数量

   /**
	 * 统计每个分数对应的数量
	 * @return
	 */
	public Map<String,Integer> lineChart(){
		
		Connection conn = null;
		PreparedStatement stmt = null;
		ResultSet resultSet = null;
		
		Map<String,Integer> mapCount = new HashMap<String, Integer>();
		
		conn = JdbcUtils.getConnection();
		String sql = null;
		int number = 99;
		for( ; number>=70; number-=1){
			//获取9.9 9.1 7.4 .....
			String s = number+"";
			String score = s.charAt(0)+"."+s.charAt(1);
			sql = "SELECT COUNT(1) FROM movie WHERE score=" + score ;
			try {
				stmt = conn.prepareStatement(sql);
				resultSet = stmt.executeQuery();
				while (resultSet.next()) {
					mapCount.put(score, resultSet.getInt(1));
				}
			} catch (SQLException e) {
				e.printStackTrace();
			}
		}
		return mapCount;
	}
}

     两分钟抓取数据2000+并保存至数据库中，感觉还是挺慢的，有待优化代码

代码源码：

GitHub：https://github.com/YanKuan-IT/DouBanMoviesInfo_DB.git

注：如有什么做的不对的，请指教

转载于:https://my.oschina.net/u/3285916/blog/1501619