简单的网络爬虫(获取页面中的url)
package text;
import java.io.BufferedReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class SpiderNet {
public static void main(String[] args) {
URL url = null;
URLConnection urlConnection = null;
PrintWriter out = null;
BufferedReader in = null;
//编写正则表达式
String regex = "http://[\\w+\\.?/?]+\\.[A-Za-z]+";
//定义正则表达式
Pattern pattern = Pattern.compile(regex);
try {
//访问链接,这里选择steam官网主页
url = new URL("http://store.steampowered.com/");
//建立url的连接connection
urlConnection = url.openConnection();
//创建新 PrintWriter,路径D:/mzsds_mzm.txt
out = new PrintWriter(new FileWriter("D:/mzsds_mzm.txt"),true);
//将url地址的响应读到in中
in = new BufferedReader(new InputStreamReader(urlConnection.getInputStream()));
String buffer = null;
Set<String> set = new HashSet<String>();
//读响应数据到.txt文件中
while((buffer=in.readLine())!=null){
//匹配正则表达式
Matcher matcher = pattern.matcher(buffer);
if(matcher.find()){
//输出匹配内容
set.add(matcher.group());
}
}
for (String str : set) {
out.println(str);
}
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
}
结果输出:
http://cdn.steamstatic.com.8686c.com/steam/apps/668210/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/346110/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/672730/capsule_184x69.jpg
http://www.valvesoftware.com/jobs.html
http://translation.steampowered.com
http://www.steampowered.com
http://www.valvesoftware.com
http://store.akamai.steamstatic.com/public/css/v6/home.css
http://www.w3.org
http://store.akamai.steamstatic.com
http://store.akamai.steamstatic.com/public/shared/css/motiva_sans.css
http://cdn.steamstatic.com.8686c.com/steam/apps/434460/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/566160/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/images/responsive/header_logo.png
http://www.valvesoftware.com/about.html
http://store.akamai.steamstatic.com/public/css/v6/store.css
http://store.steampowered.com
http://cdn.steamstatic.com.8686c.com/steam/apps/686190/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/668630/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/spotlights/68ca004cca70f207cddabbb4/spotlight_image_english.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/465240/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/javascript/cluster.js
http://store.akamai.steamstatic.com/public/shared/images/header/globalheader_logo.png
http://twitter.com
http://store.akamai.steamstatic.com/public/javascript/home.js
http://cdn.steamstatic.com.8686c.com/steam/apps/619880/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/641990/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/css/buttons.css
http://cdn.steamstatic.com.8686c.com/steam/apps/680320/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/css/shared_responsive.css
http://store.akamai.steamstatic.com/public/shared/css/shared_global.css
http://cdn.steamstatic.com.8686c.com/steam/apps/515220/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/544610/capsule_184x69.jpg
http://steamcommunity.com
http://store.akamai.steamstatic.com/public/javascript/main.js
http://store.akamai.steamstatic.com/public/shared/javascript/shared_global.js
http://store.akamai.steamstatic.com/public/images/v5/globalheader_logo.png
http://store.akamai.steamstatic.com/public/shared/javascript/shared_responsive_adapter.js
http://cdn.steamstatic.com.8686c.com/steam/apps/555150/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/javascript/tooltip.js
http://cdn.steamstatic.com.8686c.com/steam/apps/577800/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/650780/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/images/responsive/header_menu_hamburger.png
http://cdn.steamstatic.com.8686c.com/steam/apps/629960/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/689090/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/373930/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/551770/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/559910/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/614570/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/635320/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/435150/capsule_184x69.jpg
http://www.facebook.com
http://cdn.steamstatic.com.8686c.com/steam/apps/554620/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/images/responsive/logo_valve_footer.png
http://cdn.steamstatic.com.8686c.com/steam/apps/607880/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/javascript/dynamicstore.js
http://store.akamai.steamstatic.com/public/images/blank.gif
http://cdn.steamstatic.com.8686c.com/steam/apps/449840/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/spotlights/7830c4a8f396478f946d265b/spotlight_image_english.jpg
http://store.akamai.steamstatic.com/public/shared/images/trans.gif
http://store.akamai.steamstatic.com/public/shared/images/responsive/share_steam_logo.png
http://www.valvesoftware.com/legal.htm
http://cdn.steamstatic.com.8686c.com/steam/apps/357310/capsule_184x69.jpg