python能爬国外的网站吗_Python爬取国外天气预报网站的方法

本文实例讲述了Python爬取国外天气预报网站的方法。分享给大家供大家参考。具体如下：

crawl_weather.py如下：

#encoding=utf-8

import httplib

import urllib2

import time

from threading import Thread

import threading

from Queue import Queue

from time import sleep

import re

import copy

lang = "fr"

count = 0

class Location:

# Location(False, "中国", "北京", "zh")

# Location(True, "", "亚洲", "zh")

def __init__(self, is_beyond_country, country_name, loc_name, lang):

self.country_name = country_name

self.loc_name = loc_name

self.lang = lang

self.is_beyond_country = is_beyond_country

prn_lock = threading.RLock()

def GetLocationURLs(url, recursive):

global count

if url.find("weather-forecast") != -1:

count = count + 1

if count % 500 == 0:

prn_lock.acquire()

print "count:%d" % (count)

prn_lock.release()

return [url]

page = urllib2.urlopen(url).read()

time.sleep(0.01)

#"

Africa

"

pattern = "

(.*)

"

locs = re.findall(pattern, page)

locs = [(url, name) for url, name in locs if url.find("browse-locations") != -1 or url.find("weather-forecast") != -1]

if not recursive:

urls = [url for url, name in locs]

return urls

urls =