1 importurllib.request2 from bs4 importBeautifulSoup3 importre4 importtime5 import_thread6
7 #Input your Url here####################################
8 BaseURL = '127.0.0.1/'
9 #########################################################
10 TaxURL = ".html"
11
12 #Input your data-saving path ############################
13 SavePath = ""
14 #########################################################
15
16 #Input your threads count ###############################
17 thread_count = 1
18 #########################################################
19
20 #Set each spider will spy how many pages ################
21 thread_spy_count_ench = 5
22 #########################################################
23 defmkdir(path):24 #Create the directory
25 importos26 path=path.strip()27 path=path.rstrip("\\")28 isExists=os.path.exists(path)29 if notisExists:30 os.makedirs(path)31 returnTrue32 else:33 returnFalse34
35 defdownload(start, count):36 #Spider main
37 for i in range(start,start +count):38 try:39 #DEBUG##################################################
40 #print("[INFO] Connecting to page #" + str(i) + "...")
41 ########################################################
42
43 #Used to record time
44 time_start=time.time()45
46 #Construct url
47 #This only work like
48 #https://127.0.0.1/articles/00001.html
49 #https://127.0.0.1/articles/00002.html
50 #https://127.0.0.1/articles/00003.html
51 TargetURL = BaseURL + str(i) +TaxURL52
53
54 #create Request object
55 req =urllib.request.Request(TargetURL)56 #create headers using general header, you could find this by Fiddler(R) or by Chrome(R)
57 req.add_header('Host','') #Your Host, usally set as url-base
58 req.add_header('Referer',TargetURL) #Your Referer, usally set as url
59 req.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19')60 #finishing create Request object
61
62 #get information
63 res =urllib.request.urlopen(req)64 #decode the html
65 soup = BeautifulSoup(res,"html.parser")66 ##############################################################
67 #add your functions here....
68
69 #operate_data(data)
70
71 #soup find div to get inforamtion.
72 #soup is able to opreate html tag very easily, by using soup.find(...)
73 ##############################################################
74
75 #Change saving path here.
76 savetarget =SavePath77
78 #trying to saving files
79 try:80 #create directory if it doesn't existed
81 mkdir(SavePath+"\\"+str(zone)+"\\")82 #using open...
83 f = open(savetarget,'w')84
85 #edit this
86 f.write("data")87
88 exceptException as e:89 time_end=time.time()90 print("[Failed] - #" + str(i) + "Error :" +str(e))91 else:92
93 time_end=time.time()94 print("[Succeed] - #" + str(i) + "has saved to path.("+str(time_end-time_start)+"s)")95
96 pass
97 exceptException as e:98 print("[Global Failure] - #" + str(i) + "Error :" +str(e))99 pass
100
101
102
103 #if __name__ == __main__:
104 try:105 #Multithreading
106 print("Spidering webiste...")107 print("Current configuration :")108 print("--Will create" + str(thread_count) + "threads to access.")109 print("--Will save to" +SavePath)110 print("-------------START---------------------------")111 #press any key to continue
112 #this won't work under linux
113 importos114 os.system('pause')115 try:116 for i inrange(0,thread_count):117 print("[Thread #"+ str (i) +"] started successfully")118 _thread.start_new_thread(download, (thread_spy_count_ench *i,thread_spy_count_ench))119 exceptException as e:120 print("[Threading@" + str(i) +"] Error:"+str(e))121 exceptException as e:122 print("[Global Failure] Error:"+str(e))123 while 1:124 pass