python-爬虫
# import urllib.request # import http.cookiejar # import pymysql # conn = pymysql.connect("localhost", "root", "123456", "test") # cursor = conn.cursor() # cursor.execute("DROP TABLE IF EXISTS employee") # sql = """CREATE TABLE employee(first_name CHAR(20) NOT NULL, # last_name CHAR(20), # age INT, # sex CHAR(1))""" # cursor.execute(sql) # sqlInsert = """INSERT INTO employee(first_name,last_name,age,sex) VALUES('李白','白居易',20,'男')""" # try: # cursor.execute(sqlInsert) # cursor.execute(sqlInsert) # conn.commit() # except: # conn.rollback() # conn.close() # 爬虫 # import requests # from bs4 import BeautifulSoup # import pymysql # # # 本地数据库 # sql_host = 'localhost' # # 数据库的用户名 # sql_user = 'root' # # 数据库密码 # sql_password = '123456' # # 数据的名 # sql_name = 'test' # SQL_INSERT = """INSERT INTO user_data(author,page,sex,age,vote,content) VALUES(%s,%s,%s,%s,%s,%s)""" # # def download_page(http_url): # headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0"} # call_back = requests.get(http_url, headers=headers) # return call_back.text # # # def get_page_content(html, page): # conn = pymysql.connect(sql_host, sql_user, sql_password, sql_name) # cursor = conn.cursor() # soup = BeautifulSoup(html, 'html.parser') # con = soup.find(id='content-left') # con_list = con.find_all('div', class_='article') # for item in con_list: # author = item.find('h2').string # content = item.find('div', class_='content').find('span').get_text() # stats = item.find('div', class_='stats') # vote = stats.find('span', class_='stats-vote').find('i', class_='number').get_text() # comments = stats.find('span', class_='stats-comments').find('i', class_='number').string # author_info = item.find('div', class_='articleGender') # if author_info is not None: # class_list = author_info['class'] # age = author_info.string # if 'womenIcon' in class_list: # sex = '女' # elif 'manIcon' in class_list: # sex = '男' # else: # sex = '' # else: # sex = '' # age = '' # # cursor.execute(SQL_INSERT, ("name","data","gg","sd","dd")) # cursor.execute(SQL_INSERT,(author,page,sex,age,vote,content)) # conn.commit() # # conn.close() # # def main(): # conn = pymysql.connect(sql_host, sql_user, sql_password, sql_name) # cursor = conn.cursor() # cursor.execute("""DELETE FROM user_data""") # conn.commit() # conn.close() # for i in range(1, 14): # http_url = 'https://qiushibaike.com/text/page/{}'.format(i) # html = download_page(http_url) # get_page_content(html,i) # # # if __name__ == '__main__': # main() import requests from bs4 import BeautifulSoup import pymysql # 网页格式Mozilla/5.0 (Windows NT 6.1; WOW64; rv:62.0) Gecko/20100101 Firefox/62.0 <div class="articleGender manIcon">20</div> def get_html(url): headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:62.0) Gecko/20100101 Firefox/62.0'} call_back = requests.get(url,headers = headers) return call_back.text def get_page_content(html,page): soup = BeautifulSoup(html,'html.parser') content = soup.find(id='content-left') content_list = content.find_all('div',class_='article') for list_item in content_list: author_div = list_item.find('div',class_='author') author = author_div.find('h2').string author_info = author_div.find('div',class_='articleGender') if author_info is not None: info_list = author_info['class'] age = author_info.string if 'manIcon' in info_list: sex = '男' elif 'womenIcon' in info_list: sex = '女' else: sex = '' else: age = '' sex = '' print(author,sex,age) if __name__ == '__main__': url = 'https://www.baidu.com/' html = get_html(url) get_page_content(html,1)
java -spring_boo-mysql
数据类
@Entity public class UserData { @Id private int id; private String author; private String page; private String sex; private String age; private String vote; private String content; } @Repository public interface User extends JpaRepository<UserData,String> { }
接口控制器
@RestController @RequestMapping(value = "/name") public class Data { @Autowired private User user; @RequestMapping(value = "/data") public List<UserData> name() { return user.findAll(); } }