|
# -*- coding:utf-8 -*-
import requests
from bs4 import BeautifulSoup
if __name__ == '__main__':
url = 'https://www.shicimingju.com/book/sanguoyanyi.html'
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
}
page_text = requests.get(url=url,headers=headers).text
soup = BeautifulSoup(page_text,'lxml')
li_list = soup.select('.book-mulu li')
fp = open('./sanguo.txt','w',encoding='UTF-8')
for li in li_list:
title = li.a.string
detail_url = 'https://www.shicimingju.com/'+li.a['href']
detail_page_text = requests.get(url=detail_url,headers=headers).text
detail_soup = BeautifulSoup(detail_page_text,'lxml')
div_tag = detail_soup.find('div',class_='chapter_content')
content = div_tag.text
fp.write(title+':'+content+'\n')
print(title,'爬取成功')
就这么一段代码 , WIN10系统 爬完就是乱码 搜索上的回答都没有用呢
|
|