python2.7.6 和BeautifulSoup4.4.1 抓取糗事百科，带评论

1. __author__ = 'KS'
   
2. # -*- coding: utf-8 -*-
   
3. 
   
4. import urllib2
   
5. import re
   
6. from bs4 import BeautifulSoup
   
7. 
   
8. 
   
9. def getContentOrComment(urlJoin):
   
10.     user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
    
11.     headers = {'User-Agent': user_agent}
    
12.     req = urllib2.Request(url=urlJoin, headers=headers)
    
13.     response = urllib2.urlopen(req)
    
14.     content = response.read().decode('utf-8','ignore')
    
15.     return content
    
16. 
    
17. 
    
18. # 文章地址
    
19. articleUrl = "http://www.qiushibaike.com/textnew/page/%d"
    
20. # 评论地址
    
21. commentUrl = "http://www.qiushibaike.com/article/%s"
    
22. page = 0
    
23. while True:
    
24.     getFromCustomer = raw_input("next page ? print Enter key to continue or 'exit' to stop\n")
    
25.     if getFromCustomer == "exit":
    
26.         break
    
27.     page += 1
    
28.     urlJoin = articleUrl % page
    
29.     print urlJoin
    
30.     articlePage = getContentOrComment(urlJoin)
    
31.     soupArticle = BeautifulSoup(articlePage, 'html.parser')
    
32.     # print soup.prettify()
    
33.     articleFloor = 1
    
34.     for string in soupArticle.find_all(attrs="article block untagged mb15"):
    
35.         commentId = str(string.get('id')).strip()[11:]
    
36.         print "\n"
    
37.         print articleFloor, ".", string.find(attrs="content").get_text().strip()
    
38.         articleFloor += 1
    
39.         commentPage = getContentOrComment(commentUrl % commentId)
    
40.         soupComment = BeautifulSoup(commentPage, 'html.parser')
    
41.         commentFloor = 1
    
42.         for comment in soupComment.find_all(attrs="body"):
    
43.             print "     ", commentFloor, "楼回复:", comment.get_text().strip()
    
44.             commentFloor += 1
    

复制代码

学习了，谢谢楼主

学习了.{:8_204:}