1import urllib
  2#import urllib2
  3import urllib.request
  4import re
  5#from urllib import urlopen
  6from urllib.request import urlopen
  7class Tool:
  8     #去除img标签,7位长空格
  9    removeImg = re.compile('<img.*?>| {7}|')
 10    #删除超链接标签
 11    removeAddr = re.compile('<a.*?>|</a>')
 12    #把换行的标签换为\n
 13    replaceLine = re.compile('<tr>|<div>|</div>|</p>')
 14    #将表格制表<td>替换为\t
 15    replaceTD= re.compile('<td>')
 16    #把段落开头换为\n加空两格
 17    replacePara = re.compile('<p.*?>')
 18    #将换行符或双换行符替换为\n
 19    replaceBR = re.compile('<br><br>|<br>')
 20    #将其余标签剔除
 21    removeExtraTag = re.compile('<.*?>')
 22    def replace(self,x):
 23        x = re.sub(self.removeImg,"",x)
 24        x = re.sub(self.removeAddr,"",x)
 25        x = re.sub(self.replaceLine,"\n",x)
 26        x = re.sub(self.replaceTD,"\t",x)
 27        x = re.sub(self.replacePara,"\n    ",x)
 28        x = re.sub(self.replaceBR,"\n",x)
 29        x = re.sub(self.removeExtraTag,"",x)
 30        #strip()将前后多余内容删除
 31        return x.strip()
 32 
 33class XLBK:
 34    def __init__(self,baseUrl,articleTag,fileName):
 35        self.baseURL=baseUrl
 36        self.tool=Tool()
 37        self.file=None
 38        self.article=1
 39        self.defaultTitle=u'新浪博客'
 40        self.articleTag=articleTag
 41        self.fileName=fileName
 42 
 43    def getPage(self,pageNum):
 44        try:
 45            url=self.baseURL+str(pageNum)+'.html'
 46            print ('即将抓取的网页：'+url)
 47            #request= urllib2.Request(url)
 48            request= urllib.request.Request(url)
 49            #response=urllib2.urlopen(request)
 50            response=urllib.request.urlopen(request)
 51            return response.read().decode('utf-8')
 52 
 53        #except urllib2.URLError ,e:
 54        except urllib.error.URLError as e:
 55            if hasattr(e,"reason"):
 56                #print u"连接新浪博客失败,错误原因",e.reason
 57                print ("连接新浪博客失败,错误原因",e.reason)
 58 
 59                return None
 60    def getTitle(self,page):
 61        pattern = re.compile('blogname.*?blognamespan.*?>(.*?)</span>', re.S)
 62        result = re.search(pattern,page)
 63        print ("title"+result.group(1).strip())
 64        if result:
 65            return result.group(1).strip()
 66        else:
 67            return None
 68 
 69    def getPageNum(self,page):
 70        #pattern= re.compile(ur'<span style.*?>共(.*?)页</span>',re.S)
 71        pattern= re.compile(u'<span style.*?>共(.*?)页</span>',re.S)
 72        result = re.search(pattern,page)
 73        if result:
 74            #print "pagenum"+result.group(1).strip()
 75            return result.group(1).strip()
 76        else:
 77            print (result)
 78            return 1
 79 
 80    def getContent(self,page):
 81        pattern = re.compile('<span class="atc_title">.*?<a.*?href.*?.html">(.*?)</a>.*?</span>',re.S)
 82        items = re.findall(pattern,page)
 83        contents = []
 84        for item in items:
 85            content = "\n"+self.tool.replace(item)+"\n"
 86            contents.append(content.encode('utf-8'))
 87            #print (content)# 返回博客标题了
 88        return contents
 89 
 90    def getUrl(self,page):
 91        pattern =re.compile('<span class="atc_title">.*?<a.*?href="(.*?)">.*?</a>.*?</span>',re.S)
 92        items = re.findall(pattern,page)
 93        urls = []
 94        for item in items:
 95            url = item
 96            #urls.append(url.encode('utf-8')) #会导致首部多出字符 b'
 97            urls.append(url)
 98            
 99            #print (url) #返回博客url
100        return urls
101 
102 
103    def getText(self,url):
104         text=urlopen(url).read().decode('utf-8')
105         start=text.find(u"<!-- 正文开始 -->")
106         print (start)
107         end=text.find(u"<!-- 正文结束 -->")
108         print (end)
109         text=text[start:end]
110         text = re.sub(re.compile('<p.*?>'),"\n    ",text)
111         text = re.sub(re.compile('<p>'),"\n    ",text)
112         text=re.sub(r'<(S*?)[^>]*>.*?|<.*? /> ','',text)
113         text=re.sub(r'&[^>]*?\;',' ',text)
114         #return text.encode('utf-8') #编码混乱
115         return text
116 
117    def setFileTitle(self,title):
118        if title is not None:
119            self.file = open(title + ".doc","w")
120        else:
121            self.file = open(self.defaultTitle + ".doc","w")
122 
123 
124    def writeData_original(self,contents,urls):
125        for item in contents:
126            #print('item=='+str(item))
127            if self.articleTag == '1':
128 
129                articleLine = "\n" + str(self.article) + u"--------------------------------------------------------------------------------\n"
130                self.file.write(articleLine)
131            #self.file.write(item) #TypeError: write() argument must be str, not bytes
132            self.file.write(str(item))
133            #print item
134            #self.file.write(urls[contents.index(item)]) #TypeError: write() argument must be str, not bytes
135            self.file.write(urls[contents.index(str(item))])
136            #print urls[contents.index(item)]
137            text=self.getText(urls[contents.index(item)])   
138            print (text)
139            self.file.write(str(text))
140            self.article += 1
141 
142    def writeData(self,contents,urls):
143        print(urls)
144        for item in urls:
145            #item ='http://blog.sina.com.cn/s/blog_67493dad0102uxwl.html'
146            print('item=='+str(item))
147            if self.articleTag == '1':
148 
149                articleLine = "\n" + str(self.article) + u"--------------------------------------------------------------------------------\n"
150                self.file.write(articleLine)
151            #self.file.write(item) #TypeError: write() argument must be str, not bytes
152            self.file.write(str(item))
153            #print item
154            #self.file.write(urls[contents.index(item)]) #TypeError: write() argument must be str, not bytes
155            #self.file.write(urls[contents.index(str(item))])
156            item=str(item)
157            item=item.replace("b'", "");
158            #self.file.write(contents[urls.index(item)])
159 
160            #print urls[contents.index(item)]
161            #text=self.getText(urls[contents.index(item)]) 
162            text=self.getText(item)
163            #text=self.getText('http://blog.sina.com.cn/s/blog_67493dad0102uxwl.html')
164            
165 
166            print (text)
167            self.file.write(str(text))
168            self.article += 1 
169 
170    def start(self):
171        indexPage = self.getPage(1)
172        pageNum = self.getPageNum(indexPage)
173        title = self.getTitle(indexPage)
174        self.setFileTitle(self.fileName)
175        if pageNum == None:
176            print ("URL已失效，请重试")
177            return
178        try:
179            print ("该博客共有" + str(pageNum) + "页")
180            for i in range(1,int(pageNum)+1):
181                print ("正在写入第" + str(i) + "页数据")
182                page = self.getPage(i)
183                contents = self.getContent(page)
184                urls =self.getUrl(page)
185                self.writeData(contents,urls)
186        #except IOError,e:
187        except IOError as e:
188            print ("写入异常，原因" + e.message)
189        finally:
190            print ("写入任务完成")
191 
192 
193 
194#print u"打开一个新浪博客的博文目录\n如http://blog.sina.com.cn/s/articlelist_1866629225_0_1.html \n那么该博客的代号为1866629225_0_   \n请输入博客代号"
195#baseURL = 'http://blog.sina.com.cn/s/articlelist_' + str(raw_input(""))
196#baseURL = 'http://blog.sina.com.cn/s/articlelist_' + str(input(""))
197#
198#假设本例要测试的新浪博客全部文章目录地址为： http://blog.sina.com.cn/s/articlelist_1732853165_0_1.html
199baseURL =  'http://blog.sina.com.cn/s/articlelist_1732853165_0_' #这里是直接赋值的形式
200#articleTag = raw_input("是否写入文章编号信息，是输入1，否输入0\n")
201articleTag = input("是否写入文章编号信息，是输入1，否输入0\n") # 运行后输入 1
202#fileName=raw_input("请输入保存文档的名称\n")
203fileName=input("请输入保存文档的名称\n") #文件名可以随便输入-------也可以直接赋值  fileName=sina_blog_save
204xlbk = XLBK(baseURL,articleTag,fileName)
205xlbk.start()