文本密度算法提取网页正文

import re 
import requests


reBODY = r'<body.*?>([\s\S]*?)<\/body>' 
reCOMM = r'<!--.*?-->' 
reTRIM = r'<{0}.*?>([\s\S]*?)<\/{0}>' 
reTAG  = r'<[\s\S]*?>|[ \t\r\f\v]' 
reIMG  = re.compile(r'<img[\s\S]*?src=[\'|"]([\s\S]*?)[\'|"][\s\S]*?>')




class Extractor(object): 
    """
    根据文本密度提取正文区
    """
    def __init__(self, html='', blockSize=3, saveImage=False): 
        self.blockSize = blockSize
        self.saveImage = saveImage
        self.html = html
        self.ctexts = []
        self.cblocks = []


    def __processTags(self): 
        """
        处理html标签
        """
        self.body = re.sub(reCOMM, '', self.body)
        self.body = re.sub(reTRIM.format('script'), '', re.sub(reTRIM.format('style'), '', self.body))
        self.body = re.sub(reTRIM.format('SCRIPT'), '', re.sub(reTRIM.format('STYLE'), '', self.body))
        self.body = re.sub(reTAG, '', self.body)


    def __processBlocks(self): 
        """
        处理body区块, 获取文本密度最大的区块
        """
        self.ctexts   = self.body.split('\n')
        self.textLens = [len(text) for text in self.ctexts]
        self.cblocks  = [0]*(len(self.ctexts) - self.blockSize - 1)
        lines = len(self.ctexts)
        for i in range(self.blockSize):
            self.cblocks = list(map(lambda x,y: x+y, self.textLens[i : lines-1-self.blockSize+i], self.cblocks))
        maxTextLen = max(self.cblocks)
        self.start = self.end = self.cblocks.index(maxTextLen)
        while self.start > 0 and self.cblocks[self.start] > min(self.textLens):
            self.start -= 1
        while self.end < lines - self.blockSize and self.cblocks[self.end] > min(self.textLens):
            self.end += 1
        return ''.join(self.ctexts[self.start:self.end])


    def __processImages(self): 
        """
        处理图片
        """
        self.body = reIMG.sub(r'{{\1}}', self.body)


    def getContent(self): 
        """
        获取纯文本(不带html标签)
        """
        body = re.findall(reBODY, self.html)
        self.body = ''
        if body:
            self.body = body[0]
        if self.saveImage:
            self.__processImages() 
        self.__processTags()
        return self.__processBlocks()

if __name__ == '__main__': 
    html = requests.get('https://dy.163.com/article/FFPLSI8P05439A3X.html').text 
    ex = Extractor(html=html, blockSize=3, saveImage=False) 
    content = ex.getContent() 
    print(content)

　　中国式的父母大多是不善于表达的　　可能从来没听过他们嘴里说过“我爱你”　　却从小到大用行动偷偷爱了你一辈子　
 ......
  马宗谦说：“我小的时候我母亲为了照顾我付出了很大的心血，现在我母亲身体不行了，有病了，我就是再照顾60年也报答不了她的恩情。”　　　　即使看不见这个世界，但我能看见你爱我。　　28　　　　告别总有最后一次，希望不是每次都满含愧疚。父母在，人生尚有来处，父母去，人生只剩归途。　　　　在一起的时候，当时只道是寻常　　寻常到不懂得怎么去珍惜　　转眼他们都老了，他们也走了　　时光却不能倒流.....　　树欲静而风不止,子欲养而亲不待　　且行且珍惜！

if __name__ == '__main__': 
    html = requests.get('https://sports.sina.com.cn/basketball/nba/2020-06-22/doc-iirczymk8281587.shtml').content 
    html = str(html, 'utf-8') 
    ex = Extractor(html=html, blockSize=3, saveImage=False) 
    content = ex.getContent() 
    print(content)