用python中htmlParser实现的spider(python spider)

最近公司网站搞检查，发现了一些问题，一直在用的是xenu工具，速度快，小巧（写了这么多年的MFC,真的很亲切啊，呵呵）

刚好这2天才学习python，所以自己写了一个spider，逐渐也对python有了一些了解，下面把源码分享出来，大家可以玩玩看

文件是utf-8格式，但如果加了中文注释，竟然不能debug了。。。无论你是不是在文件头2行加了#encoding=utf-8等方法，有人解决了告诉我一下哦

用python实现的网站spider 1.功能：检查死链和空title，链接目前只查属于站内的（包括主站和二级站,并且会过滤重复的URL，防止一个URL多次扫描） 2.当页面中有一些错误时，也会提供出错原因和具体位置,所以也可以检查一下网站是否标准 3.目前只查页面，不查资源，过滤了 .jpg|.jpeg|.avi|.png|.gif 4.依赖：必须安装python-3.2rc1（将来也有可能是正式版，可以自己改test.bat中的python的路径，不过不能装低于这个版本的，3.1版本loggingRotating方式有bug） 5.有些不能查出的死链：有时候死链会被重定向到别的页面，如果中国电信会直接跳到它的页面，此情况暂时无法处理，在公司测试没有此现象，adsl上网会这样 6.检查网页是否有语法错误，spider.py中 except HTMLParseError as e: # log.error("HTMLParseError : msg= %s, url= %s, pos= %s" % ( e.msg, url, lParser.getpos()))pass （前提是你已经了解了python的缩进语法）去掉第2行首的“#”即可，目前由于不少网页都有好些小问题，所以默认不开放 7.这是方便检查公司网站的小工具，在公司的几个项目网站上测试可用，不过没有在其他大型网站做过性能和兼容性测试，如果你有兴趣，源码都在里面，自己动手玩玩吧 8.如果有bug，请反馈给我，也可以自己debug找一下(我的开发环境eclipse + pydev) 出错说明： 1.URL出错，一般是死链接 [2011-02-09 11:12:11,392] ERROR : URLError : http://Hardware.global-supplier.com [2011-02-09 15:37:57,148] ERROR : HTTPError : code=400, msg= Bad Request, url= http://www.fashionalgifts.com/../company_info.html 2.页面解析时出错，下例是tag中内容有问题，pos代表页面的出错位置 [2011-02-09 11:12:17,119] ERROR : msg= malformed start tag, HTMLParseError : http://fashion.global-supplier.com/cosmetic-bags/pu-cosmetic-bag-gb-c09.html pos= (238, 27) 3.urlJoinError，一般是相对路径使用错误导致的，如下，第一个是页面URL，第二个是href中的内容，一级子目录竟然要返回上2级的目录 UrlJoin Error:http://www.fashionalgifts.com/company-culture.html/, ../../company_info.html 不过这样的URL在浏览器中是可以正常执行的。。。太强悍了，帮我们屏蔽错误 4.空title title is null, url=*** 

spider.py(主程序)

#Author:Paul Wang #Date:2011-01-26 #Description: from html.parser import HTMLParser from html.parser import HTMLParseError import urllib.request import myLogger import myHtmlParse import sys debugMode = 1 site = "http://www.geekzones.com/" charset = 'utf-8' logLever = 'debuG' try:def useage():print(" python spider.py siteUrl charSet logLever")print(" example: python spider.py http://www.baidu.com utf-8 debug")def getInnerURL(url,parentUrl,siteName,log):global timesglobal urlListlParser = myHtmlParse.MyParser(log)try:try:if(url[-1] != '/'):url += '/'if url in urlList or len(url) == 0:return times += 1urlList.append(url)log.info('times = %d open url : %s' % (times , url))lParser.currentLink = url;lParser.siteName = siteName;opener = urllib.request.build_opener()opener.addheaders = [('User-agent', 'Mozilla/5.0')]htmlSource2 = opener.open(url).read()# req = urllib.request.Request(url)# htmlSource2 = urllib.request.urlopen(req).read()lParser.feed(htmlSource2.decode(charset))lParser.filterRepeatLink()if(len(lParser.title) == 0):log.error('title is null, url= %s' % url)except urllib.error.HTTPError as e:log.error("HTTPError : code=%d, msg= %s, url= %s, parentUrl=%s" % ( e.code, e.msg, url, parentUrl))except urllib.error.URLError as e:log.error("URLError : url= %s" % ( url))except HTMLParseError as e:log.error("HTMLParseError : msg= %s, url= %s, pos= %s, parentUrl=%s" % ( e.msg, url, lParser.getpos(), parentUrl))finally:lParser.close()for url1 in lParser.link: if url1 in urlList or len(url1) == 0:continue getInnerURL(url1, url, lParser.siteName,log)log = myLogger.myLogger('logging.config')if len(sys.argv) >= 3:site = sys.argv[1]charset = sys.argv[2]log.setLever(sys.argv[3])else:useage()if debugMode != 1:sys.exit()if debugMode == 1: log.setLever("debUG")urlList = urllib.parse.urlparse(site)host = urlList.scheme + '://' + urlList.hostnamesiteName = host[ host.find('.') +1: ].rstrip('/')print(siteName) # sys.exit()times = 0;urlList = []getInnerURL(site, '/',siteName,log)except KeyboardInterrupt:sys.exit()

mylogger.py

#Author:Paul Wang #Date:2011-01-26 #Description:logging wrapper import logging.config class myLogger:LEVELS = {'debug': logging.DEBUG,'info': logging.INFO,'warning': logging.WARNING,'error': logging.ERROR,'critical': logging.CRITICAL}def __init__(self, configName):print('init logger...')logging.config.fileConfig(configName)self.logger = logging.getLogger("simpleExample")def setLever(self, leverName):if (len(leverName)==0 ):self.logger.setLevel(logging.NOTSET)else:self.logger.setLevel(self.LEVELS.get(leverName.lower(), logging.NOTSET))print('set lever %s...' % logging.getLevelName(self.logger.level) )def writeLog(self, msg):self.logger.log(self.logger.level, msg)def debug(self, msg):self.logger.debug(msg)def info(self, msg):self.logger.info(msg)def warn(self, msg):self.logger.warn(msg)def error(self, msg):self.logger.error(msg)def critical(self, msg):self.logger.critical(msg)def __del__(self):print('destructor logger...') 

myHtmlParse.py

#Author:Paul Wang #Date:2011-01-26 from html.parser import HTMLParser import urllib import re import myLogger class MyParser(HTMLParser):def __init__ (self,log):HTMLParser. __init__ (self)self.currentLink = ''self.link = []self.exlink = []self.host = ''self.siteName = '' #like 'baidu.com'self.title = ''self.titleFlag = 0self.log = logdef handle_starttag(self, tag, attrs): if tag == 'a' :if len(attrs)==0:passfor name, value in attrs:if name == 'href' :pattern = re.compile('(.jpg|.jpeg|.avi|.png|.gif)</p>,re.I)match = pattern.search(value)if match:continueif len(value) and value[0] != '#' and value.find('@') == -1:if value.find(self.siteName) != -1 /or value[0] == '/' /or value[0:2] == './' /or value[0:3] == '../':if(self.currentLink[-1] != '/'):self.currentLink += '/'url = urllib.request.urljoin(self.currentLink, value)if(url.find("..") != -1 ):self.log.error("UrlJoin Error:%s, %s" % (self.currentLink, value))self.link.append( url)else:self.exlink.append(value.rstrip('/'))elif tag == 'title':if len(attrs)==0:passself.titleFlag = 1def handle_data(self, data):if self.titleFlag == 1 and len(self.title)== 0:self.title=dataself.titleFlag = 0def filterRepeatLink(self): # print('count1: %d' % len(self.link))dict = {}for keys in self.link:dict.setdefault(keys, '111')# print('count2: %d' % len(dict))self.link = dict.keys()# print('count2: %d' % len(self.link)) 

logging.config(配置档，用过log4c等的一看就明白了)

[loggers] keys=root,example [handlers] keys=consoleHandler,timedRotatingFileHandler [formatters] keys=simpleFormatter [formatter_simpleFormatter] #format=[%(asctime)s]%(name)s : %(message)s format=[%(asctime)s] %(levelname)s : %(message)s [logger_root] level=DEBUG handlers=consoleHandler,timedRotatingFileHandler [logger_example] level=DEBUG handlers=consoleHandler,timedRotatingFileHandler qualname=example propagate=0 [handler_consoleHandler] class=StreamHandler level=DEBUG formatter=simpleFormatter args=(sys.stdout,) [handler_rotateFileHandler] class=handlers.RotatingFileHandler level=DEBUG formatter=simpleFormatter args=('test.log', 'a', 1024*1024, 9) [handler_timedRotatingFileHandler] class=handlers.TimedRotatingFileHandler level=ERROR formatter=simpleFormatter args=('app.log','d')