爬虫抓取网页中的url

忙于考试没有写博客，现在总结一下之前做的一个爬虫。实现了抓取网页中的所有指向网页的url，当然可以自己定制想抓取的url格式。

class Spider:
	def __init__(self,turl):		
		self.url=turl
		self.response = urllib2.urlopen(self.url,timeout=5)
		self.html = self.response.read()
		self.outdegree = []
		reg = r'(http://.*?/)'
		select = re.compile(reg)
		self.domain = select.findall(self.url)[0][:-1]
		l = len(self.domain)
		t = list(self.domain)
		if self.domain[l-1]=='.':
			self.domain[l-1]='/'
		self.domain = "".join(self.domain)
	def parse_url(self):
		soup = BeautifulSoup(self.html,'html.parser')
		self.i = 0
		for link in soup.find_all('a'):
				tdomain = None
				string =  link.get('href')
				if string == None or string == '':
					string = '/'
				string = string.replace('\\','/')
				if string[0] == '#':
					string = '/'
				if string[0] == '.' or '/' not in string:
					t = os.path.split(self.url)
					tdomain = t[0]
					t = os.path.split(string)
					string = t[1]
				if 'http' in string:
					pass
				else:
					if tdomain:
						string = tdomain + '/' + string
					elif string[0]!= '/':
						string = self.domain+'/'+string
					else:
						string = self.domain+string
				self.outdegree.append(string)
				self.i+=1
				yield string