利用selenium+PhantomJS爬取ajax网页

from selenium import webdriver
driver = webdriver.PhantomJS() #调试时用 driver=webdirver.Chrome()
driver.get(url)

driver.find_element_by_id('companyDetail').click()

# -*- coding: utf-8 -*-
from selenium import webdriver

driver = webdriver.Firefox()
driver.implicitly_wait(30)  # 隐性等待，最长等30秒
driver.get('https://huilansame.github.io')

print driver.current_url
driver.quit()

# -*- coding: utf-8 -*-
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

driver = webdriver.Firefox()
driver.implicitly_wait(10)  # 隐性等待和显性等待可以同时用，但要注意：等待的最长时间取两者之中的大者
driver.get('https://huilansame.github.io')
locator = (By.LINK_TEXT, 'CSDN')

try:
    WebDriverWait(driver, 20, 0.5).until(EC.presence_of_element_located(locator))
    print driver.find_element_by_link_text('CSDN').get_attribute('href')
finally:
    driver.close()

#coding=UTF-8
import urllib2
from bs4 import BeautifulSoup
from selenium import webdriver
from time import sleep
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import MySQLdb
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

db = MySQLdb.connect('127.0.0.1','root','********','****')
cursor =  db.cursor()
db.set_character_set('utf8')
cursor.execute('SET NAMES utf8;')
cursor.execute('SET CHARACTER SET utf8;')
cursor.execute('SET character_set_connection=utf8;')
sentense1 = 'INSERT INTO survey VALUES ("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")'
sentense2 = 'INSERT INTO shareholder VALUES ("%s","%s","%s","%s","%s")'
sentense3 = 'INSERT INTO finance VALUES ("%s","%s","%s","%s","%s","%s","%s","%s","%s","%s","%s")'
driver = webdriver.PhantomJS()
url = 'http://www.neeq.com.cn/nq/detailcompany.html?companyCode='
try_count = 0

count = 430002

while count<870700:
	driver.get(url+str(count))
	company_id = 0
	locator = (By.XPATH,"//li[@id='companyDetail']")
	try:
		WebDriverWait(driver,3).until(EC.presence_of_element_located(locator))
	except Exception as e:
		print count,'not found part1.'
		count+=1
		continue
	driver.find_element_by_id('companyDetail').click()
	locator = (By.XPATH,"//div[@class='list_l' and @style='display:block;']")
	try:
		WebDriverWait(driver,3).until(EC.presence_of_element_located(locator))
	except Exception as e:
		print count,'not found part1.'
		count+=1
		continue
	soup = BeautifulSoup(driver.page_source,'html.parser')
	table = soup.find('div',class_='list_l',style='display:block;')
	if not table:
		print "Occurred a loading error."
		continue
	tds = table.find_all('td',class_='')
	company_id = tds[1].string
	target = []
	for x in tds:
		if x.string:
			target.append(MySQLdb.escape_string(unicode(x.string).encode('utf8')))
		else:
			target.append('Null')
	cursor.execute(sentense1 % tuple(target))
	db.commit()



	locator = (By.XPATH,"//a[@onclick='changeTab();']")
	try:
		WebDriverWait(driver,3).until(EC.presence_of_element_located(locator))
	except Exception as e:
		print count,'not found part2.'
		count+=1
		continue
	point = driver.find_elements_by_xpath("//a[@onclick='changeTab();']")
	try:
		point[2].click()
	except Exception as e:
		print 'Occurred a loading error'
		continue
	locator = (By.XPATH,"//div[@class='list_l' and @style='display: block;']")
	try:
		WebDriverWait(driver,3).until(EC.presence_of_element_located(locator))
	except Exception as e:
		print count,'not found part2.'
		count+=1
		continue
	soup = BeautifulSoup(driver.page_source,'html.parser')
	table = soup.find('div',class_='list_l',style='display: block;')
	trs = table.find_all('tr')
	t = 1
	for x in trs:
		if t==1:
			t=0
			continue
		target = []
		tds = x.find_all('td')
		target.append(company_id)
		for y in tds:
			if y.string:
				target.append(MySQLdb.escape_string(unicode(y.string).encode('utf8')))
			else:
				target.append('Null')
		cursor.execute(sentense2 % tuple(target))
		db.commit()


	locator = (By.XPATH,"//a[@onclick='changeTab();']")
	try:
		WebDriverWait(driver,3).until(EC.presence_of_element_located(locator))
	except Exception as e:
		print count,'not found part3.'
		count+=1
		continue
	point = driver.find_elements_by_xpath("//a[@onclick='changeTab();']")
	try:
		point[3].click()
	except Exception as e:
		print 'Occurred a loading error'
		continue
	locator = (By.XPATH,"//div[@class='list_l' and @style='display: block;']")
	try:
		WebDriverWait(driver,3).until(EC.presence_of_element_located(locator))
	except Exception as e:
		print count,'not found part3.'
		count+=1
		continue
	soup = BeautifulSoup(driver.page_source,'html.parser')
	table = soup.find('div',class_='list_l',style='display: block;')
	tds = table.find_all('td',align='center')
	target = []
	target.append(company_id)
	for x in tds:
		if x.string:
			target.append(MySQLdb.escape_string(unicode(x.string).encode('utf8')))
		else:
			target.append('Null')
	cursor.execute(sentense3 % tuple(target))
	db.commit()
	
	print 'finish.',count
	count+=1
	if count>430760 and count < 830000:
		count=830760
	elif count>840000 and count < 870000:
		count=870000