简述
代码直接借助python的selenium的库来控制浏览器，与requests和bs4相比，selenium允许你用高级多的方法实现网页交互，但是因为他启动了Web浏览器，下载文件会比较慢，并且难以在后台运行。假如你考虑的不是通过此程序来练习，那么就直接退出这篇文章。
思路上面，在父页面获取10部电影或者电视剧的link，然后跳转到电影详情页，进行信息爬取。
import os,refrom time import sleepfrom selenium import webdriverURL = "/"URLmovie = '/explore#!type=movie&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_start=0'URLseries = ''#后面直接加入跳转，无需这个变量webdri_path = r'C:\Users\86151\AppData\Local\Google\Chrome\Application\chromedriver.exe'global flag os.mkdir(os.path.join(".","douban"))os.mkdir(os.path.join(".","douban","movie"))os.mkdir(os.path.join(".","douban","series"))# 定义一个函数来获取电影的相关信息，并且存放在文件里面def getInfo(titleOfWork,aim):# 获取电影发布时间和作品名称releaseTime = driver.find_element_by_class_name('year')if int(re.sub(r"\D","",releaseTime.text)) < :return# 利用正则表达式取出发布时间中的非数字内容fileName = re.sub(r"\D","",releaseTime.text) + re.sub(r"\W","",titleOfWork)print(fileName)os.mkdir(os.path.join(".","douban",aim,fileName))# 存放电影图片driver.find_element_by_xpath('//*[@id="mainpic"]/a/img').screenshot(os.path.join(".","douban",aim,fileName,"pic.png"))# 创建一个txt文件，写入作品的基本内容，名称，上映年份，导演，主演，作品简介，豆瓣关于这个作品的链接with open(os.path.join(".","douban",aim,fileName,"info.txt"),"a") as movieFile:# 写入作品名称movieFile.write(titleOfWork+"\n")# 写入作品上映年份movieFile.write("上映日期:" + re.sub(r"\D","",releaseTime.text)+"\n")# 写入作品导演director = driver.find_element_by_class_name('attrs').textmovieFile.write("导演:" + director+"\n")# 写入作品主演starring = driver.find_element_by_class_name('actor').textmovieFile.write(starring+"\n")# 写入作品简介# introduction = driver.find_element_by_xpath('//*[@id="link-report"]/span[1]').textintroduction = driver.find_element_by_class_name('related-info').textmovieFile.write(introduction+"\n")# 写如链接movieFile.write("链接:"+driver.current_url)global flag flag +=1def openURL(aim,Links):global flagflag = 0for link in Links:# 获取电影名称titleOfWork = link.textprint(type(link.text[-3:]))if ("." not in titleOfWork[-3:]) or (float(titleOfWork[-3:]) < 7):continuedriver.execute_script(js.format(link.get_attribute("href")))driver.switch_to_window(driver.window_handles[-1])getInfo(titleOfWork[:-3],aim)# 关闭新建的电影窗口driver.close()driver.switch_to_window(driver.window_handles[0])if flag == 10:breaktry:driver = webdriver.Chrome(webdri_path)driver.get(URLmovie)driver.implicitly_wait(3)driver.maximize_window()sleep(1)js = "window.open('{}','_blank');"# 选10部电影，按评价排序选取电影driver.find_element_by_xpath('//*[@id="content"]/div/div[1]/div/div[2]/div[1]/form/div[3]/div[1]/label[2]/input').click()driver.find_elements_by_css_selector('.tag-list label')[4].click()# 将所有的电影链接存放在links数组里面movieLinks = driver.find_elements_by_class_name("item")openURL("movie",movieLinks)# 处理电视剧部分driver.find_element_by_xpath('//*[@id="db-nav-movie"]/div[2]/div/ul/li[3]/a').click()seriesLinks = driver.find_elements_by_class_name("item")openURL("series",seriesLinks)driver.quit()except Exception as e:print(e)
