import time import random import re import sys import os import django import logging sys.path.insert(0, "../") os.environ.setdefault('DJANGO_SETTINGS_MODULE', "apollo.settings") django.setup() from lxml import etree from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.action_chains import ActionChains from selenium.common.exceptions import WebDriverException from spiders.spiders.operation_model import save_data from spiders.spiders.operation_model import fail_obj_save_data use_agent = [ 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3)', 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; GTB7.0)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E) QQBrowser/6.9.11079.201', ] def get_track(distance): """ 根据偏移量获取拟人的移动轨迹 :param distance: 偏移量 :return: 移动轨迹 """ track = [] current = 0 mid = distance * 3 / 4 t = random.randint(2, 3) / 10 v = 0 while current < distance: if current < mid: a = 2 else: a = -3 v0 = v v = v0 + a * t move = v0 * t + 1 / 2 * a * t * t current += move track.append(round(move)) return track def drag_the_ball(driver, track, slider): """ 根据运动轨迹拖拽 :param driver: webdriver 对象 :param track: 运动轨迹 :param slider: 滑块 """ ActionChains(driver).click_and_hold(slider).perform() while track: x = random.choice(track) ActionChains(driver).move_by_offset(xoffset=x, yoffset=0).perform() track.remove(x) time.sleep(0.1) # 模拟人往回滑动 imitate = ActionChains(driver).move_by_offset(xoffset=-2, yoffset=0) time.sleep(0.015) imitate.perform() time.sleep(random.randint(6, 10) / 10) imitate.perform() time.sleep(0.04) imitate.perform() time.sleep(0.012) imitate.perform() time.sleep(0.019) imitate.perform() time.sleep(0.033) ActionChains(driver).move_by_offset(xoffset=1, yoffset=0).perform() SUCCESSFUL_SAVE_DATA = list() class YoukuSpider: def __init__(self): # 驱动路径 self.wait_time = 10 self.crawl_url = { "少儿": 'https://child.youku.com/', "动漫": 'https://comic.youku.com/', "电影": 'https://movie.youku.com/', "电视剧": 'https://tv.youku.com/' } self.fail_obj = dict() self.invalid = dict() # 配置参数 self.heads = '--headless' self.disable = '--disable-gpu' # 存储数据 self.lv1_body = list() # 存储1级页面body体 self.son_msg = list() self.save_data = list() def close(self): self.browser.quit() def judge_verification_code(self, idx, title, url, leixing, channel): if "优酷 web 页面-验证码-临时" in self.browser.page_source: slide_span = self.browser.find_element_by_css_selector("#nc_1_n1z") outside_div = self.browser.find_element_by_css_selector('#nc_1__scale_text') outside_width = outside_div.size['width'] slide_width = slide_span.size['width'] track = get_track(outside_width - slide_width) drag_the_ball(self.browser, track, slide_span) time.sleep(2) if "优酷 web 页面-验证码-临时" in self.browser.page_source: self.browser.refresh() time.sleep(2) # self.judge_verification_code(idx, title, url, leixing, channel) return # 页面提取附加点击简介按钮 def browser_load_page_content(self, url, title, width, height, idx, cate, channel, onclick_css=None, add_operation=False): # 创建参数对象和无头浏览器 self.chrome_options = Options() self.chrome_options.add_argument('lang=zh_CN.UTF-8') self.chrome_options.add_argument(self.heads) self.chrome_options.add_argument(self.disable) self.chrome_options.add_experimental_option("excludeSwitches", ['enable-automation']) self.chrome_options.add_argument('user-agent="{}"'.format(random.choice(use_agent))) self.browser = webdriver.Chrome(options=self.chrome_options) self.browser.set_window_size(width, height) js1 = '''Object.defineProperties(navigator,{ webdriver:{ get: () => false } }) ''' js2 = '''window.navigator.chrome = { runtime: {}, }; ''' js3 = '''Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); ''' js4 = '''Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5,6], }); ''' self.browser.execute_script(js1) self.browser.execute_script(js2) self.browser.execute_script(js3) self.browser.execute_script(js4) try: self.browser.get(url) self.browser.implicitly_wait(self.wait_time) if title: self.judge_verification_code(idx, title, url, cate, channel) if add_operation: action = ActionChains(self.browser) try: onclick = self.browser.find_element_by_css_selector(onclick_css) action.move_to_element(onclick).click().perform() time.sleep(2) response_body = self.browser.page_source except Exception: response_body = None self.invalid["{}_{}".format(title, cate)] = {"idx": idx, "title": title, "url": url, "cate": cate, "channel": channel} else: response_body = self.browser.page_source if "优酷 web 页面-验证码-临时" in response_body: response_body = None self.fail_obj["{}_{}".format(title, cate)] = {"idx": idx, "title": title, "url": url, "cate": cate, "channel": channel} # self.browser.save_screenshot('C:\\Users\\BXJH\\Desktop\\XiaoZ\\image\\{}{}{}t.png'.format(cate, channel, idx)) except WebDriverException: response_body = None self.fail_obj["{}_{}".format(title, cate)] = {"idx": idx, "title": title, "url": url, "cate": cate, "channel": channel} finally: self.close() return response_body # 爬取一级页面进行解析 def crawl_lv1_page(self): i = 0 for channel, lv1_link in self.crawl_url.items(): i += 1 time.sleep(self.wait_time) lv1_body = self.browser_load_page_content(lv1_link, "", 1920, 1200, i, '0', channel) self.lv1_body.append({"channel": channel, "body": lv1_body}) # 解析一级页面 def params_lv1_page_data(self): for lb in self.lv1_body: selector = etree.HTML(lb["body"]) recommend = selector.xpath('//div[@class="focussidebarswiper_contentswiper"]/div/div/a')[:8] top_rank = selector.xpath('//div[@aria-hidden="false"]/div/div/div/div/div/div/div/div/div/a')[:8] for idx, rd in enumerate(recommend): title = rd.xpath('@title')[0] url = rd.xpath('@href')[0] self.son_msg.append({ "title": title, "url": url if "http" in url else "https:" + url, "idx": len(recommend) - idx, "cate": "recommend", "channel": lb["channel"] }) for idx, tr in enumerate(top_rank): title = tr.xpath('@title')[0] url = tr.xpath('@href')[0] self.son_msg.append({ "title": title, "url": url if "http" in url else "https:" + url, "idx": len(top_rank) - idx, "cate": "rank", "channel": lb["channel"] }) # 子集页面解析并保存数据 def params_extract_data(self, data): for idx, sub in enumerate(data): time.sleep(6) div, mod = divmod(idx, 10) if div and mod == 0: time.sleep(300) title = sub.get("title", "") url = sub.get("url", "") cate = sub.get("cate", "") idx = sub.get("idx", "") channel = sub.get("channel", "") body = self.browser_load_page_content(url, title, 1920, 1450, idx=idx, cate=cate, channel=channel, onclick_css=".desc div div.intro-text", add_operation=True) if body: selector = etree.HTML(body) try: hot_value = selector.xpath('//div[@class="desc"]/span/span[@class="video-heat-text"]/text()')[0] except IndexError: hot_value = "" _tags = selector.xpath('//div[@class="desc"]/div[@data-sn="tags"]/a/text()') tags = ','.join(_tags) _img = selector.xpath('//div[@class="intro-wrap"]/a/@style')[0] img = _img.split('"')[1] _cleaned = selector.xpath('//div[@class="content-wrap"]/div') cleaned = {"标签": tags} for info_wrap in _cleaned: _span = info_wrap.xpath('span/text()')[0] span_title = re.sub("[:：]", '', _span, 1) _info = info_wrap.xpath('div[@class="info"]/text()') if not len(_info): _info = info_wrap.xpath('div[@class="info"]/a/text()') info = ','.join(_info) cleaned[span_title] = info global SUCCESSFUL_SAVE_DATA SUCCESSFUL_SAVE_DATA.append((title, img, url, cleaned, channel, idx, hot_value, 'youku', cate)) logging.warning((title, img, url, cleaned, channel, idx, hot_value, 'youku', cate)) # 失败数据存储 def fail_obj_save(self, ago_data): print(ago_data) for _, sub in ago_data.items(): time.sleep(60) logging.warning(sub) title = sub.get("title", "") url = sub.get("url", "") cate = sub.get("cate", "") idx = sub.get("idx", "") channel = sub.get("channel", "") body = self.browser_load_page_content(url, title, 1920, 1200, idx=idx, cate=cate, channel=channel, onclick_css=".desc div div.intro-text", add_operation=True) if body: selector = etree.HTML(body) try: hot_value = selector.xpath('//div[@class="desc"]/span/span[@class="video-heat-text"]/text()')[0] except IndexError: hot_value = "" _tags = selector.xpath('//div[@class="desc"]/div[@data-sn="tags"]/a/text()') tags = ','.join(_tags) _img = selector.xpath('//div[@class="intro-wrap"]/a/@style')[0] img = _img.split('"')[1] _cleaned = selector.xpath('//div[@class="content-wrap"]/div') cleaned = {"标签": tags} for info_wrap in _cleaned: _span = info_wrap.xpath('span/text()')[0] span_title = re.sub("[:：]", '', _span, 1) _info = info_wrap.xpath('div[@class="info"]/text()') if not len(_info): _info = info_wrap.xpath('div[@class="info"]/a/text()') info = ','.join(_info) cleaned[span_title] = info global SUCCESSFUL_SAVE_DATA SUCCESSFUL_SAVE_DATA.append((title, img, url, cleaned, channel, idx, hot_value, 'youku', cate)) logging.warning((title, img, url, cleaned, channel, idx, hot_value, 'youku', cate)) def save_to_mysql(): if len(SUCCESSFUL_SAVE_DATA): for obj in SUCCESSFUL_SAVE_DATA: title, img, url, cleaned, channel, idx, hot_value, plat, cate = obj save_data(title, img, url, cleaned, channel, idx, hot_value, plat, cate) def main(): youku = YoukuSpider() youku.crawl_lv1_page() time.sleep(120) youku.params_lv1_page_data() logging.error(youku.son_msg) youku.params_extract_data(youku.son_msg) time.sleep(500) print("""重新爬取失败数据""") youku.fail_obj_save(youku.fail_obj) time.sleep(300) print("""重新爬取失效数据""") youku.fail_obj_save(youku.invalid) save_to_mysql() if __name__ == '__main__': main()