文章目录

前言

今天爬取淘宝商品图片，遇到登录拦截，看了淘宝的登录 url 里的请求参数，好多加密，果断放弃。

不过可以用selenium模拟登录，然后获取cookies给requests使用，开整。

代码

from selenium import webdriverfrom selenium.webdriver.support import waitfrom mon.by import Byimport requestsfrom requests.cookies import RequestsCookieJarfrom fake_useragent import UserAgentclass SeleniumGetCookies:def __init__(self, user, pwd):self.user = userself.pwd = pwdua = UserAgent()headers = {'User-Agent': ua.random}self.session = requests.Session()self.session.headers.update(headers)# 配置chrome，绕过淘宝的检测option = webdriver.ChromeOptions()# option.add_argument('--headless') # 无头模式option.add_experimental_option('excludeSwitches', ['enable-automation'])option.add_experimental_option('useAutomationExtension', False)self.driver = webdriver.Chrome(options=option)self.driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {"source": """Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"""})self.driver.maximize_window()self.driver.get('/?spm=a230r.1.0.0.2dbc56bcMjHM4u')def login(self):# 点击登录wait.WebDriverWait(self.driver, 5).until(lambda x: x.find_element(By.CSS_SELECTOR, '.site-nav-sign > a'))self.driver.find_element(By.CSS_SELECTOR, '.site-nav-sign > a').click()# 输入账号密码登录self.driver.find_element(By.CSS_SELECTOR, '#fm-login-id').send_keys(self.user)self.driver.find_element(By.CSS_SELECTOR, '#fm-login-password').send_keys(self.pwd)self.driver.find_element(By.CSS_SELECTOR, '.fm-btn > button').click()# 检测是否登录成功def inspect_login(self):wait.WebDriverWait(self.driver, 10).until(lambda x: x.find_element(By.CSS_SELECTOR, '.site-nav-user > a'))user = self.driver.find_element(By.CSS_SELECTOR, '.site-nav-user > a').textif user == '你的用户名':print('登录成功')return Trueelse:print('登录失败')return False# 加载cookies给requestsdef load_to_requests(self):selenium_cookies = self.driver.get_cookies()tmp_cookies = RequestsCookieJar()for item in selenium_cookies:tmp_cookies.set(item["name"], item["value"])self.session.cookies.update(tmp_cookies)def run(self):self.login()if self.inspect_login():self.load_to_requests()self.driver.close()self.driver.quit()return self.sessionelse:# 这里可以加登陆失败后的处理return False

总结

直接导入SeleniumGetCookies类，运行run()函数，获取session返回给爬虫文件即可。

全部代码看这里 github地址