Skip to content

Commit da5bebf

Browse files
Ehco1996Ehco1996
committed
知乎信息抓取
1 parent 9a96f20 commit da5bebf

6 files changed

Lines changed: 105 additions & 0 deletions

File tree

‎zhihu/zhihu_hard/.vscode/settings.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"python.pythonPath": "/Users/ehco/.pyenv/versions/venv-spider/bin/python"
3+
}

‎zhihu/zhihu_hard/src/__init__.py‎

Whitespace-only changes.

‎zhihu/zhihu_hard/src/configs.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
from lazyspider.lazyheaders import LazyHeaders
2+
3+
# 登录之后的curl字符串
4+
CURL = ''''''
5+
# 轮子哥的主页地址
6+
VZCH = 'https://www.zhihu.com/people/excited-vczh/activities'
7+
8+
# 获取你的cookie和headers
9+
lz = LazyHeaders(CURL)
10+
COOKIES = lz.getCookies()
11+
HEDADERS = lz.getHeaders()
12+
# print(COOKIES, HEDADERS)

‎zhihu/zhihu_hard/src/parse.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
from bs4 import BeautifulSoup
2+
3+
4+
def to_soup(page):
5+
return BeautifulSoup(page, 'lxml')
6+
7+
8+
with open('1.html', 'r') as f:
9+
html = f.read()
10+
11+
12+
soup = to_soup(html)
13+
14+
res = soup.find_all('div', class_="List-item")
15+
for item in res:
16+
ele = item.find('h2', class_='ContentItem-title')
17+
title = ele.text
18+
url = 'https://www.zhihu.com' + ele.a['href']
19+
print(title, url)

‎zhihu/zhihu_hard/src/spider.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
2+
3+
from configs import COOKIES, HEDADERS
4+
from tools import get_driver
5+
from parse import to_soup
6+
7+
8+
class UserActivities():
9+
'''
10+
用户的动态信息
11+
'''
12+
13+
def __init__(self, url):
14+
self.driver = get_driver()
15+
self.peopple_url = url
16+
self.url_list = set()
17+
18+
def get_page_source(self):
19+
'''
20+
获取html文本
21+
'''
22+
self.driver.get(self.peopple_url)
23+
return self.driver.page_source
24+
25+
def parse_user_html(self):
26+
'''
27+
解析用户动态
28+
'''
29+
soup = to_soup(self.get_page_source())
30+

‎zhihu/zhihu_hard/src/tools.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import shutil
2+
3+
import requests
4+
from selenium.webdriver import PhantomJS
5+
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
6+
7+
from configs import COOKIES, HEDADERS
8+
9+
10+
def my_session():
11+
session = requests.Session()
12+
session.get('https://www.zhihu.com',
13+
cookies=COOKIES, headers=HEDADERS)
14+
return session
15+
16+
17+
def get_image(url, path):
18+
res = requests.get(url, stream=True)
19+
with open(path, 'wb') as f:
20+
shutil.copyfileobj(res.raw, f)
21+
22+
23+
def save_html(text, name):
24+
with open(name, 'w') as f:
25+
f.write(text)
26+
27+
28+
def get_driver():
29+
# 设置请求头
30+
dcap = dict(DesiredCapabilities.PHANTOMJS)
31+
dcap["phantomjs.page.settings.userAgent"] = (HEDADERS.get("User-Agent"))
32+
# 初始化driver
33+
driver = PhantomJS(desired_capabilities=dcap)
34+
# 加入cookies
35+
for c in my_session().cookies:
36+
driver.add_cookie({'name': c.name, 'value': c.value,
37+
'path': c.path, 'expiry': c.expires, 'domain': c.domain})
38+
# 设置窗口大小
39+
driver.set_window_position(0, 0)
40+
driver.set_window_size(1920, 1080)
41+
return driver

0 commit comments

Comments
 (0)