撰写金融爬虫 – 爬取雪球网用户动态（1）- 基础功能实现
利用Python爬虫爬取雪球网用户动态，存储为CSV文件（以大V望京博格举例）
#encoding: utf-8
#作者：世欺子，http://www.zhy.world
import requests
import re
import json
import csv
import time
import datetime
# cookie替换成从自己的浏览器获取到的cookie，需要登录后再获取否则无法爬取超过5页数据
cookie = '[替换成自己的cookie]'
headers = {
'Host':"xueqiu.com",
'Referer': 'https://xueqiu.com/',
'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36",
'Accept':"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
'Connection':'keep-alive',
'Cookie': cookie
}
# 初始化数据
page = 1
userid = 4579887327 # 这里填写你要爬取的用户UID
type = 0
url = 'https://xueqiu.com/v4/statuses/user_timeline.json'+'?page='+str(page)+'&user_id='+str(userid)+'&type='+str(type)
response = requests.get(url, headers = headers)
text = response.text
dict_text = json.loads(text)
# 获取maxPage
maxPage = dict_text['maxPage']
userName = dict_text['statuses'][0]['user']['screen_name']
csv_headers = ['id','user_id','source','title','created_at','retweet_count','reply_count','fav_count','truncated','commentid','retweet_status_id','symbol_id','description','type','source_link','edited_at','mark', 'reward', 'liked', 'quote_cards', 'rqtype', 'paid_mention', 'is_answer', 'donate_count', 'text', 'is_no_archive', 'topic_desc', 'firstImg', 'offer', 'topic_symbol', 'longTextForIOS', 'cover_pic', 'promotion_url', 'target', 'topic_pic_headOrPad', 'retweeted_status', 'like_count', 'commentId', 'status_industry', 'meta_keywords', 'is_column', 'timeBefore', 'is_bonus', 'favorited', 'recommend_cards', 'is_original_declare', 'topic_pic_thumbnail', 'score', 'is_refused', 'reward_amount', 'notice_tag', 'promotion_pic', 'excellent_comments', 'talk_count', 'show_cover_pic', 'expend', 'is_private', 'pic_sizes', 'new_card', 'rqid', 'fragment', 'topic_pic', 'rawTitle', 'stockCorrelation', 'promotion_id', 'canEdit', 'blocking', 'pic', 'blocked', 'topic_title', 'reward_user_count', 'cover_pic_size', 'reward_count', 'forbidden_retweet', 'card', 'topic_pic_thumbnail_small', 'view_count', 'reply_user_count', 'is_ss_multi_pic', 'tagStr', 'common_emotion', 'answers', 'weixin_retweet_count', 'tags', 'favorited_created_at', 'controversial', 'donate_snowcoin', 'mp_not_show_status', 'mark_desc', 'user', 'reply_user_images', 'tagsForWeb', 'order_id', 'current_stock_price', 'forbidden_comment']
last_index = 0
for page in range(1,maxPage+1):
url = 'https://xueqiu.com/v4/statuses/user_timeline.json'+'?page='+str(page)+'&user_id='+str(userid)+'&type='+str(type)
response = requests.get(url, headers = headers)
text = response.text
dict_text = json.loads(text)
print('正在爬取第'+str(page)+'页...')
if page == 1:
statuses = dict_text['statuses']
else:
statuses = statuses + dict_text['statuses']
print('当前爬取的数据量:'+str(len(statuses))+'条')
# 格式化created_time
for index in range(last_index,len(statuses)):
before_formatting = dict_text['statuses'][index-last_index]['created_at']
if before_formatting:
before_formatting = datetime.datetime.fromtimestamp(int(before_formatting) / 1000).strftime('%Y-%m-%d %H:%M:%S')
dict_text['statuses'][index-last_index]['created_at'] = before_formatting
last_index = len(statuses)
# 防止反爬虫机制，每爬取100条休息15秒
if page%100 == 99:
time.sleep(15)
with open(str(userName)+'.csv', 'w', encoding='utf-8', newline='') as fp:
writer = csv.DictWriter(fp,csv_headers)
writer.writeheader()
writer.writerows(statuses)