使用python爬取抖音视频列表信息

:authority: www.iesdouyin.com
:method: GET
:path: /web/api/v2/aweme/post/"text-align: center">

通过分析ajax请求的URL https://www.iesdouyin.com/web/api/v2/aweme/post/"external nofollow" href="https://www.iesdouyin.com/share/user/110677980134">https://www.iesdouyin.com/share/user/110677980134




(function() {
  $(function(){
    __M.require('douyin_falcon:page/reflow_user/index').init({
      uid: "110677980134",
      dytk: '061ae6e81229e178146aa674327eba89'
    });
  });
})();


通过正则获取到此参数

  
_signature 获取比较复杂，抖音对前端的js代码进行了混淆压缩，不易直接分析出算法过程，不过可以执行签名算法代码，并返回对应的签名结果。
  
执行js代码的可以使用nodejs或者selenium webdriver，这里推荐使用selenium webdriver ， nodejs的js执行环境与浏览器有区别，计算出的签名结果，并不能通过验证，selenium webdriver 可以调用本地浏览器，计算出的签名可以跟浏览器直接访问访问计算出的签名一致。
  
格式化之后的js代码，点击查看， 执行js方法 _bytedAcrawler.sign("110677980134") 对参数进行签名

代码实现导出主页视频列表


def get_user_video_list_by_uid(user_id, cursor=0):
  url = 'https://www.iesdouyin.com/web/api/v2/aweme/post/"sign:%s,dytk:%s" % (sign, dytk))
  if sign is None or dytk is None:
    tk_logger.log("sign [%s] or dytk [%s] is none" % (sign, dytk))
    return None
  headers = dict_merge(CHROME_HEADER, {
    "Accept": "application/json",
    "X-Requested-With": "XMLHttpRequest",
  })
  params = {
    "user_id": user_id,
    "count": "21",
    "max_cursor": cursor,
    "aid": "1128",
    "_signature": sign,
    "dytk": dytk
  }
  res = requests.get(url, headers=headers, params=params)
  tk_logger.info("request url: %s" % res.url)
  content = res.content.decode("utf8")
  jsn = json.loads(content)
  return jsn


获取的视频列表信息



获取视频信息代码片段


def get_video_detail_by_id(video_id):
  url = "https://aweme-hl.snssdk.com/aweme/v1/aweme/detail/"
  data = {"aweme_id": video_id}
  headers = {
    "sdk-version": "1",
    "x-Tt-Token": "00fc1e7950db67b5f43a312e9265cdfee513ea70c36d918c871f3bb553347f3db50ffca143b8722327b345816a75efca071d",
    "User-Agent": "Aweme 6.5.0 rv:65014 (iPhone; iOS 12.3.1; en_CN) Cronet",
    "Content-Type": "application/x-www-form-urlencoded",
    "Cookie": "tt_webid=6636348554880222728; __tea_sdk__user_unique_id=6636348554880222728; odin_tt=76d9b82d6e6f2ddfc99719a5b5d44a7d703cf977f0f7bddf8537f93920d57cb9ec33162ee47868b760f6b09e69209bb2f90bad220b75678af850a0dfa9f056e2; install_id=74239983401; ttreq=1$dab0516952a4157c0c11d4993533c09d6e45fc94; sid_guard=fc1e7950db67b5f43a312e9265cdfee5%7C1559955316%7C5184000%7CWed%2C+07-Aug-2019+00%3A55%3A16+GMT; uid_tt=0afcb06309f632d872799ec0ac3b2c80; sid_tt=fc1e7950db67b5f43a312e9265cdfee5; sessionid=fc1e7950db67b5f43a312e9265cdfee5",
    "X-Khronos": "1559956401",
    "X-Gorgon": "8300000000002e40eee38cad71d14037bd1385d18bc973f094f5",
  }
  ret = {}
  res = requests.post(url, data=data, headers=headers)
  if res.status_code == 200:
    # tk_logger.info("video detail raw:%s" % res.content.decode("utf8"))
    jsn = json.loads(res.content)
    detail = jsn.get("aweme_detail", {})
    video_info = get_video_info(detail)
    user_info = get_user_info(detail)
    play_addr = get_play_address(detail)
    video_cover = get_video_cover(detail)
    ret["video_info"] = video_info
    ret["user_info"] = user_info
    ret["play_addr"] = play_addr
    ret["video_cover"] = video_cover
  else:
    raise TKException("get video detail failed [%s][%d]" % (url, res.status_code))
  return ret


下载视频代码片段


detail = get_video_detail_by_id(video_id)
def download_video(detail):
  url = detail.get("play_addr", {}).get("url_list", [])
  if len(url) == 0:
    raise TKException("cannot get video url list [%s]" % detail)

  url = url[0]
  folder = DOWNLOAD_DIR + '/' + detail.get('user_info', {}).get("uid", "unknown")
  if not os.path.exists(folder):
    os.mkdir(folder)
  video_id = detail.get('video_info', {}).get('statistics', {}).get('aweme_id')
  # filename = "%s/%s" % (folder, detail.get("video_info", {}).get("desc", video_id) + ".mp4")
  filename = "%s/%s" % (folder, video_id + ".mp4")
  tk_logger.info("download video %s" % url)
  if os.path.isfile(filename):
    file_size = get_remote_file_size(url)
    if file_size == os.path.getsize(filename):
      tk_logger.info("file already downloaded, skip ...")
      return
    else:
      tk_logger.info("download file , file size:%d" % file_size)
  res = requests.get(url, headers=IOS_HEADER)
  if res.status_code == 200:
    with open(filename, "wb") as fp:
      for chunk in res.iter_content(chunk_size=1024):
        fp.write(chunk)
  else:
    raise TKException("download video [%s] failed [%d]" % (url, res.status_code))


下载视频

总结
以上所述是小编给大家介绍的使用python爬取抖音视频列表信息 ,希望对大家有所帮助，如果大家有任何疑问请给我留言，小编会及时回复大家的。在此也非常感谢大家对网站的支持！

如果你觉得本文对你有帮助，欢迎转载，烦请注明出处，谢谢！