Youtube 评论数

youtube评论数单独走评论接口，无法从静态页面中直接获取。使用selenium获取非常耗时好资源。

1. 经过分析评论数中请求中的许多参数都在youtube的静态页面中，只需要将其匹配出来就可以了。

具体匹配如下：

 def get_comment(self, data_page):
        '''
        请求参数：
        # itemSectionRenderer  : ctoken / continuation
        # XSRF_TOKEN : session_token
        # {"itemSectionRenderer":{"continuations":[{"nextContinuationData":{"continuation":"EkQSCzJTeVdfQ1BYeGNVwAEAyAEA4AEBogINKP___________wFAAMICGxoXaHR0cHM6Ly93d3cueW91dHViZS5jb20iABgG","clickTrackingParams":"CJMBEMm3AiITCL3W2fbyyeoCFdNaYAod2UgKrQ==","label":{"runs":[{"text":"显示更多回复"}]}}}]
        headers参数：
        # x-youtube-variants-checksum ： VARIANTS_CHECKSUM
        # x-youtube-page-label ：PAGE_BUILD_LABEL
        # x-youtube-page-cl ：PAGE_CL
        # x-youtube-device ： DEVICE
        # x-youtube-client-version ：INNERTUBE_CONTEXT_CLIENT_VERSION
        # x-youtube-client-name ： INNERTUBE_CONTEXT_CLIENT_NAME
        # 'x-youtube-time-zone': 'Asia/Shanghai',
        # 'x-youtube-utc-offset': '480',
        :param data_page:
        :return:
        '''

        result = {}  # 存储headers 和 params 结果
        request_params = {}   # 存储params结果
        params = re.search('"itemSectionRenderer":\{"continuations":(\[.*?\}\}\}\])', data_page)
        if params.group(1):
            request_params['params'] = json.loads(params.group(1))
        session_token = re.search('"XSRF_TOKEN":"(.*?)"', data_page)
        if session_token:
            request_params['session_token'] = session_token.group(1)  # 获取session_token

        request_headers = {}  # requests_headers youtube评论数请求头
        checksum = re.search('VARIANTS_CHECKSUM":"(.*?)"', data_page)
        if checksum:
            request_headers['x-youtube-variants-checksum'] = checksum.group(1)
        devices = re.search('DEVICE":"(.*?)"', data_page)
        if devices:
            request_headers['x-youtube-device'] = devices.group(1)
        label = re.search('PAGE_BUILD_LABEL":"(.*?)"', data_page)
        if label:
            request_headers['x-youtube-page-label'] = label.group(1)
        page_cl = re.search('PAGE_CL":(.*?),', data_page)
        if page_cl:
            request_headers['x-youtube-page-cl'] = page_cl.group(1)
        client_version = re.search('INNERTUBE_CONTEXT_CLIENT_VERSION":"(.*?)"', data_page)
        if client_version:
            request_headers['x-youtube-client-version'] = client_version.group(1)
        client_name = re.search('INNERTUBE_CONTEXT_CLIENT_NAME":(.*?),', data_page)
        if client_name:
            request_headers['x-youtube-client-name'] = client_name.group(1)
        request_headers['x-youtube-time-zone'] = 'Asia/Shanghai'
        request_headers['x-youtube-utc-offset'] = '480'
        result['requests_params'] = request_params
        result['requests_headers'] = request_headers
        return result

2. 通过匹配出来的字段匹配url， data, 以及 headers

headers = {
            "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
            "cookie": "GPS=1; YSC=-8SgKkJKT6A; VISITOR_INFO1_LIVE=JmzmM2Mt83o; CONSENT=YES+GB.zh-CN+V14+BX",
        }
        proxy = {"http": "http://127.0.0.1:7890", "https": "https://127.0.0.1:7890"}
        data = self.requests_get(url, headers, proxy=proxy)
        requests = self.get_comment(data)
        requests_params = requests.get('requests_params')
        requests_headers = requests.get('requests_headers')
        # result = self.get_page(data)

        # 拼接url
        params = requests_params.get('params')[0]
        ctoken = params.get('nextContinuationData').get('continuation')
        itct = params.get('nextContinuationData').get('clickTrackingParams')
        url = 'https://www.youtube.com/comment_service_ajax?action_get_comments=1&pbj=1&ctoken={ctoken}&continuation={ctoken}&itct={itct}'.format(ctoken=ctoken, itct=quote(itct))
        # 拼接headers
        comments_headers = headers
        comments_headers.update(requests_headers)
        # 拼接 data
        fields = dict()
        fields['session_token'] = requests_params.get('session_token')

这里值得注意的是访问youtube链接的cookie应该和访问评论数接口的cookie保持一致，否则获取不到评论数据。

3. 然后对评论数据进行请求解析

 comments_page = self.requests_post(url=url, headers=comments_headers, data=fields, proxy=proxy)
        with open("comment_source.txt", 'w', encoding='utf-8')as f:
            f.write(comments_page)
        if comments_page:
            comments_data = json.loads(comments_page)
            comments_number = comments_data.get('response').get('continuationContents').get('itemSectionContinuation')\
                .get('header').get('commentsHeaderRenderer').get('commentsCount')
            print('comments_number:', comments_number)