api ai tutorial

import scrapy import re import pickle import json import sys class MySpider(scrapy.Spider): global allUrlFile , fullUrl allUrlFile = open ( 'allUrl.txt' , 'a' ) fullUrl = open ( 'fullUrl.txt' , 'a' ) localHost = "http://localhost:8050/render.html?url=" #youtubeUrl = "https://www.youtube.com/channel/UCv1Ybb65DkQmokXqfJn0Eig/channels" # channel with only one ajax # youtubeUrl = "https://www.youtube.com/user/khanacademy/videos" # khan academy youtubeUrl = "https://www.youtube.com/channel/UCU0kWLAbhVGxXarmE3b8rHg/videos" # khan hindi start_urls = [localHost + youtubeUrl] name = "allvideos" def start_requests( self ): for url in self .start_urls: yield scrapy.Request(url, self .parse) def parse( self , response): self .log( "this program just visited " + response.url) if not 'browse_ajax' in response.url: ...

docker run -p 8050:8050 scrapinghub/splash

import scrapy import pickle class MySpider(scrapy.Spider): start_urls = [ "http://localhost:8050/render.html?url=https://www.youtube.com/channel/UCv1Ybb65DkQmokXqfJn0Eig/channels" ] name = "youtubesc" def start_requests( self ): for url in self .start_urls: yield scrapy.Request(url, self .parse) def parse( self , response): self .log( "this program just visited" + response.url) print ( "response" ) print (response.text) # print( response.css('a.ux-thumb-wrap.yt-uix-sessionlink .spf-link').extract()) filename = "pp.html" with open (filename, 'wb' ) as f: pickle.dump((response.body), f) # yield { # 'author_name': response.css('small.author::text').extract_first() # } We are leveraging localhost of splash bcz normal methods explained on website was not working