python 爬虫基础

基础脚本

import urllib
import urllib2
import re
import time



for i in range(10):
    url = "https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_start"+str(i*20)
    
    headers={
    "GET":url,
    "host":"movie.douban.com",
    "Referer":"https://movie.douban.com/explore",
    "User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36"
    }

    req=urllib2.Request(url)
    for key in headers:
        req.add_header(key, headers[key])
    html=urllib2.urlopen(req).read()
    
    print(html)
    time.sleep(5)

结果

{"subjects":[{"rate":"8.1","cover_x":1448,"title":"目击者之追凶","url":"https:\/\/movie.douban.com\/subject\/11600078\/","playable":false,"cover":"https://img1.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2494647817.jpg","id":"11600078","cover_y":2048,"is_new":false},{"rate":"8.1","cover_x":750,"title":"星游记之风暴法米拉","url":"https:\/\/movie.douban.com\/subject\/26771100\/","playable":false,"cover":"https://img3.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2492792540.jpg","id":"26771100","cover_y":1050,"is_new":false},{"rate":"7.0","cover_x":1600,"title":"那些爱人","url":"https:\/\/movie.douban.com\/subject\/26949321\/","playable":false,"cover":"https://img3.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2410602594.jpg","id":"26949321","cover_y":2370,"is_new":true},{"rate":"6.3","cover_x":1417,"title":"绝世高手","url":"https:\/\/movie.douban.com\/subject\/26754831\/","playable":true,"cover":"https://img3.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2462744662.jpg","id":"26754831","cover_y":1984,"is_new":false},{"rate":"7.2","cover_x":1099,"title":"记忆大师","url":"https:\/\/movie.douban.com\/subject\/25884801\/","playable":true,"cover":"https://img3.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2455156816.jpg","id":"25884801","cover_y":1600,"is_new":false},{"rate":"8.1","cover_x":1418,"title":"银河护卫队2","url":"https:\/\/movie.douban.com\/subject\/25937854\/","playable":true,"cover":"https://img3.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2455261804.jpg","id":"25937854","cover_y":1985,"is_new":false},{"rate":"7.9","cover_x":1142,"title":"一念无明","url":"https:\/\/movie.douban.com\/subject\/26704590\/","playable":false,"cover":"https://img3.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2452970761.jpg","id":"26704590","cover_y":1600,"is_new":false},{"rate":"7.1","cover_x":666,"title":"巴霍巴利王(下):终结","url":"https:\/\/movie.douban.com\/subject\/26420932\/","playable":false,"cover":"https://img3.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2452075545.jpg","id":"26420932","cover_y":1000,"is_new":false},{"rate":"6.5","cover_x":1200,"title":"猎杀星期一","url":"https:\/\/movie.douban.com\/subject\/7154408\/","playable":false,"cover":"https://img3.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2461637453.jpg","id":"7154408","cover_y":1600,"is_new":true},{"rate":"7.6","cover_x":674,"title":"三个老枪手","url":"https:\/\/movie.douban.com\/subject\/26266085\/","playable":true,"cover":"https://img3.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2493595992.jpg","id":"26266085","cover_y":1000,"is_new":false},{"rate":"9.1","cover_x":4500,"title":"摔跤吧!爸爸","url":"https:\/\/movie.douban.com\/subject\/26387939\/","playable":true,"cover":"https://img3.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2457983084.jpg","id":"26387939","cover_y":6300,"is_new":false},{"rate":"4.2","cover_x":705,"title":"青禾男高","url":"https:\/\/movie.douban.com\/subject\/26787124\/","playable":false,"cover":"https://img3.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2460295681.jpg","id":"26787124","cover_y":1000,"is_new":false},{"rate":"7.2","cover_x":3000,"title":"亚瑟王:斗兽争霸","url":"https:\/\/movie.douban.com\/subject\/25864124\/","playable":true,"cover":"https://img1.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2453433569.jpg","id":"25864124","cover_y":4286,"is_new":false},{"rate":"7.4","cover_x":638,"title":"一锤定音","url":"https:\/\/movie.douban.com\/subject\/26383898\/","playable":false,"cover":"https://img1.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2455068697.jpg","id":"26383898","cover_y":900,"is_new":false},{"rate":"6.8","cover_x":1000,"title":"原谅他77次","url":"https:\/\/movie.douban.com\/subject\/26857793\/","playable":true,"cover":"https://img1.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2464315908.jpg","id":"26857793","cover_y":1407,"is_new":false},{"rate":"7.4","cover_x":1500,"title":"雄狮","url":"https:\/\/movie.douban.com\/subject\/26220650\/","playable":true,"cover":"https://img3.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2462515303.jpg","id":"26220650","cover_y":2115,"is_new":false},{"rate":"8.1","cover_x":1000,"title":"天才少女","url":"https:\/\/movie.douban.com\/subject\/26593587\/","playable":true,"cover":"https://img1.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2394315478.jpg","id":"26593587","cover_y":1481,"is_new":false},{"rate":"7.1","cover_x":1263,"title":"速度与激情8","url":"https:\/\/movie.douban.com\/subject\/26260853\/","playable":true,"cover":"https://img3.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2444256500.jpg","id":"26260853","cover_y":2000,"is_new":false},{"rate":"5.4","cover_x":2244,"title":"真实","url":"https:\/\/movie.douban.com\/subject\/26362764\/","playable":false,"cover":"https://img1.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2460120818.jpg","id":"26362764","cover_y":3219,"is_new":false},{"rate":"4.8","cover_x":2000,"title":"新木乃伊","url":"https:\/\/movie.douban.com\/subject\/20451290\/","playable":false,"cover":"https://img1.doubanio.com\/view\/movie_poster_cover\/lpst\/public\/p2459198108.jpg","id":"20451290","cover_y":2999,"is_new":false}]}

参考资料

python爬虫之采集搜素引擎联想词

2017-08-12 22:31 37 技术
Comments
Write a Comment