Python 爬取 gank.io 今日力推 - DoooReynの星际航行日志

Python 爬取 gank.io 今日力推¶

这是入门级的基础训练,没用使用什么框架,所以代码很简单,就不说了,直接贴出来:

# -*- encoding=utf-8 -*-

import sys, re, os
import urllib
import urllib.request as request
from collections import deque

def scrapy_url(start_url):
    cnt = 0
    visited = set()
    queue = deque()
    queue.append(start_url)
    tryagain = set()
    log = list()


    while True:
        if len(queue) == 0:
            print('no url to scrapy. well done!')
            break
        url = queue.popleft()
        visited |= {url}

        print('已经抓取: ' + str(cnt))
        print('  正在抓取: ' + url)
        cnt += 1

        try:
            resp = request.urlopen(url, timeout=100)
            if 'html' not in resp.getheader('Content-Type'):
                continue
        except:
            if url not in tryagain:
                tryagain |= {url}
                queue.append(url)
            print('  open "{0}" err'.format(url))
            continue

        try:
            data = resp.read().decode('utf-8')
        except:
            if url not in tryagain:
                tryagain |= {url}
                queue.append(url)
            print('  "{0}" decode err'.format(url))
            continue

        linkre = re.compile('href=\"(/\d+/\d+/\d+)\">(.+?)</a>')
        for section in linkre.findall(data):
            real_url = start_url+section[0]
            if real_url not in visited and real_url not in queue:
                queue.append(real_url)
                print('  加入队列: [%s]' % (start_url+section[0]))
                log.append('[%s](%s)' % (section[1], start_url+section[0]) )

    write_path = os.path.join(os.getcwd(), 'log.md')
    with open( write_path, 'wt', encoding='utf-8' ) as f:
        lines = [ str(row+1)+'. '+line+'\n' for row, line in enumerate(log)]
        f.writelines(lines)


if __name__ == '__main__':
    start_url = 'http://gank.io'
    scrapy_url(start_url)

生成的Markdown结果如下:

Keys	Action
`?`	Open this help
`n`	Next page
`p`	Previous page
`s`	Search