简单的爬虫示例.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from lxml import html\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_content(url, pattern):\n",
    "    page=requests.Session().get(url) \n",
    "    tree=html.fromstring(page.text) \n",
    "    result=tree.xpath(pattern) #获取需要的数据\n",
    "    result=json.dumps(result, ensure_ascii=False) \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"升级\", \"祈祷落幕时\", \"遗传厄运\", \"蚁人2：黄蜂女现身\", \"大三儿\", \"犬猿\", \"今夜在浪漫剧场\", \"窒息\", \"瞒天过海：美人计\", \"致所有我曾爱过的男孩\"]\n"
     ]
    }
   ],
   "source": [
    "url='https://movie.douban.com/' #需要爬数据的网址\n",
    "pattern = '//td[@class=\"title\"]//a/text()'\n",
    "result = get_content(url, pattern)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"蚁人2：黄蜂...\", \"大师兄\", \"一出好戏\", \"巨齿鲨\", \"精灵旅社3：...\", \"快把我哥带走...\", \"西虹市首富\", \"深海历险记\", \"黑子的篮球·...\", \"小偷家族\", \"奇怪的袜子精...\", \"大三儿\", \"神秘世界历险...\", \"新乌龙院之笑...\", \"我不是药神\", \"欧洲攻略\", \"信仰者\", \"风语咒\", \"狄仁杰之四大...\", \"惊慌失色之诡...\", \"爸，我一定行...\", \"的士速递5\", \"爱情公寓\", \"最后的棒棒\", \"解码游戏\", \"妈妈咪呀2\", \"天下第一镖局...\", \"新大头儿子和...\", \"长征\", \"我们诞生在中...\", \"动物世界\", \"三十那年\", \"熊猫回家路\", \"心迷宫\", \"疯狂暑期之哈...\", \"灰姑娘\", \"李保国\", \"茶花女\"]\n"
     ]
    }
   ],
   "source": [
    "url='https://movie.douban.com/' #需要爬数据的网址\n",
    "pattern = '//li[@class=\"title\"]//a/text()'\n",
    "result = get_content(url, pattern)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}