用 Python 写第一个 Web 爬虫

python3 --version
pip3 --version

mkdir crawler-tutorial && cd crawler-tutorial
python3 -m venv .venv
source .venv/bin/activate

pip install requests beautifulsoup4

python3 -c "import requests, bs4; print(requests.__version__, bs4.__version__)"
pip freeze > requirements.txt

crawler-tutorial/
├── .venv/             # 虚拟环境（不提交到 git）
├── requirements.txt   # 依赖清单
├── crawler.py         # 主脚本（先留空）
└── data/              # 输出目录（先建好）
    └── quotes.csv     # 之后爬下来的数据

import requests

url = "https://quotes.toscrape.com/"
resp = requests.get(url)
print(resp.status_code)

print("状态码:", resp.status_code, resp.reason)
print("编码:", resp.encoding)
print("响应头 Content-Type:", resp.headers.get("Content-Type"))
print("响应体前 200 个字符:\n", resp.text[:200])

状态码: 200 OK
编码: ISO-8859-1
响应头 Content-Type: text/html; charset=utf-8
响应体前 200 个字符:

headers = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/126.0 Safari/537.36",
}

resp = requests.get(url, headers=headers, timeout=10)
print("服务器看到的 UA:", resp.request.headers["User-Agent"])

<div class="quote">
  <span class="text">“The world as we have created it is a process of our thinking...”</span>
  <span>by <small class="author">Albert Einstein</small></span>
</div>

from bs4 import BeautifulSoup

soup = BeautifulSoup(resp.text, "html.parser")

first_quote = soup.find("div", class_="quote")
print(first_quote.text[:120])

# 写法一：find_all 按标签 + class
quotes = soup.find_all("div", class_="quote")
print("用 find_all 找到", len(quotes), "条")

# 写法二：select 用 CSS 选择器（效果相同）
quotes2 = soup.select("div.quote")
print("用 select 找到", len(quotes2), "条")

quote = quotes[0]

text = quote.find("span", class_="text").text
author = quote.find("small", class_="author").text
tags = [tag.text for tag in quote.find_all("a", class_="tag")]

print("名言:", text)
print("作者:", author)
print("标签:", tags)

名言: “The world as we have created it is a process of our thinking...”  # 截断示意
作者: Albert Einstein
标签: ['change', 'deep-thoughts', 'thinking', 'world']

def clean_text(s):
    """去掉首尾空白并剥掉中文/英文引号"""
    s = s.strip()
    s = s.strip("\u201c\u201d\u2018\u2019\"'")
    return s.strip()

raw = "“The world as we have created it...”"
print(clean_text(raw))

import csv

rows = []                      # 每行一个 dict：{"text": ..., "author": ..., "tags": ...}
for q in quotes:
    rows.append({
        "text": clean_text(q.find("span", class_="text").text),
        "author": q.find("small", class_="author").text.strip(),
        "tags": ", ".join(t.text for t in q.find_all("a", class_="tag")),
    })

with open("data/quotes.csv", "w", encoding="utf-8", newline="") as f:
    writer = csv.DictWriter(f, fieldnames=["text", "author", "tags"])
    writer.writeheader()
    writer.writerows(rows)

print("已写入", len(rows), "行")

text,author,tags
The world as we have created it is a process of our thinking...,Albert Einstein,"change, deep-thoughts, thinking, world"
It is our choices, Harry, that show what we truly are...,J.K. Rowling,"abilities, choices"

def append_rows(filepath, rows):
    write_header = not Path(filepath).exists()
    with open(filepath, "a", encoding="utf-8", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=["text", "author", "tags"])
        if write_header:
            writer.writeheader()
        writer.writerows(rows)

import requests

for page in range(1, 4):
    url = f"https://quotes.toscrape.com/page/{page}/"
    resp = requests.get(url, headers=headers, timeout=10)
    print(f"第 {page} 页 -> {resp.status_code}, 长度 {len(resp.text)}")

from bs4 import BeautifulSoup

all_rows = []
for page in range(1, 11):
    resp = requests.get(f"https://quotes.toscrape.com/page/{page}/",
                        headers=headers, timeout=10)
    if resp.status_code != 200:
        print(f"第 {page} 页失败: {resp.status_code}")
        continue

    soup = BeautifulSoup(resp.text, "html.parser")
    for q in soup.select("div.quote"):
        all_rows.append({
            "text": clean_text(q.select_one(".text").text),
            "author": q.select_one(".author").text.strip(),
            "tags": ", ".join(t.text for t in q.select("a.tag")),
        })

print("共抓到", len(all_rows), "条")

url = "https://quotes.toscrape.com/"
while url:                       # 下一页为空时退出
    resp = requests.get(url, headers=headers, timeout=10)
    soup = BeautifulSoup(resp.text, "html.parser")

    next_link = soup.select_one("li.next a")
    url = "https://quotes.toscrape.com" + next_link["href"] if next_link else None
    print("抓取:", url)

import time

def get_with_retry(url, max_retries=3):
    for attempt in range(max_retries):
        try:
            resp = requests.get(url, headers=headers, timeout=10)
            if resp.status_code == 200:
                return resp
        except requests.RequestException:
            pass
        time.sleep(2 ** attempt)      # 指数退避：2s, 4s, 8s
    return None

resp = get_with_retry("https://quotes.toscrape.com/page/2/")
print("结果:", resp.status_code if resp else "多次重试仍失败")

curl -s https://quotes.toscrape.com/robots.txt
# User-agent: *
# Allow: /page/*
# Disallow: /

BASE_URL = "https://quotes.toscrape.com"
HEADERS = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) ..."}
DELAY   = 1.0     # 每页之间延时（秒）
MAX_RETRIES = 3   # 最大重试次数
OUT_CSV = "data/quotes.csv"

import csv
import time
from pathlib import Path
import requests
from bs4 import BeautifulSoup

BASE_URL = "https://quotes.toscrape.com"
HEADERS = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
                         "AppleWebKit/537.36 (KHTML, like Gecko) "
                         "Chrome/126.0 Safari/537.36"}
DELAY = 1.0
MAX_RETRIES = 3
OUT_CSV = "data/quotes.csv"
FIELD_NAMES = ["text", "author", "tags"]


def clean_text(s):
    """去掉首尾空白并剥掉中文/英文引号"""
    s = s.strip().strip("\u201c\u201d\u2018\u2019\"'")
    return s.strip()


def get_with_retry(url, max_retries=MAX_RETRIES):
    """带指数退避的请求，失败返回 None"""
    for attempt in range(max_retries):
        try:
            resp = requests.get(url, headers=HEADERS, timeout=10)
            if resp.status_code == 200:
                return resp
        except requests.RequestException:
            pass
        time.sleep(2 ** attempt)
    return None


def parse_quotes(html):
    """从一页 HTML 中提取名言列表"""
    soup = BeautifulSoup(html, "html.parser")
    rows = []
    for q in soup.select("div.quote"):
        rows.append({
            "text": clean_text(q.select_one(".text").text),
            "author": q.select_one(".author").text.strip(),
            "tags": ", ".join(t.text for t in q.select("a.tag")),
        })
    return rows


def append_rows(rows):
    """追加写入 CSV，文件不存在时先写表头"""
    write_header = not Path(OUT_CSV).exists()
    Path(OUT_CSV).parent.mkdir(parents=True, exist_ok=True)
    with open(OUT_CSV, "a", encoding="utf-8", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=FIELD_NAMES)
        if write_header:
            writer.writeheader()
        writer.writerows(rows)


def crawl():
    """从第一页开始，跟着『下一页』一直抓到没有为止"""
    url = BASE_URL + "/page/1/"
    total = 0
    while url:
        resp = get_with_retry(url)
        if resp is None:
            print("跳过:", url)
            break

        rows = parse_quotes(resp.text)
        append_rows(rows)
        total += len(rows)
        print(f"已抓 {len(rows)} 条（{url}），累计 {total}")

        soup = BeautifulSoup(resp.text, "html.parser")
        next_link = soup.select_one("li.next a")
        url = BASE_URL + next_link["href"] if next_link else None
        time.sleep(DELAY)
    print(f"完成，共 {total} 条，写入 {OUT_CSV}")


if __name__ == "__main__":
    crawl()

python3 crawler.py

已抓 10 条（https://quotes.toscrape.com/page/1/），累计 10
已抓 10 条（https://quotes.toscrape.com/page/2/），累计 20
...
完成，共 100 条，写入 data/quotes.csv