行政院本院新聞爬蟲：使用分頁抓多頁新聞

```在原本的「列表頁 → 詳細頁」外面，再加一層分頁迴圈```
import time

import requests
import pandas as pd
from bs4 import BeautifulSoup
from urllib.parse import urljoin


### 設定網址
base_url = "https://www.ey.gov.tw"
list_url = "https://www.ey.gov.tw/Page/6485009ABEC1CB9C"

headers = {
    "User-Agent": "Mozilla/5.0"
}


### 設定抓取範圍
start_page = 1 # 從第幾頁開始
end_page = 3 # 抓到第幾頁
page_size = 15 # 每頁新聞數量


### 建立 Session
session = requests.Session() # 多次請求時共用連線
session.headers.update(headers)


### 抓取新聞詳細頁內文
def get_news_content(news_url):
    try:
        response = session.get(
            news_url,
            timeout=10
        ) # 進入新聞詳細頁

        response.raise_for_status()

        soup = BeautifulSoup(
            response.text,
            "lxml"
        )

        content_element = soup.select_one(
            ".words_content .data_left"
        ) # 找到新聞主要內文區域

        if not content_element:
            return ""

        paragraphs = content_element.select("p") # 取得所有內文段落

        content_list = []

        for paragraph in paragraphs:
            text = paragraph.get_text(
                " ",
                strip=True
            ) # 取得段落文字並移除多餘空白

            if text:
                content_list.append(text)

        content = "\n\n".join(content_list) # 每個段落之間空一行

        return content

    except requests.exceptions.RequestException as error:
        print("詳細頁抓取失敗：", news_url)
        print("錯誤訊息：", error)

        return ""


### 儲存所有頁面的新聞
news_list = []


### 逐頁抓取新聞列表
for page in range(start_page, end_page + 1):
    print(f"\n正在抓取第 {page} 頁")

    params = {
        "page": page,
        "PS": page_size
    } # 設定分頁參數

    try:
        response = session.get(
            list_url,
            params=params,
            timeout=10
        ) # 取得指定頁面的新聞列表

        response.raise_for_status()

    except requests.exceptions.RequestException as error:
        print(f"第 {page} 頁抓取失敗")
        print("錯誤訊息：", error)

        continue # 跳過失敗頁面，繼續抓下一頁

    print("列表頁網址：", response.url)
    print("狀態碼：", response.status_code)

    soup = BeautifulSoup(
        response.text,
        "lxml"
    )


    ### 找到這一頁的所有新聞
    news_items = soup.select(
        "ul.list-group-item > li"
    )

    print("本頁新聞數量：", len(news_items))

    if not news_items:
        print("這一頁沒有新聞，停止抓取")
        break


    ### 逐則處理新聞
    for item_index, item in enumerate(news_items, start=1):
        title_element = item.select_one(".title")
        date_element = item.select_one(".date")
        summary_element = item.select_one("p")
        link_element = item.select_one("a")

        if not title_element or not date_element or not link_element:
            continue

        title = title_element.get_text(
            " ",
            strip=True
        )

        date = date_element.get_text(
            strip=True
        )

        if summary_element:
            summary = summary_element.get_text(
                " ",
                strip=True
            )
        else:
            summary = ""

        relative_url = link_element.get("href")

        news_url = urljoin(
            base_url,
            relative_url
        ) # 將相對網址轉換成完整網址

        print(
            f"第 {page} 頁，第 {item_index} 則：{title}"
        )

        content = get_news_content(
            news_url
        ) # 進入詳細頁抓取完整內文

        news_list.append({
            "頁數": page,
            "標題": title,
            "日期": date,
            "摘要": summary,
            "完整內文": content,
            "網址": news_url
        })

        time.sleep(1) # 每則新聞暫停 1 秒


    ### 每抓完一頁先儲存一次
    news_df = pd.DataFrame(news_list)

    news_df.to_csv(
        "行政院多頁新聞.csv",
        index=False,
        encoding="utf-8-sig"
    ) # 中途儲存，避免程式意外中斷後資料全部消失

    print(f"第 {page} 頁完成，目前共抓到 {len(news_list)} 則新聞")

    time.sleep(2) # 每抓完一頁暫停 2 秒


### 顯示抓取結果
news_df = pd.DataFrame(news_list)

print("\n新聞資料：")
print(news_df[["頁數", "標題", "日期", "網址"]])


### 最終儲存
news_df.to_csv(
    "行政院多頁新聞.csv",
    index=False,
    encoding="utf-8-sig"
)


### 執行完成
print("\n資料抓取完成")
print("抓取頁數：", start_page, "到", end_page)
print("共抓到：", len(news_list), "則新聞")
print("已產生：行政院多頁新聞.csv")