music-score/scripts/download_musicxml_spec.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
MusicXML 4.0 规范文档下载器

下载 W3C MusicXML 4.0 官方规范的所有页面，保存为本地文件供学习使用。
官方网站: https://www.w3.org/2021/06/musicxml40/
"""

import os
import re
import time
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor, as_completed

# 配置
BASE_URL = "https://www.w3.org/2021/06/musicxml40/"
OUTPUT_DIR = Path(__file__).parent.parent / "docs" / "musicxml-spec"
MAX_WORKERS = 5  # 并发下载数量
REQUEST_DELAY = 0.5  # 请求间隔（秒），避免对服务器造成压力

# 请求头
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.5",
}


def ensure_output_dir():
    """确保输出目录存在"""
    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
    (OUTPUT_DIR / "elements").mkdir(exist_ok=True)
    (OUTPUT_DIR / "data-types").mkdir(exist_ok=True)
    (OUTPUT_DIR / "examples").mkdir(exist_ok=True)
    print(f"输出目录: {OUTPUT_DIR}")


def fetch_page(url: str, retries: int = 3) -> str | None:
    """获取页面内容，带重试机制"""
    for attempt in range(retries):
        try:
            response = requests.get(url, headers=HEADERS, timeout=30)
            response.raise_for_status()
            response.encoding = 'utf-8'
            return response.text
        except requests.RequestException as e:
            print(f"  获取 {url} 失败 (尝试 {attempt + 1}/{retries}): {e}")
            if attempt < retries - 1:
                time.sleep(2)
    return None


def extract_text_content(html: str) -> str:
    """从 HTML 中提取主要文本内容，转换为 Markdown 格式"""
    soup = BeautifulSoup(html, 'html.parser')
    
    # 移除不需要的元素
    for element in soup.find_all(['script', 'style', 'nav', 'header', 'footer']):
        element.decompose()
    
    # 获取主要内容区域
    main_content = soup.find('main') or soup.find('article') or soup.find('body')
    if not main_content:
        return ""
    
    lines = []
    
    # 获取标题
    title = soup.find('title')
    if title:
        lines.append(f"# {title.get_text(strip=True)}\n")
    
    # 处理内容
    for element in main_content.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'pre', 'code', 'ul', 'ol', 'li', 'table', 'tr', 'td', 'th', 'dl', 'dt', 'dd']):
        tag_name = element.name
        text = element.get_text(strip=True)
        
        if not text:
            continue
            
        if tag_name == 'h1':
            lines.append(f"\n# {text}\n")
        elif tag_name == 'h2':
            lines.append(f"\n## {text}\n")
        elif tag_name == 'h3':
            lines.append(f"\n### {text}\n")
        elif tag_name == 'h4':
            lines.append(f"\n#### {text}\n")
        elif tag_name == 'h5':
            lines.append(f"\n##### {text}\n")
        elif tag_name == 'h6':
            lines.append(f"\n###### {text}\n")
        elif tag_name == 'p':
            lines.append(f"{text}\n")
        elif tag_name == 'pre':
            # 代码块
            code_text = element.get_text()
            lines.append(f"\n```xml\n{code_text}\n```\n")
        elif tag_name == 'li':
            lines.append(f"- {text}")
        elif tag_name == 'dt':
            lines.append(f"\n**{text}**")
        elif tag_name == 'dd':
            lines.append(f": {text}\n")
    
    return '\n'.join(lines)


def save_page(url: str, content: str, subdir: str = ""):
    """保存页面内容到文件"""
    # 从 URL 生成文件名
    parsed = urlparse(url)
    path = parsed.path.rstrip('/')
    
    if path.endswith('.html') or path.endswith('.htm'):
        filename = os.path.basename(path)
        filename = filename.rsplit('.', 1)[0] + '.md'
    elif path.endswith('/') or not os.path.basename(path):
        filename = 'index.md'
    else:
        filename = os.path.basename(path) + '.md'
    
    # 清理文件名
    filename = re.sub(r'[<>:"/\\|?*]', '_', filename)
    
    # 确定保存路径
    if subdir:
        save_dir = OUTPUT_DIR / subdir
        save_dir.mkdir(exist_ok=True)
    else:
        save_dir = OUTPUT_DIR
    
    filepath = save_dir / filename
    
    # 转换为 Markdown
    md_content = extract_text_content(content)
    
    # 添加来源信息
    header = f"---\nsource: {url}\ndownloaded: {time.strftime('%Y-%m-%d %H:%M:%S')}\n---\n\n"
    
    with open(filepath, 'w', encoding='utf-8') as f:
        f.write(header + md_content)
    
    return filepath


def get_all_links(html: str, base_url: str) -> list[tuple[str, str]]:
    """
    从页面中提取所有相关链接
    返回: [(url, 分类), ...]
    """
    soup = BeautifulSoup(html, 'html.parser')
    links = []
    seen = set()
    
    for a_tag in soup.find_all('a', href=True):
        href = a_tag['href']
        full_url = urljoin(base_url, href)
        
        # 只处理同域名下的链接
        if not full_url.startswith(BASE_URL):
            continue
        
        # 跳过锚点链接
        if '#' in href and not href.startswith('http'):
            continue
        
        # 去重
        if full_url in seen:
            continue
        seen.add(full_url)
        
        # 分类
        subdir = ""
        if '/musicxml-reference/elements/' in full_url:
            subdir = "elements"
        elif '/musicxml-reference/data-types/' in full_url:
            subdir = "data-types"
        elif '/musicxml-reference/examples/' in full_url:
            subdir = "examples"
        
        links.append((full_url, subdir))
    
    return links


def download_index_page():
    """下载首页并提取所有链接"""
    print("正在获取首页...")
    html = fetch_page(BASE_URL)
    if not html:
        print("无法获取首页，退出")
        return []
    
    # 保存首页
    save_page(BASE_URL, html)
    print("首页已保存")
    
    # 提取链接
    links = get_all_links(html, BASE_URL)
    print(f"找到 {len(links)} 个链接")
    
    return links


def download_page(url: str, subdir: str) -> tuple[str, bool, str]:
    """下载单个页面"""
    time.sleep(REQUEST_DELAY)  # 添加延迟
    
    html = fetch_page(url)
    if html:
        filepath = save_page(url, html, subdir)
        return url, True, str(filepath)
    return url, False, ""


def download_all_pages(links: list[tuple[str, str]]):
    """并发下载所有页面"""
    print(f"\n开始下载 {len(links)} 个页面...")
    
    success_count = 0
    fail_count = 0
    
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        futures = {
            executor.submit(download_page, url, subdir): url 
            for url, subdir in links
        }
        
        for i, future in enumerate(as_completed(futures), 1):
            url = futures[future]
            try:
                result_url, success, filepath = future.result()
                if success:
                    success_count += 1
                    print(f"[{i}/{len(links)}] ✓ {os.path.basename(filepath)}")
                else:
                    fail_count += 1
                    print(f"[{i}/{len(links)}] ✗ {url}")
            except Exception as e:
                fail_count += 1
                print(f"[{i}/{len(links)}] ✗ {url}: {e}")
    
    print(f"\n下载完成: 成功 {success_count}, 失败 {fail_count}")


def create_index_file():
    """创建目录索引文件"""
    index_path = OUTPUT_DIR / "README.md"
    
    content = """# MusicXML 4.0 规范文档

> 来源: https://www.w3.org/2021/06/musicxml40/
> 下载时间: {time}

## 目录结构

- `elements/` - 元素定义（所有 MusicXML 元素的详细说明）
- `data-types/` - 数据类型定义
- `examples/` - 示例文件

## 主要文档

请从 `index.md` 开始阅读。

## 元素列表

""".format(time=time.strftime('%Y-%m-%d %H:%M:%S'))
    
    # 列出所有元素文件
    elements_dir = OUTPUT_DIR / "elements"
    if elements_dir.exists():
        element_files = sorted(elements_dir.glob("*.md"))
        for f in element_files:
            name = f.stem
            content += f"- [{name}](elements/{f.name})\n"
    
    with open(index_path, 'w', encoding='utf-8') as f:
        f.write(content)
    
    print(f"\n索引文件已创建: {index_path}")


def main():
    """主函数"""
    print("=" * 60)
    print("MusicXML 4.0 规范文档下载器")
    print("=" * 60)
    print()
    
    # 创建输出目录
    ensure_output_dir()
    
    # 下载首页并获取所有链接
    links = download_index_page()
    
    if not links:
        print("未找到任何链接，尝试下载主要页面...")
        # 手动添加一些已知的重要页面
        links = [
            (BASE_URL + "musicxml-reference/", ""),
            (BASE_URL + "musicxml-reference/elements/", ""),
            (BASE_URL + "musicxml-reference/data-types/", ""),
            (BASE_URL + "tutorial/introduction/", ""),
        ]
    
    # 下载所有页面
    download_all_pages(links)
    
    # 创建索引
    create_index_file()
    
    print("\n" + "=" * 60)
    print(f"所有文档已保存到: {OUTPUT_DIR}")
    print("=" * 60)


if __name__ == "__main__":
    main()