.agents/skills/core_specification/scripts/downloader.py

# Copyright 2026 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

"""Downloads and formats Bluetooth core specification documents."""

import os
import re
import sys
from typing import Optional
from urllib.request import urlopen, Request
from urllib.error import URLError
from urllib.parse import urlparse
from bs4 import BeautifulSoup, Tag

# Root project directory for specification cache
SPEC_DIR = os.path.join(
    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
    "specifications",
)

# Hardcoded list of official Bluetooth Core Specification URLs
CORE_SPEC_URLS = [
    (
        "https://www.bluetooth.com/wp-content/uploads/Files/Specification/"
        "HTML/Core-62/out/en/host-controller-interface/"
        "host-controller-interface-functional-specification.html"
    ),
    (
        "https://www.bluetooth.com/wp-content/uploads/Files/Specification/"
        "HTML/Core-62/out/en/host/"
        "logical-link-control-and-adaptation-protocol-specification.html"
    ),
    (
        "https://www.bluetooth.com/wp-content/uploads/Files/Specification/"
        "HTML/Core-62/out/en/host/attribute-protocol--att-.html"
    ),
    (
        "https://www.bluetooth.com/wp-content/uploads/Files/Specification/"
        "HTML/Core-62/out/en/host/generic-access-profile.html"
    ),
    (
        "https://www.bluetooth.com/wp-content/uploads/Files/Specification/"
        "HTML/Core-62/out/en/host/generic-attribute-profile--gatt-.html"
    ),
    (
        "https://www.bluetooth.com/wp-content/uploads/Files/Specification/"
        "HTML/Core-62/out/en/host/security-manager-specification.html"
    ),
    (
        "https://www.bluetooth.com/wp-content/uploads/Files/Specification/"
        "HTML/Core-62/out/en/br-edr-controller/"
        "link-manager-protocol-specification.html"
    ),
    (
        "https://www.bluetooth.com/wp-content/uploads/Files/Specification/"
        "HTML/Core-62/out/en/low-energy-controller/"
        "link-layer-specification.html"
    ),
]


def fetch_html(url: str) -> Optional[str]:
    """Downloads HTML content from the given URL."""
    print(f"Downloading: {url}...")
    headers = {'User-Agent': 'Mozilla/5.0'}
    try:
        req = Request(url, headers=headers)
        with urlopen(req, timeout=30) as response:
            return response.read().decode('utf-8')
    except URLError as e:
        print(f"Error downloading {url}: {e}")
        return None


def extract_metadata_header(soup: BeautifulSoup, url: str) -> str:
    """Extracts the document title and formats an initial metadata header."""
    meta_header = f"Source URL: {url}\n"
    title_div = soup.find('div', class_='titlepage')

    if title_div:
        title_element = title_div.find(class_='title')
        raw_title_source = title_element if title_element else title_div
        raw_title = raw_title_source.get_text(separator=' ', strip=True)
        # Remove internal SIG codenames/revisions like 'vAtlanta r00'
        clean_title = re.sub(r'\bv[A-Za-z]+\s+r\d{2,}\b', '', raw_title)
        # Clean up any double spaces left behind
        clean_title = re.sub(r'\s{2,}', ' ', clean_title).strip()

        meta_header += clean_title + "\n"
        meta_header += "=" * max(10, len(clean_title)) + "\n\n"

    return meta_header


def stitch_headers(content_root: Tag) -> None:
    """Combines section numbers and titles into single Markdown headers."""
    for header in content_root.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']):
        num = header.find("span", class_="formal-number")
        title = header.find("span", class_="formal-title")

        if num and title:
            # Prefix with Markdown header hashes based on heading level
            level = int(header.name[1])
            hashes = "#" * level
            stitched = (
                f"{hashes} {num.get_text().strip()}. {title.get_text().strip()}"
            )
            header.clear()
            header.append(stitched)


def convert_table_to_markdown(table: Tag) -> str:
    """Converts a BeautifulSoup table element into a Markdown table string."""
    rows = []
    max_cols = 0
    for tr in table.find_all('tr'):
        cells = []
        for cell in tr.find_all(['th', 'td']):
            # Clean up cell text: remove extra newlines/spaces and escape pipes
            text = (
                cell.get_text(separator=' ', strip=True)
                .replace('|', '\\|')
                .replace('\n', ' ')
            )
            cells.append(text)

        max_cols = max(max_cols, len(cells))
        if cells:
            rows.append(f"| {' | '.join(cells)} |")

    if not rows:
        return ""

    # Create the separator row
    if max_cols > 0:
        separator = f"|{'---|' * max_cols}"
        # Insert after header if possible
        if len(rows) > 1:
            rows.insert(1, separator)
        else:
            rows.append(separator)

    return '\n\n' + '\n'.join(rows) + '\n\n'


def replace_tables_with_markdown(
    content_root: Tag, soup: BeautifulSoup
) -> None:
    """Finds all tables in the HTML and replaces them with Markdown text."""
    for table in content_root.find_all('table'):
        md_table = convert_table_to_markdown(table)
        table.replace_with(soup.new_string(md_table))


def extract_clean_text(meta_header: str, content_root: Tag) -> str:
    """Extracts text content and cleans up excessive newlines."""
    text_content = content_root.get_text(separator='\n', strip=True)
    # Remove internal SIG codenames/revisions like 'vAtlanta r00'
    # (on its own line)
    text_content = re.sub(r'(?im)^v[a-z]+\s+r\d{2,}\n?$', '', text_content)
    full_text = meta_header + text_content
    return re.sub(r'\n{3,}', '\n\n', full_text)


def process_spec(url: str) -> None:
    """
    Coordinates the process of downloading, parsing, and saving a specification.
    """
    sys.setrecursionlimit(10000)

    # Determine filenames
    parsed_url = urlparse(url)
    page_name = os.path.basename(parsed_url.path).replace('.html', '')
    raw_path = os.path.join(SPEC_DIR, f"{page_name}.html")
    pretty_path = os.path.join(SPEC_DIR, f"{page_name}_pretty.html")
    md_path = os.path.join(SPEC_DIR, f"{page_name}.md")

    # Ensure output directory exists
    os.makedirs(SPEC_DIR, exist_ok=True)

    if (
        os.path.exists(md_path)
        and os.path.exists(raw_path)
        and os.path.exists(pretty_path)
    ):
        print(f"Skipping cached spec: {page_name}")
        return

    html_content = fetch_html(url)
    if not html_content:
        return
    # Save raw HTML
    with open(raw_path, 'w', encoding='utf-8') as f:
        f.write(html_content)

    soup = BeautifulSoup(html_content, 'html.parser')

    # Save prettified HTML
    with open(pretty_path, 'w', encoding='utf-8') as f:
        f.write(soup.prettify())

    meta_header = extract_metadata_header(soup, url)

    main_article = soup.find('article', class_='topic')
    content_root = main_article if main_article else soup

    stitch_headers(content_root)
    replace_tables_with_markdown(content_root, soup)
    text_content = extract_clean_text(meta_header, content_root)

    # Save Markdown
    with open(md_path, 'w', encoding='utf-8') as f:
        f.write(text_content)
    print(f"Generated enhanced Markdown: {md_path}")


def generate_index() -> None:
    """Scans all .md files and generates a global index.md."""
    print("\nGenerating Section Index...")
    index_entries = []

    # Regex to find lines starting with section numbers (prefixed with hashes)
    section_pattern = re.compile(r'^(#+)\s+(\d+(\.\d+)*\.)\s+(.+)$')

    for filename in sorted(os.listdir(SPEC_DIR)):
        if filename.endswith(".md") and filename != "index.md":
            filepath = os.path.join(SPEC_DIR, filename)
            spec_name = ""

            with open(filepath, 'r', encoding='utf-8') as f:
                for line_num, line in enumerate(f, 1):
                    stripped = line.strip()
                    # The second line is the spec title
                    if line_num == 2 and not spec_name:
                        spec_name = stripped

                    match = section_pattern.match(stripped)
                    if match:
                        section_num = match.group(2)
                        title = match.group(4)
                        index_entries.append(
                            f"| {section_num} | {title} | {spec_name} | "
                            f"{filename}:{line_num} |"
                        )

    index_path = os.path.join(SPEC_DIR, "index.md")
    with open(index_path, 'w', encoding='utf-8') as f:
        f.write("# Bluetooth Core Specification Index\n\n")
        f.write("| Section | Title | Specification | File:Line |\n")
        f.write("|---|---|---|---|\n")

        # Remove duplicates while keeping order
        seen = set()
        for entry in index_entries:
            if entry not in seen:
                f.write(entry + "\n")
                seen.add(entry)

    print(f"Index created: {index_path}")


if __name__ == '__main__':
    for spec_url in CORE_SPEC_URLS:
        process_spec(spec_url)

    generate_index()
    print("\nAll core specifications synchronized and indexed successfully.")