.agents/skills/le_profiles_specifications/scripts/downloader.py

import os
import re
from urllib.request import urlopen, Request
from urllib.error import URLError
from bs4 import BeautifulSoup

SPEC_DIR = os.path.join(
    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
    "specifications",
)

ROOT_URL = "https://www.bluetooth.com/wp-content/uploads/Files/Specification/HTML/"

# Targets to download
TARGETS = ['BAP', 'CAP', 'TMAP', 'HAP', 'PBP', 'MCP', 'CCP', 'MICP', 'VCP', 'ASCS', 'PACS', 'CSIS', 'CSIP', 'BASS', 'AICS']

def get_all_index_links():
    print(f"Fetching directory listing from {ROOT_URL}...")
    headers = {'User-Agent': 'Mozilla/5.0'}
    try:
        req = Request(ROOT_URL, headers=headers)
        with urlopen(req, timeout=30) as response:
            html_content = response.read()
            soup = BeautifulSoup(html_content, 'html.parser')
            links = []
            for a in soup.find_all('a'):
                href = a.get('href', '')
                if "index-en.html" in href:
                    links.append(href)
            return links
    except URLError as e:
        print(f"Error fetching root directory: {e}")
        return []

def match_target(path, acronym):
    pattern = re.compile(rf"(?:^|/|_|-)({acronym})(?:[._-]|html5)", re.IGNORECASE)
    return bool(pattern.search(path))

def parse_version(path, acronym):
    pattern = re.compile(rf"{acronym}_v(\d+(?:[.-]\d+)*)", re.IGNORECASE)
    match = pattern.search(path)
    if match:
        return match.group(1).replace('-', '.')
    return "0.0"

def get_latest_link(links, acronym):
    target_links = [l for l in links if match_target(l, acronym)]
    if not target_links:
        return None

    timestamp_pattern = re.compile(r"_(\d{10})/")

    def sort_key(path):
        version_str = parse_version(path, acronym)
        version_parts = []
        for x in version_str.split('.'):
            try:
                version_parts.append(int(x))
            except ValueError:
                version_parts.append(0)

        timestamp_match = timestamp_pattern.search(path)
        timestamp = int(timestamp_match.group(1)) if timestamp_match else 0

        return (version_parts, timestamp)

    sorted_links = sorted(target_links, key=sort_key, reverse=True)
    return sorted_links[0]

def download_spec(name, relative_url):
    url = ROOT_URL + relative_url
    print(f"Downloading {name} from {url}...")
    headers = {'User-Agent': 'Mozilla/5.0'}
    try:
        req = Request(url, headers=headers)
        with urlopen(req, timeout=30) as response:
            html_content = response.read()

            # Create spec dir if not exists
            os.makedirs(SPEC_DIR, exist_ok=True)

            # Save HTML
            html_path = os.path.join(SPEC_DIR, f"{name}.html")
            with open(html_path, 'wb') as f:
                f.write(html_content)
            print(f"Saved HTML to {html_path}")

            # Save Text/Markdown for searching
            soup = BeautifulSoup(html_content, 'html.parser')
            text_content = soup.get_text(separator='\n', strip=True)

            md_path = os.path.join(SPEC_DIR, f"{name}.md")
            with open(md_path, 'w', encoding='utf-8') as f:
                f.write(text_content)
            print(f"Saved Markdown to {md_path}")

    except URLError as e:
        print(f"Error downloading {name}: {e}")
    except Exception as e:
        print(f"An error occurred: {e}")

def main():
    links = get_all_index_links()
    if not links:
        print("No links found.")
        return

    for target in TARGETS:
        latest_link = get_latest_link(links, target)
        if latest_link:
            version = parse_version(latest_link, target)
            name = f"{target}_v{version}" if version != "0.0" else target
            if version == "0.0":
                match = re.search(rf"(\d+)-{target}-html5", latest_link, re.IGNORECASE)
                if match:
                    name = f"{target}_ID_{match.group(1)}"

            download_spec(name, latest_link)
        else:
            print(f"No link found for {target}")

if __name__ == "__main__":
    main()