diff --git a/docs.py b/docs.py index a21d306aa..167a3ae4d 100755 --- a/docs.py +++ b/docs.py @@ -8,7 +8,6 @@ import json import logging import os -import re import shutil import subprocess from functools import lru_cache @@ -42,240 +41,240 @@ def language_docs_dir(lang: str) -> Path: - return docs_path / lang / "docs" + return docs_path / lang / "docs" def get_missing_translation_snippet() -> str: - missing_translation_file_path = (Path(__file__).parent / "help/missing-translation.md") - return missing_translation_file_path.read_text(encoding="utf-8") + missing_translation_file_path = (Path(__file__).parent / "help/missing-translation.md") + return missing_translation_file_path.read_text(encoding="utf-8") def get_mkdocs_yaml_for_lang(lang: str) -> str: - return dedent(f""" - INHERIT: ../../mkdocs.yml - docs_dir: './docs' - theme: - custom_dir: ../../overrides - language: {lang} - """).lstrip() + return dedent(f""" + INHERIT: ../../mkdocs.yml + docs_dir: './docs' + theme: + custom_dir: ../../overrides + language: {lang} + """).lstrip() def get_alternate_langs_config() -> Dict[str, Any]: - return mkdocs.utils.yaml_load(alternate_langs_config_path.read_text(encoding="utf-8")) + return mkdocs.utils.yaml_load(alternate_langs_config_path.read_text(encoding="utf-8")) def get_lang_paths() -> List[Path]: - return sorted(docs_path.iterdir()) + return sorted(docs_path.iterdir()) def lang_callback(lang: Optional[str]) -> Union[str, None]: - if lang is None: - return None - lang = lang.lower() - return lang + if lang is None: + return None + lang = lang.lower() + return lang def complete_existing_lang(incomplete: str): - lang_path: Path - for lang_path in get_lang_paths(): - if lang_path.is_dir() and lang_path.name.startswith(incomplete): - yield lang_path.name + lang_path: Path + for lang_path in get_lang_paths(): + if lang_path.is_dir() and lang_path.name.startswith(incomplete): + yield lang_path.name @app.command() def new_lang(lang: str = typer.Argument(..., callback=lang_callback)): - """ - Generate a new docs translation directory for the language LANG. - """ - new_path: Path = docs_path / lang - if new_path.exists(): - typer.echo(f"The language was already created: {lang}") - raise typer.Abort() - new_path.mkdir() - new_config_path: Path = Path(new_path) / alternate_langs_config_name - new_config_path.write_text(get_mkdocs_yaml_for_lang(lang), encoding="utf-8") - new_config_docs_path: Path = language_docs_dir(lang) - new_config_docs_path.mkdir() - en_index_path: Path = language_docs_dir('en') / "index.md" - new_index_path: Path = new_config_docs_path / "index.md" - en_index_content = en_index_path.read_text(encoding="utf-8") - new_index_content = f"{get_missing_translation_snippet()}\n\n{en_index_content}" - new_index_path.write_text(new_index_content, encoding="utf-8") - typer.secho(f"Successfully initialized: {new_path}", color=typer.colors.GREEN) - update_languages() + """ + Generate a new docs translation directory for the language LANG. + """ + new_path: Path = docs_path / lang + if new_path.exists(): + typer.echo(f"The language was already created: {lang}") + raise typer.Abort() + new_path.mkdir() + new_config_path: Path = Path(new_path) / alternate_langs_config_name + new_config_path.write_text(get_mkdocs_yaml_for_lang(lang), encoding="utf-8") + new_config_docs_path: Path = language_docs_dir(lang) + new_config_docs_path.mkdir() + en_index_path: Path = language_docs_dir('en') / "index.md" + new_index_path: Path = new_config_docs_path / "index.md" + en_index_content = en_index_path.read_text(encoding="utf-8") + new_index_content = f"{get_missing_translation_snippet()}\n\n{en_index_content}" + new_index_path.write_text(new_index_content, encoding="utf-8") + typer.secho(f"Successfully initialized: {new_path}", color=typer.colors.GREEN) + update_languages() @app.command() def build_lang( - lang: str = typer.Argument( - ..., callback=lang_callback, autocompletion=complete_existing_lang - ), + lang: str = typer.Argument( + ..., callback=lang_callback, autocompletion=complete_existing_lang + ), ) -> None: - """ - Build the docs for a language. - """ - lang_path: Path = docs_path / lang - if not lang_path.is_dir(): - typer.echo(f"The language translation doesn't seem to exist yet: {lang}") - raise typer.Abort() - typer.echo(f"Building docs for: {lang}") - build_site_dist_path = build_site_path / lang - if lang == "en": - dist_path = site_path - # Don't remove en dist_path as it might already contain other languages. - # When running build_all(), that function already removes site_path. - # All this is only relevant locally, on GitHub Actions all this is done through - # artifacts and multiple workflows, so it doesn't matter if directories are - # removed or not. - else: - dist_path = site_path / lang - shutil.rmtree(dist_path, ignore_errors=True) - current_dir = os.getcwd() - os.chdir(lang_path) - shutil.rmtree(build_site_dist_path, ignore_errors=True) - subprocess.run(["mkdocs", "build", "--site-dir", build_site_dist_path], check=True) - shutil.copytree(build_site_dist_path, dist_path, dirs_exist_ok=True) - os.chdir(current_dir) - typer.secho(f"Successfully built docs for: {lang}", color=typer.colors.GREEN) + """ + Build the docs for a language. + """ + lang_path: Path = docs_path / lang + if not lang_path.is_dir(): + typer.echo(f"The language translation doesn't seem to exist yet: {lang}") + raise typer.Abort() + typer.echo(f"Building docs for: {lang}") + build_site_dist_path = build_site_path / lang + if lang == "en": + dist_path = site_path + # Don't remove en dist_path as it might already contain other languages. + # When running build_all(), that function already removes site_path. + # All this is only relevant locally, on GitHub Actions all this is done through + # artifacts and multiple workflows, so it doesn't matter if directories are + # removed or not. + else: + dist_path = site_path / lang + shutil.rmtree(dist_path, ignore_errors=True) + current_dir = os.getcwd() + os.chdir(lang_path) + shutil.rmtree(build_site_dist_path, ignore_errors=True) + subprocess.run(["mkdocs", "build", "--site-dir", build_site_dist_path], check=True) + shutil.copytree(build_site_dist_path, dist_path, dirs_exist_ok=True) + os.chdir(current_dir) + typer.secho(f"Successfully built docs for: {lang}", color=typer.colors.GREEN) @app.command() def build_all() -> None: - """ - Build mkdocs site for en, and then build each language inside, end result is located - at directory ./site/ with each language inside. - """ - update_languages() - shutil.rmtree(site_path, ignore_errors=True) - langs = [lang.name for lang in get_lang_paths() if lang.is_dir()] - cpu_count = os.cpu_count() or 1 - process_pool_size = cpu_count * 4 - typer.echo(f"Using process pool size: {process_pool_size}") - with Pool(process_pool_size) as p: - p.map(build_lang, langs) + """ + Build mkdocs site for en, and then build each language inside, end result is located + at directory ./site/ with each language inside. + """ + update_languages() + shutil.rmtree(site_path, ignore_errors=True) + langs = [lang.name for lang in get_lang_paths() if lang.is_dir()] + cpu_count = os.cpu_count() or 1 + process_pool_size = cpu_count * 4 + typer.echo(f"Using process pool size: {process_pool_size}") + with Pool(process_pool_size) as p: + p.map(build_lang, langs) @app.command() def update_languages() -> None: - """ - Update the mkdocs.yml file Languages section including all the available languages. - """ - update_config() + """ + Update the mkdocs.yml file Languages section including all the available languages. + """ + update_config() @app.command() def serve() -> None: - """ - A quick server to preview a built site with translations. + """ + A quick server to preview a built site with translations. - For development, prefer the command live (or just mkdocs serve). + For development, prefer the command live (or just mkdocs serve). - This is here only to preview a site with translations already built. + This is here only to preview a site with translations already built. - Make sure you run the build-all command first. - """ - typer.echo("Warning: this is a very simple server.") - typer.echo("For development, use the command live instead.") - typer.echo("This is here only to preview a site with translations already built.") - typer.echo("Make sure you run the build-all command first.") - os.chdir("site") - server_address = ("", 8000) - server = HTTPServer(server_address, SimpleHTTPRequestHandler) - typer.echo("Serving at: http://127.0.0.1:8000") - server.serve_forever() + Make sure you run the build-all command first. + """ + typer.echo("Warning: this is a very simple server.") + typer.echo("For development, use the command live instead.") + typer.echo("This is here only to preview a site with translations already built.") + typer.echo("Make sure you run the build-all command first.") + os.chdir("site") + server_address = ("", 8000) + server = HTTPServer(server_address, SimpleHTTPRequestHandler) + typer.echo("Serving at: http://127.0.0.1:8000") + server.serve_forever() @app.command() def live( - lang: str = typer.Argument( - None, callback=lang_callback, autocompletion=complete_existing_lang - ), + lang: str = typer.Argument( + None, callback=lang_callback, autocompletion=complete_existing_lang + ), ) -> None: - """ - Serve with livereload a docs site for a specific language. + """ + Serve with livereload a docs site for a specific language. - This only shows the actual translated files, not the placeholders created with - build-all. + This only shows the actual translated files, not the placeholders created with + build-all. - Takes an optional LANG argument with the name of the language to serve, by default - en. - """ - # Enable line numbers during local development to make it easier to highlight - os.environ["LINENUMS"] = "true" - if lang is None: - lang = "en" - lang_path: Path = docs_path / lang - os.chdir(lang_path) - mkdocs.commands.serve.serve(dev_addr="127.0.0.1:8000") + Takes an optional LANG argument with the name of the language to serve, by default + en. + """ + # Enable line numbers during local development to make it easier to highlight + os.environ["LINENUMS"] = "true" + if lang is None: + lang = "en" + lang_path: Path = docs_path / lang + os.chdir(lang_path) + mkdocs.commands.serve.serve(dev_addr="127.0.0.1:8000") def get_updated_config_content() -> Dict[str, Any]: - config = get_alternate_langs_config() - languages = [{"en": "/"}] - new_alternate: List[Dict[str, str]] = [] - # Language names sourced from https://quickref.me/iso-639-1 - # Contributors may wish to update or change these, e.g. to fix capitalization. - language_names_path = Path(__file__).parent / docs_folder_name / "language_names.yml" - local_language_names: Dict[str, str] = mkdocs.utils.yaml_load( - language_names_path.read_text(encoding="utf-8") - ) - for lang_path in get_lang_paths(): - if lang_path.name == "en" or not lang_path.is_dir(): - continue - code = lang_path.name - languages.append({code: f"/{code}/"}) - for lang_dict in languages: - code = list(lang_dict.keys())[0] - url = lang_dict[code] - if code not in local_language_names: - print( - f"Missing language name for: {code}, " - f"update it in {docs_folder_name}/language_names.yml" - ) - raise typer.Abort() - use_name = f"{code} - {local_language_names[code]}" - new_alternate.append({"link": url, "name": use_name}) - config["extra"]["alternate"] = new_alternate - return config + config = get_alternate_langs_config() + languages = [{"en": "/"}] + new_alternate: List[Dict[str, str]] = [] + # Language names sourced from https://quickref.me/iso-639-1 + # Contributors may wish to update or change these, e.g. to fix capitalization. + language_names_path = Path(__file__).parent / docs_folder_name / "language_names.yml" + local_language_names: Dict[str, str] = mkdocs.utils.yaml_load( + language_names_path.read_text(encoding="utf-8") + ) + for lang_path in get_lang_paths(): + if lang_path.name == "en" or not lang_path.is_dir(): + continue + code = lang_path.name + languages.append({code: f"/{code}/"}) + for lang_dict in languages: + code = list(lang_dict.keys())[0] + url = lang_dict[code] + if code not in local_language_names: + print( + f"Missing language name for: {code}, " + f"update it in {docs_folder_name}/language_names.yml" + ) + raise typer.Abort() + use_name = f"{code} - {local_language_names[code]}" + new_alternate.append({"link": url, "name": use_name}) + config["extra"]["alternate"] = new_alternate + return config def update_config() -> None: - config = get_updated_config_content() - alternate_langs_config_path.write_text( - "# WARNING: this file is auto-generated by ./docs.py\n" + - yaml.dump(config, sort_keys=False, width=200, allow_unicode=True), - encoding="utf-8", - ) + config = get_updated_config_content() + alternate_langs_config_path.write_text( + "# WARNING: this file is auto-generated by ./docs.py\n" + + yaml.dump(config, sort_keys=False, width=200, allow_unicode=True), + encoding="utf-8", + ) @app.command() def verify_config() -> None: - """ - Verify main mkdocs.yml content to make sure it uses the latest language names. - """ - typer.echo("Verifying mkdocs-alternate-lang.yml") - config = get_alternate_langs_config() - print(config) - updated_config = get_updated_config_content() - print(updated_config) - if config != updated_config: - typer.secho( - f"./mkdocs-alternate-lang.yml outdated from {docs_folder_name}/language_names.yml, " - "update language_names.yml and run " - "python ./docs.py update-languages", - color=typer.colors.RED, - ) - raise typer.Abort() - typer.echo("Valid mkdocs-alternate-lang.yml ✅") + """ + Verify main mkdocs.yml content to make sure it uses the latest language names. + """ + typer.echo("Verifying mkdocs-alternate-lang.yml") + config = get_alternate_langs_config() + print(config) + updated_config = get_updated_config_content() + print(updated_config) + if config != updated_config: + typer.secho( + f"./mkdocs-alternate-lang.yml outdated from {docs_folder_name}/language_names.yml, " + "update language_names.yml and run " + "python ./docs.py update-languages", + color=typer.colors.RED, + ) + raise typer.Abort() + typer.echo("Valid mkdocs-alternate-lang.yml ✅") @app.command() def langs_json(): - langs = [] - for lang_path in get_lang_paths(): - if lang_path.is_dir(): - langs.append(lang_path.name) - print(json.dumps(langs)) + langs = [] + for lang_path in get_lang_paths(): + if lang_path.is_dir(): + langs.append(lang_path.name) + print(json.dumps(langs)) if __name__ == "__main__": - app() + app() diff --git a/mkdocs-utils/hooks.py b/mkdocs-utils/hooks.py index f130f350a..ba2eb372f 100644 --- a/mkdocs-utils/hooks.py +++ b/mkdocs-utils/hooks.py @@ -3,6 +3,7 @@ # Source: https://github.com/tiangolo/fastapi/blob/master/scripts/mkdocs_hooks.py # Original Author: Sebastián Ramírez and contributors +import glob from functools import lru_cache from pathlib import Path from typing import Any, List, Union @@ -13,127 +14,121 @@ from mkdocs.structure.pages import Page from mkdocs.utils.yaml import yaml_load -import glob - non_translated_sections = [] # Add the sections that are not translated here @lru_cache def get_missing_translation_content(docs_dir: str) -> str: - docs_dir_path = Path(docs_dir) - missing_translation_path = docs_dir_path.parent.parent / "missing-translation.md" - return missing_translation_path.read_text(encoding="utf-8") + docs_dir_path = Path(docs_dir) + missing_translation_path = docs_dir_path.parent.parent / "missing-translation.md" + return missing_translation_path.read_text(encoding="utf-8") class EnFile(File): - pass + pass def resolve_file(*, item: str, files: Files, config: MkDocsConfig) -> None: - item_path = Path(config.docs_dir) / item - if not item_path.is_file(): - en_src_dir = (Path(config.docs_dir) / "../../en/docs").resolve() - potential_path = en_src_dir / item - if potential_path.is_file(): - files.append( - EnFile( - path=item, - src_dir=str(en_src_dir), - dest_dir=config.site_dir, - use_directory_urls=config.use_directory_urls, - ) - ) + item_path = Path(config.docs_dir) / item + if not item_path.is_file(): + en_src_dir = (Path(config.docs_dir) / "../../en/docs").resolve() + potential_path = en_src_dir / item + if potential_path.is_file(): + files.append( + EnFile( + path=item, + src_dir=str(en_src_dir), + dest_dir=config.site_dir, + use_directory_urls=config.use_directory_urls, + ) + ) def resolve_files(*, items: List[Any], files: Files, config: MkDocsConfig) -> None: - for item in items: - if isinstance(item, str): - resolve_file(item=item, files=files, config=config) - elif isinstance(item, dict): - assert len(item) == 1 - values = list(item.values()) - if not values: - continue - if isinstance(values[0], str): - resolve_file(item=values[0], files=files, config=config) - elif isinstance(values[0], list): - resolve_files(items=values[0], files=files, config=config) - else: - raise ValueError(f"Unexpected value: {values}") + for item in items: + if isinstance(item, str): + resolve_file(item=item, files=files, config=config) + elif isinstance(item, dict): + assert len(item) == 1 + values = list(item.values()) + if not values: + continue + if isinstance(values[0], str): + resolve_file(item=values[0], files=files, config=config) + elif isinstance(values[0], list): + resolve_files(items=values[0], files=files, config=config) + else: + raise ValueError(f"Unexpected value: {values}") def get_images_relative_paths(docs_dir: str) -> List[str]: - en_docs_path = (Path(docs_dir) / "../../en/docs").resolve() - images = glob.glob(f"{en_docs_path}/images/**/*", recursive=True) - return list(map(lambda i: str(Path(i).relative_to(en_docs_path)), images)) + en_docs_path = (Path(docs_dir) / "../../en/docs").resolve() + images = glob.glob(f"{en_docs_path}/images/**/*", recursive=True) + return list(map(lambda i: str(Path(i).relative_to(en_docs_path)), images)) def on_files(files: Files, *, config: MkDocsConfig) -> Files: - # We circumvent the fact that the config.nav object is not available anymore - # due to the removal made by mkdocs-awesome-pages-plugin. - # - # So we read the config file again to get the nav object. - with open(config.config_file_path, 'rb') as f: - untouched_config = yaml_load(f) - resolve_files(items=untouched_config['nav'] or [], files=files, config=config) + # We circumvent the fact that the config.nav object is not available anymore + # due to the removal made by mkdocs-awesome-pages-plugin. + # + # So we read the config file again to get the nav object. + with open(config.config_file_path, 'rb') as f: + untouched_config = yaml_load(f) + resolve_files(items=untouched_config['nav'] or [], files=files, config=config) - images = get_images_relative_paths(config.docs_dir) + images = get_images_relative_paths(config.docs_dir) - resolve_files(items=images, files=files, config=config) - if "logo" in config.extra: - resolve_file(item=config.extra["logo"], files=files, config=config) - if "favicon" in config.theme: - resolve_file(item=config.theme["favicon"], files=files, config=config) - resolve_files(items=config.extra_css, files=files, config=config) - resolve_files(items=config.extra_javascript, files=files, config=config) - return files + resolve_files(items=images, files=files, config=config) + if "logo" in config.extra: + resolve_file(item=config.extra["logo"], files=files, config=config) + if "favicon" in config.theme: + resolve_file(item=config.theme["favicon"], files=files, config=config) + resolve_files(items=config.extra_css, files=files, config=config) + resolve_files(items=config.extra_javascript, files=files, config=config) + return files def generate_renamed_section_items( - items: List[Union[Page, Section, Link]], *, config: MkDocsConfig + items: List[Union[Page, Section, Link]], *, config: MkDocsConfig ) -> List[Union[Page, Section, Link]]: - new_items: List[Union[Page, Section, Link]] = [] - for item in items: - if isinstance(item, Section): - new_title = item.title - new_children = generate_renamed_section_items(item.children, config=config) - first_child = new_children[0] - if isinstance(first_child, Page): - if first_child.file.src_path.endswith("index.md"): - # Read the source so that the title is parsed and available - first_child.read_source(config=config) - new_title = first_child.title or new_title - # Creating a new section makes it render it collapsed by default - # no idea why, so, let's just modify the existing one - # new_section = Section(title=new_title, children=new_children) - item.title = new_title - item.children = new_children - new_items.append(item) - else: - new_items.append(item) - return new_items + new_items: List[Union[Page, Section, Link]] = [] + for item in items: + if isinstance(item, Section): + new_title = item.title + new_children = generate_renamed_section_items(item.children, config=config) + first_child = new_children[0] + if isinstance(first_child, Page): + if first_child.file.src_path.endswith("index.md"): + # Read the source so that the title is parsed and available + first_child.read_source(config=config) + new_title = first_child.title or new_title + # Creating a new section makes it render it collapsed by default + # no idea why, so, let's just modify the existing one + # new_section = Section(title=new_title, children=new_children) + item.title = new_title + item.children = new_children + new_items.append(item) + else: + new_items.append(item) + return new_items def on_nav( - nav: Navigation, *, config: MkDocsConfig, files: Files, **kwargs: Any + nav: Navigation, *, config: MkDocsConfig, _: Files ) -> Navigation: - new_items = generate_renamed_section_items(nav.items, config=config) - return Navigation(items=new_items, pages=nav.pages) - - -def on_pre_page(page: Page, *, config: MkDocsConfig, files: Files) -> Page: - return page + new_items = generate_renamed_section_items(nav.items, config=config) + return Navigation(items=new_items, pages=nav.pages) def on_page_markdown( - markdown: str, *, page: Page, config: MkDocsConfig, files: Files + markdown: str, *, page: Page, config: MkDocsConfig ) -> str: - if isinstance(page.file, EnFile): - for excluded_section in non_translated_sections: - if page.file.src_path.startswith(excluded_section): - return markdown - missing_translation_content = get_missing_translation_content(config.docs_dir) - header = "" - body = markdown - if markdown.startswith("#"): - header, _, body = markdown.partition("\n\n") - return f"{header}\n\n{missing_translation_content}\n\n{body}" - return markdown + if isinstance(page.file, EnFile): + for excluded_section in non_translated_sections: + if page.file.src_path.startswith(excluded_section): + return markdown + missing_translation_content = get_missing_translation_content(config.docs_dir) + header = "" + body = markdown + if markdown.startswith("#"): + header, _, body = markdown.partition("\n\n") + return f"{header}\n\n{missing_translation_content}\n\n{body}" + return markdown