diff --git a/.gitignore b/.gitignore index 0e4be4579..1de3851f9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /site +/site_build /hscout/site /env diff --git a/README.md b/README.md index 0ca69a970..18848ee9c 100644 --- a/README.md +++ b/README.md @@ -16,17 +16,18 @@ Conversation tab. ## Setting up local environment ``` sh +# initiate a python virtual environment python3 -m venv env -./env/bin/pip install -r requirements.txt -./env/bin/mkdocs serve +source ./env/bin/activate # for most shells, or source ./env/bin/activate.fish or source ./env/bin/activate.csh depending on your shell +pip install -r requirements.txt +./docs.py live en # or whatever code for language you want to see ``` Then visit to preview documentation. All the articles, -as well as static files, are under `/help/en/docs` subdirectory. While `mkdocs serve` -is running, you may make changes there, and the open page will refresh to show -the changes. +as well as static files, are under `/help/en/docs` subdirectory. While `./docs.py live` +is running, you may make changes there, and the open page will refresh to show the changes. -While `mkdocs serve` is running, you can run `./check_links.sh` to check +While `./docs.py live` is running, you can run `./check_links.sh` to check for broken links in the site. It will print out a lot of chatter, then if there are broken links, conclude with a section like this: @@ -45,6 +46,16 @@ Total wall clock time: 1.3s Downloaded: 36 files, 2.0M in 0.009s (226 MB/s) ``` +In order to build the website with all the languages, run both `build-all` +(so it builds the website) and then `serve` for preview: +```sh +source ./env/bin/activate +./docs.py build-all +./docs.py serve +``` + +Also see `./docs.py --help` for the full list of commands and options available. + ## Publishing for preview This is now automatically done by Netlify. Whenever you create a pull request, Netlify will build @@ -101,3 +112,5 @@ You need to first run `yarn install` in your Grist checkout directory. This work is licensed under a [Creative Commons Attribution-ShareAlike 4.0 International License](LICENSE.txt). + +The script [docs.py](./docs.py) is MIT licensed, credits to Sebastián Ramírez and the FastAPI project contributors. diff --git a/build-doc.sh b/build-doc.sh index 3565466eb..3d83d13d2 100755 --- a/build-doc.sh +++ b/build-doc.sh @@ -8,4 +8,4 @@ if [ -e env ]; then source ./env/bin/activate fi -mkdocs build +python3 ./docs.py build-all diff --git a/docs.py b/docs.py new file mode 100755 index 000000000..cdbd14f3c --- /dev/null +++ b/docs.py @@ -0,0 +1,290 @@ +#!./env/bin/python + +# Python script adapted from the fastapi project to manage translations +# License: MIT +# Source: https://github.com/tiangolo/fastapi/blob/master/scripts/docs.py +# Original Author: Sebastián Ramírez and contributors + +import json +import logging +import os +import shutil +import subprocess +from http.server import HTTPServer, SimpleHTTPRequestHandler +from multiprocessing import Pool +from pathlib import Path +from typing import Any, Dict, List, Optional, Union +from textwrap import dedent, indent + +import mkdocs.commands.build +import mkdocs.commands.serve +import mkdocs.config +import mkdocs.utils +import typer +import yaml + +logging.basicConfig(level=logging.INFO) + +app = typer.Typer() + +mkdocs_config_name = "mkdocs.yml" + +docs_folder_name = "help" +docs_path = Path("help") +en_docs_path = Path("help/en") +alternate_langs_config_path: Path = Path("mkdocs-alternate-langs.yml") +site_path = Path("site").absolute() +build_site_path = Path("site_build").absolute() + + +def language_docs_dir(lang: str) -> Path: + return docs_path / lang / "docs" + + +def get_missing_translation_snippet() -> str: + missing_translation_file_path = (Path(__file__).parent / "help/en/docs/MISSING-TRANSLATION.md") + missing_translation_content = missing_translation_file_path.read_text(encoding="utf-8") + return "!!!warning\n\n" + indent(missing_translation_content, " ") + + +def get_mkdocs_yaml_for_lang(lang: str) -> str: + return dedent(f""" + INHERIT: ../../mkdocs.yml + docs_dir: './docs' + theme: + custom_dir: ../../overrides + language: {lang} + """).lstrip() + + +def get_alternate_langs_config() -> Dict[str, Any]: + return mkdocs.utils.yaml_load(alternate_langs_config_path.read_text(encoding="utf-8")) + + +def get_lang_paths() -> List[Path]: + return sorted(docs_path.iterdir()) + + +def lang_callback(lang: Optional[str]) -> Union[str, None]: + if lang is None: + return None + lang = lang.lower() + return lang + + +def complete_existing_lang(incomplete: str): + lang_path: Path + for lang_path in get_lang_paths(): + if lang_path.is_dir() and lang_path.name.startswith(incomplete): + yield lang_path.name + + +@app.command() +def new_lang(lang: str = typer.Argument(..., callback=lang_callback)): + """ + Generate a new docs translation directory for the language LANG. + """ + new_path: Path = docs_path / lang + if new_path.exists(): + typer.echo(f"The language was already created: {lang}") + raise typer.Abort() + new_path.mkdir() + new_config_path: Path = Path(new_path) / mkdocs_config_name + new_config_path.write_text(get_mkdocs_yaml_for_lang(lang), encoding="utf-8") + new_config_docs_path: Path = language_docs_dir(lang) + new_config_docs_path.mkdir() + + en_index_path: Path = language_docs_dir('en') / "index.md" + new_index_path: Path = new_config_docs_path / "index.md" + en_index_content = en_index_path.read_text(encoding="utf-8") + new_index_content = f"{get_missing_translation_snippet()}\n\n{en_index_content}" + new_index_path.write_text(new_index_content, encoding="utf-8") + + # Copy the MISSING-TRANSLATION.md file, which can be localized later + en_missing_translation_path: Path = language_docs_dir('en') / "MISSING-TRANSLATION.md" + new_missing_translation_path: Path = new_config_docs_path / "MISSING-TRANSLATION.md" + new_missing_translation_path.write_bytes(en_missing_translation_path.read_bytes()) + + # Create the images directory + images_dir = new_config_docs_path / "images" + images_dir.mkdir() + (images_dir / ".gitkeep").touch() + + typer.secho(f"Successfully initialized: {new_path}", color=typer.colors.GREEN) + update_languages() + + +@app.command() +def build_lang( + lang: str = typer.Argument( + ..., callback=lang_callback, autocompletion=complete_existing_lang + ), +) -> None: + """ + Build the docs for a language. + """ + lang_path: Path = docs_path / lang + if not lang_path.is_dir(): + typer.echo(f"The language translation doesn't seem to exist yet: {lang}") + raise typer.Abort() + typer.echo(f"Building docs for: {lang}") + build_site_dist_path = build_site_path / lang + if lang == "en": + dist_path = site_path + # Don't remove en dist_path as it might already contain other languages. + # When running build_all(), that function already removes site_path. + # All this is only relevant locally, on GitHub Actions all this is done through + # artifacts and multiple workflows, so it doesn't matter if directories are + # removed or not. + else: + dist_path = site_path / lang + shutil.rmtree(dist_path, ignore_errors=True) + current_dir = os.getcwd() + os.chdir(lang_path) + shutil.rmtree(build_site_dist_path, ignore_errors=True) + subprocess.run(["mkdocs", "build", "--site-dir", build_site_dist_path], check=True) + shutil.copytree(build_site_dist_path, dist_path, dirs_exist_ok=True) + os.chdir(current_dir) + typer.secho(f"Successfully built docs for: {lang}", color=typer.colors.GREEN) + + +@app.command() +def build_all() -> None: + """ + Build mkdocs site for en, and then build each language inside, end result is located + at directory ./site/ with each language inside. + """ + update_languages() + shutil.rmtree(site_path, ignore_errors=True) + langs = [lang.name for lang in get_lang_paths() if lang.is_dir()] + cpu_count = os.cpu_count() or 1 + process_pool_size = cpu_count * 4 + typer.echo(f"Using process pool size: {process_pool_size}") + with Pool(process_pool_size) as p: + p.map(build_lang, langs) + + +@app.command() +def update_languages() -> None: + """ + Update the mkdocs.yml file Languages section including all the available languages. + """ + update_config() + + +@app.command() +def serve() -> None: + """ + A quick server to preview a built site with translations. + + For development, prefer the command live (or just mkdocs serve). + + This is here only to preview a site with translations already built. + + Make sure you run the build-all command first. + """ + typer.echo("Warning: this is a very simple server.") + typer.echo("For development, use the command live instead.") + typer.echo("This is here only to preview a site with translations already built.") + typer.echo("Make sure you run the build-all command first.") + os.chdir("site") + server_address = ("", 8000) + server = HTTPServer(server_address, SimpleHTTPRequestHandler) + typer.echo("Serving at: http://127.0.0.1:8000") + server.serve_forever() + + +@app.command() +def live( + lang: str = typer.Argument( + None, callback=lang_callback, autocompletion=complete_existing_lang + ), +) -> None: + """ + Serve with livereload a docs site for a specific language. + + This only shows the actual translated files, not the placeholders created with + build-all. + + Takes an optional LANG argument with the name of the language to serve, by default + en. + """ + # Enable line numbers during local development to make it easier to highlight + os.environ["LINENUMS"] = "true" + if lang is None: + lang = "en" + lang_path: Path = docs_path / lang + os.chdir(lang_path) + os.environ["DISABLE_LANGUAGE_SELECTOR"] = "true" + mkdocs.commands.serve.serve(dev_addr="127.0.0.1:8000") + + +def get_updated_config_content() -> Dict[str, Any]: + config = get_alternate_langs_config() + languages = [{"en": "/"}] + new_alternate: List[Dict[str, str]] = [] + # Language names sourced from https://quickref.me/iso-639-1 + # Contributors may wish to update or change these, e.g. to fix capitalization. + language_names_path = Path(__file__).parent / docs_folder_name / "language_names.yml" + local_language_names: Dict[str, str] = mkdocs.utils.yaml_load( + language_names_path.read_text(encoding="utf-8") + ) + for lang_path in get_lang_paths(): + if lang_path.name == "en" or not lang_path.is_dir(): + continue + code = lang_path.name + languages.append({code: f"/{code}/"}) + for lang_dict in languages: + code = list(lang_dict.keys())[0] + url = lang_dict[code] + if code not in local_language_names: + print( + f"Missing language name for: {code}, " + f"update it in {docs_folder_name}/language_names.yml" + ) + raise typer.Abort() + use_name = f"{code} - {local_language_names[code]}" + new_alternate.append({"link": url, "name": use_name, "code": code}) + config["extra"]["alternate"] = new_alternate + return config + + +def update_config() -> None: + config = get_updated_config_content() + alternate_langs_config_path.write_text( + "# WARNING: this file is auto-generated by ./docs.py\n" + + yaml.dump(config, sort_keys=False, width=200, allow_unicode=True), + encoding="utf-8", + ) + + +@app.command() +def verify_config() -> None: + """ + Verify main mkdocs.yml content to make sure it uses the latest language names. + """ + typer.echo("Verifying mkdocs-alternate-lang.yml") + config = get_alternate_langs_config() + updated_config = get_updated_config_content() + if config != updated_config: + typer.secho( + f"./mkdocs-alternate-lang.yml outdated from {docs_folder_name}/language_names.yml, " + "update language_names.yml and run " + "python ./docs.py update-languages", + color=typer.colors.RED, + ) + raise typer.Abort() + typer.echo("Valid mkdocs-alternate-lang.yml ✅") + + +@app.command() +def langs_json(): + langs = [] + for lang_path in get_lang_paths(): + if lang_path.is_dir(): + langs.append(lang_path.name) + print(json.dumps(langs)) + + +if __name__ == "__main__": + app() diff --git a/help/en/docs/MISSING-TRANSLATION.md b/help/en/docs/MISSING-TRANSLATION.md new file mode 100644 index 000000000..c69a3b193 --- /dev/null +++ b/help/en/docs/MISSING-TRANSLATION.md @@ -0,0 +1,3 @@ +The current page still doesn't have a translation for this language. + +But you can help translating it: [Contributing](https://hosted.weblate.org/engage/grist-help/){.internal-link target=_blank}. diff --git a/help/en/docs/afterschool-program.md b/help/en/docs/afterschool-program.md index a9a4d697e..29c07b8ba 100644 --- a/help/en/docs/afterschool-program.md +++ b/help/en/docs/afterschool-program.md @@ -39,14 +39,14 @@ instructor. When starting from scratch, you'll create a new empty document (see [Creating a document](creating-doc.md)), rename the initial empty table "Table1" to "Classes", add the columns you need, and type in some classes. To follow the steps of this tutorial, you can instead import -[Classes.csv](/unlocalized-assets/afterschool-program/Classes.csv){: data-wm-adjusted=1 } +[Classes.csv](./unlocalized-assets/afterschool-program/Classes.csv){: data-wm-adjusted=1 } (or simply refer to the "Afterschool Program" example document). ![add-classes](images/afterschool-program/add-classes.png) For the Staff table, click the "Add New" button and select "Add Empty Table". Rename it to "Staff", create some columns, and enter some data about instructors. Or import -[Staff.csv](/unlocalized-assets/afterschool-program/Staff.csv) to use sample data and save a few steps. +[Staff.csv](./unlocalized-assets/afterschool-program/Staff.csv) to use sample data and save a few steps. ![add-staff](images/afterschool-program/add-staff.png) @@ -108,7 +108,7 @@ Next, we will continue with students and their enrollments. Each class has a number of students. So, we’ll need a table of students. Again, add a new empty table, rename it to "Students", and fill it with the students’ names, grade levels, etc. Or -import [Students.csv](/unlocalized-assets/afterschool-program/Students.csv) to use sample data and save a few steps. +import [Students.csv](./unlocalized-assets/afterschool-program/Students.csv) to use sample data and save a few steps. ![students-table](images/afterschool-program/students-table.png) @@ -143,7 +143,7 @@ becomes this: {: .screenshot-half } So, let’s add a new table, name it "Enrollments", and add the columns we need. Here too, to follow -along, you may import sample data from [Enrollments.csv](/unlocalized-assets/afterschool-program/Enrollments.csv). +along, you may import sample data from [Enrollments.csv](./unlocalized-assets/afterschool-program/Enrollments.csv). ![enrollments-table](images/afterschool-program/enrollments-table.png) @@ -271,7 +271,7 @@ will likely simplify your daily workflow. So, let’s add one more table: `Families`. We’ll include the parent name and contact info, and link each child to a record here. You can import sample data from -[Families.csv](/unlocalized-assets/afterschool-program/Families.csv). +[Families.csv](./unlocalized-assets/afterschool-program/Families.csv). ![families1](images/afterschool-program/families1.png) diff --git a/help/en/docs/api.md b/help/en/docs/api.md index 3b9b4a96c..a5588be6d 100644 --- a/help/en/docs/api.md +++ b/help/en/docs/api.md @@ -272,7 +272,7 @@ Also note that some query parameters alter this behavior.

Responses

Request samples

Content type
application/json
{
  • "sql": "select * from Pets where popularity >= ?",
  • "args": [
    • 50
    ],
  • "timeout": 500
}

Response samples

Content type
application/json
{
  • "statement": "select * from Pets ...",
  • "records": [
    • {
      • "fields": {
        • "id": 1,
        • "pet": "cat",
        • "popularity": 67
        }
      },
    • {
      • "fields": {
        • "id": 2,
        • "pet": "dog",
        • "popularity": 95
        }
      }
    ]
}