Modify doc generation after repositories merge (#757)

* Modify doc generation after repositories merge
* Adjustements for links to repositories

.gitignore

@@ -110,7 +110,7 @@ site/
 # Doc generation
 mkdocs.yml
 /taipy
-/gui
+/taipy-fe
 docs/manuals/reference/**/*.md
 !docs/manuals/reference/index.md
 docs/manuals/gui/controls.md

Pipfile

@@ -5,15 +5,13 @@ name = "pypi"
 
 [packages]
 "backports.zoneinfo" = {version="==0.2.1", markers="python_version < '3.9'", extras=["tzdata"]}
-apispec = {extras = ["yaml"], version = "==5.0"}
+apispec = {extras = ["yaml"], version = "==6.3"}
 apispec-webframeworks = "==0.5.2"
 bcrypt = "==3.2.2"
 cookiecutter = "==2.1.1"
 deepdiff = "==6.2.2"
 flask = "==3.0.0"
 flask-cors = "==4.0.0"
-flask-marshmallow = ">=0.14"
-flask-migrate = ">=3.0"
 flask-restful = ">=0.3.9"
 flask-socketio = "==5.3.6"
 gevent = "==23.7.0"
@@ -22,15 +20,16 @@ gitignore-parser = "==0.1.1"
 kthread = "==0.2.3"
 ldap3 = "==2.9.1"
 markdown = "==3.4.4"
-markupsafe = "==2.1.1"
+marshmallow = "==3.20.1"
 modin = "==0.23.0"
 networkx = "==2.6"
 openpyxl = "==3.1.2"
+opentelemetry-api = "==1.20.0"
+opentelemetry-exporter-otlp = "==1.20.0"
+opentelemetry-sdk = "==1.20.0"
 pandas = "==2.0.0"
-passlib = "==1.7.4"
 pyarrow = "==10.0.1"
 pymongo = {extras = ["srv"], version = "==4.2.0"}
-pyngrok = "==5.1"
 python-dotenv = "==1.0.0"
 python-magic = {version = "==0.4.24", markers="sys_platform != 'win32'"}
 python-magic-bin = {version = "==0.4.14", markers="sys_platform == 'win32'"}

docs/contributing/contributing.md

@@ -53,14 +53,17 @@ sprint is attached to the `current sprint`. Please, do not work on it, the Taipy
 
 # Code organization
 
-Taipy is organized in several repositories:
-
-- [taipy-config](https://github.com/Avaiga/taipy-config).
-- [taipy-core](https://github.com/Avaiga/taipy-core).
-- [taipy-gui](https://github.com/Avaiga/taipy-gui).
-- [taipy-rest](https://github.com/Avaiga/taipy-rest).
-- [taipy-templates](https://github.com/Avaiga/taipy-templates).
-- [taipy](https://github.com/Avaiga/taipy) brings previous packages in a single one.
+The Taipy source code is located in the is organized [taipy](https://github.com/Avaiga/taipy)
+repository, in the `taipy` directory.
+
+Packages sources are organized in sub-directories from there:
+
+- `taipy-config`
+- `taipy-core`
+- `taipy-gui`
+- `taipy-rest`
+- `taipy-templates`
+
 
 # Best practices
 

docs/manuals/gui/extension_assets.md

@@ -18,4 +18,4 @@ are requested, and return the actual files according to the application setting.
     Please look into the `doc/extension` directory where Taipy GUI is
     installed for more information.<br/>
     You can also look at this example directly on
-    [GitHub](https://github.com/Avaiga/taipy-gui/tree/develop/doc/extension).
+    [GitHub](https://github.com/Avaiga/taipy/tree/[BRANCH]/doc/gui/extension).

docs/manuals/gui/extension_data.md

@@ -7,4 +7,4 @@
     Please look into the `doc/extension` directory where Taipy GUI is
     installed for more information.<br/>
     You can also look at this example directly on
-    [GitHub](https://github.com/Avaiga/taipy-gui/tree/develop/doc/extension).
+    [GitHub](https://github.com/Avaiga/taipy/tree/[BRANCH]/doc/gui/extension).

docs/manuals/gui/extension_list_of_values.md

@@ -7,4 +7,4 @@
     Please look into the `doc/extension` directory where Taipy GUI is
     installed for more information.<br/>
     You can also look at this example directly on
-    [GitHub](https://github.com/Avaiga/taipy-gui/tree/develop/doc/extension).
+    [GitHub](https://github.com/Avaiga/taipy/tree/[BRANCH]/doc/gui/extension).

docs/manuals/gui/extension_packaging.md

@@ -7,4 +7,4 @@
     Please look into the `doc/extension` directory where Taipy GUI is
     installed for more information.<br/>
     You can also look at this example directly on
-    [GitHub](https://github.com/Avaiga/taipy-gui/tree/develop/doc/extension).
+    [GitHub](https://github.com/Avaiga/taipy/tree/[BRANCH]/doc/gui/extension).

docs/manuals/gui/viselements/controls.md_template

@@ -10,7 +10,7 @@ advanced graphical controls such as selectors, tables, and charts.
 !!! note "Available in Taipy GUI, Taipy Community, and Enterprise editions"
 
     The controls listed in this section are provided in the
-    [`taipy-gui`](https://pypi.org/project/taip-gui/) Python package. These controls are
+    [`taipy-gui`](https://pypi.org/project/taipy-gui/) Python package. These controls are
     also present when [`taipy`](https://pypi.org/project/taipy/) is installed.
 
 Here is the list of all available controls in Taipy:

tools/_setup_generation/step_gui_ext_refman.py

@@ -37,10 +37,10 @@ def enter(self, setup: Setup):
     def setup(self, setup: Setup) -> None:
         if self.npm_path:
             saved_cwd = os.getcwd()
-            gui_path = os.path.join(setup.root_dir, "gui")
+            gui_path = os.path.join(setup.root_dir, "taipy-fe")
             os.chdir(gui_path)
             print(f"... Installing node modules...", flush=True)
-            subprocess.run(f"{self.npm_path} run inst", shell=True)
+            subprocess.run(f"{self.npm_path} i", shell=True)
             print(f"... Generating documentation...", flush=True)
             subprocess.run(f"{self.npm_path} run mkdocs", shell=True)
             # Process and copy files to docs/manuals

tools/fetch_source_files.py

@@ -13,7 +13,7 @@
 # Where all the code from all directories/repositories is copied
 DEST_DIR_NAME = "taipy"
 
-REPOS = ["config", "core", "gui", "rest", "taipy", "templates"]
+REPOS = ["taipy"]
 PRIVATE_REPOS = ["auth", "enterprise"]
 
 OPTIONAL_PACKAGES = {
@@ -143,8 +143,9 @@ def safe_rmtree(dir: str):
 safe_rmtree(os.path.join(TOOLS_PATH, DEST_DIR_NAME))
 
 pipfile_packages = {}
-PIPFILE_PACKAGE_RE = re.compile(r"(.*?)\s?=\s?(.*)")
+PIPFILE_PACKAGE_RE = re.compile(r"(..*?)\s?=\s?(.*)")
 
+frontend_dir = os.path.join(ROOT_DIR, f"taipy-fe")
 
 # Fetch files
 def move_files(repo: str, src_path: str):
@@ -196,13 +197,6 @@ def move_files(repo: str, src_path: str):
                        text=True)
         os.chdir(saved_dir)
     else:
-        tmp_dir = os.path.join(ROOT_DIR, f"{repo}.tmp")
-        safe_rmtree(tmp_dir)
-        gui_dir = os.path.join(ROOT_DIR, f"gui") if repo == "taipy-gui" else None
-        if repo == "taipy-gui" and os.path.isdir(gui_dir):
-            if os.path.isdir(os.path.join(gui_dir, "node_modules")):
-                shutil.move(os.path.join(gui_dir, "node_modules"), os.path.join(ROOT_DIR, f"gui_node_modules"))
-            shutil.rmtree(gui_dir)
         try:
             def copy_source(src_path: str, repo: str):
                 def copy(item: str, src: str, dst: str, rel_path: str):
@@ -229,26 +223,33 @@ def copy(item: str, src: str, dst: str, rel_path: str):
                 dest_path = os.path.join(ROOT_DIR, "taipy")
                 if not os.path.exists(dest_path):
                     os.makedirs(dest_path)
-                src_path = os.path.join(src_path, "src", "taipy")
-                for item in os.listdir(src_path):
-                    copy(item, src_path, dest_path, "")
+                sources_path = os.path.join(src_path, "taipy")
+                # Packages using the 'src' directory to hold source files
+                if not os.path.exists(sources_path):
+                    sources_path = os.path.join(src_path, "src", "taipy")
+                for item in os.listdir(sources_path):
+                    copy(item, sources_path, dest_path, "")
             copy_source(src_path, repo)
 
-            if gui_dir:
-                if not os.path.isdir(gui_dir):
-                    os.mkdir(gui_dir)
-                src_gui_dir = os.path.join(src_path, "gui")
-                shutil.copytree(os.path.join(src_gui_dir, "src"), os.path.join(gui_dir, "src"))
-                for f in [f for f in os.listdir(src_gui_dir) if f.endswith(".md") or f.endswith(".json")]:
-                    shutil.copy(os.path.join(src_gui_dir, f), os.path.join(gui_dir, f))
-                if os.path.isdir(os.path.join(ROOT_DIR, "gui_node_modules")):
-                    shutil.move(os.path.join(ROOT_DIR, "gui_node_modules"), os.path.join(gui_dir, "node_modules"))
+            # Copy Taipy GUI front end code
+            if repo == "taipy":
+                if not os.path.isdir(frontend_dir):
+                    os.mkdir(frontend_dir)
+                fe_src_dir = os.path.join(src_path, "frontend", "taipy-gui")
+                shutil.copytree(os.path.join(fe_src_dir, "src"), os.path.join(frontend_dir, "src"))
+                for f in [f for f in os.listdir(fe_src_dir) if f.endswith(".md") or f.endswith(".json")]:
+                    shutil.copy(os.path.join(fe_src_dir, f), os.path.join(frontend_dir, f))
         finally:
             pass
             """
             shutil.rmtree(tmp_dir)
             """
 
+frontend_dir = os.path.join(ROOT_DIR, f"taipy-fe")
+if os.path.isdir(os.path.join(frontend_dir, "node_modules")):
+    shutil.move(os.path.join(frontend_dir, "node_modules"), os.path.join(ROOT_DIR, f"fe_node_modules"))
+if os.path.isdir(os.path.join(frontend_dir)):
+    shutil.rmtree(frontend_dir)
 
 for repo in repo_defs.keys():
     if repo_defs[repo].get("skip", False):
@@ -291,6 +292,9 @@ def handleRemoveReadonly(func, path, exc):
 
         shutil.rmtree(clone_dir, onerror=handleRemoveReadonly)
 
+if os.path.isdir(os.path.join(ROOT_DIR, "fe_node_modules")) and os.path.isdir(os.path.join(frontend_dir)):
+    shutil.move(os.path.join(ROOT_DIR, "fe_node_modules"), os.path.join(frontend_dir, "node_modules"))
+
 # Manually add the taipy.run() function.
 # TODO: Automate this, grabbing the function from the 'taipy' repository,
 # so we benefit from potential updates.

tools/postprocess.py

@@ -499,8 +499,13 @@ def process_data_source_attr(html: str, env):
         ref = m.group(3)
         repo_m = re.search(r"^([\w\d]+):", ref)
         if repo_m:
-            ref = ("https://github.com/Avaiga/taipy-" +
-                   f"{repo_m.group(0)[:-1]}/blob/{env.conf['branch']}/{ref[repo_m.end():]}")
+            target = ref[repo_m.end():]
+            if target.startswith("doc/"): #gui/examples/charts/advanced-annotations.py
+                target = f"{repo_m.group(0)[:-1]}/{target[4:]}"
+                ref = f"https://github.com/Avaiga/taipy/blob/{env.conf['branch']}/doc/{target}"
+            else:
+                logging.warning("Suspicious data-source attribute: {m.group(0)}")
+                ref = f"https://github.com/Avaiga/taipy-{repo_m.group(0)[:-1]}/blob/{env.conf['branch']}/{target}"
         new_content += (html[last_location:m.start()]
                         + f"{m.group(1)}{m.group(4)}"
                         + f"\n<small>You can download the entire source code used in this "
@@ -514,7 +519,7 @@ def process_data_source_attr(html: str, env):
 
 
 def process_links_to_github(html: str, env):
-    _LINK_RE = re.compile(r"(?<=href=\"https://github.com/Avaiga/)(taipy-gui/tree/)\[BRANCH\](.*?\")")
+    _LINK_RE = re.compile(r"(?<=href=\"https://github.com/Avaiga/)(taipy/tree/)\[BRANCH\](.*?\")")
     new_content = ""
     last_location = 0
     for m in _LINK_RE.finditer(html):