diff --git a/tools/_setup_generation/step_contributors.py b/tools/_setup_generation/step_contributors.py index c8c3b9997..1f2196776 100644 --- a/tools/_setup_generation/step_contributors.py +++ b/tools/_setup_generation/step_contributors.py @@ -13,7 +13,6 @@ class ContributorsStep(SetupStep): - def __init__(self): self.GH_TOKEN = os.getenv("GITHUB_TOKEN", None) self.BASE_URL = "https://api.github.com" @@ -52,30 +51,30 @@ def get_repo_urls(self): print(f"WARNING - Couldn't get repositories. response.status_code: {response.status_code}", flush=True) return repos = response.json() - self.REPO_URLS = list(map(lambda _: _['url'], repos)) + self.REPO_URLS = list(map(lambda _: _["url"], repos)) def get_avaiga_members(self): - response = self.__get(self.MEMBERS_URL, with_token=False) + response = self.__get(self.MEMBERS_URL) if response.status_code != 200: print(f"WARNING - Couldn't get members. response.status_code: {response.status_code}", flush=True) return members = response.json() for member in members: - login = member['login'] + login = member["login"] if login not in self.MEMBERS and login not in self.ANONYMOUS: - self.MEMBERS[login] = {"avatar_url": member['avatar_url'], "html_url": member['html_url']} + self.MEMBERS[login] = {"avatar_url": member["avatar_url"], "html_url": member["html_url"]} def get_contributors(self): for url in self.REPO_URLS: response = self.__get(url + "/contributors") if response.status_code != 200: - print(f"WARNING - Couldn't get collaborators. response.status_code: {response.status_code}", flush=True) + print(f"WARNING - Couldn't get contributors. response.status_code: {response.status_code}", flush=True) return contributors = response.json() for contrib in contributors: - login = contrib['login'] + login = contrib["login"] if login not in self.CONTRIBUTORS and login not in self.ANONYMOUS: - self.CONTRIBUTORS[login] = {"avatar_url": contrib['avatar_url'], "html_url": contrib['html_url']} + self.CONTRIBUTORS[login] = {"avatar_url": contrib["avatar_url"], "html_url": contrib["html_url"]} def build_content(self, *members_pattern_tuples): pattern_content_tuples = [] @@ -87,9 +86,11 @@ def build_content(self, *members_pattern_tuples): random.shuffle(members_list) for login, member_info in members_list: if login not in self.ANONYMOUS: - content += f"\n- [{login} GitHub avatar" \ - f"{login}]" \ - f"({member_info['html_url']})" + content += ( + f"\n- [{login} GitHub avatar" + f"{login}]" + f"({member_info['html_url']})" + ) content += "\n" pattern_content_tuples.append((pattern, content)) @@ -97,7 +98,7 @@ def build_content(self, *members_pattern_tuples): def _replace(self, path, *pattern_content_tuples): # Read in the file - with open(path + self.TEMPLATE_SUFFIX, 'r') as file: + with open(path + self.TEMPLATE_SUFFIX, "r") as file: file_data = file.read() # Replace the patterns by the contents @@ -107,17 +108,14 @@ def _replace(self, path, *pattern_content_tuples): file_data = file_data.replace(pattern, content) # Write the file out without the template suffix - with open(path, 'w') as file: + with open(path, "w") as file: file.write(file_data) def __get(self, url, with_token=True): if with_token and self.GH_TOKEN: - headers = { - "Accept": "application/vnd.github+json", - "Authorization": "Bearer "+self.GH_TOKEN - } - # {'Authorization': f'token {self.GH_TOKEN}'} - return requests.get(url, headers=headers) + return requests.get( + url, headers={"Accept": "application/vnd.github+json", "Authorization": "Bearer " + self.GH_TOKEN} + ) else: return requests.get(url) diff --git a/tools/_setup_generation/step_designer.py b/tools/_setup_generation/step_designer.py index b37652dbf..c6e76fa62 100644 --- a/tools/_setup_generation/step_designer.py +++ b/tools/_setup_generation/step_designer.py @@ -23,15 +23,15 @@ def get_id(self) -> str: def get_description(self) -> str: return "Retrieve the Designer documentation files." - def setup(self, setup: Setup): ... + def setup(self, _: Setup): ... def enter(self, setup: Setup): if os.path.exists(os.path.join(setup.docs_dir, DesignerStep.PREFIX)): self.DESIGNER_PATH = os.path.join( setup.docs_dir, *DesignerStep.PREFIX.split("/") ) - self.MKDOCS_TMPL = os.path.join(self.DESIGNER_PATH, "mkdocs.yml_template") - if not os.access(self.MKDOCS_TMPL, os.R_OK): + self.MKDOCS_TEMPLATE = os.path.join(self.DESIGNER_PATH, "mkdocs.yml_template") + if not os.access(self.MKDOCS_TEMPLATE, os.R_OK): raise FileNotFoundError( f"FATAL - Could not read docs/{DesignerStep.PREFIX}/mkdocs.yml_template" ) @@ -46,7 +46,7 @@ def exit(self, setup: Setup): def _read_mkdocs_template(self) -> str: lines = [] indentation = 0 - with open(self.MKDOCS_TMPL) as file: + with open(self.MKDOCS_TEMPLATE) as file: collect = False for line in file: if line.startswith("nav:"): # Start collecting navigation @@ -54,10 +54,10 @@ def _read_mkdocs_template(self) -> str: elif re.match(r"^[\w_]+\s*?:", line): # Stop collecting navigation if collect: navigation = StringIO() - for navline in lines: + for nav_line in lines: # Add each line with indentation removed navigation.write(" ") - navigation.write(navline[indentation:]) + navigation.write(nav_line[indentation:]) navigation.write("\n") return navigation.getvalue() elif collect: diff --git a/tools/_setup_generation/step_file_injection.py b/tools/_setup_generation/step_file_injection.py index 0d7f26d37..e757ea97c 100644 --- a/tools/_setup_generation/step_file_injection.py +++ b/tools/_setup_generation/step_file_injection.py @@ -33,24 +33,19 @@ def get_description(self) -> str: def setup(self, setup: Setup) -> None: try: - with open(self.src_path, 'r') as file: + with open(self.src_path, "r", encoding="utf-8") as file: content = file.read() self._replace(self.dst_tpl_path, self.pattern, content, self.dst_path) except Exception as e: - print(f"Error: cannot generate page: {e}") + print(f"ERROR - Cannot generate page: {e}") @staticmethod def _replace(in_tpl_file_path, pattern, by, into_file_path): # Read template file - with open(in_tpl_file_path, 'r') as tpl_file: - from_file_content = tpl_file.read() - + with open(in_tpl_file_path, "r", encoding="utf-8") as file: + content = file.read() # Replace the pattern by the contents - content = from_file_content.replace(pattern, by) - + content = content.replace(pattern, by) # Write the file to_file - with open(into_file_path, 'w') as into_file: - into_file.write(content) - - def exit(self, setup: Setup): - pass + with open(into_file_path, "w", encoding="utf-8") as file: + file.write(content) diff --git a/tools/fetch_source_files.py b/tools/fetch_source_files.py index c280cc204..905046bdc 100644 --- a/tools/fetch_source_files.py +++ b/tools/fetch_source_files.py @@ -32,8 +32,7 @@ # Gather version information for each repository repo_defs = { - repo if repo == "taipy" else f"taipy-{repo}": {"version": "local", "tag": None} - for repo in REPOS + PRIVATE_REPOS + repo if repo == "taipy" else f"taipy-{repo}": {"version": "local", "tag": None} for repo in REPOS + PRIVATE_REPOS } CATCH_VERSION_RE = re.compile(r"(^\d+\.\d+?)(?:(\.\d+)(\..*)?)?|develop|local$") for version in args.version: @@ -76,9 +75,7 @@ if repo_desc is None: repo = f"taipy-{repo}" repo_desc = repo_defs.get(repo, None) - if repo_desc and ( - remapped_version := version_remap_desc.get(repo_desc["version"], None) - ): + if repo_desc and (remapped_version := version_remap_desc.get(repo_desc["version"], None)): repo_desc["version"] = remapped_version # Test git, if needed @@ -87,11 +84,7 @@ git_command = None else: git_path = shutil.which(git_command) - if ( - git_path is None - or subprocess.run(f'"{git_path}" --version', shell=True, capture_output=True) - is None - ): + if git_path is None or subprocess.run(f'"{git_path}" --version', shell=True, capture_output=True) is None: raise IOError(f'Couldn\'t find command "{git_command}"') git_command = git_path @@ -124,9 +117,7 @@ repo_defs[repo]["skip"] = True continue else: - raise SystemError( - f"Problem with {repo}:\nOutput: {cmd.stdout}\nError: {cmd.stderr}" - ) + raise SystemError(f"Problem with {repo}:\nOutput: {cmd.stdout}\nError: {cmd.stderr}") else: with GitContext(repo, PRIVATE_REPOS): cmd = subprocess.run( @@ -140,13 +131,9 @@ repo_defs[repo]["skip"] = True continue else: - raise SystemError( - f"Couldn't query branches from {loggable_github_root}{repo}." - ) + raise SystemError(f"Couldn't query branches from {loggable_github_root}{repo}.") if f"release/{version}\n" not in cmd.stdout: - raise ValueError( - f"No branch 'release/{version}' in repository '{repo}'." - ) + raise ValueError(f"No branch 'release/{version}' in repository '{repo}'.") tag = repo_defs[repo]["tag"] if tag: cmd = subprocess.run( @@ -191,8 +178,6 @@ def safe_rmtree(dir: str): pipfile_packages = {} PIPFILE_PACKAGE_RE = re.compile(r"(..*?)\s?=\s?(.*)") -# frontend_dir = os.path.join(ROOT_DIR, "taipy-fe") - # Fetch files def move_files(repo: str, src_path: str): @@ -204,9 +189,7 @@ def move_files(repo: str, src_path: str): with open(pipfile_path, "r") as pipfile: while True: line = pipfile.readline() - if str(line) == "" or ( - reading_packages and (not line.strip() or line[0] == "[") - ): + if str(line) == "" or (reading_packages and (not line.strip() or line[0] == "[")): break line = line.strip() if line == "[packages]": @@ -216,10 +199,7 @@ def move_files(repo: str, src_path: str): if match and not match.group(1).startswith("taipy"): package = match.group(1).lower() version = match.group(2) - if ( - repo_optional_packages is None - or package not in repo_optional_packages - ): + if repo_optional_packages is None or package not in repo_optional_packages: if package in pipfile_packages: versions = pipfile_packages[package] if version in versions: @@ -235,24 +215,18 @@ def move_files(repo: str, src_path: str): for step_dir in [ step_dir for step_dir in os.listdir(gs_dir) - if step_dir.startswith("step_") - and os.path.isdir(os.path.join(gs_dir, step_dir)) + if step_dir.startswith("step_") and os.path.isdir(os.path.join(gs_dir, step_dir)) ]: safe_rmtree(os.path.join(gs_dir, step_dir)) for step_dir in [ step_dir for step_dir in os.listdir(src_path) - if step_dir.startswith("step_") - and os.path.isdir(os.path.join(src_path, step_dir)) + if step_dir.startswith("step_") and os.path.isdir(os.path.join(src_path, step_dir)) ]: - shutil.copytree( - os.path.join(src_path, step_dir), os.path.join(gs_dir, step_dir) - ) + shutil.copytree(os.path.join(src_path, step_dir), os.path.join(gs_dir, step_dir)) safe_rmtree(os.path.join(gs_dir, "src")) shutil.copytree(os.path.join(src_path, "src"), os.path.join(gs_dir, "src")) - shutil.copy( - os.path.join(src_path, "index.md"), os.path.join(gs_dir, "index.md") - ) + shutil.copy(os.path.join(src_path, "index.md"), os.path.join(gs_dir, "index.md")) saved_dir = os.getcwd() os.chdir(os.path.join(ROOT_DIR, "docs", "getting_started", repo[6:])) subprocess.run( @@ -263,27 +237,52 @@ def move_files(repo: str, src_path: str): ) os.chdir(saved_dir) elif repo == "taipy-designer": - designer_doc_dir = os.path.join( - ROOT_DIR, "docs", "userman", "ecosystem", "designer" - ) + designer_doc_dir = os.path.join(ROOT_DIR, "docs", "userman", "ecosystem", "designer") safe_rmtree(designer_doc_dir) - src_documentation_dir = os.path.join(src_path, "documentation") - saved_dir = os.getcwd() - os.chdir(saved_dir) + doc_src_dir = os.path.join(src_path, "documentation") + taipy_docs_src_dir = os.path.join(doc_src_dir, "taipy_docs") + tools_src_dir = os.path.join(doc_src_dir, "tools") + shutil.copytree(os.path.join(doc_src_dir, "taipy_docs"), designer_doc_dir) + shutil.copy( + os.path.join(doc_src_dir, "mkdocs_taipy.yml"), + os.path.join(designer_doc_dir, "mkdocs.yml_template"), + ) + dest_dir = os.path.join(designer_doc_dir, "designer-examples") + safe_rmtree(dest_dir) + os.mkdir(dest_dir) subprocess.run( - f"python {os.path.join(src_path, 'copy_examples.py')}", + f"python \"{os.path.join(tools_src_dir, 'zip_examples.py')}\" " + # src_directory + f"\"{taipy_docs_src_dir}\" " + # intermediate_directory + f"\"{dest_dir}\" " + # target_directory + f"\"{designer_doc_dir}\" " + # zip)file + "examples.zip", shell=True, capture_output=True, text=True, ) - os.chdir(saved_dir) - shutil.copytree( - os.path.join(src_documentation_dir, "taipy_docs"), designer_doc_dir - ) - shutil.copy( - os.path.join(src_documentation_dir, "mkdocs_taipy.yml"), - os.path.join(designer_doc_dir, "mkdocs.yml_template"), + safe_rmtree(dest_dir) + dest_dir = os.path.join(designer_doc_dir, "designer-training") + safe_rmtree(dest_dir) + os.mkdir(dest_dir) + subprocess.run( + f"python \"{os.path.join(tools_src_dir, 'zip_examples.py')}\" " + # src_directory + f"\"{os.path.join(taipy_docs_src_dir, 'training')}\" " + # intermediate_directory + f"\"{dest_dir}\" " + # target_directory + f"\"{os.path.join(designer_doc_dir, 'training')}\" " + # zip)file + "training.zip", + shell=True, + capture_output=True, + text=True, ) + safe_rmtree(dest_dir) else: try: @@ -299,9 +298,7 @@ def copy(item: str, src: str, dst: str, rel_path: str): rel_path = f"{rel_path}/{item}" for sub_item in os.listdir(full_src): copy(sub_item, full_src, full_dst, rel_path) - elif any( - item.endswith(ext) for ext in [".py", ".pyi", ".json", ".ipynb"] - ): + elif any(item.endswith(ext) for ext in [".py", ".pyi", ".json", ".ipynb"]): if os.path.isfile(full_dst): # File exists - compare with open(full_src, "r") as f: src = f.read() @@ -331,29 +328,31 @@ def copy(item: str, src: str, dst: str, rel_path: str): # Copy Taipy GUI front end code if not os.path.isdir(frontend_dir): os.mkdir(frontend_dir) - print("NOTE: Copying taipy-gui front-end source files") fe_src_dir = os.path.join(src_path, "frontend", "taipy-gui") shutil.copytree(os.path.join(fe_src_dir, "src"), os.path.join(frontend_dir, "src")) for f in [f for f in os.listdir(fe_src_dir) if f.endswith(".md") or f.endswith(".json")]: shutil.copy(os.path.join(fe_src_dir, f), os.path.join(frontend_dir, f)) - print("NOTE: Copying taipy front-end source files") + # Copy Taipy front end code (Core Elements) core_fe_src_dir = os.path.join(src_path, "frontend", "taipy") shutil.copytree(os.path.join(core_fe_src_dir, "src"), os.path.join(frontend_dir, "core_src")) # Copy INSTALLATION.md, CONTRIBUTING.md, and CODE_OF_CONDUCT.md if not os.path.isdir(taipy_documentation_files_dir): os.mkdir(taipy_documentation_files_dir) - shutil.copy(os.path.join(src_path, "INSTALLATION.md"), - os.path.join(taipy_documentation_files_dir, "INSTALLATION.md")) - shutil.copy(os.path.join(src_path, "CONTRIBUTING.md"), - os.path.join(taipy_documentation_files_dir, "CONTRIBUTING.md")) - shutil.copy(os.path.join(src_path, "CODE_OF_CONDUCT.md"), - os.path.join(taipy_documentation_files_dir, "CODE_OF_CONDUCT.md")) + shutil.copy( + os.path.join(src_path, "INSTALLATION.md"), + os.path.join(taipy_documentation_files_dir, "INSTALLATION.md"), + ) + shutil.copy( + os.path.join(src_path, "CONTRIBUTING.md"), + os.path.join(taipy_documentation_files_dir, "CONTRIBUTING.md"), + ) + shutil.copy( + os.path.join(src_path, "CODE_OF_CONDUCT.md"), + os.path.join(taipy_documentation_files_dir, "CODE_OF_CONDUCT.md"), + ) finally: pass - """ - shutil.rmtree(tmp_dir) - """ frontend_dir = os.path.join(ROOT_DIR, "taipy-fe") @@ -379,9 +378,7 @@ def copy(item: str, src: str, dst: str, rel_path: str): if not args.no_pull: cwd = os.getcwd() os.chdir(src_path) - subprocess.run( - f'"{git_path}" pull', shell=True, capture_output=True, text=True - ) + subprocess.run(f'"{git_path}" pull', shell=True, capture_output=True, text=True) os.chdir(cwd) print(f" Copying from {src_path}...", flush=True) move_files(repo, src_path) @@ -424,9 +421,7 @@ def handleRemoveReadonly(func, path, exc): shutil.rmtree(clone_dir, onerror=handleRemoveReadonly) -if os.path.isdir(os.path.join(ROOT_DIR, "fe_node_modules")) and os.path.isdir( - os.path.join(frontend_dir) -): +if os.path.isdir(os.path.join(ROOT_DIR, "fe_node_modules")) and os.path.isdir(os.path.join(frontend_dir)): shutil.move( os.path.join(ROOT_DIR, "fe_node_modules"), os.path.join(frontend_dir, "node_modules"), @@ -493,9 +488,7 @@ def run(*services: t.Union[Gui, Rest, Orchestrator], **kwargs) -> t.Optional[t.U ) new_pipfile.write(f"{package} = {version}\n") if package not in legacy_pipfile_packages: - pipfile_changes.append( - f"Package '{package}' added ({version})" - ) + pipfile_changes.append(f"Package '{package}' added ({version})") elif legacy_pipfile_packages[package] != version: pipfile_changes.append( f"Package '{package}' version changed from "