diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1827289b76..41ea643642 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.6 + rev: v0.9.3 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix @@ -13,7 +13,7 @@ repos: - prettier@3.3.3 - repo: https://github.com/editorconfig-checker/editorconfig-checker.python - rev: "3.0.3" + rev: "3.1.2" hooks: - id: editorconfig-checker alias: ec diff --git a/CHANGELOG.md b/CHANGELOG.md index 1197a379b6..7527b374cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # nf-core/tools: Changelog +## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] + +### Template + +- Remove automated release tweets ([#3419](https://github.com/nf-core/tools/pull/3419)) +- Update template components ([#3426](https://github.com/nf-core/tools/pull/3426)) +- Fix `process.shell` in `nextflow.config` ([#3416](https://github.com/nf-core/tools/pull/3416)) and split into new lines ([#3425](https://github.com/nf-core/tools/pull/3425)) + +### Modules + +- Modules created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) + +### Subworkflows + +- Subworkflows created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) + +### General + +- Update pre-commit hook editorconfig-checker/editorconfig-checker.python to v3.1.2 ([#3414](https://github.com/nf-core/tools/pull/3414)) +- Update python:3.12-slim Docker digest to 123be56 ([#3421](https://github.com/nf-core/tools/pull/3421)) + ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] ### Template diff --git a/Dockerfile b/Dockerfile index f5a6796a27..d26381a066 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim@sha256:10f3aaab98db50cba827d3b33a91f39dc9ec2d02ca9b85cbc5008220d07b17f3 +FROM python:3.12-slim@sha256:123be5684f39d8476e64f47a5fddf38f5e9d839baff5c023c815ae5bdfae0df7 LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools" diff --git a/docs/api/_src/pipeline_lint_tests/local_component_structure.md b/docs/api/_src/pipeline_lint_tests/local_component_structure.md new file mode 100644 index 0000000000..1884d862be --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/local_component_structure.md @@ -0,0 +1,5 @@ +# modules_structure + +```{eval-rst} +.. automethod:: nf_core.pipelines.lint.PipelineLint.local_component_structure +``` diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index f25fb33a6f..f04bb7da6b 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -71,6 +71,10 @@ def get_local_components(self) -> List[str]: """ local_component_dir = Path(self.directory, self.component_type, "local") return [ + str(Path(directory).relative_to(local_component_dir)) + for directory, _, files in os.walk(local_component_dir) + if "main.nf" in files + ] + [ str(path.relative_to(local_component_dir)) for path in local_component_dir.iterdir() if path.suffix == ".nf" ] diff --git a/nf_core/components/components_differ.py b/nf_core/components/components_differ.py index db51c1910d..18454471f4 100644 --- a/nf_core/components/components_differ.py +++ b/nf_core/components/components_differ.py @@ -271,9 +271,7 @@ def print_diff( console = Console(force_terminal=nf_core.utils.rich_force_colors()) if current_version is not None and new_version is not None: log.info( - f"Changes in component '{Path(repo_path, component)}' between" - f" ({current_version}) and" - f" ({new_version})" + f"Changes in component '{Path(repo_path, component)}' between ({current_version}) and ({new_version})" ) else: log.info(f"Changes in component '{Path(repo_path, component)}'") diff --git a/nf_core/components/create.py b/nf_core/components/create.py index c781905618..0a2c6aaff7 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -75,11 +75,11 @@ def create(self) -> bool: e.g bam_sort or bam_sort_samtools, respectively. If is a pipeline, this function creates a file called: - '/modules/local/tool.nf' + '/modules/local/tool/main.nf' OR - '/modules/local/tool_subtool.nf' + '/modules/local/tool/subtool/main.nf' OR for subworkflows - '/subworkflows/local/subworkflow_name.nf' + '/subworkflows/local/subworkflow_name/main.nf' If is a clone of nf-core/modules, it creates or modifies the following files: @@ -355,70 +355,46 @@ def _get_component_dirs(self) -> Dict[str, Path]: """ file_paths = {} if self.repo_type == "pipeline": - local_component_dir = Path(self.directory, self.component_type, "local") - # Check whether component file already exists - component_file = local_component_dir / f"{self.component_name}.nf" - if component_file.exists() and not self.force_overwrite: - raise UserWarning( - f"{self.component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite" - ) - - if self.component_type == "modules": - # If a subtool, check if there is a module called the base tool name already - if self.subtool and (local_component_dir / f"{self.component}.nf").exists(): - raise UserWarning( - f"Module '{self.component}' exists already, cannot make subtool '{self.component_name}'" - ) - - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob(f"{local_component_dir}/{self.component}_*.nf") - if not self.subtool and tool_glob: - raise UserWarning( - f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" - ) - - # Set file paths - file_paths["main.nf"] = component_file + component_dir = Path(self.directory, self.component_type, "local", self.component_dir) elif self.repo_type == "modules": component_dir = Path(self.directory, self.component_type, self.org, self.component_dir) - # Check if module/subworkflow directories exist already - if component_dir.exists() and not self.force_overwrite and not self.migrate_pytest: - raise UserWarning( - f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite" - ) + else: + raise ValueError("`repo_type` not set correctly") - if self.component_type == "modules": - # If a subtool, check if there is a module called the base tool name already - parent_tool_main_nf = Path( - self.directory, - self.component_type, - self.org, - self.component, - "main.nf", + # Check if module/subworkflow directories exist already + if component_dir.exists() and not self.force_overwrite and not self.migrate_pytest: + raise UserWarning( + f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite" + ) + + if self.component_type == "modules": + # If a subtool, check if there is a module called the base tool name already + parent_tool_main_nf = Path( + self.directory, + self.component_type, + self.org, + self.component, + "main.nf", + ) + if self.subtool and parent_tool_main_nf.exists() and not self.migrate_pytest: + raise UserWarning( + f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" ) - if self.subtool and parent_tool_main_nf.exists() and not self.migrate_pytest: - raise UserWarning( - f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" - ) - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob( - f"{Path(self.directory, self.component_type, self.org, self.component)}/*/main.nf" + # If no subtool, check that there isn't already a tool/subtool + tool_glob = glob.glob(f"{Path(self.directory, self.component_type, self.org, self.component)}/*/main.nf") + if not self.subtool and tool_glob and not self.migrate_pytest: + raise UserWarning( + f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" ) - if not self.subtool and tool_glob and not self.migrate_pytest: - raise UserWarning( - f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" - ) - # Set file paths - # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure - file_paths["main.nf"] = component_dir / "main.nf" - file_paths["meta.yml"] = component_dir / "meta.yml" - if self.component_type == "modules": - file_paths["environment.yml"] = component_dir / "environment.yml" - file_paths["tests/main.nf.test.j2"] = component_dir / "tests" / "main.nf.test" - else: - raise ValueError("`repo_type` not set correctly") + # Set file paths + # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure + file_paths["main.nf"] = component_dir / "main.nf" + file_paths["meta.yml"] = component_dir / "meta.yml" + if self.component_type == "modules": + file_paths["environment.yml"] = component_dir / "environment.yml" + file_paths["tests/main.nf.test.j2"] = component_dir / "tests" / "main.nf.test" return file_paths diff --git a/nf_core/components/info.py b/nf_core/components/info.py index 31769785a1..4cf7dc946c 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -265,9 +265,9 @@ def generate_component_info_help(self): intro_text.append( Text.from_markup( ":globe_with_meridians: Repository: " - f"{ '[link={self.remote_location}]' if self.remote_location.startswith('http') else ''}" + f"{'[link={self.remote_location}]' if self.remote_location.startswith('http') else ''}" f"{self.remote_location}" - f"{'[/link]' if self.remote_location.startswith('http') else '' }" + f"{'[/link]' if self.remote_location.startswith('http') else ''}" "\n" ) ) diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index 69740135a8..0a4fcbb3d1 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -102,7 +102,7 @@ def __init__( continue if isinstance(components, str): raise LookupError( - f"Error parsing modules.json: {components}. " f"Please check the file for errors or try again." + f"Error parsing modules.json: {components}. Please check the file for errors or try again." ) for org, comp in components: self.all_remote_components.append( @@ -162,6 +162,10 @@ def _set_registry(self, registry) -> None: self.registry = registry log.debug(f"Registry set to {self.registry}") + @property + def local_module_exclude_tests(self): + return ["module_version", "module_changes", "modules_patch"] + @staticmethod def get_all_module_lint_tests(is_pipeline): if is_pipeline: diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 81c0ba98e7..b04cb71e35 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -74,8 +74,8 @@ def __init__( repo_dir = self.component_dir.parts[:name_index][-1] self.org = repo_dir - self.nftest_testdir = Path(self.component_dir, "tests") - self.nftest_main_nf = Path(self.nftest_testdir, "main.nf.test") + self.nftest_testdir: Optional[Path] = Path(self.component_dir, "tests") + self.nftest_main_nf: Optional[Path] = Path(self.nftest_testdir, "main.nf.test") if self.repo_type == "pipeline": patch_fn = f"{self.component_name.replace('/', '-')}.diff" @@ -85,15 +85,23 @@ def __init__( self.patch_path = patch_path else: # The main file is just the local module - self.main_nf = self.component_dir - self.component_name = self.component_dir.stem - # These attributes are only used by nf-core modules - # so just initialize them to None - self.meta_yml = None - self.environment_yml = None - self.test_dir = None - self.test_yml = None - self.test_main_nf = None + if self.component_dir.is_dir(): + self.main_nf = Path(self.component_dir, "main.nf") + self.component_name = self.component_dir.stem + # These attributes are only required by nf-core modules + # so just set them to None if they don't exist + self.meta_yml = p if (p := Path(self.component_dir, "meta.yml")).exists() else None + self.environment_yml = p if (p := Path(self.component_dir, "environment.yml")).exists() else None + self.nftest_testdir = p if (p := Path(self.component_dir, "tests")).exists() else None + if self.nftest_testdir is not None: + self.nftest_main_nf = p if (p := Path(self.nftest_testdir, "main.nf.test")).exists() else None + else: + self.main_nf = self.component_dir + self.component_dir = self.component_dir.parent + self.meta_yml = None + self.environment_yml = None + self.nftest_testdir = None + self.nftest_main_nf = None self.process_name: str = self._get_process_name() diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index 37208629c0..316b8e7cb4 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -176,6 +176,6 @@ def remove(self, component, removed_by=None, removed_components=None, force=Fals f"Did not remove '{component}', because it was also manually installed. Only updated 'installed_by' entry in modules.json." ) log.info( - f"""Did not remove {self.component_type[:-1]} '{component}', because it was also installed by {', '.join(f"'{d}'" for d in installed_by)}. Only updated the 'installed_by' entry in modules.json.""" + f"""Did not remove {self.component_type[:-1]} '{component}', because it was also installed by {", ".join(f"'{d}'" for d in installed_by)}. Only updated the 'installed_by' entry in modules.json.""" ) return removed diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 49012cff40..5f64953116 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -117,7 +117,7 @@ def lint( """ # TODO: consider unifying modules and subworkflows lint() function and add it to the ComponentLint class # Prompt for module or all - if module is None and not all_modules and len(self.all_remote_components) > 0: + if module is None and not (local or all_modules) and len(self.all_remote_components) > 0: questions = [ { "type": "list", @@ -170,7 +170,7 @@ def lint( self.lint_modules(local_modules, registry=registry, local=True, fix_version=fix_version) # Lint nf-core modules - if len(remote_modules) > 0: + if not local and len(remote_modules) > 0: self.lint_modules(remote_modules, registry=registry, local=False, fix_version=fix_version) if print_results: @@ -234,7 +234,23 @@ def lint_module( # TODO: consider unifying modules and subworkflows lint_module() function and add it to the ComponentLint class # Only check the main script in case of a local module if local: - self.main_nf(mod, fix_version, self.registry, progress_bar) + mod.get_inputs_from_main_nf() + mod.get_outputs_from_main_nf() + # Update meta.yml file if requested + if self.fix and mod.meta_yml is not None: + self.update_meta_yml_file(mod) + + for test_name in self.lint_tests: + if test_name in self.local_module_exclude_tests: + continue + if test_name == "main_nf": + getattr(self, test_name)(mod, fix_version, self.registry, progress_bar) + elif test_name in ["meta_yml", "environment_yml"]: + # Allow files to be missing for local + getattr(self, test_name)(mod, allow_missing=True) + else: + getattr(self, test_name)(mod) + self.passed += [LintResult(mod, *m) for m in mod.passed] warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)] if not self.fail_warned: diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py index 4488b0befa..6571f07ae8 100644 --- a/nf_core/modules/lint/environment_yml.py +++ b/nf_core/modules/lint/environment_yml.py @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) -def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None: +def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent, allow_missing: bool = False) -> None: """ Lint an ``environment.yml`` file. @@ -23,6 +23,15 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) env_yml = None # load the environment.yml file if module.environment_yml is None: + if allow_missing: + module.warned.append( + ( + "environment_yml_exists", + "Module's `environment.yml` does not exist", + Path(module.component_dir, "environment.yml"), + ), + ) + return raise LintExceptionError("Module does not have an `environment.yml` file") try: with open(module.environment_yml) as fh: diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index d0268a40cc..977ea819ae 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -13,7 +13,7 @@ log = logging.getLogger(__name__) -def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None: +def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent, allow_missing: bool = False) -> None: """ Lint a ``meta.yml`` file @@ -42,7 +42,13 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None module (NFCoreComponent): The module to lint """ - + if module.meta_yml is None: + if allow_missing: + module.warned.append( + ("meta_yml_exists", "Module `meta.yml` does not exist", Path(module.component_dir, "meta.yml")) + ) + return + raise LintExceptionError("Module does not have a `meta.yml` file") # Check if we have a patch file, get original file in that case meta_yaml = read_meta_yml(module_lint_object, module) if module.is_patched and module_lint_object.modules_repo.repo_path is not None: @@ -56,9 +62,7 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None ).get("meta.yml") if lines is not None: yaml = ruamel.yaml.YAML() - meta_yaml = yaml.safe_load("".join(lines)) - if module.meta_yml is None: - raise LintExceptionError("Module does not have a `meta.yml` file") + meta_yaml = yaml.load("".join(lines)) if meta_yaml is None: module.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", module.meta_yml)) return @@ -78,7 +82,7 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None if len(e.path) > 0: hint = f"\nCheck the entry for `{e.path[0]}`." if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: - hint = f"\nCheck that the child entries of {str(e.path[0])+'.'+str(e.path[2])} are indented correctly." + hint = f"\nCheck that the child entries of {str(e.path[0]) + '.' + str(e.path[2])} are indented correctly." if e.schema and isinstance(e.schema, dict) and "message" in e.schema: e.message = e.schema["message"] incorrect_value = meta_yaml diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 6722c12129..6826b2e743 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -9,12 +9,13 @@ import yaml +from nf_core.components.lint import LintExceptionError from nf_core.components.nfcore_component import NFCoreComponent log = logging.getLogger(__name__) -def module_tests(_, module: NFCoreComponent): +def module_tests(_, module: NFCoreComponent, allow_missing: bool = False): """ Lint the tests of a module in ``nf-core/modules`` @@ -22,6 +23,30 @@ def module_tests(_, module: NFCoreComponent): and contains a ``main.nf.test`` and a ``main.nf.test.snap`` """ + if module.nftest_testdir is None: + if allow_missing: + module.warned.append( + ( + "test_dir_exists", + "nf-test directory is missing", + Path(module.component_dir, "tests"), + ) + ) + return + raise LintExceptionError("Module does not have a `tests` dir") + + if module.nftest_main_nf is None: + if allow_missing: + module.warned.append( + ( + "test_main_nf_exists", + "test `main.nf.test` does not exist", + Path(module.component_dir, "tests", "main.nf.test"), + ) + ) + return + raise LintExceptionError("Module does not have a `tests` dir") + repo_dir = module.component_dir.parts[: module.component_dir.parts.index(module.component_name.split("/")[0])][-1] test_dir = Path(module.base_dir, "tests", "modules", repo_dir, module.component_name) pytest_main_nf = Path(test_dir, "main.nf") diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index 63b20bb311..b0b6755858 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 + uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8 with: workflow: linting.yml workflow_conclusion: completed diff --git a/nf_core/pipeline-template/.github/workflows/release-announcements.yml b/nf_core/pipeline-template/.github/workflows/release-announcements.yml index e1b654d34b..820cc49695 100644 --- a/nf_core/pipeline-template/.github/workflows/release-announcements.yml +++ b/nf_core/pipeline-template/.github/workflows/release-announcements.yml @@ -27,39 +27,6 @@ jobs: ${{ steps.get_topics.outputs.topics }} #nfcore #openscience #nextflow #bioinformatics - send-tweet: - runs-on: ubuntu-latest - - steps: - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.10" - - name: Install dependencies - run: pip install tweepy==4.14.0 - - name: Send tweet - shell: python - run: | - import os - import tweepy - - client = tweepy.Client( - access_token=os.getenv("TWITTER_ACCESS_TOKEN"), - access_token_secret=os.getenv("TWITTER_ACCESS_TOKEN_SECRET"), - consumer_key=os.getenv("TWITTER_CONSUMER_KEY"), - consumer_secret=os.getenv("TWITTER_CONSUMER_SECRET"), - ) - tweet = os.getenv("TWEET") - client.create_tweet(text=tweet) - env: - TWEET: | - Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}! - - Please see the changelog: ${{ github.event.release.html_url }} - TWITTER_CONSUMER_KEY: ${{ secrets.TWITTER_CONSUMER_KEY }} - TWITTER_CONSUMER_SECRET: ${{ secrets.TWITTER_CONSUMER_SECRET }} - TWITTER_ACCESS_TOKEN: ${{ secrets.TWITTER_ACCESS_TOKEN }} - TWITTER_ACCESS_TOKEN_SECRET: ${{ secrets.TWITTER_ACCESS_TOKEN_SECRET }} - bsky-post: runs-on: ubuntu-latest steps: diff --git a/nf_core/pipeline-template/.pre-commit-config.yaml b/nf_core/pipeline-template/.pre-commit-config.yaml index 9e9f0e1c4e..1dec865026 100644 --- a/nf_core/pipeline-template/.pre-commit-config.yaml +++ b/nf_core/pipeline-template/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: - prettier@3.2.5 - repo: https://github.com/editorconfig-checker/editorconfig-checker.python - rev: "3.0.3" + rev: "3.1.2" hooks: - id: editorconfig-checker alias: ec diff --git a/nf_core/pipeline-template/docs/output.md b/nf_core/pipeline-template/docs/output.md index a9be6620e1..d9bc3a188f 100644 --- a/nf_core/pipeline-template/docs/output.md +++ b/nf_core/pipeline-template/docs/output.md @@ -15,7 +15,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d {% if fastqc -%} - [FastQC](#fastqc) - Raw read QC{% endif %} - {%- if multiqc -%} + {%- if multiqc %} - [MultiQC](#multiqc) - Aggregate report describing results and QC from the whole pipeline {%- endif %} - [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution @@ -34,7 +34,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d [FastQC](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/) gives general quality metrics about your sequenced reads. It provides information about the quality score distribution across your reads, per base sequence content (%A/T/G/C), adapter contamination and overrepresented sequences. For further reading and documentation see the [FastQC help pages](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/). -{%- endif -%} +{%- endif %} {% if multiqc -%} @@ -53,7 +53,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d [MultiQC](http://multiqc.info) is a visualization tool that generates a single HTML report summarising all samples in your project. Most of the pipeline QC results are visualised in the report and further statistics are available in the report data directory. Results generated by MultiQC collate pipeline QC from supported tools e.g. FastQC. The pipeline has special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see . -{%- endif -%} +{%- endif %} ### Pipeline information diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 7d2761d290..750826ffac 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -8,12 +8,12 @@ {%- if fastqc %} "fastqc": { "branch": "master", - "git_sha": "dc94b6ee04a05ddb9f7ae050712ff30a13149164", + "git_sha": "08108058ea36a63f141c25c4e75f9f872a5b2296", "installed_by": ["modules"] }{% endif %}{%- if multiqc %}{% if fastqc %},{% endif %} "multiqc": { "branch": "master", - "git_sha": "cf17ca47590cc578dfb47db1c2a44ef86f89976d", + "git_sha": "f0719ae309075ae4a291533883847c3f7c441dad", "installed_by": ["modules"] } {%- endif %} diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf index 752c3a10c6..033f4154a4 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf @@ -1,5 +1,5 @@ process FASTQC { - tag "$meta.id" + tag "${meta.id}" label 'process_medium' conda "${moduleDir}/environment.yml" @@ -19,30 +19,30 @@ process FASTQC { task.ext.when == null || task.ext.when script: - def args = task.ext.args ?: '' - def prefix = task.ext.prefix ?: "${meta.id}" + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" // Make list of old name and new name pairs to use for renaming in the bash while loop def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } - def rename_to = old_new_pairs*.join(' ').join(' ') + def rename_to = old_new_pairs*.join(' ').join(' ') def renamed_files = old_new_pairs.collect{ _old_name, new_name -> new_name }.join(' ') // The total amount of allocated RAM by FastQC is equal to the number of threads defined (--threads) time the amount of RAM defined (--memory) // https://github.com/s-andrews/FastQC/blob/1faeea0412093224d7f6a07f777fad60a5650795/fastqc#L211-L222 // Dividing the task.memory by task.cpu allows to stick to requested amount of RAM in the label - def memory_in_mb = MemoryUnit.of("${task.memory}").toUnit('MB') / task.cpus + def memory_in_mb = task.memory ? task.memory.toUnit('MB').toFloat() / task.cpus : null // FastQC memory value allowed range (100 - 10000) def fastqc_memory = memory_in_mb > 10000 ? 10000 : (memory_in_mb < 100 ? 100 : memory_in_mb) """ - printf "%s %s\\n" $rename_to | while read old_name new_name; do + printf "%s %s\\n" ${rename_to} | while read old_name new_name; do [ -f "\${new_name}" ] || ln -s \$old_name \$new_name done fastqc \\ - $args \\ - --threads $task.cpus \\ - --memory $fastqc_memory \\ - $renamed_files + ${args} \\ + --threads ${task.cpus} \\ + --memory ${fastqc_memory} \\ + ${renamed_files} cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml deleted file mode 100644 index 7834294ba0..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -fastqc: - - modules/nf-core/fastqc/** diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml index 6f5b867b76..a27122ce1a 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml @@ -2,4 +2,4 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::multiqc=1.25.1 + - bioconda::multiqc=1.27 diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf index cc0643e1d5..58d9313c6e 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf @@ -3,8 +3,8 @@ process MULTIQC { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.25.1--pyhdfd78af_0' : - 'biocontainers/multiqc:1.25.1--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.27--pyhdfd78af_0' : + 'biocontainers/multiqc:1.27--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap index 2fcbb5ff7d..7b7c132205 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -2,14 +2,14 @@ "multiqc_versions_single": { "content": [ [ - "versions.yml:md5,41f391dcedce7f93ca188f3a3ffa0916" + "versions.yml:md5,8f3b8c1cec5388cf2708be948c9fa42f" ] ], "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.4" }, - "timestamp": "2024-10-02T17:51:46.317523" + "timestamp": "2025-01-27T09:29:57.631982377" }, "multiqc_stub": { "content": [ @@ -17,25 +17,25 @@ "multiqc_report.html", "multiqc_data", "multiqc_plots", - "versions.yml:md5,41f391dcedce7f93ca188f3a3ffa0916" + "versions.yml:md5,8f3b8c1cec5388cf2708be948c9fa42f" ] ], "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.4" }, - "timestamp": "2024-10-02T17:52:20.680978" + "timestamp": "2025-01-27T09:30:34.743726958" }, "multiqc_versions_config": { "content": [ [ - "versions.yml:md5,41f391dcedce7f93ca188f3a3ffa0916" + "versions.yml:md5,8f3b8c1cec5388cf2708be948c9fa42f" ] ], "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.4" }, - "timestamp": "2024-10-02T17:52:09.185842" + "timestamp": "2025-01-27T09:30:21.44383553" } } \ No newline at end of file diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 5bbe47d9d9..d58ca72196 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -238,14 +238,14 @@ env { } // Set bash options -process.shell = """\ -bash - -set -e # Exit if a tool returns a non-zero status/exit code -set -u # Treat unset variables and parameters as an error -set -o pipefail # Returns the status of the last command to exit with a non-zero status or zero if all successfully execute -set -C # No clobber - prevent output redirection from overwriting files. -""" +process.shell = [ + "bash", + "-C", // No clobber - prevent output redirection from overwriting files. + "-e", // Exit if a tool returns a non-zero status/exit code + "-u", // Treat unset variables and parameters as an error + "-o", // Returns the status of the last command to exit.. + "pipefail" // ..with a non-zero status or zero if all successfully execute +] // Disable process selector warnings by default. Use debug profile to enable warnings. nextflow.enable.configProcessNamesValidation = false diff --git a/nf_core/pipelines/download.py b/nf_core/pipelines/download.py index c0f6e8c2f3..3d371ca681 100644 --- a/nf_core/pipelines/download.py +++ b/nf_core/pipelines/download.py @@ -202,7 +202,7 @@ def download_workflow(self): raise DownloadError(e) from e summary_log = [ - f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',['+str(len(self.revision)-2)+' more revisions],'+self.revision[-1]}'", + f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0] + ',[' + str(len(self.revision) - 2) + ' more revisions],' + self.revision[-1]}'", f"Use containers: '{self.container_system}'", ] if self.container_system: @@ -1908,8 +1908,8 @@ def __init__( log.error(self.error_type.message) log.info(self.error_type.helpmessage) - log.debug(f'Failed command:\n{" ".join(singularity_command)}') - log.debug(f'Singularity error messages:\n{"".join(error_msg)}') + log.debug(f"Failed command:\n{' '.join(singularity_command)}") + log.debug(f"Singularity error messages:\n{''.join(error_msg)}") raise self.error_type diff --git a/nf_core/pipelines/lint/__init__.py b/nf_core/pipelines/lint/__init__.py index 154e38aea6..36f1bfca52 100644 --- a/nf_core/pipelines/lint/__init__.py +++ b/nf_core/pipelines/lint/__init__.py @@ -27,7 +27,7 @@ from nf_core import __version__ from nf_core.components.lint import ComponentLint from nf_core.pipelines.lint_utils import console -from nf_core.utils import NFCoreYamlConfig, NFCoreYamlLintConfig, strip_ansi_codes +from nf_core.utils import NFCoreYamlLintConfig, strip_ansi_codes from nf_core.utils import plural_s as _s from .actions_awsfulltest import actions_awsfulltest @@ -38,6 +38,7 @@ from .files_exist import files_exist from .files_unchanged import files_unchanged from .included_configs import included_configs +from .local_component_structure import local_component_structure from .merge_markers import merge_markers from .modules_json import modules_json from .modules_structure import modules_structure @@ -89,6 +90,7 @@ class PipelineLint(nf_core.utils.Pipeline): merge_markers = merge_markers modules_json = modules_json modules_structure = modules_structure + local_component_structure = local_component_structure multiqc_config = multiqc_config nextflow_config = nextflow_config nfcore_yml = nfcore_yml @@ -151,6 +153,7 @@ def _get_all_lint_tests(release_mode): "modules_json", "multiqc_config", "modules_structure", + "local_component_structure", "base_config", "modules_config", "nfcore_yml", diff --git a/nf_core/pipelines/lint/local_component_structure.py b/nf_core/pipelines/lint/local_component_structure.py new file mode 100644 index 0000000000..71f02ba545 --- /dev/null +++ b/nf_core/pipelines/lint/local_component_structure.py @@ -0,0 +1,38 @@ +import logging +from pathlib import Path + +log = logging.getLogger(__name__) + + +def local_component_structure(self): + """ + Check that the local modules and subworkflows directories in a pipeline have the correct format: + + .. code-block:: bash + + modules/local/TOOL/SUBTOOL + + Prior to nf-core/tools release 3.1.0 the directory structure allowed top-level `*.nf` files: + + .. code-block:: bash + + modules/local/modules/TOOL_SUBTOOL.nf + """ + warned_mods = [] + for nf_file in Path(self.wf_path, "modules", "local").glob("*.nf"): + warned_mods.append(f"{nf_file.name} in modules/local should be moved to a TOOL/SUBTOOL/main.nf structure") + # If there are modules installed in the wrong location + passed = [] + if len(warned_mods) == 0: + passed = ["local modules directory structure is correct 'modules/local/TOOL/SUBTOOL'"] + + warned_swfs = [] + for nf_file in Path(self.wf_path, "subworkflows", "local").glob("*.nf"): + warned_swfs.append( + f"{nf_file.name} in subworkflows/local should be moved to a SUBWORKFLOW_NAME/main.nf structure" + ) + + if len(warned_swfs) == 0: + passed = ["local subworkflows directory structure is correct 'subworkflows/local/TOOL/SUBTOOL'"] + + return {"passed": passed, "warned": warned_mods + warned_swfs, "failed": [], "ignored": []} diff --git a/nf_core/pipelines/lint/multiqc_config.py b/nf_core/pipelines/lint/multiqc_config.py index fec5b518e3..49f632faa8 100644 --- a/nf_core/pipelines/lint/multiqc_config.py +++ b/nf_core/pipelines/lint/multiqc_config.py @@ -122,7 +122,7 @@ def multiqc_config(self) -> Dict[str, List[str]]: f"The expected comment is: \n" f"```{hint}``` \n" f"The current comment is: \n" - f"```{ mqc_yml['report_comment'].strip()}```" + f"```{mqc_yml['report_comment'].strip()}```" ) else: passed.append("`assets/multiqc_config.yml` contains a matching 'report_comment'.") diff --git a/nf_core/pipelines/lint/version_consistency.py b/nf_core/pipelines/lint/version_consistency.py index 2f9cead83c..e347e8e825 100644 --- a/nf_core/pipelines/lint/version_consistency.py +++ b/nf_core/pipelines/lint/version_consistency.py @@ -53,7 +53,7 @@ def version_consistency(self): # Check if they are consistent if len(set(versions.values())) != 1: failed.append( - "The versioning is not consistent between container, release tag " "and config. Found {}".format( + "The versioning is not consistent between container, release tag and config. Found {}".format( ", ".join([f"{k} = {v}" for k, v in versions.items()]) ) ) diff --git a/nf_core/pipelines/params_file.py b/nf_core/pipelines/params_file.py index 69326c142d..48e4dfe46c 100644 --- a/nf_core/pipelines/params_file.py +++ b/nf_core/pipelines/params_file.py @@ -19,7 +19,7 @@ "of nextflow run with the {pipeline_name} pipeline." ) -USAGE = "Uncomment lines with a single '#' if you want to pass the parameter " "to the pipeline." +USAGE = "Uncomment lines with a single '#' if you want to pass the parameter to the pipeline." H1_SEPERATOR = "## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" H2_SEPERATOR = "## ----------------------------------------------------------------------------" @@ -107,7 +107,7 @@ def get_pipeline(self) -> Optional[bool]: # Prompt for pipeline if not supplied if self.pipeline is None: launch_type = questionary.select( - "Generate parameter file for local pipeline " "or remote GitHub pipeline?", + "Generate parameter file for local pipeline or remote GitHub pipeline?", choices=["Remote pipeline", "Local path"], style=nf_core.utils.nfcore_question_style, ).unsafe_ask() diff --git a/nf_core/pipelines/rocrate.py b/nf_core/pipelines/rocrate.py index b0f9611f1f..2bf46beede 100644 --- a/nf_core/pipelines/rocrate.py +++ b/nf_core/pipelines/rocrate.py @@ -212,7 +212,7 @@ def set_main_entity(self, main_entity_filename: str): else: url = self.version self.crate.mainEntity.append_to( - "url", f"https://nf-co.re/{self.crate.name.replace('nf-core/','')}/{url}/", compact=True + "url", f"https://nf-co.re/{self.crate.name.replace('nf-core/', '')}/{url}/", compact=True ) self.crate.mainEntity.append_to("version", self.version, compact=True) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index b425ec64ed..47990b1064 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -770,7 +770,7 @@ def build_schema(self, pipeline_dir, no_prompts, web_only, url): if self.web_schema_build_web_url: log.info( "To save your work, open {}\n" - f"Click the blue 'Finished' button, copy the schema and paste into this file: { self.web_schema_build_web_url, self.schema_filename}" + f"Click the blue 'Finished' button, copy the schema and paste into this file: {self.web_schema_build_web_url, self.schema_filename}" ) return False diff --git a/nf_core/pipelines/sync.py b/nf_core/pipelines/sync.py index 14365da3f8..f0c8f4a321 100644 --- a/nf_core/pipelines/sync.py +++ b/nf_core/pipelines/sync.py @@ -345,7 +345,7 @@ def create_merge_base_branch(self): if merge_branch_format.match(branch) ] ) - new_branch = f"{self.original_merge_branch}-{max_branch+1}" + new_branch = f"{self.original_merge_branch}-{max_branch + 1}" log.info(f"Branch already existed: '{self.merge_branch}', creating branch '{new_branch}' instead.") self.merge_branch = new_branch diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index cedae62f11..c1be26d0d5 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -99,7 +99,7 @@ def lint( """ # TODO: consider unifying modules and subworkflows lint() function and add it to the ComponentLint class # Prompt for subworkflow or all - if subworkflow is None and not all_subworkflows: + if subworkflow is None and not (local or all_subworkflows): questions = [ { "type": "list", @@ -152,7 +152,7 @@ def lint( self.lint_subworkflows(local_subworkflows, registry=registry, local=True) # Lint nf-core subworkflows - if len(remote_subworkflows) > 0: + if not local and len(remote_subworkflows) > 0: self.lint_subworkflows(remote_subworkflows, registry=registry, local=False) if print_results: @@ -208,6 +208,8 @@ def lint_subworkflow(self, swf, progress_bar, registry, local=False): # Only check the main script in case of a local subworkflow if local: self.main_nf(swf) + self.meta_yml(swf, allow_missing=True) + self.subworkflow_todos(swf) self.passed += [LintResult(swf, *s) for s in swf.passed] warned = [LintResult(swf, *m) for m in (swf.warned + swf.failed)] if not self.fail_warned: diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index be282bc453..8a2120ed0a 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -3,14 +3,15 @@ from pathlib import Path import jsonschema.validators -import yaml +import ruamel.yaml import nf_core.components.components_utils +from nf_core.components.lint import LintExceptionError log = logging.getLogger(__name__) -def meta_yml(subworkflow_lint_object, subworkflow): +def meta_yml(subworkflow_lint_object, subworkflow, allow_missing: bool = False): """ Lint a ``meta.yml`` file @@ -28,9 +29,22 @@ def meta_yml(subworkflow_lint_object, subworkflow): """ # Read the meta.yml file + if subworkflow.meta_yml is None: + if allow_missing: + subworkflow.warned.append( + ( + "meta_yml_exists", + "Subworkflow `meta.yml` does not exist", + Path(subworkflow.component_dir, "meta.yml"), + ) + ) + return + raise LintExceptionError("Subworkflow does not have a `meta.yml` file") + try: with open(subworkflow.meta_yml) as fh: - meta_yaml = yaml.safe_load(fh) + yaml = ruamel.yaml.YAML(typ="safe") + meta_yaml = yaml.load(fh) subworkflow.passed.append(("meta_yml_exists", "Subworkflow `meta.yml` exists", subworkflow.meta_yml)) except FileNotFoundError: subworkflow.failed.append(("meta_yml_exists", "Subworkflow `meta.yml` does not exist", subworkflow.meta_yml)) @@ -49,7 +63,7 @@ def meta_yml(subworkflow_lint_object, subworkflow): if len(e.path) > 0: hint = f"\nCheck the entry for `{e.path[0]}`." if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: - hint = f"\nCheck that the child entries of {e.path[0]+'.'+e.path[2]} are indented correctly." + hint = f"\nCheck that the child entries of {str(e.path[0]) + '.' + str(e.path[2])} are indented correctly." subworkflow.failed.append( ( "meta_yml_valid", @@ -96,10 +110,9 @@ def meta_yml(subworkflow_lint_object, subworkflow): ) # confirm that all included components in ``main.nf`` are specified in ``meta.yml`` - included_components = nf_core.components.components_utils.get_components_to_install(subworkflow.component_dir) - included_components = ( - included_components[0] + included_components[1] - ) # join included modules and included subworkflows in a single list + included_components_ = nf_core.components.components_utils.get_components_to_install(subworkflow.component_dir) + included_components = included_components_[0] + included_components_[1] + # join included modules and included subworkflows in a single list if "components" in meta_yaml: meta_components = [x for x in meta_yaml["components"]] for component in set(included_components): diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 8e9e62430a..74b170416d 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -9,12 +9,13 @@ import yaml +from nf_core.components.lint import LintExceptionError from nf_core.components.nfcore_component import NFCoreComponent log = logging.getLogger(__name__) -def subworkflow_tests(_, subworkflow: NFCoreComponent): +def subworkflow_tests(_, subworkflow: NFCoreComponent, allow_missing: bool = False): """ Lint the tests of a subworkflow in ``nf-core/modules`` @@ -23,6 +24,29 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): Additionally, checks that all included components in test ``main.nf`` are specified in ``test.yml`` """ + if subworkflow.nftest_testdir is None: + if allow_missing: + subworkflow.warned.append( + ( + "test_dir_exists", + "nf-test directory is missing", + Path(subworkflow.component_dir, "tests"), + ) + ) + return + raise LintExceptionError("Module does not have a `tests` dir") + + if subworkflow.nftest_main_nf is None: + if allow_missing: + subworkflow.warned.append( + ( + "test_main_nf_exists", + "test `main.nf.test` does not exist", + Path(subworkflow.component_dir, "tests", "main.nf.test"), + ) + ) + return + raise LintExceptionError("Subworkflow does not have a `tests` dir") repo_dir = subworkflow.component_dir.parts[ : subworkflow.component_dir.parts.index(subworkflow.component_name.split("/")[0]) diff --git a/nf_core/utils.py b/nf_core/utils.py index 27334d473c..c0d7029e46 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1244,6 +1244,8 @@ class NFCoreYamlLintConfig(BaseModel): """ Lint for version consistency """ included_configs: Optional[bool] = None """ Lint for included configs """ + local_component_structure: Optional[bool] = None + """ Lint local components use correct structure mirroring remote""" def __getitem__(self, item: str) -> Any: return getattr(self, item) diff --git a/setup.py b/setup.py index fb1621adfc..d426478786 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "3.1.2" +version = "3.2.0" with open("README.md") as f: readme = f.read() diff --git a/tests/modules/test_create.py b/tests/modules/test_create.py index 219f869997..4ae9f9b5c4 100644 --- a/tests/modules/test_create.py +++ b/tests/modules/test_create.py @@ -31,7 +31,7 @@ def test_modules_create_succeed(self): ) with requests_cache.disabled(): module_create.create() - assert os.path.exists(os.path.join(self.pipeline_dir, "modules", "local", "trimgalore.nf")) + assert os.path.exists(os.path.join(self.pipeline_dir, "modules", "local", "trimgalore/main.nf")) def test_modules_create_fail_exists(self): """Fail at creating the same module twice""" @@ -46,7 +46,7 @@ def test_modules_create_fail_exists(self): with pytest.raises(UserWarning) as excinfo: with requests_cache.disabled(): module_create.create() - assert "Module file exists already" in str(excinfo.value) + assert "module directory exists:" in str(excinfo.value) def test_modules_create_nfcore_modules(self): """Create a module in nf-core/modules clone""" diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index 5372807987..c641b6dc97 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -1,4 +1,5 @@ import json +import shutil from pathlib import Path from typing import Union @@ -158,7 +159,7 @@ ] -class TestModulesCreate(TestModules): +class TestModulesLint(TestModules): def _setup_patch(self, pipeline_dir: Union[str, Path], modify_module: bool): install_obj = nf_core.modules.install.ModuleInstall( pipeline_dir, @@ -295,15 +296,15 @@ def test_modules_lint_check_url(self): if line.strip(): check_container_link_line(mocked_ModuleLint, line, registry="quay.io") - assert ( - len(mocked_ModuleLint.passed) == passed - ), f"{test}: Expected {passed} PASS, got {len(mocked_ModuleLint.passed)}." - assert ( - len(mocked_ModuleLint.warned) == warned - ), f"{test}: Expected {warned} WARN, got {len(mocked_ModuleLint.warned)}." - assert ( - len(mocked_ModuleLint.failed) == failed - ), f"{test}: Expected {failed} FAIL, got {len(mocked_ModuleLint.failed)}." + assert len(mocked_ModuleLint.passed) == passed, ( + f"{test}: Expected {passed} PASS, got {len(mocked_ModuleLint.passed)}." + ) + assert len(mocked_ModuleLint.warned) == warned, ( + f"{test}: Expected {warned} WARN, got {len(mocked_ModuleLint.warned)}." + ) + assert len(mocked_ModuleLint.failed) == failed, ( + f"{test}: Expected {failed} FAIL, got {len(mocked_ModuleLint.failed)}." + ) def test_modules_lint_update_meta_yml(self): """update the meta.yml of a module""" @@ -760,6 +761,46 @@ def test_modules_empty_file_in_stub_snapshot(self): with open(snap_file, "w") as fh: fh.write(content) + def test_modules_lint_local(self): + assert self.mods_install.install("trimgalore") + installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore") + local = Path(self.pipeline_dir, "modules", "local", "trimgalore") + shutil.move(installed, local) + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, local=True) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + + def test_modules_lint_local_missing_files(self): + assert self.mods_install.install("trimgalore") + installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore") + local = Path(self.pipeline_dir, "modules", "local", "trimgalore") + shutil.move(installed, local) + Path(self.pipeline_dir, "modules", "local", "trimgalore", "environment.yml").unlink() + Path(self.pipeline_dir, "modules", "local", "trimgalore", "meta.yml").unlink() + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, local=True) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + warnings = [x.message for x in module_lint.warned] + assert "Module's `environment.yml` does not exist" in warnings + assert "Module `meta.yml` does not exist" in warnings + + def test_modules_lint_local_old_format(self): + Path(self.pipeline_dir, "modules", "local").mkdir() + assert self.mods_install.install("trimgalore") + installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore", "main.nf") + local = Path(self.pipeline_dir, "modules", "local", "trimgalore.nf") + shutil.move(installed, local) + self.mods_remove.remove("trimgalore", force=True) + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, local=True) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + # A skeleton object with the passed/warned/failed list attrs # Use this in place of a ModuleLint object to test behaviour of diff --git a/tests/pipelines/lint/test_local_component_structure.py b/tests/pipelines/lint/test_local_component_structure.py new file mode 100644 index 0000000000..93dc3174a3 --- /dev/null +++ b/tests/pipelines/lint/test_local_component_structure.py @@ -0,0 +1,28 @@ +from pathlib import Path + +import nf_core.pipelines.lint + +from ..test_lint import TestLint + + +class TestLintLocalComponentStructure(TestLint): + def setUp(self) -> None: + super().setUp() + self.new_pipeline = self._make_pipeline_copy() + + def test_local_component_structure(self): + local_modules = Path(self.new_pipeline, "modules", "local") + local_swf = Path(self.new_pipeline, "subworkflows", "local") + local_modules.mkdir(parents=True, exist_ok=True) + local_swf.mkdir(parents=True, exist_ok=True) + + (local_modules / "dummy_module.nf").touch() + (local_swf / "dummy_subworkflow.nf").touch() + + lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) + lint_obj._load() + + results = lint_obj.local_component_structure() + assert len(results.get("warned", [])) == 2 + assert len(results.get("failed", [])) == 0 + assert len(results.get("ignored", [])) == 0 diff --git a/tests/pipelines/test_create.py b/tests/pipelines/test_create.py index f83cc274fc..7c0eebe7d6 100644 --- a/tests/pipelines/test_create.py +++ b/tests/pipelines/test_create.py @@ -158,9 +158,9 @@ def test_template_customisation_all_files_grouping(self): str_path = str((Path(root) / file).relative_to(PIPELINE_TEMPLATE)) if str_path not in base_required_files: try: - assert ( - str_path in all_skipped_files - ), f"Template file `{str_path}` not present in a group for pipeline customisation in `template_features.yml`." + assert str_path in all_skipped_files, ( + f"Template file `{str_path}` not present in a group for pipeline customisation in `template_features.yml`." + ) except AssertionError: if "/" in str_path: # Check if the parent directory is in the skipped files @@ -170,6 +170,8 @@ def test_template_customisation_all_files_grouping(self): if upper_dir in all_skipped_files: upper_dir_present = True break - assert upper_dir_present, f"Template file `{str_path}` not present in a group for pipeline customisation in `template_features.yml`." + assert upper_dir_present, ( + f"Template file `{str_path}` not present in a group for pipeline customisation in `template_features.yml`." + ) else: raise diff --git a/tests/subworkflows/test_create.py b/tests/subworkflows/test_create.py index 48cb482260..704a23772e 100644 --- a/tests/subworkflows/test_create.py +++ b/tests/subworkflows/test_create.py @@ -19,7 +19,7 @@ def test_subworkflows_create_succeed(self): self.pipeline_dir, "test_subworkflow_local", "@author", True ) subworkflow_create.create() - assert Path(self.pipeline_dir, "subworkflows", "local", "test_subworkflow_local.nf").exists() + assert Path(self.pipeline_dir, "subworkflows", "local", "test_subworkflow_local/main.nf").exists() def test_subworkflows_create_fail_exists(self): """Fail at creating the same subworkflow twice""" @@ -29,7 +29,7 @@ def test_subworkflows_create_fail_exists(self): subworkflow_create.create() with pytest.raises(UserWarning) as excinfo: subworkflow_create.create() - assert "Subworkflow file exists already" in str(excinfo.value) + assert "subworkflow directory exists" in str(excinfo.value) def test_subworkflows_create_nfcore_modules(self): """Create a subworkflow in nf-core/modules clone""" diff --git a/tests/subworkflows/test_lint.py b/tests/subworkflows/test_lint.py index d94b55b3d3..8f6ff353a1 100644 --- a/tests/subworkflows/test_lint.py +++ b/tests/subworkflows/test_lint.py @@ -31,7 +31,6 @@ def test_subworkflows_lint_new_subworkflow(self): subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.nfcore_modules) subworkflow_lint.lint(print_results=True, all_subworkflows=True) assert len(subworkflow_lint.failed) == 0 - assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -397,3 +396,41 @@ def test_subworkflows_empty_file_in_stub_snapshot(self): # reset the file with open(snap_file, "w") as fh: fh.write(content) + + def test_subworkflows_lint_local(self): + assert self.subworkflow_install.install("fastq_align_bowtie2") + installed = Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_bowtie2") + local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2") + shutil.move(installed, local) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, local=True) + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + + def test_subworkflows_lint_local_missing_files(self): + assert self.subworkflow_install.install("fastq_align_bowtie2") + installed = Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_bowtie2") + local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2") + shutil.move(installed, local) + Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2", "meta.yml").unlink() + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, local=True) + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + warnings = [x.message for x in subworkflow_lint.warned] + assert "Subworkflow `meta.yml` does not exist" in warnings + + def test_subworkflows_lint_local_old_format(self): + assert self.subworkflow_install.install("fastq_align_bowtie2") + installed = Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_bowtie2", "main.nf") + Path(self.pipeline_dir, "subworkflows", "local").mkdir(exist_ok=True) + local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2.nf") + shutil.copy(installed, local) + self.subworkflow_remove.remove("fastq_align_bowtie2", force=True) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, local=True) + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0