From 1682d06fdf67e9d09dcaef221c23b603bd28c6a2 Mon Sep 17 00:00:00 2001 From: lmReef Date: Tue, 21 Jan 2025 15:00:48 +1300 Subject: [PATCH 01/65] add linting for ifEmpty(null) in pipeline and subworkflow linters --- docs/api/_src/pipeline_lint_tests/index.md | 1 + .../pipeline_if_empty_null.md | 5 ++ docs/api/_src/subworkflow_lint_tests/index.md | 1 + .../subworkflow_if_empty_null.md | 5 ++ nf_core/components/lint/__init__.py | 11 +++- nf_core/pipelines/lint/__init__.py | 3 + .../pipelines/lint/pipeline_if_empty_null.py | 58 +++++++++++++++++++ nf_core/subworkflows/lint/__init__.py | 2 + .../lint/subworkflow_if_empty_null.py | 24 ++++++++ nf_core/utils.py | 2 + 10 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 docs/api/_src/pipeline_lint_tests/pipeline_if_empty_null.md create mode 100644 docs/api/_src/subworkflow_lint_tests/subworkflow_if_empty_null.md create mode 100644 nf_core/pipelines/lint/pipeline_if_empty_null.py create mode 100644 nf_core/subworkflows/lint/subworkflow_if_empty_null.py diff --git a/docs/api/_src/pipeline_lint_tests/index.md b/docs/api/_src/pipeline_lint_tests/index.md index 4dd93442d2..86319b035a 100644 --- a/docs/api/_src/pipeline_lint_tests/index.md +++ b/docs/api/_src/pipeline_lint_tests/index.md @@ -17,6 +17,7 @@ - [nfcore_yml](./nfcore_yml/) - [pipeline_name_conventions](./pipeline_name_conventions/) - [pipeline_todos](./pipeline_todos/) + - [pipeline_if_empty_null](./pipeline_if_empty_null/) - [plugin_includes](./plugin_includes/) - [readme](./readme/) - [schema_description](./schema_description/) diff --git a/docs/api/_src/pipeline_lint_tests/pipeline_if_empty_null.md b/docs/api/_src/pipeline_lint_tests/pipeline_if_empty_null.md new file mode 100644 index 0000000000..5adf770fee --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/pipeline_if_empty_null.md @@ -0,0 +1,5 @@ +# pipeline_if_empty_null + +```{eval-rst} +.. automethod:: nf_core.pipelines.lint.PipelineLint.pipeline_if_empty_null +``` diff --git a/docs/api/_src/subworkflow_lint_tests/index.md b/docs/api/_src/subworkflow_lint_tests/index.md index da8db49a7b..e94bbbde34 100644 --- a/docs/api/_src/subworkflow_lint_tests/index.md +++ b/docs/api/_src/subworkflow_lint_tests/index.md @@ -5,4 +5,5 @@ - [subworkflow_changes](./subworkflow_changes/) - [subworkflow_tests](./subworkflow_tests/) - [subworkflow_todos](./subworkflow_todos/) + - [subworkflow_if_empty_null](./subworkflow_if_empty_null/) - [subworkflow_version](./subworkflow_version/) diff --git a/docs/api/_src/subworkflow_lint_tests/subworkflow_if_empty_null.md b/docs/api/_src/subworkflow_lint_tests/subworkflow_if_empty_null.md new file mode 100644 index 0000000000..50f8f3beb5 --- /dev/null +++ b/docs/api/_src/subworkflow_lint_tests/subworkflow_if_empty_null.md @@ -0,0 +1,5 @@ +# subworkflow_if_empty_null + +```{eval-rst} +.. automethod:: nf_core.subworkflows.lint.SubworkflowLint.subworkflow_if_empty_null +``` diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index 69740135a8..9409274160 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -181,9 +181,16 @@ def get_all_module_lint_tests(is_pipeline): @staticmethod def get_all_subworkflow_lint_tests(is_pipeline): if is_pipeline: - return ["main_nf", "meta_yml", "subworkflow_changes", "subworkflow_todos", "subworkflow_version"] + return [ + "main_nf", + "meta_yml", + "subworkflow_changes", + "subworkflow_todos", + "subworkflow_if_empty_null", + "subworkflow_version", + ] else: - return ["main_nf", "meta_yml", "subworkflow_todos", "subworkflow_tests"] + return ["main_nf", "meta_yml", "subworkflow_todos", "subworkflow_if_empty_null", "subworkflow_tests"] def set_up_pipeline_files(self): self.load_lint_config() diff --git a/nf_core/pipelines/lint/__init__.py b/nf_core/pipelines/lint/__init__.py index 154e38aea6..675312166b 100644 --- a/nf_core/pipelines/lint/__init__.py +++ b/nf_core/pipelines/lint/__init__.py @@ -44,6 +44,7 @@ from .multiqc_config import multiqc_config from .nextflow_config import nextflow_config from .nfcore_yml import nfcore_yml +from .pipeline_if_empty_null import pipeline_if_empty_null from .pipeline_name_conventions import pipeline_name_conventions from .pipeline_todos import pipeline_todos from .plugin_includes import plugin_includes @@ -94,6 +95,7 @@ class PipelineLint(nf_core.utils.Pipeline): nfcore_yml = nfcore_yml pipeline_name_conventions = pipeline_name_conventions pipeline_todos = pipeline_todos + pipeline_if_empty_null = pipeline_if_empty_null plugin_includes = plugin_includes readme = readme schema_description = schema_description @@ -139,6 +141,7 @@ def _get_all_lint_tests(release_mode): "actions_awsfulltest", "readme", "pipeline_todos", + "pipeline_if_empty_null", "plugin_includes", "pipeline_name_conventions", "template_strings", diff --git a/nf_core/pipelines/lint/pipeline_if_empty_null.py b/nf_core/pipelines/lint/pipeline_if_empty_null.py new file mode 100644 index 0000000000..a2a516e7a5 --- /dev/null +++ b/nf_core/pipelines/lint/pipeline_if_empty_null.py @@ -0,0 +1,58 @@ +import fnmatch +import logging +import os +import re +from pathlib import Path + +log = logging.getLogger(__name__) + + +def pipeline_if_empty_null(self, root_dir=None): + """Check for ifEmpty(null) + + There are two general cases for workflows to use the channel operator `ifEmpty`: + 1. `ifEmpty( [ ] )` to ensure a process executes, for example when an input file is optional (although this can be replaced by `toList()`). + 2. When a channel should not be empty and throws an error `ifEmpty { error ... }`, e.g. reading from an empty samplesheet. + + There are multiple examples of workflows that inject null objects into channels using ifEmpty(null), which can cause unhandled null pointer exceptions. + This lint test throws warnings for those instances. + """ + passed = [] + warned = [] + file_paths = [] + pattern = re.compile(r"ifEmpty\(\s*null\s*\)") + + # Pipelines don't provide a path, so use the workflow path. + # Modules run this function twice and provide a string path + if root_dir is None: + root_dir = self.wf_path + + ignore = [".git"] + if Path(root_dir, ".gitignore").is_file(): + with open(Path(root_dir, ".gitignore"), encoding="latin1") as fh: + for line in fh: + ignore.append(Path(line.strip().rstrip("/")).name) + for root, dirs, files in os.walk(root_dir, topdown=True): + # Ignore files + for i_base in ignore: + i = str(Path(root, i_base)) + dirs[:] = [d for d in dirs if not fnmatch.fnmatch(str(Path(root, d)), i)] + files[:] = [f for f in files if not fnmatch.fnmatch(str(Path(root, f)), i)] + for fname in files: + try: + with open(Path(root, fname), encoding="latin1") as fh: + for line in fh: + if re.findall(pattern, line): + warned.append(f"`ifEmpty(null)` found in `{fname}`: _{line}_") + file_paths.append(Path(root, fname)) + except FileNotFoundError: + log.debug( + f"Could not open file { + fname} in pipeline_if_empty_null lint test" + ) + + if len(warned) == 0: + passed.append("No `ifEmpty(null)` strings found") + + # return file_paths for use in subworkflow lint + return {"passed": passed, "warned": warned, "file_paths": file_paths} diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index cedae62f11..3c4219be99 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -24,6 +24,7 @@ from .main_nf import main_nf # type: ignore[misc] from .meta_yml import meta_yml # type: ignore[misc] from .subworkflow_changes import subworkflow_changes # type: ignore[misc] +from .subworkflow_if_empty_null import subworkflow_if_empty_null # type: ignore[misc] from .subworkflow_tests import subworkflow_tests # type: ignore[misc] from .subworkflow_todos import subworkflow_todos # type: ignore[misc] from .subworkflow_version import subworkflow_version # type: ignore[misc] @@ -40,6 +41,7 @@ class SubworkflowLint(ComponentLint): subworkflow_changes = subworkflow_changes subworkflow_tests = subworkflow_tests subworkflow_todos = subworkflow_todos + subworkflow_if_empty_null = subworkflow_if_empty_null subworkflow_version = subworkflow_version def __init__( diff --git a/nf_core/subworkflows/lint/subworkflow_if_empty_null.py b/nf_core/subworkflows/lint/subworkflow_if_empty_null.py new file mode 100644 index 0000000000..ce683e0787 --- /dev/null +++ b/nf_core/subworkflows/lint/subworkflow_if_empty_null.py @@ -0,0 +1,24 @@ +import logging + +from nf_core.pipelines.lint.pipeline_if_empty_null import pipeline_if_empty_null + +log = logging.getLogger(__name__) + + +def subworkflow_if_empty_null(_, subworkflow): + """Check for ifEmpty(null) + + There are two general cases for workflows to use the channel operator `ifEmpty`: + 1. `ifEmpty( [ ] )` to ensure a process executes, for example when an input file is optional (although this can be replaced by `toList()`). + 2. When a channel should not be empty and throws an error `ifEmpty { error ... }`, e.g. reading from an empty samplesheet. + + There are multiple examples of workflows that inject null objects into channels using ifEmpty(null), which can cause unhandled null pointer exceptions. + This lint test throws warnings for those instances. + """ + + # Main subworkflow directory + swf_results = pipeline_if_empty_null(None, root_dir=subworkflow.component_dir) + for i, warning in enumerate(swf_results["warned"]): + subworkflow.warned.append(("subworkflow_if_empty_null", warning, swf_results["file_paths"][i])) + for i, passed in enumerate(swf_results["passed"]): + subworkflow.passed.append(("subworkflow_if_empty_null", passed, subworkflow.component_dir)) diff --git a/nf_core/utils.py b/nf_core/utils.py index 30b0743493..c731c59030 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1198,6 +1198,8 @@ class NFCoreYamlLintConfig(BaseModel): """ Lint all required files to run full tests on AWS """ pipeline_todos: Optional[bool] = None """ Lint for TODOs statements""" + pipeline_if_empty_null: Optional[bool] = None + """ Lint for ifEmpty(null) statements""" plugin_includes: Optional[bool] = None """ Lint for nextflow plugin """ pipeline_name_conventions: Optional[bool] = None From e832a97ed8bdab07f35d1ae965beda041bc4f1bf Mon Sep 17 00:00:00 2001 From: lmReef Date: Tue, 21 Jan 2025 15:31:48 +1300 Subject: [PATCH 02/65] comment formatting --- nf_core/pipelines/lint/pipeline_if_empty_null.py | 2 +- nf_core/subworkflows/lint/subworkflow_if_empty_null.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipelines/lint/pipeline_if_empty_null.py b/nf_core/pipelines/lint/pipeline_if_empty_null.py index a2a516e7a5..240e3c0021 100644 --- a/nf_core/pipelines/lint/pipeline_if_empty_null.py +++ b/nf_core/pipelines/lint/pipeline_if_empty_null.py @@ -14,7 +14,7 @@ def pipeline_if_empty_null(self, root_dir=None): 1. `ifEmpty( [ ] )` to ensure a process executes, for example when an input file is optional (although this can be replaced by `toList()`). 2. When a channel should not be empty and throws an error `ifEmpty { error ... }`, e.g. reading from an empty samplesheet. - There are multiple examples of workflows that inject null objects into channels using ifEmpty(null), which can cause unhandled null pointer exceptions. + There are multiple examples of workflows that inject null objects into channels using `ifEmpty(null)`, which can cause unhandled null pointer exceptions. This lint test throws warnings for those instances. """ passed = [] diff --git a/nf_core/subworkflows/lint/subworkflow_if_empty_null.py b/nf_core/subworkflows/lint/subworkflow_if_empty_null.py index ce683e0787..c1eff3b688 100644 --- a/nf_core/subworkflows/lint/subworkflow_if_empty_null.py +++ b/nf_core/subworkflows/lint/subworkflow_if_empty_null.py @@ -12,7 +12,7 @@ def subworkflow_if_empty_null(_, subworkflow): 1. `ifEmpty( [ ] )` to ensure a process executes, for example when an input file is optional (although this can be replaced by `toList()`). 2. When a channel should not be empty and throws an error `ifEmpty { error ... }`, e.g. reading from an empty samplesheet. - There are multiple examples of workflows that inject null objects into channels using ifEmpty(null), which can cause unhandled null pointer exceptions. + There are multiple examples of workflows that inject null objects into channels using `ifEmpty(null)`, which can cause unhandled null pointer exceptions. This lint test throws warnings for those instances. """ From c198c452947b0a800674513127d47f7faf9be638 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 21 Jan 2025 02:35:58 +0000 Subject: [PATCH 03/65] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4d8b44878..66195bca93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ ### Linting - allow mixed `str` and `dict` entries in lint config ([#3228](https://github.com/nf-core/tools/pull/3228)) +- Add linting for if empty null dev ([#3411](https://github.com/nf-core/tools/pull/3411)) ### Modules From 62118b7c70d5025264dbd5da00267c69d7ddac83 Mon Sep 17 00:00:00 2001 From: lmReef <60915116+lmReef@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:37:22 +1300 Subject: [PATCH 04/65] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66195bca93..c6573d5c7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ ### Linting - allow mixed `str` and `dict` entries in lint config ([#3228](https://github.com/nf-core/tools/pull/3228)) -- Add linting for if empty null dev ([#3411](https://github.com/nf-core/tools/pull/3411)) +- Add linting for ifEmpty(null) ([#3411](https://github.com/nf-core/tools/pull/3411)) ### Modules From c1692f5b886d15e09f3d343e6bac8336ef26acf4 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:52:52 +0100 Subject: [PATCH 05/65] fix(1929): more type-gating --- nf_core/components/components_command.py | 4 + nf_core/components/create.py | 96 +++++++------------ nf_core/components/nfcore_component.py | 29 +++--- nf_core/modules/lint/module_tests.py | 2 + nf_core/pipelines/lint/__init__.py | 3 + .../lint/local_component_structure.py | 31 ++++++ .../subworkflows/lint/subworkflow_tests.py | 3 +- tests/modules/test_create.py | 4 +- 8 files changed, 98 insertions(+), 74 deletions(-) create mode 100644 nf_core/pipelines/lint/local_component_structure.py diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index f25fb33a6f..f04bb7da6b 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -71,6 +71,10 @@ def get_local_components(self) -> List[str]: """ local_component_dir = Path(self.directory, self.component_type, "local") return [ + str(Path(directory).relative_to(local_component_dir)) + for directory, _, files in os.walk(local_component_dir) + if "main.nf" in files + ] + [ str(path.relative_to(local_component_dir)) for path in local_component_dir.iterdir() if path.suffix == ".nf" ] diff --git a/nf_core/components/create.py b/nf_core/components/create.py index c781905618..0a2c6aaff7 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -75,11 +75,11 @@ def create(self) -> bool: e.g bam_sort or bam_sort_samtools, respectively. If is a pipeline, this function creates a file called: - '/modules/local/tool.nf' + '/modules/local/tool/main.nf' OR - '/modules/local/tool_subtool.nf' + '/modules/local/tool/subtool/main.nf' OR for subworkflows - '/subworkflows/local/subworkflow_name.nf' + '/subworkflows/local/subworkflow_name/main.nf' If is a clone of nf-core/modules, it creates or modifies the following files: @@ -355,70 +355,46 @@ def _get_component_dirs(self) -> Dict[str, Path]: """ file_paths = {} if self.repo_type == "pipeline": - local_component_dir = Path(self.directory, self.component_type, "local") - # Check whether component file already exists - component_file = local_component_dir / f"{self.component_name}.nf" - if component_file.exists() and not self.force_overwrite: - raise UserWarning( - f"{self.component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite" - ) - - if self.component_type == "modules": - # If a subtool, check if there is a module called the base tool name already - if self.subtool and (local_component_dir / f"{self.component}.nf").exists(): - raise UserWarning( - f"Module '{self.component}' exists already, cannot make subtool '{self.component_name}'" - ) - - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob(f"{local_component_dir}/{self.component}_*.nf") - if not self.subtool and tool_glob: - raise UserWarning( - f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" - ) - - # Set file paths - file_paths["main.nf"] = component_file + component_dir = Path(self.directory, self.component_type, "local", self.component_dir) elif self.repo_type == "modules": component_dir = Path(self.directory, self.component_type, self.org, self.component_dir) - # Check if module/subworkflow directories exist already - if component_dir.exists() and not self.force_overwrite and not self.migrate_pytest: - raise UserWarning( - f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite" - ) + else: + raise ValueError("`repo_type` not set correctly") - if self.component_type == "modules": - # If a subtool, check if there is a module called the base tool name already - parent_tool_main_nf = Path( - self.directory, - self.component_type, - self.org, - self.component, - "main.nf", + # Check if module/subworkflow directories exist already + if component_dir.exists() and not self.force_overwrite and not self.migrate_pytest: + raise UserWarning( + f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite" + ) + + if self.component_type == "modules": + # If a subtool, check if there is a module called the base tool name already + parent_tool_main_nf = Path( + self.directory, + self.component_type, + self.org, + self.component, + "main.nf", + ) + if self.subtool and parent_tool_main_nf.exists() and not self.migrate_pytest: + raise UserWarning( + f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" ) - if self.subtool and parent_tool_main_nf.exists() and not self.migrate_pytest: - raise UserWarning( - f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" - ) - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob( - f"{Path(self.directory, self.component_type, self.org, self.component)}/*/main.nf" + # If no subtool, check that there isn't already a tool/subtool + tool_glob = glob.glob(f"{Path(self.directory, self.component_type, self.org, self.component)}/*/main.nf") + if not self.subtool and tool_glob and not self.migrate_pytest: + raise UserWarning( + f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" ) - if not self.subtool and tool_glob and not self.migrate_pytest: - raise UserWarning( - f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" - ) - # Set file paths - # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure - file_paths["main.nf"] = component_dir / "main.nf" - file_paths["meta.yml"] = component_dir / "meta.yml" - if self.component_type == "modules": - file_paths["environment.yml"] = component_dir / "environment.yml" - file_paths["tests/main.nf.test.j2"] = component_dir / "tests" / "main.nf.test" - else: - raise ValueError("`repo_type` not set correctly") + # Set file paths + # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure + file_paths["main.nf"] = component_dir / "main.nf" + file_paths["meta.yml"] = component_dir / "meta.yml" + if self.component_type == "modules": + file_paths["environment.yml"] = component_dir / "environment.yml" + file_paths["tests/main.nf.test.j2"] = component_dir / "tests" / "main.nf.test" return file_paths diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 81c0ba98e7..03ee51b18a 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -74,8 +74,8 @@ def __init__( repo_dir = self.component_dir.parts[:name_index][-1] self.org = repo_dir - self.nftest_testdir = Path(self.component_dir, "tests") - self.nftest_main_nf = Path(self.nftest_testdir, "main.nf.test") + self.nftest_testdir: Optional[Path] = Path(self.component_dir, "tests") + self.nftest_main_nf: Optional[Path] = Path(self.nftest_testdir, "main.nf.test") if self.repo_type == "pipeline": patch_fn = f"{self.component_name.replace('/', '-')}.diff" @@ -85,15 +85,22 @@ def __init__( self.patch_path = patch_path else: # The main file is just the local module - self.main_nf = self.component_dir - self.component_name = self.component_dir.stem - # These attributes are only used by nf-core modules - # so just initialize them to None - self.meta_yml = None - self.environment_yml = None - self.test_dir = None - self.test_yml = None - self.test_main_nf = None + if self.component_dir.is_dir(): + self.main_nf = Path(self.component_dir, "main.nf") + self.component_name = self.component_dir.stem + # These attributes are only required by nf-core modules + # so just set them to None if they don't exist + self.meta_yml = p if (p := Path(self.component_dir, "meta.yml")).exists() else None + self.environment_yml = p if (p := Path(self.component_dir, "environment.yml")).exists() else None + self.nftest_testdir = p if (p := Path(self.component_dir, "tests")).exists() else None + if self.nftest_testdir is not None: + self.nftest_main_nf = p if (p := Path(self.nftest_testdir, "main.nf.test")).exists() else None + else: + self.main_nf = self.component_dir + self.meta_yml = None + self.environment_yml = None + self.nftest_testdir = None + self.nftest_main_nf = None self.process_name: str = self._get_process_name() diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 6722c12129..8f5fc91425 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -22,6 +22,8 @@ def module_tests(_, module: NFCoreComponent): and contains a ``main.nf.test`` and a ``main.nf.test.snap`` """ + if module.nftest_testdir is None or module.nftest_main_nf is None: + raise ValueError() repo_dir = module.component_dir.parts[: module.component_dir.parts.index(module.component_name.split("/")[0])][-1] test_dir = Path(module.base_dir, "tests", "modules", repo_dir, module.component_name) pytest_main_nf = Path(test_dir, "main.nf") diff --git a/nf_core/pipelines/lint/__init__.py b/nf_core/pipelines/lint/__init__.py index 154e38aea6..7f15b092cc 100644 --- a/nf_core/pipelines/lint/__init__.py +++ b/nf_core/pipelines/lint/__init__.py @@ -38,6 +38,7 @@ from .files_exist import files_exist from .files_unchanged import files_unchanged from .included_configs import included_configs +from .local_component_structure import local_component_structure from .merge_markers import merge_markers from .modules_json import modules_json from .modules_structure import modules_structure @@ -89,6 +90,7 @@ class PipelineLint(nf_core.utils.Pipeline): merge_markers = merge_markers modules_json = modules_json modules_structure = modules_structure + local_component_structure = local_component_structure multiqc_config = multiqc_config nextflow_config = nextflow_config nfcore_yml = nfcore_yml @@ -151,6 +153,7 @@ def _get_all_lint_tests(release_mode): "modules_json", "multiqc_config", "modules_structure", + "local_component_structure", "base_config", "modules_config", "nfcore_yml", diff --git a/nf_core/pipelines/lint/local_component_structure.py b/nf_core/pipelines/lint/local_component_structure.py new file mode 100644 index 0000000000..e5e1a00dee --- /dev/null +++ b/nf_core/pipelines/lint/local_component_structure.py @@ -0,0 +1,31 @@ +import logging +from pathlib import Path + +log = logging.getLogger(__name__) + + +def local_component_structure(self): + """ + Check that the local modules and subworkflows directories in a pipeline have the correct format: + + .. code-block:: bash + + modules/local/TOOL/SUBTOOL + + Prior to nf-core/tools release 3.0.3 the directory structure allowed top-level `*.nf` files: + + .. code-block:: bash + + modules/local/modules/TOOL_SUBTOOL.nf + """ + warned = [] + for nf_file in Path(self.wf_path, "modules", "local").glob("*.nf"): + warned.append(f"{nf_file.name} in modules/local should be moved to a TOOL/SUBTOOL/main.nf structure") + for nf_file in Path(self.wf_path, "subworkflows", "local").glob("*.nf"): + warned.append(f"{nf_file.name} in subworkflows/local should be moved to a TOOL/SUBTOOL/main.nf structure") + + # If there are modules installed in the wrong location + passed = [] + if len(warned) == 0: + passed = ["modules directory structure is correct 'modules/nf-core/TOOL/SUBTOOL'"] + return {"passed": passed, "warned": warned, "failed": [], "ignored": []} diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 8e9e62430a..1caf73d00b 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -23,7 +23,8 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): Additionally, checks that all included components in test ``main.nf`` are specified in ``test.yml`` """ - + if subworkflow.nftest_testdir is None or subworkflow.nftest_main_nf is None: + raise ValueError() repo_dir = subworkflow.component_dir.parts[ : subworkflow.component_dir.parts.index(subworkflow.component_name.split("/")[0]) ][-1] diff --git a/tests/modules/test_create.py b/tests/modules/test_create.py index 219f869997..4ae9f9b5c4 100644 --- a/tests/modules/test_create.py +++ b/tests/modules/test_create.py @@ -31,7 +31,7 @@ def test_modules_create_succeed(self): ) with requests_cache.disabled(): module_create.create() - assert os.path.exists(os.path.join(self.pipeline_dir, "modules", "local", "trimgalore.nf")) + assert os.path.exists(os.path.join(self.pipeline_dir, "modules", "local", "trimgalore/main.nf")) def test_modules_create_fail_exists(self): """Fail at creating the same module twice""" @@ -46,7 +46,7 @@ def test_modules_create_fail_exists(self): with pytest.raises(UserWarning) as excinfo: with requests_cache.disabled(): module_create.create() - assert "Module file exists already" in str(excinfo.value) + assert "module directory exists:" in str(excinfo.value) def test_modules_create_nfcore_modules(self): """Create a module in nf-core/modules clone""" From 90978ead6f36252e192bfee77e4a756145710f13 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 28 Oct 2024 15:54:44 +0000 Subject: [PATCH 06/65] fix CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5aa7f22fe..331e2fcb56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### Modules +- feat(1929): Make local modules use remote structure ([#3256](https://github.com/nf-core/tools/pull/3256)) + ### Subworkflows ### General From a5bb0f0ed40244c7283fe57155acf6b3ebf0fb6f Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:03:36 +0100 Subject: [PATCH 07/65] fix(1929): update subworkflow tests and add lint test markdown --- .../_src/pipeline_lint_tests/local_component_structure.md | 5 +++++ tests/subworkflows/test_create.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 docs/api/_src/pipeline_lint_tests/local_component_structure.md diff --git a/docs/api/_src/pipeline_lint_tests/local_component_structure.md b/docs/api/_src/pipeline_lint_tests/local_component_structure.md new file mode 100644 index 0000000000..1884d862be --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/local_component_structure.md @@ -0,0 +1,5 @@ +# modules_structure + +```{eval-rst} +.. automethod:: nf_core.pipelines.lint.PipelineLint.local_component_structure +``` diff --git a/tests/subworkflows/test_create.py b/tests/subworkflows/test_create.py index 48cb482260..704a23772e 100644 --- a/tests/subworkflows/test_create.py +++ b/tests/subworkflows/test_create.py @@ -19,7 +19,7 @@ def test_subworkflows_create_succeed(self): self.pipeline_dir, "test_subworkflow_local", "@author", True ) subworkflow_create.create() - assert Path(self.pipeline_dir, "subworkflows", "local", "test_subworkflow_local.nf").exists() + assert Path(self.pipeline_dir, "subworkflows", "local", "test_subworkflow_local/main.nf").exists() def test_subworkflows_create_fail_exists(self): """Fail at creating the same subworkflow twice""" @@ -29,7 +29,7 @@ def test_subworkflows_create_fail_exists(self): subworkflow_create.create() with pytest.raises(UserWarning) as excinfo: subworkflow_create.create() - assert "Subworkflow file exists already" in str(excinfo.value) + assert "subworkflow directory exists" in str(excinfo.value) def test_subworkflows_create_nfcore_modules(self): """Create a subworkflow in nf-core/modules clone""" From 732c78635f237e3edfdc7b88050d469299b1f49f Mon Sep 17 00:00:00 2001 From: awgymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:47:50 +0100 Subject: [PATCH 08/65] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/pipelines/lint/local_component_structure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipelines/lint/local_component_structure.py b/nf_core/pipelines/lint/local_component_structure.py index e5e1a00dee..61c1d45e29 100644 --- a/nf_core/pipelines/lint/local_component_structure.py +++ b/nf_core/pipelines/lint/local_component_structure.py @@ -12,7 +12,7 @@ def local_component_structure(self): modules/local/TOOL/SUBTOOL - Prior to nf-core/tools release 3.0.3 the directory structure allowed top-level `*.nf` files: + Prior to nf-core/tools release 3.1.0 the directory structure allowed top-level `*.nf` files: .. code-block:: bash @@ -22,7 +22,7 @@ def local_component_structure(self): for nf_file in Path(self.wf_path, "modules", "local").glob("*.nf"): warned.append(f"{nf_file.name} in modules/local should be moved to a TOOL/SUBTOOL/main.nf structure") for nf_file in Path(self.wf_path, "subworkflows", "local").glob("*.nf"): - warned.append(f"{nf_file.name} in subworkflows/local should be moved to a TOOL/SUBTOOL/main.nf structure") + warned.append(f"{nf_file.name} in subworkflows/local should be moved to a SUBWORKFLOW_NAME/main.nf structure") # If there are modules installed in the wrong location passed = [] From 7ba6f79f7c6800bf5fe5f3c71f9358c393a12e72 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:17:00 +0100 Subject: [PATCH 09/65] feat(1929): allow missing files when linting local modules --- CHANGELOG.md | 9 ++++++++ nf_core/components/lint/__init__.py | 4 ++++ nf_core/modules/lint/__init__.py | 18 +++++++++++++++ nf_core/modules/lint/environment_yml.py | 11 +++++++++- nf_core/modules/lint/meta_yml.py | 12 ++++++---- nf_core/modules/lint/module_tests.py | 29 ++++++++++++++++++++++--- 6 files changed, 75 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 331e2fcb56..c910b9cd02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,11 +96,16 @@ ### Modules - add a panel around diff previews when updating ([#3246](https://github.com/nf-core/tools/pull/3246)) +- Modules created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) ### Subworkflows +<<<<<<< HEAD - Add `nf-core subworkflows patch` command ([#2861](https://github.com/nf-core/tools/pull/2861)) - Improve subworkflow nf-test migration warning ([#3298](https://github.com/nf-core/tools/pull/3298)) +======= +- Subworkflows created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) +>>>>>>> 31cffdd5 (feat(1929): allow missing files when linting local modules) ### General @@ -110,6 +115,7 @@ - use correct `--profile` options for `nf-core subworkflows test` ([#3233](https://github.com/nf-core/tools/pull/3233)) - Update GitHub Actions ([#3237](https://github.com/nf-core/tools/pull/3237)) - add `--dir/-d` option to schema commands ([#3247](https://github.com/nf-core/tools/pull/3247)) +<<<<<<< HEAD - fix headers in api docs ([#3323](https://github.com/nf-core/tools/pull/3323)) - handle new schema structure in `nf-core pipelines create-params-file` ([#3276](https://github.com/nf-core/tools/pull/3276)) - Update Gitpod image to use Miniforge instead of Miniconda([#3274](https://github.com/nf-core/tools/pull/3274)) @@ -132,6 +138,9 @@ - Update codecov/codecov-action action to v5 ([#3283](https://github.com/nf-core/tools/pull/3283)) - Update gitpod/workspace-base Docker digest to 12853f7 ([#3309](https://github.com/nf-core/tools/pull/3309)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.8.2 ([#3325](https://github.com/nf-core/tools/pull/3325)) +======= +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.1 ([#3250](https://github.com/nf-core/tools/pull/3250)) +>>>>>>> 31cffdd5 (feat(1929): allow missing files when linting local modules) ## [v3.0.2 - Titanium Tapir Patch](https://github.com/nf-core/tools/releases/tag/3.0.2) - [2024-10-11] diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index 69740135a8..96b611cc20 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -162,6 +162,10 @@ def _set_registry(self, registry) -> None: self.registry = registry log.debug(f"Registry set to {self.registry}") + @property + def local_module_exclude_tests(self): + return ["module_version", "module_changes", "modules_patch"] + @staticmethod def get_all_module_lint_tests(is_pipeline): if is_pipeline: diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 49012cff40..1ce9902a31 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -234,13 +234,31 @@ def lint_module( # TODO: consider unifying modules and subworkflows lint_module() function and add it to the ComponentLint class # Only check the main script in case of a local module if local: + mod.get_inputs_from_main_nf() + mod.get_outputs_from_main_nf() + # Update meta.yml file if requested + if self.fix and mod.meta_yml is not None: + self.update_meta_yml_file(mod) + + for test_name in self.lint_tests: + if test_name in self.local_module_exclude_tests: + continue + if test_name == "main_nf": + getattr(self, test_name)(mod, fix_version, self.registry, progress_bar) + elif test_name in ["meta_yml", "environment_yml"]: + # Allow files to be missing for local + getattr(self, test_name)(mod, allow_missing=True) + """ self.main_nf(mod, fix_version, self.registry, progress_bar) + """ + self.passed += [LintResult(mod, *m) for m in mod.passed] warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)] if not self.fail_warned: self.warned += warned else: self.failed += warned + self.failed += [LintResult(mod, *m) for m in mod.failed] # Otherwise run all the lint tests else: diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py index 4488b0befa..6571f07ae8 100644 --- a/nf_core/modules/lint/environment_yml.py +++ b/nf_core/modules/lint/environment_yml.py @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) -def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None: +def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent, allow_missing: bool = False) -> None: """ Lint an ``environment.yml`` file. @@ -23,6 +23,15 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) env_yml = None # load the environment.yml file if module.environment_yml is None: + if allow_missing: + module.warned.append( + ( + "environment_yml_exists", + "Module's `environment.yml` does not exist", + Path(module.component_dir, "environment.yml"), + ), + ) + return raise LintExceptionError("Module does not have an `environment.yml` file") try: with open(module.environment_yml) as fh: diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index d0268a40cc..76345bdcb3 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -13,7 +13,7 @@ log = logging.getLogger(__name__) -def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None: +def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent, allow_missing: bool = False) -> None: """ Lint a ``meta.yml`` file @@ -42,7 +42,13 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None module (NFCoreComponent): The module to lint """ - + if module.meta_yml is None: + if allow_missing: + module.warned.append( + ("meta_yml_exists", "Module `meta.yml` does not exist", Path(module.component_dir, "meta.yml")) + ) + return + raise LintExceptionError("Module does not have a `meta.yml` file") # Check if we have a patch file, get original file in that case meta_yaml = read_meta_yml(module_lint_object, module) if module.is_patched and module_lint_object.modules_repo.repo_path is not None: @@ -57,8 +63,6 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None if lines is not None: yaml = ruamel.yaml.YAML() meta_yaml = yaml.safe_load("".join(lines)) - if module.meta_yml is None: - raise LintExceptionError("Module does not have a `meta.yml` file") if meta_yaml is None: module.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", module.meta_yml)) return diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 8f5fc91425..6826b2e743 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -9,12 +9,13 @@ import yaml +from nf_core.components.lint import LintExceptionError from nf_core.components.nfcore_component import NFCoreComponent log = logging.getLogger(__name__) -def module_tests(_, module: NFCoreComponent): +def module_tests(_, module: NFCoreComponent, allow_missing: bool = False): """ Lint the tests of a module in ``nf-core/modules`` @@ -22,8 +23,30 @@ def module_tests(_, module: NFCoreComponent): and contains a ``main.nf.test`` and a ``main.nf.test.snap`` """ - if module.nftest_testdir is None or module.nftest_main_nf is None: - raise ValueError() + if module.nftest_testdir is None: + if allow_missing: + module.warned.append( + ( + "test_dir_exists", + "nf-test directory is missing", + Path(module.component_dir, "tests"), + ) + ) + return + raise LintExceptionError("Module does not have a `tests` dir") + + if module.nftest_main_nf is None: + if allow_missing: + module.warned.append( + ( + "test_main_nf_exists", + "test `main.nf.test` does not exist", + Path(module.component_dir, "tests", "main.nf.test"), + ) + ) + return + raise LintExceptionError("Module does not have a `tests` dir") + repo_dir = module.component_dir.parts[: module.component_dir.parts.index(module.component_name.split("/")[0])][-1] test_dir = Path(module.base_dir, "tests", "modules", repo_dir, module.component_name) pytest_main_nf = Path(test_dir, "main.nf") From 8a1d44a09941e040b5b6c0a2961ccb97f00c9a53 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:35:33 +0100 Subject: [PATCH 10/65] feat(1929): local linting option now *only* lints local modules. Add tests --- nf_core/modules/lint/__init__.py | 7 ++- nf_core/subworkflows/lint/__init__.py | 3 +- nf_core/subworkflows/lint/meta_yml.py | 24 ++++++++--- .../subworkflows/lint/subworkflow_tests.py | 29 +++++++++++-- tests/modules/test_lint.py | 43 ++++++++++++++++++- tests/subworkflows/test_lint.py | 39 ++++++++++++++++- 6 files changed, 129 insertions(+), 16 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 1ce9902a31..af057b67fc 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -170,7 +170,7 @@ def lint( self.lint_modules(local_modules, registry=registry, local=True, fix_version=fix_version) # Lint nf-core modules - if len(remote_modules) > 0: + if not local and len(remote_modules) > 0: self.lint_modules(remote_modules, registry=registry, local=False, fix_version=fix_version) if print_results: @@ -248,9 +248,8 @@ def lint_module( elif test_name in ["meta_yml", "environment_yml"]: # Allow files to be missing for local getattr(self, test_name)(mod, allow_missing=True) - """ - self.main_nf(mod, fix_version, self.registry, progress_bar) - """ + else: + getattr(self, test_name)(mod) self.passed += [LintResult(mod, *m) for m in mod.passed] warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)] diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index cedae62f11..ee46f3f3b2 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -152,7 +152,7 @@ def lint( self.lint_subworkflows(local_subworkflows, registry=registry, local=True) # Lint nf-core subworkflows - if len(remote_subworkflows) > 0: + if not local and len(remote_subworkflows) > 0: self.lint_subworkflows(remote_subworkflows, registry=registry, local=False) if print_results: @@ -208,6 +208,7 @@ def lint_subworkflow(self, swf, progress_bar, registry, local=False): # Only check the main script in case of a local subworkflow if local: self.main_nf(swf) + self.meta_yml(swf, allow_missing=True) self.passed += [LintResult(swf, *s) for s in swf.passed] warned = [LintResult(swf, *m) for m in (swf.warned + swf.failed)] if not self.fail_warned: diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index be282bc453..262c80f948 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -6,11 +6,12 @@ import yaml import nf_core.components.components_utils +from nf_core.components.lint import LintExceptionError log = logging.getLogger(__name__) -def meta_yml(subworkflow_lint_object, subworkflow): +def meta_yml(subworkflow_lint_object, subworkflow, allow_missing: bool = False): """ Lint a ``meta.yml`` file @@ -28,6 +29,18 @@ def meta_yml(subworkflow_lint_object, subworkflow): """ # Read the meta.yml file + if subworkflow.meta_yml is None: + if allow_missing: + subworkflow.warned.append( + ( + "meta_yml_exists", + "Subworkflow `meta.yml` does not exist", + Path(subworkflow.component_dir, "meta.yml"), + ) + ) + return + raise LintExceptionError("Subworkflow does not have a `meta.yml` file") + try: with open(subworkflow.meta_yml) as fh: meta_yaml = yaml.safe_load(fh) @@ -49,7 +62,7 @@ def meta_yml(subworkflow_lint_object, subworkflow): if len(e.path) > 0: hint = f"\nCheck the entry for `{e.path[0]}`." if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: - hint = f"\nCheck that the child entries of {e.path[0]+'.'+e.path[2]} are indented correctly." + hint = f"\nCheck that the child entries of {e.path[0]}.{e.path[2]} are indented correctly." subworkflow.failed.append( ( "meta_yml_valid", @@ -96,10 +109,9 @@ def meta_yml(subworkflow_lint_object, subworkflow): ) # confirm that all included components in ``main.nf`` are specified in ``meta.yml`` - included_components = nf_core.components.components_utils.get_components_to_install(subworkflow.component_dir) - included_components = ( - included_components[0] + included_components[1] - ) # join included modules and included subworkflows in a single list + included_components_ = nf_core.components.components_utils.get_components_to_install(subworkflow.component_dir) + included_components = included_components_[0] + included_components_[1] + # join included modules and included subworkflows in a single list if "components" in meta_yaml: meta_components = [x for x in meta_yaml["components"]] for component in set(included_components): diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 1caf73d00b..74b170416d 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -9,12 +9,13 @@ import yaml +from nf_core.components.lint import LintExceptionError from nf_core.components.nfcore_component import NFCoreComponent log = logging.getLogger(__name__) -def subworkflow_tests(_, subworkflow: NFCoreComponent): +def subworkflow_tests(_, subworkflow: NFCoreComponent, allow_missing: bool = False): """ Lint the tests of a subworkflow in ``nf-core/modules`` @@ -23,8 +24,30 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): Additionally, checks that all included components in test ``main.nf`` are specified in ``test.yml`` """ - if subworkflow.nftest_testdir is None or subworkflow.nftest_main_nf is None: - raise ValueError() + if subworkflow.nftest_testdir is None: + if allow_missing: + subworkflow.warned.append( + ( + "test_dir_exists", + "nf-test directory is missing", + Path(subworkflow.component_dir, "tests"), + ) + ) + return + raise LintExceptionError("Module does not have a `tests` dir") + + if subworkflow.nftest_main_nf is None: + if allow_missing: + subworkflow.warned.append( + ( + "test_main_nf_exists", + "test `main.nf.test` does not exist", + Path(subworkflow.component_dir, "tests", "main.nf.test"), + ) + ) + return + raise LintExceptionError("Subworkflow does not have a `tests` dir") + repo_dir = subworkflow.component_dir.parts[ : subworkflow.component_dir.parts.index(subworkflow.component_name.split("/")[0]) ][-1] diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index 5372807987..37f992a7c9 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -1,4 +1,5 @@ import json +import shutil from pathlib import Path from typing import Union @@ -158,7 +159,7 @@ ] -class TestModulesCreate(TestModules): +class TestModulesLint(TestModules): def _setup_patch(self, pipeline_dir: Union[str, Path], modify_module: bool): install_obj = nf_core.modules.install.ModuleInstall( pipeline_dir, @@ -760,6 +761,46 @@ def test_modules_empty_file_in_stub_snapshot(self): with open(snap_file, "w") as fh: fh.write(content) + def test_modules_lint_local(self): + assert self.mods_install.install("trimgalore") + installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore") + local = Path(self.pipeline_dir, "modules", "local", "trimgalore") + shutil.move(installed, local) + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, local=True, all_modules=True) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + + def test_modules_lint_local_missing_files(self): + assert self.mods_install.install("trimgalore") + installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore") + local = Path(self.pipeline_dir, "modules", "local", "trimgalore") + shutil.move(installed, local) + Path(self.pipeline_dir, "modules", "local", "trimgalore", "environment.yml").unlink() + Path(self.pipeline_dir, "modules", "local", "trimgalore", "meta.yml").unlink() + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, local=True, all_modules=True) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + warnings = [x.message for x in module_lint.warned] + assert "Module's `environment.yml` does not exist" in warnings + assert "Module `meta.yml` does not exist" in warnings + + def test_modules_lint_local_old_format(self): + assert self.mods_install.install("trimgalore") + installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore", "main.nf") + Path(self.pipeline_dir, "modules", "local").mkdir() + local = Path(self.pipeline_dir, "modules", "local", "trimgalore.nf") + shutil.copy(installed, local) + self.mods_remove.remove("trimgalore", force=True) + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, local=True, all_modules=True) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + # A skeleton object with the passed/warned/failed list attrs # Use this in place of a ModuleLint object to test behaviour of diff --git a/tests/subworkflows/test_lint.py b/tests/subworkflows/test_lint.py index d94b55b3d3..49c671c088 100644 --- a/tests/subworkflows/test_lint.py +++ b/tests/subworkflows/test_lint.py @@ -31,7 +31,6 @@ def test_subworkflows_lint_new_subworkflow(self): subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.nfcore_modules) subworkflow_lint.lint(print_results=True, all_subworkflows=True) assert len(subworkflow_lint.failed) == 0 - assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -397,3 +396,41 @@ def test_subworkflows_empty_file_in_stub_snapshot(self): # reset the file with open(snap_file, "w") as fh: fh.write(content) + + def test_subworkflows_lint_local(self): + assert self.subworkflow_install.install("fastq_align_bowtie2") + installed = Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_bowtie2") + local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2") + shutil.move(installed, local) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + + def test_subworkflows_lint_local_missing_files(self): + assert self.subworkflow_install.install("fastq_align_bowtie2") + installed = Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_bowtie2") + local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2") + shutil.move(installed, local) + Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2", "meta.yml").unlink() + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + warnings = [x.message for x in subworkflow_lint.warned] + assert "Subworkflow `meta.yml` does not exist" in warnings + + def test_subworkflows_lint_local_old_format(self): + assert self.subworkflow_install.install("fastq_align_bowtie2") + installed = Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_bowtie2", "main.nf") + Path(self.pipeline_dir, "subworkflows", "local").mkdir(exist_ok=True) + local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2.nf") + shutil.copy(installed, local) + self.subworkflow_remove.remove("fastq_align_bowtie2", force=True) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 From 6c9c1b146bb69d87a513cfc0ff9d4fb0f5490e84 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:46:36 +0100 Subject: [PATCH 11/65] feat(1929): don't ask for a module/subworkflow name to lint if linting local --- nf_core/modules/lint/__init__.py | 3 +-- nf_core/subworkflows/lint/__init__.py | 2 +- tests/modules/test_lint.py | 6 +++--- tests/subworkflows/test_lint.py | 6 +++--- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index af057b67fc..5f64953116 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -117,7 +117,7 @@ def lint( """ # TODO: consider unifying modules and subworkflows lint() function and add it to the ComponentLint class # Prompt for module or all - if module is None and not all_modules and len(self.all_remote_components) > 0: + if module is None and not (local or all_modules) and len(self.all_remote_components) > 0: questions = [ { "type": "list", @@ -257,7 +257,6 @@ def lint_module( self.warned += warned else: self.failed += warned - self.failed += [LintResult(mod, *m) for m in mod.failed] # Otherwise run all the lint tests else: diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index ee46f3f3b2..8538cc3bf8 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -99,7 +99,7 @@ def lint( """ # TODO: consider unifying modules and subworkflows lint() function and add it to the ComponentLint class # Prompt for subworkflow or all - if subworkflow is None and not all_subworkflows: + if subworkflow is None and not (local or all_subworkflows): questions = [ { "type": "list", diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index 37f992a7c9..38597b44a0 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -767,7 +767,7 @@ def test_modules_lint_local(self): local = Path(self.pipeline_dir, "modules", "local", "trimgalore") shutil.move(installed, local) module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) - module_lint.lint(print_results=False, local=True, all_modules=True) + module_lint.lint(print_results=False, local=True) assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 @@ -780,7 +780,7 @@ def test_modules_lint_local_missing_files(self): Path(self.pipeline_dir, "modules", "local", "trimgalore", "environment.yml").unlink() Path(self.pipeline_dir, "modules", "local", "trimgalore", "meta.yml").unlink() module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) - module_lint.lint(print_results=False, local=True, all_modules=True) + module_lint.lint(print_results=False, local=True) assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 @@ -796,7 +796,7 @@ def test_modules_lint_local_old_format(self): shutil.copy(installed, local) self.mods_remove.remove("trimgalore", force=True) module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) - module_lint.lint(print_results=False, local=True, all_modules=True) + module_lint.lint(print_results=False, local=True) assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 diff --git a/tests/subworkflows/test_lint.py b/tests/subworkflows/test_lint.py index 49c671c088..8f6ff353a1 100644 --- a/tests/subworkflows/test_lint.py +++ b/tests/subworkflows/test_lint.py @@ -403,7 +403,7 @@ def test_subworkflows_lint_local(self): local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2") shutil.move(installed, local) subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) - subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + subworkflow_lint.lint(print_results=False, local=True) assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -415,7 +415,7 @@ def test_subworkflows_lint_local_missing_files(self): shutil.move(installed, local) Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2", "meta.yml").unlink() subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) - subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + subworkflow_lint.lint(print_results=False, local=True) assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -430,7 +430,7 @@ def test_subworkflows_lint_local_old_format(self): shutil.copy(installed, local) self.subworkflow_remove.remove("fastq_align_bowtie2", force=True) subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) - subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + subworkflow_lint.lint(print_results=False, local=True) assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 From 19df0447c06c7b1e8266ca60a706039e686aa667 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:59:47 +0100 Subject: [PATCH 12/65] feat(1929): add test for new pipeline lint check of local component dirs --- .../lint/local_component_structure.py | 23 +++++++++------ .../lint/test_local_component_structure.py | 28 +++++++++++++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 tests/pipelines/lint/test_local_component_structure.py diff --git a/nf_core/pipelines/lint/local_component_structure.py b/nf_core/pipelines/lint/local_component_structure.py index 61c1d45e29..71f02ba545 100644 --- a/nf_core/pipelines/lint/local_component_structure.py +++ b/nf_core/pipelines/lint/local_component_structure.py @@ -18,14 +18,21 @@ def local_component_structure(self): modules/local/modules/TOOL_SUBTOOL.nf """ - warned = [] + warned_mods = [] for nf_file in Path(self.wf_path, "modules", "local").glob("*.nf"): - warned.append(f"{nf_file.name} in modules/local should be moved to a TOOL/SUBTOOL/main.nf structure") - for nf_file in Path(self.wf_path, "subworkflows", "local").glob("*.nf"): - warned.append(f"{nf_file.name} in subworkflows/local should be moved to a SUBWORKFLOW_NAME/main.nf structure") - + warned_mods.append(f"{nf_file.name} in modules/local should be moved to a TOOL/SUBTOOL/main.nf structure") # If there are modules installed in the wrong location passed = [] - if len(warned) == 0: - passed = ["modules directory structure is correct 'modules/nf-core/TOOL/SUBTOOL'"] - return {"passed": passed, "warned": warned, "failed": [], "ignored": []} + if len(warned_mods) == 0: + passed = ["local modules directory structure is correct 'modules/local/TOOL/SUBTOOL'"] + + warned_swfs = [] + for nf_file in Path(self.wf_path, "subworkflows", "local").glob("*.nf"): + warned_swfs.append( + f"{nf_file.name} in subworkflows/local should be moved to a SUBWORKFLOW_NAME/main.nf structure" + ) + + if len(warned_swfs) == 0: + passed = ["local subworkflows directory structure is correct 'subworkflows/local/TOOL/SUBTOOL'"] + + return {"passed": passed, "warned": warned_mods + warned_swfs, "failed": [], "ignored": []} diff --git a/tests/pipelines/lint/test_local_component_structure.py b/tests/pipelines/lint/test_local_component_structure.py new file mode 100644 index 0000000000..93dc3174a3 --- /dev/null +++ b/tests/pipelines/lint/test_local_component_structure.py @@ -0,0 +1,28 @@ +from pathlib import Path + +import nf_core.pipelines.lint + +from ..test_lint import TestLint + + +class TestLintLocalComponentStructure(TestLint): + def setUp(self) -> None: + super().setUp() + self.new_pipeline = self._make_pipeline_copy() + + def test_local_component_structure(self): + local_modules = Path(self.new_pipeline, "modules", "local") + local_swf = Path(self.new_pipeline, "subworkflows", "local") + local_modules.mkdir(parents=True, exist_ok=True) + local_swf.mkdir(parents=True, exist_ok=True) + + (local_modules / "dummy_module.nf").touch() + (local_swf / "dummy_subworkflow.nf").touch() + + lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) + lint_obj._load() + + results = lint_obj.local_component_structure() + assert len(results.get("warned", [])) == 2 + assert len(results.get("failed", [])) == 0 + assert len(results.get("ignored", [])) == 0 From 5d1d44b07d0de940b69cbbf8c404d0412df65585 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 30 Oct 2024 10:04:31 +0100 Subject: [PATCH 13/65] feat(1929): run todo linting on local subworkflows --- nf_core/subworkflows/lint/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index 8538cc3bf8..c1be26d0d5 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -209,6 +209,7 @@ def lint_subworkflow(self, swf, progress_bar, registry, local=False): if local: self.main_nf(swf) self.meta_yml(swf, allow_missing=True) + self.subworkflow_todos(swf) self.passed += [LintResult(swf, *s) for s in swf.passed] warned = [LintResult(swf, *m) for m in (swf.warned + swf.failed)] if not self.fail_warned: From 5689ff6258ae5bc8f24397392b6e7f0ca87dd4d9 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:30:13 +1030 Subject: [PATCH 14/65] fix: Update to latest CHANGELOG --- CHANGELOG.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c910b9cd02..ad20861f8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,11 @@ ### Modules +<<<<<<< HEAD - feat(1929): Make local modules use remote structure ([#3256](https://github.com/nf-core/tools/pull/3256)) +======= +>>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) ### Subworkflows ### General @@ -96,16 +99,20 @@ ### Modules - add a panel around diff previews when updating ([#3246](https://github.com/nf-core/tools/pull/3246)) -- Modules created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) ### Subworkflows +<<<<<<< HEAD <<<<<<< HEAD - Add `nf-core subworkflows patch` command ([#2861](https://github.com/nf-core/tools/pull/2861)) - Improve subworkflow nf-test migration warning ([#3298](https://github.com/nf-core/tools/pull/3298)) ======= - Subworkflows created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) >>>>>>> 31cffdd5 (feat(1929): allow missing files when linting local modules) +======= +- Add `nf-core subworkflows patch` command ([#2861](https://github.com/nf-core/tools/pull/2861)) +- Improve subworkflow nf-test migration warning ([#3298](https://github.com/nf-core/tools/pull/3298)) +>>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) ### General @@ -116,6 +123,9 @@ - Update GitHub Actions ([#3237](https://github.com/nf-core/tools/pull/3237)) - add `--dir/-d` option to schema commands ([#3247](https://github.com/nf-core/tools/pull/3247)) <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) - fix headers in api docs ([#3323](https://github.com/nf-core/tools/pull/3323)) - handle new schema structure in `nf-core pipelines create-params-file` ([#3276](https://github.com/nf-core/tools/pull/3276)) - Update Gitpod image to use Miniforge instead of Miniconda([#3274](https://github.com/nf-core/tools/pull/3274)) @@ -138,9 +148,12 @@ - Update codecov/codecov-action action to v5 ([#3283](https://github.com/nf-core/tools/pull/3283)) - Update gitpod/workspace-base Docker digest to 12853f7 ([#3309](https://github.com/nf-core/tools/pull/3309)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.8.2 ([#3325](https://github.com/nf-core/tools/pull/3325)) +<<<<<<< HEAD ======= - Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.1 ([#3250](https://github.com/nf-core/tools/pull/3250)) >>>>>>> 31cffdd5 (feat(1929): allow missing files when linting local modules) +======= +>>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) ## [v3.0.2 - Titanium Tapir Patch](https://github.com/nf-core/tools/releases/tag/3.0.2) - [2024-10-11] From 3f76e999aa09694a4d9b90e8f2a08e956883cb2c Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:39:53 +1030 Subject: [PATCH 15/65] fix: Clean up CHANGELOG.md --- CHANGELOG.md | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad20861f8c..592d80ec62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,13 +8,12 @@ ### Modules -<<<<<<< HEAD -- feat(1929): Make local modules use remote structure ([#3256](https://github.com/nf-core/tools/pull/3256)) +- Modules created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) -======= ->>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) ### Subworkflows +- Subworkflows created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) + ### General ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] @@ -102,17 +101,8 @@ ### Subworkflows -<<<<<<< HEAD -<<<<<<< HEAD -- Add `nf-core subworkflows patch` command ([#2861](https://github.com/nf-core/tools/pull/2861)) -- Improve subworkflow nf-test migration warning ([#3298](https://github.com/nf-core/tools/pull/3298)) -======= -- Subworkflows created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) ->>>>>>> 31cffdd5 (feat(1929): allow missing files when linting local modules) -======= - Add `nf-core subworkflows patch` command ([#2861](https://github.com/nf-core/tools/pull/2861)) - Improve subworkflow nf-test migration warning ([#3298](https://github.com/nf-core/tools/pull/3298)) ->>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) ### General @@ -122,10 +112,6 @@ - use correct `--profile` options for `nf-core subworkflows test` ([#3233](https://github.com/nf-core/tools/pull/3233)) - Update GitHub Actions ([#3237](https://github.com/nf-core/tools/pull/3237)) - add `--dir/-d` option to schema commands ([#3247](https://github.com/nf-core/tools/pull/3247)) -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) - fix headers in api docs ([#3323](https://github.com/nf-core/tools/pull/3323)) - handle new schema structure in `nf-core pipelines create-params-file` ([#3276](https://github.com/nf-core/tools/pull/3276)) - Update Gitpod image to use Miniforge instead of Miniconda([#3274](https://github.com/nf-core/tools/pull/3274)) @@ -148,12 +134,6 @@ - Update codecov/codecov-action action to v5 ([#3283](https://github.com/nf-core/tools/pull/3283)) - Update gitpod/workspace-base Docker digest to 12853f7 ([#3309](https://github.com/nf-core/tools/pull/3309)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.8.2 ([#3325](https://github.com/nf-core/tools/pull/3325)) -<<<<<<< HEAD -======= -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.1 ([#3250](https://github.com/nf-core/tools/pull/3250)) ->>>>>>> 31cffdd5 (feat(1929): allow missing files when linting local modules) -======= ->>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) ## [v3.0.2 - Titanium Tapir Patch](https://github.com/nf-core/tools/releases/tag/3.0.2) - [2024-10-11] From def95d7315298ed2881396976098bb8464eb3b33 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 21 Jan 2025 06:57:08 +0000 Subject: [PATCH 16/65] [automated] Fix code linting --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa8db9824a..3df607ee57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,6 @@ - Add linting for ifEmpty(null) ([#3411](https://github.com/nf-core/tools/pull/3411)) - ### Modules ### Subworkflows From 660c8a7801c80f66edda3778681849874ef0b71f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 08:16:11 +0000 Subject: [PATCH 17/65] Update pre-commit hook editorconfig-checker/editorconfig-checker.python to v3.1.2 --- .pre-commit-config.yaml | 2 +- nf_core/pipeline-template/.pre-commit-config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1827289b76..54eedaff9e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: - prettier@3.3.3 - repo: https://github.com/editorconfig-checker/editorconfig-checker.python - rev: "3.0.3" + rev: "3.1.2" hooks: - id: editorconfig-checker alias: ec diff --git a/nf_core/pipeline-template/.pre-commit-config.yaml b/nf_core/pipeline-template/.pre-commit-config.yaml index 9e9f0e1c4e..1dec865026 100644 --- a/nf_core/pipeline-template/.pre-commit-config.yaml +++ b/nf_core/pipeline-template/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: - prettier@3.2.5 - repo: https://github.com/editorconfig-checker/editorconfig-checker.python - rev: "3.0.3" + rev: "3.1.2" hooks: - id: editorconfig-checker alias: ec From 8d336624f200b7e6cf7588990fa992ca93359c78 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 21 Jan 2025 08:16:57 +0000 Subject: [PATCH 18/65] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5aa7f22fe..dfa6701501 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ ### General +- Update pre-commit hook editorconfig-checker/editorconfig-checker.python to v3.1.2 ([#3414](https://github.com/nf-core/tools/pull/3414)) + ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] ### Template From d1f6dc7f2d348be313e867066448e2720b02c19a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 21 Jan 2025 14:43:13 +0100 Subject: [PATCH 19/65] Fix 'process.shell' in nextflow.config --- nf_core/pipeline-template/nextflow.config | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 5bbe47d9d9..217b1c254d 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -238,14 +238,13 @@ env { } // Set bash options -process.shell = """\ -bash - -set -e # Exit if a tool returns a non-zero status/exit code -set -u # Treat unset variables and parameters as an error -set -o pipefail # Returns the status of the last command to exit with a non-zero status or zero if all successfully execute -set -C # No clobber - prevent output redirection from overwriting files. -""" +process.shell = [ + "bash", + "-C", // No clobber - prevent output redirection from overwriting files. + "-e", // Exit if a tool returns a non-zero status/exit code + "-u", // Treat unset variables and parameters as an error + "-o pipefail" // Returns the status of the last command to exit with a non-zero status or zero if all successfully execute +] // Disable process selector warnings by default. Use debug profile to enable warnings. nextflow.enable.configProcessNamesValidation = false From 702e00c6e6c3d7663e4865b469cf915c4ab92aaf Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 21 Jan 2025 13:59:49 +0000 Subject: [PATCH 20/65] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5aa7f22fe..ecb4fcb935 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ ### General +- Fix `process.shell` in `nextflow.config` ([#3416](https://github.com/nf-core/tools/pull/3416)) + ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] ### Template From bee05205db39bcb8e680e588402f9a3a74d3ca8d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 21 Jan 2025 23:00:16 +0100 Subject: [PATCH 21/65] Remove automated release tweets --- .../workflows/release-announcements.yml | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/release-announcements.yml b/nf_core/pipeline-template/.github/workflows/release-announcements.yml index e1b654d34b..820cc49695 100644 --- a/nf_core/pipeline-template/.github/workflows/release-announcements.yml +++ b/nf_core/pipeline-template/.github/workflows/release-announcements.yml @@ -27,39 +27,6 @@ jobs: ${{ steps.get_topics.outputs.topics }} #nfcore #openscience #nextflow #bioinformatics - send-tweet: - runs-on: ubuntu-latest - - steps: - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.10" - - name: Install dependencies - run: pip install tweepy==4.14.0 - - name: Send tweet - shell: python - run: | - import os - import tweepy - - client = tweepy.Client( - access_token=os.getenv("TWITTER_ACCESS_TOKEN"), - access_token_secret=os.getenv("TWITTER_ACCESS_TOKEN_SECRET"), - consumer_key=os.getenv("TWITTER_CONSUMER_KEY"), - consumer_secret=os.getenv("TWITTER_CONSUMER_SECRET"), - ) - tweet = os.getenv("TWEET") - client.create_tweet(text=tweet) - env: - TWEET: | - Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}! - - Please see the changelog: ${{ github.event.release.html_url }} - TWITTER_CONSUMER_KEY: ${{ secrets.TWITTER_CONSUMER_KEY }} - TWITTER_CONSUMER_SECRET: ${{ secrets.TWITTER_CONSUMER_SECRET }} - TWITTER_ACCESS_TOKEN: ${{ secrets.TWITTER_ACCESS_TOKEN }} - TWITTER_ACCESS_TOKEN_SECRET: ${{ secrets.TWITTER_ACCESS_TOKEN_SECRET }} - bsky-post: runs-on: ubuntu-latest steps: From 078e4ee9e6edc435fc874976d5de5a922e1e9d52 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 21 Jan 2025 22:01:35 +0000 Subject: [PATCH 22/65] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecb4fcb935..a09eb4e9fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ ### General - Fix `process.shell` in `nextflow.config` ([#3416](https://github.com/nf-core/tools/pull/3416)) +- Remove automated release tweets ([#3419](https://github.com/nf-core/tools/pull/3419)) ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] From 48ee89cfab8d739dfe22668a500923f025047a88 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 21 Jan 2025 22:09:51 +0000 Subject: [PATCH 23/65] [automated] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a09eb4e9fc..5045622b41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Remove automated release tweets ([#3419](https://github.com/nf-core/tools/pull/3419)) + ### Linting ### Modules @@ -13,7 +15,6 @@ ### General - Fix `process.shell` in `nextflow.config` ([#3416](https://github.com/nf-core/tools/pull/3416)) -- Remove automated release tweets ([#3419](https://github.com/nf-core/tools/pull/3419)) ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] From fba7d88bb68ab3b9ab3738f838dbcf8fb476bcaa Mon Sep 17 00:00:00 2001 From: lmReef Date: Wed, 22 Jan 2025 17:24:52 +1300 Subject: [PATCH 24/65] remove latin encoding, improve regex --- nf_core/pipelines/lint/pipeline_if_empty_null.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipelines/lint/pipeline_if_empty_null.py b/nf_core/pipelines/lint/pipeline_if_empty_null.py index 240e3c0021..16b558421d 100644 --- a/nf_core/pipelines/lint/pipeline_if_empty_null.py +++ b/nf_core/pipelines/lint/pipeline_if_empty_null.py @@ -20,7 +20,7 @@ def pipeline_if_empty_null(self, root_dir=None): passed = [] warned = [] file_paths = [] - pattern = re.compile(r"ifEmpty\(\s*null\s*\)") + pattern = re.compile(r"ifEmpty\s*\(\s*null\s*\)") # Pipelines don't provide a path, so use the workflow path. # Modules run this function twice and provide a string path @@ -29,7 +29,7 @@ def pipeline_if_empty_null(self, root_dir=None): ignore = [".git"] if Path(root_dir, ".gitignore").is_file(): - with open(Path(root_dir, ".gitignore"), encoding="latin1") as fh: + with open(Path(root_dir, ".gitignore")) as fh: for line in fh: ignore.append(Path(line.strip().rstrip("/")).name) for root, dirs, files in os.walk(root_dir, topdown=True): @@ -40,7 +40,7 @@ def pipeline_if_empty_null(self, root_dir=None): files[:] = [f for f in files if not fnmatch.fnmatch(str(Path(root, f)), i)] for fname in files: try: - with open(Path(root, fname), encoding="latin1") as fh: + with open(Path(root, fname)) as fh: for line in fh: if re.findall(pattern, line): warned.append(f"`ifEmpty(null)` found in `{fname}`: _{line}_") From 3dd2e00c6b42e25744287f7d5f3ec4b75446dba2 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 22 Jan 2025 20:17:02 +1030 Subject: [PATCH 25/65] fix: broken test. Need to assign component_dir to the parent dir if a local module is an nf file only --- nf_core/components/nfcore_component.py | 1 + tests/modules/test_lint.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 03ee51b18a..b04cb71e35 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -97,6 +97,7 @@ def __init__( self.nftest_main_nf = p if (p := Path(self.nftest_testdir, "main.nf.test")).exists() else None else: self.main_nf = self.component_dir + self.component_dir = self.component_dir.parent self.meta_yml = None self.environment_yml = None self.nftest_testdir = None diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index 38597b44a0..b71c729d1c 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -789,11 +789,11 @@ def test_modules_lint_local_missing_files(self): assert "Module `meta.yml` does not exist" in warnings def test_modules_lint_local_old_format(self): + Path(self.pipeline_dir, "modules", "local").mkdir() assert self.mods_install.install("trimgalore") installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore", "main.nf") - Path(self.pipeline_dir, "modules", "local").mkdir() local = Path(self.pipeline_dir, "modules", "local", "trimgalore.nf") - shutil.copy(installed, local) + shutil.move(installed, local) self.mods_remove.remove("trimgalore", force=True) module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) module_lint.lint(print_results=False, local=True) From 262dea395dba2990dcfd7d607fd521153239c270 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Thu, 23 Jan 2025 08:08:25 +1030 Subject: [PATCH 26/65] fix: Add new lint test to lint config pydantic model --- nf_core/pipelines/lint/__init__.py | 2 +- nf_core/utils.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/pipelines/lint/__init__.py b/nf_core/pipelines/lint/__init__.py index 7f15b092cc..36f1bfca52 100644 --- a/nf_core/pipelines/lint/__init__.py +++ b/nf_core/pipelines/lint/__init__.py @@ -27,7 +27,7 @@ from nf_core import __version__ from nf_core.components.lint import ComponentLint from nf_core.pipelines.lint_utils import console -from nf_core.utils import NFCoreYamlConfig, NFCoreYamlLintConfig, strip_ansi_codes +from nf_core.utils import NFCoreYamlLintConfig, strip_ansi_codes from nf_core.utils import plural_s as _s from .actions_awsfulltest import actions_awsfulltest diff --git a/nf_core/utils.py b/nf_core/utils.py index 27334d473c..c0d7029e46 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1244,6 +1244,8 @@ class NFCoreYamlLintConfig(BaseModel): """ Lint for version consistency """ included_configs: Optional[bool] = None """ Lint for included configs """ + local_component_structure: Optional[bool] = None + """ Lint local components use correct structure mirroring remote""" def __getitem__(self, item: str) -> Any: return getattr(self, item) From 775da79dc26698ccd48ab541dd3d598ad2800b01 Mon Sep 17 00:00:00 2001 From: lmReef Date: Thu, 23 Jan 2025 16:04:22 +1300 Subject: [PATCH 27/65] fix latin1 encoding, remove unused import, remove gitignore file filtering --- .../pipelines/lint/pipeline_if_empty_null.py | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/nf_core/pipelines/lint/pipeline_if_empty_null.py b/nf_core/pipelines/lint/pipeline_if_empty_null.py index 16b558421d..e1d6651831 100644 --- a/nf_core/pipelines/lint/pipeline_if_empty_null.py +++ b/nf_core/pipelines/lint/pipeline_if_empty_null.py @@ -1,4 +1,3 @@ -import fnmatch import logging import os import re @@ -22,33 +21,26 @@ def pipeline_if_empty_null(self, root_dir=None): file_paths = [] pattern = re.compile(r"ifEmpty\s*\(\s*null\s*\)") - # Pipelines don't provide a path, so use the workflow path. + # Pipelines don"t provide a path, so use the workflow path. # Modules run this function twice and provide a string path if root_dir is None: root_dir = self.wf_path - ignore = [".git"] - if Path(root_dir, ".gitignore").is_file(): - with open(Path(root_dir, ".gitignore")) as fh: - for line in fh: - ignore.append(Path(line.strip().rstrip("/")).name) for root, dirs, files in os.walk(root_dir, topdown=True): - # Ignore files - for i_base in ignore: - i = str(Path(root, i_base)) - dirs[:] = [d for d in dirs if not fnmatch.fnmatch(str(Path(root, d)), i)] - files[:] = [f for f in files if not fnmatch.fnmatch(str(Path(root, f)), i)] for fname in files: try: - with open(Path(root, fname)) as fh: + with open(Path(root, fname), encoding="latin1") as fh: for line in fh: if re.findall(pattern, line): - warned.append(f"`ifEmpty(null)` found in `{fname}`: _{line}_") + warned.append( + f"`ifEmpty(null)` found in `{ + fname}`: _{line}_" + ) file_paths.append(Path(root, fname)) except FileNotFoundError: log.debug( f"Could not open file { - fname} in pipeline_if_empty_null lint test" + fname} in pipeline_if_empty_null lint test" ) if len(warned) == 0: From 49e67a6059fee32aaa3022687cf20e37c3b7b8b5 Mon Sep 17 00:00:00 2001 From: lmReef Date: Thu, 23 Jan 2025 16:07:29 +1300 Subject: [PATCH 28/65] add test --- tests/pipelines/lint/test_if_empty_null.py | 39 ++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tests/pipelines/lint/test_if_empty_null.py diff --git a/tests/pipelines/lint/test_if_empty_null.py b/tests/pipelines/lint/test_if_empty_null.py new file mode 100644 index 0000000000..a813a06569 --- /dev/null +++ b/tests/pipelines/lint/test_if_empty_null.py @@ -0,0 +1,39 @@ +from pathlib import Path + +import yaml + +import nf_core.pipelines.create +import nf_core.pipelines.lint + +from ..test_lint import TestLint + + +class TestLintIfEmptyNull(TestLint): + def setUp(self) -> None: + super().setUp() + self.new_pipeline = self._make_pipeline_copy() + self.nf_core_yml_path = Path(self.new_pipeline) / ".nf-core.yml" + with open(self.nf_core_yml_path) as f: + self.nf_core_yml = yaml.safe_load(f) + + def test_if_empty_null_throws_warn(self): + """Tests finding ifEmpty(null) in file throws warn in linting""" + # Create a file and add examples that should fail linting + txt_file = Path(self.new_pipeline) / "docs" / "test.txt" + with open(txt_file, "w") as f: + f.writelines( + [ + "ifEmpty(null)\n", + "ifEmpty (null)\n", + "ifEmpty( null )\n", + "ifEmpty ( null )\n", + ".ifEmpty(null)\n", + ". ifEmpty(null)\n", + "|ifEmpty(null)\n", + "| ifEmpty(null)\n", + ] + ) + lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) + lint_obj._load() + result = lint_obj.pipeline_if_empty_null() + assert len(result["warned"]) == 8 From 583809cfacbbc60fd3bb14c9523af736e4b14b69 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 23 Jan 2025 08:15:40 +0100 Subject: [PATCH 29/65] Update python:3.12-slim Docker digest to 123be56 (#3421) * Update python:3.12-slim Docker digest to 123be56 * [automated] Update CHANGELOG.md --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: nf-core-bot --- CHANGELOG.md | 1 + Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5045622b41..e67dce371e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ### General - Fix `process.shell` in `nextflow.config` ([#3416](https://github.com/nf-core/tools/pull/3416)) +- Update python:3.12-slim Docker digest to 123be56 ([#3421](https://github.com/nf-core/tools/pull/3421)) ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] diff --git a/Dockerfile b/Dockerfile index f5a6796a27..d26381a066 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim@sha256:10f3aaab98db50cba827d3b33a91f39dc9ec2d02ca9b85cbc5008220d07b17f3 +FROM python:3.12-slim@sha256:123be5684f39d8476e64f47a5fddf38f5e9d839baff5c023c815ae5bdfae0df7 LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools" From 1283406814592c2a1b5e51bc3f819d948906678b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 23 Jan 2025 21:14:52 +0000 Subject: [PATCH 30/65] Update pre-commit hook astral-sh/ruff-pre-commit to v0.9.3 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1827289b76..b902ab3ea0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.6 + rev: v0.9.3 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 52e8d4b90c9b8a99961f32514d541aeabb69ee6f Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 23 Jan 2025 22:25:38 +0000 Subject: [PATCH 31/65] [automated] Fix code linting --- nf_core/components/components_differ.py | 4 +--- nf_core/components/info.py | 4 ++-- nf_core/components/lint/__init__.py | 2 +- nf_core/components/remove.py | 2 +- nf_core/modules/lint/meta_yml.py | 2 +- nf_core/pipelines/download.py | 6 +++--- nf_core/pipelines/lint/multiqc_config.py | 2 +- nf_core/pipelines/lint/version_consistency.py | 2 +- nf_core/pipelines/params_file.py | 4 ++-- nf_core/pipelines/rocrate.py | 2 +- nf_core/pipelines/schema.py | 2 +- nf_core/pipelines/sync.py | 2 +- nf_core/subworkflows/lint/meta_yml.py | 2 +- tests/modules/test_lint.py | 18 +++++++++--------- tests/pipelines/test_create.py | 10 ++++++---- 15 files changed, 32 insertions(+), 32 deletions(-) diff --git a/nf_core/components/components_differ.py b/nf_core/components/components_differ.py index db51c1910d..18454471f4 100644 --- a/nf_core/components/components_differ.py +++ b/nf_core/components/components_differ.py @@ -271,9 +271,7 @@ def print_diff( console = Console(force_terminal=nf_core.utils.rich_force_colors()) if current_version is not None and new_version is not None: log.info( - f"Changes in component '{Path(repo_path, component)}' between" - f" ({current_version}) and" - f" ({new_version})" + f"Changes in component '{Path(repo_path, component)}' between ({current_version}) and ({new_version})" ) else: log.info(f"Changes in component '{Path(repo_path, component)}'") diff --git a/nf_core/components/info.py b/nf_core/components/info.py index 31769785a1..4cf7dc946c 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -265,9 +265,9 @@ def generate_component_info_help(self): intro_text.append( Text.from_markup( ":globe_with_meridians: Repository: " - f"{ '[link={self.remote_location}]' if self.remote_location.startswith('http') else ''}" + f"{'[link={self.remote_location}]' if self.remote_location.startswith('http') else ''}" f"{self.remote_location}" - f"{'[/link]' if self.remote_location.startswith('http') else '' }" + f"{'[/link]' if self.remote_location.startswith('http') else ''}" "\n" ) ) diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index 69740135a8..1e1dad9c34 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -102,7 +102,7 @@ def __init__( continue if isinstance(components, str): raise LookupError( - f"Error parsing modules.json: {components}. " f"Please check the file for errors or try again." + f"Error parsing modules.json: {components}. Please check the file for errors or try again." ) for org, comp in components: self.all_remote_components.append( diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index 37208629c0..316b8e7cb4 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -176,6 +176,6 @@ def remove(self, component, removed_by=None, removed_components=None, force=Fals f"Did not remove '{component}', because it was also manually installed. Only updated 'installed_by' entry in modules.json." ) log.info( - f"""Did not remove {self.component_type[:-1]} '{component}', because it was also installed by {', '.join(f"'{d}'" for d in installed_by)}. Only updated the 'installed_by' entry in modules.json.""" + f"""Did not remove {self.component_type[:-1]} '{component}', because it was also installed by {", ".join(f"'{d}'" for d in installed_by)}. Only updated the 'installed_by' entry in modules.json.""" ) return removed diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index d0268a40cc..328c15067f 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -78,7 +78,7 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None if len(e.path) > 0: hint = f"\nCheck the entry for `{e.path[0]}`." if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: - hint = f"\nCheck that the child entries of {str(e.path[0])+'.'+str(e.path[2])} are indented correctly." + hint = f"\nCheck that the child entries of {str(e.path[0]) + '.' + str(e.path[2])} are indented correctly." if e.schema and isinstance(e.schema, dict) and "message" in e.schema: e.message = e.schema["message"] incorrect_value = meta_yaml diff --git a/nf_core/pipelines/download.py b/nf_core/pipelines/download.py index c0f6e8c2f3..3d371ca681 100644 --- a/nf_core/pipelines/download.py +++ b/nf_core/pipelines/download.py @@ -202,7 +202,7 @@ def download_workflow(self): raise DownloadError(e) from e summary_log = [ - f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',['+str(len(self.revision)-2)+' more revisions],'+self.revision[-1]}'", + f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0] + ',[' + str(len(self.revision) - 2) + ' more revisions],' + self.revision[-1]}'", f"Use containers: '{self.container_system}'", ] if self.container_system: @@ -1908,8 +1908,8 @@ def __init__( log.error(self.error_type.message) log.info(self.error_type.helpmessage) - log.debug(f'Failed command:\n{" ".join(singularity_command)}') - log.debug(f'Singularity error messages:\n{"".join(error_msg)}') + log.debug(f"Failed command:\n{' '.join(singularity_command)}") + log.debug(f"Singularity error messages:\n{''.join(error_msg)}") raise self.error_type diff --git a/nf_core/pipelines/lint/multiqc_config.py b/nf_core/pipelines/lint/multiqc_config.py index fec5b518e3..49f632faa8 100644 --- a/nf_core/pipelines/lint/multiqc_config.py +++ b/nf_core/pipelines/lint/multiqc_config.py @@ -122,7 +122,7 @@ def multiqc_config(self) -> Dict[str, List[str]]: f"The expected comment is: \n" f"```{hint}``` \n" f"The current comment is: \n" - f"```{ mqc_yml['report_comment'].strip()}```" + f"```{mqc_yml['report_comment'].strip()}```" ) else: passed.append("`assets/multiqc_config.yml` contains a matching 'report_comment'.") diff --git a/nf_core/pipelines/lint/version_consistency.py b/nf_core/pipelines/lint/version_consistency.py index 2f9cead83c..e347e8e825 100644 --- a/nf_core/pipelines/lint/version_consistency.py +++ b/nf_core/pipelines/lint/version_consistency.py @@ -53,7 +53,7 @@ def version_consistency(self): # Check if they are consistent if len(set(versions.values())) != 1: failed.append( - "The versioning is not consistent between container, release tag " "and config. Found {}".format( + "The versioning is not consistent between container, release tag and config. Found {}".format( ", ".join([f"{k} = {v}" for k, v in versions.items()]) ) ) diff --git a/nf_core/pipelines/params_file.py b/nf_core/pipelines/params_file.py index 69326c142d..48e4dfe46c 100644 --- a/nf_core/pipelines/params_file.py +++ b/nf_core/pipelines/params_file.py @@ -19,7 +19,7 @@ "of nextflow run with the {pipeline_name} pipeline." ) -USAGE = "Uncomment lines with a single '#' if you want to pass the parameter " "to the pipeline." +USAGE = "Uncomment lines with a single '#' if you want to pass the parameter to the pipeline." H1_SEPERATOR = "## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" H2_SEPERATOR = "## ----------------------------------------------------------------------------" @@ -107,7 +107,7 @@ def get_pipeline(self) -> Optional[bool]: # Prompt for pipeline if not supplied if self.pipeline is None: launch_type = questionary.select( - "Generate parameter file for local pipeline " "or remote GitHub pipeline?", + "Generate parameter file for local pipeline or remote GitHub pipeline?", choices=["Remote pipeline", "Local path"], style=nf_core.utils.nfcore_question_style, ).unsafe_ask() diff --git a/nf_core/pipelines/rocrate.py b/nf_core/pipelines/rocrate.py index b0f9611f1f..2bf46beede 100644 --- a/nf_core/pipelines/rocrate.py +++ b/nf_core/pipelines/rocrate.py @@ -212,7 +212,7 @@ def set_main_entity(self, main_entity_filename: str): else: url = self.version self.crate.mainEntity.append_to( - "url", f"https://nf-co.re/{self.crate.name.replace('nf-core/','')}/{url}/", compact=True + "url", f"https://nf-co.re/{self.crate.name.replace('nf-core/', '')}/{url}/", compact=True ) self.crate.mainEntity.append_to("version", self.version, compact=True) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index b425ec64ed..47990b1064 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -770,7 +770,7 @@ def build_schema(self, pipeline_dir, no_prompts, web_only, url): if self.web_schema_build_web_url: log.info( "To save your work, open {}\n" - f"Click the blue 'Finished' button, copy the schema and paste into this file: { self.web_schema_build_web_url, self.schema_filename}" + f"Click the blue 'Finished' button, copy the schema and paste into this file: {self.web_schema_build_web_url, self.schema_filename}" ) return False diff --git a/nf_core/pipelines/sync.py b/nf_core/pipelines/sync.py index 14365da3f8..f0c8f4a321 100644 --- a/nf_core/pipelines/sync.py +++ b/nf_core/pipelines/sync.py @@ -345,7 +345,7 @@ def create_merge_base_branch(self): if merge_branch_format.match(branch) ] ) - new_branch = f"{self.original_merge_branch}-{max_branch+1}" + new_branch = f"{self.original_merge_branch}-{max_branch + 1}" log.info(f"Branch already existed: '{self.merge_branch}', creating branch '{new_branch}' instead.") self.merge_branch = new_branch diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index be282bc453..0de5ea626b 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -49,7 +49,7 @@ def meta_yml(subworkflow_lint_object, subworkflow): if len(e.path) > 0: hint = f"\nCheck the entry for `{e.path[0]}`." if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: - hint = f"\nCheck that the child entries of {e.path[0]+'.'+e.path[2]} are indented correctly." + hint = f"\nCheck that the child entries of {e.path[0] + '.' + e.path[2]} are indented correctly." subworkflow.failed.append( ( "meta_yml_valid", diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index 5372807987..b6e816d37e 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -295,15 +295,15 @@ def test_modules_lint_check_url(self): if line.strip(): check_container_link_line(mocked_ModuleLint, line, registry="quay.io") - assert ( - len(mocked_ModuleLint.passed) == passed - ), f"{test}: Expected {passed} PASS, got {len(mocked_ModuleLint.passed)}." - assert ( - len(mocked_ModuleLint.warned) == warned - ), f"{test}: Expected {warned} WARN, got {len(mocked_ModuleLint.warned)}." - assert ( - len(mocked_ModuleLint.failed) == failed - ), f"{test}: Expected {failed} FAIL, got {len(mocked_ModuleLint.failed)}." + assert len(mocked_ModuleLint.passed) == passed, ( + f"{test}: Expected {passed} PASS, got {len(mocked_ModuleLint.passed)}." + ) + assert len(mocked_ModuleLint.warned) == warned, ( + f"{test}: Expected {warned} WARN, got {len(mocked_ModuleLint.warned)}." + ) + assert len(mocked_ModuleLint.failed) == failed, ( + f"{test}: Expected {failed} FAIL, got {len(mocked_ModuleLint.failed)}." + ) def test_modules_lint_update_meta_yml(self): """update the meta.yml of a module""" diff --git a/tests/pipelines/test_create.py b/tests/pipelines/test_create.py index f83cc274fc..7c0eebe7d6 100644 --- a/tests/pipelines/test_create.py +++ b/tests/pipelines/test_create.py @@ -158,9 +158,9 @@ def test_template_customisation_all_files_grouping(self): str_path = str((Path(root) / file).relative_to(PIPELINE_TEMPLATE)) if str_path not in base_required_files: try: - assert ( - str_path in all_skipped_files - ), f"Template file `{str_path}` not present in a group for pipeline customisation in `template_features.yml`." + assert str_path in all_skipped_files, ( + f"Template file `{str_path}` not present in a group for pipeline customisation in `template_features.yml`." + ) except AssertionError: if "/" in str_path: # Check if the parent directory is in the skipped files @@ -170,6 +170,8 @@ def test_template_customisation_all_files_grouping(self): if upper_dir in all_skipped_files: upper_dir_present = True break - assert upper_dir_present, f"Template file `{str_path}` not present in a group for pipeline customisation in `template_features.yml`." + assert upper_dir_present, ( + f"Template file `{str_path}` not present in a group for pipeline customisation in `template_features.yml`." + ) else: raise From aa679d5a3c6743f7535baa5d09373cfa8865659d Mon Sep 17 00:00:00 2001 From: lmReef Date: Fri, 24 Jan 2025 15:34:33 +1300 Subject: [PATCH 32/65] fix formatting error --- nf_core/pipelines/lint/pipeline_if_empty_null.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/nf_core/pipelines/lint/pipeline_if_empty_null.py b/nf_core/pipelines/lint/pipeline_if_empty_null.py index e1d6651831..82cef45999 100644 --- a/nf_core/pipelines/lint/pipeline_if_empty_null.py +++ b/nf_core/pipelines/lint/pipeline_if_empty_null.py @@ -32,16 +32,10 @@ def pipeline_if_empty_null(self, root_dir=None): with open(Path(root, fname), encoding="latin1") as fh: for line in fh: if re.findall(pattern, line): - warned.append( - f"`ifEmpty(null)` found in `{ - fname}`: _{line}_" - ) + warned.append(f"`ifEmpty(null)` found in `{fname}`: _{line}_") file_paths.append(Path(root, fname)) except FileNotFoundError: - log.debug( - f"Could not open file { - fname} in pipeline_if_empty_null lint test" - ) + log.debug(f"Could not open file {fname} in pipeline_if_empty_null lint test") if len(warned) == 0: passed.append("No `ifEmpty(null)` strings found") From 3be4b603e9ff10e71deb0c7e165a6542d8c4538f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 09:39:54 +0100 Subject: [PATCH 33/65] chore(deps): update dawidd6/action-download-artifact action to v8 (#3424) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- nf_core/pipeline-template/.github/workflows/linting_comment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index 63b20bb311..b0b6755858 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 + uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8 with: workflow: linting.yml workflow_conclusion: completed From 61d971045728307963752d534ad3f6898ab49071 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 27 Jan 2025 09:53:22 +0100 Subject: [PATCH 34/65] fix mypy error --- nf_core/modules/lint/meta_yml.py | 2 +- nf_core/subworkflows/lint/meta_yml.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 13eee2ef96..977ea819ae 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -62,7 +62,7 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent, allow_m ).get("meta.yml") if lines is not None: yaml = ruamel.yaml.YAML() - meta_yaml = yaml.safe_load("".join(lines)) + meta_yaml = yaml.load("".join(lines)) if meta_yaml is None: module.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", module.meta_yml)) return diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index 3c63bf7353..8a2120ed0a 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -3,7 +3,7 @@ from pathlib import Path import jsonschema.validators -import yaml +import ruamel.yaml import nf_core.components.components_utils from nf_core.components.lint import LintExceptionError @@ -43,7 +43,8 @@ def meta_yml(subworkflow_lint_object, subworkflow, allow_missing: bool = False): try: with open(subworkflow.meta_yml) as fh: - meta_yaml = yaml.safe_load(fh) + yaml = ruamel.yaml.YAML(typ="safe") + meta_yaml = yaml.load(fh) subworkflow.passed.append(("meta_yml_exists", "Subworkflow `meta.yml` exists", subworkflow.meta_yml)) except FileNotFoundError: subworkflow.failed.append(("meta_yml_exists", "Subworkflow `meta.yml` does not exist", subworkflow.meta_yml)) @@ -62,7 +63,7 @@ def meta_yml(subworkflow_lint_object, subworkflow, allow_missing: bool = False): if len(e.path) > 0: hint = f"\nCheck the entry for `{e.path[0]}`." if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: - hint = f"\nCheck that the child entries of {e.path[0] + '.' + e.path[2]} are indented correctly." + hint = f"\nCheck that the child entries of {str(e.path[0]) + '.' + str(e.path[2])} are indented correctly." subworkflow.failed.append( ( "meta_yml_valid", From f8b099c5c0db44ef2f54b01329ee149e4c6be8ae Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 27 Jan 2025 10:55:28 +0100 Subject: [PATCH 35/65] Separate -o pipefail into separate list elements --- nf_core/pipeline-template/nextflow.config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 217b1c254d..d58ca72196 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -243,7 +243,8 @@ process.shell = [ "-C", // No clobber - prevent output redirection from overwriting files. "-e", // Exit if a tool returns a non-zero status/exit code "-u", // Treat unset variables and parameters as an error - "-o pipefail" // Returns the status of the last command to exit with a non-zero status or zero if all successfully execute + "-o", // Returns the status of the last command to exit.. + "pipefail" // ..with a non-zero status or zero if all successfully execute ] // Disable process selector warnings by default. Use debug profile to enable warnings. From 2d7e1a6cfea4549560468ccdeafb922e38cdbe09 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 27 Jan 2025 09:57:21 +0000 Subject: [PATCH 36/65] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00a8abd87b..0ae99a0057 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Fix `process.shell` in `nextflow.config` ([#3416](https://github.com/nf-core/tools/pull/3416)) - Update python:3.12-slim Docker digest to 123be56 ([#3421](https://github.com/nf-core/tools/pull/3421)) +- `-o pipefail` newline ([#3425](https://github.com/nf-core/tools/pull/3425)) ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] From 008dabba3ae0825786ff2d92958fcfde3a5db4d1 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 27 Jan 2025 11:46:39 +0100 Subject: [PATCH 37/65] update template components --- nf_core/pipeline-template/modules.json | 4 ++-- .../modules/nf-core/fastqc/main.nf | 20 ++++++++-------- .../modules/nf-core/fastqc/tests/tags.yml | 2 -- .../modules/nf-core/multiqc/environment.yml | 2 +- .../modules/nf-core/multiqc/main.nf | 4 ++-- .../nf-core/multiqc/tests/main.nf.test.snap | 24 +++++++++---------- 6 files changed, 27 insertions(+), 29 deletions(-) delete mode 100644 nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 7d2761d290..750826ffac 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -8,12 +8,12 @@ {%- if fastqc %} "fastqc": { "branch": "master", - "git_sha": "dc94b6ee04a05ddb9f7ae050712ff30a13149164", + "git_sha": "08108058ea36a63f141c25c4e75f9f872a5b2296", "installed_by": ["modules"] }{% endif %}{%- if multiqc %}{% if fastqc %},{% endif %} "multiqc": { "branch": "master", - "git_sha": "cf17ca47590cc578dfb47db1c2a44ef86f89976d", + "git_sha": "f0719ae309075ae4a291533883847c3f7c441dad", "installed_by": ["modules"] } {%- endif %} diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf index 752c3a10c6..033f4154a4 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf @@ -1,5 +1,5 @@ process FASTQC { - tag "$meta.id" + tag "${meta.id}" label 'process_medium' conda "${moduleDir}/environment.yml" @@ -19,30 +19,30 @@ process FASTQC { task.ext.when == null || task.ext.when script: - def args = task.ext.args ?: '' - def prefix = task.ext.prefix ?: "${meta.id}" + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" // Make list of old name and new name pairs to use for renaming in the bash while loop def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } - def rename_to = old_new_pairs*.join(' ').join(' ') + def rename_to = old_new_pairs*.join(' ').join(' ') def renamed_files = old_new_pairs.collect{ _old_name, new_name -> new_name }.join(' ') // The total amount of allocated RAM by FastQC is equal to the number of threads defined (--threads) time the amount of RAM defined (--memory) // https://github.com/s-andrews/FastQC/blob/1faeea0412093224d7f6a07f777fad60a5650795/fastqc#L211-L222 // Dividing the task.memory by task.cpu allows to stick to requested amount of RAM in the label - def memory_in_mb = MemoryUnit.of("${task.memory}").toUnit('MB') / task.cpus + def memory_in_mb = task.memory ? task.memory.toUnit('MB').toFloat() / task.cpus : null // FastQC memory value allowed range (100 - 10000) def fastqc_memory = memory_in_mb > 10000 ? 10000 : (memory_in_mb < 100 ? 100 : memory_in_mb) """ - printf "%s %s\\n" $rename_to | while read old_name new_name; do + printf "%s %s\\n" ${rename_to} | while read old_name new_name; do [ -f "\${new_name}" ] || ln -s \$old_name \$new_name done fastqc \\ - $args \\ - --threads $task.cpus \\ - --memory $fastqc_memory \\ - $renamed_files + ${args} \\ + --threads ${task.cpus} \\ + --memory ${fastqc_memory} \\ + ${renamed_files} cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml deleted file mode 100644 index 7834294ba0..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -fastqc: - - modules/nf-core/fastqc/** diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml index 6f5b867b76..a27122ce1a 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml @@ -2,4 +2,4 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::multiqc=1.25.1 + - bioconda::multiqc=1.27 diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf index cc0643e1d5..58d9313c6e 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf @@ -3,8 +3,8 @@ process MULTIQC { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.25.1--pyhdfd78af_0' : - 'biocontainers/multiqc:1.25.1--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.27--pyhdfd78af_0' : + 'biocontainers/multiqc:1.27--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap index 2fcbb5ff7d..7b7c132205 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -2,14 +2,14 @@ "multiqc_versions_single": { "content": [ [ - "versions.yml:md5,41f391dcedce7f93ca188f3a3ffa0916" + "versions.yml:md5,8f3b8c1cec5388cf2708be948c9fa42f" ] ], "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.4" }, - "timestamp": "2024-10-02T17:51:46.317523" + "timestamp": "2025-01-27T09:29:57.631982377" }, "multiqc_stub": { "content": [ @@ -17,25 +17,25 @@ "multiqc_report.html", "multiqc_data", "multiqc_plots", - "versions.yml:md5,41f391dcedce7f93ca188f3a3ffa0916" + "versions.yml:md5,8f3b8c1cec5388cf2708be948c9fa42f" ] ], "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.4" }, - "timestamp": "2024-10-02T17:52:20.680978" + "timestamp": "2025-01-27T09:30:34.743726958" }, "multiqc_versions_config": { "content": [ [ - "versions.yml:md5,41f391dcedce7f93ca188f3a3ffa0916" + "versions.yml:md5,8f3b8c1cec5388cf2708be948c9fa42f" ] ], "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.4" }, - "timestamp": "2024-10-02T17:52:09.185842" + "timestamp": "2025-01-27T09:30:21.44383553" } } \ No newline at end of file From ae92258b7307308a806990d718604db39bb6b3c0 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 27 Jan 2025 10:48:50 +0000 Subject: [PATCH 38/65] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44a21e3e05..8884ad042b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Remove automated release tweets ([#3419](https://github.com/nf-core/tools/pull/3419)) +- Update template components ([#3426](https://github.com/nf-core/tools/pull/3426)) ### Linting From 92408c301396b1b39785537006bb046d3f530764 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 27 Jan 2025 12:09:04 +0100 Subject: [PATCH 39/65] bump to v3.2.0 --- .gitpod.yml | 2 +- CHANGELOG.md | 4 +--- setup.py | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index d5948695bf..db31d01bed 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,4 @@ -image: nfcore/gitpod:dev +image: nfcore/gitpod:latest tasks: - name: install current state of nf-core/tools and setup pre-commit command: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 8884ad042b..6fcaface96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,12 @@ # nf-core/tools: Changelog -## v3.2.0dev +## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] ### Template - Remove automated release tweets ([#3419](https://github.com/nf-core/tools/pull/3419)) - Update template components ([#3426](https://github.com/nf-core/tools/pull/3426)) -### Linting - ### Modules - Modules created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) diff --git a/setup.py b/setup.py index 4a142c20fd..d426478786 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "3.2.0dev" +version = "3.2.0" with open("README.md") as f: readme = f.read() From 0d9a4f13a4390d85674d93cc8e71adc3fab0d1af Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 27 Jan 2025 12:31:43 +0100 Subject: [PATCH 40/65] template line break fixes --- nf_core/pipeline-template/docs/output.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/docs/output.md b/nf_core/pipeline-template/docs/output.md index a9be6620e1..d9bc3a188f 100644 --- a/nf_core/pipeline-template/docs/output.md +++ b/nf_core/pipeline-template/docs/output.md @@ -15,7 +15,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d {% if fastqc -%} - [FastQC](#fastqc) - Raw read QC{% endif %} - {%- if multiqc -%} + {%- if multiqc %} - [MultiQC](#multiqc) - Aggregate report describing results and QC from the whole pipeline {%- endif %} - [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution @@ -34,7 +34,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d [FastQC](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/) gives general quality metrics about your sequenced reads. It provides information about the quality score distribution across your reads, per base sequence content (%A/T/G/C), adapter contamination and overrepresented sequences. For further reading and documentation see the [FastQC help pages](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/). -{%- endif -%} +{%- endif %} {% if multiqc -%} @@ -53,7 +53,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d [MultiQC](http://multiqc.info) is a visualization tool that generates a single HTML report summarising all samples in your project. Most of the pipeline QC results are visualised in the report and further statistics are available in the report data directory. Results generated by MultiQC collate pipeline QC from supported tools e.g. FastQC. The pipeline has special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see . -{%- endif -%} +{%- endif %} ### Pipeline information From 381a6b49c744bf9223222e4c7ce06102f0f0a350 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 27 Jan 2025 13:51:16 +0100 Subject: [PATCH 41/65] update changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fcaface96..7527b374cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Remove automated release tweets ([#3419](https://github.com/nf-core/tools/pull/3419)) - Update template components ([#3426](https://github.com/nf-core/tools/pull/3426)) +- Fix `process.shell` in `nextflow.config` ([#3416](https://github.com/nf-core/tools/pull/3416)) and split into new lines ([#3425](https://github.com/nf-core/tools/pull/3425)) ### Modules @@ -18,9 +19,7 @@ ### General - Update pre-commit hook editorconfig-checker/editorconfig-checker.python to v3.1.2 ([#3414](https://github.com/nf-core/tools/pull/3414)) -- Fix `process.shell` in `nextflow.config` ([#3416](https://github.com/nf-core/tools/pull/3416)) - Update python:3.12-slim Docker digest to 123be56 ([#3421](https://github.com/nf-core/tools/pull/3421)) -- `-o pipefail` newline ([#3425](https://github.com/nf-core/tools/pull/3425)) ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] From b5770f8a507ef303a05ceb6e4f4a6c97f5894728 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 27 Jan 2025 15:34:03 +0100 Subject: [PATCH 42/65] bump to 3.3.0dev --- .gitpod.yml | 2 +- CHANGELOG.md | 12 ++++++++++++ setup.py | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index db31d01bed..d5948695bf 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,4 @@ -image: nfcore/gitpod:latest +image: nfcore/gitpod:dev tasks: - name: install current state of nf-core/tools and setup pre-commit command: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 7527b374cc..4a6e7a62c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # nf-core/tools: Changelog +## v3.3.0dev + +### Template + +### Linting + +### Modules + +### Subworkflows + +### General + ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] ### Template diff --git a/setup.py b/setup.py index d426478786..0441077fd3 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "3.2.0" +version = "3.3.0dev" with open("README.md") as f: readme = f.read() From c4c8aa94076b1ee1ab54f36bf141a2f90d78e152 Mon Sep 17 00:00:00 2001 From: lmReef <60915116+lmReef@users.noreply.github.com> Date: Tue, 28 Jan 2025 09:01:40 +1300 Subject: [PATCH 43/65] Update nf_core/pipelines/lint/pipeline_if_empty_null.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/pipelines/lint/pipeline_if_empty_null.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/lint/pipeline_if_empty_null.py b/nf_core/pipelines/lint/pipeline_if_empty_null.py index 82cef45999..dc82641a1a 100644 --- a/nf_core/pipelines/lint/pipeline_if_empty_null.py +++ b/nf_core/pipelines/lint/pipeline_if_empty_null.py @@ -21,7 +21,7 @@ def pipeline_if_empty_null(self, root_dir=None): file_paths = [] pattern = re.compile(r"ifEmpty\s*\(\s*null\s*\)") - # Pipelines don"t provide a path, so use the workflow path. + # Pipelines don't provide a path, so use the workflow path. # Modules run this function twice and provide a string path if root_dir is None: root_dir = self.wf_path From 17176f2cba25fe0ba38b57ab04ba49de9ae7cc6c Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Wed, 15 Jan 2025 12:56:23 +0100 Subject: [PATCH 44/65] Remove the on_pull_request_target trigger from the download test. --- .../.github/workflows/download_pipeline.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 977b4255dc..bb135cbfc6 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -14,15 +14,10 @@ on: pull_request: types: - opened - - edited - synchronize branches: - main - master - pull_request_target: - branches: - - main - - master env: NXF_ANSI_LOG: false From b9d61ce81a70b5bac38459ab0e1cb65622482513 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Wed, 15 Jan 2025 12:06:31 +0000 Subject: [PATCH 45/65] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 653f7cca6e..f34870d343 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ - Allow `nf-core pipelines download -r` to download commits ([#3374](https://github.com/nf-core/tools/pull/3374)) - Fix faulty Download Test Action to ensure that setup and test run as one job and on the same runner ([#3389](https://github.com/nf-core/tools/pull/3389)) +- Remove the on_pull_request_target trigger from the download test. ([#3399](https://github.com/nf-core/tools/pull/3399)) ### Modules From 19febf1f44310ad6e0349a9bf10fff76c0ac242d Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Mon, 27 Jan 2025 18:35:39 +0100 Subject: [PATCH 46/65] Remove push triggers and pull request types from CI. --- CHANGELOG.md | 3 ++- nf_core/pipeline-template/.github/workflows/ci.yml | 3 --- .../pipeline-template/.github/workflows/download_pipeline.yml | 3 --- nf_core/pipeline-template/.github/workflows/linting.yml | 3 --- 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f34870d343..c099d8360c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Remove the on `pull_request_target` trigger and `pull_request` types from the download test. Also drop `push` triggers on other CI tests. ([#3399](https://github.com/nf-core/tools/pull/3399)) + ### Linting - Add linting for ifEmpty(null) ([#3411](https://github.com/nf-core/tools/pull/3411)) @@ -46,7 +48,6 @@ - Allow `nf-core pipelines download -r` to download commits ([#3374](https://github.com/nf-core/tools/pull/3374)) - Fix faulty Download Test Action to ensure that setup and test run as one job and on the same runner ([#3389](https://github.com/nf-core/tools/pull/3389)) -- Remove the on_pull_request_target trigger from the download test. ([#3399](https://github.com/nf-core/tools/pull/3399)) ### Modules diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index f3a1073843..dd86342e51 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -1,9 +1,6 @@ name: nf-core CI # {% raw %}This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors on: - push: - branches: - - dev pull_request: release: types: [published] diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index bb135cbfc6..a3ac33d579 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -12,9 +12,6 @@ on: required: true default: "dev" pull_request: - types: - - opened - - synchronize branches: - main - master diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index cfdbcc12a9..21771001c2 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -3,9 +3,6 @@ name: nf-core linting # It runs the `nf-core pipelines lint` and markdown lint tests to ensure # that the code meets the nf-core guidelines. {%- raw %} on: - push: - branches: - - dev pull_request: release: types: [published] From e1591ae212f387f1fd299791aa49039dd28d09d2 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 28 Jan 2025 09:48:33 +0100 Subject: [PATCH 47/65] remove push event from GHA linting --- nf_core/pipelines/lint/actions_ci.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nf_core/pipelines/lint/actions_ci.py b/nf_core/pipelines/lint/actions_ci.py index 74f433ef80..35565da482 100644 --- a/nf_core/pipelines/lint/actions_ci.py +++ b/nf_core/pipelines/lint/actions_ci.py @@ -55,8 +55,6 @@ def actions_ci(self): # Check that the action is turned on for the correct events try: # NB: YAML dict key 'on' is evaluated to a Python dict key True - if "dev" not in ciwf[True]["push"]["branches"]: - raise AssertionError() pr_subtree = ciwf[True]["pull_request"] if not ( pr_subtree is None From 722514d9fec28275c06ca99a2bc71859d4c644d7 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 28 Jan 2025 16:39:40 +0100 Subject: [PATCH 48/65] Add additional step to Download action to get easy access to the Nextflow logfile for debugging. --- .../.github/workflows/download_pipeline.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index a3ac33d579..fae344e04c 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -112,6 +112,7 @@ jobs: echo "IMAGE_COUNT_AFTER=$image_count" >> "$GITHUB_OUTPUT" - name: Compare container image counts + id: count_comparison run: | if [ "${{ steps.count_initial.outputs.IMAGE_COUNT_INITIAL }}" -ne "${{ steps.count_afterwards.outputs.IMAGE_COUNT_AFTER }}" ]; then initial_count=${{ steps.count_initial.outputs.IMAGE_COUNT_INITIAL }} @@ -123,4 +124,10 @@ jobs: exit 1 else echo "The pipeline can be downloaded successfully!" - fi{% endraw %}{% endif %} + fi{% endraw %} + + - name: Upload Nextflow logfile for debugging purposes + uses: actions/upload-artifact@v4 + with: + name: nextflow_logfile.txt + path: .nextflow.log*{% endif %} From c81b818dd7510618cd72ed6781e5461201087318 Mon Sep 17 00:00:00 2001 From: Matthias Zepper Date: Tue, 28 Jan 2025 19:01:05 +0100 Subject: [PATCH 49/65] Include hidden files as well. --- .../pipeline-template/.github/workflows/download_pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index fae344e04c..1df0fa74da 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -130,4 +130,5 @@ jobs: uses: actions/upload-artifact@v4 with: name: nextflow_logfile.txt - path: .nextflow.log*{% endif %} + path: .nextflow.log* + include-hidden-files: true{% endif %} From 09b9e2add67b241fd8d58e481dae573cbaa3ccb3 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 29 Jan 2025 11:07:01 +0100 Subject: [PATCH 50/65] fix test_actions_ci_fail_wrong_trigger test --- tests/pipelines/lint/test_actions_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipelines/lint/test_actions_ci.py b/tests/pipelines/lint/test_actions_ci.py index 7319ce4b0c..4484ac74d9 100644 --- a/tests/pipelines/lint/test_actions_ci.py +++ b/tests/pipelines/lint/test_actions_ci.py @@ -34,7 +34,7 @@ def test_actions_ci_fail_wrong_trigger(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, ".github", "workflows", "ci.yml")) as fh: ci_yml = yaml.safe_load(fh) - ci_yml[True]["push"] = ["dev", "patch"] + ci_yml[True].pop("pull_request") ci_yml["jobs"]["test"]["strategy"]["matrix"] = {"nxf_versionnn": ["foo", ""]} with open(Path(new_pipeline, ".github", "workflows", "ci.yml"), "w") as fh: yaml.dump(ci_yml, fh) From 8e5777c5b8a19e2a989301e880bc5c8f861e7797 Mon Sep 17 00:00:00 2001 From: Matteo Bonfanti Date: Thu, 30 Jan 2025 10:16:19 +0100 Subject: [PATCH 51/65] output passed to write_params_file as Path object --- nf_core/commands_pipelines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/commands_pipelines.py b/nf_core/commands_pipelines.py index 3b28f4979c..c4eda78b11 100644 --- a/nf_core/commands_pipelines.py +++ b/nf_core/commands_pipelines.py @@ -218,7 +218,7 @@ def pipelines_create_params_file(ctx, pipeline, revision, output, force, show_hi """ builder = ParamsFileBuilder(pipeline, revision) - if not builder.write_params_file(output, show_hidden=show_hidden, force=force): + if not builder.write_params_file(Path(output), show_hidden=show_hidden, force=force): sys.exit(1) From 01f2e54f792cc4197352d9f7a94db09c776871ee Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 30 Jan 2025 09:26:58 +0000 Subject: [PATCH 52/65] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c099d8360c..105489e04e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ ### General +- output passed to write_params_file as Path object ([#3435](https://github.com/nf-core/tools/pull/3435)) + ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] ### Template From 1265f651aebf3a8cfb4a62d51f5bbdd0a588b55f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 30 Jan 2025 10:30:17 +0100 Subject: [PATCH 53/65] chore(deps): update python:3.12-slim docker digest to 69ce3ae (#3433) * chore(deps): update python:3.12-slim docker digest to 69ce3ae * [automated] Update CHANGELOG.md --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: nf-core-bot --- CHANGELOG.md | 2 ++ Dockerfile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c099d8360c..35b5f8865f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ ### General +- chore(deps): update python:3.12-slim docker digest to 69ce3ae ([#3433](https://github.com/nf-core/tools/pull/3433)) + ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] ### Template diff --git a/Dockerfile b/Dockerfile index d26381a066..c4bfefdedf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim@sha256:123be5684f39d8476e64f47a5fddf38f5e9d839baff5c023c815ae5bdfae0df7 +FROM python:3.12-slim@sha256:69ce3aed05675d284bee807e7c45e560e98db21fb1e4c670252b4ee0f2496b6d LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools" From 687ed9cf7206fa038b318ed8ab42f8e1e9e8c0a7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 1 Feb 2025 00:25:47 +0000 Subject: [PATCH 54/65] chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.9.4 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 41ea643642..57f756bb78 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.3 + rev: v0.9.4 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 6c52b668d2641c2dff102ff866b3daa9b8142ca6 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Sat, 1 Feb 2025 00:26:39 +0000 Subject: [PATCH 55/65] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a1dc737d2..5c772e8c08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - output passed to write_params_file as Path object ([#3435](https://github.com/nf-core/tools/pull/3435)) - chore(deps): update python:3.12-slim docker digest to 69ce3ae ([#3433](https://github.com/nf-core/tools/pull/3433)) +- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.9.4 ([#3438](https://github.com/nf-core/tools/pull/3438)) ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] From d8b2af5029b37b630c926e5e1481bc1eafad7580 Mon Sep 17 00:00:00 2001 From: Matteo Bonfanti Date: Mon, 3 Feb 2025 15:26:34 +0100 Subject: [PATCH 56/65] format name/value with YAML syntax --- nf_core/pipelines/params_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipelines/params_file.py b/nf_core/pipelines/params_file.py index 48e4dfe46c..6699068b49 100644 --- a/nf_core/pipelines/params_file.py +++ b/nf_core/pipelines/params_file.py @@ -213,7 +213,7 @@ def format_param( out += _print_wrapped("Required", mode="none", indent=4) out += _print_wrapped("\n", mode="end") - out += f"# {name} = {json.dumps(default)}\n" + out += f"# {name}: {json.dumps(default)}\n" return out From a1a5d5a962bbd66019879681406e329602b3c3af Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 3 Feb 2025 14:32:03 +0000 Subject: [PATCH 57/65] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c772e8c08..cfb8272828 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - output passed to write_params_file as Path object ([#3435](https://github.com/nf-core/tools/pull/3435)) - chore(deps): update python:3.12-slim docker digest to 69ce3ae ([#3433](https://github.com/nf-core/tools/pull/3433)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.9.4 ([#3438](https://github.com/nf-core/tools/pull/3438)) +- format name/value with YAML syntax ([#3442](https://github.com/nf-core/tools/pull/3442)) ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] From 02dcbb731ef6e54e0cd00057a9ec3b89777f838b Mon Sep 17 00:00:00 2001 From: Matteo Bonfanti Date: Mon, 3 Feb 2025 15:52:44 +0100 Subject: [PATCH 58/65] look for "# input: null" in test_params_file --- tests/pipelines/test_params_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipelines/test_params_file.py b/tests/pipelines/test_params_file.py index 0450d3f99d..723a8a561e 100644 --- a/tests/pipelines/test_params_file.py +++ b/tests/pipelines/test_params_file.py @@ -64,4 +64,4 @@ def test_build_template_content(self): out = fh.read() assert "nf-core/testpipeline" in out - assert "# input = null" in out + assert "# input: null" in out From b9640185a7993a91ddc9ed64d5b20817fd9a5f75 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 08:13:47 +0000 Subject: [PATCH 59/65] chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.9.5 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 57f756bb78..9582977ffe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.4 + rev: v0.9.5 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 27c01e58791f1f32d7b6bbae2989bfb910d74997 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 08:13:54 +0000 Subject: [PATCH 60/65] chore(deps): update pre-commit hook pre-commit/mirrors-mypy to v1.15.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 57f756bb78..7bd9252122 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: alias: ec - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.14.1" + rev: "v1.15.0" hooks: - id: mypy additional_dependencies: From 56d5c8d6fadfd82f11020fade44ea7e92305f485 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 10 Feb 2025 08:14:42 +0000 Subject: [PATCH 61/65] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfb8272828..5080464d17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - chore(deps): update python:3.12-slim docker digest to 69ce3ae ([#3433](https://github.com/nf-core/tools/pull/3433)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.9.4 ([#3438](https://github.com/nf-core/tools/pull/3438)) - format name/value with YAML syntax ([#3442](https://github.com/nf-core/tools/pull/3442)) +- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.9.5 ([#3445](https://github.com/nf-core/tools/pull/3445)) ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] From a9472e75caeda00b946c6f7f70546e588b08d65e Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 10 Feb 2025 08:14:51 +0000 Subject: [PATCH 62/65] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfb8272828..bd23b5651f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - chore(deps): update python:3.12-slim docker digest to 69ce3ae ([#3433](https://github.com/nf-core/tools/pull/3433)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.9.4 ([#3438](https://github.com/nf-core/tools/pull/3438)) - format name/value with YAML syntax ([#3442](https://github.com/nf-core/tools/pull/3442)) +- chore(deps): update pre-commit hook pre-commit/mirrors-mypy to v1.15.0 ([#3447](https://github.com/nf-core/tools/pull/3447)) ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] From e830164f9401ff023efdf5d6285e339524b5556a Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 10 Feb 2025 10:47:02 +0100 Subject: [PATCH 63/65] update prettier to 3.5.0 --- .pre-commit-config.yaml | 2 +- nf_core/pipeline-template/.pre-commit-config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8c5d71a4c3..a22aa24b45 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: hooks: - id: prettier additional_dependencies: - - prettier@3.3.3 + - prettier@3.5.0 - repo: https://github.com/editorconfig-checker/editorconfig-checker.python rev: "3.1.2" diff --git a/nf_core/pipeline-template/.pre-commit-config.yaml b/nf_core/pipeline-template/.pre-commit-config.yaml index 1dec865026..0ad2aa575e 100644 --- a/nf_core/pipeline-template/.pre-commit-config.yaml +++ b/nf_core/pipeline-template/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: prettier additional_dependencies: - - prettier@3.2.5 + - prettier@3.5.0 - repo: https://github.com/editorconfig-checker/editorconfig-checker.python rev: "3.1.2" From cab486295f5d2e21d3a40ac3724b61cb3e7bc9ed Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 10 Feb 2025 10:22:13 +0000 Subject: [PATCH 64/65] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c99e240baa..46df66a8c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - format name/value with YAML syntax ([#3442](https://github.com/nf-core/tools/pull/3442)) - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.9.5 ([#3445](https://github.com/nf-core/tools/pull/3445)) - chore(deps): update pre-commit hook pre-commit/mirrors-mypy to v1.15.0 ([#3447](https://github.com/nf-core/tools/pull/3447)) +- Update prettier to 3.5.0 ([#3448](https://github.com/nf-core/tools/pull/3448)) ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] From 38813b00c3a090f15c56119a56133c34c9c82edc Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 11 Feb 2025 15:29:25 +0000 Subject: [PATCH 65/65] chore(deps): update python:3.12-slim docker digest to 34656cd (#3450) * chore(deps): update python:3.12-slim docker digest to 34656cd * [automated] Update CHANGELOG.md --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: nf-core-bot --- CHANGELOG.md | 1 + Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46df66a8c8..7c46b9ebc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.9.5 ([#3445](https://github.com/nf-core/tools/pull/3445)) - chore(deps): update pre-commit hook pre-commit/mirrors-mypy to v1.15.0 ([#3447](https://github.com/nf-core/tools/pull/3447)) - Update prettier to 3.5.0 ([#3448](https://github.com/nf-core/tools/pull/3448)) +- chore(deps): update python:3.12-slim docker digest to 34656cd ([#3450](https://github.com/nf-core/tools/pull/3450)) ## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] diff --git a/Dockerfile b/Dockerfile index c4bfefdedf..331584d662 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim@sha256:69ce3aed05675d284bee807e7c45e560e98db21fb1e4c670252b4ee0f2496b6d +FROM python:3.12-slim@sha256:34656cd90456349040784165b9decccbcee4de66f3ead0a1168ba893455afd1e LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools"