diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index ce6a3672fe..dc19a3c7f4 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -27,11 +27,17 @@ jobs: - name: Run nf-core/tools run: | - nf-core create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" - nf-core lint nf-core-testpipeline - nf-core list - nf-core licences nf-core-testpipeline - nf-core sync nf-core-testpipeline/ - nf-core schema build nf-core-testpipeline/ --no-prompts - nf-core bump-version nf-core-testpipeline/ 1.1 - nf-core modules install nf-core-testpipeline/ fastqc + nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" + nf-core --log-file log.txt lint nf-core-testpipeline + nf-core --log-file log.txt list + nf-core --log-file log.txt licences nf-core-testpipeline + nf-core --log-file log.txt sync nf-core-testpipeline/ + nf-core --log-file log.txt schema build nf-core-testpipeline/ --no-prompts + nf-core --log-file log.txt bump-version nf-core-testpipeline/ 1.1 + nf-core --log-file log.txt modules install nf-core-testpipeline/ fastqc + + - name: Upload log file artifact + uses: actions/upload-artifact@v2 + with: + name: nf-core-log-file + path: log.txt diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index f8063c53d8..256a33570d 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -29,13 +29,17 @@ jobs: wget -qO- get.nextflow.io | bash sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow - - name: Configure git - run: | - git config user.email "core@nf-co.re" - git config user.name "nf-core-bot" - - name: Run synchronisation if: github.repository == 'nf-core/tools' env: AUTH_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - run: nf-core sync --all --username nf-core-bot --auth-token $AUTH_TOKEN + run: | + git config --global user.email "core@nf-co.re" + git config --global user.name "nf-core-bot" + nf-core --log-file sync_log.txt sync --all --username nf-core-bot --auth-token $AUTH_TOKEN + + - name: Upload sync log file artifact + uses: actions/upload-artifact@v2 + with: + name: sync-log-file + path: sync_log.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 88a2aefb90..f4d8d69258 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # nf-core/tools: Changelog +## [v1.10.1](https://github.com/nf-core/tools/releases/tag/1.10.1) - [2020-07-30] + +Patch release to fix the automatic template synchronisation, which failed in the v1.10 release. + +* Improved logging: `nf-core --log-file log.txt` now saves a verbose log to disk. +* GitHub actions sync now uploads verbose log as an artifact. +* Sync - fixed several minor bugs, improved logging. +* Python Rich library updated to `>=4.2.1` + ## [v1.10 - Copper Camel](https://github.com/nf-core/tools/releases/tag/1.10) - [2020-07-30] ### Pipeline schema diff --git a/nf_core/__main__.py b/nf_core/__main__.py index ecbccbf6ee..a52b18a101 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -24,7 +24,9 @@ import nf_core.sync import nf_core.utils -log = logging.getLogger(__name__) +# Set up logging as the root logger +# Submodules should all traverse back to this +log = logging.getLogger() def run_nf_core(): @@ -101,16 +103,30 @@ def decorator(f): @click.group(cls=CustomHelpOrder) @click.version_option(nf_core.__version__) -@click.option("-v", "--verbose", is_flag=True, default=False, help="Verbose output (print debug statements).") -def nf_core_cli(verbose): - stderr = rich.console.Console(file=sys.stderr) - logging.basicConfig( - level=logging.DEBUG if verbose else logging.INFO, - format="%(message)s", - datefmt=" ", - handlers=[rich.logging.RichHandler(console=stderr, markup=True)], +@click.option("-v", "--verbose", is_flag=True, default=False, help="Print verbose output to the console.") +@click.option("-l", "--log-file", help="Save a verbose log to a file.", metavar="") +def nf_core_cli(verbose, log_file): + + # Set the base logger to output DEBUG + log.setLevel(logging.DEBUG) + + # Set up logs to the console + log.addHandler( + rich.logging.RichHandler( + level=logging.DEBUG if verbose else logging.INFO, + console=rich.console.Console(file=sys.stderr), + show_time=False, + markup=True, + ) ) + # Set up logs to a file if we asked for one + if log_file: + log_fh = logging.FileHandler(log_file, encoding="utf-8") + log_fh.setLevel(logging.DEBUG) + log_fh.setFormatter(logging.Formatter("[%(asctime)s] %(name)-20s [%(levelname)-7s] %(message)s")) + log.addHandler(log_fh) + # nf-core list @nf_core_cli.command(help_priority=1) @@ -274,9 +290,10 @@ def create(name, description, author, new_version, no_git, force, outdir): and not os.environ.get("GITHUB_REPOSITORY", "") == "nf-core/tools", help="Execute additional checks for release-ready workflows.", ) +@click.option("-p", "--show-passed", is_flag=True, help="Show passing tests on the command line.") @click.option("--markdown", type=str, metavar="", help="File to write linting results to (Markdown)") @click.option("--json", type=str, metavar="", help="File to write linting results to (JSON)") -def lint(pipeline_dir, release, markdown, json): +def lint(pipeline_dir, release, show_passed, markdown, json): """ Check pipeline code against nf-core guidelines. @@ -286,7 +303,7 @@ def lint(pipeline_dir, release, markdown, json): """ # Run the lint tests! - lint_obj = nf_core.lint.run_linting(pipeline_dir, release, markdown, json) + lint_obj = nf_core.lint.run_linting(pipeline_dir, release, show_passed, markdown, json) if len(lint_obj.failed) > 0: sys.exit(1) diff --git a/nf_core/lint.py b/nf_core/lint.py index 9a77c991a2..34f1ac5c2f 100755 --- a/nf_core/lint.py +++ b/nf_core/lint.py @@ -39,7 +39,7 @@ logging.getLogger("urllib3").setLevel(logging.WARNING) -def run_linting(pipeline_dir, release_mode=False, md_fn=None, json_fn=None): +def run_linting(pipeline_dir, release_mode=False, show_passed=False, md_fn=None, json_fn=None): """Runs all nf-core linting checks on a given Nextflow pipeline project in either `release` mode or `normal` mode (default). Returns an object of type :class:`PipelineLint` after finished. @@ -65,7 +65,7 @@ def run_linting(pipeline_dir, release_mode=False, md_fn=None, json_fn=None): return lint_obj # Print the results - lint_obj.print_results() + lint_obj.print_results(show_passed) # Save results to Markdown file if md_fn is not None: @@ -238,6 +238,7 @@ def lint_pipeline(self, release_mode=False): ) for fun_name in check_functions: progress.update(lint_progress, advance=1, func_name=fun_name) + log.debug("Running lint test: {}".format(fun_name)) getattr(self, fun_name)() if len(self.failed) > 0: log.error("Found test failures in `{}`, halting lint run.".format(fun_name)) @@ -1225,8 +1226,7 @@ def check_schema_lint(self): """ Lint the pipeline schema """ # Only show error messages from schema - if log.getEffectiveLevel() == logging.INFO: - logging.getLogger("nf_core.schema").setLevel(logging.ERROR) + logging.getLogger("nf_core.schema").setLevel(logging.ERROR) # Lint the schema self.schema_obj = nf_core.schema.PipelineSchema() @@ -1273,7 +1273,9 @@ def check_schema_params(self): if len(removed_params) == 0 and len(added_params) == 0: self.passed.append((15, "Schema matched params returned from nextflow config")) - def print_results(self): + def print_results(self, show_passed=False): + + log.debug("Printing final results") console = Console() # Helper function to format test links nicely @@ -1294,7 +1296,7 @@ def _s(some_list): return "" # Table of passed tests - if len(self.passed) > 0 and log.getEffectiveLevel() == logging.DEBUG: + if len(self.passed) > 0 and show_passed: table = Table(style="green", box=rich.box.ROUNDED) table.add_column( "[[\u2714]] {} Test{} Passed".format(len(self.passed), _s(self.passed)), no_wrap=True, diff --git a/nf_core/sync.py b/nf_core/sync.py index 5dc109cbb8..7e455b054f 100644 --- a/nf_core/sync.py +++ b/nf_core/sync.py @@ -100,7 +100,7 @@ def sync(self): self.make_pull_request() except PullRequestException as e: self.reset_target_dir() - raise PullRequestException(pr_exception) + raise PullRequestException(e) self.reset_target_dir() @@ -140,7 +140,7 @@ def get_wf_config(self): # Try to check out target branch (eg. `origin/dev`) try: if self.from_branch and self.repo.active_branch.name != self.from_branch: - log.info("Checking out workflow branch '{}'".format(self.from_branch)) + log.debug("Checking out workflow branch '{}'".format(self.from_branch)) self.repo.git.checkout(self.from_branch) except git.exc.GitCommandError: raise SyncException("Branch `{}` not found!".format(self.from_branch)) @@ -173,7 +173,7 @@ def get_wf_config(self): ) # Fetch workflow variables - log.info("Fetching workflow config variables") + log.debug("Fetching workflow config variables") self.wf_config = nf_core.utils.fetch_wf_config(self.pipeline_dir) # Check that we have the required variables @@ -201,7 +201,7 @@ def delete_template_branch_files(self): Delete all files in the TEMPLATE branch """ # Delete everything - log.info("Deleting all files in TEMPLATE branch") + log.debug("Deleting all files in TEMPLATE branch") for the_file in os.listdir(self.pipeline_dir): if the_file == ".git": continue @@ -219,11 +219,10 @@ def make_template_pipeline(self): """ Delete all files and make a fresh template using the workflow variables """ - log.info("Making a new template pipeline using pipeline variables") + log.debug("Making a new template pipeline using pipeline variables") # Only show error messages from pipeline creation - if log.getEffectiveLevel() == logging.INFO: - logging.getLogger("nf_core.create").setLevel(logging.ERROR) + logging.getLogger("nf_core.create").setLevel(logging.ERROR) nf_core.create.PipelineCreate( name=self.wf_config["manifest.name"].strip('"').strip("'"), @@ -247,7 +246,7 @@ def commit_template_changes(self): self.repo.git.add(A=True) self.repo.index.commit("Template update for nf-core/tools version {}".format(nf_core.__version__)) self.made_changes = True - log.info("Committed changes to TEMPLATE branch") + log.debug("Committed changes to TEMPLATE branch") except Exception as e: raise SyncException("Could not commit changes to TEMPLATE:\n{}".format(e)) return True @@ -257,7 +256,7 @@ def push_template_branch(self): and try to make a PR. If we don't have the auth token, try to figure out a URL for the PR and print this to the console. """ - log.info("Pushing TEMPLATE branch to remote") + log.debug("Pushing TEMPLATE branch to remote: '{}'".format(os.path.basename(self.pipeline_dir))) try: self.repo.git.push() except git.exc.GitCommandError as e: @@ -287,23 +286,18 @@ def make_pull_request(self): ) raise PullRequestException("No GitHub authentication token set - cannot make PR") - log.info("Submitting a pull request via the GitHub API") + log.debug("Submitting a pull request via the GitHub API") - pr_body_text = """ - A new release of the main template in nf-core/tools has just been released. - This automated pull-request attempts to apply the relevant updates to this pipeline. - - Please make sure to merge this pull-request as soon as possible. - Once complete, make a new minor release of your pipeline. - - For instructions on how to merge this PR, please see - [https://nf-co.re/developers/sync](https://nf-co.re/developers/sync#merging-automated-prs). - - For more information about this release of [nf-core/tools](https://github.com/nf-core/tools), - please see the [nf-core/tools v{tag} release page](https://github.com/nf-core/tools/releases/tag/{tag}). - """.format( - tag=nf_core.__version__ - ) + pr_body_text = ( + "A new release of the main template in nf-core/tools has just been released. " + "This automated pull-request attempts to apply the relevant updates to this pipeline.\n\n" + "Please make sure to merge this pull-request as soon as possible. " + "Once complete, make a new minor release of your pipeline. " + "For instructions on how to merge this PR, please see " + "[https://nf-co.re/developers/sync](https://nf-co.re/developers/sync#merging-automated-prs).\n\n" + "For more information about this release of [nf-core/tools](https://github.com/nf-core/tools), " + "please see the [nf-core/tools v{tag} release page](https://github.com/nf-core/tools/releases/tag/{tag})." + ).format(tag=nf_core.__version__) pr_content = { "title": "Important! Template update for nf-core/tools v{}".format(nf_core.__version__), @@ -360,6 +354,7 @@ def sync_all_pipelines(gh_username=None, gh_auth_token=None): # Let's do some updating! for wf in wfs.remote_workflows: + log.info("-" * 30) log.info("Syncing {}".format(wf.full_name)) # Make a local working directory @@ -373,8 +368,7 @@ def sync_all_pipelines(gh_username=None, gh_auth_token=None): assert repo # Only show error messages from pipeline creation - if log.getEffectiveLevel() == logging.INFO: - logging.getLogger("nf_core.create").setLevel(logging.ERROR) + logging.getLogger("nf_core.create").setLevel(logging.ERROR) # Sync the repo log.debug("Running template sync") @@ -395,7 +389,7 @@ def sync_all_pipelines(gh_username=None, gh_auth_token=None): failed_syncs.append(wf.name) else: log.info( - "[green]Sync successful for {}:[/] [blue][link={1}]{1}[/link]".format( + "[green]Sync successful for {0}:[/] [blue][link={1}]{1}[/link]".format( wf.full_name, sync_obj.gh_pr_returned_data.get("html_url") ) ) diff --git a/setup.py b/setup.py index 180f97f086..778d1a6d03 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages import sys -version = "1.10" +version = "1.10.1" with open("README.md") as f: readme = f.read() @@ -40,7 +40,7 @@ "pyyaml", "requests", "requests_cache", - "rich>=4.0.0", + "rich>=4.2.1", "tabulate", ], setup_requires=["twine>=1.11.0", "setuptools>=38.6."],