Migration to pyproject.toml

.github/workflows/pythonbuild.yml

@@ -13,35 +13,22 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: "3.7"
-            tox-env: py37-core
-          - python-version: "3.8"
-            tox-env: py38-core
-          - python-version: "3.9"
-            tox-env: py39-core
-          - python-version: "3.10"
-            tox-env: py310-core
-          - python-version: "3.11"
-            tox-env: py311-core
-          - python-version: "3.12"
-            tox-env: py312-core
+          - tox-env: py38-core
+          - tox-env: py39-core
+          - tox-env: py310-core
+          - tox-env: py311-core
+          - tox-env: py312-core
 
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+      - name: Set up the latest version of uv
+        uses: astral-sh/setup-uv@v5
         with:
-          python-version: ${{ matrix.python-version }}
-      - name: Cache pip
-        uses: actions/cache@v4
-        with:
-          # This path is specific to Ubuntu
-          path: ~/.cache/pip
-          # Look to see if there is a cache hit for the corresponding requirements files
-          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }}
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip 'tox<4.0'
+          uv tool install --python-preference only-managed --python 3.12 tox --with tox-uv
       - name: Setup MySQL DB
         run: |
           sudo /etc/init.d/mysql start
@@ -51,7 +38,7 @@ jobs:
       - name: Build
         env:
           TOXENV: ${{ matrix.tox-env }}
-        run: tox
+        run: uvx --with tox-uv tox run
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5
         with:
@@ -81,42 +68,29 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: "3.7"
-            tox-env: py37-postgres
-          - python-version: "3.8"
-            tox-env: py38-postgres
-          - python-version: "3.9"
-            tox-env: py39-postgres
-          - python-version: "3.10"
-            tox-env: py310-postgres
-          - python-version: "3.11"
-            tox-env: py311-postgres
-          - python-version: "3.12"
-            tox-env: py312-postgres
+          - tox-env: py38-postgres
+          - tox-env: py39-postgres
+          - tox-env: py310-postgres
+          - tox-env: py311-postgres
+          - tox-env: py312-postgres
 
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Cache pip
-        uses: actions/cache@v4
+      - name: Set up the latest version of uv
+        uses: astral-sh/setup-uv@v5
         with:
-          # This path is specific to Ubuntu
-          path: ~/.cache/pip
-          # Look to see if there is a cache hit for the corresponding requirements files
-          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }}
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip 'tox<4.0'
+          uv tool install --python-preference only-managed --python 3.12 tox --with tox-uv
       - name: Create PSQL database
         run: |
           PGPASSWORD=postgres psql -h localhost -p 5432 -c 'create database spotify;' -U postgres
       - name: Build
         env:
           TOXENV: ${{ matrix.tox-env }}
-        run: tox
+        run: uvx --with tox-uv tox run
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5
         with:
@@ -135,91 +109,79 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: "3.7"
-            tox-env: py37-aws
-          - python-version: "3.8"
-            tox-env: py38-aws
-          - python-version: "3.9"
-            tox-env: py39-aws
-          - python-version: "3.10"
-            tox-env: py310-aws
-          - python-version: "3.11"
-            tox-env: py311-aws
-          - python-version: "3.12"
-            tox-env: py312-aws
+          - tox-env: py38-aws
+          - tox-env: py39-aws
+          - tox-env: py310-aws
+          - tox-env: py311-aws
+          - tox-env: py312-aws
 
-          - python-version: "3.7"
-            tox-env: py37-unixsocket
-            OVERRIDE_SKIP_CI_TESTS: True
-          - python-version: "3.8"
-            tox-env: py38-unixsocket
+          - tox-env: py38-unixsocket
             OVERRIDE_SKIP_CI_TESTS: True
-          - python-version: "3.9"
-            tox-env: py39-unixsocket
+          - tox-env: py39-unixsocket
             OVERRIDE_SKIP_CI_TESTS: True
-          - python-version: "3.10"
-            tox-env: py310-unixsocket
+          - tox-env: py310-unixsocket
             OVERRIDE_SKIP_CI_TESTS: True
-          - python-version: "3.11"
-            tox-env: py311-unixsocket
+          - tox-env: py311-unixsocket
             OVERRIDE_SKIP_CI_TESTS: True
-          - python-version: "3.12"
-            tox-env: py312-unixsocket
+          - tox-env: py312-unixsocket
             OVERRIDE_SKIP_CI_TESTS: True
 
-          - python-version: "3.7"
-            tox-env: py37-apache
-          - python-version: "3.8"
-            tox-env: py38-apache
-          - python-version: "3.9"
-            tox-env: py39-apache
-          - python-version: "3.10"
-            tox-env: py310-apache
-          - python-version: "3.11"
-            tox-env: py311-apache
-          - python-version: "3.12"
-            tox-env: py312-apache
+          - tox-env: py38-apache
+          - tox-env: py39-apache
+          - tox-env: py310-apache
+          - tox-env: py311-apache
+          - tox-env: py312-apache
 
-          - python-version: "3.7"
-            tox-env: py37-azureblob
-          - python-version: "3.8"
-            tox-env: py38-azureblob
-          - python-version: "3.9"
-            tox-env: py39-azureblob
-          - python-version: "3.10"
-            tox-env: py310-azureblob
-          - python-version: "3.11"
-            tox-env: py311-azureblob
-          - python-version: "3.12"
-            tox-env: py312-azureblob
-
-
-          - python-version: "3.10"
-            tox-env: flake8
-          - python-version: "3.10"
-            tox-env: docs
+          - tox-env: py38-azureblob
+          - tox-env: py39-azureblob
+          - tox-env: py310-azureblob
+          - tox-env: py311-azureblob
+          - tox-env: py312-azureblob
 
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+      - name: Set up the latest version of uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+      - name: Install dependencies
+        run: |
+          uv tool install --python-preference only-managed --python 3.12 tox --with tox-uv
+      - name: Build
+        env:
+          TOXENV: ${{ matrix.tox-env }}
+          OVERRIDE_SKIP_CI_TESTS: ${{ matrix.OVERRIDE_SKIP_CI_TESTS }}
+        run: uvx --with tox-uv tox run
+      - name: Upload coverage to Codecov
+        if: ${{ matrix.tox-env != 'flake8' && matrix.tox-env != 'docs' }}
+        uses: codecov/codecov-action@v5
         with:
-          python-version: ${{ matrix.python-version }}
-      - name: Cache pip
-        uses: actions/cache@v4
+          fail_ci_if_error: true
+          verbose: true
+  others:
+    runs-on: ubuntu-20.04
+
+    strategy:
+      matrix:
+        include:
+          - tox-env: flake8
+          - tox-env: docs
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up the latest version of uv
+        uses: astral-sh/setup-uv@v5
         with:
-          # This path is specific to Ubuntu
-          path: ~/.cache/pip
-          # Look to see if there is a cache hit for the corresponding requirements files
-          key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }}
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip 'tox<4.0'
+          uv tool install --python-preference only-managed --python 3.12 tox --with tox-uv
       - name: Build
         env:
           TOXENV: ${{ matrix.tox-env }}
           OVERRIDE_SKIP_CI_TESTS: ${{ matrix.OVERRIDE_SKIP_CI_TESTS }}
-        run: tox
+        run: uvx --with tox-uv tox run
       - name: Upload coverage to Codecov
         if: ${{ matrix.tox-env != 'flake8' && matrix.tox-env != 'docs' }}
         uses: codecov/codecov-action@v5

.gitignore

@@ -172,3 +172,5 @@ Icon
 Network Trash Folder
 Temporary Items
 .apdisk
+
+.python-version

.readthedocs.yaml

@@ -16,3 +16,5 @@ python:
   install:
     - method: pip
       path: .
+      extra_requirements:
+        - readthedocs

CONTRIBUTING.rst

@@ -13,29 +13,31 @@ We are always happy to receive Pull Requests. When you open a PR, it will
 automatically build on Travis. So you're not strictly required to test the
 patch locally before submitting it.
 
-If you do want to run the tests locally you'll need to ``pip install tox`` and
-then run one of the tox commands below.
+If you do want to run the tests locally you'll need to run the commands below
+.. code:: bash
+   pipx install uv
+   uv tool install tox --with tox-uv
 
-You will need a ``tox --version`` of at least 2.0.
+You will need a ``tox --version`` of at least 4.22.
 
 .. code:: bash
 
     # These commands are pretty fast and will tell if you've
     # broken something major:
-    tox -e flake8
-    tox -e py37-core
+    tox run -e flake8
+    tox run -e py38-core
 
     # You can also test particular files for even faster iterations
-    tox -e py37-core test/rpc_test.py
+    tox run -e py38-core -- test/rpc_test.py
 
     # The visualiser tests require phantomjs to be installed on your path
-    tox -e visualiser
+    tox run -e visualiser
 
     # And some of the others involve downloading and running Hadoop:
-    tox -e py33-cdh
-    tox -e py34-hdp
+    tox run -e py38-cdh
+    tox run -e py39-hdp
 
-Where ``flake8`` is the lint checking, ``py37`` is obviously Python 3.7.
+Where ``flake8`` is the lint checking, ``py38`` is obviously Python 3.8.
 ``core`` are tests that do not require external components and ``cdh`` and
 ``hdp`` are two different hadoop distributions. For most local development it's
 usually enough to run the lint checking and a python version for ``core``
@@ -45,7 +47,7 @@ For `cdh` and `hdp`, tox will download the hadoop distribution for you. You
 however have to have Java installed and the `JAVA_HOME` environment variable
 set.
 
-For more details, check out the ``.travis.yml`` and ``tox.ini`` files.
+For more details, check out the ``.github/workflows/pythonbuild.yml`` and ``tox.ini`` files.
 
 Writing documentation
 =====================
@@ -59,7 +61,7 @@ You verify that your documentation code compiles by running
 
 .. code:: bash
 
-    tox -e docs
+    tox run -e docs
 
 After that, you can check how it renders locally with your browser
 

README.rst

@@ -18,7 +18,7 @@
     :target: https://luigi.readthedocs.io/en/stable/?badge=stable
     :alt: Documentation Status
 
-Luigi is a Python (3.7, 3.8, 3.9, 3.10, 3.11, 3.12 tested) package that helps you build complex
+Luigi is a Python (3.8, 3.9, 3.10, 3.11, 3.12 tested) package that helps you build complex
 pipelines of batch jobs. It handles dependency resolution, workflow management,
 visualization, handling failures, command line integration, and much more.
 

RELEASE-PROCESS.rst

@@ -1,13 +1,14 @@
 For maintainers of Luigi, who have push access to pypi. Here's how you upload
 Luigi to pypi.
 
-#. Make sure [twine](https://pypi.org/project/twine/) is installed ``pip install twine``.
-#. Update version number in `luigi/__meta__.py`.
+#. Make sure [uv](https://github.com/astral-sh/uv) is installed ``curl -LsSf https://astral.sh/uv/install.sh | sh``.
+#. Update version number in `luigi/__version__.py`.
 #. Commit, perhaps simply with a commit message like ``Version x.y.z``.
 #. Push to GitHub at [spotify/luigi](https://github.com/spotify/luigi).
-#. Clean up previous distributions by executing ``rm -rf dist``
-#. Build a source distribution by executing ``python setup.py sdist``
-#. Upload to pypi by executing ``twine upload dist/*``
+#. Clean up previous distributions by executing ``rm -rf dist``.
+#. Build a source distribution by executing ``uv build``.
+#. Set pypi token on environment variable ``export UV_PUBLISH_TOKEN="LUIGI_PYPI_TOKEN_HERE"``.
+#. Upload to pypi by executing ``uv publish``.
 #. Add a tag on github (https://github.com/spotify/luigi/releases),
    including a handwritten changelog, possibly inspired from previous notes.
 

doc/conf.py

@@ -67,7 +67,10 @@ def _warn_node(self, msg, node, *args, **kwargs):
 
 sphinx.environment.BuildEnvironment.warn_node = _warn_node
 
-if os.environ.get('READTHEDOCS', None) == 'True':
+# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
+on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+if on_rtd:
     # Run sphinx-apidoc automatically in readthedocs
     # Taken from this: https://lists.torproject.org/pipermail/tor-commits/2012-September/046695.html
     os.system('sphinx-apidoc -o api -T ../luigi --separate')
@@ -83,7 +86,7 @@ def _warn_node(self, msg, node, *args, **kwargs):
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
-needs_sphinx = '1.4.4'  # Value mirrored in doc/conf.py
+needs_sphinx = '1.4.4'  # Value mirrored in pyproject.toml
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
@@ -167,9 +170,6 @@ def _warn_node(self, msg, node, *args, **kwargs):
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 
-# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
-
 if not on_rtd:  # only import and set the theme if we're building docs locally
     try:
         import sphinx_rtd_theme