Use pip-tools for dependency management

.gitlab-ci.yml

@@ -33,7 +33,7 @@ pip:
   script:
     - python -m venv env
     - source env/bin/activate
-    - pip install -r dev-requirements.txt
+    - pip install -r requirements/dev.txt
   artifacts:
     name: 'pip-$CI_JOB_ID'
     paths:
@@ -103,3 +103,10 @@ prettier:
   <<: *node_test_template
   script:
     - npm run lint
+
+pip_tools:
+  <<: *python_test_template
+  script:
+    - ./scripts/compile-requirements.sh
+    - git diff
+    - git diff-index --quiet HEAD --

Dockerfile

@@ -25,8 +25,8 @@ EXPOSE 8000
 USER website
 
 RUN python -m venv $VIRTUAL_ENV
-COPY --chown=website requirements.txt ./
-RUN pip install --upgrade pip && pip install -r requirements.txt
+COPY --chown=website requirements/base.txt ./requirements/base.txt
+RUN pip install --upgrade pip && pip install -r ./requirements/base.txt
 
 COPY --chown=website --from=frontend ./static/build ./static/build
 
@@ -50,7 +50,7 @@ RUN curl -sSf https://just.systems/install.sh | bash -s -- --to /usr/bin
 # Restore user
 USER website
 
-COPY --chown=website dev-requirements.txt ./
-RUN pip install --upgrade pip && pip install -r dev-requirements.txt
+COPY --chown=website requirements/dev.txt ./requirements/dev.txt
+RUN pip install --upgrade pip && pip install -r requirements/dev.txt
 
 CMD sleep infinity

dev-requirements.txt

@@ -1,10 +0,0 @@
--r requirements.txt
-
-honcho==1.1.0
-django-stubs[compatible-mypy]==1.11.0
-flake8==4.0.1
-isort==5.10.1
-black==22.3.0
-django-browser-reload==1.6.0
-django-debug-toolbar==3.4.0
-types-requests==2.27.1

requirements.txt

@@ -1,14 +0,0 @@
-Django==4.0.5
-wagtail==3.0
-django-environ==0.8.1
-whitenoise[brotli]==6.2.0
-pygments==2.12.0
-beautifulsoup4==4.9.3
-lxml==4.9.0
-more-itertools==8.13.0
-requests==2.27.1
-wagtail-generic-chooser==0.4.1
-django-rq==2.5.1
-django-redis==5.2.0
-wagtail-draftail-snippet==0.4.1
-gunicorn==20.1.0

requirements/base.in

@@ -0,0 +1,14 @@
+Django
+wagtail
+django-environ
+whitenoise[brotli]
+pygments
+beautifulsoup4
+lxml
+more-itertools
+requests
+wagtail-generic-chooser
+django-rq
+django-redis
+wagtail-draftail-snippet
+gunicorn

requirements/base.txt

@@ -0,0 +1,61 @@
+#
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
+#
+#    pip-compile --annotation-style=line --output-file=requirements/base.txt requirements/base.in
+#
+anyascii==0.3.1           # via wagtail
+asgiref==3.5.2            # via django
+async-timeout==4.0.2      # via redis
+beautifulsoup4==4.9.3     # via -r requirements/base.in, wagtail
+brotli==1.0.9             # via whitenoise
+certifi==2022.6.15        # via requests
+charset-normalizer==2.1.0  # via requests
+click==8.1.3              # via rq
+deprecated==1.2.13        # via redis
+django==4.0.6             # via -r requirements/base.in, django-filter, django-modelcluster, django-permissionedforms, django-redis, django-rq, django-taggit, django-treebeard, djangorestframework, wagtail
+django-environ==0.9.0     # via -r requirements/base.in
+django-filter==21.1       # via wagtail
+django-modelcluster==6.0  # via wagtail
+django-permissionedforms==0.1  # via wagtail
+django-redis==5.2.0       # via -r requirements/base.in
+django-rq==2.5.1          # via -r requirements/base.in
+django-taggit==2.1.0      # via wagtail
+django-treebeard==4.5.1   # via wagtail
+djangorestframework==3.13.1  # via wagtail
+draftjs-exporter==2.1.7   # via wagtail
+et-xmlfile==1.1.0         # via openpyxl
+gunicorn==20.1.0          # via -r requirements/base.in
+html5lib==1.1             # via wagtail
+idna==3.3                 # via requests
+l18n==2021.3              # via wagtail
+lxml==4.9.1               # via -r requirements/base.in
+more-itertools==8.13.0    # via -r requirements/base.in
+openpyxl==3.0.10          # via tablib
+packaging==21.3           # via redis
+pillow==9.2.0             # via wagtail
+pygments==2.12.0          # via -r requirements/base.in
+pyparsing==3.0.9          # via packaging
+pytz==2022.1              # via django-modelcluster, djangorestframework, l18n
+redis==4.3.4              # via django-redis, django-rq, rq
+requests==2.28.1          # via -r requirements/base.in, wagtail, wagtail-generic-chooser
+rq==1.10.1                # via django-rq
+six==1.16.0               # via html5lib, l18n
+soupsieve==2.3.2.post1    # via beautifulsoup4
+sqlparse==0.4.2           # via django
+tablib[xls,xlsx]==3.2.1   # via wagtail
+telepath==0.2             # via wagtail
+urllib3==1.26.11          # via requests
+wagtail==3.0.1            # via -r requirements/base.in, wagtail-draftail-snippet
+wagtail-draftail-snippet==0.4.1  # via -r requirements/base.in
+wagtail-generic-chooser==0.4.1  # via -r requirements/base.in
+webencodings==0.5.1       # via html5lib
+whitenoise[brotli]==6.2.0  # via -r requirements/base.in
+willow==1.4.1             # via wagtail
+wrapt==1.14.1             # via deprecated
+xlrd==2.0.1               # via tablib
+xlsxwriter==3.0.3         # via wagtail
+xlwt==1.3.0               # via tablib
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

requirements/dev.in

@@ -0,0 +1,12 @@
+-r base.txt
+
+pip-tools
+
+honcho
+django-stubs[compatible-mypy]
+flake8
+isort
+black
+django-browser-reload
+django-debug-toolbar
+types-requests

requirements/dev.txt

@@ -0,0 +1,87 @@
+#
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
+#
+#    pip-compile --annotation-style=line --output-file=requirements/dev.txt requirements/dev.in
+#
+anyascii==0.3.1           # via -r requirements/base.txt, wagtail
+asgiref==3.5.2            # via -r requirements/base.txt, django
+async-timeout==4.0.2      # via -r requirements/base.txt, redis
+beautifulsoup4==4.9.3     # via -r requirements/base.txt, wagtail
+black==22.6.0             # via -r requirements/dev.in
+brotli==1.0.9             # via -r requirements/base.txt, whitenoise
+build==0.8.0              # via pip-tools
+certifi==2022.6.15        # via -r requirements/base.txt, requests
+charset-normalizer==2.1.0  # via -r requirements/base.txt, requests
+click==8.1.3              # via -r requirements/base.txt, black, pip-tools, rq
+deprecated==1.2.13        # via -r requirements/base.txt, redis
+django==4.0.6             # via -r requirements/base.txt, django-browser-reload, django-debug-toolbar, django-filter, django-modelcluster, django-permissionedforms, django-redis, django-rq, django-stubs, django-stubs-ext, django-taggit, django-treebeard, djangorestframework, wagtail
+django-browser-reload==1.6.0  # via -r requirements/dev.in
+django-debug-toolbar==3.5.0  # via -r requirements/dev.in
+django-environ==0.9.0     # via -r requirements/base.txt
+django-filter==21.1       # via -r requirements/base.txt, wagtail
+django-modelcluster==6.0  # via -r requirements/base.txt, wagtail
+django-permissionedforms==0.1  # via -r requirements/base.txt, wagtail
+django-redis==5.2.0       # via -r requirements/base.txt
+django-rq==2.5.1          # via -r requirements/base.txt
+django-stubs[compatible-mypy]==1.12.0  # via -r requirements/dev.in
+django-stubs-ext==0.5.0   # via django-stubs
+django-taggit==2.1.0      # via -r requirements/base.txt, wagtail
+django-treebeard==4.5.1   # via -r requirements/base.txt, wagtail
+djangorestframework==3.13.1  # via -r requirements/base.txt, wagtail
+draftjs-exporter==2.1.7   # via -r requirements/base.txt, wagtail
+et-xmlfile==1.1.0         # via -r requirements/base.txt, openpyxl
+flake8==4.0.1             # via -r requirements/dev.in
+gunicorn==20.1.0          # via -r requirements/base.txt
+honcho==1.1.0             # via -r requirements/dev.in
+html5lib==1.1             # via -r requirements/base.txt, wagtail
+idna==3.3                 # via -r requirements/base.txt, requests
+isort==5.10.1             # via -r requirements/dev.in
+l18n==2021.3              # via -r requirements/base.txt, wagtail
+lxml==4.9.1               # via -r requirements/base.txt
+mccabe==0.6.1             # via flake8
+more-itertools==8.13.0    # via -r requirements/base.txt
+mypy==0.971               # via django-stubs
+mypy-extensions==0.4.3    # via black, mypy
+openpyxl==3.0.10          # via -r requirements/base.txt, tablib
+packaging==21.3           # via -r requirements/base.txt, build, redis
+pathspec==0.9.0           # via black
+pep517==0.12.0            # via build
+pillow==9.2.0             # via -r requirements/base.txt, wagtail
+pip-tools==6.8.0          # via -r requirements/dev.in
+platformdirs==2.5.2       # via black
+pycodestyle==2.8.0        # via flake8
+pyflakes==2.4.0           # via flake8
+pygments==2.12.0          # via -r requirements/base.txt
+pyparsing==3.0.9          # via -r requirements/base.txt, packaging
+pytz==2022.1              # via -r requirements/base.txt, django-modelcluster, djangorestframework, l18n
+redis==4.3.4              # via -r requirements/base.txt, django-redis, django-rq, rq
+requests==2.28.1          # via -r requirements/base.txt, wagtail, wagtail-generic-chooser
+rq==1.10.1                # via -r requirements/base.txt, django-rq
+six==1.16.0               # via -r requirements/base.txt, html5lib, l18n
+soupsieve==2.3.2.post1    # via -r requirements/base.txt, beautifulsoup4
+sqlparse==0.4.2           # via -r requirements/base.txt, django, django-debug-toolbar
+tablib[xls,xlsx]==3.2.1   # via -r requirements/base.txt, wagtail
+telepath==0.2             # via -r requirements/base.txt, wagtail
+tomli==2.0.1              # via black, build, django-stubs, mypy
+types-pytz==2022.1.2      # via django-stubs
+types-pyyaml==6.0.11      # via django-stubs
+types-requests==2.28.5    # via -r requirements/dev.in
+types-urllib3==1.26.17    # via types-requests
+typing-extensions==4.3.0  # via django-stubs, django-stubs-ext, mypy
+urllib3==1.26.11          # via -r requirements/base.txt, requests
+wagtail==3.0.1            # via -r requirements/base.txt, wagtail-draftail-snippet
+wagtail-draftail-snippet==0.4.1  # via -r requirements/base.txt
+wagtail-generic-chooser==0.4.1  # via -r requirements/base.txt
+webencodings==0.5.1       # via -r requirements/base.txt, html5lib
+wheel==0.37.1             # via pip-tools
+whitenoise[brotli]==6.2.0  # via -r requirements/base.txt
+willow==1.4.1             # via -r requirements/base.txt, wagtail
+wrapt==1.14.1             # via -r requirements/base.txt, deprecated
+xlrd==2.0.1               # via -r requirements/base.txt, tablib
+xlsxwriter==3.0.3         # via -r requirements/base.txt, wagtail
+xlwt==1.3.0               # via -r requirements/base.txt, tablib
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools

scripts/compile-requirements.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd $(dirname $0)/..
+
+COMMON_ARGS="--quiet --annotation-style=line"
+
+pip-compile $COMMON_ARGS -o requirements/base.txt requirements/base.in
+pip-compile $COMMON_ARGS -o requirements/dev.txt requirements/dev.in