emails: restrict domains for default_from_email (#72173)

Django 3.2. changed the implementation of django.db.ConnectionHandler it
now uses asgiref.local.Local as a thread/asyncio-task local dictionnary
instead of threading.local. This new implementation use
threading.current_thread() to get a reference to the current thread
instead of threading._get_ident(), but inside __bootstrap_inner, the
_active dictionnary is not initialized and current_thread() returns a
dummy value instead of the current thread.

To work around this behaviour I made __bootstrap_inner wrap the run
method with the code needed to setup the tenant, so that it's run after
__boostrap_inner initialization of the current thread in the _active
dictionnary.

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
     - id: black
       args: ['--target-version', 'py37', '--skip-string-normalization', '--line-length', '110']
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.7.0
+    rev: 5.12.0
     hooks:
     - id: isort
       args: ['--profile', 'black', '--line-length', '110']

Jenkinsfile

@@ -9,23 +9,18 @@ pipeline {
     stages {
         stage('Unit Tests') {
             steps {
-                sh 'tox -rv -p 8'
+                sh """
+python3 -m venv ./venv/
+./venv/bin/pip install "tox"
+./venv/bin/tox run-parallel --workdir ./tox-workdir --recreate --parallel 8 --parallel-no-spinner
+./venv/bin/tox run -e coverage-report"""
             }
             post {
                 always {
                     script {
                         utils = new Utils()
-                        utils.publish_coverage('coverage-*.xml')
-                        utils.publish_coverage_native(
-                            'index.html', 'htmlcov-py3-django22-drf312-coverage-authentic', 'Coverage authentic tests')
-                        utils.publish_coverage_native(
-                            'index.html', 'htmlcov-py3-django22-drf312-coverage-hobo', 'Coverage hobo tests')
-                        utils.publish_coverage_native(
-                            'index.html', 'htmlcov-py3-django22-drf312-coverage-multipublik', 'Coverage multipublik tests')
-                        utils.publish_coverage_native(
-                            'index.html', 'htmlcov-py3-django22-drf312-coverage-multitenant', 'Coverage multitenant tests')
-                        utils.publish_coverage_native(
-                            'index.html', 'htmlcov-py3-django22-drf312-coverage-passerelle', 'Coverage passerelle tests')
+                        utils.publish_coverage('coverage.xml')
+                        utils.publish_coverage_native('index.html', 'htmlcov', 'Coverage')
                         utils.publish_pylint('pylint.out')
                     }
                     mergeJunitResults()

debian/debian_config_common.py

@@ -159,7 +159,7 @@ LOGGING = {
         'django.security.DisallowedRedirect': {
             'filters': ['clamp_to_warning'],
         },
-        'django.core.exceptions.DisallowedHost': {
+        'django.security.DisallowedHost': {
             'filters': ['clamp_to_warning'],
         },
         'django.template': {

hobo/agent/common/__init__.py

@@ -6,7 +6,7 @@ from kombu.common import Broadcast
 
 def notify_agents(data):
     '''Send notifications to all other tenants'''
-    tenant = connection.get_tenant()
+    tenant = connection.tenant
     if not hasattr(tenant, 'domain_url'):
         # Fake tenant, certainly during migration, do nothing
         return

hobo/applications/migrations/0011_element_type.py

@@ -0,0 +1,16 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('applications', '0010_relation_error'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='element',
+            name='type',
+            field=models.CharField(max_length=100, verbose_name='Type'),
+        ),
+    ]

hobo/applications/models.py

@@ -182,7 +182,7 @@ class Application(models.Model):
 
 
 class Element(models.Model):
-    type = models.CharField(max_length=25, verbose_name=_('Type'))
+    type = models.CharField(max_length=100, verbose_name=_('Type'))
     slug = models.SlugField(max_length=500, verbose_name=_('Slug'))
     name = models.CharField(max_length=500, verbose_name=_('Name'))
     cache = JSONField(blank=True, default=dict)
@@ -297,6 +297,8 @@ class Version(models.Model):
                 continue
             service_objects = {x.get_base_url_path(): x for x in get_installed_services(types=[service_id])}
             for service in services.values():
+                if service['url'] not in service_objects:
+                    continue
                 if service_objects[service['url']].secondary:
                     continue
                 url = urllib.parse.urljoin(service['url'], 'api/export-import/bundle-import/')

hobo/context_processors.py

@@ -83,6 +83,8 @@ class RemoteTemplate:
                 # deployed.
                 return Template('<html><body>{% block content %}{% endblock %}</body></html>')
             self.theme_skeleton_url = context['portal_url'] + '__skeleton__/'
+        elif not settings.THEME_SKELETON_URL:
+            return Template('<html><body>{% block content %}{% endblock %}</body></html>')
         else:
             self.theme_skeleton_url = settings.THEME_SKELETON_URL
         if item is None:

hobo/environment/management/commands/cook.py

@@ -122,7 +122,7 @@ class Command(BaseCommand):
         wait_operationals(services, timeout, self.verbosity, self.terminal_width, notify_agents)
 
     def create_hobo(self, url, primary=False, title=None, slug=None, **kwargs):
-        if connection.get_tenant().schema_name == 'public':
+        if connection.tenant.schema_name == 'public':
             # if we're not currently in a tenant then we force the creation of
             # a primary hobo
             primary = True
@@ -245,7 +245,7 @@ class Command(BaseCommand):
 
     def set_theme(self, theme):
         set_theme(theme)
-        HoboDeployCommand().configure_theme({'variables': {'theme': theme}}, connection.get_tenant())
+        HoboDeployCommand().configure_theme({'variables': {'theme': theme}}, connection.tenant)
 
     def set_variable(self, name, value, label=None, auto=None):
         if auto is None:
@@ -299,7 +299,7 @@ class Command(BaseCommand):
         attribute.save()
 
     def cook(self, filename):
-        current_tenant = connection.get_tenant()
+        current_tenant = connection.tenant
         self.run_cook(filename)
         connection.set_tenant(current_tenant)
 

hobo/environment/migrations/0023_populate_local_hobo.py

@@ -15,7 +15,7 @@ def populate_local_hobo(apps, schema_editor):
         pass
 
     if hasattr(connection, 'get_tenant'):
-        build_absolute_uri = getattr(connection.get_tenant(), 'build_absolute_uri', None)
+        build_absolute_uri = getattr(connection.tenant, 'build_absolute_uri', None)
         if build_absolute_uri:
             Hobo.objects.create(
                 secret_key=get_local_key(build_absolute_uri('/')),

hobo/environment/migrations/0024_remove_local_hobo.py

@@ -13,7 +13,7 @@ def populate_local_hobo(apps, schema_editor):
         pass
 
     if hasattr(connection, 'get_tenant'):
-        build_absolute_uri = getattr(connection.get_tenant(), 'build_absolute_uri', None)
+        build_absolute_uri = getattr(connection.tenant, 'build_absolute_uri', None)
         if build_absolute_uri:
             Hobo.objects.create(
                 secret_key=get_local_key(build_absolute_uri('/')),

hobo/environment/migrations/0028_clean_internal_ips.py

@@ -0,0 +1,26 @@
+# Generated by Django 2.2.28 on 2022-07-01 07:11
+
+from django.db import migrations
+
+
+def clean_internal_ips(apps, schema_editor):
+    Variable = apps.get_model('environment', 'Variable')
+    for var in Variable.objects.filter(name='SETTING_INTERNAL_IPS', service_pk__isnull=True, auto=True):
+        if Variable.objects.filter(
+            name='SETTING_INTERNAL_IPS.extend', service_pk__isnull=True, auto=True
+        ).exists():
+            var.delete()
+        else:
+            var.name = 'SETTING_INTERNAL_IPS.extend'
+            var.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('environment', '0027_allow_long_slug'),
+    ]
+
+    operations = [
+        migrations.RunPython(clean_internal_ips, migrations.RunPython.noop),
+    ]

hobo/environment/models.py

@@ -97,6 +97,25 @@ class Variable(models.Model):
         self._parse_value_as_json()
 
 
+def is_resolvable(url):
+    try:
+        netloc = urllib.parse.urlparse(url).netloc
+        if netloc and socket.gethostbyname(netloc):
+            return True
+    except socket.gaierror:
+        return False
+
+
+def has_valid_certificate(url):
+    try:
+        requests.get(url, timeout=5, verify=True, allow_redirects=False)
+        return True
+    except requests.exceptions.SSLError:
+        return False
+    except requests.exceptions.ConnectionError:
+        return False
+
+
 class ServiceBase(models.Model):
     class Meta:
         abstract = True
@@ -228,23 +247,10 @@ class ServiceBase(models.Model):
         return self.get_base_url_path() + '__provision__/'
 
     def is_resolvable(self):
-        try:
-            netloc = urllib.parse.urlparse(self.base_url).netloc
-            if netloc and socket.gethostbyname(netloc):
-                return True
-        except socket.gaierror:
-            return False
+        return is_resolvable(self.base_url)
 
     def has_valid_certificate(self):
-        if not self.is_resolvable():
-            return False
-        try:
-            requests.get(self.base_url, timeout=5, verify=True, allow_redirects=False)
-            return True
-        except requests.exceptions.SSLError:
-            return False
-        except requests.exceptions.ConnectionError:
-            return False
+        return has_valid_certificate(self.base_url)
 
     def is_running(self):
         if not self.is_resolvable():

hobo/environment/utils.py

@@ -60,8 +60,8 @@ def get_or_create_local_hobo():
         hobo = Hobo.objects.get(local=True)
     except Hobo.DoesNotExist:
         build_absolute_uri = None
-        if hasattr(connection, 'get_tenant') and hasattr(connection.get_tenant(), 'build_absolute_uri'):
-            build_absolute_uri = connection.get_tenant().build_absolute_uri
+        if hasattr(connection, 'get_tenant') and hasattr(connection.tenant, 'build_absolute_uri'):
+            build_absolute_uri = connection.tenant.build_absolute_uri
         else:
             request = StoreRequestMiddleware.get_request()
             if request:

hobo/environment/validators.py

@@ -3,18 +3,17 @@ import urllib.parse
 from django.core.exceptions import ValidationError
 from django.utils.translation import gettext_lazy as _
 
-from hobo.environment.models import ServiceBase
+from hobo.environment import models
 
 
 def validate_service_url(url):
-    service = ServiceBase(title='dummy', base_url=url)
-    if not service.is_resolvable():
+    if not models.is_resolvable(url):
         raise ValidationError(
             _('Error: %(netloc)s is not resolvable in URL %(url)s'),
             code='not-resolvable',
             params={'netloc': urllib.parse.urlsplit(url).netloc, 'url': url},
         )
-    if not service.has_valid_certificate():
+    if not models.has_valid_certificate(url):
         raise ValidationError(
             _('Error: no valid certificate for %(url)s'), code='invalid-certificate', params={'url': url}
         )

hobo/multitenant/apps.py

@@ -2,19 +2,30 @@ from django.apps import AppConfig, apps
 
 from . import settings, threads
 
+_tenant_settings_wrapper = None
+
+
+def clear_tenants_settings():
+    if _tenant_settings_wrapper is not None:
+        _tenant_settings_wrapper.__dict__['tenants_settings'] = {}
+
 
 class MultitenantAppConfig(AppConfig):
     name = 'hobo.multitenant'
     verbose_name = 'Multitenant'
 
     def ready(self):
+        global _tenant_settings_wrapper
+
         from django import conf
         from django.db import migrations
         from django.db.migrations import operations
 
         # Install tenant aware settings
         if not isinstance(conf.settings._wrapped, settings.TenantSettingsWrapper):
-            conf.settings._wrapped = settings.TenantSettingsWrapper(conf.settings._wrapped)
+            _tenant_settings_wrapper = conf.settings._wrapped = settings.TenantSettingsWrapper(
+                conf.settings._wrapped
+            )
             # reset settings getattr method to a cache-less version, to cancel
             # https://code.djangoproject.com/ticket/27625.
             conf.LazySettings.__getattr__ = lambda self, name: getattr(self._wrapped, name)

hobo/multitenant/haystack.py

@@ -28,7 +28,7 @@ from tenant_schemas.postgresql_backend.base import FakeTenant
 class WhooshSearchBackend(haystack.backends.whoosh_backend.WhooshSearchBackend):
     @property
     def use_file_storage(self):
-        tenant = connection.get_tenant()
+        tenant = connection.tenant
         return not (isinstance(connection.tenant, FakeTenant))
 
     @use_file_storage.setter
@@ -53,7 +53,7 @@ class WhooshSearchBackend(haystack.backends.whoosh_backend.WhooshSearchBackend):
 
     @property
     def path(self):
-        tenant = connection.get_tenant()
+        tenant = connection.tenant
         return os.path.join(tenant.get_directory(), 'whoosh_index')
 
     @path.setter

hobo/multitenant/log.py

@@ -24,7 +24,7 @@ class AdminEmailHandler(django.utils.log.AdminEmailHandler):
 
         subject = super().format_subject(subject)
         try:
-            subject = '[%s] %s' % (connection.get_tenant().domain_url, subject)
+            subject = '[%s] %s' % (connection.tenant.domain_url, subject)
         except AttributeError:
             pass
         return subject

hobo/multitenant/management/commands/__init__.py

@@ -105,8 +105,8 @@ https://django-tenant-schemas.readthedocs.org/en/latest/use.html#creating-a-tena
         if options.get('domain'):
             domain = options['domain']
         else:
-            if connection.get_tenant().schema_name != 'public':
-                return connection.get_tenant()
+            if connection.tenant.schema_name != 'public':
+                return connection.tenant
             displayed_tenants = all_tenants
             while True:
                 domain = input("Enter Tenant Domain ('?' to list): ")

hobo/multitenant/management/commands/tenant_command.py

@@ -59,7 +59,7 @@ def run_command_from_argv(command, argv):
             command.stderr.write(str(e), lambda x: x)
         else:
             command.stderr.write(
-                '%s: %s: %s' % (connection.get_tenant(), e.__class__.__name__, exception_to_text(e))
+                '%s: %s: %s' % (connection.tenant, e.__class__.__name__, exception_to_text(e))
             )
         return e
 

hobo/multitenant/settings.py

@@ -34,9 +34,6 @@ class TenantSettingsWrapper:
         self.__dict__['tenants_settings'] = {}
         self.__dict__['default_settings'] = default_settings
 
-    def clear_tenants_settings(self):
-        self.__dict__['tenants_settings'] = {}
-
     @property
     def loaders(self):
         loaders = getattr(self.default_settings, 'TENANT_SETTINGS_LOADERS', [])
@@ -87,7 +84,7 @@ class TenantSettingsWrapper:
             return self.default_settings
         try:
             self.local.in_get_wrapped = True
-            tenant = connection.get_tenant()
+            tenant = connection.tenant
             if not hasattr(tenant, 'domain_url'):
                 return self.default_settings
             return self.get_tenant_settings(tenant)

hobo/multitenant/settings_loaders.py

@@ -339,8 +339,10 @@ class Authentic(FileBaseSettingsLoader):
         saml_key = os.path.join(tenant_dir, 'saml.key')
         if os.path.exists(saml_crt) and os.path.exists(saml_key):
             tenant_settings.A2_IDP_SAML2_ENABLE = True
-            tenant_settings.A2_IDP_SAML2_SIGNATURE_PUBLIC_KEY = open(saml_crt).read()
-            tenant_settings.A2_IDP_SAML2_SIGNATURE_PRIVATE_KEY = open(saml_key).read()
+            with open(saml_crt) as f:
+                tenant_settings.A2_IDP_SAML2_SIGNATURE_PUBLIC_KEY = f.read()
+            with open(saml_key) as f:
+                tenant_settings.A2_IDP_SAML2_SIGNATURE_PRIVATE_KEY = f.read()
             if not getattr(tenant_settings, 'A2_IDP_OIDC_JWKSET', None):
                 from jwcrypto import jwk
 

hobo/multitenant/threads.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import functools
 import threading
 
 _Thread_start = threading.Thread.start
@@ -28,21 +29,34 @@ def _new_start(self):
     return _Thread_start(self)
 
 
+def wrap_run(self, func):
+    if getattr(func, '_wrapped', False):
+        return func
+
+    @functools.wraps(func)
+    def wrapper():
+        tenant = getattr(self, 'tenant', None)
+
+        if tenant is not None:
+            from django.db import connection
+
+            old_tenant = connection.tenant
+            connection.set_tenant(self.tenant)
+            try:
+                func()
+            finally:
+                connection.set_tenant(old_tenant)
+                connection.close()
+        else:
+            func()
+
+    wrapper._wrapped = True
+    return wrapper
+
+
 def _new__bootstrap_inner(self):
-    tenant = getattr(self, 'tenant', None)
-
-    if tenant is not None:
-        from django.db import connection
-
-        old_tenant = connection.get_tenant()
-        connection.set_tenant(self.tenant)
-        try:
-            _Thread__bootstrap_inner(self)
-        finally:
-            connection.set_tenant(old_tenant)
-            connection.close()
-    else:
-        _Thread__bootstrap_inner(self)
+    self.run = wrap_run(self, self.run)
+    _Thread__bootstrap_inner(self)
 
 
 def install_tenant_aware_threads():

hobo/test_utils.py

@@ -1,7 +1,27 @@
+# hobo - portal to configure and deploy applications
+# Copyright (C) 2022  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import hashlib
 import os
+import socket
+import struct
+from contextlib import closing
 
 
-def get_safe_db_name():
+def get_safe_db_name(max_length=53):
     """
     PostgreSQL database name limit is 63 characters, which can become
     an issue during testing, because we need to build a unique
@@ -19,4 +39,23 @@ def get_safe_db_name():
         # when we're in parallel mode, pytest-django will do this
         # for us at a later point
         parts.append(os.environ.get('TOX_ENV_NAME'))
-    return '_'.join(parts)
+    full_db_name = '_'.join(parts)
+    if len(full_db_name) < max_length:
+        return full_db_name
+    hashcode_length = 8
+    hashcode = hashlib.md5(full_db_name.encode()).hexdigest()[: hashcode_length - 2]
+    prefix_length = (max_length - hashcode_length) - (max_length - hashcode_length) // 2
+    suffix_length = (max_length - hashcode_length) // 2
+    assert hashcode_length + prefix_length + suffix_length == max_length
+    truncated_db_name = full_db_name[:prefix_length] + '_' + hashcode + '_' + full_db_name[-suffix_length:]
+    assert len(truncated_db_name) == max_length
+    return truncated_db_name
+
+
+def find_free_port():
+    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        s.bind(('', 0))
+        # SO_LINGER (man 7 socket) l_onoff=1 l_linger=0, immediately release
+        # the port on closing of the socket
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0))
+        return s.getsockname()[1]

setup.py

@@ -148,13 +148,13 @@ setup(
         'Programming Language :: Python',
     ],
     install_requires=[
-        'django>=2.2, <2.3',
+        'django>=2.2, <3.3',
         'gadjo',
         'celery<4' if sys.version_info < (3, 7) else 'celery>=4',
         'django-mellon',
         'django-tenant-schemas',
         'prometheus_client',
-        'djangorestframework>=3.4, <3.13',
+        'djangorestframework>=3.12, <3.14',
         'dnspython',
         'lxml',
         'num2words==0.5.9',

tests/test_application.py

@@ -66,6 +66,13 @@ WCS_AVAILABLE_OBJECTS = {
             "minor": True,
             "urls": {"list": "https://wcs.example.invalid/api/export-import/mail-templates/"},
         },
+        {
+            "id": "comment-templates-categories",
+            "text": "Categories (comment templates)",
+            "singular": "Category (comment templates)",
+            "minor": True,
+            "urls": {"list": "https://wcs.example.invalid/api/export-import/comment-templates-categories/"},
+        },
         {
             "id": "wscalls",
             "text": "Webservice Calls",
@@ -775,6 +782,12 @@ def get_bundle(with_icon=False):
                 {'type': 'forms', 'slug': 'test', 'name': 'test', 'auto-dependency': False},
                 {'type': 'blocks', 'slug': 'test', 'name': 'test', 'auto-dependency': True},
                 {'type': 'workflows', 'slug': 'test', 'name': 'test', 'auto-dependency': True},
+                {
+                    'type': 'comment-templates-categories',
+                    'slug': 'test',
+                    'name': 'test',
+                    'auto-dependency': True,
+                },
             ],
         }
         manifest_fd = io.BytesIO(json.dumps(manifest_json, indent=2).encode())
@@ -833,7 +846,7 @@ def test_deploy_application(app, admin_user, settings, app_bundle, app_bundle_wi
         else:
             assert version.number == '42.0'
         assert version.notes == 'foo bar blah'
-        assert application.elements.count() == 3
+        assert application.elements.count() == 4
         job = AsyncJob.objects.latest('pk')
         assert job.status == 'completed'
         assert job.progression_urls == {'wcs': {'Foobar': 'https://wcs.example.invalid/api/jobs/job-uuid/'}}
@@ -914,10 +927,11 @@ def test_deploy_application(app, admin_user, settings, app_bundle, app_bundle_wi
         resp.form.submit()
         application = Application.objects.get(slug='test')
         elements = application.elements.all().order_by('type')
-        assert len(elements) == 3
+        assert len(elements) == 4
         assert elements[0].cache == {}
-        assert elements[1].cache == form_def
-        assert elements[2].cache == {}
+        assert elements[1].cache == {}
+        assert elements[2].cache == form_def
+        assert elements[3].cache == {}
 
     def response_content(url, request):
         if url.path == '/api/export-import/forms/':

tests/test_cook.py

@@ -9,6 +9,7 @@ from django.contrib.contenttypes.models import ContentType
 from django.core.management import call_command
 from django.core.management.base import CommandError
 
+from hobo.environment import models as environment_models
 from hobo.environment.management.commands.cook import Command
 from hobo.environment.models import (
     Authentic,
@@ -57,8 +58,8 @@ def test_check_action(command, monkeypatch):
     command.server_action = 'mock a server_action handler (ex: hobo-create)'
     action, action_args = 'server-action', {'url': 'https://test.org/'}
 
-    monkeypatch.setattr(ServiceBase, 'is_resolvable', lambda x: True)
-    monkeypatch.setattr(ServiceBase, 'has_valid_certificate', lambda x: True)
+    monkeypatch.setattr(environment_models, 'is_resolvable', lambda x: True)
+    monkeypatch.setattr(environment_models, 'has_valid_certificate', lambda x: True)
     command.check_action(action, action_args)
     assert True
 
@@ -68,8 +69,8 @@ def test_check_action_unknown_action(command, monkeypatch):
     command.server_action = 'mock a server_action handler (ex: hobo-create)'
     action, action_args = 'not-a-server-action', {'url': 'https://test.org/'}
 
-    monkeypatch.setattr(ServiceBase, 'is_resolvable', lambda x: True)
-    monkeypatch.setattr(ServiceBase, 'has_valid_certificate', lambda x: True)
+    monkeypatch.setattr(environment_models, 'is_resolvable', lambda x: True)
+    monkeypatch.setattr(environment_models, 'has_valid_certificate', lambda x: True)
     with pytest.raises(CommandError) as e_info:
         command.check_action(action, action_args)
     assert 'Unknown action not-a-server-action' in str(e_info.value)
@@ -80,8 +81,8 @@ def test_check_action_unresolvable(command, monkeypatch):
     command.server_action = 'mock a server_action handler (ex: hobo-create)'
     action, action_args = 'server-action', {'url': 'https://test.org/'}
 
-    monkeypatch.setattr(ServiceBase, 'is_resolvable', lambda x: False)
-    monkeypatch.setattr(ServiceBase, 'has_valid_certificate', lambda x: True)
+    monkeypatch.setattr(environment_models, 'is_resolvable', lambda x: False)
+    monkeypatch.setattr(environment_models, 'has_valid_certificate', lambda x: True)
     with pytest.raises(CommandError) as e_info:
         command.check_action(action, action_args)
     assert 'test.org is not resolvable in URL https://test.org/' in str(e_info.value)
@@ -92,8 +93,8 @@ def test_check_action_invalid_certificat(command, monkeypatch):
     command.server_action = 'mock a server_action handler (ex: hobo-create)'
     action, action_args = 'server-action', {'url': 'https://test.org/'}
 
-    monkeypatch.setattr(ServiceBase, 'is_resolvable', lambda x: True)
-    monkeypatch.setattr(ServiceBase, 'has_valid_certificate', lambda x: False)
+    monkeypatch.setattr(environment_models, 'is_resolvable', lambda x: True)
+    monkeypatch.setattr(environment_models, 'has_valid_certificate', lambda x: False)
     with pytest.raises(CommandError) as e_info:
         command.check_action(action, action_args)
     assert 'no valid certificate for https://test.org/' in str(e_info.value)
@@ -320,7 +321,7 @@ def test_create_hobo_primary(mocked_TenantMiddleware, mocked_call_command, mocke
     tenant = Mock()
     tenant.schema_name = 'public'
     tenant.get_directory = Mock(return_value='/foo')
-    mocked_connection.get_tenant = Mock(return_value=tenant)
+    mocked_connection.tenant = tenant
     mocked_connection.set_tenant = Mock()
     mocked_TenantMiddleware.get_tenant_by_hostname = Mock(return_value=tenant)
     mocked_call_command.side_effect = CommandError
@@ -348,7 +349,7 @@ def test_create_hobo_not_primary(mocked_TenantMiddleware, mocked_call_command, m
     tenant = Mock()
     tenant.schema_name = 'other than public'
     tenant.get_directory = Mock(return_value='/foo')
-    mocked_connection.get_tenant = Mock(return_value=tenant)
+    mocked_connection.tenant = tenant
     mocked_connection.set_tenant = Mock()
     mocked_TenantMiddleware.get_tenant_by_hostname = Mock(return_value=tenant)
 
@@ -419,21 +420,19 @@ def test_set_idp(command, db):
 @patch('hobo.environment.management.commands.cook.connection')
 @patch('hobo.agent.common.management.commands.hobo_deploy.Command.configure_theme')
 def test_set_theme(mocked_configure_theme, mocked_connection, mocked_set_theme, command):
-    mocked_connection.get_tenant = Mock(return_value='the tenant')
+    mocked_connection.tenant = 'the tenant'
     command.set_theme('the theme')
 
     assert mocked_set_theme.mock_calls == [call('the theme')]
-    assert len(mocked_connection.get_tenant.mock_calls) == 1
     assert mocked_configure_theme.mock_calls == [call({'variables': {'theme': 'the theme'}}, 'the tenant')]
 
 
 @patch('hobo.environment.management.commands.cook.connection')
 def test_cook(mocked_connection, command):
-    mocked_connection.get_tenant = Mock(return_value='the tenant')
+    mocked_connection.tenant = 'the tenant'
     mocked_connection.set_tenant = Mock()
     command.run_cook = Mock()
     command.cook('a-recipe-file.json')
 
-    assert len(mocked_connection.get_tenant.mock_calls) == 1
     assert command.run_cook.mock_calls == [call('a-recipe-file.json')]
     assert mocked_connection.set_tenant.mock_calls == [call('the tenant')]

tests/test_emails.py

@@ -15,18 +15,13 @@ from dns.rdtypes.ANY import MX, TXT
 from test_manager import login
 
 from hobo.emails.validators import validate_email_address
-from hobo.environment.models import Variable, ServiceBase, Combo, Wcs
+from hobo.environment.models import Variable, ServiceBase, Combo
+from hobo.test_utils import find_free_port
 
 
 @pytest.fixture
 def port_available():
-    errno = 0
-    while not errno:
-        port = random.randint(49152, 65534)
-        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        errno = sock.connect_ex(('127.0.0.1', port))
-        sock.close()
-    return port
+    return find_free_port()
 
 
 @pytest.fixture

tests/test_environment.py

@@ -8,6 +8,7 @@ from django.utils import timezone
 from test_manager import login
 from webtest import Upload
 
+from hobo.environment import models as environment_models
 from hobo.environment.models import AVAILABLE_SERVICES, Combo, Passerelle, ServiceBase, Variable
 from hobo.environment.utils import get_installed_services_dict
 from hobo.profile.models import AttributeDefinition
@@ -143,13 +144,13 @@ def test_service_creation_url_validation(app, admin_user, monkeypatch):
     response = form.submit()
     assert 'not resolvable' in response
 
-    monkeypatch.setattr(ServiceBase, 'is_resolvable', lambda x: True)
+    monkeypatch.setattr(environment_models, 'is_resolvable', lambda x: True)
     form = response.form
     response = form.submit()
     assert 'no valid certificate' in response
 
     assert not Combo.objects.exists()
-    monkeypatch.setattr(ServiceBase, 'has_valid_certificate', lambda x: True)
+    monkeypatch.setattr(environment_models, 'has_valid_certificate', lambda x: True)
     form = response.form
     response = form.submit()
     assert Combo.objects.exists()

tests/test_migrations.py

@@ -0,0 +1,94 @@
+# hobo - portal to configure and deploy applications
+# Copyright (C) 2023  Entr'ouvert
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import pytest
+from django.core.management import call_command
+from django.db import connection
+from django.db.migrations.executor import MigrationExecutor
+
+
+@pytest.fixture()
+def migration(request, transactional_db):
+    # see https://gist.github.com/asfaltboy/b3e6f9b5d95af8ba2cc46f2ba6eae5e2
+    """
+    This fixture returns a helper object to test Django data migrations.
+    The fixture returns an object with two methods;
+     - `before` to initialize db to the state before the migration under test
+     - `after` to execute the migration and bring db to the state after the migration
+    The methods return `old_apps` and `new_apps` respectively; these can
+    be used to initiate the ORM models as in the migrations themselves.
+    For example:
+        def test_foo_set_to_bar(migration):
+            old_apps = migration.before([('my_app', '0001_inital')])
+            Foo = old_apps.get_model('my_app', 'foo')
+            Foo.objects.create(bar=False)
+            assert Foo.objects.count() == 1
+            assert Foo.objects.filter(bar=False).count() == Foo.objects.count()
+            # executing migration
+            new_apps = migration.apply([('my_app', '0002_set_foo_bar')])
+            Foo = new_apps.get_model('my_app', 'foo')
+
+            assert Foo.objects.filter(bar=False).count() == 0
+            assert Foo.objects.filter(bar=True).count() == Foo.objects.count()
+    Based on: https://gist.github.com/blueyed/4fb0a807104551f103e6
+    """
+
+    class Migrator:
+        def before(self, targets, at_end=True):
+            """Specify app and starting migration names as in:
+            before([('app', '0001_before')]) => app/migrations/0001_before.py
+            """
+            executor = MigrationExecutor(connection)
+            executor.migrate(targets)
+            executor.loader.build_graph()
+            return executor._create_project_state(with_applied_migrations=True).apps
+
+        def apply(self, targets):
+            """Migrate forwards to the "targets" migration"""
+            executor = MigrationExecutor(connection)
+            executor.migrate(targets)
+            executor.loader.build_graph()
+            return executor._create_project_state(with_applied_migrations=True).apps
+
+    yield Migrator()
+
+    call_command('migrate', verbosity=0)
+
+
+def test_0028_clean_internal_ips_case_delete(transactional_db, migration):
+    old_apps = migration.before([('environment', '0027_allow_long_slug')])
+
+    Variable = old_apps.get_model('environment', 'Variable')
+    var1 = Variable.objects.create(name='SETTING_INTERNAL_IPS', auto=True, value='["1.1.1.1"]')
+    var2 = Variable.objects.create(name='SETTING_INTERNAL_IPS.extend', auto=True, value='["2.2.2.2"]')
+
+    new_apps = migration.apply([('environment', '0028_clean_internal_ips')])
+    Variable = new_apps.get_model('environment', 'Variable')
+    assert Variable.objects.filter(id=var1.id).count() == 0
+    assert Variable.objects.filter(id=var2.id).count() == 1
+
+
+def test_0028_clean_internal_ips_case_rename(transactional_db, migration):
+    old_apps = migration.before([('environment', '0027_allow_long_slug')])
+
+    Variable = old_apps.get_model('environment', 'Variable')
+    var1 = Variable.objects.create(name='SETTING_INTERNAL_IPS', auto=True, value='["1.1.1.1"]')
+
+    new_apps = migration.apply([('environment', '0028_clean_internal_ips')])
+    Variable = new_apps.get_model('environment', 'Variable')
+    assert Variable.objects.filter(id=var1.id).count() == 1
+    var1.refresh_from_db()
+    assert var1.name == 'SETTING_INTERNAL_IPS.extend'

tests/test_theme.py

@@ -19,8 +19,8 @@ def test_theme_view(mocked_random, app, admin_user, fake_themes):
     assert Variable.objects.filter(name='theme')[0].value == 'alfortville'
     assert Variable.objects.filter(name='foo')[0].value == 'bar'
     assert resp.location == '/theme/'
-    assert "The theme has been changed" in dict(resp.headers)['Set-Cookie']
     resp = resp.follow()
+    assert "The theme has been changed" in str(resp.html)
     assert resp.form['theme'].value == 'alfortville'
 
     resp.form['theme'].value = 'publik'

tests_multitenant/conftest.py

@@ -1,5 +1,7 @@
 import pytest
 
+from hobo.multitenant.apps import clear_tenants_settings
+
 
 @pytest.fixture
 def make_tenant(tmp_path, transactional_db, settings, request):
@@ -111,11 +113,13 @@ def make_tenant(tmp_path, transactional_db, settings, request):
 
 @pytest.fixture
 def tenants(make_tenant):
+    clear_tenants_settings()
     return [make_tenant('tenant1.example.net'), make_tenant('tenant2.example.net')]
 
 
 @pytest.fixture
 def tenant(make_tenant):
+    clear_tenants_settings()
     return make_tenant('tenant.example.net')
 
 

tests_multitenant/test_middleware.py

@@ -22,12 +22,12 @@ def test_internalipmiddleware(app, tenants, settings):
     settings.DEBUG_PROPAGATE_EXCEPTIONS = False
     app.get('/?raise', status=404)
     response = app.get('/?raise', status=500, extra_environ={'HTTP_HOST': tenants[0].domain_url})
-    assert response.text == '<h1>Server Error (500)</h1>'
+    assert 'Server Error (500)' in response.text
 
     settings.INTERNAL_IPS = ['127.0.0.1']
 
     response = app.get('/?raise', status=500, extra_environ={'HTTP_HOST': tenants[0].domain_url})
-    assert 'You\'re seeing this error because you have' in response.text
+    assert 'seeing this error because you have' in response.text
 
 
 def test_samesite_middleware(app, tenants, settings):

tests_multitenant/test_settings.py

@@ -39,8 +39,6 @@ def test_tenant_middleware(tenants, client, settings):
 
 
 def test_tenant_json_settings(tenants, settings):
-    settings.clear_tenants_settings()
-
     with utilities.patch_default_settings(
         settings, TENANT_SETTINGS_LOADERS=('hobo.multitenant.settings_loaders.SettingsJSON',)
     ):
@@ -67,8 +65,6 @@ def test_tenant_json_settings(tenants, settings):
 
 
 def test_tenant_template_vars(tenants, settings, client):
-    django.conf.settings.clear_tenants_settings()
-
     with utilities.patch_default_settings(
         settings, TENANT_SETTINGS_LOADERS=('hobo.multitenant.settings_loaders.TemplateVars',)
     ):
@@ -99,8 +95,6 @@ def test_tenant_template_vars(tenants, settings, client):
 
 
 def test_tenant_settings_vars(tenants, settings, client):
-    django.conf.settings.clear_tenants_settings()
-
     with utilities.patch_default_settings(
         settings, TENANT_SETTINGS_LOADERS=('hobo.multitenant.settings_loaders.SettingsVars',)
     ):
@@ -127,8 +121,6 @@ def test_tenant_settings_vars(tenants, settings, client):
 
 
 def test_tenant_cors_settings(tenants, settings, client):
-    settings.clear_tenants_settings()
-
     with utilities.patch_default_settings(
         settings, TENANT_SETTINGS_LOADERS=('hobo.multitenant.settings_loaders.CORSSettings',)
     ):
@@ -148,8 +140,6 @@ def test_tenant_cors_settings(tenants, settings, client):
 
 
 def test_tenant_theme_settings(tenants, settings, client):
-    django.conf.settings.clear_tenants_settings()
-
     with utilities.patch_default_settings(
         settings,
         TENANT_SETTINGS_LOADERS=(
@@ -183,7 +173,6 @@ def test_shared_secret():
 def test_known_services(tenants, settings):
     from hobo.multitenant.settings_loaders import KnownServices
 
-    settings.clear_tenants_settings()
     settings.SETTINGS_MODULE = 'fake.settings'
 
     for tenant in tenants:
@@ -223,8 +212,6 @@ def test_known_services(tenants, settings):
 
 
 def test_legacy_urls_mapping(tenants, settings):
-
-    settings.clear_tenants_settings()
     settings.SETTINGS_MODULE = 'fake.settings'
 
     for tenant in tenants:
@@ -235,8 +222,6 @@ def test_legacy_urls_mapping(tenants, settings):
 
 
 def test_unique_cookies(tenants, settings):
-    settings.clear_tenants_settings()
-
     cookie_names = set()
     for tenant in tenants:
         with tenant_context(tenant):
@@ -247,8 +232,6 @@ def test_unique_cookies(tenants, settings):
 
 
 def test_tenant_json_settings_reload(tenants, settings, freezer):
-    settings.clear_tenants_settings()
-
     with utilities.patch_default_settings(
         settings, TENANT_SETTINGS_LOADERS=('hobo.multitenant.settings_loaders.SettingsJSON',)
     ):

tests_multitenant/test_tenant_command.py

@@ -49,7 +49,6 @@ def test_all_tenants(handle, tenants):
 def test_all_tenants_disable_cron(handle, tenants, settings):
     from django.core.management import execute_from_command_line
 
-    settings.clear_tenants_settings()
     settings.DISABLE_CRON_JOBS = True
     handle.side_effect = RecordTenant()
     execute_from_command_line(['manage.py', 'tenant_command', 'clearsessions', '--all-tenants'])
@@ -61,7 +60,6 @@ def test_all_tenants_disable_cron(handle, tenants, settings):
 def test_all_tenants_disable_cron_for_specific_tenant(handle, tenants, settings):
     from django.core.management import execute_from_command_line
 
-    settings.clear_tenants_settings()
     disabled_tenant = tenants[0]
     with open(os.path.join(disabled_tenant.get_directory(), 'settings.json'), 'w') as fd:
         json.dump(
@@ -78,7 +76,6 @@ def test_all_tenants_disable_cron_for_specific_tenant(handle, tenants, settings)
 def test_all_tenants_global_disable_cron_with_force_job(handle, tenants, settings):
     from django.core.management import execute_from_command_line
 
-    settings.clear_tenants_settings()
     settings.DISABLE_CRON_JOBS = True
     handle.side_effect = RecordTenant()
     execute_from_command_line(

tests_multitenant/test_threading.py

@@ -23,8 +23,6 @@ from tenant_schemas.utils import tenant_context
 
 
 def test_thread(tenants, settings, client):
-
-    settings.clear_tenants_settings()
     with utilities.patch_default_settings(
         settings, TENANT_SETTINGS_LOADERS=('hobo.multitenant.settings_loaders.TemplateVars',)
     ):
@@ -56,7 +54,8 @@ def test_thread(tenants, settings, client):
 
 def test_cache(tenants, client):
     # Clear caches
-    caches._caches.caches = {}
+    for c in caches.all():
+        c.clear()
 
     cache.set('coin', 1)
 
@@ -86,7 +85,6 @@ def test_cache(tenants, client):
 
 
 def test_timer_thread(tenants, settings, client):
-    settings.clear_tenants_settings()
     with utilities.patch_default_settings(
         settings, TENANT_SETTINGS_LOADERS=('hobo.multitenant.settings_loaders.TemplateVars',)
     ):

tests_schemas/test_cook.py

@@ -5,18 +5,18 @@ from django.core.management import call_command, load_command_class
 from django.core.management.base import CommandError
 
 from hobo.deploy.utils import get_hobo_json
-from hobo.environment.models import ServiceBase
+from hobo.environment import models as environment_models
 
 
 def test_cook(db, fake_notify, monkeypatch):
-    monkeypatch.setattr(ServiceBase, 'is_resolvable', lambda x: True)
-    monkeypatch.setattr(ServiceBase, 'has_valid_certificate', lambda x: True)
+    monkeypatch.setattr(environment_models, 'is_resolvable', lambda x: True)
+    monkeypatch.setattr(environment_models, 'has_valid_certificate', lambda x: True)
     call_command('cook', 'tests_schemas/recipe.json')
     assert len(fake_notify) == 3
 
 
 def test_cook_unresolvable(db, fake_notify, monkeypatch):
-    monkeypatch.setattr(ServiceBase, 'is_resolvable', lambda x: False)
+    monkeypatch.setattr(environment_models, 'is_resolvable', lambda x: False)
     with pytest.raises(CommandError) as e_info:
         call_command('cook', 'tests_schemas/recipe.json')
     assert 'is not resolvable' in str(e_info.value)
@@ -26,8 +26,8 @@ def test_cook_example(db, fake_notify, monkeypatch, fake_themes):
     """hobo/cook (before rabbitmq) scenario having templates.
     the resulting JSON may be helpfull to manually invoque hobo-deploy (after rabbitmq)
     """
-    monkeypatch.setattr(ServiceBase, 'is_resolvable', lambda x: True)
-    monkeypatch.setattr(ServiceBase, 'has_valid_certificate', lambda x: True)
+    monkeypatch.setattr(environment_models, 'is_resolvable', lambda x: True)
+    monkeypatch.setattr(environment_models, 'has_valid_certificate', lambda x: True)
     call_command('cook', 'tests_schemas/example_recipe.json')
 
     # notify_agents was call

tests_schemas/test_rename_hobo_service.py

@@ -5,15 +5,15 @@ from django.core.management import call_command
 from django.core.management.base import CommandError
 from tenant_schemas.utils import tenant_context
 
-from hobo.environment.models import Passerelle, ServiceBase
+from hobo.environment import models as environment_models
 from hobo.environment.utils import get_installed_services, get_or_create_local_hobo
 from hobo.multitenant.middleware import TenantMiddleware, TenantNotFound
 
 
 @pytest.fixture()
 def hobo_tenant(db, fake_notify, monkeypatch, fake_themes):
-    monkeypatch.setattr(ServiceBase, 'is_resolvable', lambda x: True)
-    monkeypatch.setattr(ServiceBase, 'has_valid_certificate', lambda x: True)
+    monkeypatch.setattr(environment_models, 'is_resolvable', lambda x: True)
+    monkeypatch.setattr(environment_models, 'has_valid_certificate', lambda x: True)
     yield call_command('cook', 'tests_schemas/example_recipe.json')
     call_command('delete_tenant', 'hobo-instance-name.dev.signalpublik.com')
 
@@ -34,8 +34,8 @@ def test_unknown_service(hobo_tenant):
 
 
 def test_rename_hobo_service_succes(db, fake_notify, monkeypatch, fake_themes):
-    monkeypatch.setattr(ServiceBase, 'is_resolvable', lambda x: True)
-    monkeypatch.setattr(ServiceBase, 'has_valid_certificate', lambda x: True)
+    monkeypatch.setattr(environment_models, 'is_resolvable', lambda x: True)
+    monkeypatch.setattr(environment_models, 'has_valid_certificate', lambda x: True)
     call_command('cook', 'tests_schemas/example_recipe.json')
     assert TenantMiddleware.get_tenant_by_hostname('hobo-instance-name.dev.signalpublik.com')
     call_command(

tests_schemas/test_rename_service.py

@@ -5,15 +5,15 @@ from django.core.management import call_command
 from django.core.management.base import CommandError
 from tenant_schemas.utils import tenant_context
 
-from hobo.environment.models import Passerelle, ServiceBase
+from hobo.environment import models as environment_models
 from hobo.environment.utils import get_installed_services
 from hobo.multitenant.middleware import TenantMiddleware
 
 
 @pytest.fixture()
 def hobo_tenant(db, fake_notify, monkeypatch, fake_themes):
-    monkeypatch.setattr(ServiceBase, 'is_resolvable', lambda x: True)
-    monkeypatch.setattr(ServiceBase, 'has_valid_certificate', lambda x: True)
+    monkeypatch.setattr(environment_models, 'is_resolvable', lambda x: True)
+    monkeypatch.setattr(environment_models, 'has_valid_certificate', lambda x: True)
     yield call_command('cook', 'tests_schemas/example_recipe.json')
     call_command('delete_tenant', 'hobo-instance-name.dev.signalpublik.com')
 
@@ -35,7 +35,7 @@ def test_secondary_service(hobo_tenant):
     tenant = TenantMiddleware.get_tenant_by_hostname('hobo-instance-name.dev.signalpublik.com')
     with tenant_context(tenant):
         assert get_installed_services()
-        Passerelle.objects.create(
+        environment_models.Passerelle.objects.create(
             title='other passerelle',
             slug='other-passerelle',
             base_url='https://other-passerelle-instance-name.dev.signalpublik.com',
@@ -56,8 +56,8 @@ def test_secondary_service(hobo_tenant):
 def test_rename_service_succes(hobo_tenant, monkeypatch):
     tenant = TenantMiddleware.get_tenant_by_hostname('hobo-instance-name.dev.signalpublik.com')
     with tenant_context(tenant):
-        assert Passerelle.objects.count() == 1
-        passerelle_service = Passerelle.objects.first()
+        assert environment_models.Passerelle.objects.count() == 1
+        passerelle_service = environment_models.Passerelle.objects.first()
         assert (
             passerelle_service.get_base_url_path() == 'https://passerelle-instance-name.dev.signalpublik.com/'
         )
@@ -69,8 +69,8 @@ def test_rename_service_succes(hobo_tenant, monkeypatch):
             'https://passerelle-instance-name.dev.signalpublik.com/',
             'https://new-passerelle-instance-name.dev.signalpublik.com/',
         )
-        assert Passerelle.objects.count() == 1
-        passerelle_service = Passerelle.objects.first()
+        assert environment_models.Passerelle.objects.count() == 1
+        passerelle_service = environment_models.Passerelle.objects.first()
         assert (
             passerelle_service.get_base_url_path()
             == 'https://new-passerelle-instance-name.dev.signalpublik.com/'

tox.ini

@@ -3,21 +3,26 @@
 # test suite on all supported python versions. To use it, "pip install tox"
 # and then run "tox" from this directory.
 [tox]
+min_version = 4
 toxworkdir = {env:TMPDIR:/tmp}/tox-{env:USER}/hobo/{env:BRANCH_NAME:}
 envlist =
-  py3-django22-drf39-{hobo,authentic,multipublik,multitenant,schemas,passerelle}
-  py3-django22-drf312-coverage-{hobo,authentic,multipublik,multitenant,schemas,passerelle}
-  py3-black
+  py3-django22-{hobo,authentic,multipublik,multitenant,schemas,passerelle}
+  py3-django32-{hobo,authentic,multipublik,multitenant,schemas,passerelle}
+  code-style
 
 [testenv]
 usedevelop = True
+parallel_show_output = True
 setenv =
+# necessary to separate coverage binary output between parallel environments
+	COVERAGE_FILE=.coverage.{envname}
 	DISABLE_GLOBAL_HANDLERS=1
 	BRANCH_NAME={env:BRANCH_NAME:}
-	COVERAGE_FILE={envdir}/coverage
 	DB_ENGINE=django.db.backends.postgresql_psycopg2
 	SETUPTOOLS_USE_DISTUTILS=stdlib
-        JUNIT=--junitxml=junit-{envname}.xml
+	JUNIT=--junitxml=junit-{envname}.xml
+	COVERAGE=--cov=hobo/ --cov-config .coveragerc --cov-report= --cov-branch
+
 	hobo: DJANGO_SETTINGS_MODULE=hobo.settings
 	hobo: HOBO_SETTINGS_FILE=tests/settings.py
 	schemas: DJANGO_SETTINGS_MODULE=hobo.settings
@@ -32,12 +37,20 @@ setenv =
 	passerelle: DEBIAN_CONFIG_COMMON=debian/debian_config_common.py
 	passerelle: PASSERELLE_SETTINGS_FILE=tests_passerelle/settings.py
 	passerelle: DJANGO_SETTINGS_MODULE=passerelle.settings
-	coverage: COVERAGE=--cov-report xml:coverage-{envname}.xml --cov-report html:htmlcov-{envname} --cov=hobo/ --cov-config .coveragerc 
-	fast: NOMIGRATIONS=--nomigrations
+# test directories
+	hobo: TEST_DIRECTORY=tests/
+	schemas: TEST_DIRECTORY=tests_schemas/
+	multitenant: TEST_DIRECTORY=tests_multitenant/
+	multipublik: TEST_DIRECTORY=tests_multipublik/
+	authentic: TEST_DIRECTORY=tests_authentic/
+	passerelle: TEST_DIRECTORY=tests_passerelle/
 deps:
-	drf39: djangorestframework>=3.9.2,<3.10
-	drf312: djangorestframework>=3.12,<3.13
 	django22: django>=2.2,<2.3
+        django22: psycopg2-binary<2.9
+        django22: psycopg2<2.9
+        django32: django>=3.2,<3.3
+        django32: psycopg2-binary
+        django32: psycopg
 	pytest!=6.0.0
 	pytest-cov
 	pytest-django
@@ -50,7 +63,6 @@ deps:
 	django-mellon
 	django-webtest
 	Markdown<3
-	django-tables2<2.0
 	authentic: https://git.entrouvert.org/authentic.git/snapshot/authentic-main.tar.gz
 	passerelle: https://git.entrouvert.org/passerelle.git/snapshot/passerelle-main.tar.gz
 	passerelle: python-memcached
@@ -69,21 +81,36 @@ deps:
     Pillow
 	black: pre-commit
 	git+https://git.entrouvert.org/publik-django-templatetags.git
+allowlist_externals =
+	./getlasso3.sh
 commands =
 	./getlasso3.sh
-	hobo: py.test {env:JUNIT:} {env:COVERAGE:} {env:NOMIGRATIONS:} {posargs:tests/}
-	schemas: py.test {env:JUNIT:} {env:COVERAGE:} {env:NOMIGRATIONS:} {posargs:tests_schemas/}
-	multitenant: py.test {env:JUNIT:} {env:COVERAGE:} {env:NOMIGRATIONS:} {posargs:tests_multitenant/}
-	multipublik: py.test {env:JUNIT:} {env:COVERAGE:} {env:NOMIGRATIONS:} {posargs:tests_multipublik/}
-	authentic: py.test {env:JUNIT:} {env:FAST:} {env:COVERAGE:} {env:NOMIGRATIONS:} {posargs:tests_authentic/}
-	passerelle: py.test {env:JUNIT:} {env:COVERAGE:} {env:NOMIGRATIONS:} {posargs:tests_passerelle/}
-	black: pre-commit run black --all-files --show-diff-on-failure
+	py.test {posargs:{tty::{env:JUNIT:}} {tty::{env:COVERAGE:}} {env:TEST_DIRECTORY:}}
+
+[testenv:code-style]
+basepython = python3
+skip_install = true
+deps =
+  pre-commit
+commands =
+  pre-commit run --all-files --show-diff-on-failure
 
 [testenv:update-locales]
 setenv =
   DJANGO_SETTINGS_MODULE=hobo.settings
-deps =
+allowlist_externals =
+	./getlasso3.sh
 commands =
     ./getlasso3.sh
     ./manage.py makemessages --add-location=file
     ./manage.py compilemessages
+
+[testenv:coverage-report]
+setenv =
+deps =
+	coverage
+commands =
+	python3 -m coverage erase --data-file=.coverage
+	python3 -m coverage combine
+	python3 -m coverage html -d htmlcov --show-contexts
+	python3 -m coverage xml