toulouse-maelis: add functional tests (#71647)

functests/toulouse_maelis/conftest.py~

@@ -0,0 +1,9 @@
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption("--cmis-connector-url", help="Url of a passerelle CMIS connector instance")
+    parser.addoption("--cmis-endpoint", help="Url of a passerelle CMIS endpoint")
+    parser.addoption("--cmis-username", help="Username for the CMIS endpoint")
+    parser.addoption("--cmis-password", help="Password for the CMIS endpoint")
+    parser.addoption("--preserve-tree", action="store_true", default=False, help="Preserve test directory")

passerelle/apps/esabora/models.py

@@ -72,7 +72,7 @@ class Esabora(BaseResource, HTTPResource):
         url = urllib.parse.urljoin(self.service_url, path)
         headers = {'Authorization': f'Bearer {self.api_key}'}
         try:
-            response = self.requests.post(url, json=payload, headers=headers, timeout=5, **kwargs)
+            return self.requests.post(url, json=payload, headers=headers, timeout=5, **kwargs)
         except requests.RequestException as e:
             raise APIError(
                 'Esabora platform "%s" connection error: %s' % (self.service_url, exception_to_text(e)),
@@ -83,26 +83,6 @@ class Esabora(BaseResource, HTTPResource):
                     'error': str(e),
                 },
             )
-        try:
-            data = response.json()
-        except requests.JSONDecodeError as e:
-            raise APIError(
-                'Esabora platform "%s" invalid JSON response: %s' % (self.service_url, exception_to_text(e)),
-                log_error=True,
-                data={
-                    'status_code': response.status_code,
-                },
-            )
-        if not response.ok:
-            raise APIError(
-                'Esabora platform "%s" answered with HTTP error' % (self.service_url),
-                log_error=True,
-                data={
-                    'status_code': response.status_code,
-                    'content': data,
-                },
-            )
-        return data
 
     @endpoint(
         name='do-search',
@@ -120,7 +100,9 @@ class Esabora(BaseResource, HTTPResource):
                 for name, value in post_data['criterions'].items()
             ],
         }
-        data = self.post('mult/', payload, params={'task': 'doSearch'})
+        response = self.post('mult/', payload, params={'task': 'doSearch'})
+        response.raise_for_status()
+        data = response.json()
         columns = {slugify(c).replace('-', '_'): c for c in data['columnList']}
         keys = {slugify(c).replace('-', '_'): c for c in data['keyList']}
         cleaned_data = {
@@ -148,7 +130,9 @@ class Esabora(BaseResource, HTTPResource):
         endpoint = post_data.pop('endpoint', None) or 'modbdd'
         payload = get_treatment_payload(post_data)
 
-        data = self.post(f'{endpoint}/', payload, params={'task': 'doTreatment'})
+        response = self.post(f'{endpoint}/', payload, params={'task': 'doTreatment'})
+        response.raise_for_status()
+        data = response.json()
         keys = [slugify(c).replace('-', '_') for c in data['keyList']]
         cleaned_data = esabora_row_to_object([], keys, data)
         cleaned_data['action'] = data['action']

passerelle/apps/franceconnect_data/fc.py

@@ -60,10 +60,9 @@ def base64url_decode(input):
 
 
 class FranceConnect:
-    def __init__(self, session, logger, dgfip_api_base_url):
+    def __init__(self, session, logger):
         self.session = session
         self.logger = logger
-        self.dgfip_api_base_url = dgfip_api_base_url
         self.items = []
         self.correlation_id = str(uuid.uuid4())
 
@@ -136,7 +135,7 @@ class FranceConnect:
         dgfip_response = self.request(
             'dgfip token endpoint',
             'POST',
-            'token',
+            'https://gwfc.impots.gouv.fr/token',
             data=data,
             auth=(dgfip_username, dgfip_password),
         )
@@ -163,7 +162,7 @@ class FranceConnect:
             dgfip_ressource_ir_response = self.request(
                 'ressource IR endpoint',
                 'GET',
-                'impotparticulier/1.0/situations/ir/assiettes/annrev/%s' % annrev,
+                'https://gwfc.impots.gouv.fr/impotparticulier/1.0/situations/ir/assiettes/annrev/%s' % annrev,
                 headers=headers,
             )
         except FranceConnectError as e:
@@ -186,9 +185,8 @@ class FranceConnect:
     def add(self, key, value):
         self.items.append((key, value))
 
-    def request(self, label, method, endpoint, *args, **kwargs):
+    def request(self, label, method, url, *args, **kwargs):
         self.logger.debug('request %s %s args:%s kwargs:%s', label, method, args, kwargs)
-        url = urllib.parse.urljoin(self.dgfip_api_base_url, endpoint)
         self.add(label.replace(' ', '_') + '_request', [method, url, args, kwargs])
         try:
             response = getattr(self.session, method.lower())(url, *args, **kwargs)

passerelle/apps/franceconnect_data/migrations/0001_initial.py

@@ -48,19 +48,23 @@ class Migration(migrations.Migration):
                 ),
                 (
                     'dgfip_username',
-                    models.CharField(blank=True, max_length=64, null=True, verbose_name='DGFIP API Username'),
+                    models.CharField(
+                        blank=True, max_length=64, null=True, verbose_name='api.impots.gouv.fr username'
+                    ),
                 ),
                 (
                     'dgfip_password',
-                    models.CharField(blank=True, max_length=64, null=True, verbose_name='DGFIP API Password'),
+                    models.CharField(
+                        blank=True, max_length=64, null=True, verbose_name='api.impots.gouv.fr password'
+                    ),
                 ),
                 (
                     'dgfip_scopes',
-                    models.TextField(blank=True, null=True, verbose_name='DGFIP API Scopes'),
+                    models.TextField(blank=True, null=True, verbose_name='api.impots.gouv.fr scopes'),
                 ),
                 (
                     'dgfip_id_teleservice',
-                    models.TextField(blank=True, null=True, verbose_name='DGFIP API ID_Teleservice'),
+                    models.TextField(blank=True, null=True, verbose_name='api.impots.gouv.fr ID_Teleservice'),
                 ),
                 (
                     'users',

passerelle/apps/franceconnect_data/migrations/0003_auto_20221124_1158.py

@@ -1,22 +0,0 @@
-# Generated by Django 2.2.24 on 2022-11-24 10:58
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('franceconnect_data', '0002_token'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='resource',
-            name='dgfip_api_base_url',
-            field=models.URLField(
-                default='https://gwfc.dgfip.finances.gouv.fr/',
-                max_length=256,
-                verbose_name='DGFIP API base URL',
-            ),
-        ),
-    ]

passerelle/apps/franceconnect_data/models.py

@@ -62,17 +62,13 @@ class Resource(BaseResource):
         ),
     )
 
-    dgfip_api_base_url = models.URLField(
-        _('DGFIP API base URL'), max_length=256, default='https://gwfc.dgfip.finances.gouv.fr/'
-    )
+    dgfip_username = models.CharField(_('api.impots.gouv.fr username'), max_length=64, blank=True, null=True)
 
-    dgfip_username = models.CharField(_('DGFIP API Username'), max_length=64, blank=True, null=True)
+    dgfip_password = models.CharField(_('api.impots.gouv.fr password'), max_length=64, blank=True, null=True)
 
-    dgfip_password = models.CharField(_('DGFIP API Password'), max_length=64, blank=True, null=True)
+    dgfip_scopes = models.TextField(_('api.impots.gouv.fr scopes'), blank=True, null=True)
 
-    dgfip_scopes = models.TextField(_('DGFIP API Scopes'), blank=True, null=True)
-
-    dgfip_id_teleservice = models.TextField(_('DGFIP API ID_Teleservice'), blank=True, null=True)
+    dgfip_id_teleservice = models.TextField(_('api.impots.gouv.fr ID_Teleservice'), blank=True, null=True)
 
     log_requests_errors = False
 
@@ -125,9 +121,7 @@ class Resource(BaseResource):
             return HttpResponseBadRequest('Missing or invalid origin')
 
         redirect_uri = self.build_callback_url(request, origin=origin, mode=mode, test=test)
-        franceconnect = fc.FranceConnect(
-            session=self.requests, logger=self.logger, dgfip_api_base_url=self.dgfip_api_base_url
-        )
+        franceconnect = fc.FranceConnect(session=self.requests, logger=self.logger)
         return HttpResponseRedirect(
             franceconnect.authorization_request(
                 platform=self.fc_platform,
@@ -158,9 +152,7 @@ class Resource(BaseResource):
         if test and not request.user.is_superuser:
             return HttpResponseBadRequest('Only admin can use test mode.')
 
-        franceconnect = fc.FranceConnect(
-            session=self.requests, logger=self.logger, dgfip_api_base_url=self.dgfip_api_base_url
-        )
+        franceconnect = fc.FranceConnect(session=self.requests, logger=self.logger)
         redirect_uri = self.build_callback_url(request, origin=origin, mode=mode, test=test)
         context = {
             'origin': origin,
@@ -230,13 +222,11 @@ class Resource(BaseResource):
     )
     def data_source(self, request, id=None, test=None, mode=None, **kwargs):
         if id:
-            token = self.retrieve(id)
-            if token:
-                return {
-                    'data': [
-                        dict(token, id=id),
-                    ]
-                }
+            return {
+                'data': [
+                    dict(self.retrieve(id), id=id),
+                ]
+            }
         url = request.build_absolute_uri(
             reverse(
                 'generic-endpoint',
@@ -272,10 +262,6 @@ class Resource(BaseResource):
         return token.guid.hex
 
     def retrieve(self, ref):
-        try:
-            ref = uuid.UUID(str(ref))
-        except ValueError:
-            return None
         token = Token.objects.filter(guid=ref).first()
         return token and token.content
 

passerelle/contrib/grandlyon_streetsections/models.py

@@ -131,7 +131,6 @@ class GrandLyonStreetSections(BaseResource):
             )
             for attribute in ('nom', 'nomcommune', 'domanialite', 'codeinsee'):
                 setattr(section, attribute, value.get(attribute) or '')
-            section.nomcommune = normalize_commune(section.nomcommune)
             for attribute in ('bornemindroite', 'bornemingauche', 'bornemaxdroite', 'bornemaxgauche', 'gid'):
                 if value.get(attribute) in (None, 'None'):
                     # data.grandlyon returned 'None' as a string at a time

passerelle/contrib/tcl/models.py

@@ -131,10 +131,6 @@ class Tcl(BaseResource):
                 response = self.requests.get(url)
                 response.raise_for_status()
                 for line_data in response.json()['values']:
-                    if 'code_trace' in line_data:
-                        line_data['code_titan'] = line_data['code_trace']
-                    if 'nom_trace' in line_data:
-                        line_data['libelle'] = line_data['nom_trace']
                     line, dummy = Line.objects.get_or_create(
                         code_titan=line_data['code_titan'],
                         defaults={'transport_key': key, 'ligne': line_data['ligne']},

passerelle/locale/fr/LC_MESSAGES/django.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: Passerelle 0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2022-11-28 15:13+0100\n"
+"POT-Creation-Date: 2022-11-03 16:06-0500\n"
 "PO-Revision-Date: 2022-11-03 22:06+0100\n"
 "Last-Translator: Frederic Peters <fpeters@entrouvert.com>\n"
 "Language: fr\n"
@@ -2390,25 +2390,20 @@ msgid "FranceConnect scopes"
 msgstr "Domaines (« scopes ») pour FranceConnect"
 
 #: apps/franceconnect_data/models.py
-msgid "DGFIP API base URL"
-msgstr "URL de base de l’API"
+msgid "api.impots.gouv.fr username"
+msgstr "identifiant pour api.impots.gouv.fr"
 
 #: apps/franceconnect_data/models.py
-msgid "DGFIP API Username"
-msgstr "Identifiant pour l’API DGFIP"
+msgid "api.impots.gouv.fr password"
+msgstr "mot de passe pour api.impots.gouv.fr"
 
 #: apps/franceconnect_data/models.py
-
-msgid "DGFIP API Password"
-msgstr "Mot de passe pour l’API DGFIP"
+msgid "api.impots.gouv.fr scopes"
+msgstr "Domaines (« scopes ») pour api.impots.gouv.fr"
 
 #: apps/franceconnect_data/models.py
-msgid "DGFIP API Scopes"
-msgstr "Scopes de l’API DGFIP"
-
-#: apps/franceconnect_data/models.py
-msgid "DGFIP API ID_Teleservice"
-msgstr "ID_Teleservice pour l’API DGFIP"
+msgid "api.impots.gouv.fr ID_Teleservice"
+msgstr "ID_Teleservice pour api.impots.gouv.fr"
 
 #: apps/franceconnect_data/models.py
 msgid "Data sources through FranceConnect"

tests/test_esabora.py

@@ -245,31 +245,3 @@ def test_do_treatment_arbitrary_endpoint(app, connector):
     assert responses.calls[0].request.params['task'] == 'doTreatment'
     response_data = json.loads(responses.calls[0].request.body)
     assert response_data == expected_payload
-
-
-@responses.activate
-def test_post_raises_proper_error(app, connector):
-    url = tests.utils.generic_endpoint_url('esabora', 'do-treatment')
-    responses.add(
-        responses.POST,
-        f'{connector.service_url}addevt/',
-        json={'foo': 'bar'},
-        status=400,
-    )
-
-    payload = {
-        'endpoint': 'addevt',
-        'treatment_name': 'Import Event',
-        'Adresse_Latitude': 12.3,
-        'Adresse_Ville': 'Marseille',
-    }
-
-    expected_response = {
-        'err': 1,
-        'err_class': 'passerelle.utils.jsonresponse.APIError',
-        'data': {'content': {'foo': 'bar'}, 'status_code': 400},
-        'err_desc': 'Esabora platform "http://example.esabora/ws/rest/" answered with HTTP error',
-    }
-
-    response = app.post_json(url, params=payload)
-    assert response.json == expected_response

tests/test_franceconnect_data.py

@@ -18,52 +18,11 @@ import json
 from urllib.parse import parse_qs, urlparse, urlunparse
 
 import pytest
-from django.utils.timezone import now
 
 import tests.utils
 from passerelle.apps.franceconnect_data.models import Resource
 from tests.test_rsa13 import mock_response
 
-CURRENT_YEAR = now().year
-
-
-USER_INFO_MOCKED_RESPONSES = [
-    ['/api/v1/token', {'access_token': 'at-1234', 'id_token': '.e30=.'}],
-    [
-        '/api/v1/userinfo',
-        {
-            'sub': 'sub-1234',
-            'given_name': 'John',
-            'family_name': 'Doe',
-            'birthdate': '2001-04-28',
-            'birthplace': '13055',
-            'birthcountry': '99100',
-            'gender': 'male',
-        },
-    ],
-]
-
-
-DGFIP_MOCKED_RESPONSES = USER_INFO_MOCKED_RESPONSES + [
-    [
-        '/token',
-        {
-            'access_token': 'eyJ4NXQiOi',
-            'expires_in': 3600,
-            'scope': 'RessourceIRDerniere2',
-            'token_type': 'Bearer',
-        },
-    ]
-]
-
-DGFIP_MOCKED_RESPONSES += [
-    [
-        '/impotparticulier/1.0/situations/ir/assiettes/annrev/%s' % year,
-        {'rfr': 0, 'revenuBrutGlobal': 0},
-    ]
-    for year in range(CURRENT_YEAR - 3, CURRENT_YEAR)
-]
-
 
 @pytest.fixture
 def fc(db):
@@ -95,7 +54,21 @@ def test_init_request(app, fc):
     }
 
 
-@mock_response(*USER_INFO_MOCKED_RESPONSES)
+@mock_response(
+    ['/api/v1/token', {'access_token': 'at-1234', 'id_token': '.e30=.'}],
+    [
+        '/api/v1/userinfo',
+        {
+            'sub': 'sub-1234',
+            'given_name': 'John',
+            'family_name': 'Doe',
+            'birthdate': '2001-04-28',
+            'birthplace': '13055',
+            'birthcountry': '99100',
+            'gender': 'male',
+        },
+    ],
+)
 def test_callback(app, fc):
     resp = app.get(
         'http://testserver/franceconnect-data/test/callback?origin=http%3A%2F%2Ftestserver&code=5678&raise=1'
@@ -122,8 +95,6 @@ def test_callback(app, fc):
         'text': 'John Doe né le April 28, 2001',
     }
 
-    app.get('/franceconnect-data/test/data_source?id=bad')  # no error
-
 
 @mock_response(
     ['/api/v1/token', ''],
@@ -148,20 +119,3 @@ def test_callback_error(app, fc):
     error = json.loads(resp.pyquery('#error').text())
     assert error
     assert 'Error in token endpoint response' in resp
-
-
-@mock_response(*DGFIP_MOCKED_RESPONSES)
-def test_dgfip_mode(app, fc):
-    resp = app.get(
-        'http://testserver/franceconnect-data/test/callback?origin=http%3A%2F%2Ftestserver&code=5678&raise=1&mode=dgfip'
-    )
-    data = json.loads(resp.pyquery('#data').text())
-    assert data
-    assert 'id' in data
-    assert data['text'] == 'John Doe né le April 28, 2001'
-
-    resp = app.get('/franceconnect-data/test/data_source?mode=dgfip&id=' + data['id'])
-    data = resp.json['data'][0]
-    assert data['dgfip_ir']
-    for year in range(CURRENT_YEAR - 3, CURRENT_YEAR):
-        assert data['dgfip_ir'][str(year)]

tests/test_grandlyon_streetsections.py

@@ -209,33 +209,6 @@ DATA_EXAMPLE = """{
          "senscircspecialise" : null,
          "senscirculation" : "Double",
          "typecirculation" : "générale"
-      },
-      {
-         "bornemaxdroite" : 10,
-         "bornemaxgauche" : 11,
-         "bornemindroite" : 2,
-         "bornemingauche" : 1,
-         "codefuv" : "21424",
-         "codeinsee" : "69266",
-         "codetroncon" : "T54753",
-         "datecreation" : "1995-07-17 18:43:00+02:00",
-         "datedomanialite" : null,
-         "datemajalpha" : "2022-11-14",
-         "datemajborne" : "2022-07-26",
-         "datemajgraph" : "2022-07-26",
-         "denomroutiere" : null,
-         "domanialite" : "Métropole",
-         "gestionnaire" : null,
-         "gid" : 30067,
-         "importance" : "Petite rue",
-         "nom" : "Rue de l'Avenir",
-         "nomcommune" : "Villeurbanne",
-         "observation" : null,
-         "particularite" : null,
-         "referencedomanialite" : null,
-         "senscircspecialise" : null,
-         "senscirculation" : "Inverse",
-         "typecirculation" : "Générale"
       }
    ]
 }
@@ -265,15 +238,3 @@ def test_daily_none_bornes(app, connector):
         connector.daily()
         assert StreetSection.objects.get(codetroncon='T5869').bornemindroite == 0
         assert StreetSection.objects.get(codetroncon='T5869').bornemaxdroite == 99999
-
-
-def test_non_uppercase_communes(app, connector):
-    StreetSection.objects.all().delete()
-    with HTTMock(data_mock):
-        connector.daily()
-        response = app.get(
-            "/grandlyon-streetsections/gl-streetsections/section_info"
-            "?streetname=Rue de l'Avenir&commune=Villeurbanne&streetnumber=8"
-        )
-        assert response.json['err'] == 0
-        assert response.json['data']['codetroncon'] == 'T54753'

tests/test_tcl.py

@@ -10,12 +10,16 @@ from passerelle.contrib.tcl.models import Line, Stop, Tcl
 LIGNE_BUS = {
     "values": [
         {
+            "indice": "",
             "last_update_fme": "2017-06-27 06:01:10",
-            "nom_trace": "Croix Rousse - Plateaux de St Rambert",
+            "infos": "",
+            "couleur": "164 203 38",
+            "libelle": "Croix Rousse - Plateaux de St Rambert",
             "last_update": "None",
-            "code_trace": "2Aa1",
+            "code_titan": "2Aa1",
             "gid": "1003",
             "ligne": "2",
+            "ut": "UTV",
             "sens": "Aller",
         },
     ]

tox.ini

@@ -41,7 +41,6 @@ deps =
   vobject
   django-ratelimit
   bleach
-  phonenumbers
   dnspython
   pyquery
   responses