cmis: use requests wrapper in cmislib (#73771) #56

Merged
bdauvergne merged 1 commits from wip/73771-cmis-utiliser-le-wrapper-request into main 2023-01-30 10:37:38 +01:00
4 changed files with 127 additions and 197 deletions

View File

@ -1,17 +0,0 @@
# passerelle - uniform access to multiple data sources and services
# Copyright (C) 2021 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
default_app_config = 'passerelle.apps.cmis.apps.CmisAppConfig'

View File

@ -1,62 +0,0 @@
# passerelle - uniform access to multiple data sources and services
# Copyright (C) 2021 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.apps import AppConfig
def add_logging_to_cmislib():
'''Monkeypatch cmislib request module to log requests and responses.'''
from cmislib.atompub import binding
from cmislib.net import RESTService as CMISRESTService
class RESTService(CMISRESTService):
def get(self, url, *args, **kwargs):
logger = kwargs.pop('passerelle_logger')
logger.debug('cmislib GET request to %s', url)
resp, content = super().get(url, *args, **kwargs)
logger.debug('cmislib GET response (%s)', resp['status'], extra={'response': content.decode()})
return resp, content
def delete(self, url, *args, **kwargs):
logger = kwargs.pop('passerelle_logger')
logger.debug('cmislib DELETE request to %s', url)
resp, content = super().delete(url, *args, **kwargs)
logger.debug('cmislib DELETE response (%s)', resp['status'], extra={'response': content.decode()})
return resp, content
def post(self, url, payload, *args, **kwargs):
logger = kwargs.pop('passerelle_logger')
logger.debug('cmislib POST request to %s', url, extra={'payload': payload.decode()})
resp, content = super().post(url, payload, *args, **kwargs)
logger.debug('cmislib POST response (%s)', resp['status'], extra={'response': content.decode()})
return resp, content
def put(self, url, payload, *args, **kwargs):
logger = kwargs.pop('passerelle_logger')
logger.debug('cmislib PUT request to %s', url, payload, extra={'payload': payload.decode()})
resp, content = super().put(url, *args, **kwargs)
logger.debug('cmislib PUT response (%s)', resp['status'], extra={'response': content.decode()})
return resp, content
binding.Rest = RESTService
class CmisAppConfig(AppConfig):
name = 'passerelle.apps.cmis'
def ready(self):
add_logging_to_cmislib()

View File

@ -34,6 +34,7 @@ from cmislib.exceptions import (
from django.db import models
from django.http import HttpResponse
from django.utils.functional import cached_property
from django.utils.http import urlencode
from django.utils.translation import gettext_lazy as _
from passerelle.base.models import BaseResource
@ -147,7 +148,14 @@ class CmisConnector(BaseResource):
@contextmanager
def get_cmis_gateway(self):
with ignore_loggers('cmislib', 'cmislib.atompub.binding'):
yield CMISGateway(self.cmis_endpoint, self.username, self.password, self.logger)
import cmislib.atompub.binding as atompub_binding
old_Rest = atompub_binding.Rest
atompub_binding.Rest = lambda: RESTService(self)
try:
yield CMISGateway(self.cmis_endpoint, self.username, self.password, self.logger)
finally:
atompub_binding.Rest = old_Rest
def _validate_inputs(self, data):
"""process dict
@ -236,7 +244,7 @@ def wrap_cmis_error(f):
class CMISGateway:
def __init__(self, cmis_endpoint, username, password, logger):
self._cmis_client = CmisClient(cmis_endpoint, username, password, passerelle_logger=logger)
self._cmis_client = CmisClient(cmis_endpoint, username, password)
self._logger = logger
@cached_property
@ -284,3 +292,41 @@ class CMISGateway:
@wrap_cmis_error
def get_object(self, object_id):
return self.repo.getObject(object_id)
# Mock API from cmilib.net.RESTService
class RESTService:
def __init__(self, resource):
self.resource = resource
def request(self, method, url, username, password, body=None, content_type=None, **kwargs):
if username or password:
auth = (username, password)
else:
auth = None
headers = kwargs.pop('headers', {})
if kwargs:
url = url + ('&' if '?' in url else '?') + urlencode(kwargs)
if content_type:
headers['Content-Type'] = content_type
response = self.resource.requests.request(
method=method, url=url, auth=auth, headers=headers, data=body
)
return {'status': str(response.status_code)}, response.content
def get(self, url, username=None, password=None, **kwargs):
return self.request('GET', url, username, password, **kwargs)
def delete(self, url, username=None, password=None, **kwargs):
return self.request('DELETE', url, username, password, **kwargs)
def put(self, url, payload, contentType, username=None, password=None, **kwargs):
return self.request('PUT', url, username, password, body=payload, content_type=contentType, **kwargs)
def post(self, url, payload, contentType, username=None, password=None, **kwargs):
return self.request('POST', url, username, password, body=payload, content_type=contentType, **kwargs)

View File

@ -1,14 +1,28 @@
# passerelle - uniform access to multiple data sources and services
# Copyright (C) 2023 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import base64
import os
import re
import xml.etree.ElementTree as ET
from unittest import mock
from unittest.mock import Mock, call
from urllib import error as urllib2
import httplib2
import py
import pytest
import responses
from cmislib import CmisClient
from cmislib.exceptions import (
CmisException,
@ -22,7 +36,7 @@ from django.urls import reverse
from django.utils.encoding import force_bytes, force_str
from passerelle.apps.cmis.models import CmisConnector
from passerelle.base.models import AccessRight, ApiUser, ResourceLog
from passerelle.base.models import AccessRight, ApiUser
from tests.test_manager import login
@ -392,9 +406,6 @@ def test_create_doc():
@pytest.mark.parametrize(
"cmis_exc,err_msg",
[
(httplib2.HttpLib2Error, "connection error"),
# FIXME used for cmslib 0.5 compat
(urllib2.URLError, "connection error"),
(PermissionDeniedException, "permission denied"),
(UpdateConflictException, "update conflict"),
(InvalidArgumentException, "invalid property"),
@ -509,8 +520,8 @@ def test_cmis_types_view(setup, app, admin_user, monkeypatch):
@pytest.mark.parametrize('debug', (False, True))
@mock.patch('httplib2.Http.request')
def test_raw_uploadfile(mocked_request, app, setup, debug, caplog):
@responses.activate
def test_raw_uploadfile(app, setup, debug, caplog):
""" Simulate the bellow bash query :
$ http https://passerelle.dev.publik.love/cmis/ged/uploadfile \
file:='{"filename": "test2", "content": "c2FsdXQK"}' path=/test-eo
@ -525,46 +536,21 @@ def test_raw_uploadfile(mocked_request, app, setup, debug, caplog):
if debug:
setup.set_log_level('DEBUG')
def cmis_mocked_request(uri, method="GET", body=None, **kwargs):
"""simulate the 3 (ordered) HTTP queries involved"""
response = {'status': '200'}
if method == 'GET' and uri == 'http://example.com/cmisatom':
with open('%s/tests/data/cmis/cmis1.out.xml' % os.getcwd(), 'rb') as fd:
content = fd.read()
elif method == 'GET' and uri == (
'http://example.com/cmisatom/test/path?path=/test-eo&filter=&includeAllowableActions=false&includeACL=false&'
'includePolicyIds=false&includeRelationships=&renditionFilter='
):
with open('%s/tests/data/cmis/cmis2.out.xml' % os.getcwd(), 'rb') as fd:
content = fd.read()
elif method == 'POST' and uri == 'http://example.com/cmisatom/test/children?id=L3Rlc3QtZW8%3D':
with open('%s/tests/data/cmis/cmis3.in.xml' % os.getcwd()) as fd:
expected_input = fd.read()
expected_input = expected_input.replace('\n', '')
expected_input = re.sub('> *<', '><', expected_input)
input1 = ET.tostring(ET.XML(expected_input))
with open('%s/tests/data/cmis/cmis1.out.xml' % os.getcwd(), 'rb') as fd:
cmis1_body = fd.read()
# reorder properties
input2 = ET.XML(body)
objects = input2.find('{http://docs.oasis-open.org/ns/cmis/restatom/200908/}object')
properties = objects.find('{http://docs.oasis-open.org/ns/cmis/core/200908/}properties')
data = []
for elem in properties:
key = elem.tag
data.append((key, elem))
data.sort()
properties[:] = [item[-1] for item in data]
input2 = ET.tostring(input2)
with open('%s/tests/data/cmis/cmis2.out.xml' % os.getcwd(), 'rb') as fd:
cmis2_body = fd.read()
if input1 != input2:
raise Exception('expect [[%s]] but get [[%s]]' % (body, expected_input))
with open('%s/tests/data/cmis/cmis3.out.xml' % os.getcwd(), 'rb') as fd:
content = fd.read()
else:
raise Exception('my fault error, url is not yet mocked: %s' % uri)
return (response, content)
with open('%s/tests/data/cmis/cmis3.out.xml' % os.getcwd(), 'rb') as fd:
cmis3_body = fd.read()
responses.add(responses.GET, 'http://example.com/cmisatom', body=cmis1_body, status=200)
responses.add(responses.GET, 'http://example.com/cmisatom/test/path', body=cmis2_body, status=200)
responses.add(responses.POST, 'http://example.com/cmisatom/test/children', body=cmis3_body, status=200)
mocked_request.side_effect = cmis_mocked_request
params = {
"path": path,
"file": {"filename": file_name, "content": b64encode(file_content), "content_type": "image/jpeg"},
@ -575,22 +561,6 @@ def test_raw_uploadfile(mocked_request, app, setup, debug, caplog):
assert json_result['data']['properties']['cmis:objectTypeId'] == "cmis:document"
assert json_result['data']['properties']['cmis:name'] == file_name
if not debug:
assert ResourceLog.objects.count() == 2
else:
assert ResourceLog.objects.count() == 11
logs = list(ResourceLog.objects.all())
assert logs[3].message == 'cmislib GET request to http://example.com/cmisatom'
assert logs[4].message == 'cmislib GET response (200)'
assert logs[4].extra['response'].startswith('<?xml')
assert (
logs[8].message
== 'cmislib POST request to http://example.com/cmisatom/test/children?id=L3Rlc3QtZW8%3D'
)
assert logs[8].extra['payload'].startswith('<?xml')
assert logs[9].message == 'cmislib POST response (200)'
assert logs[9].extra['response'].startswith('<?xml')
assert not any('cmislib' in record.name for record in caplog.records)
@ -606,73 +576,66 @@ def test_cmis_check_status(app, setup, monkeypatch):
setup.check_status()
@mock.patch('httplib2.Http.request')
def test_get_file(mocked_request, app, setup):
url = reverse('generic-endpoint', kwargs={'connector': 'cmis', 'endpoint': 'getfile', 'slug': setup.slug})
@responses.activate
def test_get_file(app, setup):
url = (
reverse('generic-endpoint', kwargs={'connector': 'cmis', 'endpoint': 'getfile', 'slug': setup.slug})
+ '?raise=1'
)
def cmis_mocked_request(uri, method="GET", body=None, **kwargs):
"""simulate the HTTP queries involved"""
response = {'status': '200'}
if method == 'GET' and uri == 'http://example.com/cmisatom':
with open('%s/tests/data/cmis/cmis1.out.xml' % os.getcwd(), 'rb') as fd:
content = fd.read()
elif method == 'GET' and (
uri.startswith('http://example.com/cmisatom/test/path?path=/test/file')
or uri.startswith(
'http://example.com/cmisatom/test/id?id=c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'
)
):
with open('%s/tests/data/cmis/cmis3.out.xml' % os.getcwd(), 'rb') as fd:
content = fd.read()
elif (
method == 'GET'
and uri == 'http://example.com/cmisatom/test/content/test2?id=L3Rlc3QtZW8vdGVzdDI%3D'
):
content = b'hello world'
else:
raise Exception('url is not yet mocked: %s' % uri)
return (response, content)
with open('%s/tests/data/cmis/cmis1.out.xml' % os.getcwd(), 'rb') as fd:
cmis1_body = fd.read()
mocked_request.side_effect = cmis_mocked_request
response = app.get(url, params={'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'})
assert response.content_type == 'application/octet-stream'
assert response.content == b'hello world'
with open('%s/tests/data/cmis/cmis3.out.xml' % os.getcwd(), 'rb') as fd:
cmis3_body = fd.read()
responses.add(responses.GET, 'http://example.com/cmisatom', body=cmis1_body, status=200)
responses.add(responses.GET, 'http://example.com/cmisatom/test/path', body=cmis3_body, status=200)
responses.add(responses.GET, 'http://example.com/cmisatom/test/id', body=cmis3_body, status=200)
responses.add(
responses.GET,
'http://example.com/cmisatom/test/content/test2?id=L3Rlc3QtZW8vdGVzdDI%3D',
body=b'hello world',
status=200,
)
response = app.get(url, params={'object_id': '/test/file'})
assert response.content_type == 'application/octet-stream'
assert response.content == b'hello world'
response = app.get(url, params={'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'})
assert response.content_type == 'application/octet-stream'
assert response.content == b'hello world'
@mock.patch('httplib2.Http.request')
def test_get_metadata(mocked_request, app, setup):
@responses.activate
def test_get_metadata(app, setup):
url = reverse(
'generic-endpoint', kwargs={'connector': 'cmis', 'endpoint': 'getmetadata', 'slug': setup.slug}
)
def cmis_mocked_request(uri, method="GET", body=None, **kwargs):
"""simulate the HTTP queries involved"""
response = {'status': '200'}
if method == 'GET' and uri == 'http://example.com/cmisatom':
with open('%s/tests/data/cmis/cmis1.out.xml' % os.getcwd(), 'rb') as fd:
content = fd.read()
elif method == 'GET' and (
uri.startswith('http://example.com/cmisatom/test/path?path=/test/file')
or uri.startswith(
'http://example.com/cmisatom/test/id?id=c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'
)
):
with open('%s/tests/data/cmis/cmis3.out.xml' % os.getcwd(), 'rb') as fd:
content = fd.read()
elif (
method == 'GET'
and uri == 'http://example.com/cmisatom/test/content/test2?id=L3Rlc3QtZW8vdGVzdDI%3D'
):
content = b'hello world'
else:
raise Exception('url is not yet mocked: %s' % uri)
return (response, content)
with open('%s/tests/data/cmis/cmis1.out.xml' % os.getcwd(), 'rb') as fd:
cmis1_body = fd.read()
with open('%s/tests/data/cmis/cmis3.out.xml' % os.getcwd(), 'rb') as fd:
cmis3_body = fd.read()
responses.add(responses.GET, 'http://example.com/cmisatom', body=cmis1_body, status=200)
responses.add(responses.GET, 'http://example.com/cmisatom/test/path', body=cmis3_body, status=200)
responses.add(responses.GET, 'http://example.com/cmisatom/test/id', body=cmis3_body, status=200)
responses.add(
responses.GET,
'http://example.com/cmisatom/test/content/test2?id=L3Rlc3QtZW8vdGVzdDI%3D',
body=b'hello world',
status=200,
)
mocked_request.side_effect = cmis_mocked_request
response = app.get(url, params={'object_id': 'c4bc9d00-5bf0-404d-8f0a-a6260f6d21ae;1.0'})
assert response.json['data']['cmis']['contentStreamFileName'] == 'test2'
assert response.json['data']['rsj']['idInsertis'] == '21N284563'