tests/api/test_carddef.py

@@ -784,6 +784,80 @@ def test_cards_import_json(pub, local_user, auth):
     assert resp.json['data']['creation_time'] <= resp.json['data']['completion_time']
 
 
+def test_cards_import_json_update(pub, local_user):
+    pub.role_class.wipe()
+    role = pub.role_class(name='test')
+    role.store()
+    local_user.roles = [role.id]
+    local_user.store()
+
+    ApiAccess.wipe()
+    access = ApiAccess()
+    access.name = 'test'
+    access.access_identifier = 'test'
+    access.access_key = '12345'
+    access.store()
+
+    app = get_app(pub)
+
+    access.roles = [role]
+    access.store()
+
+    def put_url(url, *args, **kwargs):
+        app.set_authorization(('Basic', ('test', '12345')))
+        return app.put_json(url, *args, **kwargs)
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'test'
+    carddef.fields = [
+        fields.StringField(id='0', label='foobar', varname='foo'),
+        fields.StringField(id='1', label='foobar2', varname='foo2'),
+    ]
+    carddef.workflow_roles = {'_viewer': role.id}
+    carddef.backoffice_submission_roles = [role.id]
+    carddef.digest_templates = {'default': 'bla {{ form_var_foo }} xxx'}
+    carddef.store()
+
+    carddef.data_class().wipe()
+
+    carddata = carddef.data_class()()
+    carddata.data = {'0': 'foo', '1': 'bar'}
+    carddata.just_created()
+    carddata.store()
+
+    carddata2 = carddef.data_class()()
+    carddata2.data = {'0': 'foo2', '1': 'bar2'}
+    carddata2.just_created()
+    carddata2.store()
+
+    data = {
+        'data': [
+            {
+                'uuid': carddata.uuid,
+                'fields': {
+                    'foo': 'first entry',
+                    'foo2': 'plop',
+                },
+            },
+            {
+                'fields': {
+                    'foo': 'second entry',
+                    'foo2': 'plop',
+                }
+            },
+        ]
+    }
+    resp = put_url(
+        '/api/cards/test/import-json?update=on',
+        data,
+        headers={'content-type': 'application/json'},
+    )
+    assert resp.json == {'err': 0}
+    assert carddef.data_class().count() == 3
+    assert {x.data['0'] for x in carddef.data_class().select()} == {'first entry', 'second entry', 'foo2'}
+
+
 def test_cards_restricted_api(pub, local_user):
     pub.role_class.wipe()
     role = pub.role_class(name='test')

tests/backoffice_pages/test_carddata.py

@@ -2,6 +2,7 @@ import base64
 import datetime
 import json
 import os
+import urllib.parse
 import uuid
 
 import pytest
@@ -12,10 +13,11 @@ from wcs.blocks import BlockDef
 from wcs.carddef import CardDef
 from wcs.categories import CardDefCategory
 from wcs.formdef import FormDef
+from wcs.qommon.afterjobs import AfterJob
 from wcs.qommon.http_request import HTTPRequest
 from wcs.wf.create_formdata import Mapping
 from wcs.wf.form import WorkflowFormFieldsFormDef
-from wcs.workflows import Workflow, WorkflowBackofficeFieldsFormDef
+from wcs.workflows import ContentSnapshotPart, Workflow, WorkflowBackofficeFieldsFormDef
 
 from ..utilities import clean_temporary_pub, create_temporary_pub, get_app, login
 from .test_all import create_superuser, create_user
@@ -925,7 +927,7 @@ def test_backoffice_cards_import_status_from_json(pub):
                 },
                 'workflow': {
                     'status': {
-                        'id': 'wf-%s' % st2.id,
+                        'id': str(st2.id),
                     }
                 },
             }
@@ -988,7 +990,7 @@ def test_backoffice_cards_import_backoffice_fields_from_json(pub):
     assert '/backoffice/processing?job=' in resp.location
 
     carddata = carddef.data_class().select()[0]
-    assert carddata.status == 'recorded'
+    assert carddata.status == 'wf-recorded'
     assert carddata.data == {'1': 'a string', 'bo1': 'foo'}
 
 
@@ -1034,6 +1036,127 @@ def test_backoffice_cards_import_user_from_json(pub):
     assert str(carddata.user_id) == str(user2.id)
 
 
+def test_backoffice_cards_update_data_from_json(pub):
+    user = create_user(pub)
+
+    workflow = CardDef.get_default_workflow()
+    workflow.id = None
+    workflow.add_status('status2', 'st2')
+    workflow.store()
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'test'
+    carddef.fields = [
+        fields.StringField(id='1', label='Test', varname='string'),
+    ]
+    carddef.workflow_roles = {'_editor': user.roles[0]}
+    carddef.workflow_id = workflow.id
+    carddef.backoffice_submission_roles = user.roles
+    carddef.store()
+    carddef.data_class().wipe()
+
+    card = carddef.data_class()()
+    card.data = {'1': 'plop'}
+    card.just_created()
+    card.store()
+
+    app = login(get_app(pub))
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Export Data')
+    resp.form['format'] = 'json'
+    resp = resp.form.submit('submit')
+    job_id = urllib.parse.parse_qs(urllib.parse.urlparse(resp.location).query)['job'][0]
+    job = AfterJob.get(job_id)
+    json_export = json.loads(job.file_content)
+    assert len(json_export['data']) == 1
+    json_export['data'][0]['fields']['string'] = 'plop 2'
+
+    # update
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a file')
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(json_export).encode(), 'application/json')
+    resp.form['update_existing_cards'].checked = True
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+    resp = resp.follow()
+
+    assert carddef.data_class().count() == 1
+    card.refresh_from_storage()
+    assert card.data == {'1': 'plop 2'}
+    assert isinstance(card.evolution[0].parts[-1], ContentSnapshotPart)
+    assert card.evolution[0].parts[-1].old_data == {'1': 'plop'}
+    assert card.evolution[0].parts[-1].new_data == {'1': 'plop 2'}
+
+    # no uuid -> create
+    json_export['data'][0]['uuid'] = None
+    json_export['data'][0]['fields']['string'] = 'plop 3'
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a file')
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(json_export).encode(), 'application/json')
+    resp.form['update_existing_cards'].checked = True
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+    resp = resp.follow()
+    assert carddef.data_class().count() == 2
+
+    # uuid -> create but keep uuid
+    json_export['data'][0]['uuid'] = str(uuid.uuid4())
+    json_export['data'][0]['fields']['string'] = 'plop 4'
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a file')
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(json_export).encode(), 'application/json')
+    resp.form['update_existing_cards'].checked = True
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+    resp = resp.follow()
+    assert carddef.data_class().count() == 3
+    assert carddef.data_class().get_by_uuid(json_export['data'][0]['uuid']).data == {'1': 'plop 4'}
+
+    # invalid uuid -> ignore
+    json_export['data'][0]['uuid'] = 'hello world'
+    json_export['data'][0]['fields']['string'] = 'plop 5'
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a file')
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(json_export).encode(), 'application/json')
+    resp.form['update_existing_cards'].checked = True
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+    resp = resp.follow()
+    assert carddef.data_class().count() == 4
+
+    # ask not to update
+    json_export['data'][0]['uuid'] = str(card.uuid)
+    json_export['data'][0]['fields']['string'] = 'plop 6'
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a file')
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(json_export).encode(), 'application/json')
+    resp.form['update_existing_cards'].checked = False
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+    resp = resp.follow()
+    assert carddef.data_class().count() == 5
+    assert len({x.uuid for x in carddef.data_class().select()}) == 5  # all unique UUIDs
+    assert carddef.data_class().get_by_uuid(card.uuid).data == {'1': 'plop 2'}
+
+    # update and change status
+    json_export['data'][0]['uuid'] = str(card.uuid)
+    del json_export['data'][0]['workflow']['real_status']
+    json_export['data'][0]['workflow']['real_status'] = {'id': 'st2', 'name': 'status2'}
+    json_export['data'][0]['fields']['string'] = 'plop 7'
+    resp = app.get(carddef.get_url())
+    resp = resp.click('Import data from a file')
+    resp.forms[0]['file'] = Upload('test.json', json.dumps(json_export).encode(), 'application/json')
+    resp.form['update_existing_cards'].checked = True
+    resp = resp.forms[0].submit()
+    assert '/backoffice/processing?job=' in resp.location
+    resp = resp.follow()
+    assert carddef.data_class().count() == 5
+    card.refresh_from_storage()
+    assert card.status == 'wf-st2'
+    assert [x.status for x in card.evolution] == ['wf-recorded', 'wf-st2']
+
+
 def test_backoffice_cards_wscall_failure_display(http_requests, pub):
     user = create_user(pub)
 

tests/test_carddef.py

@@ -72,6 +72,8 @@ def test_basics(pub):
     assert carddata_class.get(carddata.id).data['1'] == 'hello world'
     assert carddata_class.get(carddata.id).status == 'wf-recorded'
 
+    assert carddata_class.get(carddata.id).uuid
+
 
 def test_advertised_urls(pub):
     CardDef.wipe()

tests/test_sql.py

@@ -2273,6 +2273,38 @@ def test_migration_82_statistics_data(formdef):
     cur.close()
 
 
+def test_migration_86_card_uuid(pub):
+    CardDef.wipe()
+
+    carddef = CardDef()
+    carddef.name = 'tests'
+    carddef.fields = [fields.StringField(id='0', label='string')]
+    carddef.store()
+
+    carddata = carddef.data_class()()
+    carddata.data['0'] = 'blah'
+    carddata.store()
+
+    assert carddef.data_class().get(carddata.id).uuid
+
+    conn, cur = sql.get_connection_and_cursor()
+    cur.execute('UPDATE wcs_meta SET value = 85 WHERE key = %s', ('sql_level',))
+
+    # drop uuid column
+    cur.execute('ALTER TABLE %s DROP COLUMN uuid' % sql.get_formdef_table_name(carddef))
+    assert not column_exists_in_table(cur, sql.get_formdef_table_name(carddef), 'uuid')
+
+    sql.migrate()
+
+    assert column_exists_in_table(cur, sql.get_formdef_table_name(carddef), 'uuid')
+    assert migration_level(cur) >= 86
+
+    assert carddef.data_class().get(carddata.id).uuid
+
+    conn.commit()
+    cur.close()
+
+
 def test_logged_error_store_without_integrity_error(pub, sql_queries):
     sql.LoggedError.record('there was an error')
 

wcs/api.py

@@ -401,13 +401,16 @@ class ApiCardPage(ApiFormPageMixin, BackofficeCardPage):
             raise AccessForbiddenError('cannot import cards')
 
         afterjob = bool(get_request().form.get('async') == 'on')
+        do_update = bool(get_request().form.get('update') == 'on')
         get_response().set_content_type('application/json')
         try:
             if file_format == 'csv':
                 content = get_request().stdin.read()
                 job = self.import_csv_submit(content, afterjob=afterjob, api=True)
             elif file_format == 'json':
-                job = self.import_json_submit(get_request().json, afterjob=afterjob, api=True)
+                job = self.import_json_submit(
+                    get_request().json, update_existing_cards=do_update, afterjob=afterjob, api=True
+                )
         except ValueError as e:
             return json.dumps({'err': 1, 'err_desc': str(e)})
         if job is None:

wcs/backoffice/data_management.py

@@ -19,6 +19,7 @@ import csv
 import datetime
 import io
 import json
+import uuid
 
 from quixote import get_publisher, get_request, get_response, redirect
 from quixote.html import htmltext
@@ -26,11 +27,12 @@ from quixote.html import htmltext
 from wcs import fields
 from wcs.carddef import CardDef
 from wcs.categories import CardDefCategory
+from wcs.workflows import ContentSnapshotPart
 
 from ..qommon import _, errors, ngettext, template
 from ..qommon.afterjobs import AfterJob
 from ..qommon.backoffice.menu import html_top
-from ..qommon.form import FileWidget, Form
+from ..qommon.form import CheckboxWidget, FileWidget, Form
 from .management import FormBackOfficeStatusPage, FormPage, ManagementDirectory
 from .submission import FormFillPage
 
@@ -217,6 +219,12 @@ class CardPage(FormPage):
 
         form = Form(enctype='multipart/form-data', use_tokens=False)
         form.add(FileWidget, 'file', title=_('File'), required=True)
+        form.add(
+            CheckboxWidget,
+            'update_existing_cards',
+            title=_('Update existing cards (only for JSON imports)'),
+            value=False,
+        )
         form.add_submit('submit', _('Submit'))
         form.add_submit('cancel', _('Cancel'))
         if form.get_widget('cancel').parse():
@@ -234,7 +242,9 @@ class CardPage(FormPage):
                     form.set_error('file', e)
             else:
                 try:
-                    return self.import_json_submit(json_content)
+                    return self.import_json_submit(
+                        json_content, update_existing_cards=form.get_widget('update_existing_cards').parse()
+                    )
                 except ValueError as e:
                     form.set_error('file', e)
 
@@ -320,12 +330,14 @@ class CardPage(FormPage):
         else:
             job.execute()
 
-    def import_json_submit(self, json_content, afterjob=True, api=False):
+    def import_json_submit(self, json_content, *, update_existing_cards=False, afterjob=True, api=False):
         # basic check, looks like valid json card content?
         if not isinstance(json_content, dict) or 'data' not in json_content:
             raise ValueError(_('Invalid JSON file'))
 
-        job = ImportFromJsonAfterJob(carddef=self.formdef, json_content=json_content)
+        job = ImportFromJsonAfterJob(
+            carddef=self.formdef, json_content=json_content, update_existing_cards=update_existing_cards
+        )
         if afterjob:
             get_response().add_after_job(job)
             if api:
@@ -485,12 +497,13 @@ class ImportFromCsvAfterJob(AfterJob):
 
 
 class ImportFromJsonAfterJob(AfterJob):
-    def __init__(self, carddef, json_content):
+    def __init__(self, carddef, json_content, update_existing_cards):
         super().__init__(
             label=_('Importing data into cards'),
             carddef_class=carddef.__class__,
             carddef_id=carddef.id,
             json_content=json_content,
+            update_existing_cards=update_existing_cards,
         )
 
     @property
@@ -499,37 +512,75 @@ class ImportFromJsonAfterJob(AfterJob):
 
     def execute(self):
         json_content = self.kwargs['json_content']
+        update_existing_cards = self.kwargs['update_existing_cards']
         from wcs.api import posted_json_data_to_formdata_data
 
         for json_data in json_content['data']:
             json_data = copy.deepcopy(json_data)
             carddata = self.carddef.data_class()()
-            carddata.data = posted_json_data_to_formdata_data(self.carddef, json_data['fields'])
+            carddata.data = {}
+            if update_existing_cards:
+                if json_data.get('uuid'):
+                    try:
+                        normalized_uuid = str(uuid.UUID(json_data.get('uuid')))
+                    except ValueError:
+                        pass  # ignore invalid uuid
+                    else:
+                        try:
+                            carddata = self.carddef.data_class().get_by_uuid(normalized_uuid)
+                            orig_data = copy.copy(carddata.data)
+                        except KeyError:
+                            # create with provided uuid
+                            carddata.uuid = normalized_uuid
+
+            # load fields
+            carddata.data.update(posted_json_data_to_formdata_data(self.carddef, json_data['fields']))
+
+            # load backoffice fields if any
+            if 'fields' in (json_data.get('workflow') or {}):
+                backoffice_data_dict = posted_json_data_to_formdata_data(
+                    self.carddef, json_data['workflow']['fields']
+                )
+                carddata.data.update(backoffice_data_dict)
+
+            # set user if any
             if 'user' in json_data:
                 carddata.set_user_from_json(json_data['user'])
 
-            carddata.just_created()
+            try:
+                card_status = json_data['workflow'].get('real_status') or json_data['workflow'].get('status')
+                # check it has a valid 'id' key
+                card_status_id = self.carddef.workflow.get_status(card_status['id']).id
+            except KeyError:
+                card_status = None
+                card_status_id = None
 
-            if 'workflow' not in json_data:
-                # perform as new
-                carddata.store()
+            if carddata.id is None:
+                # no id, this is a new card
+                carddata.just_created()
 
-                get_publisher().reset_formdata_state()
-                get_publisher().substitutions.feed(carddata)
-                carddata.record_workflow_event('json-import-created')
-                carddata.perform_workflow()
+                if not card_status_id:
+                    # perform as new
+                    carddata.store()
+
+                    get_publisher().reset_formdata_state()
+                    get_publisher().substitutions.feed(carddata)
+                    carddata.record_workflow_event('json-import-created')
+                    carddata.perform_workflow()
+                else:
+                    # set to status specified in json
+                    carddata.status = f'wf-{card_status_id}'
+                    carddata.evolution[-1].status = carddata.status
+                    carddata.store()
+                    carddata.record_workflow_event('json-import-created')
             else:
-                if 'fields' in json_data['workflow']:
-                    backoffice_data_dict = posted_json_data_to_formdata_data(
-                        self.carddef, json_data['workflow']['fields']
-                    )
-                    carddata.data.update(backoffice_data_dict)
-
-                status = json_data['workflow'].get('real_status') or json_data['workflow'].get('status')
-                carddata.status = status.get('id')
-                carddata.evolution[-1].status = status
+                # update data of existing card
+                ContentSnapshotPart.take(formdata=carddata, old_data=orig_data)
+                carddata.record_workflow_event('json-import-updated')
                 carddata.store()
-                carddata.record_workflow_event('json-import-created')
+                if card_status and carddata.status != f'wf-{card_status_id}':
+                    # switch status (but do not execute)
+                    carddata.jump_status(card_status_id)
 
             self.increment_count()
 

wcs/carddata.py

@@ -19,9 +19,12 @@ from quixote import get_publisher, get_request
 from wcs.formdata import FormData
 
 from .qommon import _
+from .qommon.storage import Equal
 
 
 class CardData(FormData):
+    uuid = None
+
     def get_formdef(self):
         if self._formdef:
             return self._formdef
@@ -101,3 +104,10 @@ class CardData(FormData):
     @classmethod
     def get_submission_channels(cls):
         return {'web': _('Web'), 'file-import': _('File Import')}
+
+    @classmethod
+    def get_by_uuid(cls, value):
+        try:
+            return cls.select([Equal('uuid', value)], limit=1)[0]
+        except IndexError:
+            raise KeyError(value)

wcs/formdata.py

@@ -1393,6 +1393,8 @@ class FormData(StorableObject):
         # noqa pylint: disable=too-many-arguments
         data = {}
         data['id'] = str(self.id)
+        if hasattr(self, 'uuid'):
+            data['uuid'] = self.uuid
         data['display_id'] = self.get_display_id()
         data['display_name'] = self.get_display_name()
         data['digests'] = self.digests

wcs/sql.py

@@ -26,6 +26,7 @@ import re
 import secrets
 import shutil
 import time
+import uuid
 
 import psycopg2
 import psycopg2.extensions
@@ -2339,6 +2340,8 @@ class SqlDataMixin(SqlMixin):
                     sql_dict['workflow_roles_array'].append(str(x))
         else:
             sql_dict['workflow_roles_array'] = None
+        if hasattr(self, 'uuid'):
+            sql_dict['uuid'] = self.uuid
         for attr in ('workflow_data', 'workflow_roles', 'submission_context', 'prefilling_data'):
             if getattr(self, attr):
                 sql_dict[attr] = bytearray(pickle.dumps(getattr(self, attr), protocol=2))
@@ -2739,7 +2742,14 @@ class SqlFormData(SqlDataMixin, wcs.formdata.FormData):
 
 
 class SqlCardData(SqlDataMixin, wcs.carddata.CardData):
-    pass
+    _table_static_fields = SqlDataMixin._table_static_fields + [
+        ('uuid', 'uuid UNIQUE NOT NULL DEFAULT gen_random_uuid()')
+    ]
+
+    def store(self, *args, **kwargs):
+        if self.uuid is None:
+            self.uuid = str(uuid.uuid4())
+        return super().store(*args, **kwargs)
 
 
 class SqlUser(SqlMixin, wcs.users.User):
@@ -4929,7 +4939,7 @@ def get_period_total(
 # latest migration, number + description (description is not used
 # programmaticaly but will make sure git conflicts if two migrations are
 # separately added with the same number)
-SQL_LEVEL = (85, 'remove anonymous column from users table')
+SQL_LEVEL = (86, 'add uuid to cards')
 
 
 def migrate_global_views(conn, cur):
@@ -5198,7 +5208,6 @@ def migrate():
         #     combined formdef and formdata value.
         # 69: add auto_geoloc field to form/card tables
         # 80: add jsonb column to hold statistics data
-
         for formdef in FormDef.select() + CardDef.select():
             do_formdef_tables(formdef, rebuild_views=False, rebuild_global_views=False)
         migrate_views(conn, cur)
@@ -5236,6 +5245,11 @@ def migrate():
         if os.path.exists(nonces_dir):
             shutil.rmtree(nonces_dir, ignore_errors=True)
 
+    if sql_level < 86:
+        # 86: add uuid to cards
+        for formdef in CardDef.select():
+            do_formdef_tables(formdef, rebuild_views=False, rebuild_global_views=False)
+
     if sql_level != SQL_LEVEL[0]:
         cur.execute(
             '''UPDATE wcs_meta SET value = %s, updated_at=NOW() WHERE key = %s''',

wcs/workflow_traces.py

@@ -61,6 +61,7 @@ class WorkflowTrace(sql.WorkflowTrace):
             'global-interactive-action': _('Global action (interactive)'),
             'global-external-workflow': _('Trigger by external workflow'),
             'json-import-created': _('Created (by JSON import)'),
+            'json-import-updated': _('Updated (by JSON import)'),
             'timeout-jump': _('Timeout jump'),
             'workflow-created': _('Created (by workflow action)'),
             'workflow-edited': _('Edited (by workflow action)'),