tests/admin_pages/test_card.py

@@ -388,6 +388,56 @@ def test_card_templates(pub):
     assert 'Existing cards will be updated in the background.' not in resp.text
 
 
+def test_card_id_template(pub):
+    create_superuser(pub)
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'foo'
+    carddef.fields = [
+        fields.StringField(id='1', label='Test', varname='test'),
+    ]
+    carddef.store()
+    carddata = carddef.data_class()()
+    carddata.data = {'1': 'bar'}
+    carddata.just_created()
+    carddata.store()
+
+    app = login(get_app(pub))
+    resp = app.get('/backoffice/cards/1/')
+    resp = resp.click('Templates')
+    assert 'id_template' not in resp.text
+
+    if not pub.site_options.has_section('options'):
+        pub.site_options.add_section('options')
+    pub.site_options.set('options', 'enable-card-identifier-template', 'true')
+    with open(os.path.join(pub.app_dir, 'site-options.cfg'), 'w') as fd:
+        pub.site_options.write(fd)
+
+    resp = app.get('/backoffice/cards/1/')
+    resp = resp.click('Templates')
+    assert 'Identifier cannot be modified if there are existing cards.' in resp.text
+
+    carddef.data_class().wipe()
+
+    resp = app.get('/backoffice/cards/1/')
+    resp = resp.click('Templates')
+    assert 'Identifier cannot be modified if there are existing cards.' not in resp.text
+    resp.form['id_template'] = 'X{{form_var_test}}Y'
+    resp = resp.form.submit()
+    assert resp.location == 'http://example.net/backoffice/cards/1/'
+    resp = resp.follow()
+    assert_option_display(resp, 'Templates', 'Custom')
+    carddef = CardDef.get(carddef.id)
+    assert carddef.id_template == 'X{{form_var_test}}Y'
+
+    carddata = carddef.data_class()()
+    carddata.data = {'1': 'bar'}
+    carddata.just_created()
+    carddata.store()
+    assert carddata.id_display == 'XbarY'
+
+
 def test_card_digest_template(pub):
     create_superuser(pub)
 

tests/admin_pages/test_form.py

@@ -1344,6 +1344,7 @@ def test_form_templates(pub):
     assert_option_display(resp, 'Templates', 'None')
     assert resp.pyquery('[href="options/templates"]').attr.rel == ''  # no popup
     resp = resp.click('Templates')
+    assert 'id_template' not in resp.form.fields
     resp.form['digest_template'] = 'X{{form_var_test}}Y'
     resp = resp.form.submit()
     assert resp.location == 'http://example.net/backoffice/forms/1/'

tests/backoffice_pages/test_custom_view.py

@@ -1983,3 +1983,82 @@ def test_backoffice_custom_view_and_snapshots(pub, formdef_class):
         resp = resp.form.submit().follow()
         assert pub.custom_view_class.count() == 0
         assert pub.snapshot_class.count() == 7  # datasource custom view, store formdef
+
+
+def test_item_filter_card_custom_identifier(pub):
+    pub.role_class.wipe()
+    pub.custom_view_class.wipe()
+    CardDef.wipe()
+    FormDef.wipe()
+
+    user = create_superuser(pub)
+    role = pub.role_class(name='test')
+    role.store()
+    user.roles = [role.id]
+    user.store()
+
+    carddef = CardDef()
+    carddef.name = 'items'
+    carddef.digest_templates = {'default': '{{form_var_name}}'}
+    carddef.id_template = '{{form_var_custom_id}}'
+    carddef.fields = [
+        fields.StringField(id='0', label='string', varname='name'),
+        fields.StringField(id='1', label='string', varname='custom_id'),
+    ]
+    carddef.store()
+    carddef.data_class().wipe()
+    for i, value in enumerate(['foo', 'bar', 'baz']):
+        carddata = carddef.data_class()()
+        carddata.data = {
+            '0': value,
+            '1': 'attr%s' % i,
+        }
+        carddata.just_created()
+        carddata.store()
+
+    formdef = FormDef()
+    formdef.name = 'form title'
+    formdef.fields = [
+        fields.ItemField(
+            id='1',
+            label='string',
+            data_source={'type': 'carddef:%s' % carddef.url_name},
+            display_locations=['validation', 'summary', 'listings'],
+        )
+    ]
+    formdef.store()
+    formdef.workflow_roles = {'_receiver': role.id}
+    formdef.store()
+
+    # create one formdata with attr1, and two with attr2
+    for i in range(2):
+        for _ in range(i + 1):
+            formdata = formdef.data_class()()
+            formdata.data = {'1': 'attr%s' % (i + 1)}
+            formdata.data['1_display'] = formdef.fields[0].store_display_value(formdata.data, '1')
+            formdata.data['1_structured'] = formdef.fields[0].store_structured_value(formdata.data, '1')
+            formdata.just_created()
+            formdata.store()
+            formdata.perform_workflow()
+            formdata.store()
+
+    app = login(get_app(pub))
+
+    resp = app.get(formdef.get_backoffice_url())
+    assert len(resp.pyquery('tbody tr')) == 3
+
+    # enable item filter
+    resp.forms['listing-settings']['filter-1'].checked = True
+    resp = resp.forms['listing-settings'].submit()
+
+    assert [x[0] for x in resp.forms['listing-settings']['filter-1-value'].options] == ['', 'attr1', 'attr2']
+    resp.forms['listing-settings']['filter-1-value'] = 'attr2'
+    resp = resp.forms['listing-settings'].submit()
+    assert len(resp.pyquery('tbody tr')) == 2
+
+    resp.forms['save-custom-view']['title'] = 'custom view'
+    resp.forms['save-custom-view']['visibility'] = 'any'
+    resp = resp.forms['save-custom-view'].submit()
+    assert pub.custom_view_class.count() == 1
+    custom_view = pub.custom_view_class.select()[0]
+    assert custom_view.filters['filter-1-value'] == 'attr2'

tests/form_pages/test_item_field.py

@@ -405,6 +405,49 @@ def test_item_field_from_cards_id_identifier(pub):
     assert formdef.data_class().select()[0].data['0_structured']['name'] == 'bar'
 
 
+def test_item_field_from_cards_custom_identifier(pub):
+    create_user(pub)
+    formdef = create_formdef()
+    formdef.data_class().wipe()
+
+    CardDef.wipe()
+    carddef = CardDef()
+    carddef.name = 'items'
+    carddef.digest_templates = {'default': '{{form_var_name}}'}
+    carddef.id_template = '{{form_var_custom_id}}'
+    carddef.fields = [
+        fields.StringField(id='0', label='string', varname='name'),
+        fields.StringField(id='1', label='string', varname='custom_id'),
+    ]
+    carddef.store()
+    carddef.data_class().wipe()
+    for i, value in enumerate(['foo', 'bar', 'baz']):
+        carddata = carddef.data_class()()
+        carddata.data = {
+            '0': value,
+            '1': 'attr%s' % i,
+        }
+        carddata.just_created()
+        carddata.store()
+
+    ds = {'type': 'carddef:%s' % carddef.url_name}
+    formdef.fields = [fields.ItemField(id='0', label='string', data_source=ds, display_disabled_items=True)]
+    formdef.store()
+
+    resp = get_app(pub).get('/test/')
+    assert resp.form['f0'].options == [
+        ('attr1', False, 'bar'),
+        ('attr2', False, 'baz'),
+        ('attr0', False, 'foo'),
+    ]
+    resp.form['f0'] = 'attr1'
+    resp = resp.form.submit('submit')  # -> validation page
+    resp = resp.form.submit('submit')  # -> submit
+    assert formdef.data_class().select()[0].data['0'] == 'attr1'
+    assert formdef.data_class().select()[0].data['0_display'] == 'bar'
+    assert formdef.data_class().select()[0].data['0_structured']['name'] == 'bar'
+
+
 def test_item_field_from_cards_then_comment_related_card(pub):
     # https://dev.entrouvert.org/issues/58292
     create_user(pub)

wcs/admin/forms.py

@@ -417,7 +417,7 @@ class OptionsDirectory(Directory):
         )
         return self.handle(form, _('Appearance'))
 
-    def templates(self):
+    def get_templates_form(self):
         form = Form(enctype='multipart/form-data')
         form.add(
             StringWidget,
@@ -438,6 +438,10 @@ class OptionsDirectory(Directory):
             title=_('Submission Lateral Block'),
             value=self.formdef.submission_lateral_template,
         )
+        return form
+
+    def templates(self):
+        form = self.get_templates_form()
         result = self.handle(form, _('Templates'))
         if self.changed and self.formdef.data_class().count():
             get_response().add_after_job(UpdateDigestAfterJob(formdefs=[self.formdef]))
@@ -486,6 +490,7 @@ class OptionsDirectory(Directory):
                 'include_download_all_button',
                 'digest_template',
                 'lateral_template',
+                'id_template',
                 'submission_lateral_template',
                 'drafts_lifespan',
                 'user_support',
@@ -504,6 +509,10 @@ class OptionsDirectory(Directory):
                             if not self.formdef.digest_templates:
                                 self.formdef.digest_templates = {}
                             self.formdef.digest_templates['default'] = new_value
+                    elif attr == 'id_template':
+                        if self.formdef.id_template != new_value:
+                            self.changed = True
+                            self.formdef.id_template = new_value
                     else:
                         if getattr(self.formdef, attr, None) != new_value:
                             setattr(self.formdef, attr, new_value)
@@ -840,6 +849,7 @@ class FormDefPage(Directory, TempfileDirectoryMixin):
             self.formdef.default_digest_template
             or self.formdef.lateral_template
             or self.formdef.submission_lateral_template
+            or self.formdef.id_template
         ):
             template_status = pgettext_lazy('template', 'Custom')
         else:

wcs/backoffice/cards.py

@@ -31,6 +31,7 @@ from wcs.categories import CardDefCategory
 from wcs.sql_criterias import Null, StrictNotEqual
 
 from ..qommon import _, pgettext_lazy
+from ..qommon.form import ComputedExpressionWidget, StringWidget
 
 
 class CardDefUI(FormDefUI):
@@ -44,6 +45,28 @@ class CardDefOptionsDirectory(OptionsDirectory):
     category_empty_choice = _('Select a category for this card model')
     section = 'cards'
 
+    def get_templates_form(self):
+        form = super().get_templates_form()
+        if not get_publisher().has_site_option('enable-card-identifier-template'):
+            return form
+        criterias = [
+            StrictNotEqual('status', 'draft'),
+        ]
+        kwargs = {}
+        if self.formdef.data_class().exists(criterias):
+            kwargs['readonly'] = True
+            kwargs['hint'] = _('Identifier cannot be modified if there are existing cards.')
+        form.add(
+            StringWidget,
+            'id_template',
+            title=_('Unique identifier template'),
+            value=self.formdef.id_template,
+            validation_function=ComputedExpressionWidget.validate_template,
+            size=50,
+            **kwargs,
+        )
+        return form
+
 
 class CardFieldDefPage(FormFieldDefPage):
     section = 'cards'

wcs/backoffice/data_management.py

@@ -384,7 +384,7 @@ class CardFillPage(FormFillPage):
         if get_request().form.get('_popup'):
             popup_response_data = json.dumps(
                 {
-                    'value': str(filled.id),
+                    'value': str(filled.get_natural_key()),
                     'obj': str(filled.default_digest),
                     'edit_related_url': filled.get_edit_related_url() or '',
                     'view_related_url': filled.get_view_related_url() or '',
@@ -407,9 +407,9 @@ class CardFillPage(FormFillPage):
 class CardBackOfficeStatusPage(FormBackOfficeStatusPage):
     form_page_class = CardFillPage
 
-    sidebar_recorded_message = _('The card has been recorded on %(date)s with the number %(number)s.')
+    sidebar_recorded_message = _('The card has been recorded on %(date)s with the identifier %(identifier)s.')
     sidebar_recorded_by_agent_message = _(
-        'The card has been recorded on %(date)s with the number %(number)s by %(agent)s.'
+        'The card has been recorded on %(date)s with the identifier %(identifier)s by %(agent)s.'
     )
 
     def should_fold_summary(self, mine, request_user):

wcs/backoffice/management.py

@@ -1803,7 +1803,7 @@ class FormPage(Directory, TempfileDirectoryMixin):
             return redirect('..')
 
     def get_formdef_fields(self, include_block_items_fields=False):
-        yield FakeField('id', 'id', _('Number'))
+        yield FakeField('id', 'id', _('Identifier') if self.formdef.id_template else _('Number'))
         if self.formdef.default_digest_template:
             yield FakeField('digest', 'digest', _('Digest'))
         yield FakeField('submission_channel', 'submission_channel', _('Channel'))
@@ -3257,9 +3257,9 @@ class FormBackOfficeStatusPage(FormStatusPage):
     ]
     form_page_class = FormBackofficeEditPage
 
-    sidebar_recorded_message = _('The form has been recorded on %(date)s with the number %(number)s.')
+    sidebar_recorded_message = _('The form has been recorded on %(date)s with the number %(identifier)s.')
     sidebar_recorded_by_agent_message = _(
-        'The form has been recorded on %(date)s with the number %(number)s by %(agent)s.'
+        'The form has been recorded on %(date)s with the number %(identifier)s by %(agent)s.'
     )
 
     def _q_index(self):
@@ -3377,11 +3377,11 @@ class FormBackOfficeStatusPage(FormStatusPage):
             if agent_user:
                 r += self.sidebar_recorded_by_agent_message % {
                     'date': tm,
-                    'number': formdata.get_display_id(),
+                    'identifier': formdata.get_display_id(),
                     'agent': agent_user.get_display_name(),
                 }
             else:
-                r += self.sidebar_recorded_message % {'date': tm, 'number': formdata.get_display_id()}
+                r += self.sidebar_recorded_message % {'date': tm, 'identifier': formdata.get_display_id()}
             r += htmltext('</p>')
             try:
                 status_colour = formdata.get_status().colour

wcs/carddata.py

@@ -39,6 +39,13 @@ class CardData(FormData):
 
     formdef = property(get_formdef)
 
+    @classmethod
+    def get_by_id(cls, value):
+        try:
+            return cls.select([cls._formdef.get_by_id_criteria(value)], limit=1)[0]
+        except IndexError:
+            raise KeyError(value)
+
     def get_data_source_structured_item(
         self, digest_key='default', group_by=None, with_related_urls=False, with_files_urls=False
     ):
@@ -50,7 +57,7 @@ class CardData(FormData):
             get_publisher().record_error(summary, formdata=self)
 
         item = {
-            'id': self.id,
+            'id': self.id_display if self.formdef.id_template else self.id,
             'text': (self.digests or {}).get(digest_key) or '',
         }
 

wcs/carddef.py

@@ -143,6 +143,18 @@ class CardDef(FormDef):
         self.roles = self.backoffice_submission_roles
         return super().store(comment=comment, *args, **kwargs)
 
+    def get_by_id_criteria(self, value):
+        if self.id_template:
+            return Equal('id_display', value)
+        try:
+            if int(value) >= 2**31:
+                # out of range for postgresql integer type; would raise DataError.
+                raise OverflowError
+        except ValueError:
+            # value not an integer, it could be id_display
+            return Equal('id_display', value)
+        return Equal('id', value)
+
     @classmethod
     def get_carddefs_as_data_source(cls):
         carddefs_by_id = {}
@@ -239,15 +251,9 @@ class CardDef(FormDef):
             criterias.append(ElementILike('digests', digest_key, query))
         if get_by_id:
             try:
-                if int(get_by_id) >= 2**31:
-                    # out of range for postgresql integer type; would raise
-                    # DataError.
-                    return []
-            except ValueError:
-                # get_by_id not an integer, it could be id_display
-                criterias.append(Equal('id_display', get_by_id))
-            else:
-                criterias.append(Equal('id', get_by_id))
+                criterias.append(carddef.get_by_id_criteria(get_by_id))
+            except OverflowError:
+                return []
         if get_by_text is not None:
             if not get_by_text:
                 # don't match empty digests

wcs/fields/item.py

@@ -384,7 +384,7 @@ class ItemField(WidgetField, MapOptionsMixin, ItemFieldMixin, ItemWithImageField
         if not carddef:
             return value
         try:
-            carddata = carddef.data_class().get(value_id)
+            carddata = carddef.data_class().get_by_id(value_id)
         except KeyError:
             return value
         parts = data_source.data_source['type'].split(':')

wcs/fields/items.py

@@ -224,7 +224,7 @@ class ItemsField(WidgetField, ItemFieldMixin, ItemWithImageFieldMixin):
             r = TemplateIO(html=True)
             for value_id, value_label in zip(kwargs['value_id'], kwargs['labels']):
                 try:
-                    carddata = carddef.data_class().get(value_id)
+                    carddata = carddef.data_class().get_by_id(value_id)
                     value = (carddata.digests or {}).get('default') or value_label
                 except KeyError:
                     carddata = None

wcs/formdata.py

@@ -349,6 +349,11 @@ class FormData(StorableObject):
         obj = self.get(self.id)
         self.__dict__ = obj.__dict__
 
+    def get_natural_key(self):
+        if self.formdef.id_template:
+            return self.id_display
+        return self.id
+
     def get_user(self):
         if self.user_id and self.user_id != 'ultra-user':
             return get_publisher().user_class.get(self.user_id, ignore_errors=True)
@@ -485,9 +490,10 @@ class FormData(StorableObject):
         fields = {}
         for key, value in (self.formdef.digest_templates or {}).items():
             fields['template:%s' % key] = value
-        if not self.id_display:
-            # only set id_display once as it may have been set automatically
-            # by interpreting a webservice response.
+        if self.formdef.id_template or not self.id_display:
+            # unless a specific template is defined, only set id_display once
+            # as it may have been set automatically by interpreting a webservice
+            # response.
             fields['id_display'] = self.formdef.get_display_id_format().strip()
 
         changed = False
@@ -1403,10 +1409,16 @@ class FormData(StorableObject):
             self.store()
 
     def get_display_name(self):
-        return _('%(name)s #%(id)s') % {
-            'name': get_publisher().translate(self.formdef.name),
-            'id': self.get_display_id(),
-        }
+        if self.formdef.id_template:
+            return _('%(name)s - %(id)s') % {
+                'name': get_publisher().translate(self.formdef.name),
+                'id': self.get_display_id(),
+            }
+        else:
+            return _('%(name)s #%(id)s') % {
+                'name': get_publisher().translate(self.formdef.name),
+                'id': self.get_display_id(),
+            }
 
     @property
     def default_digest(self):

wcs/formdef.py

@@ -178,6 +178,7 @@ class FormDef(StorableObject):
     digest_templates = None
     lateral_template = None
     submission_lateral_template = None
+    id_template = None
     drafts_lifespan = None
     user_support = None
 
@@ -207,6 +208,7 @@ class FormDef(StorableObject):
         'appearance_keywords',
         'lateral_template',
         'submission_lateral_template',
+        'id_template',
         'drafts_lifespan',
         'user_support',
     ]
@@ -775,7 +777,7 @@ class FormDef(StorableObject):
         return '%s/%s/' % (base_url, self.url_name)
 
     def get_display_id_format(self):
-        return '[formdef_id]-[form_number_raw]'
+        return self.id_template or '{{formdef_id}}-{{form_number_raw}}'
 
     def get_submission_lateral_block(self):
         context = get_publisher().substitutions.get_context_variables(mode='lazy')

wcs/forms/root.py

@@ -1914,7 +1914,7 @@ class FormPage(Directory, TempfileDirectoryMixin, FormTemplateMixin):
         if get_request().form.get('_popup'):
             popup_response_data = json.dumps(
                 {
-                    'value': str(self.edited_data.id),
+                    'value': str(self.edited_data.get_natural_key()),
                     'obj': str(self.edited_data.default_digest),
                     'edit_related_url': self.edited_data.get_edit_related_url() or '',
                     'view_related_url': self.edited_data.get_view_related_url() or '',

wcs/sql.py

@@ -758,7 +758,10 @@ def do_formdef_indexes(formdef, created, conn, cur):
         '''%s %s_fid ON %s (formdata_id, id)''' % (create_index, evolutions_table_name, evolutions_table_name)
     )
 
-    for attr in ('receipt_time', 'anonymised', 'user_id', 'status'):
+    attrs = ['receipt_time', 'anonymised', 'user_id', 'status']
+    if isinstance(formdef, CardDef):
+        attrs.append('id_display')
+    for attr in attrs:
         cur.execute(
             '%(create_index)s %(table_name)s_%(attr)s_idx ON %(table_name)s (%(attr)s)'
             % {'create_index': create_index, 'table_name': table_name, 'attr': attr}
@@ -1799,13 +1802,22 @@ class SqlMixin:
                 else:
                     carddef_data_table_name = get_formdef_table_name(field.carddef)
                     carddef_table_alias = 't%s' % id(field.carddef)
-                    carddef_table_decl = 'LEFT JOIN %s AS %s ON (CAST(%s.%s AS INTEGER) = %s.id)' % (
-                        carddef_data_table_name,
-                        carddef_table_alias,
-                        cls._table_name,
-                        get_field_id(field.parent_field),
-                        carddef_table_alias,
-                    )
+                    if field.carddef.id_template:
+                        carddef_table_decl = 'LEFT JOIN %s AS %s ON (%s.%s = %s.id_display)' % (
+                            carddef_data_table_name,
+                            carddef_table_alias,
+                            cls._table_name,
+                            get_field_id(field.parent_field),
+                            carddef_table_alias,
+                        )
+                    else:
+                        carddef_table_decl = 'LEFT JOIN %s AS %s ON (CAST(%s.%s AS INTEGER) = %s.id)' % (
+                            carddef_data_table_name,
+                            carddef_table_alias,
+                            cls._table_name,
+                            get_field_id(field.parent_field),
+                            carddef_table_alias,
+                        )
 
                 if carddef_table_decl not in tables:
                     tables.append(carddef_table_decl)
@@ -4969,7 +4981,7 @@ def get_period_total(
 # latest migration, number + description (description is not used
 # programmaticaly but will make sure git conflicts if two migrations are
 # separately added with the same number)
-SQL_LEVEL = (96, 'change to fts normalization')
+SQL_LEVEL = (97, 'add index on carddata/id_display')
 
 
 def migrate_global_views(conn, cur):
@@ -5219,6 +5231,7 @@ def migrate():
         # 45 & 46: add index on formdata(status)
         # 56: add GIN indexes to concerned_roles_array and actions_roles_array
         # 74: (late migration) change evolution index to be on (fomdata_id, id)
+        # 97: add index on carddata/id_display
         for formdef in FormDef.select() + CardDef.select():
             do_formdef_indexes(formdef, created=False, conn=conn, cur=cur)
     if sql_level < 30: