Lancer un train de facturation dans un job asynchrone (#73223) #10

Merged
lguerin merged 1 commits from wip/73223-invoicing-job into main 2023-01-16 14:39:08 +01:00
12 changed files with 280 additions and 38 deletions

1
debian/uwsgi.ini vendored
View File

@ -13,6 +13,7 @@ chmod-socket = 666
vacuum = true
spooler-processes = 3
spooler-python-import = lingo.invoicing.spooler
spooler-python-import = hobo.provisionning.spooler
spooler-max-tasks = 20

View File

@ -19,7 +19,6 @@ import datetime
from django.core.management.base import BaseCommand, CommandError
from lingo.invoicing.models import Campaign
from lingo.invoicing.utils import generate_invoices
class Command(BaseCommand):
@ -59,7 +58,7 @@ class Command(BaseCommand):
campaign = Campaign.objects.create(
date_start=date_start, date_end=date_end, date_issue=date_issue
)
generate_invoices(campaign=campaign, draft=options['draft'])
campaign.generate(spool=False, draft=options['draft'])
self.stdout.write(
self.style.SUCCESS('Invoicing generation OK (start: %s, end: %s)' % (date_start, date_end))

View File

@ -0,0 +1,30 @@
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('invoicing', '0004_campaign'),
]
operations = [
migrations.AddField(
model_name='pool',
name='completed_at',
field=models.DateTimeField(null=True),
),
migrations.AddField(
model_name='pool',
name='status',
field=models.CharField(
choices=[
('registered', 'Registered'),
('running', 'Running'),
('failed', 'Failed'),
('completed', 'Completed'),
],
default='registered',
max_length=100,
),
),
]

View File

@ -0,0 +1,19 @@
from django.db import migrations
def forward(apps, schema_editor):
Pool = apps.get_model('invoicing', 'Pool')
for pool in Pool.objects.all():
pool.status = 'completed'
pool.save()
class Migration(migrations.Migration):
dependencies = [
('invoicing', '0005_pool_status'),
]
operations = [
migrations.RunPython(forward, reverse_code=migrations.RunPython.noop),
]

View File

@ -15,12 +15,14 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import copy
import sys
from django.contrib.auth.models import Group
from django.contrib.postgres.fields import JSONField
from django.db import models
from django.db import connection, models, transaction
from django.utils.formats import date_format
from django.utils.text import slugify
from django.utils.timezone import now
from django.utils.translation import gettext_lazy as _
from lingo.utils.misc import generate_slug
@ -99,11 +101,64 @@ class Campaign(models.Model):
'end': date_format(self.date_end, 'd/m/Y'),
}
def generate(self, spool=True, draft=True):
pool = self.pool_set.create(draft=draft)
if spool and 'uwsgi' in sys.modules:
from lingo.invoicing.spooler import generate_invoices
tenant = getattr(connection, 'tenant', None)
transaction.on_commit(
lambda: generate_invoices.spool(
campaign_id=str(self.pk), pool_id=str(pool.pk), domain=getattr(tenant, 'domain_url', None)
)
)
return
pool.generate_invoices()
class Pool(models.Model):
campaign = models.ForeignKey(Campaign, on_delete=models.PROTECT)
draft = models.BooleanField(default=True)
created_at = models.DateTimeField(auto_now_add=True)
completed_at = models.DateTimeField(null=True)
status = models.CharField(
choices=[
('registered', _('Registered')),
('running', _('Running')),
('failed', _('Failed')),
('completed', _('Completed')),
],
default='registered',
max_length=100,
)
def generate_invoices(self):
from lingo.invoicing import utils
self.status = 'running'
self.save()
try:
# get agendas with pricing corresponding to the period
agendas = utils.get_agendas(pool=self)
# get subscribed users for each agenda, for the period
Outdated
Review

ubscribed -> subscribed

ubscribed -> subscribed
user_external_ids = utils.get_users_from_subscriptions(agendas=agendas, pool=self)
# get invoice lines for all subscribed users, for each agenda in the corresponding period
lines = utils.get_all_invoice_lines(
agendas=agendas, user_external_ids=user_external_ids, pool=self
)
# and generate invoices
utils.generate_invoices_from_lines(agendas=agendas, all_lines=lines, pool=self)
except Exception:
self.status = 'failed'
self.save()
raise
finally:
if self.status == 'running':
self.status = 'completed'
self.completed_at = now()
self.save()
class AbstractInvoice(models.Model):

View File

@ -0,0 +1,41 @@
# lingo - payment and billing system
# Copyright (C) 2023 Entr'ouvert
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from django.db import connection
from uwsgidecorators import spool # pylint: disable=import-error
from lingo.invoicing.models import Pool
def set_connection(domain):
from hobo.multitenant.middleware import TenantMiddleware # pylint: disable=import-error
tenant = TenantMiddleware.get_tenant_by_hostname(domain)
connection.set_tenant(tenant)
@spool
def generate_invoices(args):
if args.get('domain'):
# multitenant installation
set_connection(args['domain'])
try:
pool = Pool.objects.get(campaign__pk=args['campaign_id'], pk=args['pool_id'], status='registered')
except Pool.DoesNotExist:
return
pool.generate_invoices()

View File

@ -8,7 +8,7 @@
{% block appbar %}
<h2>{{ object }}</h2>
{% if not has_real_pool %}
{% if not has_running_pool and not has_real_pool %}
<span class="actions">
<a href="{% url 'lingo-manager-invoicing-campaign-delete' pk=object.pk %}" rel="popup">{% trans "Delete" %}</a>
<a href="{% url 'lingo-manager-invoicing-campaign-edit' pk=object.pk %}" rel="popup">{% trans "Edit" %}</a>
@ -39,12 +39,13 @@
<li>
<a href="{% url 'lingo-manager-invoicing-pool-detail' pk=object.pk pool_pk=pool.pk %}">
{{ pool.created_at|date:'DATETIME_FORMAT' }}
<span class="extra-info">- {{ pool.get_status_display }}{% if pool.completed_at %} ({% blocktrans with edate=pool.completed_at|date:"DATETIME_FORMAT" %}Ended at {{ edate }}{% endblocktrans %}){% endif %}</span>
{% if pool.draft %}<span class="badge">{% trans "draft" %}</span>{% endif %}
</a>
</li>
{% endfor %}
</ul>
{% if not has_real_pool %}
{% if not has_running_pool and not has_real_pool %}
<div class="panel--buttons">
<a class="pk-button" rel="popup" href="{% url 'lingo-manager-invoicing-pool-add' pk=object.pk %}">{% trans 'Start a pool' %}</a>
</div>

View File

@ -8,7 +8,7 @@
{% block appbar %}
<h2>{{ pool.created_at|date:"DATETIME_FORMAT" }}</h2>
{% if pool.draft %}
{% if pool.draft and pool.status != 'registered' and pool.status != 'running' %}
<span class="actions">
<a href="{% url 'lingo-manager-invoicing-pool-delete' pk=object.pk pool_pk=pool.pk %}" rel="popup">{% trans "Delete" %}</a>
</span>

View File

@ -206,15 +206,3 @@ def generate_invoices_from_lines(agendas, all_lines, pool):
invoices.append(invoice)
return invoices
def generate_invoices(campaign, draft=True):
pool = campaign.pool_set.create(draft=draft)
# get agendas with pricing corresponding to the period
agendas = get_agendas(pool=pool)
# get ubscribed users for each agenda, for the period
user_external_ids = get_users_from_subscriptions(agendas=agendas, pool=pool)
# get invoice lines for all subscribed users, for each agenda in the corresponding period
lines = get_all_invoice_lines(agendas=agendas, user_external_ids=user_external_ids, pool=pool)
# and generate invoices
generate_invoices_from_lines(agendas=agendas, all_lines=lines, pool=pool)

View File

@ -46,7 +46,6 @@ from lingo.invoicing.models import (
Regie,
RegieImportError,
)
from lingo.invoicing.utils import generate_invoices
from lingo.pricing.forms import ImportForm
@ -214,6 +213,7 @@ class CampaignDetailView(DetailView):
def get_context_data(self, **kwargs):
kwargs['pools'] = self.object.pool_set.order_by('created_at')
kwargs['has_running_pool'] = any(p.status in ['registered', 'running'] for p in kwargs['pools'])
kwargs['has_real_pool'] = any(not p.draft for p in kwargs['pools'])
return super().get_context_data(**kwargs)
@ -227,7 +227,12 @@ class CampaignEditView(UpdateView):
form_class = CampaignForm
def get_queryset(self):
return super().get_queryset().exclude(pool__draft=False)
return (
super()
.get_queryset()
.exclude(pool__draft=False)
.exclude(pool__status__in=['registered', 'running'])
)
def get_success_url(self):
return reverse('lingo-manager-invoicing-campaign-detail', args=[self.object.pk])
@ -241,7 +246,12 @@ class CampaignDeleteView(DeleteView):
model = Campaign
def get_queryset(self):
return super().get_queryset().exclude(pool__draft=False)
return (
super()
.get_queryset()
.exclude(pool__draft=False)
.exclude(pool__status__in=['registered', 'running'])
)
def delete(self, request, *args, **kwargs):
self.object = self.get_object()
@ -287,7 +297,10 @@ class PoolAddView(FormView):
form_class = PoolForm
def dispatch(self, request, *args, **kwargs):
self.object = get_object_or_404(Campaign.objects.exclude(pool__draft=False), pk=kwargs['pk'])
self.object = get_object_or_404(
Campaign.objects.exclude(pool__draft=False).exclude(pool__status__in=['registered', 'running']),
pk=kwargs['pk'],
)
return super().dispatch(request, *args, **kwargs)
def get_form_kwargs(self):
@ -300,7 +313,7 @@ class PoolAddView(FormView):
return super().get_context_data(**kwargs)
def form_valid(self, form):
generate_invoices(campaign=self.object, draft=form.cleaned_data['draft'])
self.object.generate(draft=form.cleaned_data['draft'])
return super().form_valid(form)
def get_success_url(self):
@ -316,7 +329,10 @@ class PoolDeleteView(DeleteView):
pk_url_kwarg = 'pool_pk'
def dispatch(self, request, *args, **kwargs):
self.campaign = get_object_or_404(Campaign.objects.exclude(pool__draft=False), pk=kwargs['pk'])
self.campaign = get_object_or_404(
Campaign.objects.exclude(pool__draft=False).exclude(pool__status__in=['registered', 'running']),
pk=kwargs['pk'],
)
return super().dispatch(request, *args, **kwargs)
def get_queryset(self):

View File

@ -72,15 +72,35 @@ def test_detail_campaign(app, admin_user):
pool1 = Pool.objects.create(
campaign=campaign,
draft=True,
status='completed',
)
pool2 = Pool.objects.create(
campaign=campaign,
draft=True,
status='registered',
)
app = login(app)
resp = app.get('/manage/invoicing/campaigns/')
resp = resp.click(href='/manage/invoicing/campaign/%s/' % campaign.pk)
assert '/manage/invoicing/campaign/%s/edit/' % campaign.pk not in resp
assert '/manage/invoicing/campaign/%s/delete/' % campaign.pk not in resp
assert '/manage/invoicing/campaign/%s/pool/%s/' % (campaign.pk, pool1.pk) in resp
assert '/manage/invoicing/campaign/%s/pool/%s/' % (campaign.pk, pool2.pk) in resp
assert '/manage/invoicing/campaign/%s/pool/add/' % (campaign.pk) not in resp
pool2.status = 'running'
pool2.save()
resp = app.get('/manage/invoicing/campaign/%s/' % campaign.pk)
assert '/manage/invoicing/campaign/%s/edit/' % campaign.pk not in resp
assert '/manage/invoicing/campaign/%s/delete/' % campaign.pk not in resp
assert '/manage/invoicing/campaign/%s/pool/%s/' % (campaign.pk, pool1.pk) in resp
assert '/manage/invoicing/campaign/%s/pool/%s/' % (campaign.pk, pool2.pk) in resp
assert '/manage/invoicing/campaign/%s/pool/add/' % (campaign.pk) not in resp
pool2.status = 'failed'
pool2.save()
resp = app.get('/manage/invoicing/campaign/%s/' % campaign.pk)
assert '/manage/invoicing/campaign/%s/edit/' % campaign.pk in resp
assert '/manage/invoicing/campaign/%s/delete/' % campaign.pk in resp
assert '/manage/invoicing/campaign/%s/pool/%s/' % (campaign.pk, pool1.pk) in resp
@ -147,8 +167,23 @@ def test_edit_campaign(app, admin_user):
pool = Pool.objects.create(
campaign=campaign,
draft=True,
status='completed',
)
app.get('/manage/invoicing/campaign/%s/edit/' % campaign.pk)
pool.status = 'failed'
pool.save()
app.get('/manage/invoicing/campaign/%s/edit/' % campaign.pk)
pool.status = 'registered'
pool.save()
app.get('/manage/invoicing/campaign/%s/edit/' % campaign.pk, status=404)
pool.status = 'running'
pool.save()
app.get('/manage/invoicing/campaign/%s/edit/' % campaign.pk, status=404)
pool.status = 'completed'
pool.draft = False
pool.save()
app.get('/manage/invoicing/campaign/%s/edit/' % campaign.pk, status=404)
@ -171,6 +206,7 @@ def test_delete_campaign(app, admin_user):
pool = Pool.objects.create(
campaign=campaign,
draft=True,
status='completed',
)
regie = Regie.objects.create(label='Foo')
invoice = DraftInvoice.objects.create(date_issue=datetime.date.today(), regie=regie, pool=pool)
@ -187,6 +223,19 @@ def test_delete_campaign(app, admin_user):
assert Campaign.objects.count() == 0
campaign.save()
pool.status = 'failed'
pool.save()
app.get('/manage/invoicing/campaign/%s/delete/' % campaign.pk)
pool.status = 'registered'
pool.save()
app.get('/manage/invoicing/campaign/%s/delete/' % campaign.pk, status=404)
pool.status = 'running'
pool.save()
app.get('/manage/invoicing/campaign/%s/delete/' % campaign.pk, status=404)
pool.status = 'completed'
pool.draft = False
pool.save()
app.get('/manage/invoicing/campaign/%s/delete/' % campaign.pk, status=404)
@ -202,22 +251,36 @@ def test_add_pool(app, admin_user):
app = login(app)
resp = app.get('/manage/invoicing/campaign/%s/pool/add/' % campaign.pk)
assert resp.form['draft'].value == 'on'
with mock.patch('lingo.invoicing.views.generate_invoices') as mock_generate:
with mock.patch.object(Campaign, 'generate', autospec=True) as mock_generate:
resp = resp.form.submit()
assert resp.location.endswith('/manage/invoicing/campaign/%s/#open:pools' % campaign.pk)
assert mock_generate.call_args_list == [mock.call(campaign=campaign, draft=True)]
assert mock_generate.call_args_list == [mock.call(campaign, draft=True)]
pool = Pool.objects.create(
campaign=campaign,
draft=True,
status='failed',
)
resp = app.get('/manage/invoicing/campaign/%s/pool/add/' % campaign.pk)
resp.form['draft'] = False
with mock.patch('lingo.invoicing.views.generate_invoices') as mock_generate:
with mock.patch.object(Campaign, 'generate', autospec=True) as mock_generate:
resp = resp.form.submit()
assert resp.location.endswith('/manage/invoicing/campaign/%s/#open:pools' % campaign.pk)
assert mock_generate.call_args_list == [mock.call(campaign=campaign, draft=False)]
assert mock_generate.call_args_list == [mock.call(campaign, draft=False)]
pool.status = 'completed'
pool.save()
app.get('/manage/invoicing/campaign/%s/pool/add/' % campaign.pk)
pool.status = 'registered'
pool.save()
app.get('/manage/invoicing/campaign/%s/pool/add/' % campaign.pk, status=404)
pool.status = 'running'
pool.save()
app.get('/manage/invoicing/campaign/%s/pool/add/' % campaign.pk, status=404)
pool.status = 'completed'
pool.draft = False
pool.save()
app.get('/manage/invoicing/campaign/%s/pool/add/' % campaign.pk, status=404)
@ -237,6 +300,7 @@ def test_detail_pool(app, admin_user):
pool = Pool.objects.create(
campaign=campaign,
draft=True,
status='completed',
)
app = login(app)
@ -252,6 +316,22 @@ def test_detail_pool(app, admin_user):
resp = app.get('/manage/invoicing/campaign/%s/pool/%s/' % (campaign.pk, pool.pk))
assert '/manage/invoicing/campaign/%s/pool/%s/delete/' % (campaign.pk, pool.pk) not in resp
pool.draft = True
pool.status = 'registered'
pool.save()
resp = app.get('/manage/invoicing/campaign/%s/pool/%s/' % (campaign.pk, pool.pk))
assert '/manage/invoicing/campaign/%s/pool/%s/delete/' % (campaign.pk, pool.pk) not in resp
pool.status = 'running'
pool.save()
resp = app.get('/manage/invoicing/campaign/%s/pool/%s/' % (campaign.pk, pool.pk))
assert '/manage/invoicing/campaign/%s/pool/%s/delete/' % (campaign.pk, pool.pk) not in resp
pool.status = 'failed'
pool.save()
resp = app.get('/manage/invoicing/campaign/%s/pool/%s/' % (campaign.pk, pool.pk))
assert '/manage/invoicing/campaign/%s/pool/%s/delete/' % (campaign.pk, pool.pk) in resp
def test_delete_pool(app, admin_user):
campaign = Campaign.objects.create(
@ -267,6 +347,7 @@ def test_delete_pool(app, admin_user):
pool = Pool.objects.create(
campaign=campaign,
draft=True,
status='completed',
)
regie = Regie.objects.create(label='Foo')
invoice = DraftInvoice.objects.create(date_issue=datetime.date.today(), regie=regie, pool=pool)
@ -291,6 +372,19 @@ def test_delete_pool(app, admin_user):
app.get('/manage/invoicing/campaign/%s/pool/%s/delete/' % (campaign2.pk, pool.pk), status=404)
app.get('/manage/invoicing/campaign/%s/pool/%s/delete/' % (campaign.pk, 0), status=404)
pool.draft = False
pool.status = 'registered'
pool.save()
app.get('/manage/invoicing/campaign/%s/pool/%s/delete/' % (campaign.pk, pool.pk), status=404)
pool.status = 'running'
pool.save()
app.get('/manage/invoicing/campaign/%s/pool/%s/delete/' % (campaign.pk, pool.pk), status=404)
pool.draft = False
pool.status = 'error'
pool.save()
app.get('/manage/invoicing/campaign/%s/pool/%s/delete/' % (campaign.pk, pool.pk), status=404)
pool.draft = True
pool.save()
app.get('/manage/invoicing/campaign/%s/pool/%s/delete/' % (campaign.pk, pool.pk))

View File

@ -1011,7 +1011,7 @@ def test_generate_invoices(mock_generate, mock_lines, mock_users, mock_agendas):
mock_lines.return_value = ['foo', 'baz']
# check only calls between functions
utils.generate_invoices(campaign=campaign, draft=True)
campaign.generate(draft=True)
pool = Pool.objects.latest('pk')
assert pool.campaign == campaign
assert pool.draft is True
@ -1039,9 +1039,7 @@ def test_generate_invoices(mock_generate, mock_lines, mock_users, mock_agendas):
def test_generate_invoices_cmd():
with mock.patch(
'lingo.invoicing.management.commands.generate_invoices.generate_invoices'
) as mock_generate:
with mock.patch.object(Campaign, 'generate', autospec=True) as mock_generate:
with pytest.raises(CommandError) as excinfo:
call_command('generate_invoices')
assert '%s' % excinfo.value == 'Error: the following arguments are required: date_start, date_end'
@ -1069,17 +1067,17 @@ def test_generate_invoices_cmd():
assert campaign.date_start == datetime.date(2022, 9, 1)
assert campaign.date_end == datetime.date(2022, 10, 1)
assert campaign.date_issue == campaign.date_end
assert mock_generate.call_args_list == [mock.call(campaign=campaign, draft=False)]
assert mock_generate.call_args_list == [mock.call(campaign, draft=False, spool=False)]
mock_generate.reset_mock()
# again
call_command('generate_invoices', '2022-09-01', '2022-10-01')
assert Campaign.objects.count() == 1
assert mock_generate.call_args_list == [mock.call(campaign=campaign, draft=False)]
assert mock_generate.call_args_list == [mock.call(campaign, draft=False, spool=False)]
mock_generate.reset_mock()
call_command('generate_invoices', '2022-09-01', '2022-10-01', '--draft')
assert Campaign.objects.count() == 1
assert mock_generate.call_args_list == [mock.call(campaign=campaign, draft=True)]
assert mock_generate.call_args_list == [mock.call(campaign, draft=True, spool=False)]
mock_generate.reset_mock()
# with overlapping
@ -1096,7 +1094,7 @@ def test_generate_invoices_cmd():
call_command('generate_invoices', '2022-08-01', '2022-09-01', '--date-issue=2022-10-31')
assert Campaign.objects.count() == 2
campaign = Campaign.objects.latest('pk')
assert mock_generate.call_args_list == [mock.call(campaign=campaign, draft=False)]
assert mock_generate.call_args_list == [mock.call(campaign, draft=False, spool=False)]
assert campaign.date_start == datetime.date(2022, 8, 1)
assert campaign.date_end == datetime.date(2022, 9, 1)
assert campaign.date_issue == datetime.date(2022, 10, 31)
@ -1104,7 +1102,7 @@ def test_generate_invoices_cmd():
call_command('generate_invoices', '2022-10-01', '2022-11-01')
assert Campaign.objects.count() == 3
campaign = Campaign.objects.latest('pk')
assert mock_generate.call_args_list == [mock.call(campaign=campaign, draft=False)]
assert mock_generate.call_args_list == [mock.call(campaign, draft=False, spool=False)]
assert campaign.date_start == datetime.date(2022, 10, 1)
assert campaign.date_end == datetime.date(2022, 11, 1)
assert campaign.date_issue == campaign.date_end