VBV-705 feat: cron job abacus export

compose/django/supercronic_crontab

@@ -4,8 +4,11 @@
 # Run every 6 hours
 0 */6 * * * /usr/local/bin/python /app/manage.py simple_dummy_job
 
-# Run every hour
+# Run every hour at minute 17
-0 */1 * * * /usr/local/bin/python /app/manage.py edoniq_import_results
+17 * * * * /usr/local/bin/python /app/manage.py edoniq_import_results
+
+# Run every hour at minute 43
+43 * * * * /usr/local/bin/python /app/manage.py abacus_export_job
 
 # every day at 19:30
 30 19 * * * /usr/local/bin/python /app/manage.py send_email_reminders --type=all

server/integration_tests/abacus_sftp/test_abacus_sftp.py

@@ -1,13 +1,14 @@
 import os
 import shutil
 import tempfile
-from datetime import datetime
+from datetime import date
 from io import StringIO
 from subprocess import Popen
 from time import sleep
 
 import pytest
 from django.conf import settings
+from freezegun import freeze_time
 
 from vbv_lernwelt.core.admin import User
 from vbv_lernwelt.core.create_default_users import create_default_users
@@ -70,62 +71,111 @@ def test_upload_abacus_xml(setup_abacus_env):
     )
 
     feuz = User.objects.get(username="andreas.feuz@eiger-versicherungen.ch")
-    feuz_checkout_info = CheckoutInformationFactory(
-        user=feuz,
-        transaction_id="24021508331287484",
-        first_name="Andreas",
-        last_name="Feuz",
-        street="Eggersmatt",
-        street_number="32",
-        postal_code="1719",
-        city="Zumholz",
-        country_id="CH",
-        invoice_address="org",
-        organisation_detail_name="VBV",
-        organisation_street="Laupenstrasse",
-        organisation_street_number="10",
-        organisation_postal_code="3000",
-        organisation_city="Bern",
-        organisation_country_id="CH",
-    )
-    feuz_checkout_info.created_at = datetime(2024, 2, 15, 8, 33, 12, 0)
 
-    abacus_ssh_upload(feuz_checkout_info)
+    with freeze_time("2024-02-15 08:33:12"):
-
+        feuz_checkout_info = CheckoutInformationFactory(
-    # check if files got created
+            user=feuz,
-    debitor_filepath = os.path.join(tmppath, "debitor/myVBV_debi_60000012.xml")
+            transaction_id="24021508331287484",
-    assert os.path.exists(debitor_filepath)
+            first_name="Andreas",
-
+            last_name="Feuz",
-    with open(debitor_filepath) as debitor_file:
+            street="Eggersmatt",
-        debi_content = debitor_file.read()
+            street_number="32",
-        assert "<CustomerNumber>60000012</CustomerNumber>" in debi_content
+            postal_code="1719",
-        assert "<Email>andreas.feuz@eiger-versicherungen.ch</Email>" in debi_content
+            city="Zumholz",
-
+            country_id="CH",
-    order_filepath = os.path.join(
+            invoice_address="org",
-        tmppath, "order/myVBV_orde_20240215083312_60000012_6000000124.xml"
+            organisation_detail_name="VBV",
-    )
+            organisation_street="Laupenstrasse",
-    assert os.path.exists(order_filepath)
+            organisation_street_number="10",
-    with open(order_filepath) as order_file:
+            organisation_postal_code="3000",
-        order_content = order_file.read()
+            organisation_city="Bern",
-        assert (
+            organisation_country_id="CH",
-            "<ReferencePurchaseOrder>24021508331287484</ReferencePurchaseOrder>"
-            in order_content
         )
-        assert "<CustomerNumber>60000012</CustomerNumber>" in order_content
 
-    feuz_checkout_info.refresh_from_db()
+    with freeze_time("2024-02-15 09:37:41"):
-    assert feuz_checkout_info.abacus_ssh_upload_done
+        abacus_ssh_upload(feuz_checkout_info)
 
-    # calling `abacus_ssh_upload` a second time will not upload files again...
+        # check transmission data
-    os.remove(debitor_filepath)
+        feuz_checkout_info.refresh_from_db()
-    os.remove(order_filepath)
+        assert feuz_checkout_info.invoice_transmitted_at.date() == date(2024, 2, 15)
+        abacus_transmission_list = feuz_checkout_info.additional_json_data.get(
+            "abacus_transmission_list", []
+        )
+        assert len(abacus_transmission_list) == 1
+        assert abacus_transmission_list[0]["transmitted_at"][0:10] == "2024-02-15"
 
-    abacus_ssh_upload(feuz_checkout_info)
+        # check if files got created
+        debitor_filepath = os.path.join(tmppath, "debitor/myVBV_debi_60000012.xml")
+        assert os.path.exists(debitor_filepath)
 
-    debitor_filepath = os.path.join(tmppath, "debitor/myVBV_debi_60000012.xml")
+        with open(debitor_filepath) as debitor_file:
-    assert not os.path.exists(debitor_filepath)
+            debi_content = debitor_file.read()
+            assert "<CustomerNumber>60000012</CustomerNumber>" in debi_content
+            assert "<Email>andreas.feuz@eiger-versicherungen.ch</Email>" in debi_content
 
-    order_filepath = os.path.join(
+        order_filepath = os.path.join(
-        tmppath, "order/myVBV_orde_60000012_20240215083312_6000000124.xml"
+            tmppath, "order/myVBV_orde_20240215083312_60000012_6000000124.xml"
-    )
+        )
-    assert not os.path.exists(order_filepath)
+        assert os.path.exists(order_filepath)
+        with open(order_filepath) as order_file:
+            order_content = order_file.read()
+            assert (
+                "<ReferencePurchaseOrder>24021508331287484</ReferencePurchaseOrder>"
+                in order_content
+            )
+            assert "<CustomerNumber>60000012</CustomerNumber>" in order_content
+
+        feuz_checkout_info.refresh_from_db()
+        assert feuz_checkout_info.abacus_ssh_upload_done
+
+        # calling `abacus_ssh_upload` a second time will not upload files again...
+        os.remove(debitor_filepath)
+        os.remove(order_filepath)
+
+        abacus_ssh_upload(feuz_checkout_info)
+
+        debitor_filepath = os.path.join(tmppath, "debitor/myVBV_debi_60000012.xml")
+        assert not os.path.exists(debitor_filepath)
+
+        order_filepath = os.path.join(
+            tmppath, "order/myVBV_orde_60000012_20240215083312_6000000124.xml"
+        )
+        assert not os.path.exists(order_filepath)
+
+    with freeze_time("2024-04-21 17:33:19"):
+        # change customer data later will retrigger and redo abacus upload
+        feuz_checkout_info.first_name = "Peter"
+        feuz_checkout_info.last_name = "Egger"
+        feuz_checkout_info.save()
+
+        abacus_ssh_upload(feuz_checkout_info)
+
+        # check transmission data
+        feuz_checkout_info.refresh_from_db()
+        assert feuz_checkout_info.invoice_transmitted_at.date() == date(2024, 4, 21)
+        abacus_transmission_list = feuz_checkout_info.additional_json_data.get(
+            "abacus_transmission_list", []
+        )
+        assert len(abacus_transmission_list) == 2
+        assert abacus_transmission_list[1]["transmitted_at"][0:10] == "2024-04-21"
+
+        # check if files got created
+        debitor_filepath = os.path.join(tmppath, "debitor/myVBV_debi_60000012.xml")
+        assert os.path.exists(debitor_filepath)
+
+        with open(debitor_filepath) as debitor_file:
+            debi_content = debitor_file.read()
+            assert "<CustomerNumber>60000012</CustomerNumber>" in debi_content
+            assert "<Email>andreas.feuz@eiger-versicherungen.ch</Email>" in debi_content
+
+        order_filepath = os.path.join(
+            tmppath, "order/myVBV_orde_20240215083312_60000012_6000000124.xml"
+        )
+        assert os.path.exists(order_filepath)
+        with open(order_filepath) as order_file:
+            order_content = order_file.read()
+            assert (
+                "<ReferencePurchaseOrder>24021508331287484</ReferencePurchaseOrder>"
+                in order_content
+            )
+            assert "<CustomerNumber>60000012</CustomerNumber>" in order_content

server/vbv_lernwelt/shop/admin.py

@@ -64,6 +64,8 @@ class CheckoutInformationAdmin(admin.ModelAdmin):
     list_display = (
         "product_sku",
         customer,
+        "first_name",
+        "last_name",
         "product_name",
         "product_price",
         "created_at",
@@ -78,7 +80,11 @@ class CheckoutInformationAdmin(admin.ModelAdmin):
         "user__first_name",
         "user__last_name",
         "user__username",
+        "first_name",
+        "last_name",
         "transaction_id",
+        "refno2",
+        "organisation_detail_name",
         "abacus_order_id",
         "user__abacus_debitor_number",
     ]
@@ -91,6 +97,14 @@ class CheckoutInformationAdmin(admin.ModelAdmin):
         "state",
         "product_price",
         "webhook_history",
+        "refno2",
+        "product_sku",
+        "invoice_transmitted_at",
+        "cembra_byjuno_invoice",
+        "ip_address",
+        "device_fingerprint_session_key",
+        "abacus_order_id",
+        "abacus_ssh_upload_done",
     ]
 
 

server/vbv_lernwelt/shop/invoice/abacus.py

@@ -4,6 +4,8 @@ from xml.dom import minidom
 from xml.etree.ElementTree import Element, SubElement, tostring
 
 import structlog
+from django.db.models import F, Q
+from django.utils import timezone
 
 from vbv_lernwelt.shop.const import VV_PRODUCT_NUMBER
 from vbv_lernwelt.shop.invoice.abacus_sftp_client import AbacusSftpClient
@@ -12,14 +14,34 @@ from vbv_lernwelt.shop.models import CheckoutInformation, CheckoutState
 logger = structlog.get_logger(__name__)
 
 
+def filter_checkout_information_for_abacus_upload_qs():
+    return CheckoutInformation.objects.filter(
+        Q(abacus_ssh_upload_done=False)
+        | Q(updated_at__gt=F("invoice_transmitted_at") + datetime.timedelta(minutes=1)),
+        state=CheckoutState.PAID,
+    )
+
+
 def abacus_ssh_upload(checkout_information: CheckoutInformation):
     if checkout_information.state != CheckoutState.PAID:
         # only upload invoice if checkout is paid
         return True
 
     try:
-        if not checkout_information.abacus_ssh_upload_done:
+        if (
-            # only upload data for not yet uploaded invoices
+            # invoice not in abacus yet
+            not checkout_information.abacus_ssh_upload_done
+            # invoice updated more than 5 minutes ago
+            or (
+                checkout_information.abacus_ssh_upload_done
+                and checkout_information.invoice_transmitted_at is not None
+                and (
+                    checkout_information.updated_at
+                    - checkout_information.invoice_transmitted_at
+                )
+                > datetime.timedelta(minutes=1)
+            )
+        ):
             invoice_xml_filename, invoice_xml_content = create_invoice_xml(
                 checkout_information
             )
@@ -35,7 +57,22 @@ def abacus_ssh_upload(checkout_information: CheckoutInformation):
             )
 
             checkout_information.abacus_ssh_upload_done = True
-            checkout_information.invoice_transmitted_at = datetime.datetime.now()
+            checkout_information.invoice_transmitted_at = timezone.now()
+
+            abacus_transmission_list = checkout_information.additional_json_data.get(
+                "abacus_transmission_list", []
+            )
+            abacus_transmission_list.append(
+                {
+                    "invoice_filename": invoice_xml_filename,
+                    "customer_filename": customer_xml_filename,
+                    "transmitted_at": checkout_information.invoice_transmitted_at.isoformat(),
+                }
+            )
+            checkout_information.additional_json_data["abacus_transmission_list"] = (
+                abacus_transmission_list
+            )
+
             checkout_information.save()
         return True
 
@@ -45,6 +82,7 @@ def abacus_ssh_upload(checkout_information: CheckoutInformation):
             checkout_information_id=checkout_information.id,
             exception=str(e),
             exc_info=True,
+            label="abacus_ssh_upload",
         )
         return False
 

server/vbv_lernwelt/shop/management/commands/abacus_export_job.py

@@ -0,0 +1,45 @@
+import structlog
+
+from vbv_lernwelt.core.base import LoggedCommand
+from vbv_lernwelt.shop.invoice.abacus import (
+    abacus_ssh_upload,
+    filter_checkout_information_for_abacus_upload_qs,
+)
+
+logger = structlog.get_logger(__name__)
+
+
+class Command(LoggedCommand):
+    help = "Export `CheckoutInformation` invoices to Abacus"
+
+    def handle(self, *args, **options):
+        logger.info(
+            "abacus_export_job started",
+            label="abacus_ssh_upload",
+        )
+
+        stat_results = {
+            "num_invoices": 0,
+            "num_uploaded": 0,
+            "num_error": 0,
+        }
+
+        abacus_ci_qs = filter_checkout_information_for_abacus_upload_qs()
+        stat_results["num_invoices"] = abacus_ci_qs.count()
+
+        for checkout_info in abacus_ci_qs:
+            abacus_ssh_upload_result = abacus_ssh_upload(checkout_info)
+
+            if abacus_ssh_upload_result:
+                stat_results["num_uploaded"] += 1
+            else:
+                stat_results["num_error"] += 1
+
+        self.job_log.json_data = stat_results
+        self.job_log.save()
+
+        logger.info(
+            "abacus_export_job finished",
+            label="abacus_ssh_upload",
+            stat_results=stat_results,
+        )

server/vbv_lernwelt/shop/migrations/0018_checkoutinformation_additional_json_data.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.13 on 2024-08-30 13:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("shop", "0017_checkoutinformation_chosen_profile"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="checkoutinformation",
+            name="additional_json_data",
+            field=models.JSONField(blank=True, default=dict),
+        ),
+    ]

server/vbv_lernwelt/shop/migrations/0019_alter_checkoutinformation_refno2.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.13 on 2024-08-30 13:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("shop", "0018_checkoutinformation_additional_json_data"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="checkoutinformation",
+            name="refno2",
+            field=models.CharField(blank=True, default="", max_length=255),
+        ),
+    ]

server/vbv_lernwelt/shop/models.py

@@ -62,7 +62,7 @@ class CheckoutInformation(models.Model):
 
     invoice_transmitted_at = models.DateTimeField(blank=True, null=True)
     transaction_id = models.CharField(max_length=255)
-    refno2 = models.CharField(max_length=255)
+    refno2 = models.CharField(max_length=255, blank=True, default="")
 
     # end user (required)
     first_name = models.CharField(max_length=255)
@@ -116,6 +116,8 @@ class CheckoutInformation(models.Model):
     abacus_order_id = models.BigIntegerField(unique=True, null=True, blank=True)
     abacus_ssh_upload_done = models.BooleanField(default=False)
 
+    additional_json_data = models.JSONField(default=dict, blank=True)
+
     def set_increment_abacus_order_id(self):
         if self.abacus_order_id:
             return self

server/vbv_lernwelt/shop/tests/test_abacus_invoice.py

@@ -1,6 +1,8 @@
 from datetime import date, datetime
 
 from django.test import TestCase
+from django.utils import timezone
+from freezegun import freeze_time
 
 from vbv_lernwelt.core.admin import User
 from vbv_lernwelt.core.create_default_users import create_default_users
@@ -8,9 +10,11 @@ from vbv_lernwelt.core.model_utils import add_countries
 from vbv_lernwelt.shop.invoice.abacus import (
     create_customer_xml,
     create_invoice_xml,
+    filter_checkout_information_for_abacus_upload_qs,
     render_customer_xml,
     render_invoice_xml,
 )
+from vbv_lernwelt.shop.models import CheckoutState
 from vbv_lernwelt.shop.tests.factories import CheckoutInformationFactory
 
 USER_USERNAME = "testuser"
@@ -23,6 +27,44 @@ class AbacusInvoiceTestCase(TestCase):
         add_countries(small_set=True)
         create_default_users()
 
+    def test_filter_checkout_information_for_abacus_upload(self):
+        feuz = User.objects.get(username="andreas.feuz@eiger-versicherungen.ch")
+
+        with freeze_time("2024-02-15 08:33:12"):
+            feuz_checkout_info = CheckoutInformationFactory(
+                user=feuz,
+                transaction_id="24021508331287484",
+                first_name="Andreas",
+                last_name="Feuz",
+                street="Eggersmatt",
+                street_number="32",
+                postal_code="1719",
+                city="Zumholz",
+                country_id="CH",
+                invoice_address="prv",
+                state=CheckoutState.ONGOING,
+            )
+
+            qs = filter_checkout_information_for_abacus_upload_qs()
+            self.assertEqual(qs.count(), 0)
+
+            feuz_checkout_info.state = CheckoutState.PAID
+            feuz_checkout_info.save()
+            qs = filter_checkout_information_for_abacus_upload_qs()
+            self.assertEqual(qs.count(), 1)
+
+            feuz_checkout_info.abacus_ssh_upload_done = True
+            feuz_checkout_info.invoice_transmitted_at = timezone.now()
+            feuz_checkout_info.save()
+            qs = filter_checkout_information_for_abacus_upload_qs()
+            self.assertEqual(qs.count(), 0)
+
+        with freeze_time("2024-02-15 09:44:44"):
+            feuz_checkout_info.first_name = "Peter"
+            feuz_checkout_info.save()
+            qs = filter_checkout_information_for_abacus_upload_qs()
+            self.assertEqual(qs.count(), 1)
+
     def test_create_invoice_xml(self):
         # set abacus_number before
         _pat = User.objects.get(username="patrizia.huggel@eiger-versicherungen.ch")

server/vbv_lernwelt/shop/views.py

@@ -167,6 +167,7 @@ def checkout_vv(request):
     checkout_info.set_increment_abacus_order_id()
 
     refno2 = f"{request.user.abacus_debitor_number}_{VV_PRODUCT_NUMBER}"
+    checkout_info.refno2 = refno2
 
     try:
         datatrans_customer_data = None