From c89914107f87e638cbd723f1654f4193226afff0 Mon Sep 17 00:00:00 2001 From: Daniel Egger Date: Fri, 31 May 2024 16:42:42 +0200 Subject: [PATCH] Add sftp test server code --- .gitignore | 3 +- server/config/settings/base.py | 11 ++- server/config/settings/test_cypress.py | 4 +- server/config/urls.py | 6 +- server/requirements/requirements-dev.in | 3 + server/requirements/requirements-dev.txt | 14 ++-- .../shop/datatrans_fake_server.py | 11 +-- server/vbv_lernwelt/shop/invoice/abacus.py | 79 +++++++++++-------- server/vbv_lernwelt/shop/invoice/creator.py | 21 ----- .../vbv_lernwelt/shop/invoice/repositories.py | 44 ----------- ...013_checkoutinformation_abacus_order_id.py | 2 +- server/vbv_lernwelt/shop/services.py | 11 +-- start_sftpserver.sh | 10 +++ 13 files changed, 88 insertions(+), 131 deletions(-) delete mode 100644 server/vbv_lernwelt/shop/invoice/creator.py delete mode 100644 server/vbv_lernwelt/shop/invoice/repositories.py create mode 100755 start_sftpserver.sh diff --git a/.gitignore b/.gitignore index 5eb128d8..a838654f 100644 --- a/.gitignore +++ b/.gitignore @@ -278,10 +278,11 @@ cypress/test-reports git-crypt-encrypted-files-check.txt - /server/vbv_lernwelt/static/css/tailwind.css /server/vbv_lernwelt/static/vue/ /server/vbv_lernwelt/static/storybook /server/vbv_lernwelt/templates/vue/index.html /server/vbv_lernwelt/media /client/src/gql/dist/minifiedSchema.json + +/sftptest/ diff --git a/server/config/settings/base.py b/server/config/settings/base.py index 29c1f8b3..a94b145c 100644 --- a/server/config/settings/base.py +++ b/server/config/settings/base.py @@ -682,10 +682,13 @@ else: "DATATRANS_PAY_URL", default="https://pay.sandbox.datatrans.com" ) -# Only for debugging the webhook (locally) -DATATRANS_DEBUG_WEBHOOK_OVERWRITE = env( - "DATATRANS_DEBUG_WEBHOOK_OVERWRITE", default=None -) + +# default settings for python sftpserver test-server +ABACUS_EXPORT_SFTP_HOST = env("ABACUS_EXPORT_SFTP_HOST", default="localhost") +ABACUS_EXPORT_SFTP_PORT = env("ABACUS_EXPORT_SFTP_PORT", default="3373") +ABACUS_EXPORT_SFTP_USERNAME = env("ABACUS_EXPORT_SFTP_USERNAME", default="admin") +ABACUS_EXPORT_SFTP_PASSWORD = env("ABACUS_EXPORT_SFTP_PASSWORD", default="admin") + # S3 BUCKET CONFIGURATION FILE_UPLOAD_STORAGE = env("FILE_UPLOAD_STORAGE", default="s3") # local | s3 diff --git a/server/config/settings/test_cypress.py b/server/config/settings/test_cypress.py index 08ea70d2..10932526 100644 --- a/server/config/settings/test_cypress.py +++ b/server/config/settings/test_cypress.py @@ -14,8 +14,8 @@ from .base import * # noqa # https://docs.djangoproject.com/en/dev/ref/settings/#secret-key DATABASES["default"]["NAME"] = "vbv_lernwelt_cypress" -DATATRANS_API_ENDPOINT = 'http://localhost:8001/server/fakeapi/datatrans/api' -DATATRANS_PAY_URL = 'http://localhost:8001/server/fakeapi/datatrans/pay' +DATATRANS_API_ENDPOINT = "http://localhost:8001/server/fakeapi/datatrans/api" +DATATRANS_PAY_URL = "http://localhost:8001/server/fakeapi/datatrans/pay" # EMAIL # ------------------------------------------------------------------------------ diff --git a/server/config/urls.py b/server/config/urls.py index d7f64dc1..8d4dc3fa 100644 --- a/server/config/urls.py +++ b/server/config/urls.py @@ -64,14 +64,14 @@ from vbv_lernwelt.importer.views import ( ) from vbv_lernwelt.media_files.views import user_image from vbv_lernwelt.notify.views import email_notification_settings -from wagtail import urls as wagtail_urls -from wagtail.admin import urls as wagtailadmin_urls -from wagtail.documents import urls as media_library_urls from vbv_lernwelt.shop.datatrans_fake_server import ( fake_datatrans_api_view, fake_datatrans_pay_view, ) +from wagtail import urls as wagtail_urls +from wagtail.admin import urls as wagtailadmin_urls +from wagtail.documents import urls as media_library_urls class SignedIntConverter(IntConverter): diff --git a/server/requirements/requirements-dev.in b/server/requirements/requirements-dev.in index de6fc888..07fcb99b 100644 --- a/server/requirements/requirements-dev.in +++ b/server/requirements/requirements-dev.in @@ -41,3 +41,6 @@ truffleHog # deployement and CI git+https://github.com/iterativ/Caprover-API.git@5013f8fc929e8e3281b9d609e968a782e8e99530 + +# sftpserver for tests +git+https://github.com/lonetwin/sftpserver.git@1d16896d3f0f90d63d1caaf4e199f2a9dde6456f diff --git a/server/requirements/requirements-dev.txt b/server/requirements/requirements-dev.txt index 5cb8055c..affd3c35 100644 --- a/server/requirements/requirements-dev.txt +++ b/server/requirements/requirements-dev.txt @@ -341,7 +341,9 @@ packaging==23.1 # pytest # pytest-sugar paramiko==3.3.1 - # via -r requirements.in + # via + # -r requirements.in + # sftpserver parso==0.8.3 # via jedi pathspec==0.11.2 @@ -397,9 +399,7 @@ pyflakes==3.1.0 pygments==2.16.1 # via ipython pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt + # via msal pylint==2.17.5 # via # pylint-django @@ -480,6 +480,8 @@ sendgrid==6.10.0 # via -r requirements.in sentry-sdk==1.29.2 # via -r requirements.in +sftpserver @ git+https://github.com/lonetwin/sftpserver.git@1d16896d3f0f90d63d1caaf4e199f2a9dde6456f + # via -r requirements-dev.in six==1.16.0 # via # asttokens @@ -621,9 +623,7 @@ wheel==0.41.1 whitenoise[brotli]==6.5.0 # via -r requirements.in willow[heif]==1.6.1 - # via - # wagtail - # willow + # via wagtail wrapt==1.15.0 # via astroid diff --git a/server/vbv_lernwelt/shop/datatrans_fake_server.py b/server/vbv_lernwelt/shop/datatrans_fake_server.py index 699363a3..4e36616f 100644 --- a/server/vbv_lernwelt/shop/datatrans_fake_server.py +++ b/server/vbv_lernwelt/shop/datatrans_fake_server.py @@ -107,9 +107,10 @@ def fake_datatrans_pay_view(request, api_url=""): def call_transaction_complete_webhook( webhook_url, transaction_id, datatrans_status="settled" ): - import requests import time + import requests + time.sleep(1) payload = { @@ -119,14 +120,14 @@ def fake_datatrans_pay_view(request, api_url=""): key_hex_bytes = bytes.fromhex(settings.DATATRANS_HMAC_KEY) # Create sign with timestamp and payload - sign = hmac.new(key_hex_bytes, bytes(str(1) + json.dumps(payload), "utf-8"), hashlib.sha256) + sign = hmac.new( + key_hex_bytes, bytes(str(1) + json.dumps(payload), "utf-8"), hashlib.sha256 + ) response = requests.post( url=webhook_url, json=payload, - headers={ - "Datatrans-Signature": f"t=1,s0={sign.hexdigest()}" - }, + headers={"Datatrans-Signature": f"t=1,s0={sign.hexdigest()}"}, ) print(response) diff --git a/server/vbv_lernwelt/shop/invoice/abacus.py b/server/vbv_lernwelt/shop/invoice/abacus.py index 747d343b..982488a0 100644 --- a/server/vbv_lernwelt/shop/invoice/abacus.py +++ b/server/vbv_lernwelt/shop/invoice/abacus.py @@ -1,47 +1,28 @@ import datetime -from uuid import uuid4 +from io import BytesIO from xml.dom import minidom from xml.etree.ElementTree import Element, SubElement, tostring -from vbv_lernwelt.shop.invoice.creator import InvoiceCreator, Item -from vbv_lernwelt.shop.invoice.repositories import InvoiceRepository +import structlog +from django.conf import settings +from paramiko.client import AutoAddPolicy, SSHClient + from vbv_lernwelt.shop.models import CheckoutInformation -class AbacusInvoiceCreator(InvoiceCreator): - def __init__(self, repository: InvoiceRepository): - self.repository = repository +logger = structlog.get_logger(__name__) - def create_invoice( - self, - checkout_information: CheckoutInformation, - filename: str = None, - ): - customer_number = checkout_information.transaction_id - order_date = checkout_information.created_at.date() - reference_purchase_order = str(checkout_information.id) - unic_id = checkout_information.transaction_id - items = [ - Item( - product_number=checkout_information.product_sku, - quantity=1, - description=checkout_information.product_description, - ) - ] +def abacus_ssh_upload(checkout_information: CheckoutInformation): + invoice_xml_filename, invoice_xml_content = create_invoice_xml(checkout_information) + customer_xml_filename, customer_xml_content = create_customer_xml( + checkout_information + ) - invoice = self.render_invoice_xml( - customer_number, - order_date, - reference_purchase_order, - unic_id, - items, - ) - - if filename is None: - filename = f"vbv-vv-{uuid4().hex}.xml" - - self.repository.upload_invoice(invoice, filename) + abacus_ssh_upload_invoice( + customer_xml_filename, customer_xml_content, folder="debitor" + ) + abacus_ssh_upload_invoice(invoice_xml_filename, invoice_xml_content, folder="order") def create_invoice_xml(checkout_information: CheckoutInformation): @@ -223,3 +204,33 @@ def create_xml_string(container: Element, encoding: str = "utf-8") -> str: indent=" ", encoding=encoding ) return xml_pretty_str.decode(encoding) + + +def abacus_ssh_upload_invoice( + filename: str, content_xml: str, folder: str = "" +) -> None: + invoice_io = BytesIO(content_xml.encode("utf-8")) + ssh_client = SSHClient() + + print(invoice_io) + + try: + ssh_client.set_missing_host_key_policy(AutoAddPolicy()) + ssh_client.connect( + hostname=settings.ABACUS_EXPORT_SFTP_HOST, + port=settings.ABACUS_EXPORT_SFTP_PORT, + username=settings.ABACUS_EXPORT_SFTP_USERNAME, + password=settings.ABACUS_EXPORT_SFTP_PASSWORD, + ) + + with ssh_client.open_sftp() as sftp_client: + path = filename + if folder: + path = f"{folder}/{filename}" + + sftp_client.putfo(invoice_io, path) + + except Exception as e: + logger.error("Could not upload invoice", exc_info=e) + finally: + ssh_client.close() diff --git a/server/vbv_lernwelt/shop/invoice/creator.py b/server/vbv_lernwelt/shop/invoice/creator.py deleted file mode 100644 index 85c08475..00000000 --- a/server/vbv_lernwelt/shop/invoice/creator.py +++ /dev/null @@ -1,21 +0,0 @@ -from abc import ABC, abstractmethod -from dataclasses import dataclass - -from vbv_lernwelt.shop.models import CheckoutInformation - - -@dataclass -class Item: - product_number: str - quantity: int - description: str - - -class InvoiceCreator(ABC): - @abstractmethod - def create_invoice( - self, - checkout_information: CheckoutInformation, - filename: str = None, - ): - pass diff --git a/server/vbv_lernwelt/shop/invoice/repositories.py b/server/vbv_lernwelt/shop/invoice/repositories.py deleted file mode 100644 index df231324..00000000 --- a/server/vbv_lernwelt/shop/invoice/repositories.py +++ /dev/null @@ -1,44 +0,0 @@ -from abc import ABC, abstractmethod - -import structlog - -logger = structlog.get_logger(__name__) - - -class InvoiceRepository(ABC): - @abstractmethod - def upload_invoice(self, invoice: str, filename: str): - pass - - -class SFTPInvoiceRepository(InvoiceRepository): - def __init__(self, hostname: str, username: str, password: str, port: int = 22): - self.hostname = hostname - self.username = username - self.password = password - self.port = port - - def upload_invoice(self, invoice: str, filename: str) -> None: - from io import BytesIO - - from paramiko import AutoAddPolicy, SSHClient - - invoice_io = BytesIO(invoice.encode("utf-8")) - ssh_client = SSHClient() - - try: - ssh_client.set_missing_host_key_policy(AutoAddPolicy()) - ssh_client.connect( - self.hostname, - port=self.port, - username=self.username, - password=self.password, - ) - - with ssh_client.open_sftp() as sftp_client: - sftp_client.putfo(invoice_io, f"uploads/{filename}") - - except Exception as e: - logger.error("Could not upload invoice", exc_info=e) - finally: - ssh_client.close() diff --git a/server/vbv_lernwelt/shop/migrations/0013_checkoutinformation_abacus_order_id.py b/server/vbv_lernwelt/shop/migrations/0013_checkoutinformation_abacus_order_id.py index 3073b3db..9aed218c 100644 --- a/server/vbv_lernwelt/shop/migrations/0013_checkoutinformation_abacus_order_id.py +++ b/server/vbv_lernwelt/shop/migrations/0013_checkoutinformation_abacus_order_id.py @@ -105,6 +105,6 @@ class Migration(migrations.Migration): name="old_company_country", ), migrations.DeleteModel( - name='BillingAddress', + name="BillingAddress", ), ] diff --git a/server/vbv_lernwelt/shop/services.py b/server/vbv_lernwelt/shop/services.py index bad18346..a5aaef4d 100644 --- a/server/vbv_lernwelt/shop/services.py +++ b/server/vbv_lernwelt/shop/services.py @@ -53,13 +53,6 @@ def init_datatrans_transaction( redirect_url_cancel: str, webhook_url: str, ): - if overwrite := settings.DATATRANS_DEBUG_WEBHOOK_OVERWRITE: - logger.warning( - "APPLYING DEBUG DATATRANS WEBHOOK OVERWRITE!", - webhook_url=overwrite, - ) - webhook_url = overwrite - payload = { # We use autoSettle=True, so that we don't have to settle the transaction: # -> Be aware that autoSettle has implications of the possible transaction states @@ -77,8 +70,8 @@ def init_datatrans_transaction( } # add testing configuration data - if 'fakeapi' in settings.DATATRANS_API_ENDPOINT: - payload['user_id'] = str(user.id) + if "fakeapi" in settings.DATATRANS_API_ENDPOINT: + payload["user_id"] = str(user.id) logger.info("Initiating transaction", payload=payload) diff --git a/start_sftpserver.sh b/start_sftpserver.sh new file mode 100755 index 00000000..4ae79d2a --- /dev/null +++ b/start_sftpserver.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# set location to script directory +cd "${0%/*}" + +# start python sftp test server (for abacus exports) +rm -rf sftptest +mkdir -p sftptest/debitor +mkdir -p sftptest/order +(cd sftptest && sftpserver -p 3373)