From 0a5a6fa4ce2d6116438ff41e78e7d811cc5b3dcc Mon Sep 17 00:00:00 2001 From: Lorenz Padberg Date: Mon, 22 May 2023 14:56:50 +0200 Subject: [PATCH] Add azure blob storage as default media storage --- azure_deploy.sh | 4 ++ env_secrets/production_azure.env | Bin 2887 -> 2896 bytes server/backend/custom_azure.py | 8 ++-- server/config/settings/base.py | 12 +++--- server/requirements/requirements-dev.txt | 48 ++++++++++++++++++--- server/requirements/requirements.in | 4 ++ server/requirements/requirements.txt | 52 +++++++++++++++++++---- 7 files changed, 104 insertions(+), 24 deletions(-) create mode 100644 azure_deploy.sh diff --git a/azure_deploy.sh b/azure_deploy.sh new file mode 100644 index 00000000..54ce1861 --- /dev/null +++ b/azure_deploy.sh @@ -0,0 +1,4 @@ +npm install +python server/manage.py collectstatic --noinput +docker build --platform linux/amd64 -f compose/django/Dockerfile -t iterativ/vbv-lernwelt-django:azure . +docker image push iterativ/vbv-lernwelt-django:azure diff --git a/env_secrets/production_azure.env b/env_secrets/production_azure.env index a6c56c0b6d2b365e6227609961ed86309b809f59..f5900f62aaae2057973b0304494b54fb6a5b9084 100644 GIT binary patch literal 2896 zcmV-W3$OG5M@dveQdv+`0QFGoR+l`7FW0}x0|#WEed${n1NiI186A)1^b{sOKxL)` zth>n^5`S5bgIE#v9qRdB*$rGH1WwKLpf=|VddP7t%z;D&76~>kqAr0L&mUL2X}B>v zOTPKu#3CAL6w8>qAFt$$Jt&N#faOD(4fJ)rgc|O;{5Gl+(*m^|f%oUEWz$7Te|fZgvcZAZwXWg>v0FEwMLk6yo50FyKFuMucK zvCln$=Hi)dDeyHVguBBJSmhJFt6J|o|A`_&s?}aHTann8S#xs_ z!i^o_sZs4Be$6)dk$-BDL|#NBiG3TT(D6Z}U7zI>xCEECDUe75(Zy^YpP~p~N*I{@ z?WcMvBE2Cv+^-N)EaH$8aYRhIo5lc2xS8SJ$eA2QaCpfqJg_>|F|xjhc<^R!r;$;* zTVIiOp0`52(&F;Fs!Y8y0m;sLYRYjuRfJg?m(W3mx1=7XaP=!^?6wHOgf4#kX?11RwDyZcT)^7v8pI9v9j)nOIo453K;lqUUcpjEwsDg__MK>h@ zawkq%>HhSLp73HCMV9PA#)=w6x_!@_(r%k4#n2Ch6?%<6@rudcUJO|pPm%CJA}RHl zo^9Q+c{woxmG?qID_TO#D;DOzUJ&(Nsk)td55Nvk7)}e<)r80$pH@i->Da}+5e)n< z@r$5s$e)496ABIq**ExxXFXcwnUFy$wD3P*0gUV1I=-LtL7q9QK#0n+52#(9|QZ_=WK@7tCZoHSQ{osw9w1|C`^&dcu8 z|51(trI7W=?aAbdwrtTea-s0wl<1s_Nnk6k=-x6zx858FrJBXau{1BUWelH?#n!B- zc6ed-1K6vkbP*$VN6%ooNEsuaF@QCHQ1xxWq&kXMg+KqARtjVJ5LsjArq~0UQ<6eMH7#7AHQ8G-fF5mR(&7RD!$8)I-QVm2guTK*$u+h2!{{9t>;sLB)Jmkly#GfRY0w~Lrj z{B(*~zs>Z?RxTM^fkZj8PsYrV*$JJ2pK6#swtwg9>#AfyF5obr4=mx)Ka+(>{N#7r zGVVBmA(GPrN&|gGi3_TCD5b0s@q1@^iKjP5p$|NJ@!k=j2XGOBFJ$?qjgE5V8duil zOw8zv2kN|q#9EyqF@u$E1j@_eq>#tgeo(i->ZPc1>hyl|8>L^^Gtz+*lBeJJhwItD zGND7UqunQQu-O5?VGbrFzeWbbR$$NBWIekN#*0h+m=7qW+7Dvdt!;sPX`Fi$57fB*cAIH>ESPYB-IU{gdJvd!|RM(&ITNm87N6fi~ zNf+8L#yY{fRMuOZ+{zv*FxRh5WWcyrhEimus;ZtwaTit({fB`jyi?CAHm^}y|5fUF zBraIjGuKty&u!#nv|79_iR|S%;t;J#oR%vw>1SqCVul|(SGXG-jYzv!eH52!qJtYa zR?y1@gG#H7giMB_ms!qs93uZWs+2SP;j0zRq~AuBEi!sURWM#^Zfl_7htm|CBo^^S zdh!WQRjyKTSuvy`ldSFAf+xn)0jV}lB-9Oy5~kBhR$F0ZZ`^mRgm(;GCd6zxUy>Ca z;m zRop;`dwVur%7oUNG6{ONNPK&6tj&4`_W%|%-bcOLU$A&0Eo5v3NaI)G?fscC$Q|qXk@N`0g%F0}E^|o_Wz=3d{ zIsa?6V-My#bCSk+IxXXwut4XNFMnk^U8ueG(dQ+de7-xS*aXEC%SI)D%rUrXj!gdE zXoseUVHSZsGKi@n%q3zHm&=nHK1upJw5VY&xE6_!f>&F7Smfh&=feF(1~b4Q=){xT zD(}cnGC}Wy$YE@pvU8DFm&#WcwZT4hgyIV}b;jwY$D%9!g;c&x$+tVKiE;hdx}Ban z%2RDkhPrx9Z&wN&(!VLuVKv9EoYN--ri~CIBr;&tHfk`;A zo+UiQcM|-cNuf~;kY(r^M2zbt zHsKBt1J;kjA#!R?PrnOkV_@FirAX|EuyS!JWax8)%9hWht}hd$)=Z#$lk#^mR-^Iy zMoCo^R0Io>Kdf>Z}@#9wC2RV#^i*qUjQjg13Z_P z)ba5hmkwDyQRo`3s^{9CJK}91P)66W%(}HGx@9!gu=zN(yBa#+g=k0#5||dnm}VDb z6nky5+nr5?(V4fVdoxy@Iz1{CUnN*JEzwwby6&ggg6`YA_UN~(-n))2O%(dhkS~E- z-bL^_*L?h$=hWG#E>#Mkrr9u+G?psnrof99UZNL23d_{e<~usaflb`bvWp1RVQB%p zp^*lg7^LJC97!48AJQq|Rjq(RJt!_u*_nlk3v0BlbVL2A$N3(rYpi4tD;1n&>7b8E z&oO@fo93o-9YWLTtNvc%iXdSKzLJi)wFSbOW9vGXs?yP9=~L^Cs+F_AVWfw1YTdYI zk^01qM#FtxqI#F^jjg)L>e9j=9asBckNt|EwCmE_mIfX>t*_=sZi`xG42 z9o=>w(Oz4bynxxR7k>yf1AiUF}=BE_v5|CKforUo()J{9N+h}NE&;QJFn)K>Vqps^bO literal 2887 zcmV-N3%K+EM@dveQdv+`07(w>jw&@J%bw#f0G+<&I36nlnNw5TN&z7SS^-FQM-^|b zjHPJ5z3vSrSd-z*qA#$m-x?D-84bpMpPxd6ju^PVfQ}3SG5>OXauf69^zFiohDRJ_ z0mhE;areD`HaJn@LCIOxuD*D*nvqPqjMLt2d@H%UT4oaPVmPA(K&Gwgqb8#yS`2z1 zRzH)A2vR9G0C*`Qoab`NUFeN|rMUYS-cHT}<)Uvo?szml$N#h2i+{+zh<*hACWR5s zeYge$&f-}gN|xBzq^_7hRC`-bt&tb z=L;3dkGoy}ak=EZux*aSkd31DGI+@4jppL`WXyeYp;(CB%vUkEoNDzXGI;;*JLHmP zOg{2H3t3EDkhj%qy>gHfb%E!qW2qU_)pIGO>UJeqj>xtBeDh=w*EQ|4&)j9}De|S%c=r3;9M27byVIq98;K3oz<{YCyOtTU{f2tdruI zQoS9bp`rkh;hYDaR*?Bq{Yq)HQ0rjQ2_17hmpky6#2|ueT)lksSh#$@xVVd5AjScK z#ED{f%?|IBjrawQvGLWjb0F+4ZUWKM4m<#K!mw&Mc)OLA!vcw-BW@g&a{h`$IuA0> zZlINpJc7>8lnFZ41Ow*Ki`h4MN0=n@^z0I?F-R(v6iQfVsMwMbDdu|dxZ^P&Cg9r6 z$M9L+QX)QA?I{dL>qd((3qNK(o7`hEu)w0Nwucfk zkI=uA(0JOkveuqnZ5I+edu0Ys#|+AWqR;R~oD;bs4W=}K_NTT15bP=PWL5>rFu8>0 zy)E9)EvT=xtLOGmZ0Y=_Rw>_goSU1YIhb=Sqb zvfX>hKw)GcVee|h)92_T5r+!bm;BlXnVk0D$QF_h3yFkB>kW88%Yj9d(W{iu+`M(c zR6LldV%3f>6sRP73aa|7R!gA-n_x|K0-cWF14Qr!P<8hX$L)9oup=yBy!}S9GvJal z!#{0D_M?~mb5A0QDqieWT(E3^9W=s7p1S7C)W8}-$%zLs)qFG!U^21|%_O2e4wqrK zbA=uzbzOE$;&N%>d=Id9M0`^kixd>P-nu9*U{;9)p+P_J>rOQlW=GpT076&nnG7*_LJDD{23SDw`)C zYy|qCiKHmZlbA)f`YsQXo9!uJ@lK{X9HbNnQ2v0K1 z(P0K*+mS+R=3tM@rZui5hZd^O2CBM|hjxyayl=Eh#z zkUL}6@eA%7;gce2eq`R_Tug>CP#61W&-&lEq{(#mVi6)!w-8 z3GK~nf$fYN{BSJy;M2xB>k~J6mb7R@7dOb+o_r7CRwoZoyHLy*@xdR>hFH7MXkvmD0Oi)qtMvQ`+>D!*yo^gsLsm65e5b6D-;?BEU=<$BD~v-niP6X z6@z}oK8{)OgjH;>s+JIsA>(BU>Tcq(m7R;Z1oO}77d~X3qHo(cSmO`EcH1M5wUncz z^v=n|UD{72GOdclHDSjQq@N0x>3WX8Qqh(Lxmy*)ZV-*QzFC7HoRvp+S;+@8k+@=sxWS;-9~7!b}P|Hnk_5Q$eSc4t%Hi{$t-Wpxl+(hvLc|YRbqe zFzoaiW0N5lw(qmn0i6nO*tTf-L~a3>)Q&2!mVBkjmo?BVVt>S?NE;h}~mi z!36k{6MrBYWqb-F;A`x?CUt|0B$c(&w91JBr#zWmS~W7AeSO1wF|8^ zbuGVbC5(_L8*PB%L@WI51Nl~xfZx=RvpNq6irKJUtCbV<=Wdx615@Zbkfk_dJmb7l zkqF211f<*Fb*j*UprutwV1D3Jg@60$e$#XmH>;zXp%WYO^zH8^wnZ(iL*#;Cq5UF{u+oE$N`7Lc6g)ZAWhy zi&bCryh}IpO3Eo;mPC?CQsSuQ&w2Su?_qkI&V7@_J?o-ho!{_EjzN2u^Q`=xhEmpy zm&O*d_zg^~{WFU(%YmYvaDYoy^p5IN7#qw%%|uN!jcxFvXoE8zC z6begy!2+1!M({q5n}QR}SP6kaM!N1&O5c;NFhTS|By5cwqQ$k$uo25{M?%1YH>)c*@zVk zBI?ql%*Bd`U4eXN5_&tVX3HY`Xf(7R;))FurJ>U5dJ&r&23pcDLWn4?2y`uR7HELR zgIl3Oz6&TdP<^?=|3-?aau^UA~!w!JIXxLn;XgH?G8~67)1F4*;r%LFP z*6O6*4>pC)KpF*NPAvy#MM!QS1{k9F>>U@Ha80$5nvi{u;SDyJIkj2sXyBV~{K_CoJ*m?daa2 l8px#^;pcrty}WAmNHX+M%6bh#H+kf#dj@Q|?W#0a+sk|3uJQl? diff --git a/server/backend/custom_azure.py b/server/backend/custom_azure.py index fe0c9b08..1621f1c0 100644 --- a/server/backend/custom_azure.py +++ b/server/backend/custom_azure.py @@ -5,14 +5,14 @@ from environs import Env env = Env() env.read_env() class AzureMediaStorage(AzureStorage): - account_name = env("AZURE_STORAGE_ACCOUNT") - account_key = env("AZURE_STORAGE_KEY") + account_name = env("IT_AZURE_STORAGE_ACCOUNT") + account_key = env("IT_AZURE_STORAGE_KEY") azure_container = 'media' expiration_secs = None class AzureStaticStorage(AzureStorage): - account_name = env("AZURE_STORAGE_ACCOUNT") - account_key = env("AZURE_STORAGE_KEY") + account_name = env("IT_AZURE_STORAGE_ACCOUNT") + account_key = env("IT_AZURE_STORAGE_KEY") azure_container = 'static' expiration_secs = None diff --git a/server/config/settings/base.py b/server/config/settings/base.py index 3fc03441..4648133e 100644 --- a/server/config/settings/base.py +++ b/server/config/settings/base.py @@ -214,18 +214,18 @@ if USE_AWS: # https://wagtail.org/blog/amazon-s3-for-media-files/ MEDIA_URL = "https://%s/" % AWS_S3_CUSTOM_DOMAIN DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" -elif env(USE_AZURE_BLOB_STORAGE): +elif env("IT_USE_AZURE_BLOB_STORAGE", False): # https://medium.com/@DawlysD/django-using-azure-blob-storage-to-handle-static-media-assets-from-scratch-90cbbc7d56be DEFAULT_FILE_STORAGE = 'backend.custom_azure.AzureMediaStorage' - STATICFILES_STORAGE = 'backend.custom_azure.AzureStaticStorage' + #STATICFILES_STORAGE = 'backend.custom_azure.AzureStaticStorage' - STATIC_LOCATION = "static" + #STATIC_LOCATION = "static" MEDIA_LOCATION = "media" - AZURE_ACCOUNT_NAME = "djangoaccountstorage" + AZURE_ACCOUNT_NAME = env("IT_AZURE_STORAGE_ACCOUNT") AZURE_CUSTOM_DOMAIN = f'{AZURE_ACCOUNT_NAME}.blob.core.windows.net' - STATIC_URL = f'https://{AZURE_CUSTOM_DOMAIN}/{STATIC_LOCATION}/' - MEDIA_URL = f'https://{AZURE_CUSTOM_DOMAIN}/{MEDIA_LOCATION}/' + #STATIC_URL = f'https://{AZURE_CUSTOM_DOMAIN}/{APP_ENVIRONMENT}/{STATIC_LOCATION}/' + MEDIA_URL = f'https://{AZURE_CUSTOM_DOMAIN}/{APP_ENVIRONMENT}/{MEDIA_LOCATION}/' else: diff --git a/server/requirements/requirements-dev.txt b/server/requirements/requirements-dev.txt index f3875c8b..62594c31 100644 --- a/server/requirements/requirements-dev.txt +++ b/server/requirements/requirements-dev.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: # # pip-compile --output-file=requirements-dev.txt requirements-dev.in # @@ -10,6 +10,8 @@ anyascii==0.3.1 # via wagtail anyio==3.5.0 # via watchfiles +appnope==0.1.3 + # via ipython argon2-cffi==21.3.0 # via -r requirements.in argon2-cffi-bindings==21.2.0 @@ -29,6 +31,16 @@ attrs==21.4.0 # usort authlib==1.0.0 # via -r requirements.in +azure-core==1.26.4 + # via + # azure-identity + # azure-storage-blob +azure-identity==1.13.0 + # via -r requirements.in +azure-storage-blob==12.16.0 + # via + # -r requirements.in + # django-storages backcall==0.2.0 # via ipython beautifulsoup4==4.9.3 @@ -82,7 +94,12 @@ coverage==6.3.2 # -r requirements-dev.in # django-coverage-plugin cryptography==36.0.2 - # via authlib + # via + # authlib + # azure-identity + # azure-storage-blob + # msal + # pyjwt decorator==5.1.1 # via # ipdb @@ -155,7 +172,7 @@ django-ratelimit==3.0.1 # via -r requirements.in django-redis==5.2.0 # via -r requirements.in -django-storages==1.13.1 +django-storages[azure]==1.13.1 # via -r requirements.in django-stubs==1.10.1 # via @@ -241,6 +258,8 @@ ipdb==0.13.9 # via -r requirements-dev.in ipython==8.2.0 # via ipdb +isodate==0.6.1 + # via azure-storage-blob isort==5.10.1 # via # flake8-isort @@ -281,6 +300,12 @@ moreorless==0.4.0 # via # ufmt # usort +msal==1.22.0 + # via + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity mypy==0.942 # via # -r requirements-dev.in @@ -330,7 +355,9 @@ pluggy==1.0.0 polib==1.1.1 # via wagtail-localize portalocker==2.4.0 - # via concurrent-log-handler + # via + # concurrent-log-handler + # msal-extensions pre-commit==2.17.0 # via -r requirements-dev.in promise==2.3 @@ -353,6 +380,8 @@ pyflakes==2.4.0 # via flake8 pygments==2.11.2 # via ipython +pyjwt[crypto]==2.7.0 + # via msal pylint==2.13.4 # via # pylint-django @@ -409,9 +438,11 @@ redis==4.2.1 # django-redis requests==2.27.1 # via + # azure-core # caprover-api # coreapi # djangorestframework-stubs + # msal # wagtail s3transfer==0.6.0 # via boto3 @@ -422,8 +453,11 @@ sentry-sdk==1.5.8 six==1.16.0 # via # asttokens + # azure-core + # azure-identity # django-coverage-plugin # html5lib + # isodate # l18n # promise # python-dateutil @@ -490,8 +524,10 @@ types-pytz==2021.3.6 # via django-stubs types-pyyaml==6.0.5 # via django-stubs -typing-extensions==4.1.1 +typing-extensions==4.5.0 # via + # azure-core + # azure-storage-blob # django-stubs # django-stubs-ext # djangorestframework-stubs diff --git a/server/requirements/requirements.in b/server/requirements/requirements.in index cd4615ef..7910d91e 100644 --- a/server/requirements/requirements.in +++ b/server/requirements/requirements.in @@ -25,6 +25,7 @@ django-ratelimit django-ipware django-csp django-storages +django-storages[azure] django-notifications-hq django-jsonform @@ -42,4 +43,7 @@ wagtail-factories>=4 wagtail-localize>=1.5 wagtail_grapple>=0.19.2 +azure-storage-blob +azure-identity + boto3 diff --git a/server/requirements/requirements.txt b/server/requirements/requirements.txt index 230e6b7d..b3015ddd 100644 --- a/server/requirements/requirements.txt +++ b/server/requirements/requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: # # pip-compile --output-file=requirements.txt requirements.in # @@ -22,6 +22,16 @@ attrs==21.4.0 # via jsonschema authlib==1.0.0 # via -r requirements.in +azure-core==1.26.4 + # via + # azure-identity + # azure-storage-blob +azure-identity==1.13.0 + # via -r requirements.in +azure-storage-blob==12.16.0 + # via + # -r requirements.in + # django-storages beautifulsoup4==4.9.3 # via wagtail boto3==1.26.11 @@ -50,7 +60,12 @@ click==8.1.1 concurrent-log-handler==0.9.20 # via -r requirements.in cryptography==36.0.2 - # via authlib + # via + # authlib + # azure-identity + # azure-storage-blob + # msal + # pyjwt deprecated==1.2.13 # via redis dj-database-url==1.0.0 @@ -104,7 +119,7 @@ django-ratelimit==3.0.1 # via -r requirements.in django-redis==5.2.0 # via -r requirements.in -django-storages==1.13.1 +django-storages[azure]==1.13.1 # via -r requirements.in django-taggit==2.1.0 # via wagtail @@ -154,6 +169,8 @@ idna==3.3 # requests inflection==0.5.1 # via drf-spectacular +isodate==0.6.1 + # via azure-storage-blob jmespath==1.0.1 # via # boto3 @@ -166,6 +183,12 @@ l18n==2021.3 # via wagtail marshmallow==3.15.0 # via environs +msal==1.22.0 + # via + # azure-identity + # msal-extensions +msal-extensions==1.0.0 + # via azure-identity openpyxl==3.1.2 # via wagtail packaging==21.3 @@ -179,13 +202,17 @@ pillow==9.0.1 polib==1.1.1 # via wagtail-localize portalocker==2.4.0 - # via concurrent-log-handler + # via + # concurrent-log-handler + # msal-extensions promise==2.3 # via graphene-django psycopg2-binary==2.9.3 # via -r requirements.in pycparser==2.21 # via cffi +pyjwt[crypto]==2.7.0 + # via msal pyparsing==3.0.7 # via packaging pyrsistent==0.18.1 @@ -221,7 +248,10 @@ redis==4.2.1 # -r requirements.in # django-redis requests==2.27.1 - # via wagtail + # via + # azure-core + # msal + # wagtail s3transfer==0.6.0 # via boto3 sendgrid==6.9.7 @@ -230,7 +260,10 @@ sentry-sdk==1.5.8 # via -r requirements.in six==1.16.0 # via + # azure-core + # azure-identity # html5lib + # isodate # l18n # promise # python-dateutil @@ -252,8 +285,11 @@ text-unidecode==1.3 # via # graphene-django # python-slugify -typing-extensions==4.2.0 - # via wagtail-localize +typing-extensions==4.5.0 + # via + # azure-core + # azure-storage-blob + # wagtail-localize uritemplate==4.1.1 # via drf-spectacular urllib3==1.26.9