From 1a257341aaa2a946de89aaf4cc5c09fc6a7e9db4 Mon Sep 17 00:00:00 2001 From: mtyton Date: Wed, 26 Jul 2023 02:00:35 +0200 Subject: [PATCH 1/3] added product loading feature --- artel/artel/settings/base.py | 2 + artel/docgenerator/__init__.py | 0 artel/docgenerator/admin.py | 3 - artel/docgenerator/apps.py | 6 -- artel/docgenerator/generators.py | 44 --------- artel/docgenerator/migrations/__init__.py | 0 artel/docgenerator/models.py | 10 -- artel/docgenerator/tests.py | 5 - artel/docgenerator/views.py | 3 - artel/requirements.txt | 1 + artel/store/loader.py | 52 ++++++++++ .../management/commands/load_products.py | 13 +++ artel/store/models.py | 3 +- artel/store/tests/test_loader.py | 97 +++++++++++++++++++ 14 files changed, 167 insertions(+), 72 deletions(-) delete mode 100644 artel/docgenerator/__init__.py delete mode 100644 artel/docgenerator/admin.py delete mode 100644 artel/docgenerator/apps.py delete mode 100644 artel/docgenerator/generators.py delete mode 100644 artel/docgenerator/migrations/__init__.py delete mode 100644 artel/docgenerator/models.py delete mode 100644 artel/docgenerator/tests.py delete mode 100644 artel/docgenerator/views.py create mode 100644 artel/store/loader.py create mode 100644 artel/store/management/commands/load_products.py create mode 100644 artel/store/tests/test_loader.py diff --git a/artel/artel/settings/base.py b/artel/artel/settings/base.py index 9a423d2..285d9c8 100644 --- a/artel/artel/settings/base.py +++ b/artel/artel/settings/base.py @@ -228,3 +228,5 @@ LOGGING = { "level": "WARNING", }, } + +PRODUCTS_CSV_PATH = os.environ.get("PRODUCTS_CSV_PATH", "products.csv") diff --git a/artel/docgenerator/__init__.py b/artel/docgenerator/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/artel/docgenerator/admin.py b/artel/docgenerator/admin.py deleted file mode 100644 index 8c38f3f..0000000 --- a/artel/docgenerator/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/artel/docgenerator/apps.py b/artel/docgenerator/apps.py deleted file mode 100644 index 7c5f6d2..0000000 --- a/artel/docgenerator/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class DocgeneratorConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'docgenerator' diff --git a/artel/docgenerator/generators.py b/artel/docgenerator/generators.py deleted file mode 100644 index 7828e6b..0000000 --- a/artel/docgenerator/generators.py +++ /dev/null @@ -1,44 +0,0 @@ -from abc import ( - ABC, - abstractmethod -) -from typing import ( - Dict, - Any -) - -from django.db.models import Model -from docxtpl import DocxTemplate - - -class DocumentGeneratorInterface(ABC): - @abstractmethod - def load_template(self, path: str): - ... - - @abstractmethod - def get_extra_context(self) -> Dict[str, Any]: - ... - - @abstractmethod - def generate_file(self, context: Dict[str, Any] = None): - ... - - -class BaseDocumentGenerator(DocumentGeneratorInterface): - - def __init__(self, instance: Model) -> None: - super().__init__() - self.instance = instance - - def load_template(self, path: str): - return DocxTemplate(path) - - def get_extra_context(self): - return {} - - -class PdfFromDocGenerator(BaseDocumentGenerator): - def generate_file(self, context: Dict[str, Any] = None): - template = self.load_template() - context.update(self.get_extra_context()) diff --git a/artel/docgenerator/migrations/__init__.py b/artel/docgenerator/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/artel/docgenerator/models.py b/artel/docgenerator/models.py deleted file mode 100644 index f50168a..0000000 --- a/artel/docgenerator/models.py +++ /dev/null @@ -1,10 +0,0 @@ -from django.db import models -from django.core.files.storage import Storage - - -class DocumentTemplate(models.Model): - name = models.CharField(max_length=255) - file = models.FileField(upload_to="doc_templates/", ) - - def __str__(self) -> str: - return self.name diff --git a/artel/docgenerator/tests.py b/artel/docgenerator/tests.py deleted file mode 100644 index 7b37ab0..0000000 --- a/artel/docgenerator/tests.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.test import TestCase - - -class PdfFromDocGeneratorTestCase(TestCase): - ... \ No newline at end of file diff --git a/artel/docgenerator/views.py b/artel/docgenerator/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/artel/docgenerator/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/artel/requirements.txt b/artel/requirements.txt index 65539f3..2ea8975 100644 --- a/artel/requirements.txt +++ b/artel/requirements.txt @@ -10,3 +10,4 @@ factory-boy==3.2.1 pdfkit==1.0.0 num2words==0.5.12 sentry-sdk==1.28.0 +pandas==2.0.3 diff --git a/artel/store/loader.py b/artel/store/loader.py new file mode 100644 index 0000000..8933774 --- /dev/null +++ b/artel/store/loader.py @@ -0,0 +1,52 @@ +import logging +import pandas as pd + +from store.models import ( + ProductTemplate, + ProductCategoryParamValue, + Product +) + + +logger = logging.getLogger(__name__) + + +class BaseLoader: + def __init__(self, path): + self.path = path + + def load_data(self): + return pd.read_csv(self.path) + + +class ProductLoader(BaseLoader): + + def _process_row(self, row): + template = ProductTemplate.objects.get(code=row["template"]) + price = float(row["price"]) + name = row["name"] + available = bool(row["available"]) + params = [] + for param in row["params"]: + key, value = param + param = ProductCategoryParamValue.objects.get(param__key=key, value=value) + params.append(param) + product = Product.objects.get_or_create_by_params(template=template, params=params) + product.price = price + product.name = name + product.available = available + product.save() + return product + + def process(self): + data = self.load_data() + products = [] + for _, row in data.iterrows(): + try: + product = self._process_row(row) + except Exception as e: + # catch any error and log it, GlitchTip will catch it + logger.exception(str(e)) + else: + products.append(product) + logger.info(f"Loaded {len(products)} products") diff --git a/artel/store/management/commands/load_products.py b/artel/store/management/commands/load_products.py new file mode 100644 index 0000000..a277ad0 --- /dev/null +++ b/artel/store/management/commands/load_products.py @@ -0,0 +1,13 @@ +from django.core.management import BaseCommand +from django.conf import settings + +from store.loader import ProductLoader + + + +class Command(BaseCommand): + help = "Load products from csv file" + + def handle(self, *args, **options): + loader = ProductLoader(settings.PRODUCTS_CSV_PATH) + loader.process() diff --git a/artel/store/models.py b/artel/store/models.py index beef666..c38c31a 100644 --- a/artel/store/models.py +++ b/artel/store/models.py @@ -22,6 +22,7 @@ from django.template import ( ) from django.core.exceptions import ValidationError from django.db.models.signals import m2m_changed +from django.forms import CheckboxSelectMultiple from modelcluster.models import ClusterableModel from modelcluster.fields import ParentalKey @@ -217,7 +218,7 @@ class Product(ClusterableModel): panels = [ FieldPanel("template"), FieldPanel("price"), - FieldPanel("params"), + FieldPanel("params", widget=CheckboxSelectMultiple), FieldPanel("available"), FieldPanel("name"), InlinePanel("product_images", label="Variant Images"), diff --git a/artel/store/tests/test_loader.py b/artel/store/tests/test_loader.py new file mode 100644 index 0000000..9ee9c98 --- /dev/null +++ b/artel/store/tests/test_loader.py @@ -0,0 +1,97 @@ +import pandas as pd +from django.test import TestCase +from unittest.mock import patch + +from store.tests import factories +from store.loader import ProductLoader + + +class TestProductLoader(TestCase): + def setUp(self) -> None: + self.category = factories.ProductCategoryFactory() + self.template = factories.ProductTemplateFactory(category=self.category) + self.category_params = [factories.ProductCategoryParamFactory(category=self.category) for _ in range(3)] + self.category_param_values = [factories.ProductCategoryParamValueFactory(param=param) for param in self.category_params] + + def test_load_products_single_product_success(self): + fake_df = pd.DataFrame({ + "template": [self.template.code], + "price": [10.0], + "name": ["Test product"], + "available": [True], + "params": [[ + (self.category_params[0].key, self.category_param_values[0].value), + (self.category_params[1].key, self.category_param_values[1].value), + (self.category_params[2].key, self.category_param_values[2].value), + ]] + }) + with patch("store.loader.BaseLoader.load_data", return_value=fake_df): + loader = ProductLoader("fake_path") + loader.process() + + self.assertEqual(self.template.products.count(), 1) + product = self.template.products.first() + self.assertEqual(product.price, 10.0) + self.assertEqual(product.name, "Test product") + self.assertEqual(product.available, True) + + @patch("store.loader.logger") + def test_load_incorrect_data_types_failure(self, mock_logger): + fake_df = pd.DataFrame({ + "template": [self.template.code], + "price": ["FASDSADQAW"], + "name": ["Test product"], + "available": [True], + "params": [[ + (self.category_params[0].key, self.category_param_values[0].value), + (self.category_params[1].key, self.category_param_values[1].value), + (self.category_params[2].key, self.category_param_values[2].value), + ]] + }) + with patch("store.loader.BaseLoader.load_data", return_value=fake_df): + loader = ProductLoader("fake_path") + loader.process() + + self.assertEqual(self.template.products.count(), 0) + mock_logger.exception.assert_called_with("could not convert string to float: 'FASDSADQAW'") + + @patch("store.loader.logger") + def test_load_no_existing_template_code_failure(self, mock_logger): + fake_df = pd.DataFrame({ + "template": ["NOTEEXISTINGTEMPLATE"], + "price": [10.0], + "name": ["Test product"], + "available": [True], + "params": [[ + (self.category_params[0].key, self.category_param_values[0].value), + (self.category_params[1].key, self.category_param_values[1].value), + (self.category_params[2].key, self.category_param_values[2].value), + ]] + }) + with patch("store.loader.BaseLoader.load_data", return_value=fake_df): + loader = ProductLoader("fake_path") + loader.process() + + self.assertEqual(self.template.products.count(), 0) + mock_logger.exception.assert_called_with("ProductTemplate matching query does not exist.") + + @patch("store.loader.logger") + def test_not_existing_params_key_value_pairs_failure(self, mock_logger): + fake_df = pd.DataFrame({ + "template": [self.template.code], + "price": [10.0], + "name": ["Test product"], + "available": [True], + "params": [[ + (self.category_params[0].key, self.category_param_values[2].value), + (self.category_params[1].key, self.category_param_values[0].value), + (self.category_params[2].key, self.category_param_values[1].value), + ]] + }) + with patch("store.loader.BaseLoader.load_data", return_value=fake_df): + loader = ProductLoader("fake_path") + loader.process() + + self.assertEqual(self.template.products.count(), 0) + mock_logger.exception.assert_called_with("ProductCategoryParamValue matching query does not exist.") + \ No newline at end of file From 0a6544db1403d8c17a1b2f32f1ab8996a6e634f3 Mon Sep 17 00:00:00 2001 From: mtyton Date: Tue, 1 Aug 2023 22:41:10 +0200 Subject: [PATCH 2/3] smaller changes --- artel/store/loader.py | 16 +++++++++++++++- artel/store/models.py | 6 +++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/artel/store/loader.py b/artel/store/loader.py index 8933774..b4e2b6c 100644 --- a/artel/store/loader.py +++ b/artel/store/loader.py @@ -17,10 +17,20 @@ class BaseLoader: def load_data(self): return pd.read_csv(self.path) - + + +class TemplateLoader(BaseLoader): + ... + class ProductLoader(BaseLoader): + def _get_images(self, row): + urls = row["images"] + for url in urls: + ... + return None + def _process_row(self, row): template = ProductTemplate.objects.get(code=row["template"]) price = float(row["price"]) @@ -35,6 +45,10 @@ class ProductLoader(BaseLoader): product.price = price product.name = name product.available = available + + images = self._get_images(row) + for image in images: + product.images.add(image) product.save() return product diff --git a/artel/store/models.py b/artel/store/models.py index c38c31a..2919cf9 100644 --- a/artel/store/models.py +++ b/artel/store/models.py @@ -142,6 +142,7 @@ class ProductTemplate(ClusterableModel): title = models.CharField(max_length=255) code = models.CharField(max_length=255) description = models.TextField(blank=True) + # TODO - add mechanism for enabling params tags = TaggableManager() @@ -207,7 +208,8 @@ class Product(ClusterableModel): name = models.CharField(max_length=255, blank=True) template = models.ForeignKey(ProductTemplate, on_delete=models.CASCADE, related_name="products") params = models.ManyToManyField( - ProductCategoryParamValue, blank=True, through="ProductParam" + ProductCategoryParamValue, blank=True, through="ProductParam", + limit_choices_to=models.Q(param__category=models.F("product__template__category")) ) price = models.FloatField() available = models.BooleanField(default=True) @@ -296,6 +298,8 @@ class ProductListPage(Page): tags = TaggableManager(blank=True) def _get_items(self): + if not self.pk: + return ProductTemplate.objects.all() if self.tags.all(): return ProductTemplate.objects.filter(tags__in=self.tags.all()) return ProductTemplate.objects.all() From 9bdcb1bbf5289953360f74abae8c6de0e67c6c04 Mon Sep 17 00:00:00 2001 From: mtyton Date: Thu, 3 Aug 2023 21:05:54 +0200 Subject: [PATCH 3/3] added possibility to load remote images --- artel/store/loader.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/artel/store/loader.py b/artel/store/loader.py index b4e2b6c..8174711 100644 --- a/artel/store/loader.py +++ b/artel/store/loader.py @@ -1,10 +1,14 @@ import logging +import requests import pandas as pd +from django.core import files + from store.models import ( ProductTemplate, ProductCategoryParamValue, - Product + Product, + ProductImage ) @@ -25,11 +29,17 @@ class TemplateLoader(BaseLoader): class ProductLoader(BaseLoader): - def _get_images(self, row): + def _get_images(self, row) -> list[files.ContentFile]: urls = row["images"] + images = [] for url in urls: - ... - return None + response = requests.get(url, stream=True) + if response.status_code == 200: + data = response.raw + file_name = url.split("/")[-1] + image = files.ContentFile(data, name=file_name) + images.append(image) + return images def _process_row(self, row): template = ProductTemplate.objects.get(code=row["template"]) @@ -47,8 +57,8 @@ class ProductLoader(BaseLoader): product.available = available images = self._get_images(row) - for image in images: - product.images.add(image) + for i, image in enumerate(images): + ProductImage.objects.create(product=product, image=image, is_main=bool(i==0)) product.save() return product