Add indexes to the database.

Reviewed-on: #59
Co-authored-by: Krzysiej <[email protected]>
Co-committed-by: Krzysiej <[email protected]>

Dockerfile

@@ -1,4 +1,4 @@
-FROM php:8.3-cli
+FROM php:8.4-cli
 
 WORKDIR /usr/src/app
 ENV PHP_MEMORY_LIMIT=1500M

bin/cachewarmup

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+bin/cli php console.php app:cache:warm-twig

bin/composer

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
-[ -z "$1" ] && echo "Please specify a Composer command (ex. install)" && exit
+[ -z "$1" ] && bin/cli composer list && exit
 bin/cli composer "$@"

bin/update

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 echo "Updating project"
 git pull origin master
-bin/cli rm -rf var/cache
+bin/cacheclean
+bin/composer install --no-dev
 echo "Project updated"

src/Command/CacheWarmCommand.php

@@ -16,7 +16,7 @@ use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\HttpFoundation\RequestStack;
 use Twig\Environment;
 
-#[AsCommand(name: 'app:cache:warm-twig', description: '')]
+#[AsCommand(name: 'app:cache:warm-twig', description: 'Warmup twig cache')]
 class CacheWarmCommand extends Command
 {
     public function __construct(

src/Command/Migrate.php

@@ -6,11 +6,13 @@ namespace Krzysiej\RyobiCrawler\Command;
 
 use Illuminate\Database\Capsule\Manager as Capsule;
 use Illuminate\Database\Schema\Blueprint;
+use Krzysiej\RyobiCrawler\Models\Country;
 use Symfony\Component\Console\Attribute\AsCommand;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
+use function Symfony\Component\Clock\now;
 
 #[AsCommand(name: 'app:migrate', description: 'Create database and rum migrations')]
 class Migrate extends Command
@@ -24,11 +26,10 @@ class Migrate extends Command
 
     public function execute(InputInterface $input, OutputInterface $output): int
     {
-        if (true === $input->hasOption(self::RECREATE_OPTION)) {
+        if ($input->getOption(self::RECREATE_OPTION)) {
             unlink(__DIR__ . '/../../database.sqlite');
-            //sleep(5);
-        }
             touch(__DIR__ . '/../../database.sqlite');
+        }
         $capsule = new Capsule;
         $capsule->addConnection([
             'driver' => 'sqlite',
@@ -39,6 +40,9 @@ class Migrate extends Command
         $this->createProductsTable();
         $this->createPricesTable();
         $this->createStocksTable();
+        $this->addColumns();
+        $this->createCountriesTable();
+        $this->index();
 
         return Command::SUCCESS;
     }
@@ -77,6 +81,51 @@ class Migrate extends Command
         }
     }
 
+    public function createCountriesTable(): void
+    {
+        if (!Capsule::schema()->hasTable('countries')) {
+            Capsule::schema()->create('countries', function (Blueprint $table) {
+                $table->increments('id');
+                $table->text('countryName');
+                $table->text('productsUrl');
+                $table->text('cultureCode');
+                $table->text('currency');
+                $table->text('locale');
+                $table->timestamps();
+            });
+        }
+        if (Capsule::schema()->hasTable('countries') && !Country::where('countryName', 'Poland')->exists()) {
+            $id = Capsule::table('countries')->insertGetId(
+                [
+                    'countryName' => 'Poland',
+                    'productsUrl' => 'https://pl.ryobitools.eu/api/product-listing/get-products',
+                    'cultureCode' => 'pl-PL',
+                    'currency' => 'PLN',
+                    'locale' => 'pl',
+                    'created_at' => now(),
+                    'updated_at' => now(),
+                ]);
+        }
+        if (Capsule::schema()->hasTable('countries') && !Country::where('countryName', 'UK')->exists()) {
+            Capsule::table('countries')->insert([
+                    'countryName' => 'UK',
+                    'productsUrl' => 'https://uk.ryobitools.eu/api/product-listing/get-products',
+                    'cultureCode' => 'en-GB',
+                    'currency' => 'GBP',
+                    'locale' => 'uk',
+                    'created_at' => now(),
+                    'updated_at' => now(),
+                ]
+            );
+        }
+
+        if (!Capsule::schema()->hasColumn('products', 'country_id')) {
+            Capsule::schema()->table('products', function (Blueprint $table) use ($id) {
+                $table->foreignId('country_id')->default($id)->references('id')->on('countries');
+            });
+        }
+    }
+
     public function createStocksTable(): void
     {
         if (!Capsule::schema()->hasTable('stocks')) {
@@ -94,4 +143,110 @@ class Migrate extends Command
             });
         }
     }
+
+    public function addColumns(): void
+    {
+        if (!Capsule::schema()->hasColumn('products', 'priceCurrent')) {
+            Capsule::schema()->table('products', function (Blueprint $table) {
+                $table->float('priceCurrent')->default(0);
+            });
+        }
+
+        if (!Capsule::schema()->hasColumn('products', 'priceLowest')) {
+            Capsule::schema()->table('products', function (Blueprint $table) {
+                $table->float('priceLowest')->default(0);
+            });
+        }
+
+        if (!Capsule::schema()->hasColumn('products', 'productStandardPrice')) {
+            Capsule::schema()->table('products', function (Blueprint $table) {
+                $table->float('productStandardPrice')->default(0);
+            });
+        }
+
+        if (!Capsule::schema()->hasColumn('products', 'lowestProductPrice30Days')) {
+            Capsule::schema()->table('products', function (Blueprint $table) {
+                $table->float('lowestProductPrice30Days')->default(0);
+            });
+        }
+
+        if (!Capsule::schema()->hasColumn('products', 'lastSeen')) {
+            Capsule::schema()->table('products', function (Blueprint $table) {
+                $table->date('lastSeen')->nullable();
+            });
+        }
+
+        if (!Capsule::schema()->hasColumn('products', 'stock')) {
+            Capsule::schema()->table('products', function (Blueprint $table) {
+                $table->integer('stock')->nullable();
+            });
+        }
+        if (!Capsule::schema()->hasColumn('products', 'promotions')) {
+            Capsule::schema()->table('products', function (Blueprint $table) {
+                $table->json('promotions')->nullable();
+            });
+        }
+    }
+
+    public function index(): void
+    {
+        if (!$this->hasIndex('products_skuid_country_id_unique')) {
+            Capsule::schema()->table('products', function (Blueprint $table) {
+                if ($this->hasIndex('products_skuid_unique')) {
+                    $table->integer('skuID')->unique(false)->change();
+                }
+                $table->unique(['skuID', 'country_id']);
+            });
+        }
+
+        if (!$this->hasIndex('prices_product_id_index')) {
+            Capsule::schema()->table('prices', function (Blueprint $table) {
+                $table->index('product_id');
+            });
+        }
+
+        if (!$this->hasIndex('products_id_index')) {
+            Capsule::schema()->table('products', function (Blueprint $table) {
+                $table->index('id');
+            });
+        }
+
+        if (!$this->hasIndex('stocks_product_id_stock_index')) {
+            Capsule::schema()->table('stocks', function (Blueprint $table) {
+                $table->index(['product_id', 'stock']);
+            });
+        }
+
+        if (!$this->isFK('products', 'id', 'stocks', 'product_id')) {
+            Capsule::schema()->table('products', function (Blueprint $table) {
+                $table->foreign('id')->references('product_id')->on('stocks');
+            });
+        }
+
+        if (!$this->isFK('products', 'id', 'prices', 'product_id')) {
+            Capsule::schema()->table('products', function (Blueprint $table) {
+                $table->foreign('id')->references('product_id')->on('prices');
+            });
+        }
+        if (!$this->isFK('products', 'country_id', 'countries', 'id')) {
+            Capsule::schema()->table('products', function (Blueprint $table) {
+                $table->foreign('country_id')->references('id')->on('countries');
+            });
+        }
+    }
+
+    private function isFK(string $table, string $column, string $fTable, string $fColumn): bool
+    {
+        $fkColumns = Capsule::schema()->getForeignKeys($table);
+
+        return !empty(array_filter($fkColumns, fn($fkColumn) => ($fkColumn['foreign_table'] == $fTable &&
+            in_array($fColumn, $fkColumn['foreign_columns']) &&
+            in_array($column, $fkColumn['columns']))
+        ));
+    }
+
+    private function hasIndex(string $indexName): bool
+    {
+        return !!count(Capsule::select('SELECT name FROM sqlite_master WHERE type = "index" and name = ?', [$indexName]));
+    }
 }

src/Command/ScrapeWebsite.php

@@ -7,6 +7,7 @@ namespace Krzysiej\RyobiCrawler\Command;
 use GuzzleHttp\Client;
 use GuzzleHttp\Exception\GuzzleException;
 use Illuminate\Database\Capsule\Manager as Capsule;
+use Krzysiej\RyobiCrawler\Models\Country;
 use Krzysiej\RyobiCrawler\Models\Price;
 use Krzysiej\RyobiCrawler\Models\Product;
 use Krzysiej\RyobiCrawler\Models\Stock;
@@ -21,47 +22,73 @@ class ScrapeWebsite extends Command
 {
     private Client $client;
 
+    public function __construct(protected Capsule $database)
+    {
+        parent::__construct();
+    }
+
     protected function configure(): void
     {
-        $capsule = new Capsule;
-        $capsule->addConnection([
-            'driver' => 'sqlite',
-            'database' => __DIR__ . '/../../database.sqlite',
-        ]);
-        $capsule->setAsGlobal();
-        $capsule->bootEloquent();
         $this->client = new Client();
     }
 
     public function execute(InputInterface $input, OutputInterface $output): int
     {
+        $output->writeln('Scrape products');
         $progress = new ProgressBar($output);
+        $countries = Country::all();
+        foreach($countries as $country) {
+            $output->writeln('Country name: ' . $country->countryName);
             $progress->start();
-        $products = $this->getProducts();
+            $products = $this->getProducts($country);
             $progress->setMaxSteps(count($products));
             foreach ($products as $product) {
-            $this->saveProduct($product);
+                $this->saveProduct($product, $country);
                 $progress->advance();
             }
             $progress->finish();
             $output->writeln('');
-        $output->writeln('DONE');
+            $output->writeln('Scrape products - DONE');
+            $output->writeln('');
+        }
+
+        $output->writeln('Update prices');
+        $products = Product::all();
+        $progress->setMaxSteps(count($products));
+        $progress->start();
+        foreach($products as $product) {
+            $newestPrice = $product->newestPrice;
+            $currentStock = $product->currentStock;
+            $product->priceCurrent = $newestPrice->price;
+            $product->productStandardPrice = $newestPrice->productStandardPrice;
+            $product->lowestProductPrice30Days = $newestPrice->lowestProductPrice30Days;
+            $product->priceLowest = $product->lowestPrice->price;
+            $product->lastSeen = $newestPrice->created_at->format('Y-m-d');
+            $product->stock = $currentStock->stock;
+            $product->save(['timestamps' => false]);
+            $progress->advance();
+        }
+        $progress->finish();
+        $output->writeln('');
+        $output->writeln('Update prices - DONE');
+        $output->writeln('COMMAND - DONE');
 
         return Command::SUCCESS;
     }
 
-    private function getProducts(): array
+    private function getProducts(Country $country): array
     {
         $products = [];
         $page = 0;
+
         do {
             try {
-                $res = $this->client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [
+                $res = $this->client->request('POST', $country->productsUrl, [
                     'form_params' => [
                         "includePreviousPages" => false,
                         "pageIndex" => $page,
                         "pageSize" => 100,
-                        "cultureCode" => "pl-PL",
+                        "cultureCode" => $country->cultureCode,
                     ]
                 ]);
                 $responseObject = json_decode($res->getBody()->getContents());
@@ -76,21 +103,24 @@ class ScrapeWebsite extends Command
         return $products;
     }
 
-    private function saveProduct(\stdClass $product): void
+    private function saveProduct(\stdClass $product, Country $country): void
     {
         /** @var Product $productModel */
-        $productModel = Product::firstOrNew(['skuID' => $product->skuID]);
+        $productModel = Product::firstOrNew(['skuID' => $product->skuID, 'country_id' => $country->id]);
 
         $productModel->skuID = $product->skuID;
         $productModel->name = $product->name;
         $productModel->availableQuantity = $product->availableQuantity;
-        $productModel->categories = $product->categories;
+        $productModel->categories = str_replace(['/', '  '], '', $product->categories);
         $productModel->image = $product->image;
         $productModel->subTitle = $product->subTitle;
         $productModel->variantCode = $product->variantCode;
         $productModel->modelCode = $product->modelCode;
         $productModel->url = $product->url;
+        $productModel->lastSeen = date("Y-m-d");
         $productModel->touch('updated_at');
+        $productModel->country()->associate($country);
+        $productModel->promotions = $product->promotions;
         $productModel->save();
         $priceExists = $productModel->price()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();
 

src/Controller/BaseController.php

@@ -11,11 +11,7 @@ class BaseController extends AbstractController
 {
     protected Environment $twig;
 
-    public function __construct(protected FilesystemAdapter $cache)
+    public function __construct(protected FilesystemAdapter $cache, protected Capsule $database)
     {
-        $capsule = new Capsule;
-        $capsule->addConnection(['driver' => 'sqlite', 'database' => __DIR__ . '/../../database.sqlite']);
-        $capsule->setAsGlobal();
-        $capsule->bootEloquent();
     }
 }

src/Controller/CategoryController.php

@@ -8,22 +8,43 @@ use Symfony\Component\Routing\Attribute\Route;
 
 final class CategoryController extends BaseController
 {
-    #[Route('/category/{category}', name: 'app_category')]
+    #[Route('/category/{category?}',  name: 'app_category')]
-    public function __invoke(string $category): Response
+    public function __invoke(?string $category): Response
     {
         if($this->cache->getItem('list_category_'.$category)->isHit()) {
             return $this->render('productList.html.twig', ['listType' => 'category_'.$category]);
         }
 
+        /** @var Product[] $products */
         $products = Product::with(['price', 'lowestPrice'])
             ->selectRaw('products.*')
             ->distinct('products.id')
+            ->when(!is_null($category) , fn ($q) => $q->whereRaw('json_each.value = ?', [$category]))
             ->fromRaw('products, json_each(products.categories)')
-            ->whereRaw('json_each.value = ?', [$category])
             ->orderByDesc('starred')
             ->orderByDesc('created_by')
             ->get();
 
-        return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'category_'.$category]);
+        $categoriesTree = [];
+
+        foreach ($products as $product) {
+            $categoriesTree = $this->addToTree($product->categories, $categoriesTree);
+        }
+
+        return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'category_' . $category, 'category' => $category, 'categoryTree' => $categoriesTree]);
+    }
+
+    private function addToTree(array $categories, mixed $categoriesTree)
+    {
+        $tmp = &$categoriesTree;
+        foreach ($categories as $category) {
+            if (empty($tmp[$category])) {
+                $tmp[$category] = ['count' => 0];
+            }
+            $tmp = &$tmp[$category];
+            $tmp['count']++;
+        }
+
+        return $categoriesTree;
     }
 }

src/Controller/DiscontinuedController.php

@@ -17,10 +17,9 @@ final class DiscontinuedController extends BaseController
             return $this->render('productList.html.twig', ['listType' => 'discontinued']);
         }
 
-        $products = Product::where('updated_at', '<', now()->format('Y-m-d'))
+        $products = Product::where('lastSeen', '<', now()->format('Y-m-d'))
             ->orderByDesc('starred')
             ->orderByDesc('created_by')
-            ->with(['currentPrice', 'lowestPrice'])
             ->get();
         return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'discontinued']);
     }

src/Controller/IndexController.php

@@ -14,11 +14,11 @@ final class IndexController extends BaseController
         if ($this->cache->getItem('list_all')->isHit()) {
             return $this->render('productList.html.twig', ['listType' => 'all']);
         }
-        $products = Product::with(['currentStock', 'price', 'lowestPrice'])
+        $products = Product::orderByDesc('starred')
-            ->orderByDesc('starred')
             ->orderByDesc('created_by')
             ->get();
 
+
         return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'all']);
     }
 }

src/Controller/LowestPriceController.php

@@ -0,0 +1,29 @@
+<?php
+
+namespace Krzysiej\RyobiCrawler\Controller;
+
+use Krzysiej\RyobiCrawler\Models\Product;
+use Symfony\Component\HttpFoundation\Response;
+use Symfony\Component\Routing\Attribute\Route;
+use function Symfony\Component\Clock\now;
+
+final class LowestPriceController extends BaseController
+{
+    #[Route('/lowest-price', name: 'app_lowest_price', methods: ['GET'])]
+    public function __invoke(): Response
+    {
+        $listType = 'lowest_price';
+        if($this->cache->getItem('lowest_price')->isHit()) {
+            return $this->render('productList.html.twig', ['listType' => $listType]);
+        }
+
+        $products = Product::whereRaw('priceCurrent = priceLowest')
+            ->whereRaw('lastSeen = "'.now()->format('Y-m-d').'"')
+            ->whereRaw('priceCurrent < productStandardPrice')
+            ->orderByDesc('starred')
+            ->orderByDesc('created_by')
+            ->with(['currentPrice', 'lowestPrice'])
+            ->get();
+        return $this->render('productList.html.twig', ['products' => $products, 'listType' => $listType]);
+    }
+}

src/Controller/NewController.php

@@ -2,7 +2,6 @@
 
 namespace Krzysiej\RyobiCrawler\Controller;
 
-use Illuminate\Database\Eloquent\Builder;
 use Krzysiej\RyobiCrawler\Models\Product;
 use Symfony\Component\HttpFoundation\Response;
 use Symfony\Component\Routing\Attribute\Route;
@@ -17,11 +16,11 @@ final class NewController extends BaseController
         if($this->cache->getItem('list_new')->isHit()) {
             return $this->render('productList.html.twig', ['listType' => 'new']);
         }
-
+        $date = now()->modify('-30 days')->format('Y-m-d');
-        $products = Product::where('created_at', '>', now()->modify('-30 days')->format('Y-m-d'))
+        $products = Product::where('created_at', '>', $date)
+            ->with(['country', 'stock'])
             ->orderByDesc('starred')
             ->orderByDesc('created_by')
-            ->with(['currentPrice', 'lowestPrice'])
             ->get();
         return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'new']);
     }

src/Controller/ProductController.php

@@ -3,7 +3,6 @@
 namespace Krzysiej\RyobiCrawler\Controller;
 
 use Krzysiej\RyobiCrawler\Models\Product;
-use Symfony\Component\Cache\Adapter\FilesystemAdapter;
 use Symfony\Component\HttpFoundation\Response;
 use Symfony\Component\Routing\Attribute\Route;
 use Twig\Error\LoaderError;
@@ -31,26 +30,34 @@ final class ProductController extends BaseController
         if (null === $product) {
             throw $this->createNotFoundException('Product not found');
         }
-        $priceList = $product->price()->pluck('price');
+        $priceList = $product->price()->pluck('price', 'created_at')->mapWithKeys(fn($price, $createdAt) => [explode(' ', $createdAt)[0] => $price])->toArray();
-        $stockList = $product->stock()->pluck('stock');
+        $stockList = $product->stock()->pluck('stock', 'created_at')->mapWithKeys(fn($stock, $createdAt) => [explode(' ', $createdAt)[0] => $stock])->toArray();
-        $priceDates = $product->price()->pluck('created_at')->map(fn($date) => $date->format('Y-m-d'))->toArray();
-        $stockDates = $product->stock()->pluck('created_at')->map(fn($date) => $date->format('Y-m-d'))->toArray();
-
         return $this->render('product.html.twig', [
             'product' => $product,
-            'price_list' => $this->prepareChartData($priceDates, $priceList),
+            'price_list' => $this->prepareChartData($priceList),
-            'stock_list' => $this->prepareChartData($stockDates, $stockList),
+            'stock_list' => $this->prepareChartData($stockList),
-            'price_dates' => implode("','", $priceDates),
+            'price_dates' => implode("','", $this->dateRange(array_key_first($priceList), array_key_last($priceList))),
         ]);
     }
 
-    private function prepareChartData($set1, $set2): string
+    private function prepareChartData($set1): string
     {
-        $data = [];
+        $dates = $this->dateRange(array_key_first($set1), array_key_last($set1));
-        foreach ($set1 as $key => $value) {
+        $data = array_map(fn($date) => ['x' => $date, 'y' => $set1[$date] ?? null], $dates);
-            $data[] = ['x' => $value, 'y' => $set2[$key]];
-        }
         $stringData = json_encode($data);
+
         return str_replace(['"x"', '"y"'], ['x', 'y'], $stringData);
     }
+
+    private function dateRange($dateStart, $dateEnd): array
+    {
+        $from = new \DateTime($dateStart);
+        $to = new \DateTime($dateEnd);
+        $range = [];
+        for ($date = clone $from; $date < $to; $date->modify('+1 day')) {
+            $range[] = $date->format('Y-m-d');
+        }
+
+        return $range;
+    }
 }

src/Controller/PromosController.php

@@ -2,25 +2,31 @@
 
 namespace Krzysiej\RyobiCrawler\Controller;
 
-use Illuminate\Database\Eloquent\Builder;
 use Krzysiej\RyobiCrawler\Models\Product;
 use Symfony\Component\HttpFoundation\Response;
 use Symfony\Component\Routing\Attribute\Route;
 
 final class PromosController extends BaseController
 {
-    #[Route('/promos', name: 'app_promos')]
+    #[Route('/promos/{promo?}', name: 'app_promos')]
-    public function __invoke(): Response
+    public function __invoke(?string $promo): Response
     {
         if ($this->cache->getItem('list_promos')->isHit()) {
-            return $this->render('productList.html.twig', ['listType' => 'promos']);
+            return $this->render('productList.html.twig', ['listType' => 'promos' . $promo]);
         }
 
-        $products = Product::whereHas('currentPrice', fn(Builder $query) => $query->whereColumn('price', '<', 'productStandardPrice'))
+        $products = Product::when(is_null($promo), fn($q) => $q->whereRaw('priceCurrent < productStandardPrice'))
             ->orderByDesc('starred')
             ->orderByDesc('created_by')
             ->with(['currentPrice', 'lowestPrice'])
+            ->when(!is_null($promo), fn($q) => $q->whereRaw("json_extract(promotions, '$.slug') LIKE ?", $promo))
             ->get();
-        return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'promos']);
+
+
+        $promos = Product::select($this->database->getConnection()->raw("distinct json_extract(promotions, '$.slug') as slug, json_extract(promotions, '$.tag') as tag"))
+            ->whereRaw("json_extract(promotions, '$.tag') is not null")
+            ->get();
+
+        return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'promos' . $promo, 'promos' => $promos->toArray()]);
     }
 }

src/Controller/SearchController.php

@@ -17,6 +17,6 @@ final class SearchController extends BaseController
             ->orWhere([['name', 'like', "%$search%"]])
             ->orWhere([['subTitle', 'like', "%$search%"]])->get();
 
-        return $this->render('productList.html.twig', ['products' => $products ?? [], 'search' => $search]);
+        return $this->render('productList.html.twig', ['products' => $products ?? [], 'search' => $search, 'listType' => 'search_'.$search]);
     }
 }

src/Controller/StarController.php

@@ -12,7 +12,18 @@ final class StarController extends BaseController
     #[Route('/star/{productId<\d+>}', name: 'app_star')]
     public function __invoke(int $productId, Request $request): Response
     {
+        $this->cache->deleteItems(['list_all', 'list_promos', 'list_new', 'list_discontinued']);
+        $referer = $request->headers->get('referer');
+        if (str_contains($referer, '/category/')) {
+            preg_match('#/category/(.*)#i', $referer, $matches);
+            $this->cache->deleteItem('list_category_'.urldecode($matches[1]));
+        }
+        if (str_contains($referer, '/search?search=')) {
+            preg_match('#/search\?search=(.*)#i', $referer, $matches);
+            $this->cache->deleteItem('list_search_'.urldecode($matches[1]));
+        }
+
         Product::find($productId)->toggleStarred()->save();
-        return $this->redirect($request->headers->get('referer'));
+        return $this->redirect($referer);
     }
 }

src/DatabaseFactory.php

@@ -0,0 +1,14 @@
+<?php
+
+namespace Krzysiej\RyobiCrawler;
+
+use Illuminate\Database\Capsule\Manager as Capsule;
+class DatabaseFactory
+{
+    public static function create(Capsule $capsule): void
+    {
+        $capsule->addConnection(['driver' => 'sqlite', 'database' => __DIR__ . '/../database.sqlite']);
+        $capsule->setAsGlobal();
+        $capsule->bootEloquent();
+    }
+}

src/Kernel.php

@@ -2,6 +2,7 @@
 
 namespace Krzysiej\RyobiCrawler;
 
+use Illuminate\Database\Capsule\Manager;
 use Krzysiej\RyobiCrawler\Twig\AppExtension;
 use Symfony\Bundle\FrameworkBundle\FrameworkBundle;
 use Symfony\Bundle\FrameworkBundle\Kernel\MicroKernelTrait;
@@ -34,8 +35,9 @@ class Kernel extends BaseKernel
             'secret' => 'S0ME_SECRET'
         ]);
         $services = $container->services()->defaults()->autowire()->autoconfigure();
+        $services->set(Manager::class)->configurator([DatabaseFactory::class, 'create']);
         $services->load('Krzysiej\\RyobiCrawler\\', __DIR__ )
-            ->exclude('../src/{Models,Twig,Kernel.php}');
+            ->exclude('../src/{Models,Twig,DatabaseFactory.php,Kernel.php}');
         $services->set('twig.extension.cache', AppExtension::class)->tag('twig.extension');
         $services->set(CacheExtension::class)->tag('twig.extension');
         $services->set(FilesystemAdapter::class)->args([

src/Models/Country.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace Krzysiej\RyobiCrawler\Models;
+
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Relations\BelongsTo;
+use Illuminate\Database\Eloquent\Relations\HasMany;
+
+/**
+ * @property string $countryName
+ * @property string $productsUrl
+ * @property string $cultureCode
+ * @property string $currency
+ * @property string $locale
+ */
+class Country extends Model
+{
+    public $timestamps = true;
+
+    public function products(): HasMany
+    {
+        return $this->hasMany(Product::class);
+    }
+}

src/Models/Product.php

@@ -2,11 +2,14 @@
 
 namespace Krzysiej\RyobiCrawler\Models;
 
+use Carbon\Traits\Date;
 use Illuminate\Database\Eloquent\Casts\Attribute;
 use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Relations\BelongsTo;
 use Illuminate\Database\Eloquent\Relations\HasMany;
 use Illuminate\Database\Eloquent\Relations\HasOne;
 
+use Illuminate\Support\Str;
 use function Symfony\Component\Clock\now;
 
 /**
@@ -20,11 +23,23 @@ use function Symfony\Component\Clock\now;
  * @property string $modelCode
  * @property string $url
  * @property int $starred
+ * @property float $priceCurrent
+ * @property float $priceLowest
+ * @property float $productStandardPrice
+ * @property float $lowestProductPrice30Days
+ * @property Date $lastSeen
+ * @property integer $stock
+ * @property Object $promotions
  */
 class Product extends Model
 {
     public $timestamps = true;
-    public $fillable = ['skuID'];
+    public $fillable = ['skuID', 'country_id'];
+
+    public function country(): BelongsTo
+    {
+        return $this->belongsTo(Country::class);
+    }
 
     public function price(): HasMany
     {
@@ -46,6 +61,11 @@ class Product extends Model
         return $this->hasOne(Price::class)->ofMany('price', 'MIN');
     }
 
+    public function newestPrice(): HasOne
+    {
+        return $this->hasOne(Price::class)->latest()->take(1);
+    }
+
     public function stock(): HasMany
     {
         return $this->hasMany(Stock::class);
@@ -53,9 +73,7 @@ class Product extends Model
 
     public function currentStock(): HasOne
     {
-        return $this->stock()->one()->ofMany()->withDefault(function (Stock $stock) {
+        return $this->stock()->one()->ofMany()->withDefault(fn(Stock $stock) => $stock->stock = 0)->take(1);
-            $stock->stock = 0;
-        });
     }
 
     public function toggleStarred(): self
@@ -73,10 +91,23 @@ class Product extends Model
         );
     }
 
+    public function promotions(): Attribute
+    {
+        return Attribute::make(
+            get: fn(?string $value) => json_decode($value ?? '{"hasPromotion": false}', 1),
+            set: function (\stdClass $value) {
+                $value->slug = Str::slug($value->tag);
+
+                return json_encode($value);
+            }
+        );
+    }
+
     public function isDiscontinued(): bool
     {
-        return $this->updated_at->format('Y-m-d') < now()->format('Y-m-d');
+        return $this->lastSeen < now()->format('Y-m-d');
     }
+
     public function isNew(): bool
     {
         return $this->created_at->format('Y-m-d') > now()->modify('-30 days')->format('Y-m-d');

src/Twig/AppExtension.php

@@ -2,11 +2,11 @@
 
 namespace Krzysiej\RyobiCrawler\Twig;
 
-use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\Collection;
 use Krzysiej\RyobiCrawler\Models\Price;
 use Krzysiej\RyobiCrawler\Models\Product;
 use Krzysiej\RyobiCrawler\Models\Stock;
+use Symfony\Component\Routing\RouterInterface;
 use Twig\Extension\AbstractExtension;
 use Twig\TwigFilter;
 use Twig\TwigFunction;
@@ -15,6 +15,10 @@ use function Symfony\Component\Clock\now;
 
 class AppExtension extends AbstractExtension
 {
+    public function __construct(public RouterInterface $route)
+    {
+    }
+
     public function getFunctions(): array
     {
         return [
@@ -22,6 +26,8 @@ class AppExtension extends AbstractExtension
             new TwigFunction('allCount', [$this, 'allCount']),
             new TwigFunction('newCount', [$this, 'newCount']),
             new TwigFunction('discontinuedCount', [$this, 'discontinuedCount']),
+            new TwigFunction('lowestPriceCount', [$this, 'lowestPriceCount']),
+            new TwigFunction('renderCategoryTree', [$this, 'renderCategoryTree']),
         ];
     }
 
@@ -31,6 +37,7 @@ class AppExtension extends AbstractExtension
             new TwigFilter('findByCreatedAtDate', [$this, 'findByCreatedAtDate']),
         ];
     }
+
     public function allCount(): int
     {
         return Product::count();
@@ -38,7 +45,7 @@ class AppExtension extends AbstractExtension
 
     public function promosCount(): int
     {
-        return Product::whereHas('currentPrice', fn(Builder $query) => $query->whereColumn('price', '<', 'productStandardPrice'))->count();
+        return Product::whereRaw('priceCurrent < productStandardPrice')->count();
     }
 
     public function newCount(): int
@@ -48,11 +55,43 @@ class AppExtension extends AbstractExtension
 
     public function discontinuedCount(): int
     {
-        return Product::where('updated_at', '<', now()->format('Y-m-d'))->count();
+        return Product::where('lastSeen', '<>', now()->format('Y-m-d'))->count();
+    }
+
+    public function lowestPriceCount(): int
+    {
+        return Product::whereRaw('priceCurrent = priceLowest')
+            ->whereRaw('lastSeen = "' . now()->format('Y-m-d') . '"')
+            ->whereRaw('priceCurrent < productStandardPrice')
+            ->count();
     }
 
     public function findByCreatedAtDate(Collection $items, string $date): Stock|Price|null
     {
         return $items->first(fn($item) => str_starts_with($item->created_at, $date));
     }
+
+    public function renderCategoryTree($categories, $current, $level = 0): string
+    {
+        $tree = '';
+        if ($level == 0) {
+            $tree .= '<ul class="list-group list-group-flush">';
+        }
+        foreach ($categories as $categoryName => $category) {
+            $currentClass = $categoryName == $current ? 'list-group-item-primary' : '';
+            $tree .= '<a class="list-group-item list-group-item-action '.$currentClass.' text-decoration-none ms-' . ($level * 2) . '" href="' . $this->route->generate('app_category', ['category' => $categoryName]) . '">' . $categoryName . ' <span class="badge bg-primary rounded-pill">' . $category['count'] . '</span></a>';
+
+            unset($category['count']);
+            if (is_array($category) && count($category) >= 1) {
+                foreach ($category as $subcategoryName => $subCategory) {
+                    $tree .= $this->renderCategoryTree([$subcategoryName => $subCategory], $current, $level + 1);
+                }
+            }
+        }
+        if ($level == 0) {
+            $tree .= '</ul>';
+        }
+        return $tree;
+
+    }
 }

templates/product.html.twig

@@ -10,7 +10,8 @@
                 <td><img src='{{ product.image }}&width=150' class='border rounded p-1' alt='{{ product.name }}'/></td>
                 <td>
                     <a href='{{ path('app_product', {'productId': product.id}) }}' class="text-decoration-none">{{ product.name }}</a>
-                    <span class="badge text-bg-light">{{ product.subTitle }}</span>
+                    <span class="badge text-bg-light"><a href="{{ path('app_search', {'search': product.subTitle}) }}" class="link-underline link-underline-opacity-0 link-dark">{{ product.subTitle }}</a></span>
+                    {% if product.promotions is not null and product.promotions.hasPromotion %}<a href="{{ path('app_promos', {'promo': product.promotions.slug}) }}"><span class="badge bg-info">PROMO: {{ product.promotions.tag }}</span></a>{% endif %}
                 </td>
                 <td>
                     <nav aria-label="breadcrumb" style="--bs-breadcrumb-divider: '>';">
@@ -22,7 +23,7 @@
                         </ol>
                     </nav>
                 </td>
-                <td><a href='https://pl.ryobitools.eu/{{ product.url }}'>link</a></td>
+                <td><a href='https://{{ product.country.locale }}.ryobitools.eu{{ product.url }}'>link</a></td>
             </tr>
             <tr>
                 <td colspan="5">
@@ -42,13 +43,14 @@
                             <th>Stock</th>
                         </tr>
                         </thead>
+                        {% set stock = product.stock().get() %}
                         {% for price in product.price %}
                             <tr>
-                                <td>{{ price.price | format_currency('PLN', {}, 'pl') }}</td>
+                                <td>{{ price.price | format_currency(product.country.currency, {}, product.country.locale) }}</td>
-                                <td>{{ price.lowestProductPrice30Days | format_currency('PLN', {}, 'pl') }}</td>
+                                <td>{{ price.lowestProductPrice30Days | format_currency(product.country.currency, {}, product.country.locale) }}</td>
-                                <td>{{ price.productStandardPrice | format_currency('PLN', {}, 'pl') }}</td>
+                                <td>{{ price.productStandardPrice | format_currency(product.country.currency, {}, product.country.locale) }}</td>
                                 <td>{{ price.created_at }}</td>
-                                <td>{{ (product.stock | findByCreatedAtDate(price.created_at | slice(0,10))).stock ?? '' }}</td>
+                                <td>{{ (stock | findByCreatedAtDate(price.created_at | slice(0,10))).stock ?? '' }}</td>
                             </tr>
                         {% endfor %}
                     </table>
@@ -67,7 +69,7 @@
                 labels: ['{{ price_dates|raw }}'],
                 datasets: [
                     {
-                        label: 'Price (PLN)',
+                        label: 'Price ({{ product.country.currency }})',
                         data: {{ price_list|raw }},
                         yAxisID: 'yPrice',
                         tension: 0.1,
@@ -83,6 +85,7 @@
                 ]
             },
             options: {
+                spanGaps: false,
                 responsive: true,
                 animation: false,
                 scales: {
@@ -98,7 +101,7 @@
                         beginAtZero: true,
                         title: {
                             display: true,
-                            text: 'Price (PLN)'
+                            text: 'Price ({{ product.country.currency }})'
                         },
                         grid: {
                             drawOnChartArea: false

templates/productList.html.twig

@@ -2,6 +2,19 @@
 
 {% block content %}
     {% cache 'list_' ~ listType %}
+
+        {% if listType starts with 'category_' %}
+            {{ renderCategoryTree(categoryTree, category) | raw }}
+        {% endif %}
+
+        {% if listType starts with 'promos' %}
+            {% for promo in promos %}
+                <a href="{{ path('app_promos', {'promo': promo.slug}) }}"><span class="badge bg-info">PROMO: {{ promo.tag }}</span></a>
+            {% endfor %}
+        {% endif %}
+
+
+        {% if (listType starts with 'category_' and category == null)  or not (listType starts with 'category_') or (listType starts with 'category_' and category is not null)   %}
             <div class="table-responsive">
                 <table class='table table-hover'>
                     <thead>
@@ -19,40 +32,59 @@
                     {% for product in products %}
                         <tr>
                             <td class="align-middle font-weight-bold h3"><a class="text-warning text-decoration-none"
-                                                                href="{{ path('app_star', {'productId': product.id}) }}">{% if product.starred == true %}★{% else %} ☆ {% endif %}</a></td>
+                                                                            href="{{ path('app_star', {'productId': product.id}) }}">{% if product.starred == true %}★{% else %} ☆ {% endif %}</a>
-                <td class="align-middle" style="width: 120px;"><img src='{{ product.image }}&width=70' class='img-thumbnail' alt='{{ product.name }}'/></td>
+                            </td>
+                            <td class="align-middle" style="width: 120px;"><img src='{{ product.image }}&width=70'
+                                                                                class='img-thumbnail'
+                                                                                alt='{{ product.name }}'/></td>
                             <td class="align-middle">
-                    <a href='{{ path('app_product', {'productId': product.id}) }}' class="text-decoration-none">{{ product.name }}</a>
+                                <a href='{{ path('app_product', {'productId': product.id}) }}'
-                    {% if product.currentStock.stock > 0 %}
+                                   class="text-decoration-none">{{ product.name }}</a>
-                        <span class="badge text-bg-light">stock: {{ product.currentStock.stock }}</span>
+                                {% if product.stock > 0 %}
+                                    <span class="badge text-bg-light">stock: {{ product.stock }}</span>
                                 {% else %}
                                     <span class="badge text-bg-warning">out of stock</span>
                                 {% endif %}
                                 {% if product.isDiscontinued() %}
-                        <span class="badge text-bg-secondary"  data-bs-toggle="tooltip" data-bs-title="Last update: {{ product.updated_at }}">is discontinued</span>
+                                    <span class="badge text-bg-secondary" data-bs-toggle="tooltip"
+                                          data-bs-title="Last update: {{ product.lastSeen }}">is discontinued</span>
                                 {% endif %}
                                 {% if product.isNew() %}
                                     <span class="badge text-bg-success">is new</span>
                                 {% endif %}
-                    <span class="badge text-bg-light">{{ product.subTitle }}</span>
+                                <span class="badge text-bg-light"><a
+                                            href="{{ path('app_search', {'search': product.subTitle}) }}"
+                                            class="link-underline link-underline-opacity-0 link-dark">{{ product.subTitle }}</a></span>
+                                {% if product.promotions is not null and product.promotions.hasPromotion %}<a href="{{ path('app_promos', {'promo': product.promotions.slug}) }}"><span class="badge bg-info">PROMO: {{ product.promotions.tag }}</span></a>{% endif %}
                             </td>
                             <td class="align-middle">
                                 <nav aria-label="breadcrumb" style="--bs-breadcrumb-divider: '>';">
-                        <ol class="breadcrumb">
+                                    <ol class="breadcrumb mb-0">
                                         {% for category in product.categories %}
-                                <li class="breadcrumb-item" aria-current="page"><a class="breadcrumb-item text-decoration-none"  href="{{ path('app_category', {'category': category}) }}">{{ category }}</a></li>
+                                            <li class="breadcrumb-item" aria-current="page"><a
+                                                        class="breadcrumb-item text-decoration-none"
+                                                        href="{{ path('app_category', {'category': category}) }}">{{ category }}</a>
+                                            </li>
                                         {% endfor %}
                                     </ol>
                                 </nav>
                             </td>
-                <td class="align-middle"><a href='https://pl.ryobitools.eu/{{ product.url }}'>link</a></td>
+                            <td class="align-middle"><a href='https://{{ product.country.locale }}.ryobitools.eu{{ product.url }}'>link</a></td>
-                <td class="align-middle text-end">{% if product.lowestPrice.price != product.price.last.price  %}{{ product.lowestPrice.price | format_currency('PLN', {}, 'pl') }}{% endif %}</td>
+                            <td class="align-middle text-end">
-                <td class="align-middle text-end">{{ product.price.last.price | format_currency('PLN', {}, 'pl') }}</td>
+                                {% if product.isDiscontinued() or product.priceCurrent == product.productStandardPrice %}
+                                {{ product.priceLowest | format_currency(product.country.currency, {}, product.country.locale) }}
+                                {% else %}
+                                {% if product.priceLowest != product.priceCurrent %}{{ product.priceLowest | format_currency(product.country.currency, {}, product.country.locale) }}{% else %}
+                                    <span class="badge text-bg-info">now lowest</span>{% endif %}</td>
+                            {% endif %}
+
+                            <td class="align-middle text-end">{{ product.priceCurrent | format_currency(product.country.currency, {}, product.country.locale) }}</td>
                             <td class="align-middle">
                                 <div class="d-flex flex-row">
-                        {% if product.price.last.price != product.price.last.productStandardPrice %}<span
+                                    {% if product.priceCurrent != product.productStandardPrice %}<span
-                                class="badge text-bg-warning text-decoration-line-through flex-fill">{{ product.price.last.productStandardPrice | format_currency('PLN', {}, 'pl') }}</span>&nbsp;<span
+                                            class="badge text-bg-warning text-decoration-line-through flex-fill">{{ product.productStandardPrice | format_currency(product.country.currency, {}, product.country.locale) }}</span>
-                                class="badge text-bg-success flex-fill">{{ ((1 - product.price.last.price / product.price.last.productStandardPrice)*100)|number_format(0) }}%</span>
+                                        <span
+                                                class="badge text-bg-success flex-fill">{{ ((1 - product.priceCurrent / product.productStandardPrice)*100)|number_format(0) }}%</span>
                                     {% endif %}
                                 </div>
                             </td>
@@ -60,5 +92,6 @@
                     {% endfor %}
                 </table>
             </div>
+        {% endif %}
     {% endcache %}
 {% endblock %}

templates/template.html.twig

@@ -1,5 +1,5 @@
 <!doctype html>
-<html lang="en">
+<html lang="en" data-bs-theme="light">
 <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
@@ -18,13 +18,16 @@
         </button>
         <div class="collapse navbar-collapse" id="navbarNav">
             <ul class="navbar-nav me-auto mb-2 mb-lg-0">
-                {% cache "menu_count" %}
+                {% cache "menu_count" ~ listType|default('') %}
                     <li class="nav-item">
                         <a class="nav-link {% if app.request.pathinfo == path('app_home') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_home') }}">All products <span class="badge text-bg-secondary">{{ allCount() }}</span></a>
                     </li>
                     <li class="nav-item">
                         <a class="nav-link {% if app.request.pathinfo == path('app_promos') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_promos') }}">Promos <span class="badge text-bg-secondary">{{ promosCount() }}</span></a>
                     </li>
+                    <li class="nav-item">
+                        <a class="nav-link {% if app.request.pathinfo == path('app_lowest_price') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_lowest_price') }}">Lowest price <span class="badge text-bg-secondary">{{ lowestPriceCount() }}</span></a>
+                    </li>
                     <li class="nav-item">
                         <a class="nav-link {% if app.request.pathinfo == path('app_new') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_new') }}">New in last 30 days <span class="badge text-bg-secondary">{{ newCount() }}</span></a>
                     </li>
@@ -44,8 +47,7 @@
 
 </nav>
 {% block content %}{% endblock %}
-<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js" integrity="sha384-YvpcrYf0tY3lHB60NNkmXc5s9fDVZLESaAA55NDzOxhy9GkcIdslK1eN7N6jIeHz" crossorigin="anonymous"></script>
+<script src="/templates/js/bootstrap.bundle.min.js"></script>
-
 </body>
 </html>