Add update script.

.gitea/workflows/deploy.yaml

@@ -0,0 +1,15 @@
+on:
+  push:
+  workflow_dispatch:
+jobs:
+  deploy-job:
+    runs-on: server
+    steps:
+      - name: initial test
+        run: echo "test"
+      - name: get page
+        run: curl http://192.168.0.129:9001/update
+      - run: pwd
+      - run: cd ../ && pwd
+      - run: pwd
+      - run: cd /var/www/html/ryobi-crawler && bin/update

bin/cacheclean

@@ -1,2 +1,4 @@
 #!/usr/bin/env bash
+echo "Cleaning cache"
 bin/cli rm -rf var/cache
+echo "cache cleaned2"

composer.json

@@ -1,18 +1,18 @@
 {
     "require": {
-        "guzzlehttp/guzzle": "^7.0",
+        "guzzlehttp/guzzle": "^7",
         "illuminate/database": "11.26.0.0",
         "ext-json": "*",
-        "symfony/console": "^7.0",
-        "symfony/routing": "^7.1",
+        "symfony/console": "^8.0",
+        "symfony/routing": "^8.0",
         "laravel/serializable-closure": "^1.3",
-        "symfony/http-kernel": "^7.1",
-        "symfony/framework-bundle": "^7.1",
-        "symfony/twig-bundle": "^7.1",
-        "symfony/dotenv": "^7.1",
+        "symfony/http-kernel": "^8.0",
+        "symfony/framework-bundle": "^8.0",
+        "symfony/twig-bundle": "^8.0",
+        "symfony/dotenv": "^8.0",
         "twig/intl-extra": "^3.13",
         "twig/extra-bundle": "^3.13",
-        "symfony/cache": "^7.2",
+        "symfony/cache": "^8.0",
         "twig/cache-extra": "^3.21"
     },
     "autoload": {
@@ -21,6 +21,7 @@
         }
     },
     "require-dev": {
-        "symfony/var-dumper": "^7.1"
-    }
+        "symfony/var-dumper": "^8.0"
+    },
+    "minimum-stability": "stable"
 }

src/Command/CacheWarmCommand.php

@@ -1,84 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Krzysiej\RyobiCrawler\Command;
-
-use Symfony\Component\Console\Helper\ProgressBar;
-use Symfony\Component\HttpFoundation\Request;
-use Illuminate\Database\Capsule\Manager as Capsule;
-use Krzysiej\RyobiCrawler\Models\Product;
-use Symfony\Component\Cache\Adapter\FilesystemAdapter;
-use Symfony\Component\Console\Attribute\AsCommand;
-use Symfony\Component\Console\Command\Command;
-use Symfony\Component\Console\Input\InputInterface;
-use Symfony\Component\Console\Output\OutputInterface;
-use Symfony\Component\HttpFoundation\RequestStack;
-use Twig\Environment;
-
-#[AsCommand(name: 'app:cache:warm-twig', description: 'Warmup twig cache')]
-class CacheWarmCommand extends Command
-{
-    public function __construct(
-        private Environment $twig,
-        private RequestStack $requestStack,
-        private FilesystemAdapter $cache,
-    ) {
-        parent::__construct();
-    }
-
-    public function execute(InputInterface $input, OutputInterface $output): int
-    {
-        $this->cache->clear();
-
-        $capsule = new Capsule;
-        $capsule->addConnection([
-            'driver' => 'sqlite',
-            'database' => __DIR__ . '/../../database.sqlite',
-        ]);
-        $capsule->setAsGlobal();
-        $capsule->bootEloquent();
-
-        $progress = new ProgressBar($output);
-        $progress->start();
-        $products = Product::with([
-            'price' => fn($query) => $query->orderByDesc('created_at'),
-            'stock' => fn($query) => $query->orderByDesc('created_at'),
-        ])->get();
-        $progress->setMaxSteps(count($products));
-
-        $request = Request::create('/cache-warm');
-        $this->requestStack->push($request);
-
-        foreach ($products as $product) {
-            $priceList = $product->price()->pluck('price');
-            $stockList = $product->stock()->pluck('stock');
-            $priceDates = $product->price()->pluck('created_at')->map(fn($date) => $date->format('Y-m-d'))->toArray();
-            $stockDates = $product->stock()->pluck('created_at')->map(fn($date) => $date->format('Y-m-d'))->toArray();
-            $this->twig->render('product.html.twig', [
-                'product' => $product,
-                'price_list' => $this->prepareChartData($priceDates, $priceList),
-                'stock_list' => $this->prepareChartData($stockDates, $stockList),
-                'price_dates' => implode("','", $priceDates),
-            ]);
-            $progress->advance();
-        }
-
-        $progress->finish();
-        $output->writeln('');
-        $output->writeln('DONE');
-
-        return Command::SUCCESS;
-    }
-
-    private function prepareChartData($set1, $set2): string
-    {
-        $data = [];
-        foreach ($set1 as $key => $value) {
-            $data[] = ['x' => $value, 'y' => $set2[$key]];
-        }
-        $stringData = json_encode($data);
-
-        return str_replace(['"x"', '"y"'], ['x', 'y'], $stringData);
-    }
-}

src/Command/ScrapeWebsite.php

@@ -15,11 +15,13 @@ use Symfony\Component\Console\Attribute\AsCommand;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Helper\ProgressBar;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 #[AsCommand(name: 'app:scrape', description: 'Scrape all products from Ryobi website')]
 class ScrapeWebsite extends Command
 {
+    const COUNTRY_ID = 'country';
     private Client $client;
 
     public function __construct(protected Capsule $database)
@@ -30,30 +32,31 @@ class ScrapeWebsite extends Command
     protected function configure(): void
     {
         $this->client = new Client();
+        $this->addOption(self::COUNTRY_ID, 'c', InputOption::VALUE_OPTIONAL, 'Country id');
     }
 
     public function execute(InputInterface $input, OutputInterface $output): int
     {
+        $countryId = intval($input->getOption(self::COUNTRY_ID));
         $output->writeln('Scrape products');
-        $progress = new ProgressBar($output);
-        $countries = Country::all();
+        $countries = Country::all()->when($countryId, fn ($query)  => $query->where('id', $countryId));
         foreach($countries as $country) {
             $output->writeln('Country name: ' . $country->countryName);
-            $progress->start();
             $products = $this->getProducts($country);
+            $progress = new ProgressBar($output);
+            $progress->start();
             $progress->setMaxSteps(count($products));
             foreach ($products as $product) {
                 $this->saveProduct($product, $country);
                 $progress->advance();
             }
             $progress->finish();
-            $output->writeln('');
-            $output->writeln('Scrape products - DONE');
-            $output->writeln('');
+            $output->writeln("\nScrape products - DONE\n");
         }
 
         $output->writeln('Update prices');
         $products = Product::all();
+        $progress = new ProgressBar($output);
         $progress->setMaxSteps(count($products));
         $progress->start();
         foreach($products as $product) {
@@ -95,7 +98,7 @@ class ScrapeWebsite extends Command
                 $products = array_merge($products, $responseObject->products);
                 $page++;
                 $canLoadMore = $responseObject->canLoadMore;
-            } catch (GuzzleException) {
+            } catch (GuzzleException $e) {
                 return $products;
             }
         } while ($canLoadMore);

src/Controller/BaseController.php

@@ -11,7 +11,7 @@ class BaseController extends AbstractController
 {
     protected Environment $twig;
 
-    public function __construct(protected FilesystemAdapter $cache, protected Capsule $database)
+    public function __construct( protected Capsule $database)
     {
     }
 }

src/Controller/CategoryController.php

@@ -11,9 +11,6 @@ final class CategoryController extends BaseController
     #[Route('/category/{category?}',  name: 'app_category')]
     public function __invoke(?string $category): Response
     {
-        if($this->cache->getItem('list_category_'.$category)->isHit()) {
-            return $this->render('productList.html.twig', ['listType' => 'category_'.$category]);
-        }
 
         /** @var Product[] $products */
         $products = Product::with(['price', 'lowestPrice'])

src/Controller/DiscontinuedController.php

@@ -13,9 +13,6 @@ final class DiscontinuedController extends BaseController
     #[Route('/discontinued', name: 'app_discontinued')]
     public function __invoke(): Response
     {
-        if($this->cache->getItem('list_discontinued')->isHit()) {
-            return $this->render('productList.html.twig', ['listType' => 'discontinued']);
-        }
 
         $products = Product::where('lastSeen', '<', now()->format('Y-m-d'))
             ->orderByDesc('starred')

src/Controller/IndexController.php

@@ -11,9 +11,6 @@ final class IndexController extends BaseController
     #[Route('/', name: 'app_home')]
     public function __invoke(): Response
     {
-        if ($this->cache->getItem('list_all')->isHit()) {
-            return $this->render('productList.html.twig', ['listType' => 'all']);
-        }
         $products = Product::orderByDesc('starred')
             ->orderByDesc('created_by')
             ->get();

src/Controller/LowestPriceController.php

@@ -13,9 +13,6 @@ final class LowestPriceController extends BaseController
     public function __invoke(): Response
     {
         $listType = 'lowest_price';
-        if($this->cache->getItem('lowest_price')->isHit()) {
-            return $this->render('productList.html.twig', ['listType' => $listType]);
-        }
 
         $products = Product::whereRaw('priceCurrent = priceLowest')
             ->whereRaw('lastSeen = "'.now()->format('Y-m-d').'"')

src/Controller/NewController.php

@@ -13,9 +13,6 @@ final class NewController extends BaseController
     #[Route('/new', name: 'app_new')]
     public function __invoke(): Response
     {
-        if($this->cache->getItem('list_new')->isHit()) {
-            return $this->render('productList.html.twig', ['listType' => 'new']);
-        }
         $date = now()->modify('-30 days')->format('Y-m-d');
         $products = Product::where('created_at', '>', $date)
             ->with(['country', 'stock'])

src/Controller/ProductController.php

@@ -19,10 +19,6 @@ final class ProductController extends BaseController
     #[Route('/product/{productId<\d+>}', name: 'app_product')]
     public function __invoke(int $productId): Response
     {
-        if ($this->cache->getItem('product' . $productId)->isHit()) {
-            return $this->render('product.html.twig', ['product' => ['id' => $productId]]);
-        }
-
         $product = Product::with([
             'price' => fn($query) => $query->orderBy('created_at', 'desc'),
             'stock' => fn($query) => $query->orderBy('created_at', 'desc'),
@@ -32,6 +28,10 @@ final class ProductController extends BaseController
         }
         $priceList = $product->price()->pluck('price', 'created_at')->mapWithKeys(fn($price, $createdAt) => [explode(' ', $createdAt)[0] => $price])->toArray();
         $stockList = $product->stock()->pluck('stock', 'created_at')->mapWithKeys(fn($stock, $createdAt) => [explode(' ', $createdAt)[0] => $stock])->toArray();
+
+        ksort($stockList);
+        ksort($priceList);
+
         return $this->render('product.html.twig', [
             'product' => $product,
             'price_list' => $this->prepareChartData($priceList),

src/Controller/PromosController.php

@@ -11,10 +11,6 @@ final class PromosController extends BaseController
     #[Route('/promos/{promo?}', name: 'app_promos')]
     public function __invoke(?string $promo): Response
     {
-        if ($this->cache->getItem('list_promos')->isHit()) {
-            return $this->render('productList.html.twig', ['listType' => 'promos' . $promo]);
-        }
-
         $products = Product::when(is_null($promo), fn($q) => $q->whereRaw('priceCurrent < productStandardPrice'))
             ->orderByDesc('starred')
             ->orderByDesc('created_by')

src/Controller/StarController.php

@@ -12,15 +12,12 @@ final class StarController extends BaseController
     #[Route('/star/{productId<\d+>}', name: 'app_star')]
     public function __invoke(int $productId, Request $request): Response
     {
-        $this->cache->deleteItems(['list_all', 'list_promos', 'list_new', 'list_discontinued']);
         $referer = $request->headers->get('referer');
         if (str_contains($referer, '/category/')) {
             preg_match('#/category/(.*)#i', $referer, $matches);
-            $this->cache->deleteItem('list_category_'.urldecode($matches[1]));
         }
         if (str_contains($referer, '/search?search=')) {
             preg_match('#/search\?search=(.*)#i', $referer, $matches);
-            $this->cache->deleteItem('list_search_'.urldecode($matches[1]));
         }
 
         Product::find($productId)->toggleStarred()->save();

src/Controller/UpdateController.php

@@ -0,0 +1,18 @@
+<?php
+
+namespace Krzysiej\RyobiCrawler\Controller;
+
+use Symfony\Component\HttpFoundation\Request;
+use Symfony\Component\HttpFoundation\Response;
+use Symfony\Component\Routing\Attribute\Route;
+
+final class UpdateController extends BaseController
+{
+    #[Route('/update', name: 'app_update')]
+    public function __invoke(Request $request): Response
+    {
+        $output = shell_exec('bin/update');
+
+        return new Response($output);
+    }
+}

src/Kernel.php

@@ -38,12 +38,7 @@ class Kernel extends BaseKernel
         $services->set(Manager::class)->configurator([DatabaseFactory::class, 'create']);
         $services->load('Krzysiej\\RyobiCrawler\\', __DIR__ )
             ->exclude('../src/{Models,Twig,DatabaseFactory.php,Kernel.php}');
-        $services->set('twig.extension.cache', AppExtension::class)->tag('twig.extension');
-        $services->set(CacheExtension::class)->tag('twig.extension');
-        $services->set(FilesystemAdapter::class)->args([
-                '$directory' => __DIR__ . '/../var/cache/twig_blocks'
-            ]);
-        $services->set('twig.runtime.cache', CacheRuntime::class)->args([new Reference(FilesystemAdapter::class)])->tag('twig.runtime');
+        $services->set('twig.extension', AppExtension::class)->tag('twig.extension');
     }
 
     protected function configureRoutes(RoutingConfigurator $routes): void

templates/product.html.twig

@@ -1,24 +1,30 @@
 {% extends "template.html.twig" %}
 
 {% block content %}
-{% cache 'product' ~ product.id %}
     <div class="table-responsive">
         <table class='table table-hover'>
             <tr>
                 <td class="align-middle font-weight-bold h3"><a class="text-warning text-decoration-none"
-                                                                href="{{ path('app_star', {'productId': product.id}) }}">{% if product.starred %}★{% else %} ☆ {% endif %}</a></td>
+                                                                href="{{ path('app_star', {'productId': product.id}) }}">{% if product.starred %}★{% else %} ☆ {% endif %}</a>
+                </td>
                 <td><img src='{{ product.image }}&width=150' class='border rounded p-1' alt='{{ product.name }}'/></td>
                 <td>
-                    <a href='{{ path('app_product', {'productId': product.id}) }}' class="text-decoration-none">{{ product.name }}</a>
-                    <span class="badge text-bg-light"><a href="{{ path('app_search', {'search': product.subTitle}) }}" class="link-underline link-underline-opacity-0 link-dark">{{ product.subTitle }}</a></span>
-                    {% if product.promotions is not null and product.promotions.hasPromotion %}<a href="{{ path('app_promos', {'promo': product.promotions.slug}) }}"><span class="badge bg-info">PROMO: {{ product.promotions.tag }}</span></a>{% endif %}
+                    <a href='{{ path('app_product', {'productId': product.id}) }}'
+                       class="text-decoration-none">{{ product.name }}</a>
+                    <span class="badge text-bg-light"><a href="{{ path('app_search', {'search': product.subTitle}) }}"
+                                                         class="link-underline link-underline-opacity-0 link-dark">{{ product.subTitle }}</a></span>
+                    {% if product.promotions is not null and product.promotions.hasPromotion %}<a
+                        href="{{ path('app_promos', {'promo': product.promotions.slug}) }}"><span class="badge bg-info">PROMO: {{ product.promotions.tag }}</span>
+                        </a>{% endif %}
                 </td>
                 <td>
                     <nav aria-label="breadcrumb" style="--bs-breadcrumb-divider: '>';">
                         <ol class="breadcrumb">
                             {% for category in product.categories %}
-                                <li class="breadcrumb-item" aria-current="page"><a class="breadcrumb-item text-decoration-none"
-                                                                                   href="{{ path('app_category', {'category': category}) }}">{{ category }}</a></li>
+                                <li class="breadcrumb-item" aria-current="page"><a
+                                            class="breadcrumb-item text-decoration-none"
+                                            href="{{ path('app_category', {'category': category}) }}">{{ category }}</a>
+                                </li>
                             {% endfor %}
                         </ol>
                     </nav>
@@ -133,6 +139,5 @@
             }
         });
     </script>
-{% endcache %}
 {% endblock %}
 

templates/productList.html.twig

@@ -1,8 +1,6 @@
 {% extends "template.html.twig" %}
 
 {% block content %}
-    {% cache 'list_' ~ listType %}
-
         {% if listType starts with 'category_' %}
             {{ renderCategoryTree(categoryTree, category) | raw }}
         {% endif %}
@@ -93,5 +91,4 @@
                 </table>
             </div>
         {% endif %}
-    {% endcache %}
 {% endblock %}

templates/template.html.twig

@@ -18,7 +18,6 @@
         </button>
         <div class="collapse navbar-collapse" id="navbarNav">
             <ul class="navbar-nav me-auto mb-2 mb-lg-0">
-                {% cache "menu_count" ~ listType|default('') %}
                     <li class="nav-item">
                         <a class="nav-link {% if app.request.pathinfo == path('app_home') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_home') }}">All products <span class="badge text-bg-secondary">{{ allCount() }}</span></a>
                     </li>
@@ -34,7 +33,6 @@
                     <li class="nav-item">
                         <a class="nav-link {% if app.request.pathinfo == path('app_discontinued') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_discontinued') }}">Discontinued <span class="badge text-bg-secondary">{{ discontinuedCount() }}</span></a>
                     </li>
-                {% endcache %}
             </ul>
 
             <form class="form-floating d-flex col-lg-6 col-sm-8" role="search" action="{{ path('app_search') }}">