1 Commits

Author SHA1 Message Date
5fbd555bb3 Start of changing tabled to flexboxes 2026-01-07 08:18:56 +01:00
31 changed files with 826 additions and 1048 deletions

View File

@@ -1,15 +0,0 @@
on:
push:
workflow_dispatch:
jobs:
deploy-job:
runs-on: server
steps:
- name: initial test
run: echo "test"
- name: get page
run: curl http://192.168.0.129:9001/update
- run: pwd
- run: cd ../ && pwd
- run: pwd
- run: cd /var/www/html/ryobi-crawler && bin/update

View File

@@ -1,4 +1,4 @@
FROM php:8.4-cli
FROM php:8.3-cli
WORKDIR /usr/src/app
ENV PHP_MEMORY_LIMIT=1500M

View File

@@ -1,4 +1,2 @@
#!/usr/bin/env bash
echo "Cleaning cache"
bin/cli rm -rf var/cache
echo "cache cleaned2"

View File

@@ -1,6 +1,5 @@
#!/usr/bin/env bash
echo "Updating project"
git pull origin master
bin/cacheclean
bin/composer install --no-dev
bin/cli rm -rf var/cache
echo "Project updated"

View File

@@ -1,18 +1,18 @@
{
"require": {
"guzzlehttp/guzzle": "^7",
"guzzlehttp/guzzle": "^7.0",
"illuminate/database": "11.26.0.0",
"ext-json": "*",
"symfony/console": "^8.0",
"symfony/routing": "^8.0",
"symfony/console": "^7.0",
"symfony/routing": "^7.1",
"laravel/serializable-closure": "^1.3",
"symfony/http-kernel": "^8.0",
"symfony/framework-bundle": "^8.0",
"symfony/twig-bundle": "^8.0",
"symfony/dotenv": "^8.0",
"symfony/http-kernel": "^7.1",
"symfony/framework-bundle": "^7.1",
"symfony/twig-bundle": "^7.1",
"symfony/dotenv": "^7.1",
"twig/intl-extra": "^3.13",
"twig/extra-bundle": "^3.13",
"symfony/cache": "^8.0",
"symfony/cache": "^7.2",
"twig/cache-extra": "^3.21"
},
"autoload": {
@@ -21,7 +21,6 @@
}
},
"require-dev": {
"symfony/var-dumper": "^8.0"
},
"minimum-stability": "stable"
"symfony/var-dumper": "^7.1"
}
}

966
composer.lock generated

File diff suppressed because it is too large Load Diff

Binary file not shown.

Before

Width:  |  Height:  |  Size: 282 KiB

After

Width:  |  Height:  |  Size: 199 KiB

View File

@@ -0,0 +1,84 @@
<?php
declare(strict_types=1);
namespace Krzysiej\RyobiCrawler\Command;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\HttpFoundation\Request;
use Illuminate\Database\Capsule\Manager as Capsule;
use Krzysiej\RyobiCrawler\Models\Product;
use Symfony\Component\Cache\Adapter\FilesystemAdapter;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\HttpFoundation\RequestStack;
use Twig\Environment;
#[AsCommand(name: 'app:cache:warm-twig', description: 'Warmup twig cache')]
class CacheWarmCommand extends Command
{
public function __construct(
private Environment $twig,
private RequestStack $requestStack,
private FilesystemAdapter $cache,
) {
parent::__construct();
}
public function execute(InputInterface $input, OutputInterface $output): int
{
$this->cache->clear();
$capsule = new Capsule;
$capsule->addConnection([
'driver' => 'sqlite',
'database' => __DIR__ . '/../../database.sqlite',
]);
$capsule->setAsGlobal();
$capsule->bootEloquent();
$progress = new ProgressBar($output);
$progress->start();
$products = Product::with([
'price' => fn($query) => $query->orderByDesc('created_at'),
'stock' => fn($query) => $query->orderByDesc('created_at'),
])->get();
$progress->setMaxSteps(count($products));
$request = Request::create('/cache-warm');
$this->requestStack->push($request);
foreach ($products as $product) {
$priceList = $product->price()->pluck('price');
$stockList = $product->stock()->pluck('stock');
$priceDates = $product->price()->pluck('created_at')->map(fn($date) => $date->format('Y-m-d'))->toArray();
$stockDates = $product->stock()->pluck('created_at')->map(fn($date) => $date->format('Y-m-d'))->toArray();
$this->twig->render('product.html.twig', [
'product' => $product,
'price_list' => $this->prepareChartData($priceDates, $priceList),
'stock_list' => $this->prepareChartData($stockDates, $stockList),
'price_dates' => implode("','", $priceDates),
]);
$progress->advance();
}
$progress->finish();
$output->writeln('');
$output->writeln('DONE');
return Command::SUCCESS;
}
private function prepareChartData($set1, $set2): string
{
$data = [];
foreach ($set1 as $key => $value) {
$data[] = ['x' => $value, 'y' => $set2[$key]];
}
$stringData = json_encode($data);
return str_replace(['"x"', '"y"'], ['x', 'y'], $stringData);
}
}

View File

@@ -6,13 +6,11 @@ namespace Krzysiej\RyobiCrawler\Command;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
use Krzysiej\RyobiCrawler\Models\Country;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use function Symfony\Component\Clock\now;
#[AsCommand(name: 'app:migrate', description: 'Create database and rum migrations')]
class Migrate extends Command
@@ -26,10 +24,11 @@ class Migrate extends Command
public function execute(InputInterface $input, OutputInterface $output): int
{
if ($input->getOption(self::RECREATE_OPTION)) {
if (true === $input->hasOption(self::RECREATE_OPTION)) {
unlink(__DIR__ . '/../../database.sqlite');
touch(__DIR__ . '/../../database.sqlite');
//sleep(5);
}
touch(__DIR__ . '/../../database.sqlite');
$capsule = new Capsule;
$capsule->addConnection([
'driver' => 'sqlite',
@@ -40,9 +39,6 @@ class Migrate extends Command
$this->createProductsTable();
$this->createPricesTable();
$this->createStocksTable();
$this->addColumns();
$this->createCountriesTable();
$this->index();
return Command::SUCCESS;
}
@@ -81,111 +77,6 @@ class Migrate extends Command
}
}
public function createCountriesTable(): void
{
if (!Capsule::schema()->hasTable('countries')) {
Capsule::schema()->create('countries', function (Blueprint $table) {
$table->increments('id');
$table->text('countryName');
$table->text('productsUrl');
$table->text('cultureCode');
$table->text('currency');
$table->text('locale');
$table->timestamps();
});
}
if (Capsule::schema()->hasTable('countries') && !Country::where('countryName', 'Poland')->exists()) {
$id = Capsule::table('countries')->insertGetId(
[
'countryName' => 'Poland',
'productsUrl' => 'https://pl.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'pl-PL',
'currency' => 'PLN',
'locale' => 'pl',
'created_at' => now(),
'updated_at' => now(),
]);
}
if (Capsule::schema()->hasTable('countries') && !Country::where('countryName', 'UK')->exists()) {
Capsule::table('countries')->insert([
'countryName' => 'UK',
'productsUrl' => 'https://uk.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'en-GB',
'currency' => 'GBP',
'locale' => 'uk',
'created_at' => now(),
'updated_at' => now(),
]
);
}
if (Capsule::schema()->hasTable('countries') && !Country::where('countryName', 'Netherlands')->exists()) {
Capsule::table('countries')->insert([
'countryName' => 'Netherlands',
'productsUrl' => 'https://nl.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'nl-NL',
'currency' => 'EUR',
'locale' => 'nl',
'created_at' => now(),
'updated_at' => now(),
]
);
}
if (Capsule::schema()->hasTable('countries') && !Country::where('countryName', 'France')->exists()) {
Capsule::table('countries')->insert([
'countryName' => 'France',
'productsUrl' => 'https://fr.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'fr-FR',
'currency' => 'EUR',
'locale' => 'fr',
'created_at' => now(),
'updated_at' => now(),
]
);
}
if (Capsule::schema()->hasTable('countries') && !Country::where('countryName', 'Spain')->exists()) {
Capsule::table('countries')->insert([
'countryName' => 'Spain',
'productsUrl' => 'https://es.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'es-ES',
'currency' => 'EUR',
'locale' => 'es',
'created_at' => now(),
'updated_at' => now(),
]
);
}
if (Capsule::schema()->hasTable('countries') && !Country::where('countryName', 'Poland')->exists()) {
$id = Capsule::table('countries')->insertGetId(
[
'countryName' => 'Poland',
'productsUrl' => 'https://pl.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'pl-PL',
'currency' => 'PLN',
'locale' => 'pl',
'created_at' => now(),
'updated_at' => now(),
]);
}
if (Capsule::schema()->hasTable('countries') && !Country::where('countryName', 'UK')->exists()) {
Capsule::table('countries')->insert([
'countryName' => 'UK',
'productsUrl' => 'https://uk.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'en-GB',
'currency' => 'GBP',
'locale' => 'uk',
'created_at' => now(),
'updated_at' => now(),
]
);
}
if (!Capsule::schema()->hasColumn('products', 'country_id')) {
Capsule::schema()->table('products', function (Blueprint $table) use ($id) {
$table->foreignId('country_id')->default($id)->references('id')->on('countries');
});
}
}
public function createStocksTable(): void
{
if (!Capsule::schema()->hasTable('stocks')) {
@@ -203,110 +94,4 @@ class Migrate extends Command
});
}
}
public function addColumns(): void
{
if (!Capsule::schema()->hasColumn('products', 'priceCurrent')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->float('priceCurrent')->default(0);
});
}
if (!Capsule::schema()->hasColumn('products', 'priceLowest')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->float('priceLowest')->default(0);
});
}
if (!Capsule::schema()->hasColumn('products', 'productStandardPrice')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->float('productStandardPrice')->default(0);
});
}
if (!Capsule::schema()->hasColumn('products', 'lowestProductPrice30Days')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->float('lowestProductPrice30Days')->default(0);
});
}
if (!Capsule::schema()->hasColumn('products', 'lastSeen')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->date('lastSeen')->nullable();
});
}
if (!Capsule::schema()->hasColumn('products', 'stock')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->integer('stock')->nullable();
});
}
if (!Capsule::schema()->hasColumn('products', 'promotions')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->json('promotions')->nullable();
});
}
}
public function index(): void
{
if (!$this->hasIndex('products_skuid_country_id_unique')) {
Capsule::schema()->table('products', function (Blueprint $table) {
if ($this->hasIndex('products_skuid_unique')) {
$table->integer('skuID')->unique(false)->change();
}
$table->unique(['skuID', 'country_id']);
});
}
if (!$this->hasIndex('prices_product_id_index')) {
Capsule::schema()->table('prices', function (Blueprint $table) {
$table->index('product_id');
});
}
if (!$this->hasIndex('products_id_index')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->index('id');
});
}
if (!$this->hasIndex('stocks_product_id_stock_index')) {
Capsule::schema()->table('stocks', function (Blueprint $table) {
$table->index(['product_id', 'stock']);
});
}
if (!$this->isFK('products', 'id', 'stocks', 'product_id')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->foreign('id')->references('product_id')->on('stocks');
});
}
if (!$this->isFK('products', 'id', 'prices', 'product_id')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->foreign('id')->references('product_id')->on('prices');
});
}
if (!$this->isFK('products', 'country_id', 'countries', 'id')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->foreign('country_id')->references('id')->on('countries');
});
}
}
private function isFK(string $table, string $column, string $fTable, string $fColumn): bool
{
$fkColumns = Capsule::schema()->getForeignKeys($table);
return !empty(array_filter($fkColumns, fn($fkColumn) => ($fkColumn['foreign_table'] == $fTable &&
in_array($fColumn, $fkColumn['foreign_columns']) &&
in_array($column, $fkColumn['columns']))
));
}
private function hasIndex(string $indexName): bool
{
return !!count(Capsule::select('SELECT name FROM sqlite_master WHERE type = "index" and name = ?', [$indexName]));
}
}

View File

@@ -7,7 +7,6 @@ namespace Krzysiej\RyobiCrawler\Command;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use Illuminate\Database\Capsule\Manager as Capsule;
use Krzysiej\RyobiCrawler\Models\Country;
use Krzysiej\RyobiCrawler\Models\Price;
use Krzysiej\RyobiCrawler\Models\Product;
use Krzysiej\RyobiCrawler\Models\Stock;
@@ -15,13 +14,11 @@ use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
#[AsCommand(name: 'app:scrape', description: 'Scrape all products from Ryobi website')]
class ScrapeWebsite extends Command
{
const COUNTRY_ID = 'country';
private Client $client;
public function __construct(protected Capsule $database)
@@ -32,73 +29,44 @@ class ScrapeWebsite extends Command
protected function configure(): void
{
$this->client = new Client();
$this->addOption(self::COUNTRY_ID, 'c', InputOption::VALUE_OPTIONAL, 'Country id');
}
public function execute(InputInterface $input, OutputInterface $output): int
{
$countryId = intval($input->getOption(self::COUNTRY_ID));
$output->writeln('Scrape products');
$countries = Country::all()->when($countryId, fn ($query) => $query->where('id', $countryId));
foreach($countries as $country) {
$output->writeln('Country name: ' . $country->countryName);
$products = $this->getProducts($country);
$progress = new ProgressBar($output);
$progress->start();
$products = $this->getProducts();
$progress->setMaxSteps(count($products));
foreach ($products as $product) {
$this->saveProduct($product, $country);
$progress->advance();
}
$progress->finish();
$output->writeln("\nScrape products - DONE\n");
}
$output->writeln('Update prices');
$products = Product::all();
$progress = new ProgressBar($output);
$progress->setMaxSteps(count($products));
$progress->start();
foreach($products as $product) {
$newestPrice = $product->newestPrice;
$currentStock = $product->currentStock;
$product->priceCurrent = $newestPrice->price;
$product->productStandardPrice = $newestPrice->productStandardPrice;
$product->lowestProductPrice30Days = $newestPrice->lowestProductPrice30Days;
$product->priceLowest = $product->lowestPrice->price;
$product->lastSeen = $newestPrice->created_at->format('Y-m-d');
$product->stock = $currentStock->stock;
$product->save(['timestamps' => false]);
$this->saveProduct($product);
$progress->advance();
}
$progress->finish();
$output->writeln('');
$output->writeln('Update prices - DONE');
$output->writeln('COMMAND - DONE');
$output->writeln('DONE');
return Command::SUCCESS;
}
private function getProducts(Country $country): array
private function getProducts(): array
{
$products = [];
$page = 0;
do {
try {
$res = $this->client->request('POST', $country->productsUrl, [
$res = $this->client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [
'form_params' => [
"includePreviousPages" => false,
"pageIndex" => $page,
"pageSize" => 100,
"cultureCode" => $country->cultureCode,
"cultureCode" => "pl-PL",
]
]);
$responseObject = json_decode($res->getBody()->getContents());
$products = array_merge($products, $responseObject->products);
$page++;
$canLoadMore = $responseObject->canLoadMore;
} catch (GuzzleException $e) {
} catch (GuzzleException) {
return $products;
}
} while ($canLoadMore);
@@ -106,24 +74,21 @@ class ScrapeWebsite extends Command
return $products;
}
private function saveProduct(\stdClass $product, Country $country): void
private function saveProduct(\stdClass $product): void
{
/** @var Product $productModel */
$productModel = Product::firstOrNew(['skuID' => $product->skuID, 'country_id' => $country->id]);
$productModel = Product::firstOrNew(['skuID' => $product->skuID]);
$productModel->skuID = $product->skuID;
$productModel->name = $product->name;
$productModel->availableQuantity = $product->availableQuantity;
$productModel->categories = str_replace(['/', ' '], '', $product->categories);
$productModel->categories = $product->categories;
$productModel->image = $product->image;
$productModel->subTitle = $product->subTitle;
$productModel->variantCode = $product->variantCode;
$productModel->modelCode = $product->modelCode;
$productModel->url = $product->url;
$productModel->lastSeen = date("Y-m-d");
$productModel->touch('updated_at');
$productModel->country()->associate($country);
$productModel->promotions = $product->promotions;
$productModel->save();
$priceExists = $productModel->price()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();

View File

@@ -11,7 +11,7 @@ class BaseController extends AbstractController
{
protected Environment $twig;
public function __construct( protected Capsule $database)
public function __construct(protected FilesystemAdapter $cache, protected Capsule $database)
{
}
}

View File

@@ -8,40 +8,22 @@ use Symfony\Component\Routing\Attribute\Route;
final class CategoryController extends BaseController
{
#[Route('/category/{category?}', name: 'app_category')]
public function __invoke(?string $category): Response
#[Route('/category/{category}', name: 'app_category')]
public function __invoke(string $category): Response
{
if($this->cache->getItem('list_category_'.$category)->isHit()) {
return $this->render('productList.html.twig', ['listType' => 'category_'.$category]);
}
/** @var Product[] $products */
$products = Product::with(['price', 'lowestPrice'])
->selectRaw('products.*')
->distinct('products.id')
->when(!is_null($category) , fn ($q) => $q->whereRaw('json_each.value = ?', [$category]))
->fromRaw('products, json_each(products.categories)')
->whereRaw('json_each.value = ?', [$category])
->orderByDesc('starred')
->orderByDesc('created_by')
->get();
$categoriesTree = [];
foreach ($products as $product) {
$categoriesTree = $this->addToTree($product->categories, $categoriesTree);
}
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'category_' . $category, 'category' => $category, 'categoryTree' => $categoriesTree]);
}
private function addToTree(array $categories, mixed $categoriesTree)
{
$tmp = &$categoriesTree;
foreach ($categories as $category) {
if (empty($tmp[$category])) {
$tmp[$category] = ['count' => 0];
}
$tmp = &$tmp[$category];
$tmp['count']++;
}
return $categoriesTree;
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'category_'.$category]);
}
}

View File

@@ -13,10 +13,14 @@ final class DiscontinuedController extends BaseController
#[Route('/discontinued', name: 'app_discontinued')]
public function __invoke(): Response
{
if($this->cache->getItem('list_discontinued')->isHit()) {
return $this->render('productList.html.twig', ['listType' => 'discontinued']);
}
$products = Product::where('lastSeen', '<', now()->format('Y-m-d'))
$products = Product::where('updated_at', '<', now()->format('Y-m-d'))
->orderByDesc('starred')
->orderByDesc('created_by')
->with(['currentPrice', 'lowestPrice'])
->get();
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'discontinued']);
}

View File

@@ -11,11 +11,14 @@ final class IndexController extends BaseController
#[Route('/', name: 'app_home')]
public function __invoke(): Response
{
$products = Product::orderByDesc('starred')
if ($this->cache->getItem('list_all')->isHit()) {
return $this->render('productList.html.twig', ['listType' => 'all']);
}
$products = Product::with(['currentStock', 'price', 'lowestPrice'])
->orderByDesc('starred')
->orderByDesc('created_by')
->get();
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'all']);
}
}

View File

@@ -1,26 +0,0 @@
<?php
namespace Krzysiej\RyobiCrawler\Controller;
use Krzysiej\RyobiCrawler\Models\Product;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Attribute\Route;
use function Symfony\Component\Clock\now;
final class LowestPriceController extends BaseController
{
#[Route('/lowest-price', name: 'app_lowest_price', methods: ['GET'])]
public function __invoke(): Response
{
$listType = 'lowest_price';
$products = Product::whereRaw('priceCurrent = priceLowest')
->whereRaw('lastSeen = "'.now()->format('Y-m-d').'"')
->whereRaw('priceCurrent < productStandardPrice')
->orderByDesc('starred')
->orderByDesc('created_by')
->with(['currentPrice', 'lowestPrice'])
->get();
return $this->render('productList.html.twig', ['products' => $products, 'listType' => $listType]);
}
}

View File

@@ -2,6 +2,7 @@
namespace Krzysiej\RyobiCrawler\Controller;
use Illuminate\Database\Eloquent\Builder;
use Krzysiej\RyobiCrawler\Models\Product;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Attribute\Route;
@@ -13,11 +14,14 @@ final class NewController extends BaseController
#[Route('/new', name: 'app_new')]
public function __invoke(): Response
{
$date = now()->modify('-30 days')->format('Y-m-d');
$products = Product::where('created_at', '>', $date)
->with(['country', 'stock'])
if($this->cache->getItem('list_new')->isHit()) {
return $this->render('productList.html.twig', ['listType' => 'new']);
}
$products = Product::where('created_at', '>', now()->modify('-30 days')->format('Y-m-d'))
->orderByDesc('starred')
->orderByDesc('created_by')
->with(['currentPrice', 'lowestPrice'])
->get();
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'new']);
}

View File

@@ -3,6 +3,7 @@
namespace Krzysiej\RyobiCrawler\Controller;
use Krzysiej\RyobiCrawler\Models\Product;
use Symfony\Component\Cache\Adapter\FilesystemAdapter;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Attribute\Route;
use Twig\Error\LoaderError;
@@ -19,6 +20,10 @@ final class ProductController extends BaseController
#[Route('/product/{productId<\d+>}', name: 'app_product')]
public function __invoke(int $productId): Response
{
if($this->cache->getItem('product'.$productId)->isHit()) {
return $this->render('product.html.twig', ['product' => ['id' => $productId]]);
}
$product = Product::with([
'price' => fn($query) => $query->orderBy('created_at', 'desc'),
'stock' => fn($query) => $query->orderBy('created_at', 'desc'),
@@ -26,38 +31,26 @@ final class ProductController extends BaseController
if (null === $product) {
throw $this->createNotFoundException('Product not found');
}
$priceList = $product->price()->pluck('price', 'created_at')->mapWithKeys(fn($price, $createdAt) => [explode(' ', $createdAt)[0] => $price])->toArray();
$stockList = $product->stock()->pluck('stock', 'created_at')->mapWithKeys(fn($stock, $createdAt) => [explode(' ', $createdAt)[0] => $stock])->toArray();
ksort($stockList);
ksort($priceList);
$priceList = $product->price()->pluck('price');
$stockList = $product->stock()->pluck('stock');
$priceDates = $product->price()->pluck('created_at')->map(fn($date) => $date->format('Y-m-d'))->toArray();
$stockDates = $product->stock()->pluck('created_at')->map(fn($date) => $date->format('Y-m-d'))->toArray();
return $this->render('product.html.twig', [
'product' => $product,
'price_list' => $this->prepareChartData($priceList),
'stock_list' => $this->prepareChartData($stockList),
'price_dates' => implode("','", $this->dateRange(array_key_first($priceList), array_key_last($priceList))),
'price_list' => $this->prepareChartData($priceDates, $priceList),
'stock_list' => $this->prepareChartData($stockDates, $stockList),
'price_dates' => implode("','", $priceDates),
]);
}
private function prepareChartData($set1): string
private function prepareChartData($set1, $set2): string
{
$dates = $this->dateRange(array_key_first($set1), array_key_last($set1));
$data = array_map(fn($date) => ['x' => $date, 'y' => $set1[$date] ?? null], $dates);
$data = [];
foreach ($set1 as $key => $value) {
$data[] = ['x' => $value, 'y' => $set2[$key]];
}
$stringData = json_encode($data);
return str_replace(['"x"', '"y"'], ['x', 'y'], $stringData);
}
private function dateRange($dateStart, $dateEnd): array
{
$from = new \DateTime($dateStart);
$to = new \DateTime($dateEnd);
$range = [];
for ($date = clone $from; $date < $to; $date->modify('+1 day')) {
$range[] = $date->format('Y-m-d');
}
return $range;
}
}

View File

@@ -2,27 +2,25 @@
namespace Krzysiej\RyobiCrawler\Controller;
use Illuminate\Database\Eloquent\Builder;
use Krzysiej\RyobiCrawler\Models\Product;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Attribute\Route;
final class PromosController extends BaseController
{
#[Route('/promos/{promo?}', name: 'app_promos')]
public function __invoke(?string $promo): Response
#[Route('/promos', name: 'app_promos')]
public function __invoke(): Response
{
$products = Product::when(is_null($promo), fn($q) => $q->whereRaw('priceCurrent < productStandardPrice'))
if($this->cache->getItem('list_promos')->isHit()) {
return $this->render('productList.html.twig', ['listType' => 'promos']);
}
$products = Product::whereHas('currentPrice', fn(Builder $query) => $query->whereColumn('price', '<', 'productStandardPrice'))
->orderByDesc('starred')
->orderByDesc('created_by')
->with(['currentPrice', 'lowestPrice'])
->when(!is_null($promo), fn($q) => $q->whereRaw("json_extract(promotions, '$.slug') LIKE ?", $promo))
->get();
$promos = Product::select($this->database->getConnection()->raw("distinct json_extract(promotions, '$.slug') as slug, json_extract(promotions, '$.tag') as tag"))
->whereRaw("json_extract(promotions, '$.tag') is not null")
->get();
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'promos' . $promo, 'promos' => $promos->toArray()]);
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'promos']);
}
}

View File

@@ -12,15 +12,7 @@ final class StarController extends BaseController
#[Route('/star/{productId<\d+>}', name: 'app_star')]
public function __invoke(int $productId, Request $request): Response
{
$referer = $request->headers->get('referer');
if (str_contains($referer, '/category/')) {
preg_match('#/category/(.*)#i', $referer, $matches);
}
if (str_contains($referer, '/search?search=')) {
preg_match('#/search\?search=(.*)#i', $referer, $matches);
}
Product::find($productId)->toggleStarred()->save();
return $this->redirect($referer);
return $this->redirect($request->headers->get('referer'));
}
}

View File

@@ -1,18 +0,0 @@
<?php
namespace Krzysiej\RyobiCrawler\Controller;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Attribute\Route;
final class UpdateController extends BaseController
{
#[Route('/update', name: 'app_update')]
public function __invoke(Request $request): Response
{
$output = shell_exec('bin/update');
return new Response($output);
}
}

View File

@@ -38,7 +38,12 @@ class Kernel extends BaseKernel
$services->set(Manager::class)->configurator([DatabaseFactory::class, 'create']);
$services->load('Krzysiej\\RyobiCrawler\\', __DIR__ )
->exclude('../src/{Models,Twig,DatabaseFactory.php,Kernel.php}');
$services->set('twig.extension', AppExtension::class)->tag('twig.extension');
$services->set('twig.extension.cache', AppExtension::class)->tag('twig.extension');
$services->set(CacheExtension::class)->tag('twig.extension');
$services->set(FilesystemAdapter::class)->args([
'$directory' => __DIR__ . '/../var/cache/twig_blocks'
]);
$services->set('twig.runtime.cache', CacheRuntime::class)->args([new Reference(FilesystemAdapter::class)])->tag('twig.runtime');
}
protected function configureRoutes(RoutingConfigurator $routes): void

View File

@@ -1,24 +0,0 @@
<?php
namespace Krzysiej\RyobiCrawler\Models;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use Illuminate\Database\Eloquent\Relations\HasMany;
/**
* @property string $countryName
* @property string $productsUrl
* @property string $cultureCode
* @property string $currency
* @property string $locale
*/
class Country extends Model
{
public $timestamps = true;
public function products(): HasMany
{
return $this->hasMany(Product::class);
}
}

View File

@@ -2,14 +2,11 @@
namespace Krzysiej\RyobiCrawler\Models;
use Carbon\Traits\Date;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\Relations\HasOne;
use Illuminate\Support\Str;
use function Symfony\Component\Clock\now;
/**
@@ -23,23 +20,11 @@ use function Symfony\Component\Clock\now;
* @property string $modelCode
* @property string $url
* @property int $starred
* @property float $priceCurrent
* @property float $priceLowest
* @property float $productStandardPrice
* @property float $lowestProductPrice30Days
* @property Date $lastSeen
* @property integer $stock
* @property Object $promotions
*/
class Product extends Model
{
public $timestamps = true;
public $fillable = ['skuID', 'country_id'];
public function country(): BelongsTo
{
return $this->belongsTo(Country::class);
}
public $fillable = ['skuID'];
public function price(): HasMany
{
@@ -61,11 +46,6 @@ class Product extends Model
return $this->hasOne(Price::class)->ofMany('price', 'MIN');
}
public function newestPrice(): HasOne
{
return $this->hasOne(Price::class)->latest()->take(1);
}
public function stock(): HasMany
{
return $this->hasMany(Stock::class);
@@ -73,7 +53,9 @@ class Product extends Model
public function currentStock(): HasOne
{
return $this->stock()->one()->ofMany()->withDefault(fn(Stock $stock) => $stock->stock = 0)->take(1);
return $this->stock()->one()->ofMany()->withDefault(function (Stock $stock) {
$stock->stock = 0;
});
}
public function toggleStarred(): self
@@ -91,23 +73,10 @@ class Product extends Model
);
}
public function promotions(): Attribute
{
return Attribute::make(
get: fn(?string $value) => json_decode($value ?? '{"hasPromotion": false}', 1),
set: function (\stdClass $value) {
$value->slug = Str::slug($value->tag);
return json_encode($value);
}
);
}
public function isDiscontinued(): bool
{
return $this->lastSeen < now()->format('Y-m-d');
return $this->updated_at->format('Y-m-d') < now()->format('Y-m-d');
}
public function isNew(): bool
{
return $this->created_at->format('Y-m-d') > now()->modify('-30 days')->format('Y-m-d');

View File

@@ -2,11 +2,11 @@
namespace Krzysiej\RyobiCrawler\Twig;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Collection;
use Krzysiej\RyobiCrawler\Models\Price;
use Krzysiej\RyobiCrawler\Models\Product;
use Krzysiej\RyobiCrawler\Models\Stock;
use Symfony\Component\Routing\RouterInterface;
use Twig\Extension\AbstractExtension;
use Twig\TwigFilter;
use Twig\TwigFunction;
@@ -15,10 +15,6 @@ use function Symfony\Component\Clock\now;
class AppExtension extends AbstractExtension
{
public function __construct(public RouterInterface $route)
{
}
public function getFunctions(): array
{
return [
@@ -26,8 +22,6 @@ class AppExtension extends AbstractExtension
new TwigFunction('allCount', [$this, 'allCount']),
new TwigFunction('newCount', [$this, 'newCount']),
new TwigFunction('discontinuedCount', [$this, 'discontinuedCount']),
new TwigFunction('lowestPriceCount', [$this, 'lowestPriceCount']),
new TwigFunction('renderCategoryTree', [$this, 'renderCategoryTree']),
];
}
@@ -37,7 +31,6 @@ class AppExtension extends AbstractExtension
new TwigFilter('findByCreatedAtDate', [$this, 'findByCreatedAtDate']),
];
}
public function allCount(): int
{
return Product::count();
@@ -45,7 +38,7 @@ class AppExtension extends AbstractExtension
public function promosCount(): int
{
return Product::whereRaw('priceCurrent < productStandardPrice')->count();
return Product::whereHas('currentPrice', fn(Builder $query) => $query->whereColumn('price', '<', 'productStandardPrice'))->count();
}
public function newCount(): int
@@ -55,43 +48,11 @@ class AppExtension extends AbstractExtension
public function discontinuedCount(): int
{
return Product::where('lastSeen', '<>', now()->format('Y-m-d'))->count();
}
public function lowestPriceCount(): int
{
return Product::whereRaw('priceCurrent = priceLowest')
->whereRaw('lastSeen = "' . now()->format('Y-m-d') . '"')
->whereRaw('priceCurrent < productStandardPrice')
->count();
return Product::where('updated_at', '<', now()->format('Y-m-d'))->count();
}
public function findByCreatedAtDate(Collection $items, string $date): Stock|Price|null
{
return $items->first(fn($item) => str_starts_with($item->created_at, $date));
}
public function renderCategoryTree($categories, $current, $level = 0): string
{
$tree = '';
if ($level == 0) {
$tree .= '<ul class="list-group list-group-flush">';
}
foreach ($categories as $categoryName => $category) {
$currentClass = $categoryName == $current ? 'list-group-item-primary' : '';
$tree .= '<a class="list-group-item list-group-item-action '.$currentClass.' text-decoration-none ms-' . ($level * 2) . '" href="' . $this->route->generate('app_category', ['category' => $categoryName]) . '">' . $categoryName . ' <span class="badge bg-primary rounded-pill">' . $category['count'] . '</span></a>';
unset($category['count']);
if (is_array($category) && count($category) >= 1) {
foreach ($category as $subcategoryName => $subCategory) {
$tree .= $this->renderCategoryTree([$subcategoryName => $subCategory], $current, $level + 1);
}
}
}
if ($level == 0) {
$tree .= '</ul>';
}
return $tree;
}
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -1,35 +1,28 @@
{% extends "template.html.twig" %}
{% block content %}
{% cache 'product' ~ product.id %}
<div class="table-responsive">
<table class='table table-hover'>
<tr>
<td class="align-middle font-weight-bold h3"><a class="text-warning text-decoration-none"
href="{{ path('app_star', {'productId': product.id}) }}">{% if product.starred %}{% else %}{% endif %}</a>
</td>
href="{{ path('app_star', {'productId': product.id}) }}">{% if product.starred %}{% else %}{% endif %}</a></td>
<td><img src='{{ product.image }}&width=150' class='border rounded p-1' alt='{{ product.name }}'/></td>
<td>
<a href='{{ path('app_product', {'productId': product.id}) }}'
class="text-decoration-none">{{ product.name }}</a>
<span class="badge text-bg-light"><a href="{{ path('app_search', {'search': product.subTitle}) }}"
class="link-underline link-underline-opacity-0 link-dark">{{ product.subTitle }}</a></span>
{% if product.promotions is not null and product.promotions.hasPromotion %}<a
href="{{ path('app_promos', {'promo': product.promotions.slug}) }}"><span class="badge bg-info">PROMO: {{ product.promotions.tag }}</span>
</a>{% endif %}
<a href='{{ path('app_product', {'productId': product.id}) }}' class="text-decoration-none">{{ product.name }}</a>
<span class="badge text-bg-light">{{ product.subTitle }}</span>
</td>
<td>
<nav aria-label="breadcrumb" style="--bs-breadcrumb-divider: '>';">
<ol class="breadcrumb">
{% for category in product.categories %}
<li class="breadcrumb-item" aria-current="page"><a
class="breadcrumb-item text-decoration-none"
href="{{ path('app_category', {'category': category}) }}">{{ category }}</a>
</li>
<li class="breadcrumb-item" aria-current="page"><a class="breadcrumb-item text-decoration-none"
href="{{ path('app_category', {'category': category}) }}">{{ category }}</a></li>
{% endfor %}
</ol>
</nav>
</td>
<td><a href='https://{{ product.country.locale }}.ryobitools.eu{{ product.url }}'>link</a></td>
<td><a href='https://pl.ryobitools.eu/{{ product.url }}'>link</a></td>
</tr>
<tr>
<td colspan="5">
@@ -49,14 +42,13 @@
<th>Stock</th>
</tr>
</thead>
{% set stock = product.stock().get() %}
{% for price in product.price %}
<tr>
<td>{{ price.price | format_currency(product.country.currency, {}, product.country.locale) }}</td>
<td>{{ price.lowestProductPrice30Days | format_currency(product.country.currency, {}, product.country.locale) }}</td>
<td>{{ price.productStandardPrice | format_currency(product.country.currency, {}, product.country.locale) }}</td>
<td>{{ price.price | format_currency('PLN', {}, 'pl') }}</td>
<td>{{ price.lowestProductPrice30Days | format_currency('PLN', {}, 'pl') }}</td>
<td>{{ price.productStandardPrice | format_currency('PLN', {}, 'pl') }}</td>
<td>{{ price.created_at }}</td>
<td>{{ (stock | findByCreatedAtDate(price.created_at | slice(0,10))).stock ?? '' }}</td>
<td>{{ (product.stock | findByCreatedAtDate(price.created_at | slice(0,10))).stock ?? '' }}</td>
</tr>
{% endfor %}
</table>
@@ -75,7 +67,7 @@
labels: ['{{ price_dates|raw }}'],
datasets: [
{
label: 'Price ({{ product.country.currency }})',
label: 'Price (PLN)',
data: {{ price_list|raw }},
yAxisID: 'yPrice',
tension: 0.1,
@@ -91,7 +83,6 @@
]
},
options: {
spanGaps: false,
responsive: true,
animation: false,
scales: {
@@ -107,7 +98,7 @@
beginAtZero: true,
title: {
display: true,
text: 'Price ({{ product.country.currency }})'
text: 'Price (PLN)'
},
grid: {
drawOnChartArea: false
@@ -139,5 +130,6 @@
}
});
</script>
{% endcache %}
{% endblock %}

View File

@@ -1,94 +1,65 @@
{% extends "template.html.twig" %}
{% block content %}
{% if listType starts with 'category_' %}
{{ renderCategoryTree(categoryTree, category) | raw }}
{% endif %}
{% if listType starts with 'promos' %}
{% for promo in promos %}
<a href="{{ path('app_promos', {'promo': promo.slug}) }}"><span class="badge bg-info">PROMO: {{ promo.tag }}</span></a>
{% endfor %}
{% endif %}
{% if (listType starts with 'category_' and category == null) or not (listType starts with 'category_') or (listType starts with 'category_' and category is not null) %}
{% cache 'list_' ~ listType %}
<div class="table-responsive">
<table class='table table-hover'>
<thead>
<tr>
<th></th>
<th></th>
<th>Name</th>
<th>Categories</th>
<th></th>
<th class="text-end">Lowest Price</th>
<th class="text-end">Current Price</th>
<th></th>
</tr>
</thead>
{# <table class='table table-hover'>#}
{# <thead>#}
{# <tr>#}
{# <th></th>#}
{# <th></th>#}
{# <th>Name</th>#}
{# <th>Categories</th>#}
{# <th></th>#}
{# <th class="text-end">Lowest Price</th>#}
{# <th class="text-end">Current Price</th>#}
{# <th></th>#}
{# </tr>#}
{# </thead>#}
{% for product in products %}
<tr>
<div class="d-inline-flex">
<td class="align-middle font-weight-bold h3"><a class="text-warning text-decoration-none"
href="{{ path('app_star', {'productId': product.id}) }}">{% if product.starred == true %}{% else %}{% endif %}</a>
</td>
<td class="align-middle" style="width: 120px;"><img src='{{ product.image }}&width=70'
class='img-thumbnail'
alt='{{ product.name }}'/></td>
href="{{ path('app_star', {'productId': product.id}) }}">{% if product.starred == true %}{% else %}{% endif %}</a></td>
<td class="align-middle" style="width: 120px;"><img src='{{ product.image }}&width=70' class='img-thumbnail' alt='{{ product.name }}'/></td>
<td class="align-middle">
<a href='{{ path('app_product', {'productId': product.id}) }}'
class="text-decoration-none">{{ product.name }}</a>
{% if product.stock > 0 %}
<span class="badge text-bg-light">stock: {{ product.stock }}</span>
<a href='{{ path('app_product', {'productId': product.id}) }}' class="text-decoration-none">{{ product.name }}</a>
{% if product.currentStock.stock > 0 %}
<span class="badge text-bg-light">stock: {{ product.currentStock.stock }}</span>
{% else %}
<span class="badge text-bg-warning">out of stock</span>
{% endif %}
{% if product.isDiscontinued() %}
<span class="badge text-bg-secondary" data-bs-toggle="tooltip"
data-bs-title="Last update: {{ product.lastSeen }}">is discontinued</span>
<span class="badge text-bg-secondary" data-bs-toggle="tooltip" data-bs-title="Last update: {{ product.updated_at }}">is discontinued</span>
{% endif %}
{% if product.isNew() %}
<span class="badge text-bg-success">is new</span>
{% endif %}
<span class="badge text-bg-light"><a
href="{{ path('app_search', {'search': product.subTitle}) }}"
class="link-underline link-underline-opacity-0 link-dark">{{ product.subTitle }}</a></span>
{% if product.promotions is not null and product.promotions.hasPromotion %}<a href="{{ path('app_promos', {'promo': product.promotions.slug}) }}"><span class="badge bg-info">PROMO: {{ product.promotions.tag }}</span></a>{% endif %}
<span class="badge text-bg-light">{{ product.subTitle }}</span>
</td>
<td class="align-middle">
<nav aria-label="breadcrumb" style="--bs-breadcrumb-divider: '>';" >
<ol class="breadcrumb mb-0">
{% for category in product.categories %}
<li class="breadcrumb-item" aria-current="page"><a
class="breadcrumb-item text-decoration-none"
href="{{ path('app_category', {'category': category}) }}">{{ category }}</a>
</li>
<li class="breadcrumb-item" aria-current="page"><a class="breadcrumb-item text-decoration-none" href="{{ path('app_category', {'category': category}) }}">{{ category }}</a></li>
{% endfor %}
</ol>
</nav>
</td>
<td class="align-middle"><a href='https://{{ product.country.locale }}.ryobitools.eu{{ product.url }}'>link</a></td>
<td class="align-middle text-end">
{% if product.isDiscontinued() or product.priceCurrent == product.productStandardPrice %}
{{ product.priceLowest | format_currency(product.country.currency, {}, product.country.locale) }}
{% else %}
{% if product.priceLowest != product.priceCurrent %}{{ product.priceLowest | format_currency(product.country.currency, {}, product.country.locale) }}{% else %}
<span class="badge text-bg-info">now lowest</span>{% endif %}</td>
{% endif %}
<td class="align-middle text-end">{{ product.priceCurrent | format_currency(product.country.currency, {}, product.country.locale) }}</td>
<td class="align-middle"><a href='https://pl.ryobitools.eu/{{ product.url }}'>link</a></td>
<td class="align-middle text-end">{% if product.lowestPrice.price != product.price.last.price %}{{ product.lowestPrice.price | format_currency('PLN', {}, 'pl') }}{% endif %}</td>
<td class="align-middle text-end">{{ product.price.last.price | format_currency('PLN', {}, 'pl') }}</td>
<td class="align-middle">
<div class="d-flex flex-row">
{% if product.priceCurrent != product.productStandardPrice %}<span
class="badge text-bg-warning text-decoration-line-through flex-fill">{{ product.productStandardPrice | format_currency(product.country.currency, {}, product.country.locale) }}</span>
<span
class="badge text-bg-success flex-fill">{{ ((1 - product.priceCurrent / product.productStandardPrice)*100)|number_format(0) }}%</span>
{% if product.price.last.price != product.price.last.productStandardPrice %}<span
class="badge text-bg-warning text-decoration-line-through flex-fill">{{ product.price.last.productStandardPrice | format_currency('PLN', {}, 'pl') }}</span> <span
class="badge text-bg-success flex-fill">{{ ((1 - product.price.last.price / product.price.last.productStandardPrice)*100)|number_format(0) }}%</span>
{% endif %}
</div>
</td>
</tr>
{% endfor %}
</table>
</div>
{% endif %}
{% endfor %}
{# </table>#}
</div>
{% endcache %}
{% endblock %}

View File

@@ -1,5 +1,5 @@
<!doctype html>
<html lang="en" data-bs-theme="light">
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
@@ -18,21 +18,20 @@
</button>
<div class="collapse navbar-collapse" id="navbarNav">
<ul class="navbar-nav me-auto mb-2 mb-lg-0">
{% cache "menu_count" %}
<li class="nav-item">
<a class="nav-link {% if app.request.pathinfo == path('app_home') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_home') }}">All products <span class="badge text-bg-secondary">{{ allCount() }}</span></a>
</li>
<li class="nav-item">
<a class="nav-link {% if app.request.pathinfo == path('app_promos') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_promos') }}">Promos <span class="badge text-bg-secondary">{{ promosCount() }}</span></a>
</li>
<li class="nav-item">
<a class="nav-link {% if app.request.pathinfo == path('app_lowest_price') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_lowest_price') }}">Lowest price <span class="badge text-bg-secondary">{{ lowestPriceCount() }}</span></a>
</li>
<li class="nav-item">
<a class="nav-link {% if app.request.pathinfo == path('app_new') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_new') }}">New in last 30 days <span class="badge text-bg-secondary">{{ newCount() }}</span></a>
</li>
<li class="nav-item">
<a class="nav-link {% if app.request.pathinfo == path('app_discontinued') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_discontinued') }}">Discontinued <span class="badge text-bg-secondary">{{ discontinuedCount() }}</span></a>
</li>
{% endcache %}
</ul>
<form class="form-floating d-flex col-lg-6 col-sm-8" role="search" action="{{ path('app_search') }}">
@@ -45,7 +44,8 @@
</nav>
{% block content %}{% endblock %}
<script src="/templates/js/bootstrap.bundle.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js" integrity="sha384-YvpcrYf0tY3lHB60NNkmXc5s9fDVZLESaAA55NDzOxhy9GkcIdslK1eN7N6jIeHz" crossorigin="anonymous"></script>
</body>
</html>