32 Commits
Author SHA1 Message Date
krzysiej 40095ca7d6 Start working on handling multiple countries at once 2026-01-18 10:13:56 +01:00
krzysiej 22e4034ae3 Start working on handling multiple countries at once 2026-01-17 09:56:40 +01:00
krzysiej 03b74aa33d Start working on handling multiple countries at once 2026-01-16 16:20:21 +01:00
krzysiej e1340d45ed Start working on handling multiple countries at once 2026-01-16 11:20:34 +01:00
krzysiej 4983f868df Start working on handling multiple countries at once 2026-01-16 11:02:57 +01:00
krzysiej f2a9bdf993 Start working on handling multiple countries at once 2026-01-16 08:44:36 +01:00
krzysiej ebe40785fa Clear list cache after clicking on a star item. 2026-01-15 10:29:24 +01:00
krzysiej 4cf1c2f90b Clear list cache after clicking on a star item. 2026-01-15 08:47:17 +01:00
krzysiejandkrzysiej e40391eb4c Fix for the promo prices for items that never had promo. Add new screenshot file. Update bin/update script 2026-01-14 08:42:20 +01:00
krzysiej 4095037b69 Add bootstra *.map files 2026-01-13 08:43:41 +01:00
krzysiej b7ebf7374a Handle gaps in chart data when the product disappeared from the store then appeared again. 2026-01-12 09:03:14 +01:00
krzysiej 45343c9121 Filer items that have currently the lowest price. 2026-01-11 10:32:29 +01:00
krzysiej 653f94f9c9 Update bootstrap and get local bootstrap js file. 2026-01-10 09:32:38 +01:00
krzysiej 632f76aceb Handle cache of main menu 2026-01-10 09:27:09 +01:00
krzysiej cbf143c7a0 Handle lastSeen column and all the discontinued items. 2026-01-10 09:21:59 +01:00
krzysiej 7920172735 Handle lastSeen column and all the discontinued items. 2026-01-10 08:56:31 +01:00
krzysiej 914310dab8 Mark now lowest price as now lowest. 2026-01-09 08:54:34 +01:00
krzysiej 8e8ef8fe04 Increase speed of processing the prices and products. Because this is getting out of hand. 2026-01-08 17:18:46 +01:00
krzysiej de4915972c Merge pull request 'Autowire database connection.' (#29) from feature/autowired-database into master
Reviewed-on: #29
2026-01-04 20:31:32 +01:00
krzysiej 9c2405dd3f Merge branch 'refs/heads/master' into feature/autowired-database
# Conflicts:
#	templates/productList.html.twig
2026-01-04 20:29:35 +01:00
krzysiej da6a8f86c2 Update 2026-01-02 12:21:33 +01:00
krzysiej 2c40fb0e61 Update 2026-01-02 12:15:42 +01:00
krzysiej 5314a6a70a Fix search twig cache. 2026-01-01 18:46:18 +01:00
Krzysztof Płaczek f772532309 Merge remote-tracking branch 'origin/feature/autowired-database' into feature/autowired-database 2025-05-15 11:14:31 +02:00
Krzysztof Płaczek 17159e811f Remove bottom margin for breadcrumbs 2025-05-15 11:14:00 +02:00
Krzysztof Płaczek 76d8b7d9cf Autowire database connection. 2025-05-14 15:15:49 +02:00
Krzysztof Płaczek f30304cbe9 Autowire database connection. 2025-05-14 13:15:36 +02:00
krzysiej dc287aadc6 Merge pull request 'feature/lowest-price' (#27) from feature/lowest-price into master
Reviewed-on: #27
2025-05-14 12:46:12 +02:00
krzysiej 708a5eeae0 Merge pull request 'Implement a caching mechanism' (#28) from feature/cache into master
Reviewed-on: #28
2025-05-14 12:45:57 +02:00
Krzysztof Płaczek 7352eea270 Add the lowest price column to every product list view 2025-05-14 10:28:58 +02:00
Krzysztof Płaczek e97d976705 Implement a caching mechanism 2025-05-14 09:03:25 +02:00
krzysiej beb717f9b9 Add lowest price first draft. 2025-05-13 23:34:51 +02:00
31 changed files with 1013 additions and 561 deletions
+1 -1
View File
@@ -4,7 +4,7 @@
1. Clone repository using `git clone https://git.techtube.pl/krzysiej/ryobi-crawler.git` 1. Clone repository using `git clone https://git.techtube.pl/krzysiej/ryobi-crawler.git`
2. Cd into project directory `cd ryobi-crawler` 2. Cd into project directory `cd ryobi-crawler`
3. Build and start docker container `docker compose up -d` 3. Build and start docker container `docker compose up -d --build --force-recreate`
4. Run `docker compose exec php-app php console.php app:migrate` file to create `database.sqlite` and create tables. 4. Run `docker compose exec php-app php console.php app:migrate` file to create `database.sqlite` and create tables.
5. Run `docker compose exec php-app php console.php app:scrape` command to scrape all the products from the ryobi website. 5. Run `docker compose exec php-app php console.php app:scrape` command to scrape all the products from the ryobi website.
6. Access web interface using `localhost:9001` address in web browser. 6. Access web interface using `localhost:9001` address in web browser.
+2
View File
@@ -0,0 +1,2 @@
#!/usr/bin/env bash
bin/cli php console.php app:cache:warm-twig
+1 -1
View File
@@ -1,3 +1,3 @@
#!/usr/bin/env bash #!/usr/bin/env bash
[ -z "$1" ] && echo "Please specify a Composer command (ex. install)" && exit [ -z "$1" ] && bin/cli composer list && exit
bin/cli composer "$@" bin/cli composer "$@"
+1 -1
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env bash #!/usr/bin/env bash
echo "Updating project" echo "Updating project"
git pull origin master git pull origin master
bin/cli rm -rf var/cache bin/cacheclean
echo "Project updated" echo "Project updated"
Generated
+679 -472
View File
File diff suppressed because it is too large Load Diff
BIN
View File
Binary file not shown.

Before

Width:  |  Height:  |  Size: 199 KiB

After

Width:  |  Height:  |  Size: 282 KiB

+3 -3
View File
@@ -16,7 +16,7 @@ use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\HttpFoundation\RequestStack; use Symfony\Component\HttpFoundation\RequestStack;
use Twig\Environment; use Twig\Environment;
#[AsCommand(name: 'app:cache:warm-twig', description: '')] #[AsCommand(name: 'app:cache:warm-twig', description: 'Warmup twig cache')]
class CacheWarmCommand extends Command class CacheWarmCommand extends Command
{ {
public function __construct( public function __construct(
@@ -42,8 +42,8 @@ class CacheWarmCommand extends Command
$progress = new ProgressBar($output); $progress = new ProgressBar($output);
$progress->start(); $progress->start();
$products = Product::with([ $products = Product::with([
'price' => fn($query) => $query->orderBy('created_at', 'desc'), 'price' => fn($query) => $query->orderByDesc('created_at'),
'stock' => fn($query) => $query->orderBy('created_at', 'desc'), 'stock' => fn($query) => $query->orderByDesc('created_at'),
])->get(); ])->get();
$progress->setMaxSteps(count($products)); $progress->setMaxSteps(count($products));
+88 -3
View File
@@ -11,6 +11,7 @@ use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Output\OutputInterface;
use function Symfony\Component\Clock\now;
#[AsCommand(name: 'app:migrate', description: 'Create database and rum migrations')] #[AsCommand(name: 'app:migrate', description: 'Create database and rum migrations')]
class Migrate extends Command class Migrate extends Command
@@ -24,11 +25,10 @@ class Migrate extends Command
public function execute(InputInterface $input, OutputInterface $output): int public function execute(InputInterface $input, OutputInterface $output): int
{ {
if (true === $input->hasOption(self::RECREATE_OPTION)) { if ($input->getOption(self::RECREATE_OPTION)) {
unlink(__DIR__ . '/../../database.sqlite'); unlink(__DIR__ . '/../../database.sqlite');
//sleep(5); touch(__DIR__ . '/../../database.sqlite');
} }
touch(__DIR__ . '/../../database.sqlite');
$capsule = new Capsule; $capsule = new Capsule;
$capsule->addConnection([ $capsule->addConnection([
'driver' => 'sqlite', 'driver' => 'sqlite',
@@ -39,6 +39,9 @@ class Migrate extends Command
$this->createProductsTable(); $this->createProductsTable();
$this->createPricesTable(); $this->createPricesTable();
$this->createStocksTable(); $this->createStocksTable();
$this->addColumns();
$this->createCountriesTable();
$this->index();
return Command::SUCCESS; return Command::SUCCESS;
} }
@@ -77,6 +80,47 @@ class Migrate extends Command
} }
} }
public function createCountriesTable(): void
{
if (!Capsule::schema()->hasTable('countries')) {
Capsule::schema()->create('countries', function (Blueprint $table) {
$table->increments('id');
$table->text('countryName');
$table->text('productsUrl');
$table->text('cultureCode');
$table->text('currency');
$table->text('locale');
$table->timestamps();
});
}
$id = Capsule::table('countries')->insertGetId(
[
'countryName' => 'Poland',
'productsUrl' => 'https://pl.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'pl-PL',
'currency' => 'PLN',
'locale' => 'pl',
'created_at' => now(),
'updated_at' => now(),
]);
Capsule::table('countries')->insert([
'countryName' => 'UK',
'productsUrl' => 'https://uk.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'en-GB',
'currency' => 'GBP',
'locale' => 'en',
'created_at' => now(),
'updated_at' => now(),
]
);
if (!Capsule::schema()->hasColumn('products', 'country_id')) {
Capsule::schema()->table('products', function (Blueprint $table) use ($id) {
$table->foreignId('country_id')->default($id)->references('id')->on('countries');
});
}
}
public function createStocksTable(): void public function createStocksTable(): void
{ {
if (!Capsule::schema()->hasTable('stocks')) { if (!Capsule::schema()->hasTable('stocks')) {
@@ -94,4 +138,45 @@ class Migrate extends Command
}); });
} }
} }
public function addColumns(): void
{
if (!Capsule::schema()->hasColumn('products', 'priceCurrent')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->float('priceCurrent')->default(0);
});
}
if (!Capsule::schema()->hasColumn('products', 'priceLowest')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->float('priceLowest')->default(0);
});
}
if (!Capsule::schema()->hasColumn('products', 'productStandardPrice')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->float('productStandardPrice')->default(0);
});
}
if (!Capsule::schema()->hasColumn('products', 'lowestProductPrice30Days')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->float('lowestProductPrice30Days')->default(0);
});
}
if (!Capsule::schema()->hasColumn('products', 'lastSeen')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->date('lastSeen')->nullable();
});
}
}
public function index(): void
{
Capsule::schema()->table('products', function (Blueprint $table) {
$table->integer('skuID')->unique(false)->change();
$table->unique(['skuID', 'country_id']);
});
}
} }
+48 -17
View File
@@ -7,6 +7,7 @@ namespace Krzysiej\RyobiCrawler\Command;
use GuzzleHttp\Client; use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException; use GuzzleHttp\Exception\GuzzleException;
use Illuminate\Database\Capsule\Manager as Capsule; use Illuminate\Database\Capsule\Manager as Capsule;
use Krzysiej\RyobiCrawler\Models\Country;
use Krzysiej\RyobiCrawler\Models\Price; use Krzysiej\RyobiCrawler\Models\Price;
use Krzysiej\RyobiCrawler\Models\Product; use Krzysiej\RyobiCrawler\Models\Product;
use Krzysiej\RyobiCrawler\Models\Stock; use Krzysiej\RyobiCrawler\Models\Stock;
@@ -21,47 +22,71 @@ class ScrapeWebsite extends Command
{ {
private Client $client; private Client $client;
public function __construct(protected Capsule $database)
{
parent::__construct();
}
protected function configure(): void protected function configure(): void
{ {
$capsule = new Capsule;
$capsule->addConnection([
'driver' => 'sqlite',
'database' => __DIR__ . '/../../database.sqlite',
]);
$capsule->setAsGlobal();
$capsule->bootEloquent();
$this->client = new Client(); $this->client = new Client();
} }
public function execute(InputInterface $input, OutputInterface $output): int public function execute(InputInterface $input, OutputInterface $output): int
{ {
$output->writeln('Scrape products');
$progress = new ProgressBar($output); $progress = new ProgressBar($output);
$progress->start(); $countries = Country::all();
$products = $this->getProducts(); foreach($countries as $country) {
$output->writeln('Country name: ' . $country->countryName."\n");
$progress->start();
$products = $this->getProducts($country);
$progress->setMaxSteps(count($products));
foreach ($products as $product) {
$this->saveProduct($product, $country);
$progress->advance();
}
$progress->finish();
$output->writeln('');
$output->writeln('Scrape products - DONE');
$output->writeln('');
}
$output->writeln('Update prices');
$products = Product::all();
$progress->setMaxSteps(count($products)); $progress->setMaxSteps(count($products));
foreach ($products as $product) { $progress->start();
$this->saveProduct($product); foreach($products as $product) {
$newestPrice = $product->newestPrice;
$product->priceCurrent = $newestPrice->price;
$product->productStandardPrice = $newestPrice->productStandardPrice;
$product->lowestProductPrice30Days = $newestPrice->lowestProductPrice30Days;
$product->priceLowest = $product->lowestPrice->price;
$product->lastSeen = $newestPrice->created_at->format('Y-m-d');
$product->save(['timestamps' => false]);
$progress->advance(); $progress->advance();
} }
$progress->finish(); $progress->finish();
$output->writeln(''); $output->writeln('');
$output->writeln('DONE'); $output->writeln('Update prices - DONE');
$output->writeln('COMMAND - DONE');
return Command::SUCCESS; return Command::SUCCESS;
} }
private function getProducts(): array private function getProducts(Country $country): array
{ {
$products = []; $products = [];
$page = 0; $page = 0;
do { do {
try { try {
$res = $this->client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [ $res = $this->client->request('POST', $country->productsUrl, [
'form_params' => [ 'form_params' => [
"includePreviousPages" => false, "includePreviousPages" => false,
"pageIndex" => $page, "pageIndex" => $page,
"pageSize" => 100, "pageSize" => 100,
"cultureCode" => "pl-PL", "cultureCode" => $country->cultureCode,
] ]
]); ]);
$responseObject = json_decode($res->getBody()->getContents()); $responseObject = json_decode($res->getBody()->getContents());
@@ -76,10 +101,14 @@ class ScrapeWebsite extends Command
return $products; return $products;
} }
private function saveProduct(\stdClass $product): void private function saveProduct(\stdClass $product, Country $country): void
{ {
// if ($product->skuID == 0) {
// dump([$product->skuID, $product->name]);
// }
// return;
/** @var Product $productModel */ /** @var Product $productModel */
$productModel = Product::firstOrNew(['skuID' => $product->skuID]); $productModel = Product::firstOrNew(['skuID' => $product->skuID, 'country_id' => $country->id]);
$productModel->skuID = $product->skuID; $productModel->skuID = $product->skuID;
$productModel->name = $product->name; $productModel->name = $product->name;
@@ -90,7 +119,9 @@ class ScrapeWebsite extends Command
$productModel->variantCode = $product->variantCode; $productModel->variantCode = $product->variantCode;
$productModel->modelCode = $product->modelCode; $productModel->modelCode = $product->modelCode;
$productModel->url = $product->url; $productModel->url = $product->url;
$productModel->lastSeen = date("Y-m-d");
$productModel->touch('updated_at'); $productModel->touch('updated_at');
$productModel->country()->associate($country);
$productModel->save(); $productModel->save();
$priceExists = $productModel->price()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists(); $priceExists = $productModel->price()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();
+1 -5
View File
@@ -11,11 +11,7 @@ class BaseController extends AbstractController
{ {
protected Environment $twig; protected Environment $twig;
public function __construct(protected FilesystemAdapter $cache) public function __construct(protected FilesystemAdapter $cache, protected Capsule $database)
{ {
$capsule = new Capsule;
$capsule->addConnection(['driver' => 'sqlite', 'database' => __DIR__ . '/../../database.sqlite']);
$capsule->setAsGlobal();
$capsule->bootEloquent();
} }
} }
+1 -1
View File
@@ -15,7 +15,7 @@ final class CategoryController extends BaseController
return $this->render('productList.html.twig', ['listType' => 'category_'.$category]); return $this->render('productList.html.twig', ['listType' => 'category_'.$category]);
} }
$products = Product::with('price') $products = Product::with(['price', 'lowestPrice'])
->selectRaw('products.*') ->selectRaw('products.*')
->distinct('products.id') ->distinct('products.id')
->fromRaw('products, json_each(products.categories)') ->fromRaw('products, json_each(products.categories)')
+1 -2
View File
@@ -17,10 +17,9 @@ final class DiscontinuedController extends BaseController
return $this->render('productList.html.twig', ['listType' => 'discontinued']); return $this->render('productList.html.twig', ['listType' => 'discontinued']);
} }
$products = Product::where('updated_at', '<', now()->format('Y-m-d')) $products = Product::where('lastSeen', '<', now()->format('Y-m-d'))
->orderByDesc('starred') ->orderByDesc('starred')
->orderByDesc('created_by') ->orderByDesc('created_by')
->with(['currentPrice'])
->get(); ->get();
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'discontinued']); return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'discontinued']);
} }
+2 -2
View File
@@ -11,10 +11,10 @@ final class IndexController extends BaseController
#[Route('/', name: 'app_home')] #[Route('/', name: 'app_home')]
public function __invoke(): Response public function __invoke(): Response
{ {
if($this->cache->getItem('list_all')->isHit()) { if ($this->cache->getItem('list_all')->isHit()) {
return $this->render('productList.html.twig', ['listType' => 'all']); return $this->render('productList.html.twig', ['listType' => 'all']);
} }
$products = Product::with(['currentStock', 'price']) $products = Product::with(['currentStock'])
->orderByDesc('starred') ->orderByDesc('starred')
->orderByDesc('created_by') ->orderByDesc('created_by')
->get(); ->get();
+29
View File
@@ -0,0 +1,29 @@
<?php
namespace Krzysiej\RyobiCrawler\Controller;
use Krzysiej\RyobiCrawler\Models\Product;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Attribute\Route;
use function Symfony\Component\Clock\now;
final class LowestPriceController extends BaseController
{
#[Route('/lowest-price', name: 'app_lowest_price', methods: ['GET'])]
public function __invoke(): Response
{
$listType = 'lowest_price';
if($this->cache->getItem('lowest_price')->isHit()) {
return $this->render('productList.html.twig', ['listType' => $listType]);
}
$products = Product::whereRaw('priceCurrent = priceLowest')
->whereRaw('lastSeen = "'.now()->format('Y-m-d').'"')
->whereRaw('priceCurrent < productStandardPrice')
->orderByDesc('starred')
->orderByDesc('created_by')
->with(['currentPrice', 'lowestPrice'])
->get();
return $this->render('productList.html.twig', ['products' => $products, 'listType' => $listType]);
}
}
+1 -2
View File
@@ -2,7 +2,6 @@
namespace Krzysiej\RyobiCrawler\Controller; namespace Krzysiej\RyobiCrawler\Controller;
use Illuminate\Database\Eloquent\Builder;
use Krzysiej\RyobiCrawler\Models\Product; use Krzysiej\RyobiCrawler\Models\Product;
use Symfony\Component\HttpFoundation\Response; use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Attribute\Route; use Symfony\Component\Routing\Attribute\Route;
@@ -21,7 +20,7 @@ final class NewController extends BaseController
$products = Product::where('created_at', '>', now()->modify('-30 days')->format('Y-m-d')) $products = Product::where('created_at', '>', now()->modify('-30 days')->format('Y-m-d'))
->orderByDesc('starred') ->orderByDesc('starred')
->orderByDesc('created_by') ->orderByDesc('created_by')
->with(['currentPrice']) ->with(['currentPrice', 'lowestPrice'])
->get(); ->get();
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'new']); return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'new']);
} }
+22 -15
View File
@@ -3,7 +3,6 @@
namespace Krzysiej\RyobiCrawler\Controller; namespace Krzysiej\RyobiCrawler\Controller;
use Krzysiej\RyobiCrawler\Models\Product; use Krzysiej\RyobiCrawler\Models\Product;
use Symfony\Component\Cache\Adapter\FilesystemAdapter;
use Symfony\Component\HttpFoundation\Response; use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Attribute\Route; use Symfony\Component\Routing\Attribute\Route;
use Twig\Error\LoaderError; use Twig\Error\LoaderError;
@@ -20,7 +19,7 @@ final class ProductController extends BaseController
#[Route('/product/{productId<\d+>}', name: 'app_product')] #[Route('/product/{productId<\d+>}', name: 'app_product')]
public function __invoke(int $productId): Response public function __invoke(int $productId): Response
{ {
if($this->cache->getItem('product'.$productId)->isHit()) { if ($this->cache->getItem('product' . $productId)->isHit()) {
return $this->render('product.html.twig', ['product' => ['id' => $productId]]); return $this->render('product.html.twig', ['product' => ['id' => $productId]]);
} }
@@ -31,26 +30,34 @@ final class ProductController extends BaseController
if (null === $product) { if (null === $product) {
throw $this->createNotFoundException('Product not found'); throw $this->createNotFoundException('Product not found');
} }
$priceList = $product->price()->pluck('price'); $priceList = $product->price()->pluck('price', 'created_at')->mapWithKeys(fn($price, $createdAt) => [explode(' ', $createdAt)[0] => $price])->toArray();
$stockList = $product->stock()->pluck('stock'); $stockList = $product->stock()->pluck('stock', 'created_at')->mapWithKeys(fn($stock, $createdAt) => [explode(' ', $createdAt)[0] => $stock])->toArray();
$priceDates = $product->price()->pluck('created_at')->map(fn($date) => $date->format('Y-m-d'))->toArray();
$stockDates = $product->stock()->pluck('created_at')->map(fn($date) => $date->format('Y-m-d'))->toArray();
return $this->render('product.html.twig', [ return $this->render('product.html.twig', [
'product' => $product, 'product' => $product,
'price_list' => $this->prepareChartData($priceDates, $priceList), 'price_list' => $this->prepareChartData($priceList),
'stock_list' => $this->prepareChartData($stockDates, $stockList), 'stock_list' => $this->prepareChartData($stockList),
'price_dates' => implode("','", $priceDates), 'price_dates' => implode("','", $this->dateRange(array_key_first($priceList), array_key_last($priceList))),
]); ]);
} }
private function prepareChartData($set1, $set2): string private function prepareChartData($set1): string
{ {
$data = []; $dates = $this->dateRange(array_key_first($set1), array_key_last($set1));
foreach ($set1 as $key => $value) { $data = array_map(fn($date) => ['x' => $date, 'y' => $set1[$date] ?? null], $dates);
$data[] = ['x' => $value, 'y' => $set2[$key]];
}
$stringData = json_encode($data); $stringData = json_encode($data);
return str_replace(['"x"', '"y"'], ['x', 'y'], $stringData); return str_replace(['"x"', '"y"'], ['x', 'y'], $stringData);
} }
private function dateRange($dateStart, $dateEnd): array
{
$from = new \DateTime($dateStart);
$to = new \DateTime($dateEnd);
$range = [];
for ($date = clone $from; $date < $to; $date->modify('+1 day')) {
$range[] = $date->format('Y-m-d');
}
return $range;
}
} }
+2 -3
View File
@@ -2,7 +2,6 @@
namespace Krzysiej\RyobiCrawler\Controller; namespace Krzysiej\RyobiCrawler\Controller;
use Illuminate\Database\Eloquent\Builder;
use Krzysiej\RyobiCrawler\Models\Product; use Krzysiej\RyobiCrawler\Models\Product;
use Symfony\Component\HttpFoundation\Response; use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Attribute\Route; use Symfony\Component\Routing\Attribute\Route;
@@ -16,10 +15,10 @@ final class PromosController extends BaseController
return $this->render('productList.html.twig', ['listType' => 'promos']); return $this->render('productList.html.twig', ['listType' => 'promos']);
} }
$products = Product::whereHas('currentPrice', fn(Builder $query) => $query->whereColumn('price', '<', 'productStandardPrice')) $products = Product::whereRaw('priceCurrent < productStandardPrice')
->orderByDesc('starred') ->orderByDesc('starred')
->orderByDesc('created_by') ->orderByDesc('created_by')
->with(['currentPrice']) ->with(['currentPrice', 'lowestPrice'])
->get(); ->get();
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'promos']); return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'promos']);
} }
+1 -1
View File
@@ -17,6 +17,6 @@ final class SearchController extends BaseController
->orWhere([['name', 'like', "%$search%"]]) ->orWhere([['name', 'like', "%$search%"]])
->orWhere([['subTitle', 'like', "%$search%"]])->get(); ->orWhere([['subTitle', 'like', "%$search%"]])->get();
return $this->render('productList.html.twig', ['products' => $products ?? [], 'search' => $search]); return $this->render('productList.html.twig', ['products' => $products ?? [], 'search' => $search, 'listType' => 'search_'.$search]);
} }
} }
+12 -1
View File
@@ -12,7 +12,18 @@ final class StarController extends BaseController
#[Route('/star/{productId<\d+>}', name: 'app_star')] #[Route('/star/{productId<\d+>}', name: 'app_star')]
public function __invoke(int $productId, Request $request): Response public function __invoke(int $productId, Request $request): Response
{ {
$this->cache->deleteItems(['list_all', 'list_promos', 'list_new', 'list_discontinued']);
$referer = $request->headers->get('referer');
if (str_contains($referer, '/category/')) {
preg_match('#/category/(.*)#i', $referer, $matches);
$this->cache->deleteItem('list_category_'.urldecode($matches[1]));
}
if (str_contains($referer, '/search?search=')) {
preg_match('#/search\?search=(.*)#i', $referer, $matches);
$this->cache->deleteItem('list_search_'.urldecode($matches[1]));
}
Product::find($productId)->toggleStarred()->save(); Product::find($productId)->toggleStarred()->save();
return $this->redirect($request->headers->get('referer')); return $this->redirect($referer);
} }
} }
+14
View File
@@ -0,0 +1,14 @@
<?php
namespace Krzysiej\RyobiCrawler;
use Illuminate\Database\Capsule\Manager as Capsule;
class DatabaseFactory
{
public static function create(Capsule $capsule): void
{
$capsule->addConnection(['driver' => 'sqlite', 'database' => __DIR__ . '/../database.sqlite']);
$capsule->setAsGlobal();
$capsule->bootEloquent();
}
}
+5 -5
View File
@@ -2,6 +2,7 @@
namespace Krzysiej\RyobiCrawler; namespace Krzysiej\RyobiCrawler;
use Illuminate\Database\Capsule\Manager;
use Krzysiej\RyobiCrawler\Twig\AppExtension; use Krzysiej\RyobiCrawler\Twig\AppExtension;
use Symfony\Bundle\FrameworkBundle\FrameworkBundle; use Symfony\Bundle\FrameworkBundle\FrameworkBundle;
use Symfony\Bundle\FrameworkBundle\Kernel\MicroKernelTrait; use Symfony\Bundle\FrameworkBundle\Kernel\MicroKernelTrait;
@@ -34,14 +35,13 @@ class Kernel extends BaseKernel
'secret' => 'S0ME_SECRET' 'secret' => 'S0ME_SECRET'
]); ]);
$services = $container->services()->defaults()->autowire()->autoconfigure(); $services = $container->services()->defaults()->autowire()->autoconfigure();
$services->load('Krzysiej\\RyobiCrawler\\Controller\\', __DIR__ . '/Controller/*'); $services->set(Manager::class)->configurator([DatabaseFactory::class, 'create']);
$services->load('Krzysiej\\RyobiCrawler\\Command\\', __DIR__ . '/Command/*')->tag('console.command'); $services->load('Krzysiej\\RyobiCrawler\\', __DIR__ )
->exclude('../src/{Models,Twig,DatabaseFactory.php,Kernel.php}');
$services->set('twig.extension.cache', AppExtension::class)->tag('twig.extension'); $services->set('twig.extension.cache', AppExtension::class)->tag('twig.extension');
$services->set(CacheExtension::class)->tag('twig.extension'); $services->set(CacheExtension::class)->tag('twig.extension');
$services->set(FilesystemAdapter::class)->args([ $services->set(FilesystemAdapter::class)->args([
'', // namespace '$directory' => __DIR__ . '/../var/cache/twig_blocks'
0, // default lifetime
__DIR__ . '/../var/cache/twig_blocks' // custom path
]); ]);
$services->set('twig.runtime.cache', CacheRuntime::class)->args([new Reference(FilesystemAdapter::class)])->tag('twig.runtime'); $services->set('twig.runtime.cache', CacheRuntime::class)->args([new Reference(FilesystemAdapter::class)])->tag('twig.runtime');
} }
+24
View File
@@ -0,0 +1,24 @@
<?php
namespace Krzysiej\RyobiCrawler\Models;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use Illuminate\Database\Eloquent\Relations\HasMany;
/**
* @property string $countryName
* @property string $productsUrl
* @property string $cultureCode
* @property string $currency
* @property string $locale
*/
class Country extends Model
{
public $timestamps = true;
public function products(): HasMany
{
return $this->hasMany(Product::class);
}
}
+23 -4
View File
@@ -2,8 +2,10 @@
namespace Krzysiej\RyobiCrawler\Models; namespace Krzysiej\RyobiCrawler\Models;
use Carbon\Traits\Date;
use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\Relations\HasOne; use Illuminate\Database\Eloquent\Relations\HasOne;
@@ -20,12 +22,22 @@ use function Symfony\Component\Clock\now;
* @property string $modelCode * @property string $modelCode
* @property string $url * @property string $url
* @property int $starred * @property int $starred
* @property float $priceCurrent
* @property float $priceLowest
* @property float $productStandardPrice
* @property float $lowestProductPrice30Days
* @property Date $lastSeen
*/ */
class Product extends Model class Product extends Model
{ {
public $timestamps = true; public $timestamps = true;
public $fillable = ['skuID']; public $fillable = ['skuID'];
public function country(): BelongsTo
{
return $this->belongsTo(Country::class);
}
public function price(): HasMany public function price(): HasMany
{ {
return $this->hasMany(Price::class); return $this->hasMany(Price::class);
@@ -41,6 +53,15 @@ class Product extends Model
return $this->hasOne(Price::class)->latestOfMany('created_at'); return $this->hasOne(Price::class)->latestOfMany('created_at');
} }
public function lowestPrice(): HasOne
{
return $this->hasOne(Price::class)->ofMany('price', 'MIN');
}
public function newestPrice(): HasOne
{
return $this->hasOne(Price::class)->latest();
}
public function stock(): HasMany public function stock(): HasMany
{ {
return $this->hasMany(Stock::class); return $this->hasMany(Stock::class);
@@ -48,9 +69,7 @@ class Product extends Model
public function currentStock(): HasOne public function currentStock(): HasOne
{ {
return $this->stock()->one()->ofMany()->withDefault(function (Stock $stock) { return $this->stock()->one()->ofMany()->withDefault(fn (Stock $stock) => $stock->stock = 0);
$stock->stock = 0;
});
} }
public function toggleStarred(): self public function toggleStarred(): self
@@ -70,7 +89,7 @@ class Product extends Model
public function isDiscontinued(): bool public function isDiscontinued(): bool
{ {
return $this->updated_at->format('Y-m-d') < now()->format('Y-m-d'); return $this->lastSeen < now()->format('Y-m-d');
} }
public function isNew(): bool public function isNew(): bool
{ {
+12 -2
View File
@@ -22,6 +22,7 @@ class AppExtension extends AbstractExtension
new TwigFunction('allCount', [$this, 'allCount']), new TwigFunction('allCount', [$this, 'allCount']),
new TwigFunction('newCount', [$this, 'newCount']), new TwigFunction('newCount', [$this, 'newCount']),
new TwigFunction('discontinuedCount', [$this, 'discontinuedCount']), new TwigFunction('discontinuedCount', [$this, 'discontinuedCount']),
new TwigFunction('lowestPriceCount', [$this, 'lowestPriceCount']),
]; ];
} }
@@ -31,6 +32,7 @@ class AppExtension extends AbstractExtension
new TwigFilter('findByCreatedAtDate', [$this, 'findByCreatedAtDate']), new TwigFilter('findByCreatedAtDate', [$this, 'findByCreatedAtDate']),
]; ];
} }
public function allCount(): int public function allCount(): int
{ {
return Product::count(); return Product::count();
@@ -38,7 +40,7 @@ class AppExtension extends AbstractExtension
public function promosCount(): int public function promosCount(): int
{ {
return Product::whereHas('currentPrice', fn(Builder $query) => $query->whereColumn('price', '<', 'productStandardPrice'))->count(); return Product::whereRaw('priceCurrent < productStandardPrice')->count();
} }
public function newCount(): int public function newCount(): int
@@ -48,7 +50,15 @@ class AppExtension extends AbstractExtension
public function discontinuedCount(): int public function discontinuedCount(): int
{ {
return Product::where('updated_at', '<', now()->format('Y-m-d'))->count(); return Product::where('lastSeen', '<>', now()->format('Y-m-d'))->count();
}
public function lowestPriceCount(): int
{
return Product::whereRaw('priceCurrent = priceLowest')
->whereRaw('lastSeen = "'.now()->format('Y-m-d').'"')
->whereRaw('priceCurrent < productStandardPrice')
->count();
} }
public function findByCreatedAtDate(Collection $items, string $date): Stock|Price|null public function findByCreatedAtDate(Collection $items, string $date): Stock|Price|null
+3 -3
View File
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+7 -6
View File
@@ -10,7 +10,7 @@
<td><img src='{{ product.image }}&width=150' class='border rounded p-1' alt='{{ product.name }}'/></td> <td><img src='{{ product.image }}&width=150' class='border rounded p-1' alt='{{ product.name }}'/></td>
<td> <td>
<a href='{{ path('app_product', {'productId': product.id}) }}' class="text-decoration-none">{{ product.name }}</a> <a href='{{ path('app_product', {'productId': product.id}) }}' class="text-decoration-none">{{ product.name }}</a>
<span class="badge text-bg-light">{{ product.subTitle }}</span> <span class="badge text-bg-light"><a href="{{ path('app_search', {'search': product.subTitle}) }}" class="link-underline link-underline-opacity-0 link-dark">{{ product.subTitle }}</a></span>
</td> </td>
<td> <td>
<nav aria-label="breadcrumb" style="--bs-breadcrumb-divider: '>';"> <nav aria-label="breadcrumb" style="--bs-breadcrumb-divider: '>';">
@@ -44,9 +44,9 @@
</thead> </thead>
{% for price in product.price %} {% for price in product.price %}
<tr> <tr>
<td>{{ price.price | format_currency('PLN', {}, 'pl') }}</td> <td>{{ price.price | format_currency(product.country.currency, {}, product.country.locale) }}</td>
<td>{{ price.lowestProductPrice30Days | format_currency('PLN', {}, 'pl') }}</td> <td>{{ price.lowestProductPrice30Days | format_currency(product.country.currency, {}, product.country.locale) }}</td>
<td>{{ price.productStandardPrice | format_currency('PLN', {}, 'pl') }}</td> <td>{{ price.productStandardPrice | format_currency(product.country.currency, {}, product.country.locale) }}</td>
<td>{{ price.created_at }}</td> <td>{{ price.created_at }}</td>
<td>{{ (product.stock | findByCreatedAtDate(price.created_at | slice(0,10))).stock ?? '' }}</td> <td>{{ (product.stock | findByCreatedAtDate(price.created_at | slice(0,10))).stock ?? '' }}</td>
</tr> </tr>
@@ -67,7 +67,7 @@
labels: ['{{ price_dates|raw }}'], labels: ['{{ price_dates|raw }}'],
datasets: [ datasets: [
{ {
label: 'Price (PLN)', label: 'Price ({{ product.country.currency }})',
data: {{ price_list|raw }}, data: {{ price_list|raw }},
yAxisID: 'yPrice', yAxisID: 'yPrice',
tension: 0.1, tension: 0.1,
@@ -83,6 +83,7 @@
] ]
}, },
options: { options: {
spanGaps: false,
responsive: true, responsive: true,
animation: false, animation: false,
scales: { scales: {
@@ -98,7 +99,7 @@
beginAtZero: true, beginAtZero: true,
title: { title: {
display: true, display: true,
text: 'Price (PLN)' text: 'Price ({{ product.country.currency }})'
}, },
grid: { grid: {
drawOnChartArea: false drawOnChartArea: false
+16 -8
View File
@@ -11,7 +11,8 @@
<th>Name</th> <th>Name</th>
<th>Categories</th> <th>Categories</th>
<th></th> <th></th>
<th>Price</th> <th class="text-end">Lowest Price</th>
<th class="text-end">Current Price</th>
<th></th> <th></th>
</tr> </tr>
</thead> </thead>
@@ -28,16 +29,16 @@
<span class="badge text-bg-warning">out of stock</span> <span class="badge text-bg-warning">out of stock</span>
{% endif %} {% endif %}
{% if product.isDiscontinued() %} {% if product.isDiscontinued() %}
<span class="badge text-bg-secondary" data-bs-toggle="tooltip" data-bs-title="Last update: {{ product.updated_at }}">is discontinued</span> <span class="badge text-bg-secondary" data-bs-toggle="tooltip" data-bs-title="Last update: {{ product.lastSeen }}">is discontinued</span>
{% endif %} {% endif %}
{% if product.isNew() %} {% if product.isNew() %}
<span class="badge text-bg-success">is new</span> <span class="badge text-bg-success">is new</span>
{% endif %} {% endif %}
<span class="badge text-bg-light">{{ product.subTitle }}</span> <span class="badge text-bg-light"><a href="{{ path('app_search', {'search': product.subTitle}) }}" class="link-underline link-underline-opacity-0 link-dark">{{ product.subTitle }}</a></span>
</td> </td>
<td class="align-middle"> <td class="align-middle">
<nav aria-label="breadcrumb" style="--bs-breadcrumb-divider: '>';" > <nav aria-label="breadcrumb" style="--bs-breadcrumb-divider: '>';" >
<ol class="breadcrumb"> <ol class="breadcrumb mb-0">
{% for category in product.categories %} {% for category in product.categories %}
<li class="breadcrumb-item" aria-current="page"><a class="breadcrumb-item text-decoration-none" href="{{ path('app_category', {'category': category}) }}">{{ category }}</a></li> <li class="breadcrumb-item" aria-current="page"><a class="breadcrumb-item text-decoration-none" href="{{ path('app_category', {'category': category}) }}">{{ category }}</a></li>
{% endfor %} {% endfor %}
@@ -45,12 +46,19 @@
</nav> </nav>
</td> </td>
<td class="align-middle"><a href='https://pl.ryobitools.eu/{{ product.url }}'>link</a></td> <td class="align-middle"><a href='https://pl.ryobitools.eu/{{ product.url }}'>link</a></td>
<td class="align-middle">{{ product.price.last.price | format_currency('PLN', {}, 'pl') }}</td> <td class="align-middle text-end">
{% if product.isDiscontinued() or product.priceCurrent == product.productStandardPrice %}
{{ product.priceLowest | format_currency(product.country.currency, {}, product.country.locale) }}
{% else %}
{% if product.priceLowest != product.priceCurrent %}{{ product.priceLowest | format_currency(product.country.currency, {}, product.country.locale) }}{%else%}<span class="badge text-bg-info">now lowest</span>{% endif %}</td>
{% endif %}
<td class="align-middle text-end">{{ product.priceCurrent | format_currency(product.country.currency, {}, product.country.locale) }}</td>
<td class="align-middle"> <td class="align-middle">
<div class="d-flex flex-row"> <div class="d-flex flex-row">
{% if product.price.last.price != product.price.last.productStandardPrice %}<span {% if product.priceCurrent != product.productStandardPrice %}<span
class="badge text-bg-warning text-decoration-line-through flex-fill">{{ product.price.last.productStandardPrice | format_currency('PLN', {}, 'pl') }}</span>&nbsp;<span class="badge text-bg-warning text-decoration-line-through flex-fill">{{ product.productStandardPrice | format_currency(product.country.currency, {}, product.country.locale) }}</span> <span
class="badge text-bg-success flex-fill">{{ ((1 - product.price.last.price / product.price.last.productStandardPrice)*100)|number_format(0) }}%</span> class="badge text-bg-success flex-fill">{{ ((1 - product.priceCurrent / product.productStandardPrice)*100)|number_format(0) }}%</span>
{% endif %} {% endif %}
</div> </div>
</td> </td>
+5 -3
View File
@@ -18,13 +18,16 @@
</button> </button>
<div class="collapse navbar-collapse" id="navbarNav"> <div class="collapse navbar-collapse" id="navbarNav">
<ul class="navbar-nav me-auto mb-2 mb-lg-0"> <ul class="navbar-nav me-auto mb-2 mb-lg-0">
{% cache "menu_count" %} {% cache "menu_count" ~ listType|default('') %}
<li class="nav-item"> <li class="nav-item">
<a class="nav-link {% if app.request.pathinfo == path('app_home') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_home') }}">All products <span class="badge text-bg-secondary">{{ allCount() }}</span></a> <a class="nav-link {% if app.request.pathinfo == path('app_home') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_home') }}">All products <span class="badge text-bg-secondary">{{ allCount() }}</span></a>
</li> </li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link {% if app.request.pathinfo == path('app_promos') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_promos') }}">Promos <span class="badge text-bg-secondary">{{ promosCount() }}</span></a> <a class="nav-link {% if app.request.pathinfo == path('app_promos') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_promos') }}">Promos <span class="badge text-bg-secondary">{{ promosCount() }}</span></a>
</li> </li>
<li class="nav-item">
<a class="nav-link {% if app.request.pathinfo == path('app_lowest_price') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_lowest_price') }}">Lowest price <span class="badge text-bg-secondary">{{ lowestPriceCount() }}</span></a>
</li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link {% if app.request.pathinfo == path('app_new') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_new') }}">New in last 30 days <span class="badge text-bg-secondary">{{ newCount() }}</span></a> <a class="nav-link {% if app.request.pathinfo == path('app_new') %}active shadow-sm bg-body rounded{% endif %}" aria-current="page" href="{{ path('app_new') }}">New in last 30 days <span class="badge text-bg-secondary">{{ newCount() }}</span></a>
</li> </li>
@@ -44,8 +47,7 @@
</nav> </nav>
{% block content %}{% endblock %} {% block content %}{% endblock %}
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js" integrity="sha384-YvpcrYf0tY3lHB60NNkmXc5s9fDVZLESaAA55NDzOxhy9GkcIdslK1eN7N6jIeHz" crossorigin="anonymous"></script> <script src="/templates/js/bootstrap.bundle.min.js"></script>
</body> </body>
</html> </html>