krzysiej/ryobi-crawler
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases 1 Wiki Activity

Increase speed of processing the prices and products. Because this is getting out of hand.

Browse Source
This commit was merged in pull request #33.
This commit is contained in:
Krzysiej
2026-01-08 17:18:46 +01:00
parent de4915972c
commit 8e8ef8fe04
7 changed files with 67 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/Command/Migrate.php
View File

@@ -24,11 +24,10 @@ class Migrate extends Command
public function execute(InputInterface $input, OutputInterface $output): int
{
if (true === $input->hasOption(self::RECREATE_OPTION)) {
if ($input->getOption(self::RECREATE_OPTION)) {
unlink(__DIR__ . '/../../database.sqlite');
//sleep(5);
touch(__DIR__ . '/../../database.sqlite');
}
touch(__DIR__ . '/../../database.sqlite');
$capsule = new Capsule;
$capsule->addConnection([
'driver' => 'sqlite',
@@ -39,6 +38,7 @@ class Migrate extends Command
$this->createProductsTable();
$this->createPricesTable();
$this->createStocksTable();
$this->updateProductsTableAddPrices();
return Command::SUCCESS;
}
@@ -94,4 +94,31 @@ class Migrate extends Command
});
}
}
public function updateProductsTableAddPrices(): void
{
if (!Capsule::schema()->hasColumn('products', 'priceCurrent')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->float('priceCurrent')->default(0);
});
}
if (!Capsule::schema()->hasColumn('products', 'priceLowest')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->float('priceLowest')->default(0);
});
}
if (!Capsule::schema()->hasColumn('products', 'productStandardPrice')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->float('productStandardPrice')->default(0);
});
}
if (!Capsule::schema()->hasColumn('products', 'lowestProductPrice30Days')) {
Capsule::schema()->table('products', function (Blueprint $table) {
$table->float('lowestProductPrice30Days')->default(0);
});
}
}
}

20
src/Command/ScrapeWebsite.php
View File

@@ -33,6 +33,7 @@ class ScrapeWebsite extends Command
public function execute(InputInterface $input, OutputInterface $output): int
{
$output->writeln('Scrape products');
$progress = new ProgressBar($output);
$progress->start();
$products = $this->getProducts();
@@ -42,8 +43,25 @@ class ScrapeWebsite extends Command
$progress->advance();
}
$progress->finish();
$output->writeln('Scrape products - DONE');
$output->writeln('');
$output->writeln('DONE');
$output->writeln('Update prices');
$products = Product::all();
$progress->setMaxSteps(count($products));
$progress->start();
foreach($products as $product) {
$product->priceCurrent = $product->newestPrice->price;
$product->productStandardPrice = $product->newestPrice->productStandardPrice;
$product->lowestProductPrice30Days = $product->newestPrice->lowestProductPrice30Days;
$product->priceLowest = $product->lowestPrice->price;
$product->save(['timestamps' => false]);
$progress->advance();
}
$progress->finish();
$output->writeln('');
$output->writeln('Update prices - DONE');
$output->writeln('COMMAND - DONE');
return Command::SUCCESS;
}

2
src/Controller/IndexController.php
View File

@@ -14,7 +14,7 @@ final class IndexController extends BaseController
if ($this->cache->getItem('list_all')->isHit()) {
return $this->render('productList.html.twig', ['listType' => 'all']);
}
$products = Product::with(['currentStock', 'price', 'lowestPrice'])
$products = Product::with(['currentStock'])
->orderByDesc('starred')
->orderByDesc('created_by')
->get();

2
src/Controller/PromosController.php
View File

@@ -16,7 +16,7 @@ final class PromosController extends BaseController
return $this->render('productList.html.twig', ['listType' => 'promos']);
}
$products = Product::whereHas('currentPrice', fn(Builder $query) => $query->whereColumn('price', '<', 'productStandardPrice'))
$products = Product::whereRaw('priceCurrent < productStandardPrice')
->orderByDesc('starred')
->orderByDesc('created_by')
->with(['currentPrice', 'lowestPrice'])

12
src/Models/Product.php
View File

@@ -20,6 +20,10 @@ use function Symfony\Component\Clock\now;
* @property string $modelCode
* @property string $url
* @property int $starred
* @property float $priceCurrent
* @property float $priceLowest
* @property float $productStandardPrice
* @property float $lowestProductPrice30Days
*/
class Product extends Model
{
@@ -45,6 +49,10 @@ class Product extends Model
{
return $this->hasOne(Price::class)->ofMany('price', 'MIN');
}
public function newestPrice(): HasOne
{
return $this->hasOne(Price::class)->latest();
}
public function stock(): HasMany
{
@@ -53,9 +61,7 @@ class Product extends Model
public function currentStock(): HasOne
{
return $this->stock()->one()->ofMany()->withDefault(function (Stock $stock) {
$stock->stock = 0;
});
return $this->stock()->one()->ofMany()->withDefault(fn (Stock $stock) => $stock->stock = 0);
}
public function toggleStarred(): self

3
src/Twig/AppExtension.php
View File

@@ -31,6 +31,7 @@ class AppExtension extends AbstractExtension
new TwigFilter('findByCreatedAtDate', [$this, 'findByCreatedAtDate']),
];
}
public function allCount(): int
{
return Product::count();
@@ -38,7 +39,7 @@ class AppExtension extends AbstractExtension
public function promosCount(): int
{
return Product::whereHas('currentPrice', fn(Builder $query) => $query->whereColumn('price', '<', 'productStandardPrice'))->count();
return Product::whereRaw('priceCurrent < productStandardPrice')->count();
}
public function newCount(): int

10
templates/productList.html.twig
View File

@@ -46,13 +46,13 @@
</nav>
</td>
<td class="align-middle"><a href='https://pl.ryobitools.eu/{{ product.url }}'>link</a></td>
<td class="align-middle text-end">{% if product.lowestPrice.price != product.price.last.price %}{{ product.lowestPrice.price | format_currency('PLN', {}, 'pl') }}{% endif %}</td>
<td class="align-middle text-end">{{ product.price.last.price | format_currency('PLN', {}, 'pl') }}</td>
<td class="align-middle text-end">{% if product.priceLowest != product.priceCurrent %}{{ product.priceLowest | format_currency('PLN', {}, 'pl') }}{% endif %}</td>
<td class="align-middle text-end">{{ product.priceCurrent | format_currency('PLN', {}, 'pl') }}</td>
<td class="align-middle">
<div class="d-flex flex-row">
{% if product.price.last.price != product.price.last.productStandardPrice %}<span
class="badge text-bg-warning text-decoration-line-through flex-fill">{{ product.price.last.productStandardPrice | format_currency('PLN', {}, 'pl') }}</span> <span
class="badge text-bg-success flex-fill">{{ ((1 - product.price.last.price / product.price.last.productStandardPrice)*100)|number_format(0) }}%</span>
{% if product.priceCurrent != product.productStandardPrice %}<span
class="badge text-bg-warning text-decoration-line-through flex-fill">{{ product.productStandardPrice | format_currency('PLN', {}, 'pl') }}</span> <span
class="badge text-bg-success flex-fill">{{ ((1 - product.priceCurrent / product.productStandardPrice)*100)|number_format(0) }}%</span>
{% endif %}
</div>
</td>