krzysiej/ryobi-crawler
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases 1 Wiki Activity

Start working on handling multiple countries at once

Browse Source
This commit is contained in:
Krzysiej
2026-01-16 16:20:21 +01:00
parent e1340d45ed
commit 03b74aa33d
4 changed files with 35 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/Command/Migrate.php
View File

@@ -87,6 +87,8 @@ class Migrate extends Command
$table->text('countryName');
$table->text('productsUrl');
$table->text('cultureCode');
$table->text('currency');
$table->text('locale');
$table->timestamps();
});
}
@@ -95,6 +97,8 @@ class Migrate extends Command
'countryName' => 'Poland',
'productsUrl' => 'https://pl.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'pl-PL',
'currency' => 'PLN',
'locale' => 'pl',
'created_at' => now(),
'updated_at' => now(),
]);
@@ -102,6 +106,8 @@ class Migrate extends Command
'countryName' => 'UK',
'productsUrl' => 'https://uk.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'en-GB',
'currency' => 'GBP',
'locale' => 'en',
'created_at' => now(),
'updated_at' => now(),
]

37
src/Command/ScrapeWebsite.php
View File

@@ -36,35 +36,36 @@ class ScrapeWebsite extends Command
{
$output->writeln('Scrape products');
$progress = new ProgressBar($output);
$progress->start();
$countries = Country::all();
foreach($countries as $country) {
$output->writeln('Country name: ' . $country->countryName);
$output->writeln('Country name: ' . $country->countryName."\n");
$progress->start();
$products = $this->getProducts($country);
$progress->setMaxSteps(count($products));
foreach ($products as $product) {
$this->saveProduct($product, $country);
$progress->advance();
}
}
$progress->finish();
$output->writeln('');
$output->writeln('Scrape products - DONE');
$output->writeln('');
}
$output->writeln('Update prices');
$products = Product::all();
$progress->setMaxSteps(count($products));
$progress->start();
foreach($products as $product) {
$newestPrice = $product->newestPrice;
$product->priceCurrent = $newestPrice->price;
$product->productStandardPrice = $newestPrice->productStandardPrice;
$product->lowestProductPrice30Days = $newestPrice->lowestProductPrice30Days;
$product->priceLowest = $product->lowestPrice->price;
$product->lastSeen = $newestPrice->created_at->format('Y-m-d');
$product->save(['timestamps' => false]);
$progress->advance();
}
// foreach($products as $product) {
// $newestPrice = $product->newestPrice;
// $product->priceCurrent = $newestPrice->price;
// $product->productStandardPrice = $newestPrice->productStandardPrice;
// $product->lowestProductPrice30Days = $newestPrice->lowestProductPrice30Days;
// $product->priceLowest = $product->lowestPrice->price;
// $product->lastSeen = $newestPrice->created_at->format('Y-m-d');
// $product->save(['timestamps' => false]);
// $progress->advance();
// }
$progress->finish();
$output->writeln('');
$output->writeln('Update prices - DONE');
@@ -77,9 +78,10 @@ class ScrapeWebsite extends Command
{
$products = [];
$page = 0;
do {
try {
$res = $this->client->request('POST', $country->productUrl, [
$res = $this->client->request('POST', $country->productsUrl, [
'form_params' => [
"includePreviousPages" => false,
"pageIndex" => $page,
@@ -101,8 +103,12 @@ class ScrapeWebsite extends Command
private function saveProduct(\stdClass $product, Country $country): void
{
// if ($product->skuID == 0) {
// dump([$product->skuID, $product->name]);
// }
// return;
/** @var Product $productModel */
$productModel = Product::firstOrNew(['skuID' => $product->skuID]);
$productModel = Product::firstOrNew(['skuID' => $product->skuID, 'country_id' => $country->id]);
$productModel->skuID = $product->skuID;
$productModel->name = $product->name;
@@ -115,6 +121,7 @@ class ScrapeWebsite extends Command
$productModel->url = $product->url;
$productModel->lastSeen = date("Y-m-d");
$productModel->touch('updated_at');
$productModel->country()->associate($country);
$productModel->save();
$priceExists = $productModel->price()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();

4
src/Models/Country.php
View File

@@ -8,8 +8,10 @@ use Illuminate\Database\Eloquent\Relations\HasMany;
/**
* @property string $countryName
* @property string $productUrl
* @property string $productsUrl
* @property string $cultureCode
* @property string $currency
* @property string $locale
*/
class Country extends Model
{

4
templates/product.html.twig
View File

@@ -44,8 +44,8 @@
</thead>
{% for price in product.price %}
<tr>
<td>{{ price.price | format_currency('PLN', {}, 'pl') }}</td>
<td>{{ price.lowestProductPrice30Days | format_currency('PLN', {}, 'pl') }}</td>
<td>{{ price.price | format_currency('GBP', {}, 'en') }}</td>
<td>{{ price.lowestProductPrice30Days | format_currency('GBP', {}, 'en') }}</td>
<td>{{ price.productStandardPrice | format_currency('PLN', {}, 'pl') }}</td>
<td>{{ price.created_at }}</td>
<td>{{ (product.stock | findByCreatedAtDate(price.created_at | slice(0,10))).stock ?? '' }}</td>