krzysiej/ryobi-crawler
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases 1 Wiki Activity

Start working on handling multiple countries at once

Browse Source
This commit is contained in:
Krzysiej
2026-01-16 16:20:21 +01:00
parent e1340d45ed
commit 03b74aa33d
4 changed files with 35 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/Command/Migrate.php
View File

@@ -87,6 +87,8 @@ class Migrate extends Command
$table->text('countryName'); $table->text('countryName');
$table->text('productsUrl'); $table->text('productsUrl');
$table->text('cultureCode'); $table->text('cultureCode');
$table->text('currency');
$table->text('locale');
$table->timestamps(); $table->timestamps();
}); });
} }
@@ -95,6 +97,8 @@ class Migrate extends Command
'countryName' => 'Poland', 'countryName' => 'Poland',
'productsUrl' => 'https://pl.ryobitools.eu/api/product-listing/get-products', 'productsUrl' => 'https://pl.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'pl-PL', 'cultureCode' => 'pl-PL',
'currency' => 'PLN',
'locale' => 'pl',
'created_at' => now(), 'created_at' => now(),
'updated_at' => now(), 'updated_at' => now(),
]); ]);
@@ -102,6 +106,8 @@ class Migrate extends Command
'countryName' => 'UK', 'countryName' => 'UK',
'productsUrl' => 'https://uk.ryobitools.eu/api/product-listing/get-products', 'productsUrl' => 'https://uk.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'en-GB', 'cultureCode' => 'en-GB',
'currency' => 'GBP',
'locale' => 'en',
'created_at' => now(), 'created_at' => now(),
'updated_at' => now(), 'updated_at' => now(),
] ]

37
src/Command/ScrapeWebsite.php
View File

@@ -36,35 +36,36 @@ class ScrapeWebsite extends Command
{ {
$output->writeln('Scrape products'); $output->writeln('Scrape products');
$progress = new ProgressBar($output); $progress = new ProgressBar($output);
$progress->start();
$countries = Country::all(); $countries = Country::all();
foreach($countries as $country) { foreach($countries as $country) {
$output->writeln('Country name: ' . $country->countryName); $output->writeln('Country name: ' . $country->countryName."\n");
$progress->start();
$products = $this->getProducts($country); $products = $this->getProducts($country);
$progress->setMaxSteps(count($products)); $progress->setMaxSteps(count($products));
foreach ($products as $product) { foreach ($products as $product) {
$this->saveProduct($product, $country); $this->saveProduct($product, $country);
$progress->advance(); $progress->advance();
} }
}
$progress->finish(); $progress->finish();
$output->writeln('');
$output->writeln('Scrape products - DONE'); $output->writeln('Scrape products - DONE');
$output->writeln(''); $output->writeln('');
}
$output->writeln('Update prices'); $output->writeln('Update prices');
$products = Product::all(); $products = Product::all();
$progress->setMaxSteps(count($products)); $progress->setMaxSteps(count($products));
$progress->start(); $progress->start();
foreach($products as $product) { // foreach($products as $product) {
$newestPrice = $product->newestPrice; // $newestPrice = $product->newestPrice;
$product->priceCurrent = $newestPrice->price; // $product->priceCurrent = $newestPrice->price;
$product->productStandardPrice = $newestPrice->productStandardPrice; // $product->productStandardPrice = $newestPrice->productStandardPrice;
$product->lowestProductPrice30Days = $newestPrice->lowestProductPrice30Days; // $product->lowestProductPrice30Days = $newestPrice->lowestProductPrice30Days;
$product->priceLowest = $product->lowestPrice->price; // $product->priceLowest = $product->lowestPrice->price;
$product->lastSeen = $newestPrice->created_at->format('Y-m-d'); // $product->lastSeen = $newestPrice->created_at->format('Y-m-d');
$product->save(['timestamps' => false]); // $product->save(['timestamps' => false]);
$progress->advance(); // $progress->advance();
} // }
$progress->finish(); $progress->finish();
$output->writeln(''); $output->writeln('');
$output->writeln('Update prices - DONE'); $output->writeln('Update prices - DONE');
@@ -77,9 +78,10 @@ class ScrapeWebsite extends Command
{ {
$products = []; $products = [];
$page = 0; $page = 0;
do { do {
try { try {
$res = $this->client->request('POST', $country->productUrl, [ $res = $this->client->request('POST', $country->productsUrl, [
'form_params' => [ 'form_params' => [
"includePreviousPages" => false, "includePreviousPages" => false,
"pageIndex" => $page, "pageIndex" => $page,
@@ -101,8 +103,12 @@ class ScrapeWebsite extends Command
private function saveProduct(\stdClass $product, Country $country): void private function saveProduct(\stdClass $product, Country $country): void
{ {
// if ($product->skuID == 0) {
// dump([$product->skuID, $product->name]);
// }
// return;
/** @var Product $productModel */ /** @var Product $productModel */
$productModel = Product::firstOrNew(['skuID' => $product->skuID]); $productModel = Product::firstOrNew(['skuID' => $product->skuID, 'country_id' => $country->id]);
$productModel->skuID = $product->skuID; $productModel->skuID = $product->skuID;
$productModel->name = $product->name; $productModel->name = $product->name;
@@ -115,6 +121,7 @@ class ScrapeWebsite extends Command
$productModel->url = $product->url; $productModel->url = $product->url;
$productModel->lastSeen = date("Y-m-d"); $productModel->lastSeen = date("Y-m-d");
$productModel->touch('updated_at'); $productModel->touch('updated_at');
$productModel->country()->associate($country);
$productModel->save(); $productModel->save();
$priceExists = $productModel->price()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists(); $priceExists = $productModel->price()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();

4
src/Models/Country.php
View File

@@ -8,8 +8,10 @@ use Illuminate\Database\Eloquent\Relations\HasMany;
/** /**
* @property string $countryName * @property string $countryName
* @property string $productUrl * @property string $productsUrl
* @property string $cultureCode * @property string $cultureCode
* @property string $currency
* @property string $locale
*/ */
class Country extends Model class Country extends Model
{ {

4
templates/product.html.twig
View File

@@ -44,8 +44,8 @@
</thead> </thead>
{% for price in product.price %} {% for price in product.price %}
<tr> <tr>
<td>{{ price.price | format_currency('PLN', {}, 'pl') }}</td> <td>{{ price.price | format_currency('GBP', {}, 'en') }}</td>
<td>{{ price.lowestProductPrice30Days | format_currency('PLN', {}, 'pl') }}</td> <td>{{ price.lowestProductPrice30Days | format_currency('GBP', {}, 'en') }}</td>
<td>{{ price.productStandardPrice | format_currency('PLN', {}, 'pl') }}</td> <td>{{ price.productStandardPrice | format_currency('PLN', {}, 'pl') }}</td>
<td>{{ price.created_at }}</td> <td>{{ price.created_at }}</td>
<td>{{ (product.stock | findByCreatedAtDate(price.created_at | slice(0,10))).stock ?? '' }}</td> <td>{{ (product.stock | findByCreatedAtDate(price.created_at | slice(0,10))).stock ?? '' }}</td>