krzysiej/ryobi-crawler
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases 1 Wiki Activity

feature/handle-multiple-countries #45

Merged
krzysiej merged 5 commits from feature/handle-multiple-countries into master 2026-01-17 17:07:40 +01:00
Conversation 0 Commits 5 Files Changed 6 +35 -20
4 changed files with 35 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 03b74aa33d - Show all commits

6
src/Command/Migrate.php
View File

@@ -87,6 +87,8 @@ class Migrate extends Command
$table->text('countryName');
$table->text('productsUrl');
$table->text('cultureCode');
$table->text('currency');
$table->text('locale');
$table->timestamps();
});
}
@@ -95,6 +97,8 @@ class Migrate extends Command
'countryName' => 'Poland',
'productsUrl' => 'https://pl.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'pl-PL',
'currency' => 'PLN',
'locale' => 'pl',
'created_at' => now(),
'updated_at' => now(),
]);
@@ -102,6 +106,8 @@ class Migrate extends Command
'countryName' => 'UK',
'productsUrl' => 'https://uk.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'en-GB',
'currency' => 'GBP',
'locale' => 'en',
'created_at' => now(),
'updated_at' => now(),
]

37
src/Command/ScrapeWebsite.php
View File

@@ -36,35 +36,36 @@ class ScrapeWebsite extends Command
{
$output->writeln('Scrape products');
$progress = new ProgressBar($output);
$progress->start();
$countries = Country::all();
foreach($countries as $country) {
$output->writeln('Country name: ' . $country->countryName);
$output->writeln('Country name: ' . $country->countryName."\n");
$progress->start();
$products = $this->getProducts($country);
$progress->setMaxSteps(count($products));
foreach ($products as $product) {
$this->saveProduct($product, $country);
$progress->advance();
}
}
$progress->finish();
$output->writeln('');
$output->writeln('Scrape products - DONE');
$output->writeln('');
}
$output->writeln('Update prices');
$products = Product::all();
$progress->setMaxSteps(count($products));
$progress->start();
foreach($products as $product) {
$newestPrice = $product->newestPrice;
$product->priceCurrent = $newestPrice->price;
$product->productStandardPrice = $newestPrice->productStandardPrice;
$product->lowestProductPrice30Days = $newestPrice->lowestProductPrice30Days;
$product->priceLowest = $product->lowestPrice->price;
$product->lastSeen = $newestPrice->created_at->format('Y-m-d');
$product->save(['timestamps' => false]);
$progress->advance();
}
// foreach($products as $product) {
// $newestPrice = $product->newestPrice;
// $product->priceCurrent = $newestPrice->price;
// $product->productStandardPrice = $newestPrice->productStandardPrice;
// $product->lowestProductPrice30Days = $newestPrice->lowestProductPrice30Days;
// $product->priceLowest = $product->lowestPrice->price;
// $product->lastSeen = $newestPrice->created_at->format('Y-m-d');
// $product->save(['timestamps' => false]);
// $progress->advance();
// }
$progress->finish();
$output->writeln('');
$output->writeln('Update prices - DONE');
@@ -77,9 +78,10 @@ class ScrapeWebsite extends Command
{
$products = [];
$page = 0;
do {
try {
$res = $this->client->request('POST', $country->productUrl, [
$res = $this->client->request('POST', $country->productsUrl, [
'form_params' => [
"includePreviousPages" => false,
"pageIndex" => $page,
@@ -101,8 +103,12 @@ class ScrapeWebsite extends Command
private function saveProduct(\stdClass $product, Country $country): void
{
// if ($product->skuID == 0) {
// dump([$product->skuID, $product->name]);
// }
// return;
/** @var Product $productModel */
$productModel = Product::firstOrNew(['skuID' => $product->skuID]);
$productModel = Product::firstOrNew(['skuID' => $product->skuID, 'country_id' => $country->id]);
$productModel->skuID = $product->skuID;
$productModel->name = $product->name;
@@ -115,6 +121,7 @@ class ScrapeWebsite extends Command
$productModel->url = $product->url;
$productModel->lastSeen = date("Y-m-d");
$productModel->touch('updated_at');
$productModel->country()->associate($country);
$productModel->save();
$priceExists = $productModel->price()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();

4
src/Models/Country.php
View File

@@ -8,8 +8,10 @@ use Illuminate\Database\Eloquent\Relations\HasMany;
/**
* @property string $countryName
* @property string $productUrl
* @property string $productsUrl
* @property string $cultureCode
* @property string $currency
* @property string $locale
*/
class Country extends Model
{

4
templates/product.html.twig
View File

@@ -44,8 +44,8 @@
</thead>
{% for price in product.price %}
<tr>
<td>{{ price.price | format_currency('PLN', {}, 'pl') }}</td>
<td>{{ price.lowestProductPrice30Days | format_currency('PLN', {}, 'pl') }}</td>
<td>{{ price.price | format_currency('GBP', {}, 'en') }}</td>
<td>{{ price.lowestProductPrice30Days | format_currency('GBP', {}, 'en') }}</td>
<td>{{ price.productStandardPrice | format_currency('PLN', {}, 'pl') }}</td>
<td>{{ price.created_at }}</td>
<td>{{ (product.stock | findByCreatedAtDate(price.created_at | slice(0,10))).stock ?? '' }}</td>