krzysiej/ryobi-crawler
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases 1 Wiki Activity

feature/handle-multiple-countries (#45)

Browse Source 
Reviewed-on: #45
Co-authored-by: Krzysiej <krzysiej@gmail.com>
Co-committed-by: Krzysiej <krzysiej@gmail.com>
This commit was merged in pull request #45.
This commit is contained in:
Krzysiej
2026-01-17 17:07:40 +01:00
committed by krzysiej
parent ebe40785fa
commit 295a968581
6 changed files with 117 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
src/Command/Migrate.php
View File

@@ -11,6 +11,7 @@ use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use function Symfony\Component\Clock\now;
#[AsCommand(name: 'app:migrate', description: 'Create database and rum migrations')]
class Migrate extends Command
@@ -39,6 +40,8 @@ class Migrate extends Command
$this->createPricesTable();
$this->createStocksTable();
$this->addColumns();
$this->createCountriesTable();
$this->index();
return Command::SUCCESS;
}
@@ -77,6 +80,47 @@ class Migrate extends Command
}
}
public function createCountriesTable(): void
{
if (!Capsule::schema()->hasTable('countries')) {
Capsule::schema()->create('countries', function (Blueprint $table) {
$table->increments('id');
$table->text('countryName');
$table->text('productsUrl');
$table->text('cultureCode');
$table->text('currency');
$table->text('locale');
$table->timestamps();
});
}
$id = Capsule::table('countries')->insertGetId(
[
'countryName' => 'Poland',
'productsUrl' => 'https://pl.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'pl-PL',
'currency' => 'PLN',
'locale' => 'pl',
'created_at' => now(),
'updated_at' => now(),
]);
Capsule::table('countries')->insert([
'countryName' => 'UK',
'productsUrl' => 'https://uk.ryobitools.eu/api/product-listing/get-products',
'cultureCode' => 'en-GB',
'currency' => 'GBP',
'locale' => 'en',
'created_at' => now(),
'updated_at' => now(),
]
);
if (!Capsule::schema()->hasColumn('products', 'country_id')) {
Capsule::schema()->table('products', function (Blueprint $table) use ($id) {
$table->foreignId('country_id')->default($id)->references('id')->on('countries');
});
}
}
public function createStocksTable(): void
{
if (!Capsule::schema()->hasTable('stocks')) {
@@ -127,4 +171,12 @@ class Migrate extends Command
});
}
}
public function index(): void
{
Capsule::schema()->table('products', function (Blueprint $table) {
$table->integer('skuID')->unique(false)->change();
$table->unique(['skuID', 'country_id']);
});
}
}

41
src/Command/ScrapeWebsite.php
View File

@@ -7,7 +7,7 @@ namespace Krzysiej\RyobiCrawler\Command;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Support\Facades\Date;
use Krzysiej\RyobiCrawler\Models\Country;
use Krzysiej\RyobiCrawler\Models\Price;
use Krzysiej\RyobiCrawler\Models\Product;
use Krzysiej\RyobiCrawler\Models\Stock;
@@ -36,16 +36,21 @@ class ScrapeWebsite extends Command
{
$output->writeln('Scrape products');
$progress = new ProgressBar($output);
$progress->start();
$products = $this->getProducts();
$progress->setMaxSteps(count($products));
foreach ($products as $product) {
$this->saveProduct($product);
$progress->advance();
$countries = Country::all();
foreach($countries as $country) {
$output->writeln('Country name: ' . $country->countryName."\n");
$progress->start();
$products = $this->getProducts($country);
$progress->setMaxSteps(count($products));
foreach ($products as $product) {
$this->saveProduct($product, $country);
$progress->advance();
}
$progress->finish();
$output->writeln('');
$output->writeln('Scrape products - DONE');
$output->writeln('');
}
$progress->finish();
$output->writeln('Scrape products - DONE');
$output->writeln('');
$output->writeln('Update prices');
$products = Product::all();
@@ -69,18 +74,19 @@ class ScrapeWebsite extends Command
return Command::SUCCESS;
}
private function getProducts(): array
private function getProducts(Country $country): array
{
$products = [];
$page = 0;
do {
try {
$res = $this->client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [
$res = $this->client->request('POST', $country->productsUrl, [
'form_params' => [
"includePreviousPages" => false,
"pageIndex" => $page,
"pageSize" => 100,
"cultureCode" => "pl-PL",
"cultureCode" => $country->cultureCode,
]
]);
$responseObject = json_decode($res->getBody()->getContents());
@@ -95,10 +101,14 @@ class ScrapeWebsite extends Command
return $products;
}
private function saveProduct(\stdClass $product): void
private function saveProduct(\stdClass $product, Country $country): void
{
// if ($product->skuID == 0) {
// dump([$product->skuID, $product->name]);
// }
// return;
/** @var Product $productModel */
$productModel = Product::firstOrNew(['skuID' => $product->skuID]);
$productModel = Product::firstOrNew(['skuID' => $product->skuID, 'country_id' => $country->id]);
$productModel->skuID = $product->skuID;
$productModel->name = $product->name;
@@ -111,6 +121,7 @@ class ScrapeWebsite extends Command
$productModel->url = $product->url;
$productModel->lastSeen = date("Y-m-d");
$productModel->touch('updated_at');
$productModel->country()->associate($country);
$productModel->save();
$priceExists = $productModel->price()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();