krzysiej/ryobi-crawler
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases 1 Wiki Activity

Add country option to scrape command. #61

Merged
krzysiej merged 1 commits from feature/add-country-option-to-scrape-command into master 2026-02-06 08:34:59 +01:00
Conversation 0 Commits 1 Files Changed 1 +10 -5
1 changed files with 10 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 1303b4ad8a - Show all commits

15
src/Command/ScrapeWebsite.php
View File

@@ -15,11 +15,13 @@ use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\ProgressBar; use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Output\OutputInterface;
#[AsCommand(name: 'app:scrape', description: 'Scrape all products from Ryobi website')] #[AsCommand(name: 'app:scrape', description: 'Scrape all products from Ryobi website')]
class ScrapeWebsite extends Command class ScrapeWebsite extends Command
{ {
const COUNTRY_ID = 'country';
private Client $client; private Client $client;
public function __construct(protected Capsule $database) public function __construct(protected Capsule $database)
@@ -30,17 +32,19 @@ class ScrapeWebsite extends Command
protected function configure(): void protected function configure(): void
{ {
$this->client = new Client(); $this->client = new Client();
$this->addOption(self::COUNTRY_ID, 'c', InputOption::VALUE_OPTIONAL, 'Country id');
} }
public function execute(InputInterface $input, OutputInterface $output): int public function execute(InputInterface $input, OutputInterface $output): int
{ {
$countryId = intval($input->getOption(self::COUNTRY_ID));
$output->writeln('Scrape products'); $output->writeln('Scrape products');
$progress = new ProgressBar($output); $countries = Country::all()->when($countryId, fn ($query) => $query->where('id', $countryId));
$countries = Country::all();
foreach($countries as $country) { foreach($countries as $country) {
$output->writeln('Country name: ' . $country->countryName); $output->writeln('Country name: ' . $country->countryName);
$progress->start();
$products = $this->getProducts($country); $products = $this->getProducts($country);
$progress = new ProgressBar($output);
$progress->start();
$progress->setMaxSteps(count($products)); $progress->setMaxSteps(count($products));
foreach ($products as $product) { foreach ($products as $product) {
$this->saveProduct($product, $country); $this->saveProduct($product, $country);
@@ -54,6 +58,7 @@ class ScrapeWebsite extends Command
$output->writeln('Update prices'); $output->writeln('Update prices');
$products = Product::all(); $products = Product::all();
$progress = new ProgressBar($output);
$progress->setMaxSteps(count($products)); $progress->setMaxSteps(count($products));
$progress->start(); $progress->start();
foreach($products as $product) { foreach($products as $product) {
@@ -87,7 +92,7 @@ class ScrapeWebsite extends Command
'form_params' => [ 'form_params' => [
"includePreviousPages" => false, "includePreviousPages" => false,
"pageIndex" => $page, "pageIndex" => $page,
"pageSize" => 100, "pageSize" => 1000,
"cultureCode" => $country->cultureCode, "cultureCode" => $country->cultureCode,
] ]
]); ]);
@@ -95,7 +100,7 @@ class ScrapeWebsite extends Command
$products = array_merge($products, $responseObject->products); $products = array_merge($products, $responseObject->products);
$page++; $page++;
$canLoadMore = $responseObject->canLoadMore; $canLoadMore = $responseObject->canLoadMore;
} catch (GuzzleException) { } catch (GuzzleException $e) {
return $products; return $products;
} }
} while ($canLoadMore); } while ($canLoadMore);