krzysiej/ryobi-crawler
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases 1 Wiki Activity

Use progress bar to track progress

Browse Source
This commit is contained in:
Krzysztof Płaczek
2024-05-18 20:58:16 +02:00
parent 4f0f0a5f00
commit 56fd13c627
1 changed files with 14 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/Command/ScrapeWebsite.php
View File

@@ -17,24 +17,27 @@ use Symfony\Component\Console\Output\OutputInterface;
class ScrapeWebsite extends Command class ScrapeWebsite extends Command
{ {
private Client $client; private Client $client;
private Capsule $capsule;
protected function configure() protected function configure()
{ {
$this->capsule = new Capsule; $capsule = new Capsule;
$this->capsule->addConnection([ $capsule->addConnection([
'driver' => 'sqlite', 'driver' => 'sqlite',
'database' => __DIR__ .'/../../database.sqlite', 'database' => __DIR__ .'/../../database.sqlite',
]); ]);
$this->capsule->setAsGlobal(); $capsule->setAsGlobal();
$this->capsule->bootEloquent(); $capsule->bootEloquent();
$this->client = new Client(); $this->client = new Client();
} }
public function execute(InputInterface $input, OutputInterface $output): int public function execute(InputInterface $input, OutputInterface $output): int
{ {
$progress = new ProgressBar($output);
$progress->start();
$page = 0; $page = 0;
do { do {
$progress->advance();
$res = $this->client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [ $res = $this->client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [
'form_params' => [ 'form_params' => [
"includePreviousPages" => false, "includePreviousPages" => false,
@@ -46,6 +49,7 @@ class ScrapeWebsite extends Command
$page++; $page++;
$responseObject = json_decode($res->getBody()->getContents()); $responseObject = json_decode($res->getBody()->getContents());
$products = $responseObject->products; $products = $responseObject->products;
$progress->setMaxSteps($progress->getMaxSteps()+count($products));
foreach ($products as $product) { foreach ($products as $product) {
/** @var Product $productModel */ /** @var Product $productModel */
$productModel = Product::firstOrNew(['skuID' => $product->skuID]); $productModel = Product::firstOrNew(['skuID' => $product->skuID]);
@@ -68,10 +72,13 @@ class ScrapeWebsite extends Command
$price->lowestProductPrice30Days = $product->lowestProductPrice30Days; $price->lowestProductPrice30Days = $product->lowestProductPrice30Days;
$productModel->price()->save($price); $productModel->price()->save($price);
} }
echo "."; $progress->advance();
} }
echo "\n";
} while ((bool)$responseObject->canLoadMore); } while ((bool)$responseObject->canLoadMore);
$progress->finish();
$output->writeln('');
$output->writeln('DONE');
return Command::SUCCESS; return Command::SUCCESS;
} }
} }