krzysiej/ryobi-crawler
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases 1 Wiki Activity

Update packages. Add docker support, move migration to command, add stocks as a new table.

Browse Source
This commit is contained in:
Krzysztof Płaczek
2024-10-12 09:08:32 +02:00
parent 8827daec5d
commit 39e0262036
12 changed files with 391 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
src/Command/ScrapeWebsite.php
View File

@@ -5,21 +5,23 @@ declare(strict_types=1);
namespace Krzysiej\RyobiCrawler\Command;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use Illuminate\Database\Capsule\Manager as Capsule;
use Krzysiej\RyobiCrawler\Models\Price;
use Krzysiej\RyobiCrawler\Models\Product;
use Krzysiej\RyobiCrawler\Models\Stock;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
#[AsCommand(name: 'app:scrape')]
#[AsCommand(name: 'app:scrape', description: 'Scrape all products from Ryobi website')]
class ScrapeWebsite extends Command
{
private Client $client;
protected function configure()
protected function configure(): void
{
$capsule = new Capsule;
$capsule->addConnection([
@@ -44,6 +46,7 @@ class ScrapeWebsite extends Command
$progress->finish();
$output->writeln('');
$output->writeln('DONE');
return Command::SUCCESS;
}
@@ -52,22 +55,28 @@ class ScrapeWebsite extends Command
$products = [];
$page = 0;
do {
$res = $this->client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [
'form_params' => [
"includePreviousPages" => false,
"pageIndex" => $page,
"pageSize" => 100,
"cultureCode" => "pl-PL",
]
]);
$page++;
$responseObject = json_decode($res->getBody()->getContents());
$products = array_merge($products, $responseObject->products);
} while ((bool)$responseObject->canLoadMore);
try {
$res = $this->client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [
'form_params' => [
"includePreviousPages" => false,
"pageIndex" => $page,
"pageSize" => 100,
"cultureCode" => "pl-PL",
]
]);
$responseObject = json_decode($res->getBody()->getContents());
$products = array_merge($products, $responseObject->products);
$page++;
$canLoadMore = $responseObject->canLoadMore;
} catch (GuzzleException $e) {
return $products;
}
} while ($canLoadMore);
return $products;
}
private function saveProduct(\stdClass $product)
private function saveProduct(\stdClass $product): void
{
/** @var Product $productModel */
$productModel = Product::firstOrNew(['skuID' => $product->skuID]);
@@ -90,5 +99,17 @@ class ScrapeWebsite extends Command
$price->lowestProductPrice30Days = $product->lowestProductPrice30Days;
$productModel->price()->save($price);
}
$stockExist = $productModel->stock()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();
if (!$stockExist) {
$stock = new Stock();
$stock->availableQuantity = $product->availableQuantity;
$stock->stock = $product->stock;
$stock->isInStock = $product->isInStock;
$stock->previouslyHadStock = $product->previouslyHadStock;
$stock->isNew = $product->isNew;
$stock->isComingSoon = $product->isComingSoon;
$stock->isOnSale = $product->isOnSale;
$productModel->stock()->save($stock);
}
}
}
}