krzysiej/ryobi-crawler
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases 1 Wiki Activity

Update packages. Add docker support, move migration to command, add stocks as a new table.

Browse Source
This commit is contained in:
Krzysztof Płaczek
2024-10-12 09:08:32 +02:00
parent 8827daec5d
commit 39e0262036
12 changed files with 391 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
src/Command/Migrate.php Normal file
View File

@@ -0,0 +1,91 @@
<?php
declare(strict_types=1);
namespace Krzysiej\RyobiCrawler\Command;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
#[AsCommand(name: 'app:migrate', description: 'Create database and rum migrations')]
class Migrate extends Command
{
protected function configure(): void
{
touch(__DIR__ . '/../../database.sqlite');
$capsule = new Capsule;
$capsule->addConnection([
'driver' => 'sqlite',
'database' => __DIR__ . '/../../database.sqlite',
]);
$capsule->setAsGlobal();
$capsule->bootEloquent();
}
public function execute(InputInterface $input, OutputInterface $output): int
{
$this->createProductsTable();
$this->createPricesTable();
$this->createStocksTable();
return Command::SUCCESS;
}
public function createProductsTable(): void
{
if (!Capsule::schema()->hasTable('products')) {
Capsule::schema()->create('products', function (Blueprint $table) {
$table->increments('id');
$table->string('name');
$table->integer('skuID')->unique();
$table->integer('agilityID');
$table->integer('availableQuantity');
$table->integer('stock');
$table->json('categories');
$table->string('image');
$table->string('subTitle');
$table->string('variantCode');
$table->string('modelCode');
$table->string('url');
$table->boolean('starred')->default(false);
$table->timestamps();
});
}
}
public function createPricesTable(): void
{
if (!Capsule::schema()->hasTable('prices')) {
Capsule::schema()->create('prices', function (Blueprint $table) {
$table->increments('id');
$table->foreignId('product_id');
$table->float('price');
$table->float('productStandardPrice');
$table->float('lowestProductPrice30Days');
$table->timestamps();
});
}
}
public function createStocksTable(): void
{
if (!Capsule::schema()->hasTable('stocks')) {
Capsule::schema()->create('stocks', function (Blueprint $table) {
$table->increments('id');
$table->foreignId('product_id');
$table->integer('availableQuantity');
$table->integer('stock');
$table->boolean('isInStock');
$table->boolean('previouslyHadStock');
$table->boolean('isNew');
$table->boolean('isComingSoon');
$table->boolean('isOnSale');
$table->timestamps();
});
}
}
}

53
src/Command/ScrapeWebsite.php
View File

@@ -5,21 +5,23 @@ declare(strict_types=1);
namespace Krzysiej\RyobiCrawler\Command;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use Illuminate\Database\Capsule\Manager as Capsule;
use Krzysiej\RyobiCrawler\Models\Price;
use Krzysiej\RyobiCrawler\Models\Product;
use Krzysiej\RyobiCrawler\Models\Stock;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
#[AsCommand(name: 'app:scrape')]
#[AsCommand(name: 'app:scrape', description: 'Scrape all products from Ryobi website')]
class ScrapeWebsite extends Command
{
private Client $client;
protected function configure()
protected function configure(): void
{
$capsule = new Capsule;
$capsule->addConnection([
@@ -44,6 +46,7 @@ class ScrapeWebsite extends Command
$progress->finish();
$output->writeln('');
$output->writeln('DONE');
return Command::SUCCESS;
}
@@ -52,22 +55,28 @@ class ScrapeWebsite extends Command
$products = [];
$page = 0;
do {
$res = $this->client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [
'form_params' => [
"includePreviousPages" => false,
"pageIndex" => $page,
"pageSize" => 100,
"cultureCode" => "pl-PL",
]
]);
$page++;
$responseObject = json_decode($res->getBody()->getContents());
$products = array_merge($products, $responseObject->products);
} while ((bool)$responseObject->canLoadMore);
try {
$res = $this->client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [
'form_params' => [
"includePreviousPages" => false,
"pageIndex" => $page,
"pageSize" => 100,
"cultureCode" => "pl-PL",
]
]);
$responseObject = json_decode($res->getBody()->getContents());
$products = array_merge($products, $responseObject->products);
$page++;
$canLoadMore = $responseObject->canLoadMore;
} catch (GuzzleException $e) {
return $products;
}
} while ($canLoadMore);
return $products;
}
private function saveProduct(\stdClass $product)
private function saveProduct(\stdClass $product): void
{
/** @var Product $productModel */
$productModel = Product::firstOrNew(['skuID' => $product->skuID]);
@@ -90,5 +99,17 @@ class ScrapeWebsite extends Command
$price->lowestProductPrice30Days = $product->lowestProductPrice30Days;
$productModel->price()->save($price);
}
$stockExist = $productModel->stock()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();
if (!$stockExist) {
$stock = new Stock();
$stock->availableQuantity = $product->availableQuantity;
$stock->stock = $product->stock;
$stock->isInStock = $product->isInStock;
$stock->previouslyHadStock = $product->previouslyHadStock;
$stock->isNew = $product->isNew;
$stock->isComingSoon = $product->isComingSoon;
$stock->isOnSale = $product->isOnSale;
$productModel->stock()->save($stock);
}
}
}
}