krzysiej/ryobi-crawler
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases 1 Wiki Activity

Update packages. Add docker support, move migration to command, add stocks as a new table.

Browse Source
This commit is contained in:
Krzysztof Płaczek
2024-10-12 09:08:32 +02:00
parent 8827daec5d
commit 39e0262036
12 changed files with 391 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
Dockerfile Normal file
View File

@@ -0,0 +1,19 @@
FROM php:8.3-cli
WORKDIR /usr/src/app
RUN apt-get update && apt-get install -y \
git \
unzip \
libzip-dev
RUN docker-php-ext-install zip
RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer
COPY composer.json composer.lock ./
RUN composer install --no-dev --optimize-autoloader
COPY . .
CMD ["php", "browser.php"]

4
README.md
View File

@@ -1,2 +1,6 @@
# ryobi-crawler
## Project start
1. Run `migration.php` file to create `database.sqlite` and create tables.
2. Run `php index.php app:scrape` command to scrape all the products from the ryobi website.

2
browser.php
View File

@@ -35,4 +35,4 @@ if (empty($_GET)) {
$template = 'productList.html.twig';
$products = Product::with('price')->orderByDesc('starred')->orderByDesc('created_by')->get();
}
$twig->display($template, ['products' => $products, 'product' => $product, 'search' => $_GET['search']]);
$twig->display($template, ['products' => $products ?? [], 'product' => $product ?? [], 'search' => $_GET['search'] ?? '']);

4
composer.json
View File

@@ -2,9 +2,9 @@
"require": {
"guzzlehttp/guzzle": "^7.0",
"symfony/var-dumper": "^7.0",
"illuminate/database": "^11.0",
"illuminate/database": "11.26.0.0",
"ext-json": "*",
"twig/twig": "^3.0",
"twig/twig": "3.14.0.0",
"symfony/console": "^7.0"
},
"autoload": {

378
composer.lock generated
View File

File diff suppressed because it is too large Load Diff

9
docker-compose.yml Normal file
View File

@@ -0,0 +1,9 @@
services:
php-app:
build: .
volumes:
- .:/usr/src/app
command: >
sh -c "composer install && php -S 0.0.0.0:8000 -t /usr/src/app"
ports:
- "9000:8000"

5
index.php
View File

@@ -2,7 +2,7 @@
include_once 'vendor/autoload.php';
use Krzysiej\RyobiCrawler\Command\Migrate;
use Krzysiej\RyobiCrawler\Command\ScrapeWebsite;
use Symfony\Component\Console\Application;
@@ -13,6 +13,7 @@ if (php_sapi_name() !== 'cli') {
exit;
}
$application = new Application('Ryobi website scraper application', '1.0.0');
$application = new Application('Ryobi website scraper application', '1.1.0');
$application->add(new ScrapeWebsite());
$application->add(new Migrate());
$application->run();

91
src/Command/Migrate.php Normal file
View File

@@ -0,0 +1,91 @@
<?php
declare(strict_types=1);
namespace Krzysiej\RyobiCrawler\Command;
use Illuminate\Database\Capsule\Manager as Capsule;
use Illuminate\Database\Schema\Blueprint;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
#[AsCommand(name: 'app:migrate', description: 'Create database and rum migrations')]
class Migrate extends Command
{
protected function configure(): void
{
touch(__DIR__ . '/../../database.sqlite');
$capsule = new Capsule;
$capsule->addConnection([
'driver' => 'sqlite',
'database' => __DIR__ . '/../../database.sqlite',
]);
$capsule->setAsGlobal();
$capsule->bootEloquent();
}
public function execute(InputInterface $input, OutputInterface $output): int
{
$this->createProductsTable();
$this->createPricesTable();
$this->createStocksTable();
return Command::SUCCESS;
}
public function createProductsTable(): void
{
if (!Capsule::schema()->hasTable('products')) {
Capsule::schema()->create('products', function (Blueprint $table) {
$table->increments('id');
$table->string('name');
$table->integer('skuID')->unique();
$table->integer('agilityID');
$table->integer('availableQuantity');
$table->integer('stock');
$table->json('categories');
$table->string('image');
$table->string('subTitle');
$table->string('variantCode');
$table->string('modelCode');
$table->string('url');
$table->boolean('starred')->default(false);
$table->timestamps();
});
}
}
public function createPricesTable(): void
{
if (!Capsule::schema()->hasTable('prices')) {
Capsule::schema()->create('prices', function (Blueprint $table) {
$table->increments('id');
$table->foreignId('product_id');
$table->float('price');
$table->float('productStandardPrice');
$table->float('lowestProductPrice30Days');
$table->timestamps();
});
}
}
public function createStocksTable(): void
{
if (!Capsule::schema()->hasTable('stocks')) {
Capsule::schema()->create('stocks', function (Blueprint $table) {
$table->increments('id');
$table->foreignId('product_id');
$table->integer('availableQuantity');
$table->integer('stock');
$table->boolean('isInStock');
$table->boolean('previouslyHadStock');
$table->boolean('isNew');
$table->boolean('isComingSoon');
$table->boolean('isOnSale');
$table->timestamps();
});
}
}
}

53
src/Command/ScrapeWebsite.php
View File

@@ -5,21 +5,23 @@ declare(strict_types=1);
namespace Krzysiej\RyobiCrawler\Command;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use Illuminate\Database\Capsule\Manager as Capsule;
use Krzysiej\RyobiCrawler\Models\Price;
use Krzysiej\RyobiCrawler\Models\Product;
use Krzysiej\RyobiCrawler\Models\Stock;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
#[AsCommand(name: 'app:scrape')]
#[AsCommand(name: 'app:scrape', description: 'Scrape all products from Ryobi website')]
class ScrapeWebsite extends Command
{
private Client $client;
protected function configure()
protected function configure(): void
{
$capsule = new Capsule;
$capsule->addConnection([
@@ -44,6 +46,7 @@ class ScrapeWebsite extends Command
$progress->finish();
$output->writeln('');
$output->writeln('DONE');
return Command::SUCCESS;
}
@@ -52,22 +55,28 @@ class ScrapeWebsite extends Command
$products = [];
$page = 0;
do {
$res = $this->client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [
'form_params' => [
"includePreviousPages" => false,
"pageIndex" => $page,
"pageSize" => 100,
"cultureCode" => "pl-PL",
]
]);
$page++;
$responseObject = json_decode($res->getBody()->getContents());
$products = array_merge($products, $responseObject->products);
} while ((bool)$responseObject->canLoadMore);
try {
$res = $this->client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [
'form_params' => [
"includePreviousPages" => false,
"pageIndex" => $page,
"pageSize" => 100,
"cultureCode" => "pl-PL",
]
]);
$responseObject = json_decode($res->getBody()->getContents());
$products = array_merge($products, $responseObject->products);
$page++;
$canLoadMore = $responseObject->canLoadMore;
} catch (GuzzleException $e) {
return $products;
}
} while ($canLoadMore);
return $products;
}
private function saveProduct(\stdClass $product)
private function saveProduct(\stdClass $product): void
{
/** @var Product $productModel */
$productModel = Product::firstOrNew(['skuID' => $product->skuID]);
@@ -90,5 +99,17 @@ class ScrapeWebsite extends Command
$price->lowestProductPrice30Days = $product->lowestProductPrice30Days;
$productModel->price()->save($price);
}
$stockExist = $productModel->stock()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();
if (!$stockExist) {
$stock = new Stock();
$stock->availableQuantity = $product->availableQuantity;
$stock->stock = $product->stock;
$stock->isInStock = $product->isInStock;
$stock->previouslyHadStock = $product->previouslyHadStock;
$stock->isNew = $product->isNew;
$stock->isComingSoon = $product->isComingSoon;
$stock->isOnSale = $product->isOnSale;
$productModel->stock()->save($stock);
}
}
}
}

7
src/Models/Product.php
View File

@@ -29,6 +29,10 @@ class Product extends Model
{
return $this->hasMany(Price::class);
}
public function stock(): HasMany
{
return $this->hasMany(Stock::class);
}
public function toggleStarred(): self
{
@@ -42,5 +46,4 @@ class Product extends Model
get: fn(string $value) => array_reverse(json_decode($value, 1)),
);
}
}
}

25
src/Models/Stock.php Normal file
View File

@@ -0,0 +1,25 @@
<?php
namespace Krzysiej\RyobiCrawler\Models;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
/**
* @property integer $availableQuantity
* @property integer $stock
* @property boolean $isInStock
* @property boolean $previouslyHadStock
* @property boolean $isNew
* @property boolean $isComingSoon
* @property boolean $isOnSale
*/
class Stock extends Model
{
public $timestamps = true;
public function product(): BelongsTo
{
return $this->belongsTo(Product::class);
}
}

6
src/templates/template.html.twig
View File

@@ -9,11 +9,11 @@
crossorigin="anonymous"/>
</head>
<body>
<nav class="navbar sticky-top bg-body-tertiary">
<nav class="navbar sticky-top bg-body-tertiary border-bottom border-secondary border-1">
<div class="container-fluid">
<a class="navbar-brand" href="/browser.php">Crawler</a>
<form class="d-flex" role="search">
<input class="form-control me-2" type="search" name="search" placeholder="Search" value="{{ search }}" aria-label="Search">
<form class="d-flex w-50" role="search">
<input class="form-control me-2" type="search" name="search" placeholder="Search term eg. 36v or RCS18X" value="{{ search }}" aria-label="Search">
<button class="btn btn-outline-success" type="submit">Search</button>
</form>
</div>