krzysiej/ryobi-crawler
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases 1 Wiki Activity

Change application to symfony/console application

Browse Source
This commit is contained in:
Krzysztof Płaczek
2024-05-08 09:31:26 +02:00
parent b9ba72c308
commit 4f0f0a5f00
4 changed files with 509 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
composer.json
View File

@@ -4,7 +4,8 @@
"symfony/var-dumper": "^7.0",
"illuminate/database": "^11.0",
"ext-json": "*",
"twig/twig": "^3.0"
"twig/twig": "^3.0",
"symfony/console": "^7.0"
},
"autoload": {
"psr-4": {

423
composer.lock generated
View File

@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "ce7d9a1457a9f208ddadf1630309fecc",
"content-hash": "2c2a4512e1fb86ef1101792390337ad1",
"packages": [
{
"name": "brick/math",
@@ -1475,6 +1475,99 @@
],
"time": "2024-03-02T12:46:12+00:00"
},
{
"name": "symfony/console",
"version": "v7.0.7",
"source": {
"type": "git",
"url": "https://github.com/symfony/console.git",
"reference": "c981e0e9380ce9f146416bde3150c79197ce9986"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/console/zipball/c981e0e9380ce9f146416bde3150c79197ce9986",
"reference": "c981e0e9380ce9f146416bde3150c79197ce9986",
"shasum": ""
},
"require": {
"php": ">=8.2",
"symfony/polyfill-mbstring": "~1.0",
"symfony/service-contracts": "^2.5|^3",
"symfony/string": "^6.4|^7.0"
},
"conflict": {
"symfony/dependency-injection": "<6.4",
"symfony/dotenv": "<6.4",
"symfony/event-dispatcher": "<6.4",
"symfony/lock": "<6.4",
"symfony/process": "<6.4"
},
"provide": {
"psr/log-implementation": "1.0|2.0|3.0"
},
"require-dev": {
"psr/log": "^1|^2|^3",
"symfony/config": "^6.4|^7.0",
"symfony/dependency-injection": "^6.4|^7.0",
"symfony/event-dispatcher": "^6.4|^7.0",
"symfony/http-foundation": "^6.4|^7.0",
"symfony/http-kernel": "^6.4|^7.0",
"symfony/lock": "^6.4|^7.0",
"symfony/messenger": "^6.4|^7.0",
"symfony/process": "^6.4|^7.0",
"symfony/stopwatch": "^6.4|^7.0",
"symfony/var-dumper": "^6.4|^7.0"
},
"type": "library",
"autoload": {
"psr-4": {
"Symfony\\Component\\Console\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Fabien Potencier",
"email": "fabien@symfony.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Eases the creation of beautiful and testable command line interfaces",
"homepage": "https://symfony.com",
"keywords": [
"cli",
"command-line",
"console",
"terminal"
],
"support": {
"source": "https://github.com/symfony/console/tree/v7.0.7"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2024-04-18T09:29:19+00:00"
},
{
"name": "symfony/deprecation-contracts",
"version": "v3.4.0",
@@ -1621,6 +1714,165 @@
],
"time": "2024-01-29T20:11:03+00:00"
},
{
"name": "symfony/polyfill-intl-grapheme",
"version": "v1.29.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-intl-grapheme.git",
"reference": "32a9da87d7b3245e09ac426c83d334ae9f06f80f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/32a9da87d7b3245e09ac426c83d334ae9f06f80f",
"reference": "32a9da87d7b3245e09ac426c83d334ae9f06f80f",
"shasum": ""
},
"require": {
"php": ">=7.1"
},
"suggest": {
"ext-intl": "For best performance"
},
"type": "library",
"extra": {
"thanks": {
"name": "symfony/polyfill",
"url": "https://github.com/symfony/polyfill"
}
},
"autoload": {
"files": [
"bootstrap.php"
],
"psr-4": {
"Symfony\\Polyfill\\Intl\\Grapheme\\": ""
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Nicolas Grekas",
"email": "p@tchwork.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Symfony polyfill for intl's grapheme_* functions",
"homepage": "https://symfony.com",
"keywords": [
"compatibility",
"grapheme",
"intl",
"polyfill",
"portable",
"shim"
],
"support": {
"source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.29.0"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2024-01-29T20:11:03+00:00"
},
{
"name": "symfony/polyfill-intl-normalizer",
"version": "v1.29.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-intl-normalizer.git",
"reference": "bc45c394692b948b4d383a08d7753968bed9a83d"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/bc45c394692b948b4d383a08d7753968bed9a83d",
"reference": "bc45c394692b948b4d383a08d7753968bed9a83d",
"shasum": ""
},
"require": {
"php": ">=7.1"
},
"suggest": {
"ext-intl": "For best performance"
},
"type": "library",
"extra": {
"thanks": {
"name": "symfony/polyfill",
"url": "https://github.com/symfony/polyfill"
}
},
"autoload": {
"files": [
"bootstrap.php"
],
"psr-4": {
"Symfony\\Polyfill\\Intl\\Normalizer\\": ""
},
"classmap": [
"Resources/stubs"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Nicolas Grekas",
"email": "p@tchwork.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Symfony polyfill for intl's Normalizer class and related functions",
"homepage": "https://symfony.com",
"keywords": [
"compatibility",
"intl",
"normalizer",
"polyfill",
"portable",
"shim"
],
"support": {
"source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.29.0"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2024-01-29T20:11:03+00:00"
},
{
"name": "symfony/polyfill-mbstring",
"version": "v1.29.0",
@@ -1858,6 +2110,175 @@
],
"time": "2024-01-29T20:11:03+00:00"
},
{
"name": "symfony/service-contracts",
"version": "v3.5.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/service-contracts.git",
"reference": "bd1d9e59a81d8fa4acdcea3f617c581f7475a80f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/service-contracts/zipball/bd1d9e59a81d8fa4acdcea3f617c581f7475a80f",
"reference": "bd1d9e59a81d8fa4acdcea3f617c581f7475a80f",
"shasum": ""
},
"require": {
"php": ">=8.1",
"psr/container": "^1.1|^2.0",
"symfony/deprecation-contracts": "^2.5|^3"
},
"conflict": {
"ext-psr": "<1.1|>=2"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-main": "3.5-dev"
},
"thanks": {
"name": "symfony/contracts",
"url": "https://github.com/symfony/contracts"
}
},
"autoload": {
"psr-4": {
"Symfony\\Contracts\\Service\\": ""
},
"exclude-from-classmap": [
"/Test/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Nicolas Grekas",
"email": "p@tchwork.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Generic abstractions related to writing services",
"homepage": "https://symfony.com",
"keywords": [
"abstractions",
"contracts",
"decoupling",
"interfaces",
"interoperability",
"standards"
],
"support": {
"source": "https://github.com/symfony/service-contracts/tree/v3.5.0"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2024-04-18T09:32:20+00:00"
},
{
"name": "symfony/string",
"version": "v7.0.7",
"source": {
"type": "git",
"url": "https://github.com/symfony/string.git",
"reference": "e405b5424dc2528e02e31ba26b83a79fd4eb8f63"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/string/zipball/e405b5424dc2528e02e31ba26b83a79fd4eb8f63",
"reference": "e405b5424dc2528e02e31ba26b83a79fd4eb8f63",
"shasum": ""
},
"require": {
"php": ">=8.2",
"symfony/polyfill-ctype": "~1.8",
"symfony/polyfill-intl-grapheme": "~1.0",
"symfony/polyfill-intl-normalizer": "~1.0",
"symfony/polyfill-mbstring": "~1.0"
},
"conflict": {
"symfony/translation-contracts": "<2.5"
},
"require-dev": {
"symfony/error-handler": "^6.4|^7.0",
"symfony/http-client": "^6.4|^7.0",
"symfony/intl": "^6.4|^7.0",
"symfony/translation-contracts": "^2.5|^3.0",
"symfony/var-exporter": "^6.4|^7.0"
},
"type": "library",
"autoload": {
"files": [
"Resources/functions.php"
],
"psr-4": {
"Symfony\\Component\\String\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Nicolas Grekas",
"email": "p@tchwork.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Provides an object-oriented API to strings and deals with bytes, UTF-8 code points and grapheme clusters in a unified way",
"homepage": "https://symfony.com",
"keywords": [
"grapheme",
"i18n",
"string",
"unicode",
"utf-8",
"utf8"
],
"support": {
"source": "https://github.com/symfony/string/tree/v7.0.7"
},
"funding": [
{
"url": "https://symfony.com/sponsor",
"type": "custom"
},
{
"url": "https://github.com/fabpot",
"type": "github"
},
{
"url": "https://tidelift.com/funding/github/packagist/symfony/symfony",
"type": "tidelift"
}
],
"time": "2024-04-18T09:29:19+00:00"
},
{
"name": "symfony/translation",
"version": "v7.0.4",

58
index.php
View File

@@ -2,59 +2,17 @@
include_once 'vendor/autoload.php';
use Illuminate\Database\Capsule\Manager as Capsule;
use Krzysiej\RyobiCrawler\Models\Price;
use Krzysiej\RyobiCrawler\Models\Product;
use Krzysiej\RyobiCrawler\Command\ScrapeWebsite;
use Symfony\Component\Console\Application;
if (php_sapi_name() !== 'cli') {
header('Location: browser.php');
echo 'Execute this script in cli only';
exit;
}
$capsule = new Capsule;
$capsule->addConnection([
'driver' => 'sqlite',
'database' => __DIR__ . '/database.sqlite',
]);
$capsule->setAsGlobal();
$capsule->bootEloquent();
$client = new GuzzleHttp\Client();
$page = 0;
do {
$res = $client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [
'form_params' => [
"includePreviousPages" => false,
"pageIndex" => $page,
"pageSize" => 100,
"cultureCode" => "pl-PL",
]
]);
$page++;
$responseObject = json_decode($res->getBody()->getContents());
$products = $responseObject->products;
foreach ($products as $product) {
/** @var Product $productModel */
$productModel = Product::firstOrNew(['skuID' => $product->skuID]);
$productModel->skuID = $product->skuID;
$productModel->name = $product->name;
$productModel->availableQuantity = $product->availableQuantity;
$productModel->stock = $product->stock;
$productModel->categories = json_encode($product->categories);
$productModel->image = $product->image;
$productModel->subTitle = $product->subTitle;
$productModel->variantCode = $product->variantCode;
$productModel->modelCode = $product->modelCode;
$productModel->url = $product->url;
$productModel->save();
$priceExists = $productModel->price()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();
if (!$priceExists) {
$price = new Price();
$price->price = $product->productPrice;
$price->productStandardPrice = $product->productStandardPrice;
$price->lowestProductPrice30Days = $product->lowestProductPrice30Days;
$productModel->price()->save($price);
}
echo ".";
}
echo "\n";
} while ((bool)$responseObject->canLoadMore);
$application = new Application('Ryobi website scraper application', '1.0.0');
$application->add(new ScrapeWebsite());
$application->run();

77
src/Command/ScrapeWebsite.php Normal file
View File

@@ -0,0 +1,77 @@
<?php
declare(strict_types=1);
namespace Krzysiej\RyobiCrawler\Command;
use GuzzleHttp\Client;
use Illuminate\Database\Capsule\Manager as Capsule;
use Krzysiej\RyobiCrawler\Models\Price;
use Krzysiej\RyobiCrawler\Models\Product;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
#[AsCommand(name: 'app:scrape')]
class ScrapeWebsite extends Command
{
private Client $client;
private Capsule $capsule;
protected function configure()
{
$this->capsule = new Capsule;
$this->capsule->addConnection([
'driver' => 'sqlite',
'database' => __DIR__ .'/../../database.sqlite',
]);
$this->capsule->setAsGlobal();
$this->capsule->bootEloquent();
$this->client = new Client();
}
public function execute(InputInterface $input, OutputInterface $output): int
{
$page = 0;
do {
$res = $this->client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [
'form_params' => [
"includePreviousPages" => false,
"pageIndex" => $page,
"pageSize" => 100,
"cultureCode" => "pl-PL",
]
]);
$page++;
$responseObject = json_decode($res->getBody()->getContents());
$products = $responseObject->products;
foreach ($products as $product) {
/** @var Product $productModel */
$productModel = Product::firstOrNew(['skuID' => $product->skuID]);
$productModel->skuID = $product->skuID;
$productModel->name = $product->name;
$productModel->availableQuantity = $product->availableQuantity;
$productModel->stock = $product->stock;
$productModel->categories = json_encode($product->categories);
$productModel->image = $product->image;
$productModel->subTitle = $product->subTitle;
$productModel->variantCode = $product->variantCode;
$productModel->modelCode = $product->modelCode;
$productModel->url = $product->url;
$productModel->save();
$priceExists = $productModel->price()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();
if (!$priceExists) {
$price = new Price();
$price->price = $product->productPrice;
$price->productStandardPrice = $product->productStandardPrice;
$price->lowestProductPrice30Days = $product->lowestProductPrice30Days;
$productModel->price()->save($price);
}
echo ".";
}
echo "\n";
} while ((bool)$responseObject->canLoadMore);
return Command::SUCCESS;
}
}