Compare commits
15 Commits
1.1.1
...
a911d614e0
| Author | SHA1 | Date | |
|---|---|---|---|
| a911d614e0 | |||
|
|
58562746a3 | ||
|
|
5ee1bba812 | ||
| 3d47726a81 | |||
|
|
fee7495cfe | ||
|
|
53b6d33ab9 | ||
|
|
3cc5d73758 | ||
|
|
d043e8efb1 | ||
|
|
2b595c1403 | ||
|
|
f2e6cba2f5 | ||
|
|
e13a0acce1 | ||
|
|
77732689d8 | ||
|
|
22cc3f699b | ||
|
|
a1b6d65137 | ||
|
|
7ee8df32f9 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,3 +1,4 @@
|
|||||||
/vendor/
|
/vendor/
|
||||||
.idea
|
.idea
|
||||||
database.sqlite
|
database.sqlite
|
||||||
|
var/cache/
|
||||||
25
README.md
25
README.md
@@ -5,6 +5,25 @@
|
|||||||
1. Clone repository using `git clone https://git.techtube.pl/krzysiej/ryobi-crawler.git`
|
1. Clone repository using `git clone https://git.techtube.pl/krzysiej/ryobi-crawler.git`
|
||||||
2. Cd into project directory `cd ryobi-crawler`
|
2. Cd into project directory `cd ryobi-crawler`
|
||||||
3. Build and start docker container `docker compose up -d`
|
3. Build and start docker container `docker compose up -d`
|
||||||
4. Run `docker compose exec php-app php index.php app:migrate` file to create `database.sqlite` and create tables.
|
4. Run `docker compose exec php-app php console.php app:migrate` file to create `database.sqlite` and create tables.
|
||||||
5. Run `docker compose exec php-app php index.php app:scrape` command to scrape all the products from the ryobi website.
|
5. Run `docker compose exec php-app php console.php app:scrape` command to scrape all the products from the ryobi website.
|
||||||
6. Access web interface using `localhost:9000` address in web browser.
|
6. Access web interface using `localhost:9001` address in web browser.
|
||||||
|
|
||||||
|
|
||||||
|
## Update project
|
||||||
|
|
||||||
|
1. Cd into project directory
|
||||||
|
2. Run `git pull`
|
||||||
|
3. Start and build image in one go with command: `docker compose up -d --build --force-recreate`
|
||||||
|
|
||||||
|
## Running Cron
|
||||||
|
|
||||||
|
For now only way of running `app:scrape` command on schedule is to use host crontab.
|
||||||
|
1. Run `crontab -e` command to edit host crontab job file
|
||||||
|
2. Add a new line with e.g. line like this `0 1 * * * cd /var/project/directory/ && docker compose exec php-app php console.php app:scrape`
|
||||||
|
3. Save and exit file editor. Cron will execute `app:scrape` once per day.
|
||||||
|
|
||||||
|
## Screenshots
|
||||||
|
|
||||||
|
### Main screen of the web view
|
||||||
|

|
||||||
42
browser.php
42
browser.php
@@ -1,42 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
include_once 'vendor/autoload.php';
|
|
||||||
|
|
||||||
use Illuminate\Database\Capsule\Manager as Capsule;
|
|
||||||
use Krzysiej\RyobiCrawler\Models\Product;
|
|
||||||
use Twig\{Environment, Loader\FilesystemLoader};
|
|
||||||
|
|
||||||
if (!file_exists('database.sqlite')) {
|
|
||||||
exit('Database file <code>database.sqlite</code> missing. Run docker compose <blockquote>docker compose exec php-app php index.php app:migrate</blockquote> to create it.');
|
|
||||||
}
|
|
||||||
|
|
||||||
$capsule = new Capsule;
|
|
||||||
$capsule->addConnection(['driver' => 'sqlite', 'database' => __DIR__ . '/database.sqlite']);
|
|
||||||
$capsule->setAsGlobal();
|
|
||||||
$capsule->bootEloquent();
|
|
||||||
$loader = new FilesystemLoader(__DIR__ . '/src/templates');
|
|
||||||
$twig = new Environment($loader);
|
|
||||||
if (isset($_GET['product_id'])) {
|
|
||||||
$template = 'product.html.twig';
|
|
||||||
$product = Product::with('price')->find($_GET['product_id']);
|
|
||||||
}
|
|
||||||
if (isset($_GET['category'])) {
|
|
||||||
$template = 'productList.html.twig';
|
|
||||||
$products = Product::with('price')->selectRaw('products.*')->fromRaw('products, json_each(products.categories)')->whereRaw('json_each.value = ?', [$_GET['category']])
|
|
||||||
->orderByDesc('starred')->orderByDesc('created_by')->get();
|
|
||||||
}
|
|
||||||
if (isset($_GET['search'])) {
|
|
||||||
$template = 'productList.html.twig';
|
|
||||||
$products = Product::with('price')
|
|
||||||
->orWhere([['name', 'like', "%{$_GET['search']}%"]])
|
|
||||||
->orWhere([['subTitle', 'like', "%{$_GET['search']}%"]])->get();
|
|
||||||
}
|
|
||||||
if (isset($_GET['star'])) {
|
|
||||||
Product::find($_GET['star'])->toggleStarred()->save();
|
|
||||||
header('Location: '.$_SERVER['HTTP_REFERER']);
|
|
||||||
}
|
|
||||||
if (empty($_GET)) {
|
|
||||||
$template = 'productList.html.twig';
|
|
||||||
$products = Product::with('price')->orderByDesc('starred')->orderByDesc('created_by')->get();
|
|
||||||
}
|
|
||||||
$twig->display($template, ['products' => $products ?? [], 'product' => $product ?? [], 'search' => $_GET['search'] ?? '']);
|
|
||||||
@@ -4,8 +4,12 @@
|
|||||||
"symfony/var-dumper": "^7.0",
|
"symfony/var-dumper": "^7.0",
|
||||||
"illuminate/database": "11.26.0.0",
|
"illuminate/database": "11.26.0.0",
|
||||||
"ext-json": "*",
|
"ext-json": "*",
|
||||||
"twig/twig": "3.14.0.0",
|
"symfony/console": "^7.0",
|
||||||
"symfony/console": "^7.0"
|
"symfony/routing": "^7.1",
|
||||||
|
"laravel/serializable-closure": "^1.3",
|
||||||
|
"symfony/http-kernel": "^7.1",
|
||||||
|
"symfony/framework-bundle": "^7.1",
|
||||||
|
"symfony/twig-bundle": "^7.1"
|
||||||
},
|
},
|
||||||
"autoload": {
|
"autoload": {
|
||||||
"psr-4": {
|
"psr-4": {
|
||||||
|
|||||||
1589
composer.lock
generated
1589
composer.lock
generated
File diff suppressed because it is too large
Load Diff
19
console.php
Normal file
19
console.php
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
include_once 'vendor/autoload.php';
|
||||||
|
|
||||||
|
use Krzysiej\RyobiCrawler\Command\Migrate;
|
||||||
|
use Krzysiej\RyobiCrawler\Command\ScrapeWebsite;
|
||||||
|
|
||||||
|
use Symfony\Component\Console\Application;
|
||||||
|
|
||||||
|
if (php_sapi_name() !== 'cli') {
|
||||||
|
header('Location: browser.php');
|
||||||
|
echo 'Execute this script in cli only';
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
$application = new Application('Ryobi website scraper application', '1.1.1');
|
||||||
|
$application->add(new ScrapeWebsite());
|
||||||
|
$application->add(new Migrate());
|
||||||
|
$application->run();
|
||||||
@@ -1,9 +1,10 @@
|
|||||||
services:
|
services:
|
||||||
php-app:
|
php-app:
|
||||||
|
restart: unless-stopped
|
||||||
build: .
|
build: .
|
||||||
volumes:
|
volumes:
|
||||||
- .:/usr/src/app
|
- .:/usr/src/app
|
||||||
command: >
|
command: >
|
||||||
sh -c "composer install && php -S 0.0.0.0:8000 -t /usr/src/app"
|
sh -c "composer install && php -S 0.0.0.0:8000 -t /usr/src/app"
|
||||||
ports:
|
ports:
|
||||||
- "9000:8000"
|
- "9001:8000"
|
||||||
50
index.php
50
index.php
@@ -1,19 +1,45 @@
|
|||||||
<?php
|
<?php
|
||||||
|
|
||||||
include_once 'vendor/autoload.php';
|
use Symfony\Bundle\FrameworkBundle\Kernel\MicroKernelTrait;
|
||||||
|
use Symfony\Component\DependencyInjection\Loader\Configurator\ContainerConfigurator;
|
||||||
|
use Symfony\Component\HttpFoundation\Request;
|
||||||
|
use Symfony\Component\HttpKernel\Kernel as BaseKernel;
|
||||||
|
use Symfony\Component\Routing\Loader\Configurator\RoutingConfigurator;
|
||||||
|
|
||||||
use Krzysiej\RyobiCrawler\Command\Migrate;
|
require 'vendor/autoload.php';
|
||||||
use Krzysiej\RyobiCrawler\Command\ScrapeWebsite;
|
|
||||||
|
|
||||||
use Symfony\Component\Console\Application;
|
class Kernel extends BaseKernel
|
||||||
|
{
|
||||||
|
use MicroKernelTrait;
|
||||||
|
|
||||||
if (php_sapi_name() !== 'cli') {
|
public function registerBundles(): iterable
|
||||||
header('Location: browser.php');
|
{
|
||||||
echo 'Execute this script in cli only';
|
return [
|
||||||
exit;
|
new Symfony\Bundle\FrameworkBundle\FrameworkBundle(),
|
||||||
|
new Symfony\Bundle\TwigBundle\TwigBundle()
|
||||||
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
$application = new Application('Ryobi website scraper application', '1.1.1');
|
protected function configureContainer(ContainerConfigurator $container): void
|
||||||
$application->add(new ScrapeWebsite());
|
{
|
||||||
$application->add(new Migrate());
|
$container->extension('framework', [
|
||||||
$application->run();
|
'secret' => 'S0ME_SECRET'
|
||||||
|
]);
|
||||||
|
$container->services()
|
||||||
|
->load('Krzysiej\\RyobiCrawler\\Controller\\', __DIR__.'/src/Controller/*')
|
||||||
|
->autowire()
|
||||||
|
->autoconfigure()
|
||||||
|
;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected function configureRoutes(RoutingConfigurator $routes): void
|
||||||
|
{
|
||||||
|
$routes->import(__DIR__.'/src/Controller/', 'attribute');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$kernel = new Kernel('prod', false);
|
||||||
|
$request = Request::createFromGlobals();
|
||||||
|
$response = $kernel->handle($request);
|
||||||
|
$response->send();
|
||||||
|
$kernel->terminate($request, $response);
|
||||||
48
index_old.php
Normal file
48
index_old.php
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
include_once 'vendor/autoload.php';
|
||||||
|
|
||||||
|
use Krzysiej\RyobiCrawler\Controller\CategoryController;
|
||||||
|
use Krzysiej\RyobiCrawler\Controller\IndexController;
|
||||||
|
use Krzysiej\RyobiCrawler\Controller\ProductController;
|
||||||
|
use Krzysiej\RyobiCrawler\Controller\PromosController;
|
||||||
|
use Krzysiej\RyobiCrawler\Controller\SearchController;
|
||||||
|
use Krzysiej\RyobiCrawler\Controller\StarController;
|
||||||
|
use Symfony\Component\Routing\Matcher\UrlMatcher;
|
||||||
|
use Symfony\Component\Routing\RequestContext;
|
||||||
|
use Symfony\Component\Routing\Route;
|
||||||
|
use Symfony\Component\Routing\RouteCollection;
|
||||||
|
|
||||||
|
if (!file_exists('database.sqlite')) {
|
||||||
|
exit('Database file <code>database.sqlite</code> missing. Run docker compose <blockquote>docker compose exec php-app php index.php app:migrate</blockquote> to create it.');
|
||||||
|
}
|
||||||
|
$productRoute = new Route('/product/{product_id<\d+>}', ['_controller' => ProductController::class]);
|
||||||
|
$searchRoute = new Route('/?search={search_term}', ['_controller' => SearchController::class]);
|
||||||
|
$categoryRoute = new Route('/category/{category_name}', ['_controller' => CategoryController::class]);
|
||||||
|
$starRoute = new Route('/star/{product_id}', ['_controller' => StarController::class]);
|
||||||
|
$promosRoute = new Route('/promos', ['_controller' => PromosController::class]);
|
||||||
|
$indexRoute = new Route('/', ['_controller' => IndexController::class]);
|
||||||
|
$routes = new RouteCollection();
|
||||||
|
|
||||||
|
$routes->add('product_show', $productRoute);
|
||||||
|
$routes->add('search_show', $searchRoute);
|
||||||
|
$routes->add('category_show', $categoryRoute);
|
||||||
|
$routes->add('start_show', $starRoute);
|
||||||
|
$routes->add('promos_show', $promosRoute);
|
||||||
|
$routes->add('index_show', $indexRoute);
|
||||||
|
|
||||||
|
$context = new RequestContext();
|
||||||
|
$matcher = new UrlMatcher($routes, $context);
|
||||||
|
try {
|
||||||
|
$parameters = $matcher->match($_SERVER['REQUEST_URI']);
|
||||||
|
} catch (Exception $e) {
|
||||||
|
die($e->getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
match ($parameters['_controller']) {
|
||||||
|
SearchController::class => (new $parameters['_controller']())($parameters['search_term']),
|
||||||
|
CategoryController::class => (new $parameters['_controller']())($parameters['category_name']),
|
||||||
|
ProductController::class, StarController::class => (new $parameters['_controller']())($parameters['product_id']),
|
||||||
|
PromosController::class, IndexController::class => (new $parameters['_controller']())(),
|
||||||
|
default => throw new Exception('Route not found')
|
||||||
|
};
|
||||||
BIN
screenshot1.png
Normal file
BIN
screenshot1.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 133 KiB |
@@ -9,13 +9,25 @@ use Illuminate\Database\Schema\Blueprint;
|
|||||||
use Symfony\Component\Console\Attribute\AsCommand;
|
use Symfony\Component\Console\Attribute\AsCommand;
|
||||||
use Symfony\Component\Console\Command\Command;
|
use Symfony\Component\Console\Command\Command;
|
||||||
use Symfony\Component\Console\Input\InputInterface;
|
use Symfony\Component\Console\Input\InputInterface;
|
||||||
|
use Symfony\Component\Console\Input\InputOption;
|
||||||
use Symfony\Component\Console\Output\OutputInterface;
|
use Symfony\Component\Console\Output\OutputInterface;
|
||||||
|
|
||||||
#[AsCommand(name: 'app:migrate', description: 'Create database and rum migrations')]
|
#[AsCommand(name: 'app:migrate', description: 'Create database and rum migrations')]
|
||||||
class Migrate extends Command
|
class Migrate extends Command
|
||||||
{
|
{
|
||||||
|
private const RECREATE_OPTION = 'recreate';
|
||||||
|
|
||||||
protected function configure(): void
|
protected function configure(): void
|
||||||
{
|
{
|
||||||
|
$this->addOption(self::RECREATE_OPTION, null, InputOption::VALUE_OPTIONAL, 'Recreate database file event if exist and has data.');
|
||||||
|
}
|
||||||
|
|
||||||
|
public function execute(InputInterface $input, OutputInterface $output): int
|
||||||
|
{
|
||||||
|
if (true === $input->hasOption(self::RECREATE_OPTION)) {
|
||||||
|
unlink(__DIR__ . '/../../database.sqlite');
|
||||||
|
//sleep(5);
|
||||||
|
}
|
||||||
touch(__DIR__ . '/../../database.sqlite');
|
touch(__DIR__ . '/../../database.sqlite');
|
||||||
$capsule = new Capsule;
|
$capsule = new Capsule;
|
||||||
$capsule->addConnection([
|
$capsule->addConnection([
|
||||||
@@ -24,10 +36,6 @@ class Migrate extends Command
|
|||||||
]);
|
]);
|
||||||
$capsule->setAsGlobal();
|
$capsule->setAsGlobal();
|
||||||
$capsule->bootEloquent();
|
$capsule->bootEloquent();
|
||||||
}
|
|
||||||
|
|
||||||
public function execute(InputInterface $input, OutputInterface $output): int
|
|
||||||
{
|
|
||||||
$this->createProductsTable();
|
$this->createProductsTable();
|
||||||
$this->createPricesTable();
|
$this->createPricesTable();
|
||||||
$this->createStocksTable();
|
$this->createStocksTable();
|
||||||
@@ -42,7 +50,6 @@ class Migrate extends Command
|
|||||||
$table->increments('id');
|
$table->increments('id');
|
||||||
$table->string('name');
|
$table->string('name');
|
||||||
$table->integer('skuID')->unique();
|
$table->integer('skuID')->unique();
|
||||||
$table->integer('agilityID');
|
|
||||||
$table->integer('availableQuantity');
|
$table->integer('availableQuantity');
|
||||||
$table->integer('stock');
|
$table->integer('stock');
|
||||||
$table->json('categories');
|
$table->json('categories');
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ class ScrapeWebsite extends Command
|
|||||||
$products = $this->getProducts();
|
$products = $this->getProducts();
|
||||||
$progress->setMaxSteps(count($products));
|
$progress->setMaxSteps(count($products));
|
||||||
foreach ($products as $product) {
|
foreach ($products as $product) {
|
||||||
|
//dd($product);
|
||||||
$this->saveProduct($product);
|
$this->saveProduct($product);
|
||||||
$progress->advance();
|
$progress->advance();
|
||||||
}
|
}
|
||||||
@@ -80,11 +81,13 @@ class ScrapeWebsite extends Command
|
|||||||
{
|
{
|
||||||
/** @var Product $productModel */
|
/** @var Product $productModel */
|
||||||
$productModel = Product::firstOrNew(['skuID' => $product->skuID]);
|
$productModel = Product::firstOrNew(['skuID' => $product->skuID]);
|
||||||
|
//dd($productModel);
|
||||||
|
|
||||||
$productModel->skuID = $product->skuID;
|
$productModel->skuID = $product->skuID;
|
||||||
$productModel->name = $product->name;
|
$productModel->name = $product->name;
|
||||||
$productModel->availableQuantity = $product->availableQuantity;
|
$productModel->availableQuantity = $product->availableQuantity;
|
||||||
$productModel->stock = $product->stock;
|
$productModel->stock = $product->stock;
|
||||||
$productModel->categories = json_encode($product->categories);
|
$productModel->categories = $product->categories;
|
||||||
$productModel->image = $product->image;
|
$productModel->image = $product->image;
|
||||||
$productModel->subTitle = $product->subTitle;
|
$productModel->subTitle = $product->subTitle;
|
||||||
$productModel->variantCode = $product->variantCode;
|
$productModel->variantCode = $product->variantCode;
|
||||||
@@ -92,7 +95,8 @@ class ScrapeWebsite extends Command
|
|||||||
$productModel->url = $product->url;
|
$productModel->url = $product->url;
|
||||||
$productModel->save();
|
$productModel->save();
|
||||||
$priceExists = $productModel->price()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();
|
$priceExists = $productModel->price()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();
|
||||||
if (!$priceExists) {
|
|
||||||
|
if (false === $priceExists) {
|
||||||
$price = new Price();
|
$price = new Price();
|
||||||
$price->price = $product->productPrice;
|
$price->price = $product->productPrice;
|
||||||
$price->productStandardPrice = $product->productStandardPrice;
|
$price->productStandardPrice = $product->productStandardPrice;
|
||||||
@@ -100,7 +104,7 @@ class ScrapeWebsite extends Command
|
|||||||
$productModel->price()->save($price);
|
$productModel->price()->save($price);
|
||||||
}
|
}
|
||||||
$stockExist = $productModel->stock()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();
|
$stockExist = $productModel->stock()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();
|
||||||
if (!$stockExist) {
|
if (false === $stockExist) {
|
||||||
$stock = new Stock();
|
$stock = new Stock();
|
||||||
$stock->availableQuantity = $product->availableQuantity;
|
$stock->availableQuantity = $product->availableQuantity;
|
||||||
$stock->stock = $product->stock;
|
$stock->stock = $product->stock;
|
||||||
|
|||||||
20
src/Controller/BaseController.php
Normal file
20
src/Controller/BaseController.php
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace Krzysiej\RyobiCrawler\Controller;
|
||||||
|
|
||||||
|
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
|
||||||
|
use Twig\Environment;
|
||||||
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||||
|
|
||||||
|
class BaseController extends AbstractController
|
||||||
|
{
|
||||||
|
protected Environment $twig;
|
||||||
|
|
||||||
|
public function __construct()
|
||||||
|
{
|
||||||
|
$capsule = new Capsule;
|
||||||
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => __DIR__ . '/../../database.sqlite']);
|
||||||
|
$capsule->setAsGlobal();
|
||||||
|
$capsule->bootEloquent();
|
||||||
|
}
|
||||||
|
}
|
||||||
23
src/Controller/CategoryController.php
Normal file
23
src/Controller/CategoryController.php
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace Krzysiej\RyobiCrawler\Controller;
|
||||||
|
|
||||||
|
use Krzysiej\RyobiCrawler\Models\Product;
|
||||||
|
use Symfony\Component\HttpFoundation\Response;
|
||||||
|
use Symfony\Component\Routing\Attribute\Route;
|
||||||
|
|
||||||
|
final class CategoryController extends BaseController
|
||||||
|
{
|
||||||
|
#[Route('/category/{category}', name: 'app_category')]
|
||||||
|
public function __invoke(string $category): Response
|
||||||
|
{
|
||||||
|
$products = Product::with('price')
|
||||||
|
->selectRaw('products.*')
|
||||||
|
->fromRaw('products, json_each(products.categories)')
|
||||||
|
->whereRaw('json_each.value = ?', [$category])
|
||||||
|
->orderByDesc('starred')
|
||||||
|
->orderByDesc('created_by')
|
||||||
|
->get();
|
||||||
|
return $this->render('productList.html.twig', ['products' => $products]);
|
||||||
|
}
|
||||||
|
}
|
||||||
17
src/Controller/IndexController.php
Normal file
17
src/Controller/IndexController.php
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace Krzysiej\RyobiCrawler\Controller;
|
||||||
|
|
||||||
|
use Krzysiej\RyobiCrawler\Models\Product;
|
||||||
|
use Symfony\Component\HttpFoundation\Response;
|
||||||
|
use Symfony\Component\Routing\Attribute\Route;
|
||||||
|
|
||||||
|
final class IndexController extends BaseController
|
||||||
|
{
|
||||||
|
#[Route('/', name: 'app_home')]
|
||||||
|
public function __invoke(): Response
|
||||||
|
{
|
||||||
|
$products = Product::with('price')->orderByDesc('starred')->orderByDesc('created_by')->get();
|
||||||
|
return $this->render('productList.html.twig', ['products' => $products]);
|
||||||
|
}
|
||||||
|
}
|
||||||
35
src/Controller/ProductController.php
Normal file
35
src/Controller/ProductController.php
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace Krzysiej\RyobiCrawler\Controller;
|
||||||
|
|
||||||
|
use Illuminate\Support\ItemNotFoundException;
|
||||||
|
use Krzysiej\RyobiCrawler\Models\Product;
|
||||||
|
use Symfony\Component\HttpFoundation\Response;
|
||||||
|
use Symfony\Component\Routing\Attribute\Route;
|
||||||
|
use Twig\Error\LoaderError;
|
||||||
|
use Twig\Error\RuntimeError;
|
||||||
|
use Twig\Error\SyntaxError;
|
||||||
|
|
||||||
|
final class ProductController extends BaseController
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* @throws SyntaxError
|
||||||
|
* @throws RuntimeError
|
||||||
|
* @throws LoaderError
|
||||||
|
*/
|
||||||
|
#[Route('/product/{productId<\d+>}', name: 'app_product')]
|
||||||
|
public function __invoke(int $productId): Response
|
||||||
|
{
|
||||||
|
$product = Product::with([
|
||||||
|
'price' => fn($query) => $query->orderBy('created_at', 'desc')
|
||||||
|
])->find($productId);
|
||||||
|
|
||||||
|
if(null === $product) {
|
||||||
|
throw $this->createNotFoundException('Product not found');
|
||||||
|
}
|
||||||
|
|
||||||
|
$priceList = $product->price()->pluck('price')->implode(',');
|
||||||
|
$priceDates = $product->price()->pluck('created_at')->map(fn($date) => $date->format('Y-m-d'))->implode("','");
|
||||||
|
return $this->render('product.html.twig', ['product' => $product, 'price_list' => $priceList, 'price_dates' => $priceDates]);
|
||||||
|
}
|
||||||
|
}
|
||||||
18
src/Controller/PromosController.php
Normal file
18
src/Controller/PromosController.php
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace Krzysiej\RyobiCrawler\Controller;
|
||||||
|
|
||||||
|
use Illuminate\Database\Eloquent\Builder;
|
||||||
|
use Krzysiej\RyobiCrawler\Models\Product;
|
||||||
|
use Symfony\Component\HttpFoundation\Response;
|
||||||
|
use Symfony\Component\Routing\Attribute\Route;
|
||||||
|
|
||||||
|
final class PromosController extends BaseController
|
||||||
|
{
|
||||||
|
#[Route('/promos', name: 'app_promos')]
|
||||||
|
public function __invoke(): Response
|
||||||
|
{
|
||||||
|
$products = Product::whereHas('currentPrice', fn(Builder $query) => $query->whereColumn('price', '<', 'productStandardPrice'))->with(['currentPrice'])->get();
|
||||||
|
return $this->render('productList.html.twig', ['products' => $products]);
|
||||||
|
}
|
||||||
|
}
|
||||||
23
src/Controller/SearchController.php
Normal file
23
src/Controller/SearchController.php
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace Krzysiej\RyobiCrawler\Controller;
|
||||||
|
|
||||||
|
use Krzysiej\RyobiCrawler\Models\Product;
|
||||||
|
use Symfony\Component\HttpFoundation\Request;
|
||||||
|
use Symfony\Component\HttpFoundation\Response;
|
||||||
|
use Symfony\Component\Routing\Attribute\Route;
|
||||||
|
|
||||||
|
final class SearchController extends BaseController
|
||||||
|
{
|
||||||
|
#[Route('/search', name: 'app_search')]
|
||||||
|
public function __invoke(Request $request): Response
|
||||||
|
{
|
||||||
|
$search = $request->query->get('search');
|
||||||
|
//dd();
|
||||||
|
$products = Product::with('price')
|
||||||
|
->orWhere([['name', 'like', "%$search%"]])
|
||||||
|
->orWhere([['subTitle', 'like', "%$search%"]])->get();
|
||||||
|
|
||||||
|
return $this->render('productList.html.twig', ['products' => $products ?? [], 'search' => $search]);
|
||||||
|
}
|
||||||
|
}
|
||||||
18
src/Controller/StarController.php
Normal file
18
src/Controller/StarController.php
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace Krzysiej\RyobiCrawler\Controller;
|
||||||
|
|
||||||
|
use Krzysiej\RyobiCrawler\Models\Product;
|
||||||
|
use Symfony\Component\HttpFoundation\Request;
|
||||||
|
use Symfony\Component\HttpFoundation\Response;
|
||||||
|
use Symfony\Component\Routing\Attribute\Route;
|
||||||
|
|
||||||
|
final class StarController extends BaseController
|
||||||
|
{
|
||||||
|
#[Route('/star/{productId<\d+>}', name: 'app_star')]
|
||||||
|
public function __invoke(int $productId, Request $request): Response
|
||||||
|
{
|
||||||
|
Product::find($productId)->toggleStarred()->save();
|
||||||
|
return $this->redirect($request->headers->get('referer'));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,10 +5,10 @@ namespace Krzysiej\RyobiCrawler\Models;
|
|||||||
use Illuminate\Database\Eloquent\Casts\Attribute;
|
use Illuminate\Database\Eloquent\Casts\Attribute;
|
||||||
use Illuminate\Database\Eloquent\Model;
|
use Illuminate\Database\Eloquent\Model;
|
||||||
use Illuminate\Database\Eloquent\Relations\HasMany;
|
use Illuminate\Database\Eloquent\Relations\HasMany;
|
||||||
|
use Illuminate\Database\Eloquent\Relations\HasOne;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @property integer $skuID
|
* @property integer $skuID
|
||||||
* @property integer $agilityID
|
|
||||||
* @property string $name
|
* @property string $name
|
||||||
* @property integer $availableQuantity
|
* @property integer $availableQuantity
|
||||||
* @property integer $stock
|
* @property integer $stock
|
||||||
@@ -18,7 +18,7 @@ use Illuminate\Database\Eloquent\Relations\HasMany;
|
|||||||
* @property string $variantCode
|
* @property string $variantCode
|
||||||
* @property string $modelCode
|
* @property string $modelCode
|
||||||
* @property string $url
|
* @property string $url
|
||||||
* @property boolean $starred
|
* @property int $starred
|
||||||
*/
|
*/
|
||||||
class Product extends Model
|
class Product extends Model
|
||||||
{
|
{
|
||||||
@@ -29,6 +29,17 @@ class Product extends Model
|
|||||||
{
|
{
|
||||||
return $this->hasMany(Price::class);
|
return $this->hasMany(Price::class);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function isStarred(): bool
|
||||||
|
{
|
||||||
|
return (bool) $this->starred;
|
||||||
|
}
|
||||||
|
|
||||||
|
public function currentPrice(): HasOne
|
||||||
|
{
|
||||||
|
return $this->hasOne(Price::class)->latestOfMany('created_at');
|
||||||
|
}
|
||||||
|
|
||||||
public function stock(): HasMany
|
public function stock(): HasMany
|
||||||
{
|
{
|
||||||
return $this->hasMany(Stock::class);
|
return $this->hasMany(Stock::class);
|
||||||
@@ -37,13 +48,15 @@ class Product extends Model
|
|||||||
public function toggleStarred(): self
|
public function toggleStarred(): self
|
||||||
{
|
{
|
||||||
$this->starred = !$this->starred;
|
$this->starred = !$this->starred;
|
||||||
|
|
||||||
return $this;
|
return $this;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected function categories(): Attribute
|
public function categories(): Attribute
|
||||||
{
|
{
|
||||||
return Attribute::make(
|
return Attribute::make(
|
||||||
get: fn(string $value) => array_reverse(json_decode($value, 1)),
|
get: fn(string $value) => array_reverse(json_decode($value, 1)),
|
||||||
|
set: fn(array $value) => json_encode($value),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,42 +0,0 @@
|
|||||||
{% extends "template.html.twig" %}
|
|
||||||
|
|
||||||
{% block content %}
|
|
||||||
<table class='table table-hover'>
|
|
||||||
<tr>
|
|
||||||
<td><img src='{{ product.image }}&width=150' class='img-fluid' alt='{{ product.name }}'/></td>
|
|
||||||
<td><a href='?product_id={{ product.id }}'>{{ product.name }}</a></td>
|
|
||||||
<td>{{ product.subTitle }}</td>
|
|
||||||
<td>
|
|
||||||
<ul class='nav'>
|
|
||||||
{% for category in product.categories %}
|
|
||||||
<li class="nav-item"><a class="nav-link" href="?category={{ category }}"> {{ category }} </a></li>
|
|
||||||
{% endfor %}
|
|
||||||
</ul>
|
|
||||||
</td>
|
|
||||||
<td><a href='https://pl.ryobitools.eu/{{ product.url }}'>link</a></td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td colspan="4">
|
|
||||||
|
|
||||||
<table class='table table-hover table-sm mb-0'>
|
|
||||||
<thead>
|
|
||||||
<tr>
|
|
||||||
<th>price</th>
|
|
||||||
<th>lowest product price in 30 days</th>
|
|
||||||
<th colspan='2'>standard price</th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
{% for price in product.price %}
|
|
||||||
<tr>
|
|
||||||
<td>{{ price.price }}</td>
|
|
||||||
<td>{{ price.lowestProductPrice30Days }}</td>
|
|
||||||
<td>{{ price.productStandardPrice }}</td>
|
|
||||||
<td>{{ price.created_at }}</td>
|
|
||||||
</tr>
|
|
||||||
{% endfor %}
|
|
||||||
</table>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
</table>
|
|
||||||
{% endblock %}
|
|
||||||
|
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
{% extends "template.html.twig" %}
|
|
||||||
|
|
||||||
{% block content %}
|
|
||||||
<table class='table table-hover'>
|
|
||||||
{% for product in products %}
|
|
||||||
<tr>
|
|
||||||
<td class="align-middle font-weight-bold h3"><a class="text-warning text-decoration-none" href="?star={{ product.id }}">{% if product.starred %}★{% else %} ☆ {% endif %}</a></td>
|
|
||||||
<td><img src='{{ product.image }}&width=70' class='img-fluid' alt='{{ product.name }}'/></td>
|
|
||||||
<td class="align-middle"><a href='?product_id={{ product.id }}'>{{ product.name }}</a></td>
|
|
||||||
<td class="align-middle">{{ product.subTitle }}</td>
|
|
||||||
<td class="align-middle">
|
|
||||||
<ul class='nav'>
|
|
||||||
{% for category in product.categories %}
|
|
||||||
<li class="nav-item"><a class="nav-link" href="?category={{ category }}"> {{ category }} </a></li>
|
|
||||||
{% endfor %}
|
|
||||||
</ul>
|
|
||||||
</td>
|
|
||||||
<td class="align-middle"><a href='https://pl.ryobitools.eu/{{ product.url }}'>link</a></td>
|
|
||||||
<td class="align-middle">{{ product.price.last.price }}</td>
|
|
||||||
</tr>
|
|
||||||
{% endfor %}
|
|
||||||
</table>
|
|
||||||
{% endblock %}
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
<!doctype html>
|
|
||||||
<html lang="en">
|
|
||||||
<head>
|
|
||||||
<meta charset="UTF-8">
|
|
||||||
<meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
|
|
||||||
<meta http-equiv="X-UA-Compatible" content="ie=edge">
|
|
||||||
<title>Ryobi crawler</title>
|
|
||||||
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-QWTKZyjpPEjISv5WaRU9OFeRpok6YctnYmDr5pNlyT2bRjXh0JMhjY6hW+ALEwIH"
|
|
||||||
crossorigin="anonymous"/>
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
<nav class="navbar sticky-top bg-body-tertiary border-bottom border-secondary border-1">
|
|
||||||
<div class="container-fluid">
|
|
||||||
<a class="navbar-brand" href="/browser.php">Crawler</a>
|
|
||||||
<form class="d-flex w-50" role="search">
|
|
||||||
<input class="form-control me-2" type="search" name="search" placeholder="Search term eg. 36v or RCS18X" value="{{ search }}" aria-label="Search">
|
|
||||||
<button class="btn btn-outline-success" type="submit">Search</button>
|
|
||||||
</form>
|
|
||||||
</div>
|
|
||||||
</nav>
|
|
||||||
{% block content %}{% endblock %}
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
|
|
||||||
91
templates/product.html.twig
Normal file
91
templates/product.html.twig
Normal file
@@ -0,0 +1,91 @@
|
|||||||
|
{% extends "template.html.twig" %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<table class='table table-hover'>
|
||||||
|
<tr>
|
||||||
|
<td class="align-middle font-weight-bold h3"><a class="text-warning text-decoration-none"
|
||||||
|
href="{{ path('app_star', {'productId': product.id}) }}">{% if product.starred %}★{% else %} ☆ {% endif %}</a></td>
|
||||||
|
<td><img src='{{ product.image }}&width=150' class='border rounded p-1' alt='{{ product.name }}'/></td>
|
||||||
|
<td><a href='{{ path('app_home') }}/product/{{ product.id }}' class="text-decoration-none">{{ product.name }}</a></td>
|
||||||
|
<td>{{ product.subTitle }}</td>
|
||||||
|
<td>
|
||||||
|
<ul class='nav'>
|
||||||
|
{% for category in product.categories %}
|
||||||
|
<li class="nav-item"><a class="nav-link" href="{{ path('app_category', {'category': category}) }}"> {{ category }} </a></li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
</td>
|
||||||
|
<td><a href='https://pl.ryobitools.eu/{{ product.url }}'>link</a></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td colspan="4">
|
||||||
|
<div style="width: 800px;">
|
||||||
|
<canvas id="price"></canvas>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td colspan="4">
|
||||||
|
<table class='table table-hover table-sm mb-0'>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>price</th>
|
||||||
|
<th>lowest product price in 30 days</th>
|
||||||
|
<th colspan='2'>standard price</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
{% for price in product.price %}
|
||||||
|
<tr>
|
||||||
|
<td>{{ price.price }}</td>
|
||||||
|
<td>{{ price.lowestProductPrice30Days }}</td>
|
||||||
|
<td>{{ price.productStandardPrice }}</td>
|
||||||
|
<td>{{ price.created_at }}</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</table>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
</table>
|
||||||
|
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
const ctx = document.getElementById('price');
|
||||||
|
|
||||||
|
new Chart(ctx, {
|
||||||
|
type: 'line',
|
||||||
|
data: {
|
||||||
|
labels: ['{{ price_dates|raw }}'],
|
||||||
|
datasets: [{
|
||||||
|
label: 'Price (PLN)',
|
||||||
|
data: [{{ price_list }}],
|
||||||
|
borderWidth: 1
|
||||||
|
}]
|
||||||
|
},
|
||||||
|
options: {
|
||||||
|
animation: false,
|
||||||
|
plugins: {
|
||||||
|
title: {
|
||||||
|
display: true,
|
||||||
|
text: 'Price in time'
|
||||||
|
},
|
||||||
|
},
|
||||||
|
scales: {
|
||||||
|
x: {
|
||||||
|
title: {
|
||||||
|
display: true,
|
||||||
|
text: 'Date'
|
||||||
|
}
|
||||||
|
},
|
||||||
|
y: {
|
||||||
|
beginAtZero: true,
|
||||||
|
title: {
|
||||||
|
display: true,
|
||||||
|
text: 'Price'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
{% endblock %}
|
||||||
|
|
||||||
35
templates/productList.html.twig
Normal file
35
templates/productList.html.twig
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
{% extends "template.html.twig" %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<table class='table table-hover'>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th></th>
|
||||||
|
<th></th>
|
||||||
|
<th>Name</th>
|
||||||
|
<th>Code</th>
|
||||||
|
<th>Categories</th>
|
||||||
|
<th></th>
|
||||||
|
<th>Price</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
{% for product in products %}
|
||||||
|
<tr>
|
||||||
|
<td class="align-middle font-weight-bold h3"><a class="text-warning text-decoration-none" href="{{ path('app_star', {'productId': product.id}) }}">{% if product.starred == true %}★{% else %} ☆ {% endif %}</a></td>
|
||||||
|
<td class="align-middle" style="width: 120px;"><img src='{{ product.image }}&width=70' class='border rounded p-1' alt='{{ product.name }}'/></td>
|
||||||
|
<td class="align-middle"><a href='{{ path('app_product', {'productId': product.id}) }}' class="text-decoration-none">{{ product.name }}</a></td>
|
||||||
|
<td class="align-middle">{{ product.subTitle }}</td>
|
||||||
|
<td class="align-middle">
|
||||||
|
<ul class='nav'>
|
||||||
|
{% for category in product.categories %}
|
||||||
|
<li class="nav-item"><a class="nav-link" href="{{ path('app_category', {'category': category}) }}"> {{ category }} </a></li>
|
||||||
|
{% endfor %}
|
||||||
|
</ul>
|
||||||
|
</td>
|
||||||
|
<td class="align-middle"><a href='https://pl.ryobitools.eu/{{ product.url }}'>link</a></td>
|
||||||
|
<td class="align-middle">{{ product.price.last.price }}</td>
|
||||||
|
<td class="align-middle">{% if product.price.last.price != product.price.last.productStandardPrice %}<span class="text-decoration-line-through text-warning">{{ product.price.last.productStandardPrice }}</span> <span class="text-success">{{ ((1 - product.price.last.price / product.price.last.productStandardPrice)*100)|number_format(0) }}%</span>{% else %}{% endif %}</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</table>
|
||||||
|
{% endblock %}
|
||||||
36
templates/template.html.twig
Normal file
36
templates/template.html.twig
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
|
||||||
|
<meta http-equiv="X-UA-Compatible" content="ie=edge">
|
||||||
|
<title>Ryobi crawler</title>
|
||||||
|
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-QWTKZyjpPEjISv5WaRU9OFeRpok6YctnYmDr5pNlyT2bRjXh0JMhjY6hW+ALEwIH"
|
||||||
|
crossorigin="anonymous"/>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<nav class="navbar navbar-expand-lg sticky-top bg-body-tertiary border-bottom border-secondary border-1">
|
||||||
|
<div class="container-fluid">
|
||||||
|
<a class="navbar-brand" href="{{ path('app_home') }}">Crawler</a>
|
||||||
|
|
||||||
|
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarNav" aria-controls="navbarNav" aria-expanded="false" aria-label="Toggle navigation">
|
||||||
|
<span class="navbar-toggler-icon"></span>
|
||||||
|
</button>
|
||||||
|
<div class="collapse navbar-collapse" id="navbarNav">
|
||||||
|
<ul class="navbar-nav">
|
||||||
|
<li class="nav-item">
|
||||||
|
<a class="nav-link active" aria-current="page" href="{{ path('app_promos') }}">Promos</a>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<form class="d-flex w-50" role="search" action="{{ path('app_search') }}">
|
||||||
|
<input class="form-control me-2" type="search" name="search" placeholder="Search term eg. 36v or RCS18X" value="{{ search|default('') }}" aria-label="Search">
|
||||||
|
<button class="btn btn-outline-success" type="submit">Search</button>
|
||||||
|
</form>
|
||||||
|
</nav>
|
||||||
|
{% block content %}{% endblock %}
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
||||||
Reference in New Issue
Block a user