Compare commits
15 Commits
1.2.0
...
feature/fl
| Author | SHA1 | Date | |
|---|---|---|---|
| 5fbd555bb3 | |||
| de4915972c | |||
| 9c2405dd3f | |||
| da6a8f86c2 | |||
| 2c40fb0e61 | |||
| 5314a6a70a | |||
|
|
f772532309 | ||
|
|
17159e811f | ||
|
|
76d8b7d9cf | ||
|
|
f30304cbe9 | ||
| dc287aadc6 | |||
| 708a5eeae0 | |||
|
|
7352eea270 | ||
|
|
e97d976705 | ||
| beb717f9b9 |
@@ -4,7 +4,7 @@
|
|||||||
|
|
||||||
1. Clone repository using `git clone https://git.techtube.pl/krzysiej/ryobi-crawler.git`
|
1. Clone repository using `git clone https://git.techtube.pl/krzysiej/ryobi-crawler.git`
|
||||||
2. Cd into project directory `cd ryobi-crawler`
|
2. Cd into project directory `cd ryobi-crawler`
|
||||||
3. Build and start docker container `docker compose up -d`
|
3. Build and start docker container `docker compose up -d --build --force-recreate`
|
||||||
4. Run `docker compose exec php-app php console.php app:migrate` file to create `database.sqlite` and create tables.
|
4. Run `docker compose exec php-app php console.php app:migrate` file to create `database.sqlite` and create tables.
|
||||||
5. Run `docker compose exec php-app php console.php app:scrape` command to scrape all the products from the ryobi website.
|
5. Run `docker compose exec php-app php console.php app:scrape` command to scrape all the products from the ryobi website.
|
||||||
6. Access web interface using `localhost:9001` address in web browser.
|
6. Access web interface using `localhost:9001` address in web browser.
|
||||||
|
|||||||
2
bin/cachewarmup
Executable file
2
bin/cachewarmup
Executable file
@@ -0,0 +1,2 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
bin/cli php console.php app:cache:warm-twig
|
||||||
@@ -1,3 +1,3 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
[ -z "$1" ] && echo "Please specify a Composer command (ex. install)" && exit
|
[ -z "$1" ] && bin/cli composer list && exit
|
||||||
bin/cli composer "$@"
|
bin/cli composer "$@"
|
||||||
1151
composer.lock
generated
1151
composer.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -16,7 +16,7 @@ use Symfony\Component\Console\Output\OutputInterface;
|
|||||||
use Symfony\Component\HttpFoundation\RequestStack;
|
use Symfony\Component\HttpFoundation\RequestStack;
|
||||||
use Twig\Environment;
|
use Twig\Environment;
|
||||||
|
|
||||||
#[AsCommand(name: 'app:cache:warm-twig', description: '')]
|
#[AsCommand(name: 'app:cache:warm-twig', description: 'Warmup twig cache')]
|
||||||
class CacheWarmCommand extends Command
|
class CacheWarmCommand extends Command
|
||||||
{
|
{
|
||||||
public function __construct(
|
public function __construct(
|
||||||
@@ -42,8 +42,8 @@ class CacheWarmCommand extends Command
|
|||||||
$progress = new ProgressBar($output);
|
$progress = new ProgressBar($output);
|
||||||
$progress->start();
|
$progress->start();
|
||||||
$products = Product::with([
|
$products = Product::with([
|
||||||
'price' => fn($query) => $query->orderBy('created_at', 'desc'),
|
'price' => fn($query) => $query->orderByDesc('created_at'),
|
||||||
'stock' => fn($query) => $query->orderBy('created_at', 'desc'),
|
'stock' => fn($query) => $query->orderByDesc('created_at'),
|
||||||
])->get();
|
])->get();
|
||||||
$progress->setMaxSteps(count($products));
|
$progress->setMaxSteps(count($products));
|
||||||
|
|
||||||
|
|||||||
@@ -21,15 +21,13 @@ class ScrapeWebsite extends Command
|
|||||||
{
|
{
|
||||||
private Client $client;
|
private Client $client;
|
||||||
|
|
||||||
|
public function __construct(protected Capsule $database)
|
||||||
|
{
|
||||||
|
parent::__construct();
|
||||||
|
}
|
||||||
|
|
||||||
protected function configure(): void
|
protected function configure(): void
|
||||||
{
|
{
|
||||||
$capsule = new Capsule;
|
|
||||||
$capsule->addConnection([
|
|
||||||
'driver' => 'sqlite',
|
|
||||||
'database' => __DIR__ . '/../../database.sqlite',
|
|
||||||
]);
|
|
||||||
$capsule->setAsGlobal();
|
|
||||||
$capsule->bootEloquent();
|
|
||||||
$this->client = new Client();
|
$this->client = new Client();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -11,11 +11,7 @@ class BaseController extends AbstractController
|
|||||||
{
|
{
|
||||||
protected Environment $twig;
|
protected Environment $twig;
|
||||||
|
|
||||||
public function __construct(protected FilesystemAdapter $cache)
|
public function __construct(protected FilesystemAdapter $cache, protected Capsule $database)
|
||||||
{
|
{
|
||||||
$capsule = new Capsule;
|
|
||||||
$capsule->addConnection(['driver' => 'sqlite', 'database' => __DIR__ . '/../../database.sqlite']);
|
|
||||||
$capsule->setAsGlobal();
|
|
||||||
$capsule->bootEloquent();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ final class CategoryController extends BaseController
|
|||||||
return $this->render('productList.html.twig', ['listType' => 'category_'.$category]);
|
return $this->render('productList.html.twig', ['listType' => 'category_'.$category]);
|
||||||
}
|
}
|
||||||
|
|
||||||
$products = Product::with('price')
|
$products = Product::with(['price', 'lowestPrice'])
|
||||||
->selectRaw('products.*')
|
->selectRaw('products.*')
|
||||||
->distinct('products.id')
|
->distinct('products.id')
|
||||||
->fromRaw('products, json_each(products.categories)')
|
->fromRaw('products, json_each(products.categories)')
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ final class DiscontinuedController extends BaseController
|
|||||||
$products = Product::where('updated_at', '<', now()->format('Y-m-d'))
|
$products = Product::where('updated_at', '<', now()->format('Y-m-d'))
|
||||||
->orderByDesc('starred')
|
->orderByDesc('starred')
|
||||||
->orderByDesc('created_by')
|
->orderByDesc('created_by')
|
||||||
->with(['currentPrice'])
|
->with(['currentPrice', 'lowestPrice'])
|
||||||
->get();
|
->get();
|
||||||
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'discontinued']);
|
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'discontinued']);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ final class IndexController extends BaseController
|
|||||||
if ($this->cache->getItem('list_all')->isHit()) {
|
if ($this->cache->getItem('list_all')->isHit()) {
|
||||||
return $this->render('productList.html.twig', ['listType' => 'all']);
|
return $this->render('productList.html.twig', ['listType' => 'all']);
|
||||||
}
|
}
|
||||||
$products = Product::with(['currentStock', 'price'])
|
$products = Product::with(['currentStock', 'price', 'lowestPrice'])
|
||||||
->orderByDesc('starred')
|
->orderByDesc('starred')
|
||||||
->orderByDesc('created_by')
|
->orderByDesc('created_by')
|
||||||
->get();
|
->get();
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ final class NewController extends BaseController
|
|||||||
$products = Product::where('created_at', '>', now()->modify('-30 days')->format('Y-m-d'))
|
$products = Product::where('created_at', '>', now()->modify('-30 days')->format('Y-m-d'))
|
||||||
->orderByDesc('starred')
|
->orderByDesc('starred')
|
||||||
->orderByDesc('created_by')
|
->orderByDesc('created_by')
|
||||||
->with(['currentPrice'])
|
->with(['currentPrice', 'lowestPrice'])
|
||||||
->get();
|
->get();
|
||||||
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'new']);
|
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'new']);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ final class PromosController extends BaseController
|
|||||||
$products = Product::whereHas('currentPrice', fn(Builder $query) => $query->whereColumn('price', '<', 'productStandardPrice'))
|
$products = Product::whereHas('currentPrice', fn(Builder $query) => $query->whereColumn('price', '<', 'productStandardPrice'))
|
||||||
->orderByDesc('starred')
|
->orderByDesc('starred')
|
||||||
->orderByDesc('created_by')
|
->orderByDesc('created_by')
|
||||||
->with(['currentPrice'])
|
->with(['currentPrice', 'lowestPrice'])
|
||||||
->get();
|
->get();
|
||||||
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'promos']);
|
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'promos']);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,6 @@ final class SearchController extends BaseController
|
|||||||
->orWhere([['name', 'like', "%$search%"]])
|
->orWhere([['name', 'like', "%$search%"]])
|
||||||
->orWhere([['subTitle', 'like', "%$search%"]])->get();
|
->orWhere([['subTitle', 'like', "%$search%"]])->get();
|
||||||
|
|
||||||
return $this->render('productList.html.twig', ['products' => $products ?? [], 'search' => $search]);
|
return $this->render('productList.html.twig', ['products' => $products ?? [], 'search' => $search, 'listType' => 'search_'.$search]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
14
src/DatabaseFactory.php
Normal file
14
src/DatabaseFactory.php
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace Krzysiej\RyobiCrawler;
|
||||||
|
|
||||||
|
use Illuminate\Database\Capsule\Manager as Capsule;
|
||||||
|
class DatabaseFactory
|
||||||
|
{
|
||||||
|
public static function create(Capsule $capsule): void
|
||||||
|
{
|
||||||
|
$capsule->addConnection(['driver' => 'sqlite', 'database' => __DIR__ . '/../database.sqlite']);
|
||||||
|
$capsule->setAsGlobal();
|
||||||
|
$capsule->bootEloquent();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
namespace Krzysiej\RyobiCrawler;
|
namespace Krzysiej\RyobiCrawler;
|
||||||
|
|
||||||
|
use Illuminate\Database\Capsule\Manager;
|
||||||
use Krzysiej\RyobiCrawler\Twig\AppExtension;
|
use Krzysiej\RyobiCrawler\Twig\AppExtension;
|
||||||
use Symfony\Bundle\FrameworkBundle\FrameworkBundle;
|
use Symfony\Bundle\FrameworkBundle\FrameworkBundle;
|
||||||
use Symfony\Bundle\FrameworkBundle\Kernel\MicroKernelTrait;
|
use Symfony\Bundle\FrameworkBundle\Kernel\MicroKernelTrait;
|
||||||
@@ -34,14 +35,13 @@ class Kernel extends BaseKernel
|
|||||||
'secret' => 'S0ME_SECRET'
|
'secret' => 'S0ME_SECRET'
|
||||||
]);
|
]);
|
||||||
$services = $container->services()->defaults()->autowire()->autoconfigure();
|
$services = $container->services()->defaults()->autowire()->autoconfigure();
|
||||||
$services->load('Krzysiej\\RyobiCrawler\\Controller\\', __DIR__ . '/Controller/*');
|
$services->set(Manager::class)->configurator([DatabaseFactory::class, 'create']);
|
||||||
$services->load('Krzysiej\\RyobiCrawler\\Command\\', __DIR__ . '/Command/*')->tag('console.command');
|
$services->load('Krzysiej\\RyobiCrawler\\', __DIR__ )
|
||||||
|
->exclude('../src/{Models,Twig,DatabaseFactory.php,Kernel.php}');
|
||||||
$services->set('twig.extension.cache', AppExtension::class)->tag('twig.extension');
|
$services->set('twig.extension.cache', AppExtension::class)->tag('twig.extension');
|
||||||
$services->set(CacheExtension::class)->tag('twig.extension');
|
$services->set(CacheExtension::class)->tag('twig.extension');
|
||||||
$services->set(FilesystemAdapter::class)->args([
|
$services->set(FilesystemAdapter::class)->args([
|
||||||
'', // namespace
|
'$directory' => __DIR__ . '/../var/cache/twig_blocks'
|
||||||
0, // default lifetime
|
|
||||||
__DIR__ . '/../var/cache/twig_blocks' // custom path
|
|
||||||
]);
|
]);
|
||||||
$services->set('twig.runtime.cache', CacheRuntime::class)->args([new Reference(FilesystemAdapter::class)])->tag('twig.runtime');
|
$services->set('twig.runtime.cache', CacheRuntime::class)->args([new Reference(FilesystemAdapter::class)])->tag('twig.runtime');
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -41,6 +41,11 @@ class Product extends Model
|
|||||||
return $this->hasOne(Price::class)->latestOfMany('created_at');
|
return $this->hasOne(Price::class)->latestOfMany('created_at');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public function lowestPrice(): HasOne
|
||||||
|
{
|
||||||
|
return $this->hasOne(Price::class)->ofMany('price', 'MIN');
|
||||||
|
}
|
||||||
|
|
||||||
public function stock(): HasMany
|
public function stock(): HasMany
|
||||||
{
|
{
|
||||||
return $this->hasMany(Stock::class);
|
return $this->hasMany(Stock::class);
|
||||||
|
|||||||
@@ -3,20 +3,22 @@
|
|||||||
{% block content %}
|
{% block content %}
|
||||||
{% cache 'list_' ~ listType %}
|
{% cache 'list_' ~ listType %}
|
||||||
<div class="table-responsive">
|
<div class="table-responsive">
|
||||||
<table class='table table-hover'>
|
{# <table class='table table-hover'>#}
|
||||||
<thead>
|
{# <thead>#}
|
||||||
<tr>
|
{# <tr>#}
|
||||||
<th></th>
|
{# <th></th>#}
|
||||||
<th></th>
|
{# <th></th>#}
|
||||||
<th>Name</th>
|
{# <th>Name</th>#}
|
||||||
<th>Categories</th>
|
{# <th>Categories</th>#}
|
||||||
<th></th>
|
{# <th></th>#}
|
||||||
<th>Price</th>
|
{# <th class="text-end">Lowest Price</th>#}
|
||||||
<th></th>
|
{# <th class="text-end">Current Price</th>#}
|
||||||
</tr>
|
{# <th></th>#}
|
||||||
</thead>
|
{# </tr>#}
|
||||||
|
{# </thead>#}
|
||||||
{% for product in products %}
|
{% for product in products %}
|
||||||
<tr>
|
<div class="d-inline-flex">
|
||||||
|
|
||||||
<td class="align-middle font-weight-bold h3"><a class="text-warning text-decoration-none"
|
<td class="align-middle font-weight-bold h3"><a class="text-warning text-decoration-none"
|
||||||
href="{{ path('app_star', {'productId': product.id}) }}">{% if product.starred == true %}★{% else %} ☆ {% endif %}</a></td>
|
href="{{ path('app_star', {'productId': product.id}) }}">{% if product.starred == true %}★{% else %} ☆ {% endif %}</a></td>
|
||||||
<td class="align-middle" style="width: 120px;"><img src='{{ product.image }}&width=70' class='img-thumbnail' alt='{{ product.name }}'/></td>
|
<td class="align-middle" style="width: 120px;"><img src='{{ product.image }}&width=70' class='img-thumbnail' alt='{{ product.name }}'/></td>
|
||||||
@@ -37,7 +39,7 @@
|
|||||||
</td>
|
</td>
|
||||||
<td class="align-middle">
|
<td class="align-middle">
|
||||||
<nav aria-label="breadcrumb" style="--bs-breadcrumb-divider: '>';" >
|
<nav aria-label="breadcrumb" style="--bs-breadcrumb-divider: '>';" >
|
||||||
<ol class="breadcrumb">
|
<ol class="breadcrumb mb-0">
|
||||||
{% for category in product.categories %}
|
{% for category in product.categories %}
|
||||||
<li class="breadcrumb-item" aria-current="page"><a class="breadcrumb-item text-decoration-none" href="{{ path('app_category', {'category': category}) }}">{{ category }}</a></li>
|
<li class="breadcrumb-item" aria-current="page"><a class="breadcrumb-item text-decoration-none" href="{{ path('app_category', {'category': category}) }}">{{ category }}</a></li>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
@@ -45,18 +47,19 @@
|
|||||||
</nav>
|
</nav>
|
||||||
</td>
|
</td>
|
||||||
<td class="align-middle"><a href='https://pl.ryobitools.eu/{{ product.url }}'>link</a></td>
|
<td class="align-middle"><a href='https://pl.ryobitools.eu/{{ product.url }}'>link</a></td>
|
||||||
<td class="align-middle">{{ product.price.last.price | format_currency('PLN', {}, 'pl') }}</td>
|
<td class="align-middle text-end">{% if product.lowestPrice.price != product.price.last.price %}{{ product.lowestPrice.price | format_currency('PLN', {}, 'pl') }}{% endif %}</td>
|
||||||
|
<td class="align-middle text-end">{{ product.price.last.price | format_currency('PLN', {}, 'pl') }}</td>
|
||||||
<td class="align-middle">
|
<td class="align-middle">
|
||||||
<div class="d-flex flex-row">
|
<div class="d-flex flex-row">
|
||||||
{% if product.price.last.price != product.price.last.productStandardPrice %}<span
|
{% if product.price.last.price != product.price.last.productStandardPrice %}<span
|
||||||
class="badge text-bg-warning text-decoration-line-through flex-fill">{{ product.price.last.productStandardPrice | format_currency('PLN', {}, 'pl') }}</span> <span
|
class="badge text-bg-warning text-decoration-line-through flex-fill">{{ product.price.last.productStandardPrice | format_currency('PLN', {}, 'pl') }}</span> <span
|
||||||
class="badge text-bg-success flex-fill">{{ ((1 - product.price.last.price / product.price.last.productStandardPrice)*100)|number_format(0) }}%</span>
|
class="badge text-bg-success flex-fill">{{ ((1 - product.price.last.price / product.price.last.productStandardPrice)*100)|number_format(0) }}%</span>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</div>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</table>
|
{# </table>#}
|
||||||
</div>
|
</div>
|
||||||
{% endcache %}
|
{% endcache %}
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|||||||
Reference in New Issue
Block a user