6 Commits

Author SHA1 Message Date
Krzysztof Płaczek
c093003fe7 Start working on tailwind instead of bootstrap 2025-05-20 16:31:46 +02:00
dc287aadc6 Merge pull request 'feature/lowest-price' (#27) from feature/lowest-price into master
Reviewed-on: #27
2025-05-14 12:46:12 +02:00
708a5eeae0 Merge pull request 'Implement a caching mechanism' (#28) from feature/cache into master
Reviewed-on: #28
2025-05-14 12:45:57 +02:00
Krzysztof Płaczek
7352eea270 Add the lowest price column to every product list view 2025-05-14 10:28:58 +02:00
Krzysztof Płaczek
e97d976705 Implement a caching mechanism 2025-05-14 09:03:25 +02:00
beb717f9b9 Add lowest price first draft. 2025-05-13 23:34:51 +02:00
17 changed files with 1468 additions and 19 deletions

1
.gitignore vendored
View File

@@ -3,3 +3,4 @@
*.sqlite
var/cache/
.env.local
/node_modules/

View File

@@ -7,7 +7,9 @@ RUN apt-get update && apt-get install -y \
git \
unzip \
libicu-dev \
libzip-dev
libzip-dev \
nodejs \
npm
RUN docker-php-ext-install zip intl

View File

@@ -4,7 +4,7 @@
1. Clone repository using `git clone https://git.techtube.pl/krzysiej/ryobi-crawler.git`
2. Cd into project directory `cd ryobi-crawler`
3. Build and start docker container `docker compose up -d`
3. Build and start docker container `docker compose up -d --build --force-recreate`
4. Run `docker compose exec php-app php console.php app:migrate` file to create `database.sqlite` and create tables.
5. Run `docker compose exec php-app php console.php app:scrape` command to scrape all the products from the ryobi website.
6. Access web interface using `localhost:9001` address in web browser.

1079
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

20
package.json Normal file
View File

@@ -0,0 +1,20 @@
{
"name": "ryobi-crawler",
"version": "1.0.0",
"description": "## Project start",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"dev": "npx @tailwindcss/cli -i ./templates/input/style.css -o ./templates/output/style.css --watch"
},
"repository": {
"type": "git",
"url": "https://git.techtube.pl/krzysiej/ryobi-crawler.git"
},
"author": "",
"license": "ISC",
"dependencies": {
"@tailwindcss/cli": "^4.1.7",
"tailwindcss": "^4.1.7"
}
}

View File

@@ -42,8 +42,8 @@ class CacheWarmCommand extends Command
$progress = new ProgressBar($output);
$progress->start();
$products = Product::with([
'price' => fn($query) => $query->orderBy('created_at', 'desc'),
'stock' => fn($query) => $query->orderBy('created_at', 'desc'),
'price' => fn($query) => $query->orderByDesc('created_at'),
'stock' => fn($query) => $query->orderByDesc('created_at'),
])->get();
$progress->setMaxSteps(count($products));

View File

@@ -15,7 +15,7 @@ final class CategoryController extends BaseController
return $this->render('productList.html.twig', ['listType' => 'category_'.$category]);
}
$products = Product::with('price')
$products = Product::with(['price', 'lowestPrice'])
->selectRaw('products.*')
->distinct('products.id')
->fromRaw('products, json_each(products.categories)')

View File

@@ -20,7 +20,7 @@ final class DiscontinuedController extends BaseController
$products = Product::where('updated_at', '<', now()->format('Y-m-d'))
->orderByDesc('starred')
->orderByDesc('created_by')
->with(['currentPrice'])
->with(['currentPrice', 'lowestPrice'])
->get();
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'discontinued']);
}

View File

@@ -14,7 +14,7 @@ final class IndexController extends BaseController
if ($this->cache->getItem('list_all')->isHit()) {
return $this->render('productList.html.twig', ['listType' => 'all']);
}
$products = Product::with(['currentStock', 'price'])
$products = Product::with(['currentStock', 'price', 'lowestPrice'])
->orderByDesc('starred')
->orderByDesc('created_by')
->get();

View File

@@ -21,7 +21,7 @@ final class NewController extends BaseController
$products = Product::where('created_at', '>', now()->modify('-30 days')->format('Y-m-d'))
->orderByDesc('starred')
->orderByDesc('created_by')
->with(['currentPrice'])
->with(['currentPrice', 'lowestPrice'])
->get();
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'new']);
}

View File

@@ -19,7 +19,7 @@ final class PromosController extends BaseController
$products = Product::whereHas('currentPrice', fn(Builder $query) => $query->whereColumn('price', '<', 'productStandardPrice'))
->orderByDesc('starred')
->orderByDesc('created_by')
->with(['currentPrice'])
->with(['currentPrice', 'lowestPrice'])
->get();
return $this->render('productList.html.twig', ['products' => $products, 'listType' => 'promos']);
}

View File

@@ -34,14 +34,12 @@ class Kernel extends BaseKernel
'secret' => 'S0ME_SECRET'
]);
$services = $container->services()->defaults()->autowire()->autoconfigure();
$services->load('Krzysiej\\RyobiCrawler\\Controller\\', __DIR__ . '/Controller/*');
$services->load('Krzysiej\\RyobiCrawler\\Command\\', __DIR__ . '/Command/*')->tag('console.command');
$services->load('Krzysiej\\RyobiCrawler\\', __DIR__ )
->exclude('../src/{Models,Twig,Kernel.php}');
$services->set('twig.extension.cache', AppExtension::class)->tag('twig.extension');
$services->set(CacheExtension::class)->tag('twig.extension');
$services->set(FilesystemAdapter::class)->args([
'', // namespace
0, // default lifetime
__DIR__ . '/../var/cache/twig_blocks' // custom path
'$directory' => __DIR__ . '/../var/cache/twig_blocks'
]);
$services->set('twig.runtime.cache', CacheRuntime::class)->args([new Reference(FilesystemAdapter::class)])->tag('twig.runtime');
}

View File

@@ -41,6 +41,11 @@ class Product extends Model
return $this->hasOne(Price::class)->latestOfMany('created_at');
}
public function lowestPrice(): HasOne
{
return $this->hasOne(Price::class)->ofMany('price', 'MIN');
}
public function stock(): HasMany
{
return $this->hasMany(Stock::class);

View File

@@ -0,0 +1 @@
@import "tailwindcss";

340
templates/output/style.css Normal file
View File

@@ -0,0 +1,340 @@
/*! tailwindcss v4.1.7 | MIT License | https://tailwindcss.com */
@layer properties;
@layer theme, base, components, utilities;
@layer theme {
:root, :host {
--font-sans: ui-sans-serif, system-ui, sans-serif, "Apple Color Emoji",
"Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
--font-mono: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono",
"Courier New", monospace;
--spacing: 0.25rem;
--text-3xl: 1.875rem;
--text-3xl--line-height: calc(2.25 / 1.875);
--font-weight-bold: 700;
--default-font-family: var(--font-sans);
--default-mono-font-family: var(--font-mono);
}
}
@layer base {
*, ::after, ::before, ::backdrop, ::file-selector-button {
box-sizing: border-box;
margin: 0;
padding: 0;
border: 0 solid;
}
html, :host {
line-height: 1.5;
-webkit-text-size-adjust: 100%;
tab-size: 4;
font-family: var(--default-font-family, ui-sans-serif, system-ui, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji");
font-feature-settings: var(--default-font-feature-settings, normal);
font-variation-settings: var(--default-font-variation-settings, normal);
-webkit-tap-highlight-color: transparent;
}
hr {
height: 0;
color: inherit;
border-top-width: 1px;
}
abbr:where([title]) {
-webkit-text-decoration: underline dotted;
text-decoration: underline dotted;
}
h1, h2, h3, h4, h5, h6 {
font-size: inherit;
font-weight: inherit;
}
a {
color: inherit;
-webkit-text-decoration: inherit;
text-decoration: inherit;
}
b, strong {
font-weight: bolder;
}
code, kbd, samp, pre {
font-family: var(--default-mono-font-family, ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace);
font-feature-settings: var(--default-mono-font-feature-settings, normal);
font-variation-settings: var(--default-mono-font-variation-settings, normal);
font-size: 1em;
}
small {
font-size: 80%;
}
sub, sup {
font-size: 75%;
line-height: 0;
position: relative;
vertical-align: baseline;
}
sub {
bottom: -0.25em;
}
sup {
top: -0.5em;
}
table {
text-indent: 0;
border-color: inherit;
border-collapse: collapse;
}
:-moz-focusring {
outline: auto;
}
progress {
vertical-align: baseline;
}
summary {
display: list-item;
}
ol, ul, menu {
list-style: none;
}
img, svg, video, canvas, audio, iframe, embed, object {
display: block;
vertical-align: middle;
}
img, video {
max-width: 100%;
height: auto;
}
button, input, select, optgroup, textarea, ::file-selector-button {
font: inherit;
font-feature-settings: inherit;
font-variation-settings: inherit;
letter-spacing: inherit;
color: inherit;
border-radius: 0;
background-color: transparent;
opacity: 1;
}
:where(select:is([multiple], [size])) optgroup {
font-weight: bolder;
}
:where(select:is([multiple], [size])) optgroup option {
padding-inline-start: 20px;
}
::file-selector-button {
margin-inline-end: 4px;
}
::placeholder {
opacity: 1;
}
@supports (not (-webkit-appearance: -apple-pay-button)) or (contain-intrinsic-size: 1px) {
::placeholder {
color: currentcolor;
@supports (color: color-mix(in lab, red, red)) {
color: color-mix(in oklab, currentcolor 50%, transparent);
}
}
}
textarea {
resize: vertical;
}
::-webkit-search-decoration {
-webkit-appearance: none;
}
::-webkit-date-and-time-value {
min-height: 1lh;
text-align: inherit;
}
::-webkit-datetime-edit {
display: inline-flex;
}
::-webkit-datetime-edit-fields-wrapper {
padding: 0;
}
::-webkit-datetime-edit, ::-webkit-datetime-edit-year-field, ::-webkit-datetime-edit-month-field, ::-webkit-datetime-edit-day-field, ::-webkit-datetime-edit-hour-field, ::-webkit-datetime-edit-minute-field, ::-webkit-datetime-edit-second-field, ::-webkit-datetime-edit-millisecond-field, ::-webkit-datetime-edit-meridiem-field {
padding-block: 0;
}
:-moz-ui-invalid {
box-shadow: none;
}
button, input:where([type="button"], [type="reset"], [type="submit"]), ::file-selector-button {
appearance: button;
}
::-webkit-inner-spin-button, ::-webkit-outer-spin-button {
height: auto;
}
[hidden]:where(:not([hidden="until-found"])) {
display: none !important;
}
}
@layer utilities {
.collapse {
visibility: collapse;
}
.container {
width: 100%;
@media (width >= 40rem) {
max-width: 40rem;
}
@media (width >= 48rem) {
max-width: 48rem;
}
@media (width >= 64rem) {
max-width: 64rem;
}
@media (width >= 80rem) {
max-width: 80rem;
}
@media (width >= 96rem) {
max-width: 96rem;
}
}
.me-2 {
margin-inline-end: calc(var(--spacing) * 2);
}
.me-auto {
margin-inline-end: auto;
}
.mb-0 {
margin-bottom: calc(var(--spacing) * 0);
}
.mb-2 {
margin-bottom: calc(var(--spacing) * 2);
}
.block {
display: block;
}
.grid {
display: grid;
}
.table {
display: table;
}
.flex-row {
flex-direction: row;
}
.rounded {
border-radius: 0.25rem;
}
.border {
border-style: var(--tw-border-style);
border-width: 1px;
}
.border-1 {
border-style: var(--tw-border-style);
border-width: 1px;
}
.p-1 {
padding: calc(var(--spacing) * 1);
}
.text-end {
text-align: end;
}
.align-middle {
vertical-align: middle;
}
.text-3xl {
font-size: var(--text-3xl);
line-height: var(--tw-leading, var(--text-3xl--line-height));
}
.font-bold {
--tw-font-weight: var(--font-weight-bold);
font-weight: var(--font-weight-bold);
}
.underline {
text-decoration-line: underline;
}
.shadow-sm {
--tw-shadow: 0 1px 3px 0 var(--tw-shadow-color, rgb(0 0 0 / 0.1)), 0 1px 2px -1px var(--tw-shadow-color, rgb(0 0 0 / 0.1));
box-shadow: var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);
}
}
@property --tw-border-style {
syntax: "*";
inherits: false;
initial-value: solid;
}
@property --tw-font-weight {
syntax: "*";
inherits: false;
}
@property --tw-shadow {
syntax: "*";
inherits: false;
initial-value: 0 0 #0000;
}
@property --tw-shadow-color {
syntax: "*";
inherits: false;
}
@property --tw-shadow-alpha {
syntax: "<percentage>";
inherits: false;
initial-value: 100%;
}
@property --tw-inset-shadow {
syntax: "*";
inherits: false;
initial-value: 0 0 #0000;
}
@property --tw-inset-shadow-color {
syntax: "*";
inherits: false;
}
@property --tw-inset-shadow-alpha {
syntax: "<percentage>";
inherits: false;
initial-value: 100%;
}
@property --tw-ring-color {
syntax: "*";
inherits: false;
}
@property --tw-ring-shadow {
syntax: "*";
inherits: false;
initial-value: 0 0 #0000;
}
@property --tw-inset-ring-color {
syntax: "*";
inherits: false;
}
@property --tw-inset-ring-shadow {
syntax: "*";
inherits: false;
initial-value: 0 0 #0000;
}
@property --tw-ring-inset {
syntax: "*";
inherits: false;
}
@property --tw-ring-offset-width {
syntax: "<length>";
inherits: false;
initial-value: 0px;
}
@property --tw-ring-offset-color {
syntax: "*";
inherits: false;
initial-value: #fff;
}
@property --tw-ring-offset-shadow {
syntax: "*";
inherits: false;
initial-value: 0 0 #0000;
}
@layer properties {
@supports ((-webkit-hyphens: none) and (not (margin-trim: inline))) or ((-moz-orient: inline) and (not (color:rgb(from red r g b)))) {
*, ::before, ::after, ::backdrop {
--tw-border-style: solid;
--tw-font-weight: initial;
--tw-shadow: 0 0 #0000;
--tw-shadow-color: initial;
--tw-shadow-alpha: 100%;
--tw-inset-shadow: 0 0 #0000;
--tw-inset-shadow-color: initial;
--tw-inset-shadow-alpha: 100%;
--tw-ring-color: initial;
--tw-ring-shadow: 0 0 #0000;
--tw-inset-ring-color: initial;
--tw-inset-ring-shadow: 0 0 #0000;
--tw-ring-inset: initial;
--tw-ring-offset-width: 0px;
--tw-ring-offset-color: #fff;
--tw-ring-offset-shadow: 0 0 #0000;
}
}
}

View File

@@ -11,7 +11,8 @@
<th>Name</th>
<th>Categories</th>
<th></th>
<th>Price</th>
<th class="text-end">Lowest Price</th>
<th class="text-end">Current Price</th>
<th></th>
</tr>
</thead>
@@ -45,7 +46,8 @@
</nav>
</td>
<td class="align-middle"><a href='https://pl.ryobitools.eu/{{ product.url }}'>link</a></td>
<td class="align-middle">{{ product.price.last.price | format_currency('PLN', {}, 'pl') }}</td>
<td class="align-middle text-end">{% if product.lowestPrice.price != product.price.last.price %}{{ product.lowestPrice.price | format_currency('PLN', {}, 'pl') }}{% endif %}</td>
<td class="align-middle text-end">{{ product.price.last.price | format_currency('PLN', {}, 'pl') }}</td>
<td class="align-middle">
<div class="d-flex flex-row">
{% if product.price.last.price != product.price.last.productStandardPrice %}<span

View File

@@ -5,10 +5,11 @@
<meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<title>Ryobi crawler</title>
<link href="/templates/css/bootstrap.min.css" rel="stylesheet" />
<link href="/templates/output/style.css" rel="stylesheet" />
<script src="/templates/js/script.js" defer></script>
</head>
<body>
<h1 class="text-3xl font-bold underline">test header</h1>
<nav class="navbar navbar-expand-lg sticky-top bg-body-tertiary border-bottom border-secondary border-1">
<div class="container-fluid">
<a class="navbar-brand" href="{{ path('app_home') }}">Crawler</a>