Update packages. Add docker support, move migration to command, add stocks as a new table.

Dockerfile

@@ -0,0 +1,19 @@
+FROM php:8.3-cli
+
+WORKDIR /usr/src/app
+
+RUN apt-get update && apt-get install -y \
+    git \
+    unzip \
+    libzip-dev
+
+RUN docker-php-ext-install zip
+
+RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer
+
+COPY composer.json composer.lock ./
+RUN composer install --no-dev --optimize-autoloader
+
+COPY . .
+
+CMD ["php", "browser.php"]

README.md

@@ -1,2 +1,6 @@
 # ryobi-crawler
 
+## Project start
+
+1. Run `migration.php` file to create `database.sqlite` and create tables. 
+2. Run `php index.php app:scrape` command to scrape all the products from the ryobi website.

browser.php

@@ -35,4 +35,4 @@ if (empty($_GET)) {
     $template = 'productList.html.twig';
     $products = Product::with('price')->orderByDesc('starred')->orderByDesc('created_by')->get();
 }
-$twig->display($template, ['products' => $products, 'product' => $product, 'search' => $_GET['search']]);
+$twig->display($template, ['products' => $products ?? [], 'product' => $product ?? [], 'search' => $_GET['search'] ?? '']);

composer.json

@@ -2,9 +2,9 @@
     "require": {
         "guzzlehttp/guzzle": "^7.0",
         "symfony/var-dumper": "^7.0",
-        "illuminate/database": "^11.0",
+        "illuminate/database": "11.26.0.0",
         "ext-json": "*",
-        "twig/twig": "^3.0",
+        "twig/twig": "3.14.0.0",
         "symfony/console": "^7.0"
     },
     "autoload": {

docker-compose.yml

@@ -0,0 +1,9 @@
+services:
+  php-app:
+    build: .
+    volumes:
+      - .:/usr/src/app
+    command: >
+      sh -c "composer install && php -S 0.0.0.0:8000 -t /usr/src/app"
+    ports:
+      - "9000:8000"

index.php

@@ -2,7 +2,7 @@
 
 include_once 'vendor/autoload.php';
 
-
+use Krzysiej\RyobiCrawler\Command\Migrate;
 use Krzysiej\RyobiCrawler\Command\ScrapeWebsite;
 
 use Symfony\Component\Console\Application;
@@ -13,6 +13,7 @@ if (php_sapi_name() !== 'cli') {
     exit;
 }
 
-$application = new Application('Ryobi website scraper application', '1.0.0');
+$application = new Application('Ryobi website scraper application', '1.1.0');
 $application->add(new ScrapeWebsite());
+$application->add(new Migrate());
 $application->run();

src/Command/Migrate.php

@@ -0,0 +1,91 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Krzysiej\RyobiCrawler\Command;
+
+use Illuminate\Database\Capsule\Manager as Capsule;
+use Illuminate\Database\Schema\Blueprint;
+use Symfony\Component\Console\Attribute\AsCommand;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+#[AsCommand(name: 'app:migrate', description: 'Create database and rum migrations')]
+class Migrate extends Command
+{
+    protected function configure(): void
+    {
+        touch(__DIR__ . '/../../database.sqlite');
+        $capsule = new Capsule;
+        $capsule->addConnection([
+            'driver' => 'sqlite',
+            'database' => __DIR__ . '/../../database.sqlite',
+        ]);
+        $capsule->setAsGlobal();
+        $capsule->bootEloquent();
+    }
+
+    public function execute(InputInterface $input, OutputInterface $output): int
+    {
+        $this->createProductsTable();
+        $this->createPricesTable();
+        $this->createStocksTable();
+
+        return Command::SUCCESS;
+    }
+
+    public function createProductsTable(): void
+    {
+        if (!Capsule::schema()->hasTable('products')) {
+            Capsule::schema()->create('products', function (Blueprint $table) {
+                $table->increments('id');
+                $table->string('name');
+                $table->integer('skuID')->unique();
+                $table->integer('agilityID');
+                $table->integer('availableQuantity');
+                $table->integer('stock');
+                $table->json('categories');
+                $table->string('image');
+                $table->string('subTitle');
+                $table->string('variantCode');
+                $table->string('modelCode');
+                $table->string('url');
+                $table->boolean('starred')->default(false);
+                $table->timestamps();
+            });
+        }
+    }
+
+    public function createPricesTable(): void
+    {
+        if (!Capsule::schema()->hasTable('prices')) {
+            Capsule::schema()->create('prices', function (Blueprint $table) {
+                $table->increments('id');
+                $table->foreignId('product_id');
+                $table->float('price');
+                $table->float('productStandardPrice');
+                $table->float('lowestProductPrice30Days');
+                $table->timestamps();
+            });
+        }
+    }
+
+    public function createStocksTable(): void
+    {
+        if (!Capsule::schema()->hasTable('stocks')) {
+            Capsule::schema()->create('stocks', function (Blueprint $table) {
+                $table->increments('id');
+                $table->foreignId('product_id');
+                $table->integer('availableQuantity');
+                $table->integer('stock');
+                $table->boolean('isInStock');
+                $table->boolean('previouslyHadStock');
+                $table->boolean('isNew');
+                $table->boolean('isComingSoon');
+                $table->boolean('isOnSale');
+                $table->timestamps();
+            });
+        }
+    }
+}

src/Command/ScrapeWebsite.php

@@ -5,21 +5,23 @@ declare(strict_types=1);
 namespace Krzysiej\RyobiCrawler\Command;
 
 use GuzzleHttp\Client;
+use GuzzleHttp\Exception\GuzzleException;
 use Illuminate\Database\Capsule\Manager as Capsule;
 use Krzysiej\RyobiCrawler\Models\Price;
 use Krzysiej\RyobiCrawler\Models\Product;
+use Krzysiej\RyobiCrawler\Models\Stock;
 use Symfony\Component\Console\Attribute\AsCommand;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Helper\ProgressBar;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 
-#[AsCommand(name: 'app:scrape')]
+#[AsCommand(name: 'app:scrape', description: 'Scrape all products from Ryobi website')]
 class ScrapeWebsite extends Command
 {
     private Client $client;
 
-    protected function configure()
+    protected function configure(): void
     {
         $capsule = new Capsule;
         $capsule->addConnection([
@@ -44,6 +46,7 @@ class ScrapeWebsite extends Command
         $progress->finish();
         $output->writeln('');
         $output->writeln('DONE');
+
         return Command::SUCCESS;
     }
 
@@ -52,22 +55,28 @@ class ScrapeWebsite extends Command
         $products = [];
         $page = 0;
         do {
-            $res = $this->client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [
+            try {
-                'form_params' => [
+                $res = $this->client->request('POST', 'https://pl.ryobitools.eu/api/product-listing/get-products', [
-                    "includePreviousPages" => false,
+                    'form_params' => [
-                    "pageIndex" => $page,
+                        "includePreviousPages" => false,
-                    "pageSize" => 100,
+                        "pageIndex" => $page,
-                    "cultureCode" => "pl-PL",
+                        "pageSize" => 100,
-                ]
+                        "cultureCode" => "pl-PL",
-            ]);
+                    ]
-            $page++;
+                ]);
-            $responseObject = json_decode($res->getBody()->getContents());
+                $responseObject = json_decode($res->getBody()->getContents());
-            $products = array_merge($products, $responseObject->products);
+                $products = array_merge($products, $responseObject->products);
-        } while ((bool)$responseObject->canLoadMore);
+                $page++;
+                $canLoadMore = $responseObject->canLoadMore;
+            } catch (GuzzleException $e) {
+                return $products;
+            }
+        } while ($canLoadMore);
+
         return $products;
     }
 
-    private function saveProduct(\stdClass $product)
+    private function saveProduct(\stdClass $product): void
     {
         /** @var Product $productModel */
         $productModel = Product::firstOrNew(['skuID' => $product->skuID]);
@@ -90,5 +99,17 @@ class ScrapeWebsite extends Command
             $price->lowestProductPrice30Days = $product->lowestProductPrice30Days;
             $productModel->price()->save($price);
         }
+        $stockExist = $productModel->stock()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();
+        if (!$stockExist) {
+            $stock = new Stock();
+            $stock->availableQuantity = $product->availableQuantity;
+            $stock->stock = $product->stock;
+            $stock->isInStock = $product->isInStock;
+            $stock->previouslyHadStock = $product->previouslyHadStock;
+            $stock->isNew = $product->isNew;
+            $stock->isComingSoon = $product->isComingSoon;
+            $stock->isOnSale = $product->isOnSale;
+            $productModel->stock()->save($stock);
+        }
     }
-}
+}

src/Models/Product.php

@@ -29,6 +29,10 @@ class Product extends Model
     {
         return $this->hasMany(Price::class);
     }
+    public function stock(): HasMany
+    {
+        return $this->hasMany(Stock::class);
+    }
 
     public function toggleStarred(): self
     {
@@ -42,5 +46,4 @@ class Product extends Model
             get: fn(string $value) => array_reverse(json_decode($value, 1)),
         );
     }
-
+}
-}

src/Models/Stock.php

@@ -0,0 +1,25 @@
+<?php
+
+namespace Krzysiej\RyobiCrawler\Models;
+
+use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Relations\BelongsTo;
+
+/**
+ * @property integer $availableQuantity
+ * @property integer $stock
+ * @property boolean $isInStock
+ * @property boolean $previouslyHadStock
+ * @property boolean $isNew
+ * @property boolean $isComingSoon
+ * @property boolean $isOnSale
+ */
+class Stock extends Model
+{
+    public $timestamps = true;
+
+    public function product(): BelongsTo
+    {
+        return $this->belongsTo(Product::class);
+    }
+}

src/templates/template.html.twig

@@ -9,11 +9,11 @@
           crossorigin="anonymous"/>
 </head>
 <body>
-<nav class="navbar sticky-top bg-body-tertiary">
+<nav class="navbar sticky-top bg-body-tertiary border-bottom border-secondary border-1">
     <div class="container-fluid">
         <a class="navbar-brand" href="/browser.php">Crawler</a>
-        <form class="d-flex" role="search">
+        <form class="d-flex w-50" role="search">
-            <input class="form-control me-2" type="search" name="search" placeholder="Search" value="{{ search }}" aria-label="Search">
+            <input class="form-control me-2" type="search" name="search" placeholder="Search term eg. 36v or RCS18X" value="{{ search }}" aria-label="Search">
             <button class="btn btn-outline-success" type="submit">Search</button>
         </form>
     </div>