ryobi-crawler/src/Command/ScrapeWebsite.php

<?php

declare(strict_types=1);

namespace Krzysiej\RyobiCrawler\Command;

use GuzzleHttp\Client;
use GuzzleHttp\Exception\GuzzleException;
use Illuminate\Database\Capsule\Manager as Capsule;
use Krzysiej\RyobiCrawler\Models\Country;
use Krzysiej\RyobiCrawler\Models\Price;
use Krzysiej\RyobiCrawler\Models\Product;
use Krzysiej\RyobiCrawler\Models\Stock;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;

#[AsCommand(name: 'app:scrape', description: 'Scrape all products from Ryobi website')]
class ScrapeWebsite extends Command
{
    private Client $client;

    public function __construct(protected Capsule $database)
    {
        parent::__construct();
    }

    protected function configure(): void
    {
        $this->client = new Client();
    }

    public function execute(InputInterface $input, OutputInterface $output): int
    {
        $output->writeln('Scrape products');
        $progress = new ProgressBar($output);
        $countries = Country::all();
        foreach($countries as $country) {
            $output->writeln('Country name: ' . $country->countryName);
            $progress->start();
            $products = $this->getProducts($country);
            $progress->setMaxSteps(count($products));
            foreach ($products as $product) {
                $this->saveProduct($product, $country);
                $progress->advance();
            }
            $progress->finish();
            $output->writeln('');
            $output->writeln('Scrape products - DONE');
            $output->writeln('');
        }

        $output->writeln('Update prices');
        $products = Product::all();
        $progress->setMaxSteps(count($products));
        $progress->start();
        foreach($products as $product) {
            $newestPrice = $product->newestPrice;
            $currentStock = $product->currentStock;
            $product->priceCurrent = $newestPrice->price;
            $product->productStandardPrice = $newestPrice->productStandardPrice;
            $product->lowestProductPrice30Days = $newestPrice->lowestProductPrice30Days;
            $product->priceLowest = $product->lowestPrice->price;
            $product->lastSeen = $newestPrice->created_at->format('Y-m-d');
            $product->stock = $currentStock->stock;
            $product->save(['timestamps' => false]);
            $progress->advance();
        }
        $progress->finish();
        $output->writeln('');
        $output->writeln('Update prices - DONE');
        $output->writeln('COMMAND - DONE');

        return Command::SUCCESS;
    }

    private function getProducts(Country $country): array
    {
        $products = [];
        $page = 0;

        do {
            try {
                $res = $this->client->request('POST', $country->productsUrl, [
                    'form_params' => [
                        "includePreviousPages" => false,
                        "pageIndex" => $page,
                        "pageSize" => 100,
                        "cultureCode" => $country->cultureCode,
                    ]
                ]);
                $responseObject = json_decode($res->getBody()->getContents());
                $products = array_merge($products, $responseObject->products);
                $page++;
                $canLoadMore = $responseObject->canLoadMore;
            } catch (GuzzleException) {
                return $products;
            }
        } while ($canLoadMore);

        return $products;
    }

    private function saveProduct(\stdClass $product, Country $country): void
    {
        /** @var Product $productModel */
        $productModel = Product::firstOrNew(['skuID' => $product->skuID, 'country_id' => $country->id]);

        $productModel->skuID = $product->skuID;
        $productModel->name = $product->name;
        $productModel->availableQuantity = $product->availableQuantity;
        $productModel->categories = str_replace(['/', '  '], '', $product->categories);
        $productModel->image = $product->image;
        $productModel->subTitle = $product->subTitle;
        $productModel->variantCode = $product->variantCode;
        $productModel->modelCode = $product->modelCode;
        $productModel->url = $product->url;
        $productModel->lastSeen = date("Y-m-d");
        $productModel->touch('updated_at');
        $productModel->country()->associate($country);
        $productModel->save();
        $priceExists = $productModel->price()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();

        if (false === $priceExists) {
            $price = new Price();
            $price->price = $product->productPrice;
            $price->productStandardPrice = $product->productStandardPrice;
            $price->lowestProductPrice30Days = $product->lowestProductPrice30Days;
            $productModel->price()->save($price);
        }
        $stockExist = $productModel->stock()->whereRaw("strftime('%Y-%m-%d', created_at) = ?", [date('Y-m-d')])->exists();
        if (false === $stockExist) {
            $stock = new Stock();
            $stock->availableQuantity = $product->availableQuantity;
            $stock->stock = $product->stock;
            $stock->isInStock = $product->isInStock;
            $stock->previouslyHadStock = $product->previouslyHadStock;
            $stock->isNew = $product->isNew;
            $stock->isComingSoon = $product->isComingSoon;
            $stock->isOnSale = $product->isOnSale;
            $productModel->stock()->save($stock);
        }
    }
}