Working interface and parsed symfony cast data to a database.

This commit is contained in:
Krzysztof Płaczek
2022-08-23 08:09:14 +02:00
parent 119b94470f
commit a92c75c1dd
22 changed files with 488 additions and 224 deletions

View File

@@ -2,8 +2,6 @@
namespace App\Http\Controllers; namespace App\Http\Controllers;
use App\Http\SymfonyCastDl\HtmlParser;
use App\Http\SymfonyCastDl\SymfonyCastDlService;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Foundation\Bus\DispatchesJobs; use Illuminate\Foundation\Bus\DispatchesJobs;
use Illuminate\Foundation\Validation\ValidatesRequests; use Illuminate\Foundation\Validation\ValidatesRequests;
@@ -13,8 +11,4 @@ class Controller extends BaseController
{ {
use AuthorizesRequests, DispatchesJobs, ValidatesRequests; use AuthorizesRequests, DispatchesJobs, ValidatesRequests;
public function index(HtmlParser $htmlParser)
{
$service = new SymfonyCastDlService($htmlParser);
}
} }

View File

@@ -0,0 +1,25 @@
<?php
namespace App\Http\Controllers;
use App\Http\SymfonyCastDl\HtmlParser;
use App\Http\SymfonyCastDl\SymfonyCastDlService;
use App\Models\Chapter;
use App\Models\Course;
use Illuminate\View\View;
class CourseController extends Controller
{
public function index(Course $course): View
{
return view('course.index', compact('course'));
}
public function chapter(Chapter $chapter, HtmlParser $htmlParser)
{
$service = new SymfonyCastDlService($htmlParser);
$service->videoSize($chapter);
dd($chapter->toArray());
}
}

View File

@@ -0,0 +1,23 @@
<?php
namespace App\Http\Controllers;
use App\Http\SymfonyCastDl\HtmlParser;
use App\Http\SymfonyCastDl\SymfonyCastDlService;
use App\Models\Course;
use Illuminate\Http\Request;
class Index extends Controller
{
public function index()
{
$courses = Course::with('chapters')->withCount('chapters')->get();
return view('index', compact(['courses']));
}
public function download(HtmlParser $htmlParser)
{
$service = new SymfonyCastDlService($htmlParser);
$service->getInfo();
}
}

View File

@@ -2,8 +2,12 @@
namespace App\Http\SymfonyCastDl; namespace App\Http\SymfonyCastDl;
use App\Models\Chapter;
use App\Models\Course;
use Carbon\Carbon;
use DiDom\Document; use DiDom\Document;
use GuzzleHttp\Psr7\Response; use GuzzleHttp\Psr7\Response;
use Illuminate\Support\Collection;
class HtmlParser class HtmlParser
{ {
@@ -13,36 +17,55 @@ class HtmlParser
return $document->first('input[name="_csrf_token"]')->attr('value'); return $document->first('input[name="_csrf_token"]')->attr('value');
} }
public function getCourses(Response $response): array public function getCourses(Response $response): Collection
{ {
$courses = []; $courses = new Collection();
$document = new Document($response->getBody()->getContents()); $document = new Document($response->getBody()->getContents());
foreach ($document->find('div.js-course-item') as $courseItem) { foreach ($document->find('div.js-course-item') as $courseItem) {
$course = []; $courseId = $courseItem->attr('data-id');
$course['name'] = $courseItem->first('h3')->text(); $course = Course::firstOrNew(['course_id' => $courseId]);
$course['link'] = $courseItem->first('a')->attr('href'); $course->name = $courseItem->first('h3')->text();
$course['status'] = $courseItem->attr('data-status'); $course->thumbnail = $courseItem->first('img.course-list-item-img')->attr('src');
$course['chapter-count'] = $courseItem->attr('data-chapter-count'); $course->link = $courseItem->first('a')->attr('href');
$course['times-watched'] = $courseItem->attr('data-times-watched'); $course->status = $courseItem->attr('data-status');
$course->course_id = $courseItem->attr('data-id');
$courses[] = $course; $course->numberofchapters = $courseItem->attr('data-chapter-count');
$course->timeswatched = $courseItem->attr('data-times-watched');
$course->published_at = $courseItem->attr('data-date') > 0 ? Carbon::createFromTimestamp(
$courseItem->attr('data-date')
) : null;
$courses->add($course);
} }
return $courses; return $courses;
} }
public function getCourseDetails(Response $response): array public function getCourseDetails(Response $response, int $courseId): Collection
{ {
$document = new Document($response->getBody()->getContents()); $document = new Document($response->getBody()->getContents());
$info = ['chapters' => []]; $chapters = new Collection();
foreach ($document->find('ul.chapter-list li') as $chapter) { $chapterId = 0;
$info['chapters'][] = [ foreach ($document->find('ul.chapter-list li') as $chapterItem) {
'link' => $chapter->first('a')->attr('href'), if ($chapterItem->first('.col')) {
'title' => preg_replace('/\v(?:[\v\h]+)/', '', $chapter->first('.col')->text()), $chapterId++;
'duration' => $chapter->first('.length-styling')->text(),
]; // if(!$chapterItem->first('.length-styling')){
// dd($chapterItem->html());
// }
$chapter = Chapter::firstOrNew(['course_id' => $courseId, 'order' => $chapterId]);
$chapter->duration = $chapterItem->first('.length-styling')?->text();
$chapter->order = $chapterId;
$chapter->course_id = $courseId;
$chapter->link = config('symfonycast.base_url') . $chapterItem->first('a')->attr('href');
$chapter->video_link = config('symfonycast.base_url') . $chapterItem->first('a')->attr(
'href'
) . '/download/video';
$chapter->title = preg_replace('/\v(?:[\v\h]+)/', '', $chapterItem->first('.col')->text());
$chapter->video_size = 0;
$chapters->add($chapter);
}
} }
return $info; return $chapters;
} }
} }

View File

@@ -2,39 +2,64 @@
namespace App\Http\SymfonyCastDl; namespace App\Http\SymfonyCastDl;
use App\Models\Chapter;
use App\Models\Course;
use GuzzleHttp\TransferStats; use GuzzleHttp\TransferStats;
use DiDom\Document;
use GuzzleHttp\Client; use GuzzleHttp\Client;
class SymfonyCastDlService class SymfonyCastDlService
{ {
public function __construct(HtmlParser $htmlParser) public Client $client;
public function __construct(public HtmlParser $htmlParser)
{ {
$client = new Client([ $this->client = new Client([
'base_uri' => "https://symfonycasts.com", 'base_uri' => config('symfonycast.base_url'),
'cookies' => true 'cookies' => true
]); ]);
$response = $client->get('login'); $response = $this->client->get('login');
$token = $htmlParser->getCsrfToken($response); $token = $htmlParser->getCsrfToken($response);
$response = $client->post('login', [ $response = $this->client->post('login', [
'form_params' => [ 'form_params' => [
// 'email' => 'krzysiej@gmail.com', 'email' => config('symfonycast.login'),
// 'password' => '', 'password' => config('symfonycast.password'),
'_csrf_token' => $token '_csrf_token' => $token
], ],
'on_stats' => function (TransferStats $stats) use (&$currentUrl) { 'on_stats' => function (TransferStats $stats) use (&$currentUrl) {
$currentUrl = $stats->getEffectiveUri(); $currentUrl = $stats->getEffectiveUri();
} }
]); ]);
}
$coursePage = $client->get('courses/filtering'); public function getInfo()
// dump($htmlParser->getCourses($coursePage)); {
$coursePage = $this->client->get('courses/filtering');
$singleCoursePage = $client->get('screencast/api-platform'); $courses = $this->htmlParser->getCourses($coursePage);
dd($htmlParser->getCourseDetails($singleCoursePage)); $courses->each(fn($course) => $course->save());
$singleCoursePage = $this->client->get($courses[3]->link);
/** @var Course $course */
foreach ($courses as $course) {
$singleCoursePage = $this->client->get($course->link);
$chapters = $this->htmlParser->getCourseDetails($singleCoursePage, $course->id);
$chapters->each(fn($chapter) => $this->videoSize($chapter)->save());
}
}
public function videoSize(Chapter $chapter): Chapter
{
try {
$response = $this->client->head($chapter->video_link);
if ($response->hasHeader('Content-Length')) {
$chapter->video_size = $response->getHeader('Content-Length')[0];
}
} catch (\GuzzleHttp\Exception\ClientException $exception) {
}
return $chapter;
} }
} }

36
app/Models/Chapter.php Normal file
View File

@@ -0,0 +1,36 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\BelongsTo;
/**
* @property int $order
* @property string $link
* @property string $video_link
* @property integer $video_size
* @property string $title
* @property string $duration
* @property integer $course_id
*/
class Chapter extends Model
{
use HasFactory;
protected $fillable = [
'order',
'link',
'video_link',
'video_size',
'title',
'duration',
'course_id',
];
public function course(): BelongsTo
{
return $this->belongsTo(Course::class);
}
}

42
app/Models/Course.php Normal file
View File

@@ -0,0 +1,42 @@
<?php
namespace App\Models;
use Carbon\Traits\Date;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\HasMany;
/**
* @property int $course_id
* @property string $name
* @property string $thumbnail
* @property string $link
* @property string $status
* @property integer $numberofchapters
* @property integer $timeswatched
* @property Date $published_at
*/
class Course extends Model
{
use HasFactory;
protected $fillable = [
'name',
'thumbnail',
'link',
'status',
'numberofchapters',
'timeswatched',
'published_at',
];
protected $dates = [
'published_at'
];
public function chapters(): HasMany
{
return $this->hasMany(Chapter::class);
}
}

View File

@@ -1,44 +0,0 @@
<?php
namespace App\Models;
// use Illuminate\Contracts\Auth\MustVerifyEmail;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Foundation\Auth\User as Authenticatable;
use Illuminate\Notifications\Notifiable;
use Laravel\Sanctum\HasApiTokens;
class User extends Authenticatable
{
use HasApiTokens, HasFactory, Notifiable;
/**
* The attributes that are mass assignable.
*
* @var array<int, string>
*/
protected $fillable = [
'name',
'email',
'password',
];
/**
* The attributes that should be hidden for serialization.
*
* @var array<int, string>
*/
protected $hidden = [
'password',
'remember_token',
];
/**
* The attributes that should be cast.
*
* @var array<string, string>
*/
protected $casts = [
'email_verified_at' => 'datetime',
];
}

View File

@@ -13,6 +13,7 @@
"laravel/tinker": "^2.7" "laravel/tinker": "^2.7"
}, },
"require-dev": { "require-dev": {
"barryvdh/laravel-debugbar": "^3.7",
"fakerphp/faker": "^1.9.1", "fakerphp/faker": "^1.9.1",
"laravel/pint": "^1.0", "laravel/pint": "^1.0",
"laravel/sail": "^1.0.1", "laravel/sail": "^1.0.1",

152
composer.lock generated
View File

@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically" "This file is @generated automatically"
], ],
"content-hash": "6969489719f565e06eb44575a4396934", "content-hash": "e81b030ec4951ada3e6f11c989d631ea",
"packages": [ "packages": [
{ {
"name": "brick/math", "name": "brick/math",
@@ -5244,6 +5244,90 @@
} }
], ],
"packages-dev": [ "packages-dev": [
{
"name": "barryvdh/laravel-debugbar",
"version": "v3.7.0",
"source": {
"type": "git",
"url": "https://github.com/barryvdh/laravel-debugbar.git",
"reference": "3372ed65e6d2039d663ed19aa699956f9d346271"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/barryvdh/laravel-debugbar/zipball/3372ed65e6d2039d663ed19aa699956f9d346271",
"reference": "3372ed65e6d2039d663ed19aa699956f9d346271",
"shasum": ""
},
"require": {
"illuminate/routing": "^7|^8|^9",
"illuminate/session": "^7|^8|^9",
"illuminate/support": "^7|^8|^9",
"maximebf/debugbar": "^1.17.2",
"php": ">=7.2.5",
"symfony/finder": "^5|^6"
},
"require-dev": {
"mockery/mockery": "^1.3.3",
"orchestra/testbench-dusk": "^5|^6|^7",
"phpunit/phpunit": "^8.5|^9.0",
"squizlabs/php_codesniffer": "^3.5"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "3.6-dev"
},
"laravel": {
"providers": [
"Barryvdh\\Debugbar\\ServiceProvider"
],
"aliases": {
"Debugbar": "Barryvdh\\Debugbar\\Facades\\Debugbar"
}
}
},
"autoload": {
"files": [
"src/helpers.php"
],
"psr-4": {
"Barryvdh\\Debugbar\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Barry vd. Heuvel",
"email": "barryvdh@gmail.com"
}
],
"description": "PHP Debugbar integration for Laravel",
"keywords": [
"debug",
"debugbar",
"laravel",
"profiler",
"webprofiler"
],
"support": {
"issues": "https://github.com/barryvdh/laravel-debugbar/issues",
"source": "https://github.com/barryvdh/laravel-debugbar/tree/v3.7.0"
},
"funding": [
{
"url": "https://fruitcake.nl",
"type": "custom"
},
{
"url": "https://github.com/barryvdh",
"type": "github"
}
],
"time": "2022-07-11T09:26:42+00:00"
},
{ {
"name": "doctrine/instantiator", "name": "doctrine/instantiator",
"version": "1.4.1", "version": "1.4.1",
@@ -5682,6 +5766,72 @@
}, },
"time": "2022-07-21T14:33:56+00:00" "time": "2022-07-21T14:33:56+00:00"
}, },
{
"name": "maximebf/debugbar",
"version": "v1.18.0",
"source": {
"type": "git",
"url": "https://github.com/maximebf/php-debugbar.git",
"reference": "0d44b75f3b5d6d41ae83b79c7a4bceae7fbc78b6"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/maximebf/php-debugbar/zipball/0d44b75f3b5d6d41ae83b79c7a4bceae7fbc78b6",
"reference": "0d44b75f3b5d6d41ae83b79c7a4bceae7fbc78b6",
"shasum": ""
},
"require": {
"php": "^7.1|^8",
"psr/log": "^1|^2|^3",
"symfony/var-dumper": "^2.6|^3|^4|^5|^6"
},
"require-dev": {
"phpunit/phpunit": "^7.5.20 || ^9.4.2",
"twig/twig": "^1.38|^2.7|^3.0"
},
"suggest": {
"kriswallsmith/assetic": "The best way to manage assets",
"monolog/monolog": "Log using Monolog",
"predis/predis": "Redis storage"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.17-dev"
}
},
"autoload": {
"psr-4": {
"DebugBar\\": "src/DebugBar/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Maxime Bouroumeau-Fuseau",
"email": "maxime.bouroumeau@gmail.com",
"homepage": "http://maximebf.com"
},
{
"name": "Barry vd. Heuvel",
"email": "barryvdh@gmail.com"
}
],
"description": "Debug bar in the browser for php application",
"homepage": "https://github.com/maximebf/php-debugbar",
"keywords": [
"debug",
"debugbar"
],
"support": {
"issues": "https://github.com/maximebf/php-debugbar/issues",
"source": "https://github.com/maximebf/php-debugbar/tree/v1.18.0"
},
"time": "2021-12-27T18:49:48+00:00"
},
{ {
"name": "mockery/mockery", "name": "mockery/mockery",
"version": "1.5.0", "version": "1.5.0",

9
config/symfonycast.php Normal file
View File

@@ -0,0 +1,9 @@
<?php
use Illuminate\Support\Facades\Facade;
return [
'base_url' => env('SYMFONY_CAST_BASE_URL'),
'login' => env('SYMFONY_CAST_LOGIN'),
'password' => env('SYMFONY_CAST_PASSWORD'),
];

View File

@@ -1,36 +0,0 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('users', function (Blueprint $table) {
$table->id();
$table->string('name');
$table->string('email')->unique();
$table->timestamp('email_verified_at')->nullable();
$table->string('password');
$table->rememberToken();
$table->timestamps();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::dropIfExists('users');
}
};

View File

@@ -1,32 +0,0 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('password_resets', function (Blueprint $table) {
$table->string('email')->index();
$table->string('token');
$table->timestamp('created_at')->nullable();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::dropIfExists('password_resets');
}
};

View File

@@ -1,36 +0,0 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('failed_jobs', function (Blueprint $table) {
$table->id();
$table->string('uuid')->unique();
$table->text('connection');
$table->text('queue');
$table->longText('payload');
$table->longText('exception');
$table->timestamp('failed_at')->useCurrent();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::dropIfExists('failed_jobs');
}
};

View File

@@ -1,37 +0,0 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('personal_access_tokens', function (Blueprint $table) {
$table->id();
$table->morphs('tokenable');
$table->string('name');
$table->string('token', 64)->unique();
$table->text('abilities')->nullable();
$table->timestamp('last_used_at')->nullable();
$table->timestamp('expires_at')->nullable();
$table->timestamps();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::dropIfExists('personal_access_tokens');
}
};

View File

@@ -0,0 +1,38 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration {
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('courses', function (Blueprint $table) {
$table->id();
$table->integer('course_id')->unique();
$table->text('name');
$table->text('thumbnail');
$table->text('link');
$table->enum('status', ['unfinished', 'upcoming', 'complete']);
$table->integer('numberofchapters');
$table->integer('timeswatched');
$table->dateTime('published_at')->nullable();
$table->timestamps();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::dropIfExists('courses');
}
};

View File

@@ -0,0 +1,28 @@
<?php
use App\Models\Course;
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration {
public function up(): void
{
Schema::create('chapters', function (Blueprint $table) {
$table->id();
$table->foreignIdFor(Course::class);
$table->integer('order');
$table->text('title');
$table->text('duration')->nullable();
$table->text('link');
$table->text('video_link');
$table->integer('video_size');
$table->timestamps();
});
}
public function down(): void
{
Schema::dropIfExists('chapters');
}
};

View File

@@ -0,0 +1,12 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>{{ $title ?? 'Symfonycast.local' }}</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet"
integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
</head>
<body>
{{ $slot }}
</body>
</html>

View File

@@ -0,0 +1,14 @@
<x-layout>
<h1>
{{ $course->name }}
</h1>
<table class="table">
@foreach($course->chapters()->get() as $chapter)
<tr>
<td><a href="{{ route('course.chapter', ['chapter' => $chapter->id]) }}">{{ $chapter->title }}</a></td>
<td>{{ $chapter->duration }}</td>
</tr>
@endforeach
</table>
</x-layout>

View File

@@ -0,0 +1,24 @@
<x-layout>
<table class="table table-sm">
<thead>
<tr>
<th>Id</th>
<th></th>
<th>Name</th>
<th>status</th>
<th>Chapters</th>
<th>Published at</th>
</tr>
</thead>
@foreach($courses as $course)
<tr>
<td class="align-middle">{{ $course->id }}</td>
<td><img src="{{ $course->thumbnail }}" alt="{{ $course->name }}" class="img-thumbnail" style="width: 70px;"></td>
<td class="align-middle"><a href="{{ route('course.index', ['0' => $course->id]) }}"> {{ $course->name }}</a></td>
<td class="align-middle">{{ $course->status }}</td>
<td class="align-middle">{{ $course->chapters_count }} / {{ $course->numberofchapters }}</td>
<td class="align-middle"><abbr title="{{ $course->published_at?->format('Y-m-d') }}">{{ $course->published_at?->diffForHumans() }}</abbr></td>
</tr>
@endforeach
</table>
</x-layout>

View File

@@ -13,6 +13,9 @@ use Illuminate\Support\Facades\Route;
| |
*/ */
Route::get('/', [\App\Http\Controllers\Controller::class, 'index']); Route::get('/download', [\App\Http\Controllers\Index::class, 'download']);
Route::get('/', [\App\Http\Controllers\Index::class, 'index']);
Route::get('/course/{course}', [\App\Http\Controllers\CourseController::class, 'index'])->name('course.index');
Route::get('/chapter/{chapter}', [\App\Http\Controllers\CourseController::class, 'chapter'])->name('course.chapter');

2
storage/debugbar/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
*
!.gitignore