Added reference storage system, and command to re-index

Also re-named/orgranized some files for this, to make them "References"
specific instead of a subset of "Util".
This commit is contained in:
Dan Brown 2022-08-17 14:39:53 +01:00
parent 344b3a3615
commit 5d29d0cc7b
No known key found for this signature in database
GPG key ID: 46D9F943C24A2EF9
15 changed files with 253 additions and 23 deletions

View file

@ -5,6 +5,7 @@ namespace BookStack\Console\Commands;
use BookStack\Actions\Comment; use BookStack\Actions\Comment;
use BookStack\Actions\CommentRepo; use BookStack\Actions\CommentRepo;
use Illuminate\Console\Command; use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
class RegenerateCommentContent extends Command class RegenerateCommentContent extends Command
{ {
@ -43,9 +44,9 @@ class RegenerateCommentContent extends Command
*/ */
public function handle() public function handle()
{ {
$connection = \DB::getDefaultConnection(); $connection = DB::getDefaultConnection();
if ($this->option('database') !== null) { if ($this->option('database') !== null) {
\DB::setDefaultConnection($this->option('database')); DB::setDefaultConnection($this->option('database'));
} }
Comment::query()->chunk(100, function ($comments) { Comment::query()->chunk(100, function ($comments) {
@ -55,7 +56,8 @@ class RegenerateCommentContent extends Command
} }
}); });
\DB::setDefaultConnection($connection); DB::setDefaultConnection($connection);
$this->comment('Comment HTML content has been regenerated'); $this->comment('Comment HTML content has been regenerated');
return 0;
} }
} }

View file

@ -50,5 +50,6 @@ class RegeneratePermissions extends Command
DB::setDefaultConnection($connection); DB::setDefaultConnection($connection);
$this->comment('Permissions regenerated'); $this->comment('Permissions regenerated');
return 0;
} }
} }

View file

@ -0,0 +1,58 @@
<?php
namespace BookStack\Console\Commands;
use BookStack\References\ReferenceService;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
class RegenerateReferences extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'bookstack:regenerate-references {--database= : The database connection to use.}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Regenerate all the cross-item model reference index';
protected ReferenceService $references;
/**
* Create a new command instance.
*
* @return void
*/
public function __construct(ReferenceService $references)
{
$this->references = $references;
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
$connection = DB::getDefaultConnection();
if ($this->option('database')) {
DB::setDefaultConnection($this->option('database'));
}
$this->references->updateForAllPages();
DB::setDefaultConnection($connection);
$this->comment('References have been regenerated');
return 0;
}
}

View file

@ -18,6 +18,7 @@ use BookStack\Interfaces\Loggable;
use BookStack\Interfaces\Sluggable; use BookStack\Interfaces\Sluggable;
use BookStack\Interfaces\Viewable; use BookStack\Interfaces\Viewable;
use BookStack\Model; use BookStack\Model;
use BookStack\References\Reference;
use BookStack\Search\SearchIndex; use BookStack\Search\SearchIndex;
use BookStack\Search\SearchTerm; use BookStack\Search\SearchTerm;
use BookStack\Traits\HasCreatorAndUpdater; use BookStack\Traits\HasCreatorAndUpdater;
@ -203,6 +204,22 @@ abstract class Entity extends Model implements Sluggable, Favouritable, Viewable
return $this->morphMany(Deletion::class, 'deletable'); return $this->morphMany(Deletion::class, 'deletable');
} }
/**
* Get the references pointing from this entity to other items.
*/
public function referencesFrom(): MorphMany
{
return $this->morphMany(Reference::class, 'from');
}
/**
* Get the references pointing to this entity from other items.
*/
public function referencesTo(): MorphMany
{
return $this->morphMany(Reference::class, 'to');
}
/** /**
* Check if this instance or class is a certain type of entity. * Check if this instance or class is a certain type of entity.
* Examples of $type are 'page', 'book', 'chapter'. * Examples of $type are 'page', 'book', 'chapter'.

View file

@ -1,14 +1,14 @@
<?php <?php
namespace BookStack\Util\CrossLinking; namespace BookStack\References;
use BookStack\Model; use BookStack\Model;
use BookStack\Util\CrossLinking\ModelResolvers\BookLinkModelResolver; use BookStack\References\ModelResolvers\BookLinkModelResolver;
use BookStack\Util\CrossLinking\ModelResolvers\BookshelfLinkModelResolver; use BookStack\References\ModelResolvers\BookshelfLinkModelResolver;
use BookStack\Util\CrossLinking\ModelResolvers\ChapterLinkModelResolver; use BookStack\References\ModelResolvers\ChapterLinkModelResolver;
use BookStack\Util\CrossLinking\ModelResolvers\CrossLinkModelResolver; use BookStack\References\ModelResolvers\CrossLinkModelResolver;
use BookStack\Util\CrossLinking\ModelResolvers\PageLinkModelResolver; use BookStack\References\ModelResolvers\PageLinkModelResolver;
use BookStack\Util\CrossLinking\ModelResolvers\PagePermalinkModelResolver; use BookStack\References\ModelResolvers\PagePermalinkModelResolver;
use DOMDocument; use DOMDocument;
use DOMXPath; use DOMXPath;
@ -27,7 +27,7 @@ class CrossLinkParser
/** /**
* Extract any found models within the given HTML content. * Extract any found models within the given HTML content.
* *
* @returns Model[] * @return Model[]
*/ */
public function extractLinkedModels(string $html): array public function extractLinkedModels(string $html): array
{ {

View file

@ -1,6 +1,6 @@
<?php <?php
namespace BookStack\Util\CrossLinking\ModelResolvers; namespace BookStack\References\ModelResolvers;
use BookStack\Entities\Models\Book; use BookStack\Entities\Models\Book;
use BookStack\Model; use BookStack\Model;
@ -9,7 +9,7 @@ class BookLinkModelResolver implements CrossLinkModelResolver
{ {
public function resolve(string $link): ?Model public function resolve(string $link): ?Model
{ {
$pattern = '/^' . preg_quote(url('/books'), '/') . '\/([\w-]+)' . '[#?\/$]/'; $pattern = '/^' . preg_quote(url('/books'), '/') . '\/([\w-]+)' . '([#?\/]|$)/';
$matches = []; $matches = [];
$match = preg_match($pattern, $link, $matches); $match = preg_match($pattern, $link, $matches);
if (!$match) { if (!$match) {

View file

@ -1,6 +1,6 @@
<?php <?php
namespace BookStack\Util\CrossLinking\ModelResolvers; namespace BookStack\References\ModelResolvers;
use BookStack\Entities\Models\Bookshelf; use BookStack\Entities\Models\Bookshelf;
use BookStack\Model; use BookStack\Model;
@ -9,7 +9,7 @@ class BookshelfLinkModelResolver implements CrossLinkModelResolver
{ {
public function resolve(string $link): ?Model public function resolve(string $link): ?Model
{ {
$pattern = '/^' . preg_quote(url('/shelves'), '/') . '\/([\w-]+)' . '[#?\/$]/'; $pattern = '/^' . preg_quote(url('/shelves'), '/') . '\/([\w-]+)' . '([#?\/]|$)/';
$matches = []; $matches = [];
$match = preg_match($pattern, $link, $matches); $match = preg_match($pattern, $link, $matches);
if (!$match) { if (!$match) {

View file

@ -1,6 +1,6 @@
<?php <?php
namespace BookStack\Util\CrossLinking\ModelResolvers; namespace BookStack\References\ModelResolvers;
use BookStack\Entities\Models\Chapter; use BookStack\Entities\Models\Chapter;
use BookStack\Model; use BookStack\Model;
@ -9,7 +9,7 @@ class ChapterLinkModelResolver implements CrossLinkModelResolver
{ {
public function resolve(string $link): ?Model public function resolve(string $link): ?Model
{ {
$pattern = '/^' . preg_quote(url('/books'), '/') . '\/([\w-]+)' . '\/chapter\/' . '([\w-]+)' . '[#?\/$]/'; $pattern = '/^' . preg_quote(url('/books'), '/') . '\/([\w-]+)' . '\/chapter\/' . '([\w-]+)' . '([#?\/]|$)/';
$matches = []; $matches = [];
$match = preg_match($pattern, $link, $matches); $match = preg_match($pattern, $link, $matches);
if (!$match) { if (!$match) {

View file

@ -1,6 +1,6 @@
<?php <?php
namespace BookStack\Util\CrossLinking\ModelResolvers; namespace BookStack\References\ModelResolvers;
use BookStack\Model; use BookStack\Model;

View file

@ -1,6 +1,6 @@
<?php <?php
namespace BookStack\Util\CrossLinking\ModelResolvers; namespace BookStack\References\ModelResolvers;
use BookStack\Entities\Models\Page; use BookStack\Entities\Models\Page;
use BookStack\Model; use BookStack\Model;
@ -9,7 +9,7 @@ class PageLinkModelResolver implements CrossLinkModelResolver
{ {
public function resolve(string $link): ?Model public function resolve(string $link): ?Model
{ {
$pattern = '/^' . preg_quote(url('/books'), '/') . '\/([\w-]+)' . '\/page\/' . '([\w-]+)' . '[#?\/$]/'; $pattern = '/^' . preg_quote(url('/books'), '/') . '\/([\w-]+)' . '\/page\/' . '([\w-]+)' . '([#?\/]|$)/';
$matches = []; $matches = [];
$match = preg_match($pattern, $link, $matches); $match = preg_match($pattern, $link, $matches);
if (!$match) { if (!$match) {

View file

@ -1,6 +1,6 @@
<?php <?php
namespace BookStack\Util\CrossLinking\ModelResolvers; namespace BookStack\References\ModelResolvers;
use BookStack\Entities\Models\Page; use BookStack\Entities\Models\Page;
use BookStack\Model; use BookStack\Model;

View file

@ -0,0 +1,26 @@
<?php
namespace BookStack\References;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\MorphTo;
/**
* @property int $from_id
* @property string $from_type
* @property int $to_id
* @property string $to_type
*/
class Reference extends Model
{
public function from(): MorphTo
{
return $this->morphTo('from');
}
public function to(): MorphTo
{
return $this->morphTo('to');
}
}

View file

@ -0,0 +1,71 @@
<?php
namespace BookStack\References;
use BookStack\Entities\Models\Page;
use Illuminate\Database\Eloquent\Collection;
class ReferenceService
{
/**
* Update the outgoing references for the given page.
*/
public function updateForPage(Page $page): void
{
$this->updateForPages([$page]);
}
/**
* Update the outgoing references for all pages in the system.
*/
public function updateForAllPages(): void
{
Reference::query()
->where('from_type', '=', (new Page())->getMorphClass())
->truncate();
Page::query()->select(['id', 'html'])->chunk(100, function(Collection $pages) {
$this->updateForPages($pages->all());
});
}
/**
* Update the outgoing references for the pages in the given array.
*
* @param Page[] $pages
*/
protected function updateForPages(array $pages): void
{
if (count($pages) === 0) {
return;
}
$parser = CrossLinkParser::createWithEntityResolvers();
$references = [];
$pageIds = array_map(fn(Page $page) => $page->id, $pages);
Reference::query()
->where('from_type', '=', $pages[0]->getMorphClass())
->whereIn('from_id', $pageIds)
->delete();
foreach ($pages as $page) {
$models = $parser->extractLinkedModels($page->html);
foreach ($models as $model) {
$references[] = [
'from_id' => $page->id,
'from_type' => $page->getMorphClass(),
'to_id' => $model->id,
'to_type' => $model->getMorphClass(),
];
}
}
foreach (array_chunk($references, 1000) as $referenceDataChunk) {
Reference::query()->insert($referenceDataChunk);
}
}
}

View file

@ -0,0 +1,34 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
class CreateReferencesTable extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('references', function (Blueprint $table) {
$table->id();
$table->unsignedInteger('from_id')->index();
$table->string('from_type', 25)->index();
$table->unsignedInteger('to_id')->index();
$table->string('to_type', 25)->index();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::dropIfExists('references');
}
}

View file

@ -1,9 +1,10 @@
<?php <?php
namespace Tests\Util; namespace Tests\References;
use BookStack\Entities\Models\Book;
use BookStack\Entities\Models\Page; use BookStack\Entities\Models\Page;
use BookStack\Util\CrossLinking\CrossLinkParser; use BookStack\References\CrossLinkParser;
use Tests\TestCase; use Tests\TestCase;
class CrossLinkParserTest extends TestCase class CrossLinkParserTest extends TestCase
@ -38,4 +39,24 @@ class CrossLinkParserTest extends TestCase
$this->assertEquals(get_class($entities['bookshelf']), get_class($results[4])); $this->assertEquals(get_class($entities['bookshelf']), get_class($results[4]));
$this->assertEquals($entities['bookshelf']->id, $results[4]->id); $this->assertEquals($entities['bookshelf']->id, $results[4]->id);
} }
public function test_similar_page_and_book_reference_links_dont_conflict()
{
$page = Page::query()->first();
$book = $page->book;
$html = '
<a href="' . $page->getUrl() . '">Page Link</a>
<a href="' . $book->getUrl() . '">Book Link</a>
';
$parser = CrossLinkParser::createWithEntityResolvers();
$results = $parser->extractLinkedModels($html);
$this->assertCount(2, $results);
$this->assertEquals(get_class($page), get_class($results[0]));
$this->assertEquals($page->id, $results[0]->id);
$this->assertEquals(get_class($book), get_class($results[1]));
$this->assertEquals($book->id, $results[1]->id);
}
} }