From 307fae39c450f687aba93e589e947e5389357601 Mon Sep 17 00:00:00 2001 From: Dan Brown Date: Mon, 18 Dec 2023 16:23:40 +0000 Subject: [PATCH] Input WYSIWYG: Added reference store & fetch handling For book, shelves and chapters. Made much of the existing handling generic to entity types. Added new MixedEntityListLoader to help load lists somewhat efficiently. Only manually tested so far. --- .../Commands/RegenerateReferencesCommand.php | 2 +- app/Entities/Controllers/BookController.php | 2 +- .../Controllers/BookshelfController.php | 2 +- .../Controllers/ChapterController.php | 2 +- app/Entities/Controllers/PageController.php | 2 +- app/Entities/Models/Book.php | 2 +- app/Entities/Models/Bookshelf.php | 2 +- app/Entities/Models/Chapter.php | 2 +- app/Entities/Models/Entity.php | 9 +- app/Entities/Models/Page.php | 3 +- app/Entities/Repos/BaseRepo.php | 6 +- app/Entities/Repos/PageRepo.php | 4 +- app/Entities/Tools/MixedEntityListLoader.php | 103 ++++++++++++++++++ app/References/ReferenceController.php | 16 ++- app/References/ReferenceFetcher.php | 60 ++++------ app/References/ReferenceStore.php | 85 ++++++++++----- app/References/ReferenceUpdater.php | 4 +- app/Settings/MaintenanceController.php | 2 +- lang/en/entities.php | 4 +- resources/views/entities/meta.blade.php | 2 +- resources/views/shelves/show.blade.php | 2 +- 21 files changed, 219 insertions(+), 97 deletions(-) create mode 100644 app/Entities/Tools/MixedEntityListLoader.php diff --git a/app/Console/Commands/RegenerateReferencesCommand.php b/app/Console/Commands/RegenerateReferencesCommand.php index ea8ff8e00..563da100a 100644 --- a/app/Console/Commands/RegenerateReferencesCommand.php +++ b/app/Console/Commands/RegenerateReferencesCommand.php @@ -34,7 +34,7 @@ class RegenerateReferencesCommand extends Command DB::setDefaultConnection($this->option('database')); } - $references->updateForAllPages(); + $references->updateForAll(); DB::setDefaultConnection($connection); diff --git a/app/Entities/Controllers/BookController.php b/app/Entities/Controllers/BookController.php index 481c621e6..412feca2f 100644 --- a/app/Entities/Controllers/BookController.php +++ b/app/Entities/Controllers/BookController.php @@ -138,7 +138,7 @@ class BookController extends Controller 'bookParentShelves' => $bookParentShelves, 'watchOptions' => new UserEntityWatchOptions(user(), $book), 'activity' => $activities->entityActivity($book, 20, 1), - 'referenceCount' => $this->referenceFetcher->getPageReferenceCountToEntity($book), + 'referenceCount' => $this->referenceFetcher->getReferenceCountToEntity($book), ]); } diff --git a/app/Entities/Controllers/BookshelfController.php b/app/Entities/Controllers/BookshelfController.php index acc972348..2f5461cdb 100644 --- a/app/Entities/Controllers/BookshelfController.php +++ b/app/Entities/Controllers/BookshelfController.php @@ -125,7 +125,7 @@ class BookshelfController extends Controller 'view' => $view, 'activity' => $activities->entityActivity($shelf, 20, 1), 'listOptions' => $listOptions, - 'referenceCount' => $this->referenceFetcher->getPageReferenceCountToEntity($shelf), + 'referenceCount' => $this->referenceFetcher->getReferenceCountToEntity($shelf), ]); } diff --git a/app/Entities/Controllers/ChapterController.php b/app/Entities/Controllers/ChapterController.php index 73f314ab6..28ad35fa4 100644 --- a/app/Entities/Controllers/ChapterController.php +++ b/app/Entities/Controllers/ChapterController.php @@ -86,7 +86,7 @@ class ChapterController extends Controller 'pages' => $pages, 'next' => $nextPreviousLocator->getNext(), 'previous' => $nextPreviousLocator->getPrevious(), - 'referenceCount' => $this->referenceFetcher->getPageReferenceCountToEntity($chapter), + 'referenceCount' => $this->referenceFetcher->getReferenceCountToEntity($chapter), ]); } diff --git a/app/Entities/Controllers/PageController.php b/app/Entities/Controllers/PageController.php index 0a3e76daa..adafcdc7b 100644 --- a/app/Entities/Controllers/PageController.php +++ b/app/Entities/Controllers/PageController.php @@ -155,7 +155,7 @@ class PageController extends Controller 'watchOptions' => new UserEntityWatchOptions(user(), $page), 'next' => $nextPreviousLocator->getNext(), 'previous' => $nextPreviousLocator->getPrevious(), - 'referenceCount' => $this->referenceFetcher->getPageReferenceCountToEntity($page), + 'referenceCount' => $this->referenceFetcher->getReferenceCountToEntity($page), ]); } diff --git a/app/Entities/Models/Book.php b/app/Entities/Models/Book.php index 7bbe2d8a4..52674d839 100644 --- a/app/Entities/Models/Book.php +++ b/app/Entities/Models/Book.php @@ -28,7 +28,7 @@ class Book extends Entity implements HasCoverImage use HasFactory; use HasHtmlDescription; - public $searchFactor = 1.2; + public float $searchFactor = 1.2; protected $fillable = ['name']; protected $hidden = ['pivot', 'image_id', 'deleted_at']; diff --git a/app/Entities/Models/Bookshelf.php b/app/Entities/Models/Bookshelf.php index cf22195f7..464c127b8 100644 --- a/app/Entities/Models/Bookshelf.php +++ b/app/Entities/Models/Bookshelf.php @@ -15,7 +15,7 @@ class Bookshelf extends Entity implements HasCoverImage protected $table = 'bookshelves'; - public $searchFactor = 1.2; + public float $searchFactor = 1.2; protected $fillable = ['name', 'description', 'image_id']; diff --git a/app/Entities/Models/Chapter.php b/app/Entities/Models/Chapter.php index 17fccfd6c..6c5f059ac 100644 --- a/app/Entities/Models/Chapter.php +++ b/app/Entities/Models/Chapter.php @@ -17,7 +17,7 @@ class Chapter extends BookChild use HasFactory; use HasHtmlDescription; - public $searchFactor = 1.2; + public float $searchFactor = 1.2; protected $fillable = ['name', 'description', 'priority']; protected $hidden = ['pivot', 'deleted_at']; diff --git a/app/Entities/Models/Entity.php b/app/Entities/Models/Entity.php index 332510672..f07d372c3 100644 --- a/app/Entities/Models/Entity.php +++ b/app/Entities/Models/Entity.php @@ -57,12 +57,17 @@ abstract class Entity extends Model implements Sluggable, Favouritable, Viewable /** * @var string - Name of property where the main text content is found */ - public $textField = 'description'; + public string $textField = 'description'; + + /** + * @var string - Name of the property where the main HTML content is found + */ + public string $htmlField = 'description_html'; /** * @var float - Multiplier for search indexing. */ - public $searchFactor = 1.0; + public float $searchFactor = 1.0; /** * Get the entities that are visible to the current user. diff --git a/app/Entities/Models/Page.php b/app/Entities/Models/Page.php index 7e2c12c20..17d6f9a01 100644 --- a/app/Entities/Models/Page.php +++ b/app/Entities/Models/Page.php @@ -37,7 +37,8 @@ class Page extends BookChild protected $fillable = ['name', 'priority']; - public $textField = 'text'; + public string $textField = 'text'; + public string $htmlField = 'html'; protected $hidden = ['html', 'markdown', 'text', 'pivot', 'deleted_at']; diff --git a/app/Entities/Repos/BaseRepo.php b/app/Entities/Repos/BaseRepo.php index f6b9ff578..dbdaa9213 100644 --- a/app/Entities/Repos/BaseRepo.php +++ b/app/Entities/Repos/BaseRepo.php @@ -7,6 +7,7 @@ use BookStack\Entities\Models\Entity; use BookStack\Entities\Models\HasCoverImage; use BookStack\Entities\Models\HasHtmlDescription; use BookStack\Exceptions\ImageUploadException; +use BookStack\References\ReferenceStore; use BookStack\References\ReferenceUpdater; use BookStack\Uploads\ImageRepo; use Illuminate\Http\UploadedFile; @@ -16,7 +17,8 @@ class BaseRepo public function __construct( protected TagRepo $tagRepo, protected ImageRepo $imageRepo, - protected ReferenceUpdater $referenceUpdater + protected ReferenceUpdater $referenceUpdater, + protected ReferenceStore $referenceStore, ) { } @@ -42,6 +44,7 @@ class BaseRepo $entity->refresh(); $entity->rebuildPermissions(); $entity->indexForSearch(); + $this->referenceStore->updateForEntity($entity); } /** @@ -68,6 +71,7 @@ class BaseRepo $entity->rebuildPermissions(); $entity->indexForSearch(); + $this->referenceStore->updateForEntity($entity); if ($oldUrl !== $entity->getUrl()) { $this->referenceUpdater->updateEntityPageReferences($entity, $oldUrl); diff --git a/app/Entities/Repos/PageRepo.php b/app/Entities/Repos/PageRepo.php index 9a183469b..d491b7f2c 100644 --- a/app/Entities/Repos/PageRepo.php +++ b/app/Entities/Repos/PageRepo.php @@ -162,7 +162,6 @@ class PageRepo $this->baseRepo->update($draft, $input); $this->revisionRepo->storeNewForPage($draft, trans('entities.pages_initial_revision')); - $this->referenceStore->updateForPage($draft); $draft->refresh(); Activity::add(ActivityType::PAGE_CREATE, $draft); @@ -182,7 +181,6 @@ class PageRepo $this->updateTemplateStatusAndContentFromInput($page, $input); $this->baseRepo->update($page, $input); - $this->referenceStore->updateForPage($page); // Update with new details $page->revision_count++; @@ -301,7 +299,7 @@ class PageRepo $page->refreshSlug(); $page->save(); $page->indexForSearch(); - $this->referenceStore->updateForPage($page); + $this->referenceStore->updateForEntity($page); $summary = trans('entities.pages_revision_restored_from', ['id' => strval($revisionId), 'summary' => $revision->summary]); $this->revisionRepo->storeNewForPage($page, $summary); diff --git a/app/Entities/Tools/MixedEntityListLoader.php b/app/Entities/Tools/MixedEntityListLoader.php new file mode 100644 index 000000000..50079e3bf --- /dev/null +++ b/app/Entities/Tools/MixedEntityListLoader.php @@ -0,0 +1,103 @@ + ['id', 'name', 'slug', 'book_id', 'chapter_id', 'text', 'draft'], + 'chapter' => ['id', 'name', 'slug', 'book_id', 'description'], + 'book' => ['id', 'name', 'slug', 'description'], + 'bookshelf' => ['id', 'name', 'slug', 'description'], + ]; + + public function __construct( + protected EntityProvider $entityProvider + ) { + } + + /** + * Efficiently load in entities for listing onto the given list + * where entities are set as a relation via the given name. + * This will look for a model id and type via 'name_id' and 'name_type'. + * @param Model[] $relations + */ + public function loadIntoRelations(array $relations, string $relationName): void + { + $idsByType = []; + foreach ($relations as $relation) { + $type = $relation->getAttribute($relationName . '_type'); + $id = $relation->getAttribute($relationName . '_id'); + + if (!isset($idsByType[$type])) { + $idsByType[$type] = []; + } + + $idsByType[$type][] = $id; + } + + $modelMap = $this->idsByTypeToModelMap($idsByType); + + foreach ($relations as $relation) { + $type = $relation->getAttribute($relationName . '_type'); + $id = $relation->getAttribute($relationName . '_id'); + $related = $modelMap[$type][strval($id)] ?? null; + if ($related) { + $relation->setRelation($relationName, $related); + } + } + } + + /** + * @param array $idsByType + * @return array> + */ + protected function idsByTypeToModelMap(array $idsByType): array + { + $modelMap = []; + + foreach ($idsByType as $type => $ids) { + if (!isset($this->listAttributes[$type])) { + continue; + } + + $instance = $this->entityProvider->get($type); + $models = $instance->newQuery() + ->select($this->listAttributes[$type]) + ->scopes('visible') + ->whereIn('id', $ids) + ->with($this->getRelationsToEagerLoad($type)) + ->get(); + + if (count($models) > 0) { + $modelMap[$type] = []; + } + + foreach ($models as $model) { + $modelMap[$type][strval($model->id)] = $model; + } + } + + return $modelMap; + } + + protected function getRelationsToEagerLoad(string $type): array + { + $toLoad = []; + $loadVisible = fn (Relation $query) => $query->scopes('visible'); + + if ($type === 'chapter' || $type === 'page') { + $toLoad['book'] = $loadVisible; + } + + if ($type === 'page') { + $toLoad['chapter'] = $loadVisible; + } + + return $toLoad; + } +} diff --git a/app/References/ReferenceController.php b/app/References/ReferenceController.php index d6978dd5b..991f47225 100644 --- a/app/References/ReferenceController.php +++ b/app/References/ReferenceController.php @@ -10,11 +10,9 @@ use BookStack\Http\Controller; class ReferenceController extends Controller { - protected ReferenceFetcher $referenceFetcher; - - public function __construct(ReferenceFetcher $referenceFetcher) - { - $this->referenceFetcher = $referenceFetcher; + public function __construct( + protected ReferenceFetcher $referenceFetcher + ) { } /** @@ -23,7 +21,7 @@ class ReferenceController extends Controller public function page(string $bookSlug, string $pageSlug) { $page = Page::getBySlugs($bookSlug, $pageSlug); - $references = $this->referenceFetcher->getPageReferencesToEntity($page); + $references = $this->referenceFetcher->getReferencesToEntity($page); return view('pages.references', [ 'page' => $page, @@ -37,7 +35,7 @@ class ReferenceController extends Controller public function chapter(string $bookSlug, string $chapterSlug) { $chapter = Chapter::getBySlugs($bookSlug, $chapterSlug); - $references = $this->referenceFetcher->getPageReferencesToEntity($chapter); + $references = $this->referenceFetcher->getReferencesToEntity($chapter); return view('chapters.references', [ 'chapter' => $chapter, @@ -51,7 +49,7 @@ class ReferenceController extends Controller public function book(string $slug) { $book = Book::getBySlug($slug); - $references = $this->referenceFetcher->getPageReferencesToEntity($book); + $references = $this->referenceFetcher->getReferencesToEntity($book); return view('books.references', [ 'book' => $book, @@ -65,7 +63,7 @@ class ReferenceController extends Controller public function shelf(string $slug) { $shelf = Bookshelf::getBySlug($slug); - $references = $this->referenceFetcher->getPageReferencesToEntity($shelf); + $references = $this->referenceFetcher->getReferencesToEntity($shelf); return view('shelves.references', [ 'shelf' => $shelf, diff --git a/app/References/ReferenceFetcher.php b/app/References/ReferenceFetcher.php index c4a7d31b6..0d9883a3e 100644 --- a/app/References/ReferenceFetcher.php +++ b/app/References/ReferenceFetcher.php @@ -3,65 +3,51 @@ namespace BookStack\References; use BookStack\Entities\Models\Entity; -use BookStack\Entities\Models\Page; +use BookStack\Entities\Tools\MixedEntityListLoader; use BookStack\Permissions\PermissionApplicator; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Collection; -use Illuminate\Database\Eloquent\Relations\Relation; class ReferenceFetcher { - protected PermissionApplicator $permissions; - - public function __construct(PermissionApplicator $permissions) - { - $this->permissions = $permissions; + public function __construct( + protected PermissionApplicator $permissions, + protected MixedEntityListLoader $mixedEntityListLoader, + ) { } /** - * Query and return the page references pointing to the given entity. + * Query and return the references pointing to the given entity. * Loads the commonly required relations while taking permissions into account. */ - public function getPageReferencesToEntity(Entity $entity): Collection + public function getReferencesToEntity(Entity $entity): Collection { - $baseQuery = $this->queryPageReferencesToEntity($entity) - ->with([ - 'from' => fn (Relation $query) => $query->select(Page::$listAttributes), - 'from.book' => fn (Relation $query) => $query->scopes('visible'), - 'from.chapter' => fn (Relation $query) => $query->scopes('visible'), - ]); - - $references = $this->permissions->restrictEntityRelationQuery( - $baseQuery, - 'references', - 'from_id', - 'from_type' - )->get(); + $references = $this->queryReferencesToEntity($entity)->get(); + $this->mixedEntityListLoader->loadIntoRelations($references->all(), 'from'); return $references; } /** - * Returns the count of page references pointing to the given entity. + * Returns the count of references pointing to the given entity. * Takes permissions into account. */ - public function getPageReferenceCountToEntity(Entity $entity): int + public function getReferenceCountToEntity(Entity $entity): int { - $count = $this->permissions->restrictEntityRelationQuery( - $this->queryPageReferencesToEntity($entity), + return $this->queryReferencesToEntity($entity)->count(); + } + + protected function queryReferencesToEntity(Entity $entity): Builder + { + $baseQuery = Reference::query() + ->where('to_type', '=', $entity->getMorphClass()) + ->where('to_id', '=', $entity->id); + + return $this->permissions->restrictEntityRelationQuery( + $baseQuery, 'references', 'from_id', 'from_type' - )->count(); - - return $count; - } - - protected function queryPageReferencesToEntity(Entity $entity): Builder - { - return Reference::query() - ->where('to_type', '=', $entity->getMorphClass()) - ->where('to_id', '=', $entity->id) - ->where('from_type', '=', (new Page())->getMorphClass()); + ); } } diff --git a/app/References/ReferenceStore.php b/app/References/ReferenceStore.php index 4c6db35c5..78595084b 100644 --- a/app/References/ReferenceStore.php +++ b/app/References/ReferenceStore.php @@ -2,60 +2,62 @@ namespace BookStack\References; -use BookStack\Entities\Models\Page; +use BookStack\Entities\EntityProvider; +use BookStack\Entities\Models\Entity; use Illuminate\Database\Eloquent\Collection; class ReferenceStore { - /** - * Update the outgoing references for the given page. - */ - public function updateForPage(Page $page): void - { - $this->updateForPages([$page]); + public function __construct( + protected EntityProvider $entityProvider + ) { } /** - * Update the outgoing references for all pages in the system. + * Update the outgoing references for the given entity. */ - public function updateForAllPages(): void + public function updateForEntity(Entity $entity): void { - Reference::query() - ->where('from_type', '=', (new Page())->getMorphClass()) - ->delete(); - - Page::query()->select(['id', 'html'])->chunk(100, function (Collection $pages) { - $this->updateForPages($pages->all()); - }); + $this->updateForEntities([$entity]); } /** - * Update the outgoing references for the pages in the given array. + * Update the outgoing references for all entities in the system. + */ + public function updateForAll(): void + { + Reference::query()->delete(); + + foreach ($this->entityProvider->all() as $entity) { + $entity->newQuery()->select(['id', $entity->htmlField])->chunk(100, function (Collection $entities) { + $this->updateForEntities($entities->all()); + }); + } + } + + /** + * Update the outgoing references for the entities in the given array. * - * @param Page[] $pages + * @param Entity[] $entities */ - protected function updateForPages(array $pages): void + protected function updateForEntities(array $entities): void { - if (count($pages) === 0) { + if (count($entities) === 0) { return; } $parser = CrossLinkParser::createWithEntityResolvers(); $references = []; - $pageIds = array_map(fn (Page $page) => $page->id, $pages); - Reference::query() - ->where('from_type', '=', $pages[0]->getMorphClass()) - ->whereIn('from_id', $pageIds) - ->delete(); + $this->dropReferencesFromEntities($entities); - foreach ($pages as $page) { - $models = $parser->extractLinkedModels($page->html); + foreach ($entities as $entity) { + $models = $parser->extractLinkedModels($entity->getAttribute($entity->htmlField)); foreach ($models as $model) { $references[] = [ - 'from_id' => $page->id, - 'from_type' => $page->getMorphClass(), + 'from_id' => $entity->id, + 'from_type' => $entity->getMorphClass(), 'to_id' => $model->id, 'to_type' => $model->getMorphClass(), ]; @@ -66,4 +68,29 @@ class ReferenceStore Reference::query()->insert($referenceDataChunk); } } + + /** + * Delete all the existing references originating from the given entities. + * @param Entity[] $entities + */ + protected function dropReferencesFromEntities(array $entities): void + { + $IdsByType = []; + + foreach ($entities as $entity) { + $type = $entity->getMorphClass(); + if (!isset($IdsByType[$type])) { + $IdsByType[$type] = []; + } + + $IdsByType[$type][] = $entity->id; + } + + foreach ($IdsByType as $type => $entityIds) { + Reference::query() + ->where('from_type', '=', $type) + ->whereIn('from_id', $entityIds) + ->delete(); + } + } } diff --git a/app/References/ReferenceUpdater.php b/app/References/ReferenceUpdater.php index 248937339..db1bce4fb 100644 --- a/app/References/ReferenceUpdater.php +++ b/app/References/ReferenceUpdater.php @@ -35,7 +35,7 @@ class ReferenceUpdater protected function getReferencesToUpdate(Entity $entity): array { /** @var Reference[] $references */ - $references = $this->referenceFetcher->getPageReferencesToEntity($entity)->values()->all(); + $references = $this->referenceFetcher->getReferencesToEntity($entity)->values()->all(); if ($entity instanceof Book) { $pages = $entity->pages()->get(['id']); @@ -43,7 +43,7 @@ class ReferenceUpdater $children = $pages->concat($chapters); foreach ($children as $bookChild) { /** @var Reference[] $childRefs */ - $childRefs = $this->referenceFetcher->getPageReferencesToEntity($bookChild)->values()->all(); + $childRefs = $this->referenceFetcher->getReferencesToEntity($bookChild)->values()->all(); array_push($references, ...$childRefs); } } diff --git a/app/Settings/MaintenanceController.php b/app/Settings/MaintenanceController.php index 60e5fee28..62eeecf39 100644 --- a/app/Settings/MaintenanceController.php +++ b/app/Settings/MaintenanceController.php @@ -87,7 +87,7 @@ class MaintenanceController extends Controller $this->logActivity(ActivityType::MAINTENANCE_ACTION_RUN, 'regenerate-references'); try { - $referenceStore->updateForAllPages(); + $referenceStore->updateForAll(); $this->showSuccessNotification(trans('settings.maint_regen_references_success')); } catch (\Exception $exception) { $this->showErrorNotification($exception->getMessage()); diff --git a/lang/en/entities.php b/lang/en/entities.php index 354eee42e..f1f915544 100644 --- a/lang/en/entities.php +++ b/lang/en/entities.php @@ -23,7 +23,7 @@ return [ 'meta_updated' => 'Updated :timeLength', 'meta_updated_name' => 'Updated :timeLength by :user', 'meta_owned_name' => 'Owned by :user', - 'meta_reference_page_count' => 'Referenced on :count page|Referenced on :count pages', + 'meta_reference_count' => 'Referenced by :count item|Referenced by :count items', 'entity_select' => 'Entity Select', 'entity_select_lack_permission' => 'You don\'t have the required permissions to select this item', 'images' => 'Images', @@ -409,7 +409,7 @@ return [ // References 'references' => 'References', 'references_none' => 'There are no tracked references to this item.', - 'references_to_desc' => 'Shown below are all the known pages in the system that link to this item.', + 'references_to_desc' => 'Listed below is all the known content in the system that links to this item.', // Watch Options 'watch' => 'Watch', diff --git a/resources/views/entities/meta.blade.php b/resources/views/entities/meta.blade.php index 2298be8bb..9d3c4b956 100644 --- a/resources/views/entities/meta.blade.php +++ b/resources/views/entities/meta.blade.php @@ -64,7 +64,7 @@ @icon('reference')
- {!! trans_choice('entities.meta_reference_page_count', $referenceCount, ['count' => $referenceCount]) !!} + {{ trans_choice('entities.meta_reference_count', $referenceCount, ['count' => $referenceCount]) }}
@endif diff --git a/resources/views/shelves/show.blade.php b/resources/views/shelves/show.blade.php index 58fe1cd86..e475a8080 100644 --- a/resources/views/shelves/show.blade.php +++ b/resources/views/shelves/show.blade.php @@ -28,7 +28,7 @@
-

{!! nl2br(e($shelf->description)) !!}

+

{!! $shelf->descriptionHtml() !!}

@if(count($sortedVisibleShelfBooks) > 0) @if($view === 'list')