From 855a63edbf0dfae36041cc46afbe4962e9eb4d37 Mon Sep 17 00:00:00 2001 From: "d.savuljesku" Date: Fri, 8 Dec 2023 16:11:37 +0100 Subject: [PATCH] Title tree store #1 --- bootstrap.php | 2 +- route-files/title-tree-store.json | 13 + .../TitleQueryStore/PrimaryDataProvider.php | 1 + .../TitleQueryStore/SecondaryDataProvider.php | 27 +- src/Data/TitleQueryStore/TitleSchema.php | 6 +- .../TitleTreeStore/PrimaryDataProvider.php | 256 ++++++++++++++++++ src/Data/TitleTreeStore/Reader.php | 42 +++ .../TitleTreeStore/SecondaryDataProvider.php | 43 +++ src/Data/TitleTreeStore/Store.php | 55 ++++ .../TitleTreeStore/TitleTreeReaderParams.php | 34 +++ src/Data/TitleTreeStore/TitleTreeRecord.php | 14 + src/Data/TitleTreeStore/TitleTreeSchema.php | 39 +++ src/Rest/QueryStore.php | 12 +- src/Rest/TitleQueryStore.php | 15 +- src/Rest/TitleTreeStore.php | 51 ++++ 15 files changed, 588 insertions(+), 22 deletions(-) create mode 100644 route-files/title-tree-store.json create mode 100644 src/Data/TitleTreeStore/PrimaryDataProvider.php create mode 100644 src/Data/TitleTreeStore/Reader.php create mode 100644 src/Data/TitleTreeStore/SecondaryDataProvider.php create mode 100644 src/Data/TitleTreeStore/Store.php create mode 100644 src/Data/TitleTreeStore/TitleTreeReaderParams.php create mode 100644 src/Data/TitleTreeStore/TitleTreeRecord.php create mode 100644 src/Data/TitleTreeStore/TitleTreeSchema.php create mode 100644 src/Rest/TitleTreeStore.php diff --git a/bootstrap.php b/bootstrap.php index 1495a85..df7d1a1 100644 --- a/bootstrap.php +++ b/bootstrap.php @@ -4,7 +4,7 @@ return; } -define( 'MWSTAKE_MEDIAWIKI_COMPONENT_COMMONWEBAPIS_VERSION', '2.0.5' ); +define( 'MWSTAKE_MEDIAWIKI_COMPONENT_COMMONWEBAPIS_VERSION', '2.0.6' ); MWStake\MediaWiki\ComponentLoader\Bootstrapper::getInstance() ->register( 'commonwebapis', static function () { diff --git a/route-files/title-tree-store.json b/route-files/title-tree-store.json new file mode 100644 index 0000000..99c527b --- /dev/null +++ b/route-files/title-tree-store.json @@ -0,0 +1,13 @@ +[ + { + "path": "/mws/v1/title-tree-store", + "class": "MWStake\\MediaWiki\\Component\\CommonWebAPIs\\Rest\\TitleTreeStore", + "services": [ + "HookContainer", + "DBLoadBalancer", + "TitleFactory", + "ContentLanguage", + "NamespaceInfo" + ] + } +] diff --git a/src/Data/TitleQueryStore/PrimaryDataProvider.php b/src/Data/TitleQueryStore/PrimaryDataProvider.php index 32bcc7c..f01edc6 100644 --- a/src/Data/TitleQueryStore/PrimaryDataProvider.php +++ b/src/Data/TitleQueryStore/PrimaryDataProvider.php @@ -125,6 +125,7 @@ protected function appendRowToData( \stdClass $row ) { TitleRecord::PAGE_DBKEY => $row->page_title, TitleRecord::PAGE_CONTENT_MODEL => $row->page_content_model, TitleRecord::IS_CONTENT_PAGE => in_array( $row->page_namespace, $this->contentNamespaces ), + TitleRecord::PAGE_EXISTS => true, ] ); } diff --git a/src/Data/TitleQueryStore/SecondaryDataProvider.php b/src/Data/TitleQueryStore/SecondaryDataProvider.php index 97cfeae..ce3a000 100644 --- a/src/Data/TitleQueryStore/SecondaryDataProvider.php +++ b/src/Data/TitleQueryStore/SecondaryDataProvider.php @@ -28,24 +28,35 @@ public function __construct( $titleFactory, \Language $language ) { public function extend( $dataSets ) { foreach ( $dataSets as $dataSet ) { $title = $this->titleFromRecord( $dataSet ); - - $dataSet->set( TitleRecord::PAGE_TITLE, $title->getText() ); - $dataSet->set( TitleRecord::PAGE_PREFIXED, $title->getPrefixedText() ); - $dataSet->set( TitleRecord::PAGE_URL, $title->getLocalURL() ); - $dataSet->set( - TitleRecord::PAGE_NAMESPACE_TEXT, $this->language->getNsText( $title->getNamespace() ) - ); + $this->extendWithTitle( $dataSet, $title ); } return $dataSets; } + /** + * @param Record $dataSet + * @param \Title $title + * + * @return void + */ + protected function extendWithTitle( Record $dataSet, \Title $title ) { + $dataSet->set( TitleRecord::PAGE_TITLE, $title->getText() ); + $dataSet->set( TitleRecord::PAGE_PREFIXED, $title->getPrefixedText() ); + $dataSet->set( TitleRecord::PAGE_URL, $title->getLocalURL() ); + $dataSet->set( + TitleRecord::PAGE_NAMESPACE_TEXT, $this->language->getNsText( $title->getNamespace() ) + ); + } + /** * @param Record $record * * @return \Title|null */ protected function titleFromRecord( $record ) { - return $this->titleFactory->newFromID( $record->get( TitleRecord::PAGE_ID ) ); + return $this->titleFactory->makeTitle( + $record->get( TitleRecord::PAGE_NAMESPACE ), $record->get( TitleRecord::PAGE_TITLE) + ); } } diff --git a/src/Data/TitleQueryStore/TitleSchema.php b/src/Data/TitleQueryStore/TitleSchema.php index 7a7f48d..7f2f6c1 100644 --- a/src/Data/TitleQueryStore/TitleSchema.php +++ b/src/Data/TitleQueryStore/TitleSchema.php @@ -6,8 +6,8 @@ use MWStake\MediaWiki\Component\DataStore\Schema; class TitleSchema extends Schema { - public function __construct() { - parent::__construct( [ + public function __construct( array $fields = [] ) { + parent::__construct( array_merge( [ TitleRecord::PAGE_ID => [ self::FILTERABLE => false, self::SORTABLE => false, @@ -58,6 +58,6 @@ public function __construct() { self::SORTABLE => true , self::TYPE => FieldType::BOOLEAN ] - ] ); + ], $fields ) ); } } diff --git a/src/Data/TitleTreeStore/PrimaryDataProvider.php b/src/Data/TitleTreeStore/PrimaryDataProvider.php new file mode 100644 index 0000000..1a69ad3 --- /dev/null +++ b/src/Data/TitleTreeStore/PrimaryDataProvider.php @@ -0,0 +1,256 @@ +nsInfo = $nsInfo; + } + + /** + * @param ReaderParams $params + * + * @return \MWStake\MediaWiki\Component\DataStore\Record[] + */ + public function makeData( $params ) { + if ( $params->getNode() !== '' ) { + $node = $params->getNode(); + $node = $this->nodeToUniqueId( $node ); + if ( $node ) { + return $this->dataFromNode( $node ); + } + } + return array_values( parent::makeData( $params ) ); + } + + /** + * @param ReaderParams $params + * + * @return array + */ + protected function makePreFilterConds( ReaderParams $params ) { + if ( $params->getQuery() !== '' ) { + $this->query = mb_strtolower( str_replace( '_', ' ', $params->getQuery() ) ); + } + return parent::makePreFilterConds( $params ); + } + + /** + * @param \stdClass $row + * + * @return void + */ + protected function appendRowToData( \stdClass $row ) { + $indexTitle = $row->mti_title; + $uniqueId = $this->getUniqueId( $row ); + if ( $this->isSubpage( $indexTitle ) ) { + if ( $this->queryMatchesSubpage( $indexTitle ) ) { + $this->insertParents( $row, $uniqueId, true ); + } + return; + } + + // Adding root pages + $this->data[$uniqueId] = $this->makeRecord( $row, $uniqueId, false, false ); + } + + /** + * @param \stdClass $row + * @param string $uniqueId + * @param bool $expanded + * @param bool $loaded + * + * @return TitleTreeRecord + */ + private function makeRecord( $row, string $uniqueId, bool $expanded, bool $loaded ) { + return new TitleTreeRecord( (object)[ + TitleTreeRecord::ID => $uniqueId, + TitleTreeRecord::PAGE_NAMESPACE => (int)$row->page_namespace, + TitleTreeRecord::PAGE_TITLE => $row->page_title, + TitleTreeRecord::PAGE_DBKEY => $row->page_title, + TitleTreeRecord::IS_CONTENT_PAGE => in_array( $row->page_namespace, $this->contentNamespaces ), + TitleTreeRecord::LEAF => false, + TitleTreeRecord::EXPANDED => $expanded, + TitleTreeRecord::LOADED => $loaded, + TitleTreeRecord::CHILDREN => property_exists( $row, 'children' ) ? $row->children : [] + ] ); + } + + /** + * @param $row + * + * @return string + */ + private function getUniqueId( $row ): string { + return (int)$row->page_namespace . ':' . $row->page_title; + } + + /** + * @param string $indexTitle + * + * @return bool + */ + private function isSubpage( string $indexTitle ): bool { + return strpos( $indexTitle, '/' ) !== false; + } + + + /** + * @param string $indexTitle + * + * @return bool + */ + private function queryMatchesSubpage( string $indexTitle ): bool { + if ( !$this->query ) { + return true; + } + $exploded = explode( '/', $indexTitle ); + // Get rid of the first element, which is the root page name + array_shift( $exploded ); + return strpos( $indexTitle, $this->query ) !== false; + } + + /** + * @param \stdClass $row + * @param string $uniqueId + * @param bool|null $fromQuery True if row comes from the query, not from traversing the tree + * + * @return void + */ + private function insertParents( \stdClass $row, string $uniqueId, ?bool $fromQuery = false ): void { + $title = $row->page_title; + $bits = explode( '/', $title ); + if ( count( $bits ) === 1 ) { + $this->data[$uniqueId] = $this->makeRecord( $row, $uniqueId, !$fromQuery, !$fromQuery ); + return; + } + array_pop( $bits ); + $parentTitle = implode( '/', $bits ); + $parentRow = new \stdClass(); + $parentRow->page_title = $parentTitle; + $parentRow->page_namespace = $row->page_namespace; + $parentRow->children = $this->getChildren( + $parentRow, + $this->makeRecord( $row, $uniqueId, !$fromQuery, !$fromQuery, ) + ); + $this->insertParents( $parentRow, $this->getUniqueId( $parentRow ) ); + } + + /** + * @param \stdClass $row + * @param TitleTreeRecord|null $loadedChild + * + * @return TitleTreeRecord[] + */ + private function getChildren( \stdClass $row, ?TitleTreeRecord $loadedChild ): array { + $childRows = $this->getSubpages( $row ); + $children = $loadedChild ? [ $loadedChild ] : []; + foreach ( $childRows as $childRow ) { + $uniqueChildId = $this->getUniqueId( $childRow ); + if ( $loadedChild && $loadedChild->get( TitleTreeRecord::ID ) === $uniqueChildId ) { + continue; + } + if ( !$this->isDirectChildOf( $row->page_title, $childRow->page_title ) ) { + continue; + } + $child = $this->makeRecord( $childRow, $uniqueChildId, false, false ); + $children[] = $child; + } + + return $children; + } + + /** + * @param \stdClass $row + * + * @return \Wikimedia\Rdbms\IResultWrapper + */ + private function getSubpages( \stdClass $row ) { + return $this->db->select( + [ 'page' ], + [ 'page_title', 'page_namespace' ], + [ + 'page_namespace' => $row->page_namespace, + 'page_title LIKE ' . $this->db->addQuotes( $row->page_title . '/%' ) + ], + __METHOD__ + ); + } + + /** + * @param $parent + * @param $child + * + * @return bool + */ + private function isDirectChildOf( $parent, $child ) { + $parentBits = explode( '/', $parent ); + $childBits = explode( '/', $child ); + if ( count( $childBits ) !== count( $parentBits ) + 1 ) { + return false; + } + for ( $i = 0; $i < count( $parentBits ); $i++ ) { + if ( $parentBits[$i] !== $childBits[$i] ) { + return false; + } + } + return true; + } + + /** + * @param string $node + * + * @return array|null + */ + private function nodeToUniqueId( string $node ): ?array { + $bits = explode( ':', $node ); + if ( count( $bits ) === 1 ) { + return '0:' . $bits[0]; + } + $ns = $bits[0]; + $nsIndex = $this->language->getNsIndex( $ns ); + if ( $nsIndex === null ) { + return null; + } + return [ + 'page_namespace' => $nsIndex, + 'page_title' => implode( ':', array_slice( $bits, 1 ) ) + ]; + } + + /** + * @param array $node + * + * @return array|TitleTreeRecord[] + */ + private function dataFromNode( array $node ): array { + $row = $this->db->selectRow( + [ 'page' ], + [ 'page_title', 'page_namespace' ], + $node, + __METHOD__ + ); + + if ( !$row ) { + return []; + } + + return $this->getChildren( $row, null ); + } +} diff --git a/src/Data/TitleTreeStore/Reader.php b/src/Data/TitleTreeStore/Reader.php new file mode 100644 index 0000000..554a46e --- /dev/null +++ b/src/Data/TitleTreeStore/Reader.php @@ -0,0 +1,42 @@ +lb->getConnection( DB_REPLICA ), $this->getSchema(), $this->language, $this->nsInfo + ); + } + + /** + * @return SecondaryDataProvider + */ + public function makeSecondaryDataProvider() { + return new SecondaryDataProvider( $this->titleFactory, $this->language ); + } +} diff --git a/src/Data/TitleTreeStore/SecondaryDataProvider.php b/src/Data/TitleTreeStore/SecondaryDataProvider.php new file mode 100644 index 0000000..3a7611b --- /dev/null +++ b/src/Data/TitleTreeStore/SecondaryDataProvider.php @@ -0,0 +1,43 @@ +extend( $dataSet->get( TitleTreeRecord::CHILDREN ) ); + } + return $extended; + } + + /** + * @param Record $dataSet + * @param \Title $title + * + * @return void + */ + protected function extendWithTitle( Record $dataSet, \Title $title ) { + parent::extendWithTitle( $dataSet, $title ); + if ( $title->getNamespace() === NS_MAIN ) { + $dataSet->set( TitleTreeRecord::ID, ':' . $title->getDBkey() ); + } else { + $dataSet->set( TitleTreeRecord::ID, $title->getPrefixedDBkey() ); + } + $dataSet->set( TitleTreeRecord::PAGE_EXISTS, $title->exists() ); + if ( + ( + $dataSet->get( TitleTreeRecord::LOADED ) && + empty( $dataSet->get( TitleTreeRecord::CHILDREN ) ) + ) || + !$title->hasSubpages() + ) { + $dataSet->set( TitleTreeRecord::LEAF, true ); + } + } +} diff --git a/src/Data/TitleTreeStore/Store.php b/src/Data/TitleTreeStore/Store.php new file mode 100644 index 0000000..b80fc42 --- /dev/null +++ b/src/Data/TitleTreeStore/Store.php @@ -0,0 +1,55 @@ +lb = $lb; + $this->titleFactory = $titleFactory; + $this->language = $language; + $this->nsInfo = $nsInfo; + } + + /** + * @return UserSchema + */ + public function getSchema() { + return new TitleSchema(); + } + + /** + * @return Reader + */ + public function getReader() { + return new Reader( + $this->lb, $this->titleFactory, $this->language, $this->nsInfo + ); + } + + /** + * @return PrimaryDataProvider + */ + public function getWriter() { + throw new NotImplementedException(); + } +} diff --git a/src/Data/TitleTreeStore/TitleTreeReaderParams.php b/src/Data/TitleTreeStore/TitleTreeReaderParams.php new file mode 100644 index 0000000..c380a12 --- /dev/null +++ b/src/Data/TitleTreeStore/TitleTreeReaderParams.php @@ -0,0 +1,34 @@ +setIfAvailable( $this->node, $params, 'node' ); + $this->setIfAvailable( $this->expandPaths, $params, 'expand-paths' ); + } + + /** + * @return string + */ + public function getNode(): string { + return $this->node; + } + + /** + * @return string[] + */ + public function getExpandPaths(): array { + return $this->expandPaths; + } +} diff --git a/src/Data/TitleTreeStore/TitleTreeRecord.php b/src/Data/TitleTreeStore/TitleTreeRecord.php new file mode 100644 index 0000000..01edc56 --- /dev/null +++ b/src/Data/TitleTreeStore/TitleTreeRecord.php @@ -0,0 +1,14 @@ + [ + self::FILTERABLE => false, + self::SORTABLE => false, + self::TYPE => FieldType::STRING + ], + TitleTreeRecord::LEAF => [ + self::FILTERABLE => false, + self::SORTABLE => false, + self::TYPE => FieldType::BOOLEAN + ], + TitleTreeRecord::EXPANDED => [ + self::FILTERABLE => false, + self::SORTABLE => false, + self::TYPE => FieldType::BOOLEAN + ], + TitleTreeRecord::LOADED => [ + self::FILTERABLE => false, + self::SORTABLE => false, + self::TYPE => FieldType::BOOLEAN + ], + TitleTreeRecord::CHILDREN => [ + self::FILTERABLE => false, + self::SORTABLE => false, + self::TYPE => FieldType::LISTVALUE + ] + ] ); + } +} diff --git a/src/Rest/QueryStore.php b/src/Rest/QueryStore.php index 143f4bc..7a41583 100644 --- a/src/Rest/QueryStore.php +++ b/src/Rest/QueryStore.php @@ -149,21 +149,21 @@ public function getParamSettings() { /** * @return int */ - private function getOffset(): int { + protected function getOffset(): int { return (int)$this->getValidatedParams()['start']; } /** * @return int */ - private function getLimit(): int { + protected function getLimit(): int { return (int)$this->getValidatedParams()['limit']; } /** * @return array */ - private function getFilter(): array { + protected function getFilter(): array { $validated = $this->getValidatedParams(); if ( is_array( $validated ) && isset( $validated['filter'] ) ) { return json_decode( $validated['filter'], 1 ); @@ -174,7 +174,7 @@ private function getFilter(): array { /** * @return array */ - private function getSort(): array { + protected function getSort(): array { $validated = $this->getValidatedParams(); if ( is_array( $validated ) && isset( $validated['sort'] ) ) { return json_decode( $validated['sort'], 1 ); @@ -185,14 +185,14 @@ private function getSort(): array { /** * @return string */ - private function getFormat(): string { + protected function getFormat(): string { return $this->getValidatedParams()['format']; } /** * @return string */ - private function getQuery(): string { + protected function getQuery(): string { return $this->getValidatedParams()['query']; } } diff --git a/src/Rest/TitleQueryStore.php b/src/Rest/TitleQueryStore.php index 7371d9d..a292888 100644 --- a/src/Rest/TitleQueryStore.php +++ b/src/Rest/TitleQueryStore.php @@ -8,8 +8,12 @@ use Wikimedia\Rdbms\ILoadBalancer; class TitleQueryStore extends QueryStore { - /** @var Store */ - private $store; + /** @var ILoadBalancer */ + protected $lb; + /** @var \TitleFactory */ + protected $titleFactory; + /** @var \Language */ + protected $language; /** * @param HookContainer $hookContainer @@ -23,13 +27,16 @@ public function __construct( \Language $language, \NamespaceInfo $nsInfo ) { parent::__construct( $hookContainer ); - $this->store = new Store( $lb, $titleFactory, $language, $nsInfo ); + $this->lb = $lb; + $this->titleFactory = $titleFactory; + $this->language = $language; + $this->nsInfo = $nsInfo; } /** * @return IStore */ protected function getStore(): IStore { - return $this->store; + return new Store( $this->lb, $this->titleFactory, $this->language, $this->nsInfo ); } } diff --git a/src/Rest/TitleTreeStore.php b/src/Rest/TitleTreeStore.php new file mode 100644 index 0000000..4c4a3f5 --- /dev/null +++ b/src/Rest/TitleTreeStore.php @@ -0,0 +1,51 @@ +lb, $this->titleFactory, $this->language, $this->nsInfo ); + } + + /** + * @return ReaderParams + */ + protected function getReaderParams(): ReaderParams { + return new TitleTreeReaderParams( [ + 'query' => $this->getQuery(), + 'start' => $this->getOffset(), + 'limit' => $this->getLimit(), + 'filter' => $this->getFilter(), + 'sort' => $this->getSort(), + 'node' => $this->getValidatedParams()['node'] ?? '', + 'expand-paths' => $this->getValidatedParams()['expand-paths'] ?? [], + ] ); + } + + protected function getStoreSpecificParams() : array { + return [ + 'node' => [ + static::PARAM_SOURCE => 'query', + ParamValidator::PARAM_REQUIRED => false, + ParamValidator::PARAM_TYPE => 'string', + ], + 'expand-paths' => [ + static::PARAM_SOURCE => 'query', + ParamValidator::PARAM_REQUIRED => false, + ParamValidator::PARAM_TYPE => 'string', + ], + ]; + } +}