tests/phpunit/includes/search/SearchEngineTest.php

   1 <?php
   2
   3 use MediaWiki\Content\WikitextContent;
   4 use MediaWiki\MainConfigNames;
   5
   6 /**
   7  * @group Search
   8  * @group Database
   9  *
  10  * @covers \SearchEngine<extended>
  11  * @note Coverage will only ever show one of on of the Search* classes
  12  */
  13 class SearchEngineTest extends MediaWikiLangTestCase {
  14
  15         /**
  16          * @var SearchEngine
  17          */
  18         protected $search;
  19
  20         /**
  21          * Checks for database type & version.
  22          * Will skip current test if DB does not support search.
  23          */
  24         protected function setUp(): void {
  25                 parent::setUp();
  26
  27                 // Search tests require MySQL or SQLite with FTS
  28                 $dbType = $this->getDb()->getType();
  29                 $dbSupported = ( $dbType === 'mysql' )
  30                         || ( $dbType === 'sqlite' && $this->getDb()->getFulltextSearchModule() == 'FTS3' );
  31
  32                 if ( !$dbSupported ) {
  33                         $this->markTestSkipped( "MySQL or SQLite with FTS3 only" );
  34                 }
  35                 $dbProvider = $this->getServiceContainer()->getConnectionProvider();
  36
  37                 $searchType = SearchEngineFactory::getSearchEngineClass( $dbProvider );
  38                 $this->overrideConfigValues( [
  39                         MainConfigNames::SearchType => $searchType,
  40                         MainConfigNames::CapitalLinks => true,
  41                         MainConfigNames::CapitalLinkOverrides => [
  42                                 NS_CATEGORY => false // for testCompletionSearchMustRespectCapitalLinkOverrides
  43                         ],
  44                 ] );
  45
  46                 $this->search = new $searchType( $dbProvider );
  47                 $this->search->setHookContainer( $this->getServiceContainer()->getHookContainer() );
  48         }
  49
  50         protected function tearDown(): void {
  51                 unset( $this->search );
  52
  53                 parent::tearDown();
  54         }
  55
  56         public function addDBDataOnce() {
  57                 if ( !$this->isWikitextNS( NS_MAIN ) ) {
  58                         // @todo cover the case of non-wikitext content in the main namespace
  59                         return;
  60                 }
  61
  62                 // Reset the search type back to default - some extensions may have
  63                 // overridden it.
  64                 $this->overrideConfigValues( [
  65                         MainConfigNames::SearchType => null,
  66                         MainConfigNames::CapitalLinks => true,
  67                         MainConfigNames::CapitalLinkOverrides => [
  68                                 NS_CATEGORY => false // for testCompletionSearchMustRespectCapitalLinkOverrides
  69                         ],
  70                 ] );
  71
  72                 $this->insertPage( 'Not_Main_Page', 'This is not a main page' );
  73                 $this->insertPage(
  74                         'Talk:Not_Main_Page',
  75                         'This is not a talk page to the main page, see [[smithee]]'
  76                 );
  77                 $this->insertPage( 'Smithee', 'A smithee is one who smiths. See also [[Alan Smithee]]' );
  78                 $this->insertPage( 'Talk:Smithee', 'This article sucks.' );
  79                 $this->insertPage( 'Unrelated_page', 'Nothing in this page is about the S word.' );
  80                 $this->insertPage( 'Another_page', 'This page also is unrelated.' );
  81                 $this->insertPage( 'Help:Help', 'Help me!' );
  82                 $this->insertPage( 'Thppt', 'Blah blah' );
  83                 $this->insertPage( 'Alan_Smithee', 'yum' );
  84                 $this->insertPage( 'Pages', 'are\'food' );
  85                 $this->insertPage( 'HalfOneUp', 'AZ' );
  86                 $this->insertPage( 'FullOneUp', 'ＡＺ' );
  87                 $this->insertPage( 'HalfTwoLow', 'az' );
  88                 $this->insertPage( 'FullTwoLow', 'ａｚ' );
  89                 $this->insertPage( 'HalfNumbers', '1234567890' );
  90                 $this->insertPage( 'FullNumbers', '１２３４５６７８９０' );
  91                 $this->insertPage( 'DomainName', 'example.com' );
  92                 $this->insertPage( 'DomainName', 'example.com' );
  93                 $this->insertPage( 'Category:search is not Search', '' );
  94                 $this->insertPage( 'Category:Search is not search', '' );
  95                 $this->insertPage( 'Talk:1', 'Did you know titles can be numbers?' );
  96         }
  97
  98         protected function fetchIds( $results ) {
  99                 if ( !$this->isWikitextNS( NS_MAIN ) ) {
 100                         $this->markTestIncomplete( __CLASS__ . " does no yet support non-wikitext content "
 101                                 . "in the main namespace" );
 102                 }
 103                 $this->assertIsObject( $results );
 104
 105                 $matches = [];
 106                 foreach ( $results as $row ) {
 107                         $matches[] = $row->getTitle()->getPrefixedText();
 108                 }
 109                 $results->free();
 110                 # Search is not guaranteed to return results in a certain order;
 111                 # sort them numerically so we will compare simply that we received
 112                 # the expected matches.
 113                 sort( $matches );
 114
 115                 return $matches;
 116         }
 117
 118         public function testFullWidth() {
 119                 // T303046
 120                 $this->markTestSkippedIfDbType( 'sqlite' );
 121                 $this->assertEquals(
 122                         [ 'FullOneUp', 'FullTwoLow', 'HalfOneUp', 'HalfTwoLow' ],
 123                         $this->fetchIds( $this->search->searchText( 'AZ' ) ),
 124                         "Search for normalized from Half-width Upper" );
 125                 $this->assertEquals(
 126                         [ 'FullOneUp', 'FullTwoLow', 'HalfOneUp', 'HalfTwoLow' ],
 127                         $this->fetchIds( $this->search->searchText( 'az' ) ),
 128                         "Search for normalized from Half-width Lower" );
 129                 $this->assertEquals(
 130                         [ 'FullOneUp', 'FullTwoLow', 'HalfOneUp', 'HalfTwoLow' ],
 131                         $this->fetchIds( $this->search->searchText( 'ＡＺ' ) ),
 132                         "Search for normalized from Full-width Upper" );
 133                 $this->assertEquals(
 134                         [ 'FullOneUp', 'FullTwoLow', 'HalfOneUp', 'HalfTwoLow' ],
 135                         $this->fetchIds( $this->search->searchText( 'ａｚ' ) ),
 136                         "Search for normalized from Full-width Lower" );
 137         }
 138
 139         public function testTextSearch() {
 140                 // T303046
 141                 $this->markTestSkippedIfDbType( 'sqlite' );
 142                 $this->assertEquals(
 143                         [ 'Smithee' ],
 144                         $this->fetchIds( $this->search->searchText( 'smithee' ) ),
 145                         "Plain search" );
 146         }
 147
 148         public function testWildcardSearch() {
 149                 // T303046
 150                 $this->markTestSkippedIfDbType( 'sqlite' );
 151                 $res = $this->search->searchText( 'smith*' );
 152                 $this->assertEquals(
 153                         [ 'Smithee' ],
 154                         $this->fetchIds( $res ),
 155                         "Search with wildcards" );
 156
 157                 $res = $this->search->searchText( 'smithson*' );
 158                 $this->assertEquals(
 159                         [],
 160                         $this->fetchIds( $res ),
 161                         "Search with wildcards must not find unrelated articles" );
 162
 163                 $res = $this->search->searchText( 'smith* smithee' );
 164                 $this->assertEquals(
 165                         [ 'Smithee' ],
 166                         $this->fetchIds( $res ),
 167                         "Search with wildcards can be combined with simple terms" );
 168
 169                 $res = $this->search->searchText( 'smith* "one who smiths"' );
 170                 $this->assertEquals(
 171                         [ 'Smithee' ],
 172                         $this->fetchIds( $res ),
 173                         "Search with wildcards can be combined with phrase search" );
 174         }
 175
 176         public function testPhraseSearch() {
 177                 // T303046
 178                 $this->markTestSkippedIfDbType( 'sqlite' );
 179                 $res = $this->search->searchText( '"smithee is one who smiths"' );
 180                 $this->assertEquals(
 181                         [ 'Smithee' ],
 182                         $this->fetchIds( $res ),
 183                         "Search a phrase" );
 184
 185                 $res = $this->search->searchText( '"smithee is who smiths"' );
 186                 $this->assertEquals(
 187                         [],
 188                         $this->fetchIds( $res ),
 189                         "Phrase search is not sloppy, search terms must be adjacent" );
 190
 191                 $res = $this->search->searchText( '"is smithee one who smiths"' );
 192                 $this->assertEquals(
 193                         [],
 194                         $this->fetchIds( $res ),
 195                         "Phrase search is ordered" );
 196         }
 197
 198         public function testPhraseSearchHighlight() {
 199                 // T303046
 200                 $this->markTestSkippedIfDbType( 'sqlite' );
 201                 $phrase = "smithee is one who smiths";
 202                 $res = $this->search->searchText( "\"$phrase\"" );
 203                 $match = $res->getIterator()->current();
 204                 $snippet = 'A <span class="searchmatch">' . $phrase . '</span>';
 205                 $this->assertStringStartsWith( $snippet,
 206                         $match->getTextSnippet(),
 207                         "Highlight a phrase search" );
 208         }
 209
 210         public function testTextPowerSearch() {
 211                 // T303046
 212                 $this->markTestSkippedIfDbType( 'sqlite' );
 213                 $this->search->setNamespaces( [ 0, 1, 4 ] );
 214                 $this->assertEquals(
 215                         [
 216                                 'Smithee',
 217                                 'Talk:Not Main Page',
 218                         ],
 219                         $this->fetchIds( $this->search->searchText( 'smithee' ) ),
 220                         "Power search" );
 221         }
 222
 223         public function testTitleSearch() {
 224                 // T303046
 225                 $this->markTestSkippedIfDbType( 'sqlite' );
 226                 $this->assertEquals(
 227                         [
 228                                 'Alan Smithee',
 229                                 'Smithee',
 230                         ],
 231                         $this->fetchIds( $this->search->searchTitle( 'smithee' ) ),
 232                         "Title search" );
 233         }
 234
 235         public function testTextTitlePowerSearch() {
 236                 // T303046
 237                 $this->markTestSkippedIfDbType( 'sqlite' );
 238                 $this->search->setNamespaces( [ 0, 1, 4 ] );
 239                 $this->assertEquals(
 240                         [
 241                                 'Alan Smithee',
 242                                 'Smithee',
 243                                 'Talk:Smithee',
 244                         ],
 245                         $this->fetchIds( $this->search->searchTitle( 'smithee' ) ),
 246                         "Title power search" );
 247         }
 248
 249         public static function provideCompletionSearchMustRespectCapitalLinkOverrides() {
 250                 return [
 251                         'Searching for "smithee" finds Smithee on NS_MAIN' => [
 252                                 'smithee',
 253                                 'Smithee',
 254                                 [ NS_MAIN ],
 255                         ],
 256                         'Searching for "search is" will finds "search is not Search" on NS_CATEGORY' => [
 257                                 'search is',
 258                                 'Category:search is not Search',
 259                                 [ NS_CATEGORY ],
 260                         ],
 261                         'Searching for "Search is" will finds "search is not Search" on NS_CATEGORY' => [
 262                                 'Search is',
 263                                 'Category:Search is not search',
 264                                 [ NS_CATEGORY ],
 265                         ],
 266                         'Copy-pasted wikilinks with invalid characters will still find the page' => [
 267                                 '[[smithee]]',
 268                                 'Smithee',
 269                                 [ NS_MAIN ],
 270                         ],
 271                         'Numeric title works (T365565)' => [
 272                                 '1',
 273                                 'Talk:1',
 274                                 [ NS_TALK ],
 275                         ]
 276                 ];
 277         }
 278
 279         /**
 280          * Test that the search query is not munged using wrong CapitalLinks setup
 281          * (in other test that the default search backend can benefit from wgCapitalLinksOverride)
 282          * Guard against regressions like T208255
 283          * @dataProvider provideCompletionSearchMustRespectCapitalLinkOverrides
 284          * @covers \SearchEngine::completionSearch
 285          * @covers \PrefixSearch::defaultSearchBackend
 286          * @param string $search
 287          * @param string $expectedSuggestion
 288          * @param int[] $namespaces
 289          */
 290         public function testCompletionSearchMustRespectCapitalLinkOverrides(
 291                 $search,
 292                 $expectedSuggestion,
 293                 array $namespaces
 294         ) {
 295                 $this->search->setNamespaces( $namespaces );
 296                 $results = $this->search->completionSearch( $search );
 297                 $this->assertSame( 1, $results->getSize() );
 298                 $this->assertEquals( $expectedSuggestion, $results->getSuggestions()[0]->getText() );
 299         }
 300
 301         /**
 302          * @covers \SearchEngine::getSearchIndexFields
 303          */
 304         public function testSearchIndexFields() {
 305                 /**
 306                  * @var SearchEngine $mockEngine
 307                  */
 308                 $mockEngine = $this->getMockBuilder( SearchEngine::class )
 309                         ->onlyMethods( [ 'makeSearchFieldMapping' ] )->getMock();
 310
 311                 $mockFieldBuilder = function ( $name, $type ) {
 312                         $mockField =
 313                                 $this->getMockBuilder( SearchIndexFieldDefinition::class )->setConstructorArgs( [
 314                                         $name,
 315                                         $type,
 316                                 ] )->getMock();
 317
 318                         $mockField->method( 'getMapping' )->willReturn( [
 319                                 'testData' => 'test',
 320                                 'name' => $name,
 321                                 'type' => $type,
 322                         ] );
 323
 324                         $mockField->method( 'merge' )->willReturnSelf();
 325
 326                         return $mockField;
 327                 };
 328
 329                 $mockEngine->expects( $this->atLeastOnce() )
 330                         ->method( 'makeSearchFieldMapping' )
 331                         ->willReturnCallback( $mockFieldBuilder );
 332
 333                 // Not using mock since PHPUnit mocks do not work properly with references in params
 334                 $this->setTemporaryHook( 'SearchIndexFields',
 335                         static function ( &$fields, SearchEngine $engine ) use ( $mockFieldBuilder ) {
 336                                 $fields['testField'] =
 337                                         $mockFieldBuilder( "testField", SearchIndexField::INDEX_TYPE_TEXT );
 338                                 return true;
 339                         } );
 340                 $mockEngine->setHookContainer( $this->getServiceContainer()->getHookContainer() );
 341
 342                 $fields = $mockEngine->getSearchIndexFields();
 343                 $this->assertArrayHasKey( 'language', $fields );
 344                 $this->assertArrayHasKey( 'category', $fields );
 345                 $this->assertInstanceOf( SearchIndexField::class, $fields['testField'] );
 346
 347                 $mapping = $fields['testField']->getMapping( $mockEngine );
 348                 $this->assertArrayHasKey( 'testData', $mapping );
 349                 $this->assertEquals( 'test', $mapping['testData'] );
 350         }
 351
 352         public function hookSearchIndexFields( $mockFieldBuilder, &$fields, SearchEngine $engine ) {
 353                 $fields['testField'] = $mockFieldBuilder( "testField", SearchIndexField::INDEX_TYPE_TEXT );
 354                 return true;
 355         }
 356
 357         public function testAugmentorSearch() {
 358                 // T303046
 359                 $this->markTestSkippedIfDbType( 'sqlite' );
 360
 361                 $this->search->setHookContainer(
 362                         $this->createHookContainer( [ 'SearchResultsAugment' => [ $this, 'addAugmentors' ] ] )
 363                 );
 364
 365                 $this->search->setNamespaces( [ 0, 1, 4 ] );
 366                 $resultSet = $this->search->searchText( 'smithee' );
 367                 $this->search->augmentSearchResults( $resultSet );
 368                 foreach ( $resultSet as $result ) {
 369                         $id = $result->getTitle()->getArticleID();
 370                         $augmentData = "Result:$id:" . $result->getTitle()->getText();
 371                         $augmentData2 = "Result2:$id:" . $result->getTitle()->getText();
 372                         $this->assertEquals( [ 'testSet' => $augmentData, 'testRow' => $augmentData2 ],
 373                                 $result->getExtensionData() );
 374                 }
 375         }
 376
 377         public function addAugmentors( &$setAugmentors, &$rowAugmentors ) {
 378                 $setAugmentor = $this->createMock( ResultSetAugmentor::class );
 379                 $setAugmentor->expects( $this->once() )
 380                         ->method( 'augmentAll' )
 381                         ->willReturnCallback( static function ( ISearchResultSet $resultSet ) {
 382                                 $data = [];
 383                                 /** @var SearchResult $result */
 384                                 foreach ( $resultSet as $result ) {
 385                                         $id = $result->getTitle()->getArticleID();
 386                                         $data[$id] = "Result:$id:" . $result->getTitle()->getText();
 387                                 }
 388                                 return $data;
 389                         } );
 390                 $setAugmentors['testSet'] = $setAugmentor;
 391
 392                 $rowAugmentor = $this->createMock( ResultAugmentor::class );
 393                 $rowAugmentor->expects( $this->exactly( 2 ) )
 394                         ->method( 'augment' )
 395                         ->willReturnCallback( static function ( SearchResult $result ) {
 396                                 $id = $result->getTitle()->getArticleID();
 397                                 return "Result2:$id:" . $result->getTitle()->getText();
 398                         } );
 399                 $rowAugmentors['testRow'] = $rowAugmentor;
 400         }
 401
 402         public function testFiltersMissing() {
 403                 $availableResults = [];
 404                 $user = $this->getTestSysop()->getAuthority();
 405                 foreach ( range( 0, 11 ) as $i ) {
 406                         $title = "Search_Result_$i";
 407                         $availableResults[] = $title;
 408                         // pages not created must be filtered
 409                         if ( $i % 2 == 0 ) {
 410                                 $this->editPage(
 411                                         $title,
 412                                         new WikitextContent( 'TestFiltersMissing content' ),
 413                                         'TestFiltersMissing summary',
 414                                         NS_MAIN,
 415                                         $user
 416                                 );
 417                         }
 418                 }
 419                 MockCompletionSearchEngine::addMockResults( 'foo', $availableResults );
 420
 421                 $engine = new MockCompletionSearchEngine();
 422                 $engine->setLimitOffset( 10, 0 );
 423                 $engine->setHookContainer( $this->getServiceContainer()->getHookContainer() );
 424                 $results = $engine->completionSearch( 'foo' );
 425                 $this->assertEquals( 5, $results->getSize() );
 426                 $this->assertTrue( $results->hasMoreResults() );
 427
 428                 $engine->setLimitOffset( 10, 10 );
 429                 $results = $engine->completionSearch( 'foo' );
 430                 $this->assertSame( 1, $results->getSize() );
 431                 $this->assertFalse( $results->hasMoreResults() );
 432         }
 433
 434         public static function provideDataForParseNamespacePrefix() {
 435                 return [
 436                         'noop' => [
 437                                 [
 438                                         'query' => 'foo',
 439                                 ],
 440                                 false,
 441                         ],
 442                         'empty' => [
 443                                 [
 444                                         'query' => '',
 445                                 ],
 446                                 false,
 447                         ],
 448                         'namespace prefix' => [
 449                                 [
 450                                         'query' => 'help:test',
 451                                 ],
 452                                 [ 'test', [ NS_HELP ] ],
 453                         ],
 454                         'accented namespace prefix with hook' => [
 455                                 [
 456                                         'query' => 'hélp:test',
 457                                         'withHook' => true,
 458                                 ],
 459                                 [ 'test', [ NS_HELP ] ],
 460                         ],
 461                         'accented namespace prefix without hook' => [
 462                                 [
 463                                         'query' => 'hélp:test',
 464                                         'withHook' => false,
 465                                 ],
 466                                 false,
 467                         ],
 468                         'all with all keyword allowed' => [
 469                                 [
 470                                         'query' => 'all:test',
 471                                         'withAll' => true,
 472                                 ],
 473                                 [ 'test', null ],
 474                         ],
 475                         'all with all keyword disallowed' => [
 476                                 [
 477                                         'query' => 'all:test',
 478                                         'withAll' => false,
 479                                 ],
 480                                 false,
 481                         ],
 482                         'ns only' => [
 483                                 [
 484                                         'query' => 'help:',
 485                                 ],
 486                                 [ '', [ NS_HELP ] ],
 487                         ],
 488                         'all only' => [
 489                                 [
 490                                         'query' => 'all:',
 491                                         'withAll' => true,
 492                                 ],
 493                                 [ '', null ],
 494                         ],
 495                         'all wins over namespace when first' => [
 496                                 [
 497                                         'query' => 'all:help:test',
 498                                         'withAll' => true,
 499                                 ],
 500                                 [ 'help:test', null ],
 501                         ],
 502                         'ns wins over all when first' => [
 503                                 [
 504                                         'query' => 'help:all:test',
 505                                         'withAll' => true,
 506                                 ],
 507                                 [ 'all:test', [ NS_HELP ] ],
 508                         ],
 509                 ];
 510         }
 511
 512         /**
 513          * @dataProvider provideDataForParseNamespacePrefix
 514          */
 515         public function testParseNamespacePrefix( array $params, $expected ) {
 516                 $this->setTemporaryHook( 'PrefixSearchExtractNamespace', static function ( &$namespaces, &$query ) {
 517                         if ( str_starts_with( $query, 'hélp:' ) ) {
 518                                 $namespaces = [ NS_HELP ];
 519                                 $query = substr( $query, strlen( 'hélp:' ) );
 520                         }
 521                         return false;
 522                 } );
 523                 $testSet = [];
 524                 if ( isset( $params['withAll'] ) && isset( $params['withHook'] ) ) {
 525                         $testSet[] = $params;
 526                 } elseif ( isset( $params['withAll'] ) ) {
 527                         $testSet[] = $params + [ 'withHook' => true ];
 528                         $testSet[] = $params + [ 'withHook' => false ];
 529                 } elseif ( isset( $params['withHook'] ) ) {
 530                         $testSet[] = $params + [ 'withAll' => true ];
 531                         $testSet[] = $params + [ 'withAll' => false ];
 532                 } else {
 533                         $testSet[] = $params + [ 'withAll' => true, 'withHook' => true ];
 534                         $testSet[] = $params + [ 'withAll' => true, 'withHook' => false ];
 535                         $testSet[] = $params + [ 'withAll' => false, 'withHook' => false ];
 536                         $testSet[] = $params + [ 'withAll' => true, 'withHook' => false ];
 537                 }
 538
 539                 foreach ( $testSet as $test ) {
 540                         $actual = SearchEngine::parseNamespacePrefixes( $test['query'],
 541                                 $test['withAll'], $test['withHook'] );
 542                         $this->assertEquals( $expected, $actual, 'with params: ' . print_r( $test, true ) );
 543                 }
 544         }
 545 }