3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License as published by
5 * the Free Software Foundation; either version 2 of the License, or
6 * (at your option) any later version.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License along
14 * with this program; if not, write to the Free Software Foundation, Inc.,
15 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 * http://www.gnu.org/copyleft/gpl.html
19 use Wikimedia\Purtle\RdfWriter
;
20 use Wikimedia\Purtle\RdfWriterFactory
;
21 use Wikimedia\Rdbms\IDatabase
;
23 require_once __DIR__
. '/Maintenance.php';
26 * Maintenance script to provide RDF representation of the category tree.
28 * @ingroup Maintenance
31 class DumpCategoriesAsRdf
extends Maintenance
{
37 * Categories RDF helper.
40 private $categoriesRdf;
42 public function __construct() {
43 parent
::__construct();
45 $this->addDescription( "Generate RDF dump of categories in a wiki." );
47 $this->setBatchSize( 200 );
48 $this->addOption( 'output', "Output file (default is stdout). Will be overwritten.",
50 $this->addOption( 'format', "Set the dump format.", false, true );
54 * Produce row iterator for categories.
55 * @param IDatabase $dbr Database connection
56 * @param string $fname Name of the calling function
57 * @return RecursiveIterator
59 public function getCategoryIterator( IDatabase
$dbr, $fname ) {
60 $it = new BatchRowIterator(
62 [ 'page', 'page_props', 'category' ],
67 'page_namespace' => NS_CATEGORY
,
69 $it->setFetchColumns( [
77 $it->addJoinConditions(
80 'LEFT JOIN', [ 'pp_propname' => 'hiddencat', 'pp_page = page_id' ]
83 'LEFT JOIN', [ 'cat_title = page_title' ]
88 $it->setCaller( $fname );
93 * Get iterator for links for categories.
94 * @param IDatabase $dbr
95 * @param int[] $ids List of page IDs
96 * @param string $fname Name of the calling function
99 public function getCategoryLinksIterator( IDatabase
$dbr, array $ids, $fname ) {
100 $it = new BatchRowIterator(
103 [ 'cl_from', 'cl_to' ],
104 $this->getBatchSize()
106 $it->addConditions( [
107 'cl_type' => 'subcat',
110 $it->setFetchColumns( [ 'cl_from', 'cl_to' ] );
111 $it->setCaller( $fname );
112 return new RecursiveIteratorIterator( $it );
116 * @param int $timestamp
118 public function addDumpHeader( $timestamp ) {
120 $licenseUrl = $wgRightsUrl;
121 if ( substr( $licenseUrl, 0, 2 ) == '//' ) {
122 $licenseUrl = 'https:' . $licenseUrl;
124 $this->rdfWriter
->about( $this->categoriesRdf
->getDumpURI() )
125 ->a( 'schema', 'Dataset' )
126 ->a( 'owl', 'Ontology' )
127 ->say( 'cc', 'license' )->is( $licenseUrl )
128 ->say( 'schema', 'softwareVersion' )->value( CategoriesRdf
::FORMAT_VERSION
)
129 ->say( 'schema', 'dateModified' )
130 ->value( wfTimestamp( TS_ISO_8601
, $timestamp ), 'xsd', 'dateTime' )
131 ->say( 'schema', 'isPartOf' )->is( wfExpandUrl( '/', PROTO_CANONICAL
) )
132 ->say( 'owl', 'imports' )->is( CategoriesRdf
::OWL_URL
);
135 public function execute() {
136 $outFile = $this->getOption( 'output', 'php://stdout' );
138 if ( $outFile === '-' ) {
139 $outFile = 'php://stdout';
142 $output = fopen( $outFile, 'w' );
143 $this->rdfWriter
= $this->createRdfWriter( $this->getOption( 'format', 'ttl' ) );
144 $this->categoriesRdf
= new CategoriesRdf( $this->rdfWriter
);
146 $this->categoriesRdf
->setupPrefixes();
147 $this->rdfWriter
->start();
149 $this->addDumpHeader( time() );
150 fwrite( $output, $this->rdfWriter
->drain() );
152 $dbr = $this->getDB( DB_REPLICA
, [ 'vslow' ] );
154 foreach ( $this->getCategoryIterator( $dbr, __METHOD__
) as $batch ) {
156 foreach ( $batch as $row ) {
157 $this->categoriesRdf
->writeCategoryData(
159 $row->pp_propname
=== 'hiddencat',
160 (int)$row->cat_pages
- (int)$row->cat_subcats
- (int)$row->cat_files
,
161 (int)$row->cat_subcats
163 if ( $row->page_id
) {
164 $pages[$row->page_id
] = $row->page_title
;
168 foreach ( $this->getCategoryLinksIterator( $dbr, array_keys( $pages ), __METHOD__
) as $row ) {
169 $this->categoriesRdf
->writeCategoryLinkData( $pages[$row->cl_from
], $row->cl_to
);
171 fwrite( $output, $this->rdfWriter
->drain() );
174 if ( $outFile !== '-' ) {
180 * @param string $format Writer format
183 private function createRdfWriter( $format ) {
184 $factory = new RdfWriterFactory();
185 return $factory->getWriter( $factory->getFormatName( $format ) );
189 $maintClass = DumpCategoriesAsRdf
::class;
190 require_once RUN_MAINTENANCE_IF_MAIN
;