4 * Reads lines of work from an input stream and farms them out to multiple
5 * child streams. Each child has exactly one piece of work in flight at a given
6 * moment. Writes the result of child's work to an output stream. If numProcs
7 * <= zero the work will be performed in process.
9 * This class amends ForkController with the requirement that the output is
10 * produced in the same exact order as input values were.
12 * Currently used by CirrusSearch extension to implement CLI search script.
14 * @ingroup Maintenance
17 * This program is free software; you can redistribute it and/or modify
18 * it under the terms of the GNU General Public License as published by
19 * the Free Software Foundation; either version 2 of the License, or
20 * (at your option) any later version.
22 * This program is distributed in the hope that it will be useful,
23 * but WITHOUT ANY WARRANTY; without even the implied warranty of
24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * GNU General Public License for more details.
27 * You should have received a copy of the GNU General Public License along
28 * with this program; if not, write to the Free Software Foundation, Inc.,
29 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
30 * http://www.gnu.org/copyleft/gpl.html
32 class OrderedStreamingForkController
extends ForkController
{
34 protected $workCallback;
40 protected $nextOutputId;
41 /** @var string[] Int key indicates order, value is data */
42 protected $delayedOutputData = [];
45 * @param int $numProcs The number of worker processes to fork
46 * @param callable $workCallback A callback to call in the child process
47 * once for each line of work to process.
48 * @param resource $input A socket to read work lines from
49 * @param resource $output A socket to write the result of work to.
51 public function __construct( $numProcs, $workCallback, $input, $output ) {
52 parent
::__construct( $numProcs );
53 $this->workCallback
= $workCallback;
54 $this->input
= $input;
55 $this->output
= $output;
61 public function start() {
62 if ( $this->procsToStart
> 0 ) {
63 $status = parent
::start();
64 if ( $status === 'child' ) {
69 $this->consumeNoFork();
75 * @param int $numProcs
78 protected function forkWorkers( $numProcs ) {
79 $this->prepareEnvironment();
82 // Create the child processes
83 for ( $i = 0; $i < $numProcs; $i++
) {
84 $sockets = stream_socket_pair( STREAM_PF_UNIX
, STREAM_SOCK_STREAM
, STREAM_IPPROTO_IP
);
87 if ( $pid === -1 ||
$pid === false ) {
88 echo "Error creating child processes\n";
94 $this->childNumber
= $i;
95 $this->input
= $sockets[0];
96 $this->output
= $sockets[0];
97 fclose( $sockets[1] );
100 // This is the parent process
101 $this->children
[$pid] = true;
102 fclose( $sockets[0] );
103 $childSockets[] = $sockets[1];
106 $this->feedChildren( $childSockets );
107 foreach ( $childSockets as $socket ) {
108 // if a child has already shutdown the sockets will be closed,
109 // closing a second time would raise a warning.
110 if ( is_resource( $socket ) ) {
118 * Child worker process. Reads work from $this->input and writes the
119 * result of that work to $this->output when completed.
121 protected function consume() {
122 while ( !feof( $this->input
) ) {
123 $line = trim( fgets( $this->input
) );
125 list( $id, $data ) = json_decode( $line );
126 $result = call_user_func( $this->workCallback
, $data );
127 fwrite( $this->output
, json_encode( [ $id, $result ] ) . "\n" );
133 * Special cased version of self::consume() when no forking occurs
135 protected function consumeNoFork() {
136 while ( !feof( $this->input
) ) {
137 $data = fgets( $this->input
);
138 if ( substr( $data, -1 ) === "\n" ) {
139 // Strip any final new line used to delimit lines of input.
140 // The last line of input might not have it, though.
141 $data = substr( $data, 0, -1 );
143 if ( $data === '' ) {
146 $result = call_user_func( $this->workCallback
, $data );
147 fwrite( $this->output
, "$result\n" );
152 * Reads lines of work from $this->input and farms them out to
153 * the provided socket.
155 * @param resource[] $sockets
157 protected function feedChildren( array $sockets ) {
160 $this->nextOutputId
= 0;
162 while ( !feof( $this->input
) ) {
163 $data = fgets( $this->input
);
166 $this->updateAvailableSockets( $sockets, $used, $sockets ?
0 : 5 );
167 } while ( !$sockets );
169 if ( substr( $data, - 1 ) === "\n" ) {
170 // Strip any final new line used to delimit lines of input.
171 // The last line of input might not have it, though.
172 $data = substr( $data, 0, -1 );
174 if ( $data === '' ) {
177 $socket = array_pop( $sockets );
178 fwrite( $socket, json_encode( [ $id++
, $data ] ) . "\n" );
182 $this->updateAvailableSockets( $sockets, $used, 5 );
187 * Moves sockets from $used to $sockets when they are available
190 * @param resource[] &$sockets List of sockets that are waiting for work
191 * @param resource[] &$used List of sockets currently performing work
192 * @param int $timeout The number of seconds to block waiting. 0 for
193 * non-blocking operation.
195 protected function updateAvailableSockets( &$sockets, &$used, $timeout ) {
197 $write = $except = [];
198 stream_select( $read, $write, $except, $timeout );
199 foreach ( $read as $socket ) {
200 $line = fgets( $socket );
201 list( $id, $data ) = json_decode( trim( $line ) );
202 $this->receive( (int)$id, $data );
203 $sockets[] = $socket;
204 $idx = array_search( $socket, $used );
205 unset( $used[$idx] );
211 * @param string $data
213 protected function receive( $id, $data ) {
214 if ( $id !== $this->nextOutputId
) {
215 $this->delayedOutputData
[$id] = $data;
218 fwrite( $this->output
, $data . "\n" );
219 $this->nextOutputId
= $id +
1;
220 while ( isset( $this->delayedOutputData
[$this->nextOutputId
] ) ) {
221 fwrite( $this->output
, $this->delayedOutputData
[$this->nextOutputId
] . "\n" );
222 unset( $this->delayedOutputData
[$this->nextOutputId
] );
223 $this->nextOutputId++
;