From 7e4608120e53acadc61ea0dd51bcda3a30e98fd4 Mon Sep 17 00:00:00 2001 From: Ed Sanders Date: Tue, 14 Dec 2021 18:51:17 +0000 Subject: [PATCH] Create wikis in a background process * The form now points to start.php, which converts allowed POST data to environment variables and sends them to new.php * The output is written to log files in log/*.html which are then polled by the JS for new content every second * Logs are only deleted when the wiki is deleted, allowing for debugging and inspection by other users. Fixes #399, #336 --- deletewiki.sh | 2 + includes.php | 41 +++++++++-- index.php | 7 +- js/new.js | 28 ------- js/start.js | 79 ++++++++++++++++++++ log.php | 21 ++++++ new.php | 197 +++++++++----------------------------------------- setup.sh | 3 + start.php | 125 ++++++++++++++++++++++++++++++++ 9 files changed, 304 insertions(+), 199 deletions(-) delete mode 100644 js/new.js create mode 100644 js/start.js create mode 100644 log.php create mode 100644 start.php diff --git a/deletewiki.sh b/deletewiki.sh index 0284b11e..78709745 100755 --- a/deletewiki.sh +++ b/deletewiki.sh @@ -3,5 +3,7 @@ set -ex rm -rf $PATCHDEMO/wikis/$WIKI +rm $PATCHDEMO/logs/$WIKI.html + # delete database mysql -u patchdemo --password=patchdemo -e "DROP DATABASE IF EXISTS patchdemo_$WIKI"; diff --git a/includes.php b/includes.php index 9ef3eba3..c930b720 100644 --- a/includes.php +++ b/includes.php @@ -316,7 +316,17 @@ function shell( $cmd, array $env = [] ): ?string { return $error ? null : $process->getOutput(); } +/** + * Can't be called from CLI, use delete_wiki_cli_safe + * + * @param string $wiki Wiki name + * @return int Error code + */ function delete_wiki( string $wiki ): int { + return delete_wiki_cli_safe( $wiki, get_server() . get_server_path() ); +} + +function delete_wiki_cli_safe( string $wiki, string $serverUri ): int { global $mysqli; $wikiData = get_wiki_data( $wiki ); @@ -333,16 +343,12 @@ function delete_wiki( string $wiki ): int { ); foreach ( $wikiData['announcedTasks'] as $task ) { - // TODO: Deduplicate server/serverPath with variables in new.php - $server = detectProtocol() . '://' . $_SERVER['HTTP_HOST']; - $serverPath = preg_replace( '`/[^/]*$`', '', $_SERVER['REQUEST_URI'] ); - $creator = $wikiData['creator']; post_phab_comment( 'T' . $task, - "Test wiki on [[ $server$serverPath | Patch demo ]] " . ( $creator ? ' by ' . $creator : '' ) . " using patch(es) linked to this task was **deleted**:\n" . + "Test wiki on [[ $serverUri | Patch demo ]] " . ( $creator ? ' by ' . $creator : '' ) . " using patch(es) linked to this task was **deleted**:\n" . "\n" . - "~~[[ $server$serverPath/wikis/$wiki/w/ ]]~~" + "~~[[ $serverUri/wikis/$wiki/w/ ]]~~" ); } @@ -478,7 +484,14 @@ function get_repo_presets(): array { return $presets; } -function detectProtocol(): string { +function is_cli(): bool { + return PHP_SAPI === 'cli' || PHP_SAPI === 'phpdbg'; +} + +function detect_protocol(): string { + if ( is_cli() ) { + throw new Error( 'Can\'t access server variables from CLI.' ); + } // Copied from MediaWiki's WebRequest::detectProtocol if ( ( !empty( $_SERVER['HTTPS'] ) && $_SERVER['HTTPS'] !== 'off' ) || @@ -493,6 +506,20 @@ function detectProtocol(): string { } } +function get_server(): string { + if ( is_cli() ) { + throw new Error( 'Can\'t access server variables from CLI.' ); + } + return detect_protocol() . '://' . $_SERVER['HTTP_HOST']; +} + +function get_server_path(): string { + if ( is_cli() ) { + throw new Error( 'Can\'t access server variables from CLI.' ); + } + return preg_replace( '`/[^/]*$`', '', $_SERVER['REQUEST_URI'] ); +} + function get_csrf_token(): string { global $useOAuth; if ( !$useOAuth ) { diff --git a/index.php b/index.php index fca03f31..1c499cdd 100644 --- a/index.php +++ b/index.php @@ -57,7 +57,7 @@ echo new OOUI\FormLayout( [ 'infusable' => true, 'method' => 'POST', - 'action' => 'new.php', + 'action' => 'start.php', 'id' => 'new-form', 'items' => [ new OOUI\FieldsetLayout( [ @@ -348,7 +348,10 @@ '' . $linkedTasks . '' . '' . date( 'Y-m-d H:i:s', $wikiData[ 'created' ] ) . '' . ( $useOAuth ? '' . ( $creator ? user_link( $creator ) : '?' ) . '' : '' ) . - ( $canAdmin ? '' . ( $wikiData['timeToCreate'] ? $wikiData['timeToCreate'] . 's' : '' ) . '' : '' ) . + ( $canAdmin ? + '' . ( $wikiData['timeToCreate'] ? $wikiData['timeToCreate'] . 's' : '' ) . '' : + '' + ) . ( count( $actions ) ? '' . implode( ' · ', $actions ) . '' : '' diff --git a/js/new.js b/js/new.js deleted file mode 100644 index 0f8bd162..00000000 --- a/js/new.js +++ /dev/null @@ -1,28 +0,0 @@ -/* global OO, pd */ -( function () { - window.pd = window.pd || {}; - - pd.installProgressField = OO.ui.infuse( - document.getElementsByClassName( 'installProgressField' )[ 0 ] - ); - - pd.installProgressField.fieldWidget.pushPending(); - - pd.openWiki = OO.ui.infuse( - document.getElementsByClassName( 'openWiki' )[ 0 ] - ); - - pd.notify = function ( message, body ) { - if ( 'Notification' in window && +localStorage.getItem( 'patchdemo-notifications' ) ) { - // eslint-disable-next-line no-new - new Notification( - message, - { - icon: './images/favicon-32x32.png', - body: body - } - ); - } - }; - -}() ); diff --git a/js/start.js b/js/start.js new file mode 100644 index 00000000..c52c6db7 --- /dev/null +++ b/js/start.js @@ -0,0 +1,79 @@ +/* global OO, pd */ +( function () { + window.pd = window.pd || {}; + + var installProgressField = OO.ui.infuse( + document.getElementsByClassName( 'installProgressField' )[ 0 ] + ); + + installProgressField.fieldWidget.pushPending(); + + var openWiki = OO.ui.infuse( + document.getElementsByClassName( 'openWiki' )[ 0 ] + ); + + function endProgress() { + installProgressField.fieldWidget.popPending(); + pd.finished = true; + } + + pd.abandon = function ( html ) { + installProgressField.fieldWidget.setDisabled( true ); + installProgressField.setErrors( [ new OO.ui.HtmlSnippet( html ) ] ); + pd.notify( 'Your PatchDemo wiki failed to build', html ); + endProgress(); + }; + + pd.setProgress = function ( pc, label ) { + installProgressField.fieldWidget.setProgress( pc ); + installProgressField.setLabel( label ); + if ( pc === 100 ) { + openWiki.setDisabled( false ); + pd.notify( 'Your PatchDemo wiki is ready!' ); + endProgress(); + } + }; + + pd.notify = function ( message, body ) { + if ( 'Notification' in window && +localStorage.getItem( 'patchdemo-notifications' ) ) { + // eslint-disable-next-line no-new + new Notification( + message, + { + icon: './images/favicon-32x32.png', + body: body + } + ); + } + }; + + $( function () { + // eslint-disable-next-line no-jquery/no-global-selector + var $log = $( '.newWikiLog' ); + var log = ''; + var offset = 0; + function poll() { + $.get( 'log.php', { + wiki: pd.wiki, + offset: offset + } ).then( function ( result ) { + if ( result ) { + // result can be unbalanced HTML, so store it in + // a string and rewrite the whole thing each time + log += result; + $log.html( log ); + offset += result.length; + } + if ( !pd.finished ) { + setTimeout( poll, 1000 ); + } + } ); + } + + poll(); + + // Add wiki to URL so that page can be shared/reloaded + history.replaceState( null, '', 'start.php?wiki=' + pd.wiki ); + } ); + +}() ); diff --git a/log.php b/log.php new file mode 100644 index 00000000..78e932dc --- /dev/null +++ b/log.php @@ -0,0 +1,21 @@ +pd.abandon( 'Log not found.' );"; +} diff --git a/new.php b/new.php index 101d08df..ce64d8ae 100644 --- a/new.php +++ b/new.php @@ -4,143 +4,44 @@ require_once "includes.php"; -include "header.php"; - -ob_implicit_flush( true ); - -if ( $useOAuth && !$user ) { - echo oauth_signin_prompt(); - die(); -} - -if ( !isset( $_POST['csrf_token'] ) || !check_csrf_token( $_POST['csrf_token'] ) ) { - die( "Invalid session." ); +if ( !is_cli() ) { + die( 'Must be run from the command line.' ); } $startTime = time(); -$branch = trim( $_POST['branch'] ); -$patches = trim( $_POST['patches'] ); -$announce = !empty( $_POST['announce'] ); -$language = trim( $_POST['language'] ); - -$namePath = substr( md5( $branch . $patches . time() ), 0, 10 ); -$server = detectProtocol() . '://' . $_SERVER['HTTP_HOST']; -$serverPath = preg_replace( '`/[^/]*$`', '', $_SERVER['REQUEST_URI'] ); - -$branchDesc = preg_replace( '/^origin\//', '', $branch ); - -$creator = $user ? $user->username : ''; -$created = time(); - -$canAdmin = can_admin(); - -/** - * Check if the user has dropped their connection and delete the wiki if so - * - * We could check for dropped connections with register_shutdown_function(), but - * that could happen in the middle of a shell command. If we tried to delete - * a wiki while a shell command was running (e.g composer update) we may still - * be left with stray files (e.g. /vendor) - * - * Instead manually check the connection at 'safe' times in between API requests - * or shell commands. - */ -function check_connection() { - if ( connection_status() !== CONNECTION_NORMAL ) { - abandon( 'User disconnected early' ); - } -} +$branch = trim( getenv( 'branch' ) ); +$patches = trim( getenv( 'patches' ) ); +$announce = !empty( getenv( 'announce' ) ); +$language = trim( getenv( 'language' ) ); -// Don't kill the process automatcally -ignore_user_abort( true ); +$wiki = getenv( 'wiki' ); +$server = getenv( 'server' ); +$serverPath = getenv( 'serverPath' ); -// Create an entry for the wiki before we have resolved patches. -// Will be updated later. -insert_wiki_data( $namePath, $creator, $created, $branchDesc ); +$creator = getenv( 'creator' ); +$canAdmin = getenv( 'canAdmin' ); +$branchDesc = getenv( 'branchDesc' ); function abandon( string $errHtml ) { - global $namePath; + global $wiki, $server, $serverPath; + + echo '

' . $errHtml . '

'; $errJson = json_encode( $errHtml ); - echo << - pd.installProgressField.fieldWidget.setDisabled( true ); - pd.installProgressField.fieldWidget.popPending(); - pd.installProgressField.setErrors( [ new OO.ui.HtmlSnippet( $errJson ) ] ); - pd.notify( 'Your PatchDemo wiki failed to build', $errJson ); - -EOT; - delete_wiki( $namePath ); - die( $errHtml ); + echo ""; + + // Don't delete the log immediately so it can be sent to the client + sleep( 2 ); + delete_wiki_cli_safe( $wiki, $server . $serverPath ); + die(); } function set_progress( float $pc, string $label ) { echo '

' . htmlspecialchars( $label ) . '

'; $labelJson = json_encode( $label ); - echo << - pd.installProgressField.fieldWidget.setProgress( $pc ); - pd.installProgressField.setLabel( $labelJson ); - -EOT; - if ( (int)$pc === 100 ) { - echo << - pd.installProgressField.fieldWidget.popPending(); - pd.openWiki.setDisabled( false ); - pd.notify( 'Your PatchDemo wiki is ready!' ); - -EOT; - } - - ob_flush(); + echo ""; } -echo new OOUI\FieldsetLayout( [ - 'label' => null, - 'classes' => [ 'installForm' ], - 'items' => [ - new OOUI\FieldLayout( - new OOUI\ProgressBarWidget(), - [ - 'align' => 'top', - 'label' => 'Installing...', - 'classes' => [ 'installProgressField' ], - 'infusable' => true, - ] - ), - new OOUI\FieldLayout( - new OOUI\ButtonWidget( [ - 'label' => 'Open wiki', - 'flags' => [ 'progressive', 'primary' ], - 'href' => "wikis/$namePath/w/", - 'disabled' => true, - 'classes' => [ 'openWiki' ], - 'infusable' => true, - ] ), - [ - 'align' => 'inline', - 'classes' => [ 'openWikiField' ], - 'label' => "When complete, use this button to open your wiki ($namePath)", - 'help' => new OOUI\HtmlSnippet( <<patchdemo1 -
    -
  • Patch Demo (admin)
    • -
  • Alice
    • -
  • Bob
    • -
  • Mallory (blocked)
    • -
- EOT ), - 'helpInline' => true, - ] - ), - ] -] ); - -echo ''; - -echo '
'; - if ( $patches ) { $patches = array_map( 'trim', preg_split( "/\n|\|/", $patches ) ); } else { @@ -176,7 +77,6 @@ function set_progress( float $pc, string $label ) { $o = 'CURRENT_REVISION'; } $data = gerrit_query( "changes/?q=change:$query&o=LABELS&o=$o", true ); - check_connection(); if ( count( $data ) === 0 ) { $patch = htmlentities( $patch ); @@ -220,26 +120,12 @@ function set_progress( float $pc, string $label ) { $config[ 'requireVerified' ] && ( $data[0]['labels']['Verified']['approved']['_account_id'] ?? null ) !== 75 && // Admin override - !( $canAdmin && isset( $_POST['adminVerified'] ) ) + !( $canAdmin && !empty( getenv( 'adminVerified' ) ) ) ) { // The patch doesn't have V+2, check if the uploader is trusted $uploaderId = $data[0]['revisions'][$revision]['uploader']['_account_id']; $uploader = gerrit_query( 'accounts/' . $uploaderId, true ); - check_connection(); if ( !is_trusted_user( $uploader['email'] ) ) { - if ( $canAdmin ) { - echo '
'; - foreach ( $_POST as $k => $v ) { - if ( is_array( $v ) ) { - foreach ( $v as $part ) { - echo ''; - } - } else { - echo ''; - } - } - echo '
'; - } abandon( "Patch must be approved (Verified+2) by jenkins-bot, or uploaded by a trusted user." . ( can_admin() ? @@ -281,7 +167,6 @@ function set_progress( float $pc, string $label ) { // Look at all commits in this patch's tree for cross-repo dependencies to add $data = gerrit_query( "changes/$id/revisions/$revision/related", true ); - check_connection(); // Ancestor commits only, not descendants $foundCurr = false; foreach ( $data['changes'] as $change ) { @@ -294,7 +179,6 @@ function set_progress( float $pc, string $label ) { foreach ( $relatedChanges as [ $c, $r ] ) { $data = gerrit_query( "changes/$c/revisions/$r/commit", true ); - check_connection(); preg_match_all( '/^Depends-On: (.+)$/m', $data['message'], $m ); foreach ( $m[1] as $changeid ) { @@ -314,22 +198,22 @@ function set_progress( float $pc, string $label ) { ) . ")"; // Update DB record with patches applied -wiki_add_patches( $namePath, $patchesApplied ); +wiki_add_patches( $wiki, $patchesApplied ); // Choose repositories to enable $repos = get_repo_data(); -if ( $_POST['preset'] === 'custom' ) { - $allowedRepos = $_POST['repos']; +if ( getenv( 'preset' ) === 'custom' ) { + $allowedRepos = explode( '|', getenv( 'repos' ) ); } else { - $allowedRepos = get_repo_presets()[ $_POST['preset'] ]; + $allowedRepos = get_repo_presets()[ getenv( 'preset' ) ]; } // Always include repos we are trying to patch (#401) $allowedRepos = array_merge( $allowedRepos, $usedRepos ); -$useProxy = !empty( $_POST['proxy'] ); -$useInstantCommons = !empty( $_POST['instantCommons' ] ); +$useProxy = !empty( getenv( 'proxy' ) ); +$useInstantCommons = !empty( getenv( 'instantCommons' ) ); // When proxying, always enable MobileFrontend and its content provider if ( $useProxy ) { // Doesn't matter if this appears twice @@ -337,7 +221,7 @@ function set_progress( float $pc, string $label ) { $allowedRepos[] = 'mediawiki/extensions/MobileFrontendContentProvider'; } if ( $useInstantCommons ) { - if ( $_POST['instantCommonsMethod'] === 'quick' ) { + if ( getenv( 'instantCommonsMethod' ) === 'quick' ) { $allowedRepos[] = 'mediawiki/extensions/QuickInstantCommons'; $useInstantCommons = false; } @@ -385,7 +269,6 @@ function set_progress( float $pc, string $label ) { list( $t, $r, $p ) = $matches; $data = gerrit_query( "changes/$r/revisions/$p/commit", true ); - check_connection(); if ( $data ) { $t = $t . ': ' . $data[ 'subject' ]; get_linked_tasks( $data['message'], $linkedTasks ); @@ -406,7 +289,7 @@ function set_progress( float $pc, string $label ) { $baseEnv = [ 'PATCHDEMO' => __DIR__, - 'NAME' => $namePath, + 'NAME' => $wiki, ]; $start = 5; @@ -418,7 +301,6 @@ function set_progress( float $pc, string $label ) { foreach ( $repos as $source => $target ) { set_progress( $repoProgress, "Updating repositories ($n/$repoCount)..." ); - check_connection(); $error = shell_echo( __DIR__ . '/new/updaterepos.sh', $baseEnv + [ 'REPO_SOURCE' => $source, @@ -434,7 +316,6 @@ function set_progress( float $pc, string $label ) { } // Just creates empty folders so no need for progress update -check_connection(); $error = shell_echo( __DIR__ . '/new/precheckout.sh', $baseEnv ); if ( $error ) { abandon( "Could not create directories for wiki" ); @@ -448,7 +329,6 @@ function set_progress( float $pc, string $label ) { foreach ( $repos as $source => $target ) { set_progress( $repoProgress, "Checking out repositories ($n/$repoCount)..." ); - check_connection(); $error = shell_echo( __DIR__ . '/new/checkout.sh', $baseEnv + [ 'BRANCH' => $repoSpecificBranches[$source] ?? $branch, @@ -466,7 +346,6 @@ function set_progress( float $pc, string $label ) { // TODO: Make this a loop set_progress( 60, 'Fetching submodules...' ); -check_connection(); $error = shell_echo( __DIR__ . '/new/submodules.sh', $baseEnv ); if ( $error ) { abandon( "Could not fetch submodules" ); @@ -487,7 +366,6 @@ static function ( string $repo ) use ( $repos ): bool { foreach ( $composerInstallRepos as $i => $repo ) { $n = $i + 1; set_progress( $repoProgress, "Fetching dependencies ($n/$repoCount)..." ); - check_connection(); $error = shell_echo( __DIR__ . '/new/composerinstall.sh', $baseEnv + [ // Variable used by composer itself, not our script @@ -504,7 +382,6 @@ static function ( string $repo ) use ( $repos ): bool { set_progress( 65, 'Installing your wiki...' ); -check_connection(); $error = shell_echo( __DIR__ . '/new/install.sh', $baseEnv + [ 'WIKINAME' => $wikiName, @@ -525,7 +402,6 @@ static function ( string $repo ) use ( $repos ): bool { foreach ( $commands as $i => $command ) { $n = $i + 1; set_progress( $progress, "Fetching and applying patches ($n/$count)..." ); - check_connection(); $error = shell_echo( $command[1], $baseEnv + $command[0] ); if ( $error ) { abandon( "Could not apply patch {$patchesApplied[$i]}" ); @@ -535,7 +411,6 @@ static function ( string $repo ) use ( $repos ): bool { set_progress( 90, 'Setting up wiki content...' ); -check_connection(); $error = shell_echo( __DIR__ . '/new/postinstall.sh', $baseEnv + [ 'MAINPAGE' => $mainPage, @@ -560,22 +435,20 @@ static function ( string $repo ) use ( $repos ): bool { 'T' . $task, "Test wiki **created** on [[ $server$serverPath | Patch demo ]]" . ( $creator ? ' by ' . $creator : '' ) . " using patch(es) linked to this task:" . "\n" . - "$server$serverPath/wikis/$namePath/w/" . + "$server$serverPath/wikis/$wiki/w/" . ( $hasOOUI ? "\n\n" . "Also created an **OOUI Demos** page:" . "\n" . - "$server$serverPath/wikis/$namePath/w/build/ooui/demos" + "$server$serverPath/wikis/$wiki/w/build/ooui/demos" : "" ) ); } - wiki_add_announced_tasks( $namePath, $linkedTasks ); + wiki_add_announced_tasks( $wiki, $linkedTasks ); } $timeToCreate = time() - $startTime; -wiki_set_time_to_create( $namePath, $timeToCreate ); +wiki_set_time_to_create( $wiki, $timeToCreate ); set_progress( 100, 'All done! Wiki created in ' . $timeToCreate . 's.' ); - -echo '
'; diff --git a/setup.sh b/setup.sh index d177437e..b2367770 100755 --- a/setup.sh +++ b/setup.sh @@ -30,6 +30,9 @@ sudo -u www-data mkdir composer # Create folder for wikis sudo -u www-data mkdir wikis +# Create folder for wiki creation logs +sudo -u www-data mkdir logs + # Create a database user that is allowed to create databases for each wiki, # and the central patchdemo database sudo mysql -u root --password='' < sql/user.sql diff --git a/start.php b/start.php new file mode 100644 index 00000000..812c1124 --- /dev/null +++ b/start.php @@ -0,0 +1,125 @@ +username : ''; + $branchDesc = preg_replace( '/^origin\//', '', $_POST['branch'] ); + + // Start creatig the wiki + $env = [ + 'wiki' => $wiki, + 'creator' => $creator, + 'canAdmin' => can_admin(), + 'branchDesc' => $branchDesc, + + 'announce' => $_POST['announce'] ?? '', + 'branch' => $_POST['branch'], + 'instantCommons' => $_POST['instantCommons'], + 'instantCommonsMethod' => $_POST['instantCommonsMethod'], + 'language' => $_POST['language'], + 'patches' => $_POST['patches'], + 'preset' => $_POST['preset'], + 'proxy' => $_POST['proxy'] ?? '', + 'repos' => implode( '|', $_POST['repos'] ), + + 'adminVerified' => $_POST['adminVerified'] ?? '', + + 'server' => get_server(), + 'serverPath' => get_server_path(), + ]; + + $process = Process::fromShellCommandline( + 'php new.php >> logs/' . $wiki . '.html', + null, + $env + ); + $process->setTimeout( null ); + $process->start(); + + // Create an entry for the wiki before we have resolved patches. + // Will be updated later. + insert_wiki_data( $wiki, $creator, time(), $branchDesc ); + + // If we terminate this script (start.php) immediately, the process above can stop (?) + sleep( 1 ); +} + +echo new OOUI\FieldsetLayout( [ + 'label' => null, + 'classes' => [ 'installForm' ], + 'items' => [ + new OOUI\FieldLayout( + new OOUI\ProgressBarWidget( [ 'progress' => 0 ] ), + [ + 'align' => 'top', + 'label' => 'Installing...', + 'classes' => [ 'installProgressField' ], + 'infusable' => true, + ] + ), + new OOUI\FieldLayout( + new OOUI\ButtonWidget( [ + 'label' => 'Open wiki', + 'flags' => [ 'progressive', 'primary' ], + 'href' => "wikis/$wiki/w/", + 'disabled' => true, + 'classes' => [ 'openWiki' ], + 'infusable' => true, + ] ), + [ + 'align' => 'inline', + 'classes' => [ 'openWikiField' ], + 'label' => "When complete, use this button to open your wiki ($wiki)", + 'help' => new OOUI\HtmlSnippet( <<patchdemo1 +
    +
  • Patch Demo (admin)
    • +
  • Alice
    • +
  • Bob
    • +
  • Mallory (blocked)
    • +
+ EOT ), + 'helpInline' => true, + ] + ), + ] +] ); + +if ( can_admin() ) { + echo '
'; + foreach ( $_POST as $k => $v ) { + if ( is_array( $v ) ) { + foreach ( $v as $part ) { + echo ''; + } + } else { + echo ''; + } + } + echo '
'; +} + +echo '
'; + +echo ''; +echo '';