From 37b685bb1502077f29cd4df94a9ecaab603d6305 Mon Sep 17 00:00:00 2001 From: rvogel Date: Fri, 10 May 2024 15:28:37 +0200 Subject: [PATCH] Initial --- .gitignore | 7 + .vscode/settings.json | 3 + README.md | 104 ++++ add-init.d-service.sh | 8 + bin/parallel-runjobs-service | 13 + box.json | 18 + build/.gitkeep | 0 composer.json | 22 + composer.lock | 1022 ++++++++++++++++++++++++++++++++++ example.config.yaml | 18 + farm-runjobs | 108 ++++ src/RunjobsCommand.php | 112 ++++ src/RunjobsService.php | 191 +++++++ 13 files changed, 1626 insertions(+) create mode 100644 .gitignore create mode 100644 .vscode/settings.json create mode 100644 README.md create mode 100755 add-init.d-service.sh create mode 100755 bin/parallel-runjobs-service create mode 100644 box.json create mode 100644 build/.gitkeep create mode 100644 composer.json create mode 100644 composer.lock create mode 100644 example.config.yaml create mode 100644 farm-runjobs create mode 100644 src/RunjobsCommand.php create mode 100644 src/RunjobsService.php diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..37f42ea --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +venv +!logs/.gitkeep +logs/* +config.yaml +vendor/ +build/parallel-runjobs-service \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..de288e1 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.formatting.provider": "black" +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..e48b1fe --- /dev/null +++ b/README.md @@ -0,0 +1,104 @@ +## Table of contents +- [Table of contents](#table-of-contents) +- [Information](#information) +- [Requirements](#requirements) +- [Creating a build](#creating-a-build) +- [Usage](#usage) +- [Config](#config) + +## Information +This executes runjobs service parallely for multiple wiki farm instances as well as non-farm instances. + +## Requirements +- `php >= 8.0` +- `composer >= 2` +- Install and update composer dependencies using: + ```bash + composer update + ``` + +## Creating a build +1. Clone this repo +2. Run `composer update --no-dev` +3. Run `box compile` to actually create the PHAR file in `dist/`. See also https://github.com/humbug/box + +## Usage +1. Create config file: + - Copy the template config and make required changes: + ```bash + cp ./example.config.yaml ./config.yaml + ``` + If using config file then wiki-type and wiki-path options are mandatory to mention +2. Run the main script file: + - Using the default config: + ```bash + ./bin/run.php + ``` + - Using custom config: + - Create config file as explained above and then run the following: + ```bash + ./bin/run.php --config ./config.yaml + ``` + - `./bin/run.php` script cli options: + ```bash + Description: + Execute runjobs service parallelly for multiple instances. + + Usage: + runjobs [options] + + Options: + -c, --config[=CONFIG] Path to the configuration file (YAML). If not provided, the default path will be used. + --wiki-type[=WIKI-TYPE] Type of wiki (standalone or farm). + --wiki-path[=WIKI-PATH] Absolute path of the wiki. + --wiki-reference[=WIKI-REFERENCE] Reference file for the wiki (e.g., LocalSettings.php). + --runjobs-percentage[=RUNJOBS-PERCENTAGE] Maximum percentage of total jobs (per wiki, per cycle). + --runjobs-maxtime[=RUNJOBS-MAXTIME] Maximum life time of a runJobs.php (per wiki, per cycle) in seconds. + --runjobs-cooldown[=RUNJOBS-COOLDOWN] Wait time after each cycle in seconds. + --runjobs-maxforkprocesses[=RUNJOBS-MAXFORKPROCESSES] Maximum number of sub processes that can be spawned for parallel processing + --exclude-instances[=EXCLUDE-INSTANCES] Contains list of comma separated farm instances name for which the runjobs should not be run + --include-instances[=INCLUDE-INSTANCES] Contains list of comma separated farm instances name for which the runjobs should be run, if left empty, all the farm instances will be considered except the ones in "exclude-instances" list + -h, --help Display help for the given command. When no command is given display help for the runjobs command + -q, --quiet Do not output any message + -V, --version Display this application version + --ansi|--no-ansi Force (or disable --no-ansi) ANSI output + -n, --no-interaction Do not ask any interactive question + -v|vv|vvv, --verbose Increase the verbosity of messages: 1 for normal output, 2 for more verbose output and 3 for debug + ``` + +3. Start as a init.d service (optional): + - Create config file as explained above + - Setup init.d script, from inside the git clone repo run: + ```bash + ./add-init.d-service.sh + ``` + - Start service: + ```bash + service farm-runjobs start + ``` + - Look at service status: + ```bash + service farm-runjobs status + ``` + - Look at service logs: + ```bash + tail -f /var/log/farm-runjobs.log + ``` + - Stop service: + ```bash + service farm-runjobs stop + ``` + +## Config + | name | default value | accepted values | description | + | ------------------ | ---------------------- | --------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | + | wiki:type | `farm` | `farm`, `pro` | type of the bluespice edition | + | wiki:path | `/var/www/bluespice/w` | absolute path | bluespice installation path | + | wiki:reference | `LocalSettings.php` | `LocalSettings.php` | local settings filename for the wiki which has database credentials | + | runjobs:percentage | `50` | 1-100 | Maximum percentage of total jobs. (Per wiki, per cycle) | + | runjobs:maxtime | `10` | any suitable number | Maximum life time of a runJobs.php (Per wiki, per cycle) - Seconds | + | runjobs:cooldown | `3` | any suitable number | Wait time after each cycle - Seconds | + | runjobs:maxforkprocesses | `5` | any suitable number | Maximum number of sub processes that can be spawned for parallel processing | + | exclude_instances | `[]` | array of valid wiki farm instance names in double quotes separated by comma | contains list of farm instances name for which the runjobs should not be run | + | include_instances | `[]` | array of valid wiki farm instance names in double quotes separated by comma | contains list of farm instances name for which the runjobs should be run, if left empty, all the farm instances will be considered except the ones in `exclude_instances` list | + diff --git a/add-init.d-service.sh b/add-init.d-service.sh new file mode 100755 index 0000000..0f00bb4 --- /dev/null +++ b/add-init.d-service.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +rm -rf /etc/init.d/farm-runjobs +composer update +cp farm-runjobs /etc/init.d/farm-runjobs +sed -i -e "s||\"$(pwd)/bin/run.php -c $(pwd)/config.yaml\"|g" /etc/init.d/farm-runjobs +chmod +x /etc/init.d/farm-runjobs +echo "runjobs service added" diff --git a/bin/parallel-runjobs-service b/bin/parallel-runjobs-service new file mode 100755 index 0000000..2215560 --- /dev/null +++ b/bin/parallel-runjobs-service @@ -0,0 +1,13 @@ +#!/usr/bin/env php +add(new RunjobsCommand()); +$application->setDefaultCommand('runjobs', true); + +$application->run(); diff --git a/box.json b/box.json new file mode 100644 index 0000000..14ba923 --- /dev/null +++ b/box.json @@ -0,0 +1,18 @@ +{ + "directories": [ + "src" + ], + "finder": [ + { + "name": "*.php", + "exclude": ["tests"], + "in": "vendor" + } + ], + "git-version": "package_version", + "main": "bin/parallel-runjobs-service", + "output": "build/parallel-runjobs-service" +} + + + diff --git a/build/.gitkeep b/build/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..cb78bd2 --- /dev/null +++ b/composer.json @@ -0,0 +1,22 @@ +{ + "name": "bluespice/parallel-runjobs-service", + "description": "This executes runjobs service parallely for multiple wiki farm instances as well as non-farm instances.", + "autoload": { + "psr-4": { + "BlueSpice\\Service\\ParallelRunJobs\\": "src/" + } + }, + "require": { + "symfony/console": "^5.0", + "symfony/yaml": "^5.0", + "symfony/process": "^6.0" + }, + "authors": [ + { + "name": "jas", + "email": "support@hallowelt.com" + } + ], + "minimum-stability": "stable", + "prefer-stable": true +} diff --git a/composer.lock b/composer.lock new file mode 100644 index 0000000..3876ddd --- /dev/null +++ b/composer.lock @@ -0,0 +1,1022 @@ +{ + "_readme": [ + "This file locks the dependencies of your project to a known state", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", + "This file is @generated automatically" + ], + "content-hash": "8404e0e7111daed99653d788a3f75850", + "packages": [ + { + "name": "psr/container", + "version": "2.0.2", + "source": { + "type": "git", + "url": "https://github.com/php-fig/container.git", + "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/container/zipball/c71ecc56dfe541dbd90c5360474fbc405f8d5963", + "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963", + "shasum": "" + }, + "require": { + "php": ">=7.4.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Container\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "https://www.php-fig.org/" + } + ], + "description": "Common Container Interface (PHP FIG PSR-11)", + "homepage": "https://github.com/php-fig/container", + "keywords": [ + "PSR-11", + "container", + "container-interface", + "container-interop", + "psr" + ], + "support": { + "issues": "https://github.com/php-fig/container/issues", + "source": "https://github.com/php-fig/container/tree/2.0.2" + }, + "time": "2021-11-05T16:47:00+00:00" + }, + { + "name": "symfony/console", + "version": "v5.4.39", + "source": { + "type": "git", + "url": "https://github.com/symfony/console.git", + "reference": "f3e591c48688a0cfa1a3296205926c05e84b22b1" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/console/zipball/f3e591c48688a0cfa1a3296205926c05e84b22b1", + "reference": "f3e591c48688a0cfa1a3296205926c05e84b22b1", + "shasum": "" + }, + "require": { + "php": ">=7.2.5", + "symfony/deprecation-contracts": "^2.1|^3", + "symfony/polyfill-mbstring": "~1.0", + "symfony/polyfill-php73": "^1.9", + "symfony/polyfill-php80": "^1.16", + "symfony/service-contracts": "^1.1|^2|^3", + "symfony/string": "^5.1|^6.0" + }, + "conflict": { + "psr/log": ">=3", + "symfony/dependency-injection": "<4.4", + "symfony/dotenv": "<5.1", + "symfony/event-dispatcher": "<4.4", + "symfony/lock": "<4.4", + "symfony/process": "<4.4" + }, + "provide": { + "psr/log-implementation": "1.0|2.0" + }, + "require-dev": { + "psr/log": "^1|^2", + "symfony/config": "^4.4|^5.0|^6.0", + "symfony/dependency-injection": "^4.4|^5.0|^6.0", + "symfony/event-dispatcher": "^4.4|^5.0|^6.0", + "symfony/lock": "^4.4|^5.0|^6.0", + "symfony/process": "^4.4|^5.0|^6.0", + "symfony/var-dumper": "^4.4|^5.0|^6.0" + }, + "suggest": { + "psr/log": "For using the console logger", + "symfony/event-dispatcher": "", + "symfony/lock": "", + "symfony/process": "" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\Console\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Eases the creation of beautiful and testable command line interfaces", + "homepage": "https://symfony.com", + "keywords": [ + "cli", + "command-line", + "console", + "terminal" + ], + "support": { + "source": "https://github.com/symfony/console/tree/v5.4.39" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2024-04-18T08:26:06+00:00" + }, + { + "name": "symfony/deprecation-contracts", + "version": "v3.0.2", + "source": { + "type": "git", + "url": "https://github.com/symfony/deprecation-contracts.git", + "reference": "26954b3d62a6c5fd0ea8a2a00c0353a14978d05c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/26954b3d62a6c5fd0ea8a2a00c0353a14978d05c", + "reference": "26954b3d62a6c5fd0ea8a2a00c0353a14978d05c", + "shasum": "" + }, + "require": { + "php": ">=8.0.2" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "3.0-dev" + }, + "thanks": { + "name": "symfony/contracts", + "url": "https://github.com/symfony/contracts" + } + }, + "autoload": { + "files": [ + "function.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "A generic function and convention to trigger deprecation notices", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.0.2" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2022-01-02T09:55:41+00:00" + }, + { + "name": "symfony/polyfill-ctype", + "version": "v1.29.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-ctype.git", + "reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ef4d7e442ca910c4764bce785146269b30cb5fc4", + "reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4", + "shasum": "" + }, + "require": { + "php": ">=7.1" + }, + "provide": { + "ext-ctype": "*" + }, + "suggest": { + "ext-ctype": "For best performance" + }, + "type": "library", + "extra": { + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Ctype\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Gert de Pagter", + "email": "BackEndTea@gmail.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill for ctype functions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "ctype", + "polyfill", + "portable" + ], + "support": { + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.29.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2024-01-29T20:11:03+00:00" + }, + { + "name": "symfony/polyfill-intl-grapheme", + "version": "v1.29.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-intl-grapheme.git", + "reference": "32a9da87d7b3245e09ac426c83d334ae9f06f80f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/32a9da87d7b3245e09ac426c83d334ae9f06f80f", + "reference": "32a9da87d7b3245e09ac426c83d334ae9f06f80f", + "shasum": "" + }, + "require": { + "php": ">=7.1" + }, + "suggest": { + "ext-intl": "For best performance" + }, + "type": "library", + "extra": { + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Intl\\Grapheme\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill for intl's grapheme_* functions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "grapheme", + "intl", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.29.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2024-01-29T20:11:03+00:00" + }, + { + "name": "symfony/polyfill-intl-normalizer", + "version": "v1.29.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-intl-normalizer.git", + "reference": "bc45c394692b948b4d383a08d7753968bed9a83d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/bc45c394692b948b4d383a08d7753968bed9a83d", + "reference": "bc45c394692b948b4d383a08d7753968bed9a83d", + "shasum": "" + }, + "require": { + "php": ">=7.1" + }, + "suggest": { + "ext-intl": "For best performance" + }, + "type": "library", + "extra": { + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Intl\\Normalizer\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill for intl's Normalizer class and related functions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "intl", + "normalizer", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.29.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2024-01-29T20:11:03+00:00" + }, + { + "name": "symfony/polyfill-mbstring", + "version": "v1.29.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-mbstring.git", + "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9773676c8a1bb1f8d4340a62efe641cf76eda7ec", + "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec", + "shasum": "" + }, + "require": { + "php": ">=7.1" + }, + "provide": { + "ext-mbstring": "*" + }, + "suggest": { + "ext-mbstring": "For best performance" + }, + "type": "library", + "extra": { + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Mbstring\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill for the Mbstring extension", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "mbstring", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.29.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2024-01-29T20:11:03+00:00" + }, + { + "name": "symfony/polyfill-php73", + "version": "v1.29.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php73.git", + "reference": "21bd091060673a1177ae842c0ef8fe30893114d2" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php73/zipball/21bd091060673a1177ae842c0ef8fe30893114d2", + "reference": "21bd091060673a1177ae842c0ef8fe30893114d2", + "shasum": "" + }, + "require": { + "php": ">=7.1" + }, + "type": "library", + "extra": { + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php73\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 7.3+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php73/tree/v1.29.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2024-01-29T20:11:03+00:00" + }, + { + "name": "symfony/polyfill-php80", + "version": "v1.29.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php80.git", + "reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/87b68208d5c1188808dd7839ee1e6c8ec3b02f1b", + "reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b", + "shasum": "" + }, + "require": { + "php": ">=7.1" + }, + "type": "library", + "extra": { + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php80\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Ion Bazan", + "email": "ion.bazan@gmail.com" + }, + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 8.0+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php80/tree/v1.29.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2024-01-29T20:11:03+00:00" + }, + { + "name": "symfony/process", + "version": "v6.0.19", + "source": { + "type": "git", + "url": "https://github.com/symfony/process.git", + "reference": "2114fd60f26a296cc403a7939ab91478475a33d4" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/process/zipball/2114fd60f26a296cc403a7939ab91478475a33d4", + "reference": "2114fd60f26a296cc403a7939ab91478475a33d4", + "shasum": "" + }, + "require": { + "php": ">=8.0.2" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\Process\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Executes commands in sub-processes", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/process/tree/v6.0.19" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2023-01-01T08:36:10+00:00" + }, + { + "name": "symfony/service-contracts", + "version": "v3.0.2", + "source": { + "type": "git", + "url": "https://github.com/symfony/service-contracts.git", + "reference": "d78d39c1599bd1188b8e26bb341da52c3c6d8a66" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/d78d39c1599bd1188b8e26bb341da52c3c6d8a66", + "reference": "d78d39c1599bd1188b8e26bb341da52c3c6d8a66", + "shasum": "" + }, + "require": { + "php": ">=8.0.2", + "psr/container": "^2.0" + }, + "conflict": { + "ext-psr": "<1.1|>=2" + }, + "suggest": { + "symfony/service-implementation": "" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "3.0-dev" + }, + "thanks": { + "name": "symfony/contracts", + "url": "https://github.com/symfony/contracts" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Contracts\\Service\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Generic abstractions related to writing services", + "homepage": "https://symfony.com", + "keywords": [ + "abstractions", + "contracts", + "decoupling", + "interfaces", + "interoperability", + "standards" + ], + "support": { + "source": "https://github.com/symfony/service-contracts/tree/v3.0.2" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2022-05-30T19:17:58+00:00" + }, + { + "name": "symfony/string", + "version": "v6.0.19", + "source": { + "type": "git", + "url": "https://github.com/symfony/string.git", + "reference": "d9e72497367c23e08bf94176d2be45b00a9d232a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/string/zipball/d9e72497367c23e08bf94176d2be45b00a9d232a", + "reference": "d9e72497367c23e08bf94176d2be45b00a9d232a", + "shasum": "" + }, + "require": { + "php": ">=8.0.2", + "symfony/polyfill-ctype": "~1.8", + "symfony/polyfill-intl-grapheme": "~1.0", + "symfony/polyfill-intl-normalizer": "~1.0", + "symfony/polyfill-mbstring": "~1.0" + }, + "conflict": { + "symfony/translation-contracts": "<2.0" + }, + "require-dev": { + "symfony/error-handler": "^5.4|^6.0", + "symfony/http-client": "^5.4|^6.0", + "symfony/translation-contracts": "^2.0|^3.0", + "symfony/var-exporter": "^5.4|^6.0" + }, + "type": "library", + "autoload": { + "files": [ + "Resources/functions.php" + ], + "psr-4": { + "Symfony\\Component\\String\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Provides an object-oriented API to strings and deals with bytes, UTF-8 code points and grapheme clusters in a unified way", + "homepage": "https://symfony.com", + "keywords": [ + "grapheme", + "i18n", + "string", + "unicode", + "utf-8", + "utf8" + ], + "support": { + "source": "https://github.com/symfony/string/tree/v6.0.19" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2023-01-01T08:36:10+00:00" + }, + { + "name": "symfony/yaml", + "version": "v5.4.39", + "source": { + "type": "git", + "url": "https://github.com/symfony/yaml.git", + "reference": "bc780e16879000f77a1022163c052f5323b5e640" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/yaml/zipball/bc780e16879000f77a1022163c052f5323b5e640", + "reference": "bc780e16879000f77a1022163c052f5323b5e640", + "shasum": "" + }, + "require": { + "php": ">=7.2.5", + "symfony/deprecation-contracts": "^2.1|^3", + "symfony/polyfill-ctype": "^1.8" + }, + "conflict": { + "symfony/console": "<5.3" + }, + "require-dev": { + "symfony/console": "^5.3|^6.0" + }, + "suggest": { + "symfony/console": "For validating YAML files using the lint command" + }, + "bin": [ + "Resources/bin/yaml-lint" + ], + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\Yaml\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Loads and dumps YAML files", + "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/yaml/tree/v5.4.39" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2024-04-23T11:57:27+00:00" + } + ], + "packages-dev": [], + "aliases": [], + "minimum-stability": "stable", + "stability-flags": [], + "prefer-stable": true, + "prefer-lowest": false, + "platform": [], + "platform-dev": [], + "platform-overrides": { + "php": "8.0.99", + "ext-ldap": "8", + "ext-intl": "8", + "ext-pdo_sqlite": "8", + "ext-mongodb": "1.14.0" + }, + "plugin-api-version": "2.6.0" +} diff --git a/example.config.yaml b/example.config.yaml new file mode 100644 index 0000000..5c5fcda --- /dev/null +++ b/example.config.yaml @@ -0,0 +1,18 @@ +wiki: + type: "farm" # standalone or farm + path: "/var/www/bluespice/w" + reference: "LocalSettings.php" # LocalSettings.php for DB credentials +runjobs: + percentage: 50 # Maximum percentage of total jobs. (Per wiki, per cycle) + maxtime: 10 # Maximum life time of a runJobs.php (Per wiki, per cycle) - Seconds + cooldown: 3 # Wait time after each cycle - Seconds + maxforkprocesses: 5 # Maximum number of sub processes that can be spawned for parallel processing + +# Following list should be mutually exclusive, otherwise an exception is raised +# "exclude_instances" contains list of comma separated farm instances name for which +# the runjobs should not be run +exclude-instances: [] +# "include-instances" contains list of comma separated farm instances name for which +# the runjobs should be run, if left empty, all the farm instances will be considered +# except the ones in "exclude_instances" list +include-instances: [] diff --git a/farm-runjobs b/farm-runjobs new file mode 100644 index 0000000..813acc7 --- /dev/null +++ b/farm-runjobs @@ -0,0 +1,108 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: farm-runjobs +# Required-Start: $local_fs $network $named $time $syslog +# Required-Stop: $local_fs $network $named $time $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Description: This executes runjobs service parallely for multiple wiki farm instances. +### END INIT INFO + +SCRIPT= +RUNAS=root + +PIDFILE=/var/run/farm-runjobs.pid +LOG_FILE="/var/log/farm-runjobs.log" +MAX_LOG_SIZE=10 # in MB +ROTATION_INTERVAL=600 # in seconds + +start() { + if [ -f $PIDFILE ] && [ -s $PIDFILE ] && kill -0 $(cat $PIDFILE); then + echo 'Service already running' >&2 + return 1 + fi + echo 'Starting service…' >&2 + local CMD="$SCRIPT >> $LOG_FILE 2>&1 & echo \$!" + su -c "$CMD" $RUNAS > "$PIDFILE" + rotate_logs & + # Try with this command line instead of above if not workable + # su -s /bin/sh $RUNAS -c "$CMD" > "$PIDFILE" + sleep 2 + PID=$(cat $PIDFILE) + if pgrep -u $RUNAS -f $NAME > /dev/null + then + echo "$NAME is now running, the PID is $PID" + else + echo '' + echo "Error! Could not start $NAME!" + fi +} + +stop() { + if [ ! -f "$PIDFILE" ] || ! kill -0 $(cat "$PIDFILE"); then + echo 'Service not running' >&2 + return 1 + fi + echo 'Stopping service…' >&2 + kill -15 $(cat "$PIDFILE") && rm -f "$PIDFILE" + echo 'Service stopped' >&2 +} + +uninstall() { + echo -n "Are you really sure you want to uninstall this service? That cannot be undone. [yes|No] " + local SURE + read SURE + if [ "$SURE" = "yes" ]; then + stop + rm -f "$PIDFILE" + update-rc.d -f $NAME remove + rm -fv "$0" + else + echo "Abort!" + fi +} + +status() { + printf "%-50s" "Checking farm-runjobs..." + if [ -f $PIDFILE ] && [ -s $PIDFILE ]; then + PID=$(cat $PIDFILE) + if [ -z "$(ps axf | grep ${PID} | grep -v grep)" ]; then + printf "%s\n" "The process appears to be dead but pidfile still exists" + else + echo "Running, the PID is $PID" + fi + else + printf "%s\n" "Service not running" + fi +} + +rotate_logs() { + while true; do + if [ -f "$LOG_FILE" ] && [ "$(du -k "$LOG_FILE" | cut -f1)" -gt "$MAX_LOG_SIZE" ]; then + > "$LOG_FILE" + fi + sleep "$ROTATION_INTERVAL" + done +} + + +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + status + ;; + uninstall) + uninstall + ;; + restart) + stop + start + ;; + *) + echo "Usage: $0 {start|stop|status|restart|uninstall}" +esac diff --git a/src/RunjobsCommand.php b/src/RunjobsCommand.php new file mode 100644 index 0000000..cecafe2 --- /dev/null +++ b/src/RunjobsCommand.php @@ -0,0 +1,112 @@ +setDescription('Execute runjobs service parallelly for multiple instances.') + ->setHelp('This command executes the runjobs service for multiple wiki farm instances as well as non-farm instances.') + ->addOption('config', 'c', InputOption::VALUE_OPTIONAL, 'Path to the configuration file (YAML). If not provided, the default path will be used.') + ->addOption('wiki-type', null, InputOption::VALUE_OPTIONAL, 'Type of wiki (standalone or farm).') + ->addOption('wiki-path', null, InputOption::VALUE_OPTIONAL, 'Absolute path of the wiki.') + ->addOption('wiki-reference', null, InputOption::VALUE_OPTIONAL, 'Reference file for the wiki (e.g., LocalSettings.php).') + ->addOption('runjobs-percentage', null, InputOption::VALUE_OPTIONAL, 'Maximum percentage of total jobs (per wiki, per cycle).') + ->addOption('runjobs-maxtime', null, InputOption::VALUE_OPTIONAL, 'Maximum life time of a runJobs.php (per wiki, per cycle) in seconds.') + ->addOption('runjobs-cooldown', null, InputOption::VALUE_OPTIONAL, 'Wait time after each cycle in seconds.') + ->addOption('runjobs-maxforkprocesses', null, InputOption::VALUE_OPTIONAL, 'Maximum number of sub processes that can be spawned for parallel processing') + ->addOption('exclude-instances', null, InputOption::VALUE_OPTIONAL, 'Contains list of comma separated farm instances name for which the runjobs should not be run') + ->addOption('include-instances', null, InputOption::VALUE_OPTIONAL, 'Contains list of comma separated farm instances name for which the runjobs should be run, if left empty, all the farm instances will be considered except the ones in "exclude-instances" list'); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + + $this->output = $output->section(); + $runjobsOutputSection = $output->section(); + + if ($input->getOption('config') && $this->hasOtherOptions($input)) { + throw new \RuntimeException("The '--config' option cannot be used along with other options."); + } + + $config; + $configFilePath; + // Check if the config file was provided; if not, use provided options as config + if (!$input->getOption('config')) { + $this->validateMandatoryOptions($input); + $config = $this->buildConfigFromOptions($input); + } else { + $configFilePath = $input->getOption('config'); + $config = $this->loadConfig($configFilePath); + } + + $this->output->writeln('Started executing runjobs service'); + + $runjobsService = new RunjobsService($config, $runjobsOutputSection); + $runjobsService->run(); + + return Command::SUCCESS; + } + + protected function loadConfig(string $configFilePath) + { + if (!file_exists($configFilePath)) { + throw new \RuntimeException("Configuration file not found at: $configFilePath"); + } + + return Yaml::parseFile($configFilePath); + } + + protected function validateMandatoryOptions(InputInterface $input) + { + $mandatoryOptions = ['wiki-type', 'wiki-path']; + + foreach ($mandatoryOptions as $option) { + if (!$input->getOption($option)) { + throw new \RuntimeException("The '--$option' option is mandatory when no config file is provided."); + } + } + } + + protected function buildConfigFromOptions(InputInterface $input) + { + return [ + 'wiki' => [ + 'type' => $input->getOption('wiki-type'), + 'path' => $input->getOption('wiki-path'), + 'reference' => $input->getOption('wiki-reference') ?? 'LocalSettings.php', + ], + 'runjobs' => [ + 'percentage' => $input->getOption('runjobs-percentage') ?? 50, + 'maxtime' => $input->getOption('runjobs-maxtime') ?? 10, + 'cooldown' => $input->getOption('runjobs-cooldown') ?? 3, + 'maxforkprocesses' => $input->getOption('runjobs-maxforkprocesses') ?? 5, + ], + 'exclude-instances' => $input->getOption('exclude-instances') ?? [], + 'include-instances' => $input->getOption('include-instances') ?? [], + ]; + } + + protected function hasOtherOptions(InputInterface $input) + { + $otherOptions = ['wiki-type', 'wiki-path', 'wiki-reference', 'runjobs-percentage', 'runjobs-maxtime', 'runjobs-cooldown', 'runjobs-maxforkprocesses', 'exclude-instances', 'include-instances']; + + foreach ($otherOptions as $option) { + if ($input->getOption($option)) { + return true; + } + } + + return false; + } +} diff --git a/src/RunjobsService.php b/src/RunjobsService.php new file mode 100644 index 0000000..8cb3c57 --- /dev/null +++ b/src/RunjobsService.php @@ -0,0 +1,191 @@ +config = $config; + $this->output = $output; + $this->get_sql_creds(); + $this->config['wiki']['instancesDir'] = $this->config['wiki']['path'] . DIRECTORY_SEPARATOR . '_sf_instances'; + } + + public function run() + { + while (true) { + $this->output->clear(); + if ($this->config['wiki']['type'] == "farm") { + $this->output->writeln('Detected ' . $this->config['wiki']['type'] . ' wiki installation<\info>'); + $instanceList = $this->generate_instance_list(); + $this->run_parallel($instanceList); + } + $this->run_single(); + $this->output->writeln("Cooling down"); + sleep($this->config['runjobs']['cooldown']); + } + } + + private function parse_php_vars($php_file, $src_str = false) + { + $content = file_get_contents($php_file); + $reg = '/\$(?P\w+)\s*=\s*"?\'?(?P[^"\';]+)"?\'?;/i'; + preg_match_all($reg, $content, $matches, PREG_SET_ORDER); + + if (!$src_str) { + $pvalue = $matches; + } else { + $pvalue = false; + foreach ($matches as $line) { + if ($line['variable'] == $src_str) { + $pvalue = $line['value']; + break; + } + } + } + + return $pvalue; + } + + private function get_sql_creds() + { + $phpFilePath = $this->config['wiki']['path'] . '/' . $this->config['wiki']['reference']; + + $sqlHost = $this->parse_php_vars($phpFilePath, 'wgDBserver'); + $sqlUser = $this->parse_php_vars($phpFilePath, 'wgDBuser'); + $sqlPass = $this->parse_php_vars($phpFilePath, 'wgDBpassword'); + + $this->sqlHost = $sqlHost; + $this->sqlUser = $sqlUser; + $this->sqlPass = $sqlPass; + } + + private function count_jobs($instanceName) + { + $sqlDatabase = $this->parse_php_vars( + $this->config['wiki']['instancesDir'] . DIRECTORY_SEPARATOR . $instanceName . DIRECTORY_SEPARATOR . 'LocalSettings.php', + 'wgDBname' + ); + $value = 0; + $db = new mysqli($this->sqlHost, $this->sqlUser, $this->sqlPass, $sqlDatabase); + + if ($db->connect_error) { + throw new Exception("Connection failed: " . $db->connect_error); + } + $sql = $db->prepare("SELECT COUNT(*) FROM job WHERE job_token=''"); + $sql->execute(); + $sql->bind_result($jobCount); + $sql->fetch(); + $value = (int)$jobCount; + $sql->close(); + $db->close(); + + // return $value; + return 345345; + } + + private function generate_instance_list() + { + + // generate instance list + $this->output->writeln('generating instance list<\info>'); + $instanceList = []; + $allWikiInstances = array_map('basename', array_filter(glob($this->config['wiki']['instancesDir'] . '/*'), 'is_dir')); + $this->output->writeln("generated instance list: " . implode(', ', $allWikiInstances) . '<\info>'); + + // Check if both the include list and exclude list have valid instance names + if (!empty($this->config['include-instances'])) { + if (!empty(array_diff($config['include-instances'], $allWikiInstances))) { + throw new \RuntimeException("Invalid include-instances"); + } + } + + if (!empty($this->config['exclude-instances'])) { + if (!empty(array_diff($this->config['exclude-instances'], $allWikiInstances))) { + throw new \RuntimeException("Invalid exclude-instances"); + } + } + + // Determine the instances list based on include-instances and exclude-instances + if (empty($this->config['include-instances'])) { + $instances = $allWikiInstances; + } else { + $instances = $this->config['include-instances']; + } + + // Make sure the resultant list is mutually exclusive and get final instance list + $instances = array_diff($instances, $this->config['exclude-instances']); + $this->output->writeln("<\info>target instances: " . implode(', ', $instances) . "<\info>"); + + + foreach ($instances as $instance) { + $instancePath = $this->config['wiki']['instancesDir'] . DIRECTORY_SEPARATOR . $instance; + + if (!file_exists($instancePath . DIRECTORY_SEPARATOR . 'SUSPENDED') && file_exists($instancePath . DIRECTORY_SEPARATOR . 'LocalSettings.php')) { + $this->output->writeln("<\info>farm instance '$instance' is not suspended and has LocalSettings.php<\info>"); + + $jobCount = $this->count_jobs($instance, $this->config, $this->sqlHost, $this->sqlUser, $this->sqlPass); + + if ($jobCount > 0) { + $this->output->writeln("<\info>farm instance '$instance' has $jobCount pending jobs<\info>"); + + $instanceTuple = [$instance, $jobCount]; + $instanceList[] = $instanceTuple; + } else { + $this->output->writeln("<\info>farm instance '$instance' has no pending jobs<\info>"); + } + } + } + + return $instanceList; + } + + private function run_parallel($instanceList) + { + $this->output->writeln("<\info>Running jobs in parallel<\info>"); + + $jobs = []; + $numParallelJobs = $this->config['runjobs']['maxforkprocesses']; + + // Start the parallel jobs + for ($i = 0; $i < $numParallelJobs; $i++) { + + // Create a new Process for each job + $currInstanceName = $instanceList[$i][0]; + $currInstanceJobCount = $instanceList[$i][1]; + $cmd = ['/usr/bin/php', $this->config['wiki']['path'] . '/maintenance/runJobs.php', '--maxtime=' . $this->config['runjobs']['maxtime']]; + if ($currInstanceJobCount > 100) { + $maxjobs = (int) (($currInstanceJobCount / 100) * $this->config['runjobs']['percentage']); + $cmd[] = '--maxjobs=' . $maxjobs; + } + $cmd[] = '--sfr=' . $currInstanceName; + $process = new Process($cmd); + $process->start(); + $jobs[] = $process; + } + + foreach ($jobs as $process) { + $process->wait(); + } + + $this->output->writeln("All current parallel jobs have finished"); + } + + + private function run_single() + { + $output = $this->output; + $cmd = ['/usr/bin/php', $this->config['wiki']['path'] . '/maintenance/runJobs.php', '--maxtime=' . $this->config['runjobs']['maxtime']]; + $process = new Process($cmd); + $process->run(function ($type, $buffer) use ($output) { + $output->write($buffer); + }); + } +}