Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adds file as source of search/replace touples for the CLI #362

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Adds file as source of search/replace touples for the CLI #362

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
385e8da
Adds file as source of search/replace touples for the CLI
May 25, 2021
1c5fe36
Updates the readme to reflect the new file cli param
May 25, 2021
336ad0b
Halt execution if there is no workload
May 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,10 @@ Type `php srdb.cli.php` to run the program. Type `php srdb.cli.php
None empty string to replace search with or `preg_replace()`
style replacement.

-f, --file
File that contains touples of search/replace strings/patterns
separated by new lines: search1\nreplace1\nsearch2\nreplace2

-t, --tables
If set only runs the script on the specified table, comma
separate for multiple values.
Expand Down
113 changes: 92 additions & 21 deletions srdb.class.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,11 @@ class icit_srdb {
*/
public $replace = false;

/**
* @var string Path to file containing tuples of search => replace
*/
public $file = false;

/**
* @var bool Use regular expressions to perform search and replace
*/
Expand Down Expand Up @@ -219,6 +224,7 @@ public function __construct( $args ) {
'port' => 3306,
'search' => '',
'replace' => '',
'file' => '',
'tables' => array(),
'exclude_tables' => array(),
'exclude_cols' => array(),
Expand Down Expand Up @@ -314,15 +320,16 @@ public function __construct( $args ) {
} // update collation
elseif ( $this->alter_collation ) {
$report = $this->update_collation( $this->alter_collation, $this->tables );
} elseif ( $this->file ) {
$searchReplaceTuples = $this->parseTuplesFromFile( $this->file );

$report = $this->replacer( $searchReplaceTuples, $this->tables, $this->exclude_tables );
} elseif ( is_array( $this->search ) ) {
$report = array();
for ( $i = 0; $i < count( $this->search ); $i ++ ) {
$report[ $i ] = $this->replacer( $this->search[ $i ], $this->replace[ $i ], $this->tables,
$this->exclude_tables );
}
$searchReplaceTuples = $this->parseTuplesFromArray( $this->search, $this->replace );

$report = $this->replacer( $searchReplaceTuples, $this->tables, $this->exclude_tables );
} else {
$report = $this->replacer( $this->search, $this->replace, $this->tables, $this->exclude_tables );
$report = $this->replacer( [ $this->search, $this->replace ], $this->tables, $this->exclude_tables );
}

} else {
Expand Down Expand Up @@ -898,18 +905,13 @@ public function preg_fix_serialised_count( $matches ) {
* We split large tables into 50,000 row blocks when dealing with them to save
* on memmory consumption.
*
* @param string $search What we want to replace
* @param string $replace What we want to replace it with.
* @param array $searchReplaceTuples Array of Tuples of What we want to replace and What we want to replace it with.
* @param array $tables The tables we want to look at.
*
* @return array|bool Collection of information gathered during the run.
*/
public function replacer( $search = '', $replace = '', $tables = array(), $exclude_tables = array() ) {
$search = (string) $search;
// check we have a search string, bail if not
if ( '' === $search ) {
$this->add_error( 'Search string is empty', 'search' );

public function replacer( $searchReplaceTuples, $tables = array(), $exclude_tables = array() ) {
if ( empty ( $searchReplaceTuples ) ) {
return false;
}

Expand Down Expand Up @@ -988,7 +990,7 @@ public function replacer( $search = '', $replace = '', $tables = array(), $exclu
$new_table_report = $table_report;
$new_table_report['start'] = microtime( true );

$this->log( 'search_replace_table_start', $table, $search, $replace );
$this->log( 'search_replace_table_start', $table, $searchReplaceTuples );

// Count the number of rows we have in the table if large we'll split into blocks, This is a mod from Simon Wheatley
$row_count = $this->db_query( "SELECT COUNT(*) FROM `{$table}`" );
Expand Down Expand Up @@ -1036,16 +1038,32 @@ public function replacer( $search = '', $replace = '', $tables = array(), $exclu
if ( ! empty( $this->include_cols ) && ! in_array( $column, $this->include_cols ) ) {
continue;
}

foreach ( $searchReplaceTuples as $searchReplaceTuple ) {
$search = (string) $searchReplaceTuple[0];
$replace = (string) $searchReplaceTuple[1];

// check we have a search string, bail if not
if ( '' === $search ) {
$this->add_error( 'Search string is empty', 'search' );

return false;
}

// Run a search replace on the data that'll respect the serialisation.
$edited_data = $this->recursive_unserialize_replace( $search, $replace, $data_to_fix );
$previous_data = $edited_data;

// Run a search replace on the data that'll respect the serialisation.
$edited_data = $this->recursive_unserialize_replace( $search, $replace, $edited_data );

if ( $previous_data !== $edited_data ) {
$report['change'] ++;
$new_table_report['change'] ++;
}
}

// Something was changed
if ( $edited_data != $data_to_fix ) {

$report['change'] ++;
$new_table_report['change'] ++;

// log first x changes
if ( $new_table_report['change'] <= $this->report_change_num ) {
$new_table_report['changes'][] = array(
Expand Down Expand Up @@ -1103,7 +1121,7 @@ public function replacer( $search = '', $replace = '', $tables = array(), $exclu

$report['end'] = microtime( true );

$this->log( 'search_replace_end', $search, $replace, $report );
$this->log( 'search_replace_end', $searchReplaceTuples, $report );

return $report;
}
Expand Down Expand Up @@ -1331,7 +1349,60 @@ public function charset_decode_utf_8( $string ) {
return $string;
}

/**
* Parses the file containing search/replacement tuples separated by new lines
* to an array
*
* @param string $filePath
*
* @return array
*/
public function parseTuplesFromFile( $filePath )
{
if (!is_file( $filePath ) || !is_readable( $filePath )) {
$this->add_error( 'The file path provided is either not a file or it is not readable.' );

return [];
}

$fileContent = file_get_contents( $filePath );
$expandedContent = preg_split("/\R/", $fileContent);

if (count( $expandedContent ) % 2 === 1) {
$this->add_error( 'The file provided does not contain pairs of search and replacement'
. ' tuples separated by new lines, the number of lines in the file is odd.' );

return [];
}

$result = [];
for ( $i = 0; $i < count( $expandedContent ); $i += 2 ) {
if ( empty( $expandedContent[ $i ] ) ) {
$this->add_error( 'The file provided contains an empty line where it should be a search string' );

return [];
}

$result []= [ $expandedContent[ $i ], $expandedContent[ $i + 1 ] ];
}

return $result;
}

public function parseTuplesFromArray( $searchArray, $replaceArrayOrString )
{
$result = [];

foreach ( $searchArray as $i => $search ) {
if ( is_array( $replaceArrayOrString ) ) {
$result []= [ $search, $replaceArrayOrString[ $i ] ];
} else {
$result []= [ $search, $replaceArrayOrString ];
}
}

return $result;
}
}


Expand Down
34 changes: 17 additions & 17 deletions srdb.cli.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@
[ 'P:', 'port:', 'Optional. Port on database server to connect to. The default is 3306. (MySQL default port).', ],
[ 's:', 'search:', 'String to search for or `preg_replace()` style regular expression.', ],
[ 'r:', 'replace:', 'None empty string to replace search with or `preg_replace()` style replacement.', ],
[
'f:',
'file:',
'File with strings or regular expressions along with their intended replacements, separated by line endings.',
],
[ 't:', 'tables:', 'If set only runs the script on the specified table, comma separate for multiple values.', ],
[ 'w:', 'exclude-tables:', 'If set excluded the specified tables, comma separate for multuple values.', ],
[
Expand Down Expand Up @@ -246,15 +251,12 @@ public function log( $type = '' ) {
$output .= "$error_type: $error";
break;
case 'search_replace_table_start':
list( $table, $search, $replace ) = $args;
list( $table, $searchReplaceTuples ) = $args;

if ( is_array( $search ) ) {
$search = implode( ' or ', $search );
}
if ( is_array( $replace ) ) {
$replace = implode( ' or ', $replace );
}
$output .= "{$table}: replacing {$search} with {$replace}";
$output .= "{$table}: replacing " . implode( ', ', array_map(
function ( $searchReplaceTuple ) { return implode( ' => ', $searchReplaceTuple ); },
$searchReplaceTuples
));

break;
case 'search_replace_table_end':
Expand All @@ -266,20 +268,18 @@ public function log( $type = '' ) {
$output .= "{$table}: {$report['rows']} rows, {$report['change']} changes found, {$report['updates']} updates made in {$time} seconds";
break;
case 'search_replace_end':
list( $search, $replace, $report ) = $args;
if ( is_array( $search ) ) {
$search = implode( ' or ', $search );
}
if ( is_array( $replace ) ) {
$replace = implode( ' or ', $replace );
}
list( $searchReplaceTuples, $report ) = $args;

$time = number_format( floatval( $report['end'] ) - floatval( $report['start'] ), 8 );
if ( $time < 0 ) {
$time = $time * - 1;
}
$dry_run_string = $this->dry_run ? "would have been" : "were";
$output .= "
Replacing {$search} with {$replace} on {$report['tables']} tables with {$report['rows']} rows

$output .= "Replacing " . implode( ', ', array_map(
function ($searchReplaceTuple) { return implode(' => ', $searchReplaceTuple); },
$searchReplaceTuples
)) . " on {$report['tables']} tables with {$report['rows']} rows
{$report['change']} changes {$dry_run_string} made
{$report['updates']} updates were actually made
It took {$time} seconds";
Expand Down