Skip to content

Commit

Permalink
Merge pull request #50 from aligent/feature/convert_temp_table_to_rep…
Browse files Browse the repository at this point in the history 
…lica

Replace creating/dropping of table with inserting/deleting of records…
  • Loading branch information
@aligent-lturner
aligent-lturner authored Nov 18, 2024
2 parents 5a4adf9 + 68ae7ba commit fa0323e
Show file tree
Hide file tree
Showing 6 changed files with 172 additions and 149 deletions.
78 changes: 46 additions & 32 deletions src/index/Model/Changelog/InsertRecords.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,57 +6,53 @@
use Aligent\FredhopperIndexer\Model\DataHandler;
use Aligent\FredhopperIndexer\Model\ResourceModel\Changelog as ChangelogResource;
use Magento\Framework\App\ResourceConnection;
use Magento\Framework\Exception\LocalizedException;

class InsertRecords
{
/**
* @param ResourceConnection $resourceConnection
* @param ChangelogResource $changelogResource
* @param TempTable $tempTable
*/
public function __construct(
private readonly ResourceConnection $resourceConnection,
private readonly ChangelogResource $changelogResource,
private readonly TempTable $tempTable
) {
}

/**
* Insert add, update and delete records into the changelog table
*
* @param string $replicaId
* @return void
* @throws \Zend_Db_Select_Exception
* @throws LocalizedException
*/
public function execute(): void
public function execute(string $replicaId): void
{
$tempTableName = $this->tempTable->getTempTableName();
$addedProductIds = $this->getAddedOrDeletedProductsByType(
true,
DataHandler::TYPE_PRODUCT,
$tempTableName
$replicaId
);
$addedVariantIds = $this->getAddedOrDeletedProductsByType(
true,
DataHandler::TYPE_VARIANT,
$tempTableName
$replicaId
);
$this->changelogResource->insertAdditionOperations($addedProductIds, $addedVariantIds);

$updatedProductIds = $this->getUpdatedProductsByType(DataHandler::TYPE_PRODUCT, $tempTableName);
$updatedVariantIds = $this->getUpdatedProductsByType(DataHandler::TYPE_VARIANT, $tempTableName);
$updatedProductIds = $this->getUpdatedProductsByType(DataHandler::TYPE_PRODUCT, $replicaId);
$updatedVariantIds = $this->getUpdatedProductsByType(DataHandler::TYPE_VARIANT, $replicaId);
$this->changelogResource->insertUpdateOperations($updatedProductIds, $updatedVariantIds);

$deletedProductIds = $this->getAddedOrDeletedProductsByType(
false,
DataHandler::TYPE_PRODUCT,
$tempTableName
$replicaId
);
$deletedVariantIds = $this->getAddedOrDeletedProductsByType(
false,
DataHandler::TYPE_VARIANT,
$tempTableName
$replicaId
);
$this->changelogResource->insertDeleteOperations($deletedProductIds, $deletedVariantIds);
}
Expand All @@ -66,27 +62,41 @@ public function execute(): void
*
* @param bool $isAddition
* @param string $productType
* @param string $tempTableName
* @param string $replicaId
* @return array
*/
private function getAddedOrDeletedProductsByType(
bool $isAddition,
string $productType,
string $tempTableName
string $replicaId
): array {
$connection = $this->resourceConnection->getConnection();
$select = $connection->select();

$select->from(
['main_table' => ($isAddition ? DataHandler::INDEX_TABLE_NAME : $tempTableName)],
['product_id']
);
$select->joinLeft(
['temp_table' => ($isAddition ? $tempTableName : DataHandler::INDEX_TABLE_NAME)],
'temp_table.product_id = main_table.product_id AND '.
'temp_table.product_type = main_table.product_type',
[]
);
if ($isAddition) {
$select->from(
['main_table' => DataHandler::INDEX_TABLE_NAME],
['product_id']
);
$select->joinLeft(
['temp_table' => ReplicaTableMaintainer::REPLICA_TABLE_NAME],
'temp_table.product_id = main_table.product_id AND '.
'temp_table.product_type = main_table.product_type AND '.
'temp_table.replica_id = ' . "'" . $replicaId . "'"
);
} else {
$select->from(
['main_table' => ReplicaTableMaintainer::REPLICA_TABLE_NAME],
['product_id']
);
$select->joinLeft(
['temp_table' => DataHandler::INDEX_TABLE_NAME],
'temp_table.product_id = main_table.product_id AND '.
'temp_table.product_type = main_table.product_type'
);
$select->where('main_table.replica_id = ?', $replicaId);
}

$select->where('temp_table.product_id is null');
$select->where('main_table.product_type = ?', $productType);
$select->group('main_table.product_id');
Expand All @@ -95,20 +105,20 @@ private function getAddedOrDeletedProductsByType(
}

/**
* Determine which products have been updated between the main and temporary table
* Determine which products have been updated between the main and replica table
*
* @param string $productType
* @param string $tempTableName
* @param string $replicaId
* @return array
* @throws \Zend_Db_Select_Exception
*/
private function getUpdatedProductsByType(string $productType, string $tempTableName): array
private function getUpdatedProductsByType(string $productType, string $replicaId): array
{
// get all product ids and variant ids that exist in both tables
// we do not want to consider products that are being added or deleted completely
$connection = $this->resourceConnection->getConnection();
$existingProductsSelect = $connection->select();
$existingProductsSelect->from(['temp_table' => $tempTableName], ['product_id']);
$existingProductsSelect->from(['temp_table' => ReplicaTableMaintainer::REPLICA_TABLE_NAME], ['product_id']);
$existingProductsSelect->joinInner(
['main_table' => DataHandler::INDEX_TABLE_NAME],
'main_table.product_id = temp_table.product_id AND ' .
Expand All @@ -117,6 +127,7 @@ private function getUpdatedProductsByType(string $productType, string $tempTable
);
$existingProductsSelect->distinct();
$existingProductsSelect->where('temp_table.product_type = ?', $productType);
$existingProductsSelect->where('temp_table.replica_id = ?', $replicaId);
$existingProductIds = $connection->fetchCol($existingProductsSelect);

// records that are in the new table, but not in the old table
Expand All @@ -126,10 +137,11 @@ private function getUpdatedProductsByType(string $productType, string $tempTable
['product_id']
);
$existingProductsTempMissingSelect->joinLeft(
['temp_table' => $tempTableName],
['temp_table' => ReplicaTableMaintainer::REPLICA_TABLE_NAME],
'main_table.product_id = temp_table.product_id AND ' .
'main_table.product_type = temp_table.product_type AND ' .
'main_table.store_id = temp_table.store_id',
'main_table.store_id = temp_table.store_id AND ' .
'temp_table.replica_id = ' . "'" . $replicaId . "'",
[]
);
$existingProductsTempMissingSelect->where('temp_table.product_id IS NULL');
Expand All @@ -139,7 +151,7 @@ private function getUpdatedProductsByType(string $productType, string $tempTable
// records that are in the old table, but not in the new table
$existingProductsMainMissingSelect = $connection->select();
$existingProductsMainMissingSelect->from(
['temp_table' => $tempTableName],
['temp_table' => ReplicaTableMaintainer::REPLICA_TABLE_NAME],
['product_id']
);
$existingProductsMainMissingSelect->joinLeft(
Expand All @@ -152,6 +164,7 @@ private function getUpdatedProductsByType(string $productType, string $tempTable
$existingProductsMainMissingSelect->where('main_table.product_id IS NULL');
$existingProductsMainMissingSelect->where('temp_table.product_type = ?', $productType);
$existingProductsMainMissingSelect->where('temp_table.product_id in (?)', $existingProductIds);
$existingProductsMainMissingSelect->where('temp_table.replica_id = ?', $replicaId);

// records that differ by parent_id or attribute_data
$existingProductsDifferenceSelect = $connection->select();
Expand All @@ -160,10 +173,11 @@ private function getUpdatedProductsByType(string $productType, string $tempTable
['product_id']
);
$existingProductsDifferenceSelect->joinInner(
['temp_table' => $tempTableName],
['temp_table' => ReplicaTableMaintainer::REPLICA_TABLE_NAME],
'main_table.product_id = temp_table.product_id AND ' .
'main_table.product_type = temp_table.product_type AND ' .
'main_table.store_id = temp_table.store_id AND '.
'temp_table.replica_id = ' . "'" . $replicaId . "'" . ' AND ' .
'NOT (main_table.parent_id <=> temp_table.parent_id AND ' .
'main_table.attribute_data <=> temp_table.attribute_data)',
[]
Expand Down
83 changes: 83 additions & 0 deletions src/index/Model/Changelog/ReplicaTableMaintainer.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
<?php

declare(strict_types=1);
namespace Aligent\FredhopperIndexer\Model\Changelog;

use Aligent\FredhopperIndexer\Model\DataHandler;
use Magento\Framework\App\ResourceConnection;
use Random\RandomException;

class ReplicaTableMaintainer
{

public const REPLICA_TABLE_NAME = DataHandler::INDEX_TABLE_NAME . '_replica';

/**
* @param ResourceConnection $resourceConnection
*/
public function __construct(
private readonly ResourceConnection $resourceConnection
) {
}

/**
* Generate a random string of characters to use as an ID when comparing records before/after indexing
*
* @return string
*/
public function generateUniqueId(): string
{
try {
$id = bin2hex(random_bytes(8));
} catch (RandomException) {
$id = uniqid( '',true);
}
return substr($id, 0, 8);
}

/**
* Insert records from the main index table into the replica table using the given ID
*
* @param string $id
* @return void
*/
public function insertRecords(string $id): void
{
$connection = $this->resourceConnection->getConnection();
$copySelect = $connection->select();
$selectColumns = [
'replica_id' => new \Zend_Db_Expr("'$id'"),
'store_id' => 'store_id',
'product_type' => 'product_type',
'product_id' => 'product_id',
'parent_id' => 'parent_id',
'attribute_data' => 'attribute_data',

];
$copySelect->from(
DataHandler::INDEX_TABLE_NAME,
$selectColumns
);

$copyInsert = $connection->insertFromSelect(
$copySelect,
self::REPLICA_TABLE_NAME
);
$connection->query($copyInsert);
}

/**
* Delete records from the replica table with the given ID
*
* @param string $id
* @return void
*/
public function deleteRecords(string $id): void
{
$connection = $this->resourceConnection->getConnection();
$connection->delete(
self::REPLICA_TABLE_NAME,
['replica_id = ?' => $id]
);
}
}
102 changes: 0 additions & 102 deletions src/index/Model/Changelog/TempTable.php
View file

This file was deleted.

Loading

0 comments on commit fa0323e

Please sign in to comment.