Skip to content

Commit

Permalink
Merge pull request #49 from aligent/fix/changelog_temp_table
Browse files Browse the repository at this point in the history 
Use random suffix for temp table to avoid conflicts
  • Loading branch information
@aligent-lturner
aligent-lturner authored Nov 17, 2024
2 parents 8f73e42 + 3cf2a53 commit 5a4adf9
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 30 deletions.
42 changes: 27 additions & 15 deletions src/index/Model/Changelog/InsertRecords.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,19 @@
use Aligent\FredhopperIndexer\Model\DataHandler;
use Aligent\FredhopperIndexer\Model\ResourceModel\Changelog as ChangelogResource;
use Magento\Framework\App\ResourceConnection;
use Magento\Framework\Exception\LocalizedException;

class InsertRecords
{
/**
* @param ResourceConnection $resourceConnection
* @param ChangelogResource $changelogResource
* @param TempTable $tempTable
*/
public function __construct(
private readonly ResourceConnection $resourceConnection,
private readonly ChangelogResource $changelogResource
private readonly ChangelogResource $changelogResource,
private readonly TempTable $tempTable
) {
}

Expand All @@ -24,30 +27,36 @@ public function __construct(
*
* @return void
* @throws \Zend_Db_Select_Exception
* @throws LocalizedException
*/
public function execute(): void
{
$tempTableName = $this->tempTable->getTempTableName();
$addedProductIds = $this->getAddedOrDeletedProductsByType(
true,
DataHandler::TYPE_PRODUCT
DataHandler::TYPE_PRODUCT,
$tempTableName
);
$addedVariantIds = $this->getAddedOrDeletedProductsByType(
true,
DataHandler::TYPE_VARIANT
DataHandler::TYPE_VARIANT,
$tempTableName
);
$this->changelogResource->insertAdditionOperations($addedProductIds, $addedVariantIds);

$updatedProductIds = $this->getUpdatedProductsByType(DataHandler::TYPE_PRODUCT);
$updatedVariantIds = $this->getUpdatedProductsByType(DataHandler::TYPE_VARIANT);
$updatedProductIds = $this->getUpdatedProductsByType(DataHandler::TYPE_PRODUCT, $tempTableName);
$updatedVariantIds = $this->getUpdatedProductsByType(DataHandler::TYPE_VARIANT, $tempTableName);
$this->changelogResource->insertUpdateOperations($updatedProductIds, $updatedVariantIds);

$deletedProductIds = $this->getAddedOrDeletedProductsByType(
false,
DataHandler::TYPE_PRODUCT
DataHandler::TYPE_PRODUCT,
$tempTableName
);
$deletedVariantIds = $this->getAddedOrDeletedProductsByType(
false,
DataHandler::TYPE_VARIANT
DataHandler::TYPE_VARIANT,
$tempTableName
);
$this->changelogResource->insertDeleteOperations($deletedProductIds, $deletedVariantIds);
}
Expand All @@ -57,21 +66,23 @@ public function execute(): void
*
* @param bool $isAddition
* @param string $productType
* @param string $tempTableName
* @return array
*/
private function getAddedOrDeletedProductsByType(
bool $isAddition,
string $productType
string $productType,
string $tempTableName
): array {
$connection = $this->resourceConnection->getConnection();
$select = $connection->select();

$select->from(
['main_table' => ($isAddition ? DataHandler::INDEX_TABLE_NAME : TempTable::TEMP_TABLE_NAME)],
['main_table' => ($isAddition ? DataHandler::INDEX_TABLE_NAME : $tempTableName)],
['product_id']
);
$select->joinLeft(
['temp_table' => ($isAddition ? TempTable::TEMP_TABLE_NAME : DataHandler::INDEX_TABLE_NAME)],
['temp_table' => ($isAddition ? $tempTableName : DataHandler::INDEX_TABLE_NAME)],
'temp_table.product_id = main_table.product_id AND '.
'temp_table.product_type = main_table.product_type',
[]
Expand All @@ -87,16 +98,17 @@ private function getAddedOrDeletedProductsByType(
* Determine which products have been updated between the main and temporary table
*
* @param string $productType
* @param string $tempTableName
* @return array
* @throws \Zend_Db_Select_Exception
*/
private function getUpdatedProductsByType(string $productType): array
private function getUpdatedProductsByType(string $productType, string $tempTableName): array
{
// get all product ids and variant ids that exist in both tables
// we do not want to consider products that are being added or deleted completely
$connection = $this->resourceConnection->getConnection();
$existingProductsSelect = $connection->select();
$existingProductsSelect->from(['temp_table' => TempTable::TEMP_TABLE_NAME], ['product_id']);
$existingProductsSelect->from(['temp_table' => $tempTableName], ['product_id']);
$existingProductsSelect->joinInner(
['main_table' => DataHandler::INDEX_TABLE_NAME],
'main_table.product_id = temp_table.product_id AND ' .
Expand All @@ -114,7 +126,7 @@ private function getUpdatedProductsByType(string $productType): array
['product_id']
);
$existingProductsTempMissingSelect->joinLeft(
['temp_table' => TempTable::TEMP_TABLE_NAME],
['temp_table' => $tempTableName],
'main_table.product_id = temp_table.product_id AND ' .
'main_table.product_type = temp_table.product_type AND ' .
'main_table.store_id = temp_table.store_id',
Expand All @@ -127,7 +139,7 @@ private function getUpdatedProductsByType(string $productType): array
// records that are in the old table, but not in the new table
$existingProductsMainMissingSelect = $connection->select();
$existingProductsMainMissingSelect->from(
['temp_table' => TempTable::TEMP_TABLE_NAME],
['temp_table' => $tempTableName],
['product_id']
);
$existingProductsMainMissingSelect->joinLeft(
Expand All @@ -148,7 +160,7 @@ private function getUpdatedProductsByType(string $productType): array
['product_id']
);
$existingProductsDifferenceSelect->joinInner(
['temp_table' => TempTable::TEMP_TABLE_NAME],
['temp_table' => $tempTableName],
'main_table.product_id = temp_table.product_id AND ' .
'main_table.product_type = temp_table.product_type AND ' .
'main_table.store_id = temp_table.store_id AND '.
Expand Down
65 changes: 61 additions & 4 deletions src/index/Model/Changelog/TempTable.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,30 +5,82 @@

use Aligent\FredhopperIndexer\Model\DataHandler;
use Magento\Framework\App\ResourceConnection;
use Magento\Framework\Exception\LocalizedException;
use Random\RandomException;

class TempTable
{
public const string TEMP_TABLE_NAME = DataHandler::INDEX_TABLE_NAME . '_temp';
private const string TEMP_TABLE_PREFIX = DataHandler::INDEX_TABLE_NAME . '_temp_';

private string $tempTableName;

/**
* @param ResourceConnection $resourceConnection
*/
public function __construct(
private readonly ResourceConnection $resourceConnection
) {
}

/**
* Gets the current temporary table name
*
* @return string
* @throws LocalizedException
*/
public function getTempTableName(): string
{
if (!isset($this->tempTableName)) {
throw new LocalizedException(__(__METHOD__ . ': temp table name not set'));
}
return $this->tempTableName;
}

/**
* Sets the current temporary table to a unique value
*
* @return void
* @throws LocalizedException
*/
public function generateTempTableName(): void
{
if (isset($this->tempTableName)) {
throw new LocalizedException(__(__METHOD__ . ': temp table name already set'));
}
try {
$tempTableName = self::TEMP_TABLE_PREFIX . bin2hex(random_bytes(4));
} catch (RandomException) {
$tempTableName = uniqid(self::TEMP_TABLE_PREFIX, true);
}
$this->tempTableName = $tempTableName;
}

/**
* Creates a temporary copy of the index table for use in generating changelog records
*
* @return void
* @throws LocalizedException
*/
public function create(): void
{
if (!isset($this->tempTableName)) {
throw new LocalizedException(__(__METHOD__ . ': temp table name not set'));
}
$connection = $this->resourceConnection->getConnection();
$connection->createTemporaryTableLike(self::TEMP_TABLE_NAME, DataHandler::INDEX_TABLE_NAME);
$table = $connection->createTableByDdl(DataHandler::INDEX_TABLE_NAME, $this->tempTableName);
if ($connection->isTableExists($this->tempTableName)) {
throw new LocalizedException(__(__METHOD__ . ': temp table already exists'));
}
try {
$connection->createTable($table);
} catch (\Exception $e) {
throw new LocalizedException(__(__METHOD__ . ': ' . $e->getMessage()), $e);
}
$copySelect = $connection->select();
$copySelect->from(DataHandler::INDEX_TABLE_NAME);
$copyInsert = $connection->insertFromSelect(
$copySelect,
self::TEMP_TABLE_NAME
$this->tempTableName
);
$connection->query($copyInsert);
}
Expand All @@ -37,9 +89,14 @@ public function create(): void
* Drops the temporary table if it exists
*
* @return void
* @throws LocalizedException
*/
public function drop(): void
{
$this->resourceConnection->getConnection()->dropTemporaryTable(self::TEMP_TABLE_NAME);
if (!isset($this->tempTableName)) {
throw new LocalizedException(__(__METHOD__ . ': temp table name not set'));
}
$this->resourceConnection->getConnection()->dropTable($this->tempTableName);
unset($this->tempTableName);
}
}
28 changes: 17 additions & 11 deletions src/index/Model/ProductIndexer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,25 +70,31 @@ public function executeFull(): void
/**
* @inheritDoc
*
* @throws LocalizedException
*/
public function executeList(array $ids): void
{
// create temporary table to handle changelogs
$this->tempTable->create();
foreach ($this->dimensionProvider->getIterator() as $dimension) {
try {
// create temporary table to handle changelogs
$this->tempTable->generateTempTableName();
$this->tempTable->create();
// try block here is nested to ensure that if the table was created, it gets dropped at the end
try {
$this->executeByDimensions($dimension, new \ArrayIterator($ids));
} catch (FileSystemException|RuntimeException) {
continue;
foreach ($this->dimensionProvider->getIterator() as $dimension) {
try {
$this->executeByDimensions($dimension, new \ArrayIterator($ids));
} catch (FileSystemException|RuntimeException) {
continue;
}
}
$this->insertChangelogRecords->execute();
} finally {
// we want to ensure that the "temporary" table is always dropped
$this->tempTable->drop();
}
}
try {
$this->insertChangelogRecords->execute();
} catch (\Exception $e) {
$this->logger->critical($e->getMessage(), ['exception' => $e]);
}
$this->tempTable->drop();

}

/**
Expand Down

0 comments on commit 5a4adf9

Please sign in to comment.