From 6ac21aac98346946caddb276b6a491b8554929c2 Mon Sep 17 00:00:00 2001 From: Andrey Nikolaev Date: Sun, 3 Jan 2016 14:48:15 +0300 Subject: [PATCH] Bug fixes --- Model/Source/Config/Reader.php | 6 +- Model/Source/Type/AbstractType.php | 2 +- Model/Source/Type/Ftp.php | 5 + Plugin/Block/Import/Form.php | 5 +- Plugin/Model/Import.php | 8 +- Plugin/Model/Import/Product.php | 207 ++++++++---------- etc/import.xml | 2 +- .../templates/import/form/before.phtml | 2 +- 8 files changed, 109 insertions(+), 128 deletions(-) diff --git a/Model/Source/Config/Reader.php b/Model/Source/Config/Reader.php index 3d4a513..b18e2f7 100755 --- a/Model/Source/Config/Reader.php +++ b/Model/Source/Config/Reader.php @@ -27,9 +27,9 @@ public function __construct( $this->_fileName = $fileName; $this->_idAttributes = array_replace($this->_idAttributes, $idAttributes); $this->_schemaFile = $schemaLocator->getSchema(); - $this->_isValidated = $validationState->isValidated(); - $this->_perFileSchema = $schemaLocator->getPerFileSchema() && - $this->_isValidated ? $schemaLocator->getPerFileSchema() : null; + $this->validationState = $validationState; + $this->_perFileSchema = $schemaLocator->getPerFileSchema() && $validationState->isValidationRequired() + ? $schemaLocator->getPerFileSchema() : null; $this->_domDocumentClass = $domDocumentClass; $this->_defaultScope = $defaultScope; } diff --git a/Model/Source/Type/AbstractType.php b/Model/Source/Type/AbstractType.php index 2f039bf..eae0422 100755 --- a/Model/Source/Type/AbstractType.php +++ b/Model/Source/Type/AbstractType.php @@ -8,7 +8,7 @@ use Magento\Framework\App\Filesystem\DirectoryList; -abstract class AbstractType extends \Magento\Framework\Object { +abstract class AbstractType extends \Magento\Framework\DataObject { const IMPORT_DIR = 'var/import'; diff --git a/Model/Source/Type/Ftp.php b/Model/Source/Type/Ftp.php index 09bea67..1c3d169 100755 --- a/Model/Source/Type/Ftp.php +++ b/Model/Source/Type/Ftp.php @@ -16,6 +16,11 @@ public function uploadSource() $sourceFilePath = $this->getData($this->_code . '_file_path'); $fileName = basename($sourceFilePath); $filePath = $this->_directory->getAbsolutePath($this->getImportPath() . '/' . $fileName); + + $filesystem = new \Magento\Framework\Filesystem\Io\File(); + $filesystem->setAllowCreateFolders(true); + $filesystem->checkAndCreateFolder($this->_directory->getAbsolutePath($this->getImportPath())); + $result = $client->read($sourceFilePath, $filePath); if($result) { diff --git a/Plugin/Block/Import/Form.php b/Plugin/Block/Import/Form.php index 0050420..54b0b2f 100755 --- a/Plugin/Block/Import/Form.php +++ b/Plugin/Block/Import/Form.php @@ -28,7 +28,10 @@ public function beforeSetForm(\Magento\ImportExport\Block\Adminhtml\Import\Edit\ $fileFieldset->setClass('source-fieldset ' . $oldClass); $types = $this->_config->get(); - $sources = [['label' => __('-- Please Select --'), 'value' => '']]; + $sources = [ + ['label' => __('-- Please Select --'), 'value' => ''], + ['label' => __('File'), 'value' => 'file'] + ]; foreach ($types as $typeName => $type) { $sources[] = ['label' => $type['label'], 'value' => $typeName]; } diff --git a/Plugin/Model/Import.php b/Plugin/Model/Import.php index b97fe19..8d61443 100755 --- a/Plugin/Model/Import.php +++ b/Plugin/Model/Import.php @@ -27,7 +27,6 @@ class Import extends \Magento\ImportExport\Model\Import { * @param \Magento\Framework\App\Config\ScopeConfigInterface $coreConfig * @param \Magento\ImportExport\Model\Import\ConfigInterface $importConfig * @param \Magento\ImportExport\Model\Import\Entity\Factory $entityFactory - * @param \Magento\ImportExport\Model\Resource\Import\Data $importData * @param \Magento\ImportExport\Model\Export\Adapter\CsvFactory $csvFactory * @param \Magento\Framework\HTTP\Adapter\FileTransferFactory $httpFactory * @param \Magento\MediaStorage\Model\File\UploaderFactory $uploaderFactory @@ -47,12 +46,12 @@ public function __construct( \Magento\Framework\App\Config\ScopeConfigInterface $coreConfig, \Magento\ImportExport\Model\Import\ConfigInterface $importConfig, \Magento\ImportExport\Model\Import\Entity\Factory $entityFactory, - \Magento\ImportExport\Model\Resource\Import\Data $importData, + \Magento\ImportExport\Model\ResourceModel\Import\Data $importData, \Magento\ImportExport\Model\Export\Adapter\CsvFactory $csvFactory, \Magento\Framework\HTTP\Adapter\FileTransferFactory $httpFactory, \Magento\MediaStorage\Model\File\UploaderFactory $uploaderFactory, \Magento\ImportExport\Model\Source\Import\Behavior\Factory $behaviorFactory, - \Magento\Indexer\Model\IndexerRegistry $indexerRegistry, + \Magento\Framework\Indexer\IndexerRegistry $indexerRegistry, \Magento\ImportExport\Model\History $importHistoryModel, \Magento\Framework\Stdlib\DateTime\DateTime $localeDate, array $data = [] @@ -88,7 +87,8 @@ public function uploadSource() { $result = null; - if($sourceType = $this->getImportSource()) { + if($this->getImportSource() && $this->getImportSource() != 'file') { + $sourceType = $this->getImportSource(); $source = $this->_helper->getSourceModelByType($sourceType); $source->setData($this->getData()); diff --git a/Plugin/Model/Import/Product.php b/Plugin/Model/Import/Product.php index 4c6019b..2424b41 100755 --- a/Plugin/Model/Import/Product.php +++ b/Plugin/Model/Import/Product.php @@ -3,6 +3,9 @@ use Magento\Framework\Stdlib\DateTime; use Magento\CatalogImportExport\Model\Import\Product as MagentoProduct; +use Magento\ImportExport\Model\Import; +use Magento\Framework\Model\ResourceModel\Db\TransactionManagerInterface; +use Magento\Framework\Model\ResourceModel\Db\ObjectRelationProcessor; class Product extends \Magento\CatalogImportExport\Model\Import\Product { @@ -17,37 +20,39 @@ public function __construct( \Firebear\ImportExport\Helper\Data $helper, \Magento\Framework\Json\Helper\Data $jsonHelper, \Magento\ImportExport\Helper\Data $importExportData, - \Magento\ImportExport\Model\Resource\Import\Data $importData, + \Magento\ImportExport\Model\ResourceModel\Import\Data $importData, \Magento\Eav\Model\Config $config, - \Magento\Framework\App\Resource $resource, - \Magento\ImportExport\Model\Resource\Helper $resourceHelper, - \Magento\Framework\Stdlib\String $string, + \Magento\Framework\App\ResourceConnection $resource, + \Magento\ImportExport\Model\ResourceModel\Helper $resourceHelper, + \Magento\Framework\Stdlib\StringUtils $string, + \Magento\ImportExport\Model\Import\ErrorProcessing\ProcessingErrorAggregatorInterface $errorAggregator, \Magento\Framework\Event\ManagerInterface $eventManager, \Magento\CatalogInventory\Api\StockRegistryInterface $stockRegistry, \Magento\CatalogInventory\Api\StockConfigurationInterface $stockConfiguration, \Magento\CatalogInventory\Model\Spi\StockStateProviderInterface $stockStateProvider, \Magento\Catalog\Helper\Data $catalogData, \Magento\ImportExport\Model\Import\Config $importConfig, - \Magento\CatalogImportExport\Model\Import\Proxy\Product\ResourceFactory $resourceFactory, + \Magento\CatalogImportExport\Model\Import\Proxy\Product\ResourceModelFactory $resourceFactory, \Magento\CatalogImportExport\Model\Import\Product\OptionFactory $optionFactory, - \Magento\Eav\Model\Resource\Entity\Attribute\Set\CollectionFactory $setColFactory, + \Magento\Eav\Model\ResourceModel\Entity\Attribute\Set\CollectionFactory $setColFactory, \Magento\CatalogImportExport\Model\Import\Product\Type\Factory $productTypeFactory, - \Magento\Catalog\Model\Resource\Product\LinkFactory $linkFactory, + \Magento\Catalog\Model\ResourceModel\Product\LinkFactory $linkFactory, \Magento\CatalogImportExport\Model\Import\Proxy\ProductFactory $proxyProdFactory, \Magento\CatalogImportExport\Model\Import\UploaderFactory $uploaderFactory, \Magento\Framework\Filesystem $filesystem, - \Magento\CatalogInventory\Model\Resource\Stock\ItemFactory $stockResItemFac, + \Magento\CatalogInventory\Model\ResourceModel\Stock\ItemFactory $stockResItemFac, \Magento\Framework\Stdlib\DateTime\TimezoneInterface $localeDate, DateTime $dateTime, \Psr\Log\LoggerInterface $logger, - \Magento\Indexer\Model\IndexerRegistry $indexerRegistry, - MagentoProduct\StoreResolver $storeResolver, - MagentoProduct\SkuProcessor $skuProcessor, - MagentoProduct\CategoryProcessor $categoryProcessor, - MagentoProduct\Validator $validator, - \Magento\Framework\Model\Resource\Db\ObjectRelationProcessor $objectRelationProcessor, - \Magento\Framework\Model\Resource\Db\TransactionManagerInterface $transactionManager, - MagentoProduct\TaxClassProcessor $taxClassProcessor, + \Magento\Framework\Indexer\IndexerRegistry $indexerRegistry, + \Magento\CatalogImportExport\Model\Import\Product\StoreResolver $storeResolver, + \Magento\CatalogImportExport\Model\Import\Product\SkuProcessor $skuProcessor, + \Magento\CatalogImportExport\Model\Import\Product\CategoryProcessor $categoryProcessor, + \Magento\CatalogImportExport\Model\Import\Product\Validator $validator, + ObjectRelationProcessor $objectRelationProcessor, + TransactionManagerInterface $transactionManager, + \Magento\CatalogImportExport\Model\Import\Product\TaxClassProcessor $taxClassProcessor, + \Magento\Framework\Model\Entity\MetadataPool $metadataPool, array $data = [] ){ $this->_request = $request; @@ -61,6 +66,7 @@ public function __construct( $resource, $resourceHelper, $string, + $errorAggregator, $eventManager, $stockRegistry, $stockConfiguration, @@ -87,6 +93,7 @@ public function __construct( $objectRelationProcessor, $transactionManager, $taxClassProcessor, + $metadataPool, $data ); } @@ -134,8 +141,8 @@ protected function _saveProducts() if($this->_sourceType) { $bunch = $this->_prepareImagesFromSource($bunch); } - $allImagesFromBunch = $this->_getAllBunchImages($bunch); - $existingImages = $this->_prepareAllMediaFiles($allImagesFromBunch); + $bunchImages = $this->getBunchImages($bunch); + $existingImages = $this->getExistingImages($bunchImages); foreach ($bunch as $rowNum => $rowData) { if (!$this->validateRow($rowData, $rowNum)) { @@ -146,7 +153,7 @@ protected function _saveProducts() $rowSku = $rowData[self::COL_SKU]; if (null === $rowSku) { - $this->_rowsToSkip[$rowNum] = true; + $this->getErrorAggregator()->addRowToSkip($rowNum); // skip rows when SKU is NULL continue; } elseif (self::SCOPE_STORE == $rowScope) { @@ -177,12 +184,15 @@ protected function _saveProducts() } else { $rowSku = null; // sign for child rows to be skipped - $this->_rowsToSkip[$rowNum] = true; + $this->getErrorAggregator()->addRowToSkip($rowNum); continue; } } - $this->websitesCache[$rowSku] = []; + if (!array_key_exists($rowSku, $this->websitesCache)) { + $this->websitesCache[$rowSku] = []; + } + // 2. Product-to-Website phase if (!empty($rowData[self::COL_PRODUCT_WEBSITES])) { $websiteCodes = explode($this->getMultipleValueSeparator(), $rowData[self::COL_PRODUCT_WEBSITES]); @@ -193,12 +203,12 @@ protected function _saveProducts() } // 3. Categories phase - $categoriesString = empty($rowData[self::COL_CATEGORY]) ? '' : $rowData[self::COL_CATEGORY]; - $this->categoriesCache[$rowSku] = []; - if (!empty($categoriesString)) { - foreach ($this->categoryProcessor->upsertCategories($categoriesString) as $categoryId) { - $this->categoriesCache[$rowSku][$categoryId] = true; - } + if (!array_key_exists($rowSku, $this->categoriesCache)) { + $this->categoriesCache[$rowSku] = []; + } + $categoryIds = $this->processRowCategories($rowData); + foreach ($categoryIds as $id) { + $this->categoriesCache[$rowSku][$id] = true; } // 4.1. Tier prices phase @@ -214,6 +224,10 @@ protected function _saveProducts() ]; } + if (!$this->validateRow($rowData, $rowNum)) { + continue; + } + // 4.2. Group prices phase if (!empty($rowData['_group_price_website'])) { $groupPrices[$rowSku][] = [ @@ -231,83 +245,52 @@ protected function _saveProducts() } // 5. Media gallery phase - $mediaGalleryImages = array(); - $mediaGalleryLabels = array(); - if (!empty($rowData[self::COL_MEDIA_IMAGE])) { - $mediaGalleryImages = - explode($this->getMultipleValueSeparator(), $rowData[self::COL_MEDIA_IMAGE]); - if (isset($rowData['_media_image_label'])) { - $mediaGalleryLabels = - explode($this->getMultipleValueSeparator(), $rowData['_media_image_label']); - } else { - $mediaGalleryLabels = []; - } - if (count($mediaGalleryLabels) > count($mediaGalleryImages)) { - $mediaGalleryLabels = array_slice($mediaGalleryLabels, 0, count($mediaGalleryImages)); - } elseif (count($mediaGalleryLabels) < count($mediaGalleryImages)) { - $mediaGalleryLabels = array_pad($mediaGalleryLabels, count($mediaGalleryImages), ''); - } + $disabledImages = []; + list($rowImages, $rowLabels) = $this->getImagesFromRow($rowData); + if (isset($rowData['_media_is_disabled'])) { + $disabledImages = array_flip( + explode($this->getMultipleValueSeparator(), $rowData['_media_is_disabled']) + ); } - - foreach ($this->_imagesArrayKeys as $imageCol) { - if (!empty($rowData[$imageCol]) - && ($imageCol != self::COL_MEDIA_IMAGE) - && !in_array($rowData[$imageCol], $mediaGalleryImages) - ) { - $mediaGalleryImages[] = $rowData[$imageCol]; - if (isset($mediaGalleryLabels)) { - $mediaGalleryLabels[] = isset($rowData[$imageCol . '_label']) - ? $rowData[$imageCol . '_label'] - : ''; + $rowData[self::COL_MEDIA_IMAGE] = []; + foreach ($rowImages as $column => $columnImages) { + foreach ($columnImages as $position => $columnImage) { + if (!isset($uploadedImages[$columnImage])) { + $uploadedFile = $this->uploadMediaFiles(trim($columnImage), true); + if ($uploadedFile) { + $uploadedImages[$columnImage] = $uploadedFile; + } else { + $this->addRowError( + ValidatorInterface::ERROR_MEDIA_URL_NOT_ACCESSIBLE, + $rowNum, + null, + null, + ProcessingError::ERROR_LEVEL_NOT_CRITICAL + ); + } } else { - $mediaGalleryLabels[] = ''; - } - } - } - $rowData[self::COL_MEDIA_IMAGE] = array(); - foreach ($mediaGalleryImages as $mediaImage) { - $imagePath = $allImagesFromBunch[$mediaImage]; - if (isset($existingImages[$imagePath]) && in_array($rowSku, $existingImages[$imagePath])) { - if (!array_key_exists($mediaImage, $uploadedGalleryFiles)) { - $uploadedGalleryFiles[$mediaImage] = $this->_uploadMediaFiles( - trim($mediaImage), - true - ); + $uploadedFile = $uploadedImages[$columnImage]; } - } elseif (!isset($existingImages[$imagePath])) { - if (!array_key_exists($mediaImage, $uploadedGalleryFiles)) { - $uploadedGalleryFiles[$mediaImage] = $this->_uploadMediaFiles( - trim($mediaImage), - true - ); - $newImagePath = $uploadedGalleryFiles[$mediaImage]; - $existingImages[$newImagePath][] = $rowSku; + + if ($uploadedFile && $column !== self::COL_MEDIA_IMAGE) { + $rowData[$column] = $uploadedFile; } - $rowData[self::COL_MEDIA_IMAGE][] = $uploadedGalleryFiles[$mediaImage]; - if (!empty($rowData[self::COL_MEDIA_IMAGE]) && is_array($rowData[self::COL_MEDIA_IMAGE])) { - $position = array_search($mediaImage, $mediaGalleryImages); - foreach ($rowData[self::COL_MEDIA_IMAGE] as $mediaImage) { - $mediaGallery[$rowSku][] = [ - 'attribute_id' => $this->getMediaGalleryAttributeId(), - 'label' => isset($mediaGalleryLabels[$position]) ? $mediaGalleryLabels[$position] : '', - 'position' => $position, - 'disabled' => '', - 'value' => $mediaImage, - ]; + + $imageNotAssigned = !isset($existingImages[$uploadedFile]) + || !in_array($rowSku, $existingImages[$uploadedFile]); + + if ($uploadedFile && $imageNotAssigned) { + if ($column == self::COL_MEDIA_IMAGE) { + $rowData[$column][] = $uploadedFile; } - } - } - foreach ($this->_imagesArrayKeys as $imageCol) { - if (empty($rowData[$imageCol]) || ($imageCol == self::COL_MEDIA_IMAGE)) { - continue; - } - if (isset($existingImages[$imagePath]) - && !in_array($rowSku, $existingImages[$imagePath]) - && (($rowData[$imageCol] == $imagePath) || ($rowData[$imageCol] == $mediaImage)) - ) { - unset($rowData[$imageCol]); - } elseif (isset($uploadedGalleryFiles[$rowData[$imageCol]])) { - $rowData[$imageCol] = $uploadedGalleryFiles[$rowData[$imageCol]]; + $mediaGallery[$rowSku][] = [ + 'attribute_id' => $this->getMediaGalleryAttributeId(), + 'label' => isset($rowLabels[$column][$position]) ? $rowLabels[$column][$position] : '', + 'position' => $position + 1, + 'disabled' => isset($disabledImages[$columnImage]) ? '1' : '0', + 'value' => $uploadedFile, + ]; + $existingImages[$uploadedFile][] = $rowSku; } } } @@ -342,7 +325,7 @@ protected function _saveProducts() $this->taxClassProcessor->upsertTaxClass($rowData['tax_class_name'], $productTypeModel); } - if ($this->getBehavior() == \Magento\ImportExport\Model\Import::BEHAVIOR_APPEND || + if ($this->getBehavior() == Import::BEHAVIOR_APPEND || empty($rowData[self::COL_SKU]) ) { $rowData = $productTypeModel->clearEmptyData($rowData); @@ -355,10 +338,7 @@ protected function _saveProducts() $product = $this->_proxyProdFactory->create(['data' => $rowData]); foreach ($rowData as $attrCode => $attrValue) { - if (!isset($this->_attributeCache[$attrCode])) { - $this->_attributeCache[$attrCode] = $resource->getAttribute($attrCode); - } - $attribute = $this->_attributeCache[$attrCode]; + $attribute = $this->retrieveAttributeByCode($attrCode); if ('multiselect' != $attribute->getFrontendInput() && self::SCOPE_NULL == $rowScope) { // skip attribute processing for SCOPE_NULL rows @@ -370,8 +350,10 @@ protected function _saveProducts() $storeIds = [0]; if ('datetime' == $attribute->getBackendType() && strtotime($attrValue)) { - $attrValue = (new \DateTime())->setTimestamp(strtotime($attrValue)); - $attrValue = $attrValue->format(DateTime::DATETIME_PHP_FORMAT); + $attrValue = $this->dateTime->gmDate( + 'Y-m-d H:i:s', + $this->_localeDate->date($attrValue)->getTimestamp() + ); } elseif ($backModel) { $attribute->getBackend()->beforeSave($product); $attrValue = $product->getData($attribute->getAttributeCode()); @@ -390,14 +372,7 @@ protected function _saveProducts() } } foreach ($storeIds as $storeId) { - if ('multiselect' == $attribute->getFrontendInput()) { - if (!isset($attributes[$attrTable][$rowSku][$attrId][$storeId])) { - $attributes[$attrTable][$rowSku][$attrId][$storeId] = ''; - } else { - $attributes[$attrTable][$rowSku][$attrId][$storeId] .= ','; - } - $attributes[$attrTable][$rowSku][$attrId][$storeId] .= $attrValue; - } else { + if (!isset($attributes[$attrTable][$rowSku][$attrId][$storeId])) { $attributes[$attrTable][$rowSku][$attrId][$storeId] = $attrValue; } } @@ -415,8 +390,6 @@ protected function _saveProducts() $this->categoriesCache )->_saveProductTierPrices( $tierPrices - )->_saveProductGroupPrices( - $groupPrices )->_saveMediaGallery( $mediaGallery )->_saveProductAttributes( @@ -463,7 +436,7 @@ protected function _prepareImagesFromSource($bunch) * @param $bunch * @return array */ - protected function _getAllBunchImages($bunch) + protected function getBunchImages($bunch) { $allImagesFromBunch = []; foreach ($bunch as $rowData) { diff --git a/etc/import.xml b/etc/import.xml index 0d842be..78141bc 100755 --- a/etc/import.xml +++ b/etc/import.xml @@ -5,6 +5,6 @@ * See COPYING.txt for license details. */ --> - + diff --git a/view/adminhtml/templates/import/form/before.phtml b/view/adminhtml/templates/import/form/before.phtml index aa3d9b3..b9bc8f5 100755 --- a/view/adminhtml/templates/import/form/before.phtml +++ b/view/adminhtml/templates/import/form/before.phtml @@ -8,7 +8,7 @@ /** @var $block \Magento\ImportExport\Block\Adminhtml\Import\Edit\Before */ ?>