1: <?php
  2: /**
  3:  * Magento
  4:  *
  5:  * NOTICE OF LICENSE
  6:  *
  7:  * This source file is subject to the Open Software License (OSL 3.0)
  8:  * that is bundled with this package in the file LICENSE.txt.
  9:  * It is also available through the world-wide-web at this URL:
 10:  * http://opensource.org/licenses/osl-3.0.php
 11:  * If you did not receive a copy of the license and are unable to
 12:  * obtain it through the world-wide-web, please send an email
 13:  * to license@magentocommerce.com so we can send you a copy immediately.
 14:  *
 15:  * DISCLAIMER
 16:  *
 17:  * Do not edit or add to this file if you wish to upgrade Magento to newer
 18:  * versions in the future. If you wish to customize Magento for your
 19:  * needs please refer to http://www.magentocommerce.com for more information.
 20:  *
 21:  * @category    Mage
 22:  * @package     Mage_ImportExport
 23:  * @copyright   Copyright (c) 2012 Magento Inc. (http://www.magentocommerce.com)
 24:  * @license     http://opensource.org/licenses/osl-3.0.php  Open Software License (OSL 3.0)
 25:  */
 26: 
 27: /**
 28:  * Import entity abstract model
 29:  *
 30:  * @category    Mage
 31:  * @package     Mage_ImportExport
 32:  * @author      Magento Core Team <core@magentocommerce.com>
 33:  */
 34: abstract class Mage_ImportExport_Model_Import_Entity_Abstract
 35: {
 36:     /**
 37:      * Database constants
 38:      *
 39:      */
 40:     const DB_MAX_PACKET_COEFFICIENT = 900000;
 41:     const DB_MAX_PACKET_DATA        = 1048576;
 42:     const DB_MAX_VARCHAR_LENGTH     = 256;
 43:     const DB_MAX_TEXT_LENGTH        = 65536;
 44: 
 45:     /**
 46:      * DB connection.
 47:      *
 48:      * @var Varien_Adapter_Interface
 49:      */
 50:     protected $_connection;
 51: 
 52:     /**
 53:      * Has data process validation done?
 54:      *
 55:      * @var bool
 56:      */
 57:     protected $_dataValidated = false;
 58: 
 59:     /**
 60:      * DB data source model.
 61:      *
 62:      * @var Mage_ImportExport_Model_Mysql4_Import_Data
 63:      */
 64:     protected $_dataSourceModel;
 65: 
 66:     /**
 67:      * Entity type id.
 68:      *
 69:      * @var int
 70:      */
 71:     protected $_entityTypeId;
 72: 
 73:     /**
 74:      * Error codes with arrays of corresponding row numbers.
 75:      *
 76:      * @var array
 77:      */
 78:     protected $_errors = array();
 79: 
 80:     /**
 81:      * Error counter.
 82:      *
 83:      * @var int
 84:      */
 85:     protected $_errorsCount = 0;
 86: 
 87:     /**
 88:      * Limit of errors after which pre-processing will exit.
 89:      *
 90:      * @var int
 91:      */
 92:     protected $_errorsLimit = 100;
 93: 
 94:     /**
 95:      * Flag to disable import.
 96:      *
 97:      * @var bool
 98:      */
 99:     protected $_importAllowed = true;
100: 
101:     /**
102:      * Attributes with index (not label) value.
103:      *
104:      * @var array
105:      */
106:     protected $_indexValueAttributes = array();
107: 
108:     /**
109:      * Array of invalid rows numbers.
110:      *
111:      * @var array
112:      */
113:     protected $_invalidRows = array();
114: 
115:     /**
116:      * Validation failure message template definitions.
117:      *
118:      * @var array
119:      */
120:     protected $_messageTemplates = array();
121: 
122:     /**
123:      * Notice messages.
124:      *
125:      * @var array
126:      */
127:     protected $_notices = array();
128: 
129:     /**
130:      * Entity model parameters.
131:      *
132:      * @var array
133:      */
134:     protected $_parameters = array();
135: 
136:     /**
137:      * Column names that holds values with particular meaning.
138:      *
139:      * @var array
140:      */
141:     protected $_particularAttributes = array();
142: 
143:     /**
144:      * Permanent entity columns.
145:      *
146:      * @var array
147:      */
148:     protected $_permanentAttributes = array();
149: 
150:     /**
151:      * Number of entities processed by validation.
152:      *
153:      * @var int
154:      */
155:     protected $_processedEntitiesCount = 0;
156: 
157:     /**
158:      * Number of rows processed by validation.
159:      *
160:      * @var int
161:      */
162:     protected $_processedRowsCount = 0;
163: 
164:     /**
165:      * Rows to skip. Valid rows but we have some reasons to skip them.
166:      *
167:      * [Row number 1] => true,
168:      * ...
169:      * [Row number N] => true
170:      *
171:      * @var array
172:      */
173:     protected $_rowsToSkip = array();
174: 
175:     /**
176:      * Array of numbers of validated rows as keys and boolean TRUE as values.
177:      *
178:      * @var array
179:      */
180:     protected $_validatedRows = array();
181: 
182:     /**
183:      * Source model.
184:      *
185:      * @var Mage_ImportExport_Model_Import_Adapter_Abstract
186:      */
187:     protected $_source;
188: 
189:     /**
190:      * Array of unique attributes
191:      *
192:      * @var array
193:      */
194:     protected $_uniqueAttributes = array();
195: 
196:     /**
197:      * Constructor.
198:      *
199:      * @return void
200:      */
201:     public function __construct()
202:     {
203:         $entityType = Mage::getSingleton('eav/config')->getEntityType($this->getEntityTypeCode());
204:         $this->_entityTypeId    = $entityType->getEntityTypeId();
205:         $this->_dataSourceModel = Mage_ImportExport_Model_Import::getDataSourceModel();
206:         $this->_connection      = Mage::getSingleton('core/resource')->getConnection('write');
207:     }
208: 
209:     /**
210:      * Inner source object getter.
211:      *
212:      * @return Mage_ImportExport_Model_Import_Adapter_Abstract
213:      */
214:     protected function _getSource()
215:     {
216:         if (!$this->_source) {
217:             Mage::throwException(Mage::helper('importexport')->__('No source specified'));
218:         }
219:         return $this->_source;
220:     }
221: 
222:     /**
223:      * Import data rows.
224:      *
225:      * @abstract
226:      * @return boolean
227:      */
228:     abstract protected function _importData();
229: 
230:     /**
231:      * Returns boolean TRUE if row scope is default (fundamental) scope.
232:      *
233:      * @param array $rowData
234:      * @return bool
235:      */
236:     protected function _isRowScopeDefault(array $rowData)
237:     {
238:         return true;
239:     }
240: 
241:     /**
242:      * Change row data before saving in DB table.
243:      *
244:      * @param array $rowData
245:      * @return array
246:      */
247:     protected function _prepareRowForDb(array $rowData)
248:     {
249:         /**
250:          * Convert all empty strings to null values, as
251:          * a) we don't use empty string in DB
252:          * b) empty strings instead of numeric values will product errors in Sql Server
253:          */
254:         foreach ($rowData as $key => $val) {
255:             if ($val === '') {
256:                 $rowData[$key] = null;
257:             }
258:         }
259:         return $rowData;
260:     }
261: 
262:     /**
263:      * Validate data rows and save bunches to DB.
264:      *
265:      * @return Mage_ImportExport_Model_Import_Entity_Abstract
266:      */
267:     protected function _saveValidatedBunches()
268:     {
269:         $source          = $this->_getSource();
270:         $productDataSize = 0;
271:         $bunchRows       = array();
272:         $startNewBunch   = false;
273:         $nextRowBackup   = array();
274:         $maxDataSize = Mage::getResourceHelper('importexport')->getMaxDataSize();
275:         $bunchSize = Mage::helper('importexport')->getBunchSize();
276: 
277:         $source->rewind();
278:         $this->_dataSourceModel->cleanBunches();
279: 
280:         while ($source->valid() || $bunchRows) {
281:             if ($startNewBunch || !$source->valid()) {
282:                 $this->_dataSourceModel->saveBunch($this->getEntityTypeCode(), $this->getBehavior(), $bunchRows);
283: 
284:                 $bunchRows       = $nextRowBackup;
285:                 $productDataSize = strlen(serialize($bunchRows));
286:                 $startNewBunch   = false;
287:                 $nextRowBackup   = array();
288:             }
289:             if ($source->valid()) {
290:                 if ($this->_errorsCount >= $this->_errorsLimit) { // errors limit check
291:                     return;
292:                 }
293:                 $rowData = $source->current();
294: 
295:                 $this->_processedRowsCount++;
296: 
297:                 if ($this->validateRow($rowData, $source->key())) { // add row to bunch for save
298:                     $rowData = $this->_prepareRowForDb($rowData);
299:                     $rowSize = strlen(Mage::helper('core')->jsonEncode($rowData));
300: 
301:                     $isBunchSizeExceeded = ($bunchSize > 0 && count($bunchRows) >= $bunchSize);
302: 
303:                     if (($productDataSize + $rowSize) >= $maxDataSize || $isBunchSizeExceeded) {
304:                         $startNewBunch = true;
305:                         $nextRowBackup = array($source->key() => $rowData);
306:                     } else {
307:                         $bunchRows[$source->key()] = $rowData;
308:                         $productDataSize += $rowSize;
309:                     }
310:                 }
311:                 $source->next();
312:             }
313:         }
314:         return $this;
315:     }
316: 
317:     /**
318:      * Add error with corresponding current data source row number.
319:      *
320:      * @param string $errorCode Error code or simply column name
321:      * @param int $errorRowNum Row number.
322:      * @param string $colName OPTIONAL Column name.
323:      * @return Mage_ImportExport_Model_Import_Adapter_Abstract
324:      */
325:     public function addRowError($errorCode, $errorRowNum, $colName = null)
326:     {
327:         $this->_errors[$errorCode][] = array($errorRowNum + 1, $colName); // one added for human readability
328:         $this->_invalidRows[$errorRowNum] = true;
329:         $this->_errorsCount ++;
330: 
331:         return $this;
332:     }
333: 
334:     /**
335:      * Add message template for specific error code from outside.
336:      *
337:      * @param string $errorCode Error code
338:      * @param string $message Message template
339:      * @return Mage_ImportExport_Model_Import_Entity_Abstract
340:      */
341:     public function addMessageTemplate($errorCode, $message)
342:     {
343:         $this->_messageTemplates[$errorCode] = $message;
344: 
345:         return $this;
346:     }
347: 
348:     /**
349:      * Returns attributes all values in label-value or value-value pairs form. Labels are lower-cased.
350:      *
351:      * @param Mage_Eav_Model_Entity_Attribute_Abstract $attribute
352:      * @param array $indexValAttrs OPTIONAL Additional attributes' codes with index values.
353:      * @return array
354:      */
355:     public function getAttributeOptions(Mage_Eav_Model_Entity_Attribute_Abstract $attribute, $indexValAttrs = array())
356:     {
357:         $options = array();
358: 
359:         if ($attribute->usesSource()) {
360:             // merge global entity index value attributes
361:             $indexValAttrs = array_merge($indexValAttrs, $this->_indexValueAttributes);
362: 
363:             // should attribute has index (option value) instead of a label?
364:             $index = in_array($attribute->getAttributeCode(), $indexValAttrs) ? 'value' : 'label';
365: 
366:             // only default (admin) store values used
367:             $attribute->setStoreId(Mage_Catalog_Model_Abstract::DEFAULT_STORE_ID);
368: 
369:             try {
370:                 foreach ($attribute->getSource()->getAllOptions(false) as $option) {
371:                     $value = is_array($option['value']) ? $option['value'] : array($option);
372:                     foreach ($value as $innerOption) {
373:                         if (strlen($innerOption['value'])) { // skip ' -- Please Select -- ' option
374:                             $options[strtolower($innerOption[$index])] = $innerOption['value'];
375:                         }
376:                     }
377:                 }
378:             } catch (Exception $e) {
379:                 // ignore exceptions connected with source models
380:             }
381:         }
382:         return $options;
383:     }
384: 
385:     /**
386:      * Import behavior getter.
387:      *
388:      * @return string
389:      */
390:     public function getBehavior()
391:     {
392:         if (!isset($this->_parameters['behavior'])
393:             || ($this->_parameters['behavior'] != Mage_ImportExport_Model_Import::BEHAVIOR_APPEND
394:             && $this->_parameters['behavior'] != Mage_ImportExport_Model_Import::BEHAVIOR_REPLACE
395:             && $this->_parameters['behavior'] != Mage_ImportExport_Model_Import::BEHAVIOR_DELETE)) {
396:             return Mage_ImportExport_Model_Import::getDefaultBehavior();
397:         }
398:         return $this->_parameters['behavior'];
399:     }
400: 
401:     /**
402:      * EAV entity type code getter.
403:      *
404:      * @abstract
405:      * @return string
406:      */
407:     abstract public function getEntityTypeCode();
408: 
409:     /**
410:      * Entity type ID getter.
411:      *
412:      * @return int
413:      */
414:     public function getEntityTypeId()
415:     {
416:         return $this->_entityTypeId;
417:     }
418: 
419:     /**
420:      * Returns error information grouped by error types and translated (if possible).
421:      *
422:      * @return array
423:      */
424:     public function getErrorMessages()
425:     {
426:         $translator = Mage::helper('importexport');
427:         $messages   = array();
428: 
429:         foreach ($this->_errors as $errorCode => $errorRows) {
430:             if (isset($this->_messageTemplates[$errorCode])) {
431:                 $errorCode = $translator->__($this->_messageTemplates[$errorCode]);
432:             }
433:             foreach ($errorRows as $errorRowData) {
434:                 $key = $errorRowData[1] ? sprintf($errorCode, $errorRowData[1]) : $errorCode;
435:                 $messages[$key][] = $errorRowData[0];
436:             }
437:         }
438:         return $messages;
439:     }
440: 
441:     /**
442:      * Returns error counter value.
443:      *
444:      * @return int
445:      */
446:     public function getErrorsCount()
447:     {
448:         return $this->_errorsCount;
449:     }
450: 
451:     /**
452:      * Returns error limit value.
453:      *
454:      * @return int
455:      */
456:     public function getErrorsLimit()
457:     {
458:         return $this->_errorsLimit;
459:     }
460: 
461:     /**
462:      * Returns invalid rows count.
463:      *
464:      * @return int
465:      */
466:     public function getInvalidRowsCount()
467:     {
468:         return count($this->_invalidRows);
469:     }
470: 
471:     /**
472:      * Returns model notices.
473:      *
474:      * @return array
475:      */
476:     public function getNotices()
477:     {
478:         return $this->_notices;
479:     }
480: 
481:     /**
482:      * Returns number of checked entities.
483:      *
484:      * @return int
485:      */
486:     public function getProcessedEntitiesCount()
487:     {
488:         return $this->_processedEntitiesCount;
489:     }
490: 
491:     /**
492:      * Returns number of checked rows.
493:      *
494:      * @return int
495:      */
496:     public function getProcessedRowsCount()
497:     {
498:         return $this->_processedRowsCount;
499:     }
500: 
501:     /**
502:      * Source object getter.
503:      *
504:      * @throws Exception
505:      * @return Mage_ImportExport_Model_Import_Adapter_Abstract
506:      */
507:     public function getSource()
508:     {
509:         if (!$this->_source) {
510:             Mage::throwException(Mage::helper('importexport')->__('Source is not set'));
511:         }
512:         return $this->_source;
513:     }
514: 
515:     /**
516:      * Import process start.
517:      *
518:      * @return bool Result of operation.
519:      */
520:     public function importData()
521:     {
522:         return $this->_importData();
523:     }
524: 
525:     /**
526:      * Is attribute contains particular data (not plain entity attribute).
527:      *
528:      * @param string $attrCode
529:      * @return bool
530:      */
531:     public function isAttributeParticular($attrCode)
532:     {
533:         return in_array($attrCode, $this->_particularAttributes);
534:     }
535: 
536:     /**
537:      * Check one attribute. Can be overridden in child.
538:      *
539:      * @param string $attrCode Attribute code
540:      * @param array $attrParams Attribute params
541:      * @param array $rowData Row data
542:      * @param int $rowNum
543:      * @return boolean
544:      */
545:     public function isAttributeValid($attrCode, array $attrParams, array $rowData, $rowNum)
546:     {
547:         switch ($attrParams['type']) {
548:             case 'varchar':
549:                 $val   = Mage::helper('core/string')->cleanString($rowData[$attrCode]);
550:                 $valid = Mage::helper('core/string')->strlen($val) < self::DB_MAX_VARCHAR_LENGTH;
551:                 break;
552:             case 'decimal':
553:                 $val   = trim($rowData[$attrCode]);
554:                 $valid = (float)$val == $val;
555:                 break;
556:             case 'select':
557:             case 'multiselect':
558:                 $valid = isset($attrParams['options'][strtolower($rowData[$attrCode])]);
559:                 break;
560:             case 'int':
561:                 $val   = trim($rowData[$attrCode]);
562:                 $valid = (int)$val == $val;
563:                 break;
564:             case 'datetime':
565:                 $val   = trim($rowData[$attrCode]);
566:                 $valid = strtotime($val) !== false
567:                     || preg_match('/^\d{2}.\d{2}.\d{2,4}(?:\s+\d{1,2}.\d{1,2}(?:.\d{1,2})?)?$/', $val);
568:                 break;
569:             case 'text':
570:                 $val   = Mage::helper('core/string')->cleanString($rowData[$attrCode]);
571:                 $valid = Mage::helper('core/string')->strlen($val) < self::DB_MAX_TEXT_LENGTH;
572:                 break;
573:             default:
574:                 $valid = true;
575:                 break;
576:         }
577: 
578:         if (!$valid) {
579:             $this->addRowError(Mage::helper('importexport')->__("Invalid value for '%s'"), $rowNum, $attrCode);
580:         } elseif (!empty($attrParams['is_unique'])) {
581:             if (isset($this->_uniqueAttributes[$attrCode][$rowData[$attrCode]])) {
582:                 $this->addRowError(Mage::helper('importexport')->__("Duplicate Unique Attribute for '%s'"), $rowNum, $attrCode);
583:                 return false;
584:             }
585:             $this->_uniqueAttributes[$attrCode][$rowData[$attrCode]] = true;
586:         }
587:         return (bool) $valid;
588:     }
589: 
590:     /**
591:      * Is all of data valid?
592:      *
593:      * @return bool
594:      */
595:     public function isDataValid()
596:     {
597:         $this->validateData();
598:         return 0 == $this->_errorsCount;
599:     }
600: 
601:     /**
602:      * Import possibility getter.
603:      *
604:      * @return bool
605:      */
606:     public function isImportAllowed()
607:     {
608:         return $this->_importAllowed;
609:     }
610: 
611:     /**
612:      * Returns TRUE if row is valid and not in skipped rows array.
613:      *
614:      * @param array $rowData
615:      * @param int $rowNum
616:      * @return bool
617:      */
618:     public function isRowAllowedToImport(array $rowData, $rowNum)
619:     {
620:         return $this->validateRow($rowData, $rowNum) && !isset($this->_rowsToSkip[$rowNum]);
621:     }
622: 
623:     /**
624:      * Validate data row.
625:      *
626:      * @param array $rowData
627:      * @param int $rowNum
628:      * @return boolean
629:      */
630:     abstract public function validateRow(array $rowData, $rowNum);
631: 
632:     /**
633:      * Set data from outside to change behavior. I.e. for setting some default parameters etc.
634:      *
635:      * @param array $params
636:      * @return Mage_ImportExport_Model_Import_Entity_Abstract
637:      */
638:     public function setParameters(array $params)
639:     {
640:         $this->_parameters = $params;
641:         return $this;
642:     }
643: 
644:     /**
645:      * Source model setter.
646:      *
647:      * @param Mage_ImportExport_Model_Import_Adapter_Abstract $source
648:      * @return Mage_ImportExport_Model_Import_Entity_Abstract
649:      */
650:     public function setSource(Mage_ImportExport_Model_Import_Adapter_Abstract $source)
651:     {
652:         $this->_source = $source;
653:         $this->_dataValidated = false;
654: 
655:         return $this;
656:     }
657: 
658:     /**
659:      * Validate data.
660:      *
661:      * @throws Exception
662:      * @return Mage_ImportExport_Model_Import_Entity_Abstract
663:      */
664:     public function validateData()
665:     {
666:         if (!$this->_dataValidated) {
667:             // does all permanent columns exists?
668:             if (($colsAbsent = array_diff($this->_permanentAttributes, $this->_getSource()->getColNames()))) {
669:                 Mage::throwException(
670:                     Mage::helper('importexport')->__('Can not find required columns: %s', implode(', ', $colsAbsent))
671:                 );
672:             }
673: 
674:             // initialize validation related attributes
675:             $this->_errors = array();
676:             $this->_invalidRows = array();
677: 
678:             // check attribute columns names validity
679:             $invalidColumns = array();
680: 
681:             foreach ($this->_getSource()->getColNames() as $colName) {
682:                 if (!preg_match('/^[a-z][a-z0-9_]*$/', $colName) && !$this->isAttributeParticular($colName)) {
683:                     $invalidColumns[] = $colName;
684:                 }
685:             }
686:             if ($invalidColumns) {
687:                 Mage::throwException(
688:                     Mage::helper('importexport')->__('Column names: "%s" are invalid', implode('", "', $invalidColumns))
689:                 );
690:             }
691:             $this->_saveValidatedBunches();
692: 
693:             $this->_dataValidated = true;
694:         }
695:         return $this;
696:     }
697: }
698: