Commit a0d1ac8716ee35e76c563a45bc914d303476f189
1 parent
eff1ed96
add articul filter into converter
Showing
6 changed files
with
93 additions
and
34 deletions
Show diff stats
backend/controllers/ParserController.php
| ... | ... | @@ -111,7 +111,10 @@ class ParserController extends BaseController |
| 111 | 111 | // === ручная загрузка =========== |
| 112 | 112 | //запускаем парсинг |
| 113 | 113 | // доп. опции для парсера |
| 114 | - $options = []; | |
| 114 | + $options = ['converter_conf' => | |
| 115 | + ['importer_id' => $files_model->importer_id] | |
| 116 | + ]; | |
| 117 | + | |
| 115 | 118 | if( ! $model->action ) // обработка с кастомным разделителем |
| 116 | 119 | $options['$delimiter'] = $model->delimiter; |
| 117 | 120 | ... | ... |
backend/models/Details.php
| ... | ... | @@ -130,14 +130,21 @@ class Details extends BaseActiveRecord |
| 130 | 130 | $data = array_chunk($data, $this::BATCH); |
| 131 | 131 | foreach ($data as $current_batch_array) { |
| 132 | 132 | |
| 133 | - //воспользуемся пакетной вставкой от фреймворка, плюс сразу с экранированием и защитой от инъекций | |
| 133 | + //воспользуемся пакетной вставкой от фреймворка | |
| 134 | 134 | $query_insert = Yii::$app->db->createCommand()->batchInsert($table_name, $keys_arr, $current_batch_array)->sql; |
| 135 | + if ($this->delete_prefix) { | |
| 136 | + $query_insert = $this->prepareArticul( $query_insert ); | |
| 137 | + } | |
| 135 | 138 | // добавим фрагмент с апдейтом при дубляже |
| 136 | 139 | $query = "{$query_insert} {$query_update}"; |
| 137 | 140 | // \common\components\CustomVarDamp::dumpAndDie($query); |
| 138 | - $res = Yii::$app->db->createCommand($query)->execute(); | |
| 141 | + Yii::$app->db->createCommand($query)->execute(); | |
| 139 | 142 | |
| 140 | 143 | } |
| 141 | 144 | } |
| 142 | 145 | |
| 146 | + private function prepareArticul( $query_insert ){ | |
| 147 | + //CustomVarDamp::dumpAndDie($query_insert); | |
| 148 | + return $query_insert; | |
| 149 | + } | |
| 143 | 150 | } | ... | ... |
common/components/parsers/CustomConverter.php
| 1 | 1 | <?php |
| 2 | 2 | namespace common\components\parsers; |
| 3 | + | |
| 3 | 4 | use common\components\CustomVarDamp; |
| 4 | 5 | use yii\multiparser\Converter; |
| 5 | 6 | use backend\models\Details; |
| 7 | +use backend\models\ImportersPrefix; | |
| 6 | 8 | |
| 7 | -class CustomConverter extends Converter { | |
| 9 | +class CustomConverter extends Converter | |
| 10 | +{ | |
| 8 | 11 | |
| 9 | 12 | /** |
| 10 | 13 | * @param $value_arr - двумерный массив значений, которому нужно присвоить ключи |
| ... | ... | @@ -13,38 +16,43 @@ class CustomConverter extends Converter { |
| 13 | 16 | */ |
| 14 | 17 | public static $sign; |
| 15 | 18 | public static $multiplier; |
| 19 | + public static $importer_id; | |
| 16 | 20 | |
| 17 | - public static function convertToAssocArray ( array $value_arr, array $key_array, $key_prefix = '' ) | |
| 21 | + public static function convertToAssocArray(array $value_arr, array $key_array, $key_prefix = '') | |
| 18 | 22 | { |
| 19 | 23 | // очистка служебного префикса в массиве заголовков |
| 20 | 24 | if ($key_prefix) { |
| 21 | 25 | // @todo оптимизировать - два переворота массива - избыточно |
| 22 | - $key_array = array_flip( $key_array ); | |
| 26 | + $key_array = array_flip($key_array); | |
| 23 | 27 | |
| 24 | - array_walk( $key_array, function ( &$value, $key, $key_prefix ){ $value = str_replace( $key_prefix, '',$value ); }, $key_prefix ); | |
| 28 | + array_walk($key_array, function (&$value, $key, $key_prefix) { | |
| 29 | + $value = str_replace($key_prefix, '', $value); | |
| 30 | + }, $key_prefix); | |
| 25 | 31 | |
| 26 | - $key_array = array_flip( $key_array ); | |
| 32 | + $key_array = array_flip($key_array); | |
| 27 | 33 | //уберем пустые элементы |
| 28 | - $key_array = array_filter($key_array, function ($value){ return $value !==''; }); | |
| 34 | + $key_array = array_filter($key_array, function ($value) { | |
| 35 | + return $value !== ''; | |
| 36 | + }); | |
| 29 | 37 | } |
| 30 | 38 | |
| 31 | - array_walk( $value_arr, | |
| 32 | - function ( &$value, $key, $key_array ) { | |
| 39 | + array_walk($value_arr, | |
| 40 | + function (&$value, $key, $key_array) { | |
| 33 | 41 | $res = $value; |
| 34 | 42 | foreach ($res as $sub_key => $sub_value) { |
| 35 | 43 | if (isset($key_array[$sub_key])) { |
| 36 | 44 | // если такой ключ в базовом массиве (массиве ключей) есть, то заменим новым, иначе просто удалим |
| 37 | 45 | $new_key = $key_array[$sub_key]; |
| 38 | - if( !array_key_exists( $new_key , $res ) ){ | |
| 39 | - $res[ $new_key ] = $value[$sub_key]; | |
| 46 | + if (!array_key_exists($new_key, $res)) { | |
| 47 | + $res[$new_key] = $value[$sub_key]; | |
| 40 | 48 | } |
| 41 | 49 | } |
| 42 | - unset( $res[$sub_key] ); | |
| 50 | + unset($res[$sub_key]); | |
| 43 | 51 | $value = $res; |
| 44 | 52 | } |
| 45 | 53 | |
| 46 | 54 | }, |
| 47 | - $key_array); | |
| 55 | + $key_array); | |
| 48 | 56 | |
| 49 | 57 | return $value_arr; |
| 50 | 58 | } |
| ... | ... | @@ -54,7 +62,7 @@ class CustomConverter extends Converter { |
| 54 | 62 | * @param $add_array - массив с колонками (ключи) и значениями колонок |
| 55 | 63 | * @return mixed |
| 56 | 64 | */ |
| 57 | - public function addColumns ( array $value_arr , array $add_array ) | |
| 65 | + public function addColumns(array $value_arr, array $add_array) | |
| 58 | 66 | { |
| 59 | 67 | $i = 0; |
| 60 | 68 | while ($i < count($value_arr)) { |
| ... | ... | @@ -66,7 +74,7 @@ class CustomConverter extends Converter { |
| 66 | 74 | return $value_arr; |
| 67 | 75 | } |
| 68 | 76 | |
| 69 | - public static function convertToDetails ( array $row ) | |
| 77 | + public static function convertToDetails(array $row) | |
| 70 | 78 | { |
| 71 | 79 | // присвоим полный артикул |
| 72 | 80 | $row['FULL_ARTICLE'] = $row['ARTICLE']; |
| ... | ... | @@ -75,19 +83,19 @@ class CustomConverter extends Converter { |
| 75 | 83 | // проверим все ли обязательные колонки были указаны пользователем |
| 76 | 84 | $details_model->load(['Details' => $row]); |
| 77 | 85 | |
| 78 | - if (!$details_model->validate()){ | |
| 86 | + if (!$details_model->validate()) { | |
| 79 | 87 | $errors = ''; |
| 80 | - foreach ( $details_model->errors as $key => $arr_errors ) { | |
| 81 | - $errors .= "Аттрибут $key - " . implode( ' , ', $arr_errors ); | |
| 88 | + foreach ($details_model->errors as $key => $arr_errors) { | |
| 89 | + $errors .= "Аттрибут $key - " . implode(' , ', $arr_errors); | |
| 82 | 90 | } |
| 83 | - throw new \ErrorException( $errors ); | |
| 91 | + throw new \ErrorException($errors); | |
| 84 | 92 | } |
| 85 | 93 | return $row; |
| 86 | 94 | } |
| 87 | 95 | |
| 88 | - public function ConvertToMultiply ( array $row ) | |
| 96 | + public function ConvertToMultiply(array $row) | |
| 89 | 97 | { |
| 90 | - $PRICE = $row[ 'PRICE' ]; | |
| 98 | + $PRICE = $row['PRICE']; | |
| 91 | 99 | $sign = self::$sign; |
| 92 | 100 | $multiplier = self::$multiplier; |
| 93 | 101 | //CustomVarDamp::dumpAndDie(self); |
| ... | ... | @@ -96,28 +104,68 @@ class CustomConverter extends Converter { |
| 96 | 104 | if ($multiplier > 0) { |
| 97 | 105 | $PRICE += $multiplier; |
| 98 | 106 | } |
| 99 | - } | |
| 100 | - else if ($sign == '-') { | |
| 107 | + } else if ($sign == '-') { | |
| 101 | 108 | if ($multiplier > 0) { |
| 102 | 109 | $PRICE -= $multiplier; |
| 103 | 110 | } |
| 104 | - } | |
| 105 | - else if ($sign == '*') { | |
| 111 | + } else if ($sign == '*') { | |
| 106 | 112 | if ($multiplier > 0) { |
| 107 | 113 | $PRICE *= $multiplier; |
| 108 | 114 | } |
| 109 | - } | |
| 110 | - else if ($sign == '/') { | |
| 115 | + } else if ($sign == '/') { | |
| 111 | 116 | if ($multiplier > 0) { |
| 112 | 117 | $PRICE /= $multiplier; |
| 113 | 118 | } |
| 114 | 119 | } |
| 115 | 120 | } |
| 116 | 121 | |
| 117 | - $row[ 'PRICE' ] = $PRICE; | |
| 122 | + $row['PRICE'] = $PRICE; | |
| 123 | + | |
| 124 | + return $row; | |
| 125 | + | |
| 126 | + } | |
| 127 | + | |
| 128 | + public static function convertToArticul(array $row) | |
| 129 | + { | |
| 130 | + if (isset($row['ARTICLE']) && isset($row['BRAND']) && isset(self::$importer_id)) { | |
| 131 | + | |
| 132 | + // 1. Уберем префикс который разделен пробелом (если он есть) | |
| 133 | + $words = explode(" ", $row['ARTICLE']); | |
| 134 | + if (count($words) > 1) { | |
| 135 | + array_shift($words); | |
| 136 | + $row['ARTICLE'] = implode(" ", $words); | |
| 137 | + } | |
| 138 | + | |
| 139 | + // 2. Уберем брендовый префикс (если он есть) | |
| 140 | + $prefix = ''; | |
| 141 | + // запрос закешируем | |
| 142 | + $prefix = ImportersPrefix::getDb()->cache( function ($db, $configuration, $row ) { | |
| 143 | + return ImportersPrefix::find()->where([ 'importer_id' => self::$importer_id, | |
| 144 | + 'brand' => $row['BRAND'] ])->one(); | |
| 145 | + }); | |
| 118 | 146 | |
| 147 | + if ($prefix) { | |
| 148 | + $row['BRAND'] = str_replace($prefix, "", $row['BRAND']); | |
| 149 | + } | |
| 150 | + } | |
| 119 | 151 | return $row; |
| 152 | + } | |
| 153 | + | |
| 154 | + public static function convertToBrand($value) | |
| 155 | + { | |
| 156 | + $res = $value; | |
| 157 | + $res = trim(strtoupper($res)); | |
| 158 | + $res = str_replace("Ä", "A", str_replace("Ö", "O", str_replace("Ü", "U", str_replace("Ë", "E", str_replace("Ò", "O", $res))))); | |
| 159 | + $res = str_replace(array('@', '#', '~', '"', "'", "?", "!"), '', $res); | |
| 160 | + | |
| 161 | + return $res; | |
| 162 | + } | |
| 163 | + | |
| 164 | + public static function convertToString($value) | |
| 165 | + { | |
| 166 | + $value = parent::convertToString($value); | |
| 120 | 167 | |
| 168 | + return str_replace(array('!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '_', '+', '=', '-', '~', '`', '"', "'", ' ', '№', '%', ';', ':', '[', ']', '{', '}', '*', '?', '/', '\'', '|', '.', ',', '<', '>', '\\'), '', $value); | |
| 121 | 169 | } |
| 122 | 170 | |
| 123 | 171 | ... | ... |
common/components/parsers/CustomCsvParser.php
| ... | ... | @@ -16,7 +16,7 @@ use yii\base\ErrorException; |
| 16 | 16 | |
| 17 | 17 | class CustomCsvParser extends \yii\multiparser\CsvParser { |
| 18 | 18 | |
| 19 | - public $last_line = 10; | |
| 19 | + public $last_line = 100; | |
| 20 | 20 | //public $hasHeaderRow = true; |
| 21 | 21 | // public $keys = ['first','second', 'third', 'forth', 'fifth']; |
| 22 | 22 | public function setupConverter() | ... | ... |
common/components/parsers/config.php
| ... | ... | @@ -17,9 +17,11 @@ |
| 17 | 17 | 'hasKey' => 1, |
| 18 | 18 | 'configuration' => ["string" => 'DESCR', |
| 19 | 19 | "float" => 'PRICE', |
| 20 | + "brand" => 'BRAND', | |
| 20 | 21 | "integer" => ['BOX','ADD_BOX'], |
| 21 | 22 | "multiply" => [], |
| 22 | - "details" => [] // @todo сделать отдельно конфигурирование валидации | |
| 23 | + "details" => [], | |
| 24 | + "articul" => [] | |
| 23 | 25 | |
| 24 | 26 | ] |
| 25 | 27 | ],], | ... | ... |
console/controllers/ParserController.php
| ... | ... | @@ -32,8 +32,7 @@ class ParserController extends Controller |
| 32 | 32 | 'importer_id' => $importer_id, |
| 33 | 33 | 'parser_config' => ['keys' => $keys, |
| 34 | 34 | 'converter_conf' => |
| 35 | - ['sign' => $sign, | |
| 36 | - 'multiplier' => $multiplier], | |
| 35 | + [ 'sign' => $sign, 'multiplier' => $multiplier, 'importer_id' => $importer_id ], | |
| 37 | 36 | 'mode' => 'console'] |
| 38 | 37 | ]; |
| 39 | 38 | if ($this->parseFileConsole($file_path, $config)) { | ... | ... |