diff --git a/backend/.gitignore b/backend/.gitignore index da2fd6f..11cf485 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -1,2 +1,3 @@ /temp -/uploads \ No newline at end of file +/uploads +/runtime \ No newline at end of file diff --git a/backend/components/parsers/CustomConverter.php b/backend/components/parsers/CustomConverter.php index 599a796..b054b8c 100644 --- a/backend/components/parsers/CustomConverter.php +++ b/backend/components/parsers/CustomConverter.php @@ -1,6 +1,8 @@ load(['Details' => $row]); + + if (!$details_model->validate()){ + $errors = ''; + foreach ( $details_model->errors as $key => $arr_errors ) { + $errors .= "Аттрибут $key - " . implode( ' , ', $arr_errors ); + } + throw new \ErrorException( $errors ); + } + return $row; + } + + + } \ No newline at end of file diff --git a/backend/components/parsers/CustomCsvParser.php b/backend/components/parsers/CustomCsvParser.php index 938a1ed..8eca20d 100644 --- a/backend/components/parsers/CustomCsvParser.php +++ b/backend/components/parsers/CustomCsvParser.php @@ -10,6 +10,7 @@ namespace backend\components\parsers; use common\components\CustomVarDamp; +use yii\base\ErrorException; class CustomCsvParser extends \yii\multiparser\CsvParser { @@ -35,7 +36,8 @@ class CustomCsvParser extends \yii\multiparser\CsvParser { */ protected function convert($arr) { - $arr = \Yii::$app->multiparser->convertByConfiguration($arr, $this->converter_conf); + $arr = \Yii::$app->multiparser->convertByConfiguration($arr, $this->converter_conf); + return $arr; } diff --git a/backend/components/parsers/config.php b/backend/components/parsers/config.php index 6731f4b..1d2b28e 100644 --- a/backend/components/parsers/config.php +++ b/backend/components/parsers/config.php @@ -26,7 +26,8 @@ 'hasKey' => 1, 'configuration' => ["string" => 'DESCR', "float" => 'PRICE', - "integer" => ['BOX','ADD_BOX'] + "integer" => ['BOX','ADD_BOX'], + "details" => [], // @todo сделать отдельно конфигурирование валидации ] ],], diff --git a/backend/controllers/ParserController.php b/backend/controllers/ParserController.php index 3af9d94..a892c27 100644 --- a/backend/controllers/ParserController.php +++ b/backend/controllers/ParserController.php @@ -95,7 +95,7 @@ class ParserController extends BaseController $file_name = $model->record_id . '.' . $model->file->extension; if ($model->mode) { - $model->file_path = Yii::getAlias('@auto_upload') . '/' . $file_name; + $model->file_path = Yii::getAlias('@temp_upload') . '/' . $file_name; } else { $model->file_path = Yii::getAlias('@manual_upload') . '/' . $file_name; } @@ -105,6 +105,7 @@ class ParserController extends BaseController // для авто загрузки, обработка завершена if ($model->mode) { $model->success = true; + return $this->render('index', ['model' => $model]); } @@ -218,13 +219,15 @@ class ParserController extends BaseController public function actionServerFiles () { - $arr_id_files = []; + + $arr_id_files = []; + // получим список файлов которые ожидают к загрузке - foreach (glob(Yii::getAlias('@auto_upload') . '/*') as $server_file) { + foreach ( glob(Yii::getAlias('@temp_upload') . '/*.csv') as $server_file ) { $file_id = basename($server_file,".csv"); $arr_id_files[] = (int) $file_id; } - Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) ); + // Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) ); $query = ImportersFiles::find()->where(['in', 'id', $arr_id_files])->orderBy(['upload_time' => SORT_DESC]); $provider = new ActiveDataProvider([ @@ -247,17 +250,17 @@ class ParserController extends BaseController $id = Yii::$app->request->post()['id']; try { $files_model->delete($id); - unlink(Yii::getAlias('@auto_upload') . '/' . $id . '.csv' ); + unlink(Yii::getAlias('@temp_upload') . '/' . $id . '.csv' ); // удалим этот id и из кэша - if( $arr_id_files = Yii::$app->cache->get( 'files_to_parse' ) ){ - $arr_id_files = json_decode($arr_id_files); - if (isset( $arr_id_files[$id] ) ) { - unset( $arr_id_files[$id] ); - // положем уже обновленный массив - Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) ); - } - } +// if( $arr_id_files = Yii::$app->cache->get( 'files_to_parse' ) ){ +// $arr_id_files = json_decode($arr_id_files); +// if (isset( $arr_id_files[$id] ) ) { +// unset( $arr_id_files[$id] ); +// // положем уже обновленный массив +// Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) ); +// } +// } // сообщим скрипту что все ОК echo 1; } catch (ErrorException $e) { @@ -271,63 +274,76 @@ class ParserController extends BaseController } + public function actionLaunchCroneUploads () + { + + foreach (glob(Yii::getAlias('@temp_upload') . '/*.csv') as $server_file) { + $file_name = basename($server_file,".csv"); + rename( $server_file, Yii::getAlias('@auto_upload') . '/' . $file_name . '.csv'); + + } + + //$this->redirect('server-files'); + $this->actionParseCsv(); + } - public function actionParse () + public function actionParseCsv () { -// $comand = "/usr/bin/php -f ".Yii::getAlias('@console') ."/Controllers/ParserController.php"; -// exec($comand); - - if( $arr_id_files = Yii::$app->cache->get( 'files_to_parse' ) ) { - $arr_id_files = json_decode( $arr_id_files ); - foreach ( $arr_id_files as $file_name ) { - - $importer_id = ImportersFiles::findOne(['id' => $file_name])->importer_id; - $keys = Importers::findOne( ['id' => $importer_id] )->keys; - - $file_path = Yii::getAlias('@auto_upload') . '/' . $file_name . '.csv'; - $config = ['record_id' => $file_name, - 'importer_id' => $importer_id, - 'parser_config' => ['keys' => $keys, - 'mode' => 'console'] - ]; - if( $this->parseFileConsole( $file_path, $config ) ){ - unlink( $file_path ); - if (isset( $arr_id_files[$file_path] ) ) { - unset($arr_id_files[$file_path]); - } - } else { - // Yii::$app->log-> - // не дошли до конца по этому остатки вернем в кеш - Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) ); + // Console::output('1'); + foreach (glob( \Yii::getAlias('@auto_upload') . '/*.csv' ) as $file_path) { + $file_name = basename( $file_path,".csv" ); + // Console::output('2'); + $importer_id = ImportersFiles::findOne(['id' => $file_name])->importer_id; + $keys = Importers::findOne( ['id' => $importer_id] )->keys; + + $config = ['record_id' => $file_name, + 'importer_id' => $importer_id, + 'parser_config' => ['keys' => $keys, + 'mode' => 'console'] + ]; + // Console::output('3'); + if( $this->parseFileConsole( $file_path, $config ) ){ + // Console::output('4'); + unlink( $file_path ); + if (isset( $arr_id_files[$file_path] ) ) { + unset( $arr_id_files[$file_path] ); } } - if ( !count( $arr_id_files ) ) { - Yii::$app->cache->delete( 'files_to_parse' ); - } } - return $this->redirect('serverFiles'); + + //return $this->redirect('serverFiles'); } + protected function parseFileConsole( $file_path, $configuration ){ + + if( !file_exists( $file_path ) ) + throw new ErrorException("$file_path - файл не найден!"); + $parser_config = []; if ( isset( $configuration['parser_config'] ) ) { $parser_config = $configuration['parser_config']; } + try { + $data = \Yii::$app->multiparser->parse( $file_path, $parser_config ); + } catch (ErrorException $e) { + throw $e; + } - $data = Yii::$app->multiparser->parse( $file_path, $parser_config ); - CustomVarDamp::dumpAndDie($data); + // CustomVarDamp::dumpAndDie($data); $writer = new PriceWriter(); $writer->configuration = $configuration; $writer->data = $data; $writer->mode = 1; //console-режим if ( $writer->writeDataToDB() ){ - //Console::output('It is working'); + // Console::output('It is working'); return true; } return false; } + } diff --git a/backend/models/Details.php b/backend/models/Details.php index 38c3e07..e1b829d 100644 --- a/backend/models/Details.php +++ b/backend/models/Details.php @@ -41,6 +41,7 @@ class Details extends BaseActiveRecord [[ 'BRAND', 'ARTICLE', 'PRICE', 'DESCR', 'BOX'], 'required'], // [['IMPORT_ID', 'BOX', 'ADD_BOX'], 'integer'], [['PRICE'], 'number'], + [['BOX'], 'integer'], [['timestamp'], 'safe'], [['BRAND', 'ARTICLE'], 'string', 'max' => 100], [['FULL_ARTICLE'], 'string', 'max' => 150], @@ -97,6 +98,7 @@ class Details extends BaseActiveRecord $res = Yii::$app->db->createCommand($query)->execute(); } - } + + } diff --git a/backend/views/parser/server-files.php b/backend/views/parser/server-files.php index cd7160b..dbdc326 100644 --- a/backend/views/parser/server-files.php +++ b/backend/views/parser/server-files.php @@ -45,7 +45,7 @@ Pjax::begin(['id' => 'server_files_grid']); ]] );?> - 'btn btn-success']) ?> + 'btn btn-success']) ?> configuration['record_id'] ); - //$files_model->load(['ImportersFiles' => $this->configuration->toArray()]); $update_date = date('Y-m-d H:i:s'); $files_model->time_start = $update_date; // запишем дату начала загрузки if (!$files_model->save()) { - $this->errors[] = implode( ', ', $files_model->getErrors()); - return false; - //CustomVarDamp::dumpAndDie($files_model->implode ( ', ', getErrors())()); + throw new \ErrorException(implode( ', ', $files_model->getErrors())); } - // 2. запишем полученные данные в таблицу товаров (Details) $details_model = new Details(); - // преобразуем числовые значения - foreach ($this->data as &$row) { - $row['PRICE'] = \Yii::$app->multiparser->convertToFloat($row['PRICE']); - $row['BOX'] = \Yii::$app->multiparser->convertToInteger($row['BOX']); - // присвоим полный артикул - $row['FULL_ARTICLE'] = $row['ARTICLE']; - if(isset($row['ADD_BOX'])) - $row['ADD_BOX'] = \Yii::$app->multiparser->convertToInteger($row['ADD_BOX']); - - // проверим все ли обязательные колонки были указаны пользователем - $details_model->load(['Details' => $row]); - - if (!$details_model->validate()) - break; - } - if ($details_model->hasErrors()) { - //@todo предоставить более детальную информацию об ошибке - throw new \ErrorException('Ошибка записи товаров'); + // только для ручной загрузки, в авто режиме все делает конвертер при первом же проходе (в процессе парсинга) + if ($this->mode = 0) { + // преобразуем числовые значения + foreach ($this->data as &$row) { + $row['PRICE'] = \Yii::$app->multiparser->convertToFloat($row['PRICE']); + $row['BOX'] = \Yii::$app->multiparser->convertToInteger($row['BOX']); + // присвоим полный артикул + $row['FULL_ARTICLE'] = $row['ARTICLE']; + if(isset($row['ADD_BOX'])) + $row['ADD_BOX'] = \Yii::$app->multiparser->convertToInteger($row['ADD_BOX']); + + // проверим все ли обязательные колонки были указаны пользователем + $details_model->load(['Details' => $row]); + + if (!$details_model->validate()) + //@todo предоставить более детальную информацию об ошибке + throw new \ErrorException('Ошибка записи товаров'); + + } } - else{ + // дополним данные значением импортера и даты обновления цены $this->data = \Yii::$app->multiparser->addColumns($this->data, ['IMPORT_ID' => $this->configuration['importer_id'], 'timestamp' => $update_date]); @@ -70,13 +66,8 @@ class PriceWriter { // 3. зафиксируем дату конца загрузки в файлах поставщика - $files_model->time_end = date('Y-m-d H:i:s'); - // CustomVarDamp::dumpAndDie($files_model); if (!$files_model->save()) { throw new \ErrorException(implode( ', ', $files_model->getErrors())); -// $this->errors[] = $files_model->implode ( ', ', getErrors()); -// return false; - // CustomVarDamp::dumpAndDie($files_model->implode ( ', ', getErrors())()); } // 4. зафиксируем дату загрузки в таблице поставщиков @@ -84,18 +75,12 @@ class PriceWriter { $imp_model->price_date_update = $update_date; if (!$imp_model->save()) { -// $this->errors[] = $imp_model->implode ( ', ', getErrors())(); -// return false; throw new \ErrorException(implode( ', ', $imp_model->getErrors())); - // CustomVarDamp::dumpAndDie($imp_model->implode ( ', ', getErrors())()); } - - } catch (ErrorException $e) { - //CustomVarDamp::dump($e->getMessage()); throw new \ErrorException( $e->getMessage() ); } - } + return true; diff --git a/common/config/bootstrap.php b/common/config/bootstrap.php index df33c86..3f897d3 100644 --- a/common/config/bootstrap.php +++ b/common/config/bootstrap.php @@ -5,3 +5,4 @@ Yii::setAlias('backend', dirname(dirname(__DIR__)) . '/backend'); Yii::setAlias('console', dirname(dirname(__DIR__)) . '/console'); Yii::setAlias('auto_upload', dirname(dirname(__DIR__)) . '/backend/uploads/auto'); Yii::setAlias('manual_upload', dirname(dirname(__DIR__)) . '/backend/uploads/manual'); +Yii::setAlias('temp_upload', dirname(dirname(__DIR__)) . '/backend/uploads/temp'); diff --git a/console/config/main.php b/console/config/main.php index 7a67404..1e734ea 100644 --- a/console/config/main.php +++ b/console/config/main.php @@ -20,12 +20,14 @@ return [ ], [ 'class' => 'yii\log\FileTarget', - 'logFile' => 'C:\xampp\htdocs\ital\console\runtime\logs\parser.log', + 'logFile' => __DIR__ . '/../runtime/logs/parser.log', 'levels' => ['info', 'error'], - // 'categories' => ['parser'], + 'categories' => ['yii\base\ErrorException:*', 'parser'], + 'logVars' => [], ], ], ], + ], 'params' => $params, -] ]; + diff --git a/console/controllers/ParserController.php b/console/controllers/ParserController.php index 43d8f61..fa8eccd 100644 --- a/console/controllers/ParserController.php +++ b/console/controllers/ParserController.php @@ -5,46 +5,54 @@ use yii\console\Controller; use yii\helpers\Console; use common\components\PriceWriter; use backend\models\ImportersFiles; +use backend\models\Importers; +use yii\base\ErrorException; class ParserController extends Controller{ - public function actionParseCSV () + + public function actionParseCsv () { - \common\components\CustomVarDamp::dumpAndDie(45); - if( $arr_id_files = Yii::$app->cache->get( 'files_to_parse' ) ) { - $arr_id_files = json_decode( $arr_id_files ); - foreach ( $arr_id_files as $file_name ) { - $file_path = Yii::getAlias('@auto_upload') . '/' . $file_name . '.csv'; + Console::output('1'); + foreach (glob( \Yii::getAlias('@auto_upload') . '/*.csv' ) as $file_path) { + $file_name = basename($file_path,".csv"); + Console::output('2'); + $importer_id = ImportersFiles::findOne(['id' => $file_name])->importer_id; + $keys = Importers::findOne( ['id' => $importer_id] )->keys; + $config = ['record_id' => $file_name, - 'importer_id' => ImportersFiles::findOne(['id' => $file_name])->id, - 'parser_config' => ['keys' => ['DESCR', 'ARTICLE', 'BRAND', 'PRICE', 'BOX']] + 'importer_id' => $importer_id, + 'parser_config' => ['keys' => $keys, + 'mode' => 'console'] ]; + Console::output('3'); if( $this->parseFileConsole( $file_path, $config ) ){ + Console::output('4'); unlink( $file_path ); if (isset( $arr_id_files[$file_path] ) ) { - unset($arr_id_files[$file_path]); + unset( $arr_id_files[$file_path] ); } - } else { - // Yii::$app->log-> - // не дошли до конца по этому остаки вернем в кеш - Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) ); } } - if ( !count( $arr_id_files ) ) { - Yii::$app->cache->delete( 'files_to_parse' ); - } - } + + + //return $this->redirect('serverFiles'); } - public function actionParseXML () + + public function actionParseXml () { } protected function parseFileConsole( $file_path, $configuration ){ + + if( !file_exists( $file_path ) ) + throw new ErrorException("$file_path does not exist!"); + $parser_config = []; if ( isset( $configuration['parser_config'] ) ) { $parser_config = $configuration['parser_config']; } - $data = Yii::$app->multiparser->parse( $file_path, $parser_config ); + $data = \Yii::$app->multiparser->parse( $file_path, $parser_config ); $writer = new PriceWriter(); $writer->configuration = $configuration; @@ -60,11 +68,8 @@ class ParserController extends Controller{ public function actionTest () { - // Console::output('It is working '); - \Yii::warning('1'); - \Yii::info('2'); -// \Yii::info('3'); -// \Yii::warning('4'); + Console::output('It is working '); + \Yii::info('2','parser'); } } \ No newline at end of file -- libgit2 0.21.4