From 1fe29bbeba36c8d36a957bf6d5d7b56d39672cc9 Mon Sep 17 00:00:00 2001 From: Mihail Date: Thu, 1 Oct 2015 11:50:13 +0300 Subject: [PATCH] fixed parser and converter for console needs --- backend/components/parsers/CustomCsvParser.php | 7 ++++--- backend/components/parsers/config.php | 2 ++ backend/controllers/ParserController.php | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----- backend/views/parser/server-files.php | 3 ++- common/components/PriceWriter.php | 4 ++-- console/controllers/ParserController.php | 46 +++++++++++++++++++++++++++++++++++++++++++--- 6 files changed, 113 insertions(+), 14 deletions(-) diff --git a/backend/components/parsers/CustomCsvParser.php b/backend/components/parsers/CustomCsvParser.php index dd28725..938a1ed 100644 --- a/backend/components/parsers/CustomCsvParser.php +++ b/backend/components/parsers/CustomCsvParser.php @@ -9,6 +9,8 @@ namespace backend\components\parsers; +use common\components\CustomVarDamp; + class CustomCsvParser extends \yii\multiparser\CsvParser { public $last_line = 10; @@ -33,9 +35,8 @@ class CustomCsvParser extends \yii\multiparser\CsvParser { */ protected function convert($arr) { - $result = \Yii::$app->multiparser->convertByConfiguration( $arr, $this->converter_conf ); - - return $result; + $arr = \Yii::$app->multiparser->convertByConfiguration($arr, $this->converter_conf); + return $arr; } diff --git a/backend/components/parsers/config.php b/backend/components/parsers/config.php index 3db7f86..6731f4b 100644 --- a/backend/components/parsers/config.php +++ b/backend/components/parsers/config.php @@ -21,7 +21,9 @@ 'console' => ['class' => 'backend\components\parsers\CustomCsvParser', 'auto_detect_first_line' => true, + 'hasHeaderRow' => true, 'converter_conf' => ['class' => ' backend\components\parsers\CustomConverter', + 'hasKey' => 1, 'configuration' => ["string" => 'DESCR', "float" => 'PRICE', "integer" => ['BOX','ADD_BOX'] diff --git a/backend/controllers/ParserController.php b/backend/controllers/ParserController.php index 4a0271f..e995fc6 100644 --- a/backend/controllers/ParserController.php +++ b/backend/controllers/ParserController.php @@ -16,7 +16,6 @@ use backend\models\Importers; use yii\base\ErrorException; use common\components\PriceWriter; use common\components\CustomVarDamp; -use yii\web\ErrorAction; /** * Parser controller @@ -225,7 +224,7 @@ class ParserController extends BaseController $file_id = basename($server_file,".csv"); $arr_id_files[] = (int) $file_id; } - Yii::$app->cache->set( 'files_to_delete',json_encode( $arr_id_files ) ); + Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) ); $query = ImportersFiles::find()->where(['in', 'id', $arr_id_files])->orderBy(['upload_time' => SORT_DESC]); $provider = new ActiveDataProvider([ @@ -251,23 +250,79 @@ class ParserController extends BaseController unlink(Yii::getAlias('@auto_upload') . '/' . $id . '.csv' ); // удалим этот id и из кэша - if( $arr_id_files = Yii::$app->cache->get( 'files_to_delete' ) ){ + if( $arr_id_files = Yii::$app->cache->get( 'files_to_parse' ) ){ $arr_id_files = json_decode($arr_id_files); if (isset( $arr_id_files[$id] ) ) { unset( $arr_id_files[$id] ); // положем уже обновленный массив - Yii::$app->cache->set( 'files_to_delete',json_encode( $arr_id_files ) ); + Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) ); } } // сообщим скрипту что все ОК echo 1; } catch (ErrorException $e) { - CustomVarDamp::dump($e->getMessage()); + //CustomVarDamp::dump($e->getMessage()); + throw $e; } } } } + + + + + public function actionParse () + { +// $comand = "/usr/bin/php -f ".Yii::getAlias('@console') ."/Controllers/ParserController.php"; +// exec($comand); + if( $arr_id_files = Yii::$app->cache->get( 'files_to_parse' ) ) { + $arr_id_files = json_decode( $arr_id_files ); + foreach ( $arr_id_files as $file_name ) { + $file_path = Yii::getAlias('@auto_upload') . '/' . $file_name . '.csv'; + $config = ['record_id' => $file_name, + 'importer_id' => ImportersFiles::findOne(['id' => $file_name])->id, + 'parser_config' => ['keys' => ['DESCR', 'ARTICLE', 'BRAND', 'PRICE', 'BOX'], + 'mode' => 'console'] + ]; + if( $this->parseFileConsole( $file_path, $config ) ){ + unlink( $file_path ); + if (isset( $arr_id_files[$file_path] ) ) { + unset($arr_id_files[$file_path]); + } + } else { + // Yii::$app->log-> + // не дошли до конца по этому остаки вернем в кеш + Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) ); + } + } + if ( !count( $arr_id_files ) ) { + Yii::$app->cache->delete( 'files_to_parse' ); + } + } + + return $this->redirect('serverFiles'); + } + + protected function parseFileConsole( $file_path, $configuration ){ + $parser_config = []; + if ( isset( $configuration['parser_config'] ) ) { + $parser_config = $configuration['parser_config']; + } + + $data = Yii::$app->multiparser->parse( $file_path, $parser_config ); + CustomVarDamp::dumpAndDie($data); + $writer = new PriceWriter(); + $writer->configuration = $configuration; + $writer->data = $data; + $writer->mode = 1; //console-режим + if ( $writer->writeDataToDB() ){ + //Console::output('It is working'); + return true; + } + + return false; + } } diff --git a/backend/views/parser/server-files.php b/backend/views/parser/server-files.php index 2bc638a..cd7160b 100644 --- a/backend/views/parser/server-files.php +++ b/backend/views/parser/server-files.php @@ -1,5 +1,6 @@ 'server_files_grid']); ]] );?> - + 'btn btn-success']) ?> configuration->record_id ); + $files_model = ImportersFiles::findOne( $this->configuration['record_id'] ); //$files_model->load(['ImportersFiles' => $this->configuration->toArray()]); $update_date = date('Y-m-d H:i:s'); @@ -61,7 +61,7 @@ class PriceWriter { } else{ // дополним данные значением импортера и даты обновления цены - $this->data = \Yii::$app->multiparser->addColumns($this->data, ['IMPORT_ID' => $this->configuration->importer_id, 'timestamp' => $update_date]); + $this->data = \Yii::$app->multiparser->addColumns($this->data, ['IMPORT_ID' => $this->configuration['importer_id'], 'timestamp' => $update_date]); try { //@todo add transaction diff --git a/console/controllers/ParserController.php b/console/controllers/ParserController.php index a600e17..109a588 100644 --- a/console/controllers/ParserController.php +++ b/console/controllers/ParserController.php @@ -6,19 +6,59 @@ * Time: 14:38 */ use yii\console\Controller; +use yii\helpers\Console; +use common\components\PriceWriter; +use backend\models\ImportersFiles; class ParserController extends Controller{ public function actionParseCSV () { - + \common\components\CustomVarDamp::dumpAndDie(45); + if( $arr_id_files = Yii::$app->cache->get( 'files_to_parse' ) ) { + $arr_id_files = json_decode( $arr_id_files ); + foreach ( $arr_id_files as $file_name ) { + $file_path = Yii::getAlias('@auto_upload') . '/' . $file_name . '.csv'; + $config = ['record_id' => $file_name, + 'importer_id' => ImportersFiles::findOne(['id' => $file_name])->id, + 'parser_config' => ['keys' => ['DESCR', 'ARTICLE', 'BRAND', 'PRICE', 'BOX']] + ]; + if( $this->parseFileConsole( $file_path, $config ) ){ + unlink( $file_path ); + if (isset( $arr_id_files[$file_path] ) ) { + unset($arr_id_files[$file_path]); + } + } else { + // Yii::$app->log-> + // не дошли до конца по этому остаки вернем в кеш + Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) ); + } + } + if ( !count( $arr_id_files ) ) { + Yii::$app->cache->delete( 'files_to_parse' ); + } + } } public function actionParseXML () { } - protected function parseFileConsole( $file_path ){ + protected function parseFileConsole( $file_path, $configuration ){ + $parser_config = []; + if ( isset( $configuration['parser_config'] ) ) { + $parser_config = $configuration['parser_config']; + } + + $data = Yii::$app->multiparser->parse( $file_path, $parser_config ); - $data = Yii::$app->multiparser->parse( $file_path ); + $writer = new PriceWriter(); + $writer->configuration = $configuration; + $writer->data = $data; + $writer->mode = 1; //console-режим + if ( $writer->writeDataToDB() ){ + Console::output('It is working'); + return true; + } + return false; } } \ No newline at end of file -- libgit2 0.21.4