Administrator / test_1

Commit 1fe29bbeba36c8d36a957bf6d5d7b56d39672cc9

Authored by Mihail
1 parent f0dbd829

fixed parser and converter for console needs

Showing 6 changed files with 113 additions and 14 deletions   Show diff stats
Inline Side-by-side
backend/components/parsers/CustomCsvParser.php
  Show/Hide comments   View file @1fe29bb
... ... @@ -9,6 +9,8 @@
9 9 namespace backend\components\parsers;
10 10  
11 11  
  12 +use common\components\CustomVarDamp;
  13 +
12 14 class CustomCsvParser extends \yii\multiparser\CsvParser {
13 15  
14 16 public $last_line = 10;
... ... @@ -33,9 +35,8 @@ class CustomCsvParser extends \yii\multiparser\CsvParser {
33 35 */
34 36 protected function convert($arr)
35 37 {
36   - $result = \Yii::$app->multiparser->convertByConfiguration( $arr, $this->converter_conf );
37   -
38   - return $result;
  38 + $arr = \Yii::$app->multiparser->convertByConfiguration($arr, $this->converter_conf);
  39 + return $arr;
39 40  
40 41 }
41 42  
... ...
backend/components/parsers/config.php
  Show/Hide comments   View file @1fe29bb
... ... @@ -21,7 +21,9 @@
21 21 'console' =>
22 22 ['class' => 'backend\components\parsers\CustomCsvParser',
23 23 'auto_detect_first_line' => true,
  24 + 'hasHeaderRow' => true,
24 25 'converter_conf' => ['class' => ' backend\components\parsers\CustomConverter',
  26 + 'hasKey' => 1,
25 27 'configuration' => ["string" => 'DESCR',
26 28 "float" => 'PRICE',
27 29 "integer" => ['BOX','ADD_BOX']
... ...
backend/controllers/ParserController.php
  Show/Hide comments   View file @1fe29bb
... ... @@ -16,7 +16,6 @@ use backend\models\Importers;
16 16 use yii\base\ErrorException;
17 17 use common\components\PriceWriter;
18 18 use common\components\CustomVarDamp;
19   -use yii\web\ErrorAction;
20 19  
21 20 /**
22 21 * Parser controller
... ... @@ -225,7 +224,7 @@ class ParserController extends BaseController
225 224 $file_id = basename($server_file,".csv");
226 225 $arr_id_files[] = (int) $file_id;
227 226 }
228   - Yii::$app->cache->set( 'files_to_delete',json_encode( $arr_id_files ) );
  227 + Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) );
229 228 $query = ImportersFiles::find()->where(['in', 'id', $arr_id_files])->orderBy(['upload_time' => SORT_DESC]);
230 229  
231 230 $provider = new ActiveDataProvider([
... ... @@ -251,23 +250,79 @@ class ParserController extends BaseController
251 250 unlink(Yii::getAlias('@auto_upload') . '/' . $id . '.csv' );
252 251  
253 252 // удалим этот id и из кэша
254   - if( $arr_id_files = Yii::$app->cache->get( 'files_to_delete' ) ){
  253 + if( $arr_id_files = Yii::$app->cache->get( 'files_to_parse' ) ){
255 254 $arr_id_files = json_decode($arr_id_files);
256 255 if (isset( $arr_id_files[$id] ) ) {
257 256 unset( $arr_id_files[$id] );
258 257 // положем уже обновленный массив
259   - Yii::$app->cache->set( 'files_to_delete',json_encode( $arr_id_files ) );
  258 + Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) );
260 259 }
261 260 }
262 261 // сообщим скрипту что все ОК
263 262 echo 1;
264 263 } catch (ErrorException $e) {
265 264  
266   - CustomVarDamp::dump($e->getMessage());
  265 + //CustomVarDamp::dump($e->getMessage());
  266 + throw $e;
267 267  
268 268 }
269 269 }
270 270 }
271 271  
272 272 }
  273 +
  274 +
  275 +
  276 +
  277 + public function actionParse ()
  278 + {
  279 +// $comand = "/usr/bin/php -f ".Yii::getAlias('@console') ."/Controllers/ParserController.php";
  280 +// exec($comand);
  281 + if( $arr_id_files = Yii::$app->cache->get( 'files_to_parse' ) ) {
  282 + $arr_id_files = json_decode( $arr_id_files );
  283 + foreach ( $arr_id_files as $file_name ) {
  284 + $file_path = Yii::getAlias('@auto_upload') . '/' . $file_name . '.csv';
  285 + $config = ['record_id' => $file_name,
  286 + 'importer_id' => ImportersFiles::findOne(['id' => $file_name])->id,
  287 + 'parser_config' => ['keys' => ['DESCR', 'ARTICLE', 'BRAND', 'PRICE', 'BOX'],
  288 + 'mode' => 'console']
  289 + ];
  290 + if( $this->parseFileConsole( $file_path, $config ) ){
  291 + unlink( $file_path );
  292 + if (isset( $arr_id_files[$file_path] ) ) {
  293 + unset($arr_id_files[$file_path]);
  294 + }
  295 + } else {
  296 + // Yii::$app->log->
  297 + // не дошли до конца по этому остаки вернем в кеш
  298 + Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) );
  299 + }
  300 + }
  301 + if ( !count( $arr_id_files ) ) {
  302 + Yii::$app->cache->delete( 'files_to_parse' );
  303 + }
  304 + }
  305 +
  306 + return $this->redirect('serverFiles');
  307 + }
  308 +
  309 + protected function parseFileConsole( $file_path, $configuration ){
  310 + $parser_config = [];
  311 + if ( isset( $configuration['parser_config'] ) ) {
  312 + $parser_config = $configuration['parser_config'];
  313 + }
  314 +
  315 + $data = Yii::$app->multiparser->parse( $file_path, $parser_config );
  316 + CustomVarDamp::dumpAndDie($data);
  317 + $writer = new PriceWriter();
  318 + $writer->configuration = $configuration;
  319 + $writer->data = $data;
  320 + $writer->mode = 1; //console-режим
  321 + if ( $writer->writeDataToDB() ){
  322 + //Console::output('It is working');
  323 + return true;
  324 + }
  325 +
  326 + return false;
  327 + }
273 328 }
... ...
backend/views/parser/server-files.php
  Show/Hide comments   View file @1fe29bb
1 1 <?php
2 2 use yii\helpers\Html;
  3 +use yii\helpers\Url;
3 4 use yii\grid\GridView;
4 5 use yii\grid\SerialColumn;
5 6 use yii\grid\ActionColumn;
... ... @@ -44,7 +45,7 @@ Pjax::begin([&#39;id&#39; =&gt; &#39;server_files_grid&#39;]);
44 45  
45 46 ]] );?>
46 47  
47   -
  48 + <?= Html::a('Загрузить файлы', ['parse'],['class' => 'btn btn-success']) ?>
48 49  
49 50 </div>
50 51 <?php
... ...
common/components/PriceWriter.php
  Show/Hide comments   View file @1fe29bb
... ... @@ -24,7 +24,7 @@ class PriceWriter {
24 24 {
25 25 // 1. запишем дату старта в таблицу файлов поставщика (ImportersFiles)
26 26 // id загруженного файла получим из конфигурации
27   - $files_model = ImportersFiles::findOne( $this->configuration->record_id );
  27 + $files_model = ImportersFiles::findOne( $this->configuration['record_id'] );
28 28  
29 29 //$files_model->load(['ImportersFiles' => $this->configuration->toArray()]);
30 30 $update_date = date('Y-m-d H:i:s');
... ... @@ -61,7 +61,7 @@ class PriceWriter {
61 61 }
62 62 else{
63 63 // дополним данные значением импортера и даты обновления цены
64   - $this->data = \Yii::$app->multiparser->addColumns($this->data, ['IMPORT_ID' => $this->configuration->importer_id, 'timestamp' => $update_date]);
  64 + $this->data = \Yii::$app->multiparser->addColumns($this->data, ['IMPORT_ID' => $this->configuration['importer_id'], 'timestamp' => $update_date]);
65 65  
66 66 try {
67 67 //@todo add transaction
... ...
console/controllers/ParserController.php
  Show/Hide comments   View file @1fe29bb
... ... @@ -6,19 +6,59 @@
6 6 * Time: 14:38
7 7 */
8 8 use yii\console\Controller;
  9 +use yii\helpers\Console;
  10 +use common\components\PriceWriter;
  11 +use backend\models\ImportersFiles;
9 12  
10 13 class ParserController extends Controller{
11 14 public function actionParseCSV ()
12 15 {
13   -
  16 + \common\components\CustomVarDamp::dumpAndDie(45);
  17 + if( $arr_id_files = Yii::$app->cache->get( 'files_to_parse' ) ) {
  18 + $arr_id_files = json_decode( $arr_id_files );
  19 + foreach ( $arr_id_files as $file_name ) {
  20 + $file_path = Yii::getAlias('@auto_upload') . '/' . $file_name . '.csv';
  21 + $config = ['record_id' => $file_name,
  22 + 'importer_id' => ImportersFiles::findOne(['id' => $file_name])->id,
  23 + 'parser_config' => ['keys' => ['DESCR', 'ARTICLE', 'BRAND', 'PRICE', 'BOX']]
  24 + ];
  25 + if( $this->parseFileConsole( $file_path, $config ) ){
  26 + unlink( $file_path );
  27 + if (isset( $arr_id_files[$file_path] ) ) {
  28 + unset($arr_id_files[$file_path]);
  29 + }
  30 + } else {
  31 + // Yii::$app->log->
  32 + // не дошли до конца по этому остаки вернем в кеш
  33 + Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) );
  34 + }
  35 + }
  36 + if ( !count( $arr_id_files ) ) {
  37 + Yii::$app->cache->delete( 'files_to_parse' );
  38 + }
  39 + }
14 40 }
15 41 public function actionParseXML ()
16 42 {
17 43  
18 44 }
19   - protected function parseFileConsole( $file_path ){
  45 + protected function parseFileConsole( $file_path, $configuration ){
  46 + $parser_config = [];
  47 + if ( isset( $configuration['parser_config'] ) ) {
  48 + $parser_config = $configuration['parser_config'];
  49 + }
  50 +
  51 + $data = Yii::$app->multiparser->parse( $file_path, $parser_config );
20 52  
21   - $data = Yii::$app->multiparser->parse( $file_path );
  53 + $writer = new PriceWriter();
  54 + $writer->configuration = $configuration;
  55 + $writer->data = $data;
  56 + $writer->mode = 1; //console-режим
  57 + if ( $writer->writeDataToDB() ){
  58 + Console::output('It is working');
  59 + return true;
  60 + }
22 61  
  62 + return false;
23 63 }
24 64 }
25 65 \ No newline at end of file
... ...