fixed parser and converter for console needs

Mihail
1 parent f0dbd829
Showing 6 changed files with 113 additions and 14 deletions Show diff stats
backend/components/parsers/CustomCsvParser.php
backend/components/parsers/config.php
backend/controllers/ParserController.php
backend/views/parser/server-files.php
common/components/PriceWriter.php
console/controllers/ParserController.php
@@ -9,6 +9,8 @@
 namespace backend\components\parsers;
+use common\components\CustomVarDamp;
+
 class CustomCsvParser extends  \yii\multiparser\CsvParser {
     public $last_line = 10;
@@ -33,9 +35,8 @@ class CustomCsvParser extends  \yii\multiparser\CsvParser {
      */
     protected function convert($arr)
     {
-        $result = \Yii::$app->multiparser->convertByConfiguration( $arr, $this->converter_conf );
-
-        return $result;
+        $arr = \Yii::$app->multiparser->convertByConfiguration($arr, $this->converter_conf);
+        return $arr;
     }
@@ -21,7 +21,9 @@
           'console' =>
              ['class' => 'backend\components\parsers\CustomCsvParser',
                  'auto_detect_first_line' => true,
+                 'hasHeaderRow' => true,
                  'converter_conf' => ['class' => ' backend\components\parsers\CustomConverter',
+                     'hasKey' => 1,
                      'configuration' => ["string"   => 'DESCR',
                                          "float"   => 'PRICE',
                                         "integer"   => ['BOX','ADD_BOX']
@@ -16,7 +16,6 @@ use backend\models\Importers;
 use yii\base\ErrorException;
 use common\components\PriceWriter;
 use common\components\CustomVarDamp;
-use yii\web\ErrorAction;
 /**
  * Parser controller
@@ -225,7 +224,7 @@ class ParserController extends BaseController
             $file_id = basename($server_file,".csv");
             $arr_id_files[] = (int) $file_id;
         }
-        Yii::$app->cache->set( 'files_to_delete',json_encode( $arr_id_files ) );
+        Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) );
         $query = ImportersFiles::find()->where(['in', 'id', $arr_id_files])->orderBy(['upload_time' => SORT_DESC]);
         $provider = new ActiveDataProvider([
@@ -251,23 +250,79 @@ class ParserController extends BaseController
                     unlink(Yii::getAlias('@auto_upload') . '/' . $id . '.csv' );
                     // удалим этот id и из кэша
-                   if( $arr_id_files = Yii::$app->cache->get( 'files_to_delete' ) ){
+                   if( $arr_id_files = Yii::$app->cache->get( 'files_to_parse' ) ){
                        $arr_id_files = json_decode($arr_id_files);
                        if (isset( $arr_id_files[$id] ) ) {
                            unset( $arr_id_files[$id] );
                            // положем уже обновленный массив
-                           Yii::$app->cache->set( 'files_to_delete',json_encode( $arr_id_files ) );
+                           Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) );
                        }
                    }
                     // сообщим скрипту что все ОК
                     echo 1;
                 } catch (ErrorException  $e) {
-                    CustomVarDamp::dump($e->getMessage());
+                    //CustomVarDamp::dump($e->getMessage());
+                    throw $e;
                 }
             }
         }
     }
+
+
+
+
+    public function actionParse ()
+    {
+//        $comand = "/usr/bin/php -f ".Yii::getAlias('@console') ."/Controllers/ParserController.php";
+//        exec($comand);
+        if( $arr_id_files = Yii::$app->cache->get( 'files_to_parse' ) ) {
+            $arr_id_files = json_decode( $arr_id_files );
+            foreach ( $arr_id_files as $file_name ) {
+                $file_path = Yii::getAlias('@auto_upload') . '/' . $file_name . '.csv';
+                $config = ['record_id' => $file_name,
+                    'importer_id' => ImportersFiles::findOne(['id' => $file_name])->id,
+                    'parser_config' => ['keys' => ['DESCR', 'ARTICLE', 'BRAND', 'PRICE', 'BOX'],
+                                        'mode' => 'console']
+                ];
+                if( $this->parseFileConsole( $file_path, $config ) ){
+                    unlink( $file_path );
+                    if (isset( $arr_id_files[$file_path] ) ) {
+                        unset($arr_id_files[$file_path]);
+                    }
+                } else {
+                    // Yii::$app->log->
+                    // не дошли до конца по этому остаки вернем в кеш
+                    Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) );
+                }
+            }
+            if ( !count( $arr_id_files ) ) {
+                Yii::$app->cache->delete( 'files_to_parse' );
+            }
+        }
+
+        return $this->redirect('serverFiles');
+    }
+
+    protected function parseFileConsole( $file_path, $configuration ){
+        $parser_config = [];
+        if ( isset( $configuration['parser_config'] ) ) {
+            $parser_config = $configuration['parser_config'];
+        }
+
+        $data = Yii::$app->multiparser->parse( $file_path, $parser_config );
+        CustomVarDamp::dumpAndDie($data);
+        $writer = new PriceWriter();
+        $writer->configuration = $configuration;
+        $writer->data = $data;
+        $writer->mode = 1; //console-режим
+        if ( $writer->writeDataToDB() ){
+            //Console::output('It is working');
+            return true;
+        }
+
+        return false;
+    }
 }
 <?php
 use yii\helpers\Html;
+use yii\helpers\Url;
 use yii\grid\GridView;
 use yii\grid\SerialColumn;
 use yii\grid\ActionColumn;
@@ -44,7 +45,7 @@ Pjax::begin([&#39;id&#39; =&gt; &#39;server_files_grid&#39;]);
             ]] );?>
-
+        <?= Html::a('Загрузить файлы', ['parse'],['class' => 'btn btn-success']) ?>
     </div>
 <?php
@@ -24,7 +24,7 @@ class PriceWriter {
     {
         // 1. запишем дату старта в таблицу файлов поставщика (ImportersFiles)
         // id  загруженного файла получим из конфигурации
-        $files_model = ImportersFiles::findOne( $this->configuration->record_id );
+        $files_model = ImportersFiles::findOne( $this->configuration['record_id'] );
         //$files_model->load(['ImportersFiles' => $this->configuration->toArray()]);
         $update_date = date('Y-m-d H:i:s');
@@ -61,7 +61,7 @@ class PriceWriter {
         }
             else{
             // дополним данные значением импортера и даты обновления цены
-            $this->data = \Yii::$app->multiparser->addColumns($this->data, ['IMPORT_ID' => $this->configuration->importer_id, 'timestamp' => $update_date]);
+            $this->data = \Yii::$app->multiparser->addColumns($this->data, ['IMPORT_ID' => $this->configuration['importer_id'], 'timestamp' => $update_date]);
             try {
                 //@todo add transaction
@@ -6,19 +6,59 @@
  * Time: 14:38
  */
 use yii\console\Controller;
+use yii\helpers\Console;
+use common\components\PriceWriter;
+use backend\models\ImportersFiles;
 class ParserController extends Controller{
     public function actionParseCSV ()
     {
-
+        \common\components\CustomVarDamp::dumpAndDie(45);
+        if( $arr_id_files = Yii::$app->cache->get( 'files_to_parse' ) ) {
+            $arr_id_files = json_decode( $arr_id_files );
+            foreach ( $arr_id_files as $file_name ) {
+                $file_path = Yii::getAlias('@auto_upload') . '/' . $file_name . '.csv';
+                $config = ['record_id' => $file_name,
+                            'importer_id' => ImportersFiles::findOne(['id' => $file_name])->id,
+                    'parser_config' => ['keys' => ['DESCR', 'ARTICLE', 'BRAND', 'PRICE', 'BOX']]
+                ];
+                if( $this->parseFileConsole( $file_path, $config ) ){
+                    unlink( $file_path );
+                    if (isset( $arr_id_files[$file_path] ) ) {
+                        unset($arr_id_files[$file_path]);
+                    }
+                } else {
+                   // Yii::$app->log->
+                    // не дошли до конца по этому остаки вернем в кеш
+                    Yii::$app->cache->set( 'files_to_parse',json_encode( $arr_id_files ) );
+                }
+            }
+            if ( !count( $arr_id_files ) ) {
+                Yii::$app->cache->delete( 'files_to_parse' );
+            }
+        }
     }
     public function actionParseXML ()
     {
     }
-    protected function parseFileConsole( $file_path ){
+    protected function parseFileConsole( $file_path, $configuration ){
+        $parser_config = [];
+        if ( isset( $configuration['parser_config'] ) ) {
+            $parser_config = $configuration['parser_config'];
+        }
+
+        $data = Yii::$app->multiparser->parse( $file_path, $parser_config );
-        $data = Yii::$app->multiparser->parse( $file_path );
+        $writer = new PriceWriter();
+        $writer->configuration = $configuration;
+        $writer->data = $data;
+        $writer->mode = 1; //console-режим
+        if ( $writer->writeDataToDB() ){
+            Console::output('It is working');
+            return true;
+        }
+        return false;
     }
 }
 \ No newline at end of file