Snippets

Idoenk . Heri Purnomo Draft Pull tables from production

Created by Idoenk . Heri Purnomo
<?php

namespace App\Console\Commands;

use Illuminate\Console\Command;
use Illuminate\Support\Facades\Cache;

use App\Models\Tracker;
use App\Models\Task;

use Carbon\Carbon;
use DB;

class PullMaster extends Command
{
  /**
   * The name and signature of the console command.
   *
   * @var string
   */
  protected $signature = 'sdb:pull-master'
    .' {--force : Force process without checking lockfile }'
  ;

  /**
   * The console command description.
   *
   * @var string
   */
  protected $description = 'Sync tables from other database. In env it specified as DB2_*';

  /**
   * Create a new command instance.
   *
   * @return void
   */
  public function __construct() {
    parent::__construct();

    $this->cacheFile = Cache::store('file');
    $this->cmd_name = (new \ReflectionClass($this))->getShortName();

    // define lockfile to stay in minutes
    $this->expire_lockfile = 20;

    // Size of chunked rows
    $this->chunked_size = 30;

    $this->overwrite_row = true;

    // Main key process
    $this->key_process = 'sdb:pullmaster-lock';
  }

  /**
   * Execute the console command.
   *
   * @return mixed
   */
  public function handle() {
    if (self::isThereProcess()) {
      $this->info('There is still process running: '.$this->key_process);
      $this->info('To allow command run anyway, use option: --force');

      if($this->expireProcess) {
        $expCarbon = Carbon::createFromTimestamp($this->expireProcess);
        $this->info('Try again later: '.$expCarbon->format('d M Y, H:i:s'));
      }

      return null;
    }
    else {
      $start_time = microtime(true);

      \Log::info($this->cmd_name.' initiated.');
      $this->info($this->cmd_name.' started...');
      $this->info('');

      $params = $ret = [];
      $options = $this->options();
      $params['verbose'] = $options['verbose'];
      $is_verbose = !empty($params['verbose']);


      $maptables = [
        'task'    => '\App\Models\Task',
        'tracker' => '\App\Models\Tracker',
        'tracker_tasks' => '\App\Models\TrackerTasks',
      ];
      $todos = array_keys($maptables);

      foreach($todos as $todo){

        $model_name = (isset($maptables[$todo]) ? $maptables[$todo] : null);
        if (empty($model_name))
          continue;

        $this->line(' > Proceeding: '.$todo);

        $Model = null;
        try{
          $Model = new $model_name;
        }catch (Exception $e) {
          \Log::error('Unable initiating model: '.$model_name);
        }
        if (empty($Model)) 
          continue;
 
        $this->importTable($Model);
        $this->line('');
        drd('HALTED');
      }
      // end: foreach(todos)
    }
  }

  /**
   * Proceed import table
   */
  private function importTable($Model){

    /**
     * Insert bulk of collection rows into its Model's table
     * @param Eloquent collection results
     * @return void
     */
    $insertHandler = function($_rows) use ($Model){
      if (empty($_rows) || (!empty($_rows) && $_rows->isEmpty())){
        \Log::warn('Empty row');
        return null;
      }
      $is_verbose = $this->option('verbose');

      $count_inserted = 0;
      foreach($_rows as $row){
        $item_data = json_decode(json_encode($row), true);

        // Find existing by id
        $existing = $Model->find($row->id);

        if (!empty($existing)){
          if ($is_verbose)
            $this->info(' > Row exists, ID: '.$row->id);

          if ($this->overwrite_row){
            // crossed-finger
            $existing->forceDelete();
          }
          else{

            if ($is_verbose)
              $this->info(' > Skipping..');
          }
        }

        if ($ret_item = $Model::create($item_data))
          $count_inserted++;
      }
      // end: foreach

      return $count_inserted;
    };
    // end: insertData;

    $fields = '*';
    // $fields = ['id', 'track_value'];

    // Get task items
    $items = $Model->setConnection('mysql2')
      ->select($fields)
      ->orderBy('id', 'ASC')
    ;
    $count_items = $items->count();

    $chunkSize = $this->chunked_size;
    $chunk_step = 1;

    $this->info(' > Proceed with chunked data: '.$chunkSize);

    // Save ur arse, memory preserve
    $items->chunk($chunkSize, function($rows) use ($insertHandler, &$chunk_step){
      $this->info(' > Chunk-step #'.$chunk_step);

      $insertHandler($rows);

      $chunk_step++;
    });
  }

  private function isThereProcess() {
    $runningProcess = null;

    if($this->option('force'))
      self::destroyLockFile();

    $key_process = $this->key_process;
    $theCache = $this->cacheFile;
    $lock_value = $theCache->get($key_process);
    $now = Carbon::now();

    $generateLockFile = function($expireAt=null) use ($theCache, $key_process) {
      if (empty($expireAt))
        $expireAt = Carbon::now();

      $expireAt = $expireAt->addMinutes($this->expire_lockfile);

      $this->info($this->cmd_name.' initiated: '.Carbon::now()->format('d M Y, H:i:s'));
      $this->info('Lock file created. Expire in '.$this->expire_lockfile.' minutes ('.$expireAt->format('d M Y, H:i:s').')');

      $theCache->put(
        $key_process,
        $expireAt->timestamp,
        $this->expire_lockfile
      );

      $this->info('');
    };

    if (empty($lock_value)) {
      $generateLockFile($now);
      $runningProcess = false;
    }
    else {
      if ($now->timestamp > $lock_value) {
        $generateLockFile($now);
        $runningProcess = false;
      }
      else {
        $runningProcess = true;
        $this->expireProcess = $lock_value;
      }
    }

    return $runningProcess;
  }

  /**
   * Once process is done we destroy lock-file like
   * to allow another instance to run
   */
  private function destroyLockFile() {
    $theCache = $this->cacheFile;

    if ($theCache->has($this->key_process)) {
      $this->info('Destroying lockfile');

      $theCache->forget($this->key_process);
    }

    return true;
  }
}

Comments (0)

HTTPS SSH

You can clone a snippet to your computer for local editing. Learn more.