Merge remote-tracking branch 'origin/develop'

app/Console/Commands/RestoreFromBackup.php

@@ -5,6 +5,151 @@ namespace App\Console\Commands;
 use Illuminate\Console\Command;
 use ZipArchive;
 
+class SQLStreamer {
+    private $input;
+    private $output;
+    // embed the prefix here?
+    public ?string $prefix;
+
+    private bool $reading_beginning_of_line = true;
+
+    public static $buffer_size = 1024 * 1024;  // use a 1MB buffer, ought to work fine for most cases?
+
+    public array $tablenames = [];
+    private bool $should_guess = false;
+    private bool $statement_is_permitted = false;
+
+    public function __construct($input, $output, string $prefix = null)
+    {
+        $this->input = $input;
+        $this->output = $output;
+        $this->prefix = $prefix;
+    }
+
+    public function parse_sql(string $line): string {
+        // take into account the 'start of line or not' setting as an instance variable?
+        // 'continuation' lines for a permitted statement are PERMITTED.
+        if($this->statement_is_permitted && $line[0] === ' ') {
+            return $line;
+        }
+
+        $table_regex = '`?([a-zA-Z0-9_]+)`?';
+        $allowed_statements = [
+            "/^(DROP TABLE (?:IF EXISTS )?)`$table_regex(.*)$/" => false,
+            "/^(CREATE TABLE )$table_regex(.*)$/" => true, //sets up 'continuation'
+            "/^(LOCK TABLES )$table_regex(.*)$/" => false,
+            "/^(INSERT INTO )$table_regex(.*)$/" => false,
+            "/^UNLOCK TABLES/" => false,
+            // "/^\\) ENGINE=InnoDB AUTO_INCREMENT=16 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;/" => false, // FIXME not sure what to do here?
+            "/^\\)[a-zA-Z0-9_= ]*;$/" => false
+            // ^^^^^^ that bit should *exit* the 'perimitted' black
+        ];
+
+        foreach($allowed_statements as $statement => $statechange) {
+//            $this->info("Checking regex: $statement...\n");
+            $matches = [];
+            if (preg_match($statement,$line,$matches)) {
+                $this->statement_is_permitted = $statechange;
+                // matches are: 1 => first part of the statement, 2 => tablename, 3 => rest of statement
+                // (with of course 0 being "the whole match")
+                if (@$matches[2]) {
+//                    print "Found a tablename! It's: ".$matches[2]."\n";
+                    if ($this->should_guess) {
+                        @$this->tablenames[$matches[2]] += 1;
+                        continue; //oh? FIXME
+                    } else {
+                        $cleaned_tablename = \DB::getTablePrefix().preg_replace('/^'.$this->prefix.'/','',$matches[2]);
+                        $line = preg_replace($statement,'$1`'.$cleaned_tablename.'`$3' , $line);
+                    }
+                } else {
+                    // no explicit tablename in this one, leave the line alone
+                }
+                //how do we *replace* the tablename?
+//                print "RETURNING LINE: $line";
+                return $line;
+            }
+        }
+        // all that is not allowed is denied.
+        return "";
+    }
+
+    //this is used in exactly *TWO* places, and in both cases should return a prefix I think?
+    // first - if you do the --sanitize-only one (which is mostly for testing/development)
+    // next - when you run *without* a guessed prefix, this is run first to figure out the prefix
+    // I think we have to *duplicate* the call to be able to run it again?
+    public static function guess_prefix($input):string
+    {
+        $parser = new self($input, null);
+        $parser->should_guess = true;
+        $parser->line_aware_piping(); // <----- THIS is doing the heavy lifting!
+
+        $check_tables = ['settings' => null, 'migrations' => null /* 'assets' => null */]; //TODO - move to statics?
+        //can't use 'users' because the 'accessories_users' table?
+        // can't use 'assets' because 'ver1_components_assets'
+        foreach($check_tables as $check_table => $_ignore) {
+            foreach ($parser->tablenames as $tablename => $_count) {
+//                print "Comparing $tablename to $check_table\n";
+                if (str_ends_with($tablename,$check_table)) {
+//                    print "Found one!\n";
+                    $check_tables[$check_table] = substr($tablename,0,-strlen($check_table));
+                }
+            }
+        }
+        $guessed_prefix = null;
+        foreach ($check_tables as $clean_table => $prefix_guess) {
+            if(is_null($prefix_guess)) {
+                print("Couldn't find table $clean_table\n");
+                die();
+            }
+            if(is_null($guessed_prefix)) {
+                $guessed_prefix = $prefix_guess;
+            } else {
+                if ($guessed_prefix != $prefix_guess) {
+                    print("Prefix mismatch! Had guessed $guessed_prefix but got $prefix_guess\n");
+                    die();
+                }
+            }
+        }
+
+        return $guessed_prefix;
+
+    }
+
+    public function line_aware_piping(): int
+    {
+        $bytes_read = 0;
+        if (! $this->input) {
+            throw new \Exception("No Input available for line_aware_piping");
+        }
+
+        while (($buffer = fgets($this->input, SQLStreamer::$buffer_size)) !== false) {
+            $bytes_read += strlen($buffer);
+            if ($this->reading_beginning_of_line) {
+                // \Log::debug("Buffer is: '$buffer'");
+                $cleaned_buffer = $this->parse_sql($buffer);
+                if ($this->output) {
+                    $bytes_written = fwrite($this->output, $cleaned_buffer);
+
+                    if ($bytes_written === false) {
+                        throw new \Exception("Unable to write to pipe");
+                    }
+                }
+            }
+            // if we got a newline at the end of this, then the _next_ read is the beginning of a line
+            if($buffer[strlen($buffer)-1] === "\n") {
+                $this->reading_beginning_of_line = true;
+            } else {
+                $this->reading_beginning_of_line = false;
+            }
+
+        }
+        return $bytes_read;
+
+    }
+}
+
+
+
 class RestoreFromBackup extends Command
 {
     /**
@@ -12,10 +157,13 @@ class RestoreFromBackup extends Command
      *
      * @var string
      */
+    // FIXME - , stripping prefixes and nonstandard SQL statements. Without --prefix, guess and return the correct prefix to strip
     protected $signature = 'snipeit:restore 
                                             {--force : Skip the danger prompt; assuming you enter "y"} 
                                             {filename : The zip file to be migrated}
-                                            {--no-progress : Don\'t show a progress bar}';
+                                            {--no-progress : Don\'t show a progress bar}
+                                            {--sanitize-guess-prefix : Guess and output the table-prefix needed to "sanitize" the SQL}
+                                            {--sanitize-with-prefix= : "Sanitize" the SQL, using the passed-in table prefix (can be learned from --sanitize-guess-prefix). Pass as just \'--sanitize-with-prefix=\' to use no prefix}';
 
     /**
      * The console command description.
@@ -34,8 +182,6 @@ class RestoreFromBackup extends Command
         parent::__construct();
     }
 
-    public static $buffer_size = 1024 * 1024;  // use a 1MB buffer, ought to work fine for most cases?
-
     /**
      * Execute the console command.
      *
@@ -55,7 +201,7 @@ class RestoreFromBackup extends Command
             return $this->error('Missing required filename');
         }
 
-        if (! $this->option('force') && ! $this->confirm('Are you sure you wish to restore from the given backup file? This can lead to MASSIVE DATA LOSS!')) {
+        if (! $this->option('force') && ! $this->option('sanitize-guess-prefix') && ! $this->confirm('Are you sure you wish to restore from the given backup file? This can lead to MASSIVE DATA LOSS!')) {
             return $this->error('Data loss not confirmed');
         }
 
@@ -196,7 +342,6 @@ class RestoreFromBackup extends Command
             }
             $boring_files[] = $raw_path; //if we've gotten to here and haven't continue'ed our way into the next iteration, we don't want this file
         } // end of pre-processing the ZIP file for-loop
-
         // print_r($interesting_files);exit(-1);
 
         if (count($sqlfiles) != 1) {
@@ -208,6 +353,17 @@ class RestoreFromBackup extends Command
             //older Snipe-IT installs don't have the db-dumps subdirectory component
         }
 
+        $sql_stat = $za->statIndex($sqlfile_indices[0]);
+        //$this->info("SQL Stat is: ".print_r($sql_stat,true));
+        $sql_contents = $za->getStream($sql_stat['name']); // maybe copy *THIS* thing?
+
+        // OKAY, now that we *found* the sql file if we're doing just the guess-prefix thing, we can do that *HERE* I think?
+        if ($this->option('sanitize-guess-prefix')) {
+            $prefix = SQLStreamer::guess_prefix($sql_contents);
+            $this->line($prefix);
+            return $this->info("Re-run this command with '--sanitize-with-prefix=".$prefix."' to see an attempt to sanitze your SQL.");
+        }
+
         //how to invoke the restore?
         $pipes = [];
 
@@ -228,6 +384,7 @@ class RestoreFromBackup extends Command
             return $this->error('Unable to invoke mysql via CLI');
         }
 
+        // I'm not sure about these?
         stream_set_blocking($pipes[1], false); // use non-blocking reads for stdout
         stream_set_blocking($pipes[2], false); // use non-blocking reads for stderr
 
@@ -238,9 +395,9 @@ class RestoreFromBackup extends Command
 
         //$sql_contents = fopen($sqlfiles[0], "r"); //NOPE! This isn't a real file yet, silly-billy!
 
-        $sql_stat = $za->statIndex($sqlfile_indices[0]);
+        // FIXME - this feels like it wants to go somewhere else?
-        //$this->info("SQL Stat is: ".print_r($sql_stat,true));
+        // and it doesn't seem 'right' - if you can't get a stream to the .sql file,
-        $sql_contents = $za->getStream($sql_stat['name']);
+        // why do we care what's happening with pipes and stdout and stderr?!
         if ($sql_contents === false) {
             $stdout = fgets($pipes[1]);
             $this->info($stdout);
@@ -249,10 +406,12 @@ class RestoreFromBackup extends Command
 
             return false;
         }
-        $bytes_read = 0;
 
         try {
-            while (($buffer = fgets($sql_contents, self::$buffer_size)) !== false) {
+            if ( $this->option('sanitize-with-prefix') === null) {
+                // "Legacy" direct-piping
+                $bytes_read = 0;
+                while (($buffer = fgets($sql_contents, SQLStreamer::$buffer_size)) !== false) {
                     $bytes_read += strlen($buffer);
                     // \Log::debug("Buffer is: '$buffer'");
                     $bytes_written = fwrite($pipes[0], $buffer);
@@ -261,8 +420,13 @@ class RestoreFromBackup extends Command
                         throw new Exception("Unable to write to pipe");
                     }
                 }
+            } else {
+                $sql_importer = new SQLStreamer($sql_contents, $pipes[0], $this->option('sanitize-with-prefix'));
+                $bytes_read = $sql_importer->line_aware_piping();
+            }
         } catch (\Exception $e) {
             \Log::error("Error during restore!!!! ".$e->getMessage());
+            // FIXME - put these back and/or put them in the right places?!
             $err_out = fgets($pipes[1]);
             $err_err = fgets($pipes[2]);
             \Log::error("Error OUTPUT: ".$err_out);
@@ -271,7 +435,6 @@ class RestoreFromBackup extends Command
             $this->error($err_err);
             throw $e;
         }
-        
         if (!feof($sql_contents) || $bytes_read == 0) {
             return $this->error("Not at end of file for sql file, or zero bytes read. aborting!");
         }
@@ -303,7 +466,7 @@ class RestoreFromBackup extends Command
             $fp = $za->getStream($ugly_file_name);
             //$this->info("Weird problem, here are file details? ".print_r($file_details,true));
             $migrated_file = fopen($file_details['dest'].'/'.basename($pretty_file_name), 'w');
-            while (($buffer = fgets($fp, self::$buffer_size)) !== false) {
+            while (($buffer = fgets($fp, SQLStreamer::$buffer_size)) !== false) {
                 fwrite($migrated_file, $buffer);
             }
             fclose($migrated_file);

composer.json

@@ -77,7 +77,8 @@
     "watson/validating": "^6.1"
   },
   "suggest": {
-    "ext-ldap": "*"
+    "ext-ldap": "*",
+    "ext-zip": "*"
   },
   "require-dev": {
     "brianium/paratest": "^6.6",