Enigma: Fixed multi-host syncronization of private and deleted keys and pubring.kbx file

Added context column to filestore table for easier listing of stored files.

CHANGELOG

@@ -15,6 +15,7 @@ CHANGELOG Roundcube Webmail
 - Plugin API: Added 'common_headers' hook (#6385)
 - Plugin API: Added 'ldap_connected' hook
 - Enigma: Update to OpenPGPjs 4.2.1 - fixes user name encoding issues in key generation (#6524)
+- Enigma: Fixed multi-host synchronization of private and deleted keys and pubring.kbx file
 - Managesieve: Added support for 'editheader' extension - RFC5293 (#5954)
 - Markasjunk: Integrate markasjunk2 features into markasjunk - marking as non-junk + learning engine (#6504)
 - Password: Added 'modoboa' driver (#6361)

SQL/mssql.initial.sql

@@ -126,6 +126,7 @@ GO
 CREATE TABLE [dbo].[filestore] (
 	[file_id] [int] IDENTITY (1, 1) NOT NULL ,
 	[user_id] [int] NOT NULL ,
+	[context] [varchar] (32) COLLATE Latin1_General_CI_AI NOT NULL ,
 	[filename] [varchar] (128) COLLATE Latin1_General_CI_AI NOT NULL ,
 	[mtime] [int] NOT NULL ,
 	[data] [text] COLLATE Latin1_General_CI_AI NULL ,
@@ -360,6 +361,9 @@ GO
 CREATE UNIQUE INDEX [IX_searches_user_type_name] ON [dbo].[searches]([user_id],[type],[name]) ON [PRIMARY]
 GO
 
+CREATE UNIQUE INDEX [IX_filestore_user_id_context_filename] ON [dbo].[filestore]([user_id],[context],[filename]) ON [PRIMARY]
+GO
+
 ALTER TABLE [dbo].[identities] ADD CONSTRAINT [FK_identities_user_id] 
     FOREIGN KEY ([user_id]) REFERENCES [dbo].[users] ([user_id])
     ON DELETE CASCADE ON UPDATE CASCADE
@@ -418,6 +422,6 @@ CREATE TRIGGER [contact_delete_member] ON [dbo].[contacts]
     WHERE [contact_id] IN (SELECT [contact_id] FROM deleted)
 GO
 
-INSERT INTO [dbo].[system] ([name], [value]) VALUES ('roundcube-version', '2018021600')
+INSERT INTO [dbo].[system] ([name], [value]) VALUES ('roundcube-version', '2018122300')
 GO
 

SQL/mssql/2018122300.sql

@@ -0,0 +1,9 @@
+ALTER TABLE [dbo].[filestore] ADD COLUMN [context] [varchar] (32) COLLATE Latin1_General_CI_AI NOT NULL
+GO
+
+UPDATE [dbo].[filestore] SET [dbo].[context] = 'enigma'
+GO
+
+CREATE UNIQUE INDEX [IX_filestore_user_id_context_filename] ON [dbo].[filestore]([user_id],[context],[filename]) ON [PRIMARY]
+GO
+

SQL/mysql.initial.sql

@@ -203,13 +203,14 @@ CREATE TABLE `searches` (
 CREATE TABLE `filestore` (
  `file_id` int(10) UNSIGNED NOT NULL AUTO_INCREMENT,
  `user_id` int(10) UNSIGNED NOT NULL,
+ `context` varchar(32) NOT NULL,
  `filename` varchar(128) NOT NULL,
  `mtime` int(10) NOT NULL,
  `data` longtext NOT NULL,
  PRIMARY KEY (`file_id`),
  CONSTRAINT `user_id_fk_filestore` FOREIGN KEY (`user_id`)
    REFERENCES `users` (`user_id`) ON DELETE CASCADE ON UPDATE CASCADE,
- UNIQUE `uniqueness` (`user_id`, `filename`)
+ UNIQUE `uniqueness` (`user_id`, `context`, `filename`)
 );
 
 -- Table structure for table `system`
@@ -222,4 +223,4 @@ CREATE TABLE `system` (
 
 /*!40014 SET FOREIGN_KEY_CHECKS=1 */;
 
-INSERT INTO `system` (`name`, `value`) VALUES ('roundcube-version', '2018021600');
+INSERT INTO `system` (`name`, `value`) VALUES ('roundcube-version', '2018122300');

SQL/mysql/2018122300.sql

@@ -0,0 +1,7 @@
+ALTER TABLE `filestore` ADD COLUMN `context` varchar(32) NOT NULL;
+UPDATE `filestore` SET `context` = 'enigma';
+ALTER TABLE `filestore` DROP FOREIGN KEY `user_id_fk_filestore`;
+ALTER TABLE `filestore` DROP INDEX `uniqueness`;
+ALTER TABLE `filestore` ADD UNIQUE INDEX `uniqueness` (`user_id`, `context`, `filename`);
+ALTER TABLE `filestore` ADD CONSTRAINT `user_id_fk_filestore` FOREIGN KEY (`user_id`)
+    REFERENCES `users` (`user_id`) ON DELETE CASCADE ON UPDATE CASCADE;

SQL/oracle.initial.sql

@@ -216,10 +216,11 @@ CREATE TABLE "filestore" (
     "file_id" integer PRIMARY KEY,
     "user_id" integer NOT NULL
         REFERENCES "users" ("user_id") ON DELETE CASCADE ON UPDATE CASCADE,
+    "context" varchar(32) NOT NULL,
     "filename" varchar(128) NOT NULL,
     "mtime" integer NOT NULL,
     "data" long,
-    CONSTRAINT "filestore_user_id_key" UNIQUE ("user_id", "filename")
+    CONSTRAINT "filestore_user_id_key" UNIQUE ("user_id", "context", "filename")
 );
 
 CREATE SEQUENCE "filestore_seq"
@@ -237,4 +238,4 @@ CREATE TABLE "system" (
     "value" long
 );
 
-INSERT INTO "system" ("name", "value") VALUES ('roundcube-version', '2018021600');
+INSERT INTO "system" ("name", "value") VALUES ('roundcube-version', '2018122300');

SQL/oracle/2018122300.sql

@@ -0,0 +1,4 @@
+ALTER TABLE "filestore" ADD COLUMN "context" varchar(32) NOT NULL;
+UPDATE "filestore" SET "context" = 'enigma';
+ALTER TABLE "filestore" DROP CONSTRAINT "filestore_user_id_key";
+ALTER TABLE "filestore" ADD CONSTRAINT "filestore_user_id_key" UNIQUE ("user_id", "context", "filename");

SQL/postgres.initial.sql

@@ -297,10 +297,11 @@ CREATE TABLE "filestore" (
     file_id integer DEFAULT nextval('filestore_seq'::text) PRIMARY KEY,
     user_id integer NOT NULL
         REFERENCES users (user_id) ON DELETE CASCADE ON UPDATE CASCADE,
+    context varchar(32) NOT NULL,
     filename varchar(128) NOT NULL,
     mtime integer NOT NULL,
     data text NOT NULL,
-    CONSTRAINT filestore_user_id_filename UNIQUE (user_id, filename)
+    CONSTRAINT filestore_user_id_filename UNIQUE (user_id, context, filename)
 );
 
 --
@@ -313,4 +314,4 @@ CREATE TABLE "system" (
     value text
 );
 
-INSERT INTO "system" (name, value) VALUES ('roundcube-version', '2018021600');
+INSERT INTO "system" (name, value) VALUES ('roundcube-version', '2018122300');

SQL/postgres/2018122300.sql

@@ -0,0 +1,4 @@
+ALTER TABLE "filestore" ADD COLUMN context varchar(32) NOT NULL;
+UPDATE "filestore" SET context = 'enigma';
+ALTER TABLE "filestore" DROP CONSTRAINT "filestore_user_id_filename";
+ALTER TABLE "filestore" ADD CONSTRAINT "filestore_user_id_context_filename" UNIQUE (user_id, context, filename);

SQL/sqlite.initial.sql

@@ -196,14 +196,15 @@ CREATE INDEX ix_cache_messages_expires ON cache_messages (expires);
 --
 
 CREATE TABLE filestore (
-    file_id integer PRIMARY KEY,
+    file_id integer NOT NULL PRIMARY KEY,
     user_id integer NOT NULL,
+    context varchar(32) NOT NULL,
     filename varchar(128) NOT NULL,
     mtime integer NOT NULL,
     data text NOT NULL
 );
 
-CREATE UNIQUE INDEX ix_filestore_user_id ON filestore(user_id, filename);
+CREATE UNIQUE INDEX ix_filestore_user_id ON filestore(user_id, context, filename);
 
 --
 -- Table structure for table system
@@ -214,4 +215,4 @@ CREATE TABLE system (
   value text NOT NULL
 );
 
-INSERT INTO system (name, value) VALUES ('roundcube-version', '2018021600');
+INSERT INTO system (name, value) VALUES ('roundcube-version', '2018122300');

SQL/sqlite/2018122300.sql

@@ -0,0 +1,29 @@
+CREATE TABLE tmp_filestore (
+    file_id integer PRIMARY KEY,
+    user_id integer NOT NULL,
+    filename varchar(128) NOT NULL,
+    mtime integer NOT NULL,
+    data text NOT NULL
+);
+
+INSERT INTO tmp_filestore (file_id, user_id, filename, mtime, data)
+    SELECT file_id, user_id, filename, mtime, data FROM filestore;
+
+DROP TABLE filestore;
+
+CREATE TABLE filestore (
+    file_id integer NOT NULL PRIMARY KEY,
+    user_id integer NOT NULL,
+    context varchar(32) NOT NULL,
+    filename varchar(128) NOT NULL,
+    mtime integer NOT NULL,
+    data text NOT NULL
+);
+
+INSERT INTO filestore (file_id, user_id, filename, mtime, data, context)
+    SELECT file_id, user_id, filename, mtime, data, 'enigma' FROM tmp_filestore;
+
+CREATE UNIQUE INDEX ix_filestore_user_id ON filestore(user_id, context, filename);
+
+DROP TABLE tmp_filestore;
+

plugins/enigma/lib/enigma_driver_gnupg.php

@@ -25,7 +25,7 @@ class enigma_driver_gnupg extends enigma_driver
     protected $user;
     protected $last_sig_algorithm;
     protected $debug    = false;
-    protected $db_files = array('pubring.gpg', 'secring.gpg');
+    protected $db_files = array('pubring.gpg', 'secring.gpg', 'pubring.kbx');
 
 
     function __construct($user)
@@ -560,16 +560,16 @@ class enigma_driver_gnupg extends enigma_driver
 
         $db    = $this->rc->get_dbh();
         $table = $db->table_name('filestore', true);
+        $files = array();
 
         $result = $db->query(
-            "SELECT `file_id`, `filename`, `mtime` FROM $table"
+            "SELECT `file_id`, `filename`, `mtime` FROM $table WHERE `user_id` = ? AND `context` = ?",
-            . " WHERE `user_id` = ? AND `filename` IN (" . $db->array2list($this->db_files) . ")",
+            $this->rc->user->ID, 'enigma');
-            $this->rc->user->ID
-        );
 
         while ($record = $db->fetch_assoc($result)) {
             $file  = $this->homedir . '/' . $record['filename'];
             $mtime = @filemtime($file);
+            $files[] = $record['filename'];
 
             if ($mtime < $record['mtime']) {
                 $data_result = $db->query("SELECT `data`, `mtime` FROM $table"
@@ -609,6 +609,19 @@ class enigma_driver_gnupg extends enigma_driver
             }
         }
 
+        // Remove files not in database
+        if (!$db->is_error($result)) {
+            foreach (array_diff($this->db_files_list(), $files) as $file) {
+                $file = $this->homedir . '/' . $file;
+
+                if (unlink($file)) {
+                    if ($this->debug) {
+                        $this->debug("SYNC: Removed file: $file");
+                    }
+                }
+            }
+        }
+
         // No records found, do initial sync if already have the keyring
         if (!$db->is_error($result) && empty($file)) {
             $this->db_save(true);
@@ -630,9 +643,8 @@ class enigma_driver_gnupg extends enigma_driver
 
         if (!$is_empty) {
             $result = $db->query(
-                "SELECT `file_id`, `filename`, `mtime` FROM $table"
+                "SELECT `file_id`, `filename`, `mtime` FROM $table WHERE `user_id` = ? AND `context` = ?",
-                . " WHERE `user_id` = ? AND `filename` IN (" . $db->array2list($this->db_files) . ")",
+                $this->rc->user->ID, 'enigma'
-                $this->rc->user->ID
             );
 
             while ($record = $db->fetch_assoc($result)) {
@@ -640,11 +652,14 @@ class enigma_driver_gnupg extends enigma_driver
             }
         }
 
-        foreach ($this->db_files as $filename) {
+        foreach ($this->db_files_list() as $filename) {
             $file  = $this->homedir . '/' . $filename;
             $mtime = @filemtime($file);
 
-            if ($mtime && (empty($records[$filename]) || $mtime > $records[$filename]['mtime'])) {
+            $existing = $records[$filename];
+            unset($records[$filename]);
+
+            if ($mtime && (empty($existing) || $mtime > $existing['mtime'])) {
                 $data     = file_get_contents($file);
                 $data     = base64_encode($data);
                 $datasize = strlen($data);
@@ -662,16 +677,16 @@ class enigma_driver_gnupg extends enigma_driver
                     continue;
                 }
 
-                if (empty($records[$filename])) {
+                if (empty($existing)) {
                     $result = $db->query(
-                        "INSERT INTO $table (`user_id`, `filename`, `mtime`, `data`)"
+                        "INSERT INTO $table (`user_id`, `context`, `filename`, `mtime`, `data`)"
-                        . " VALUES(?, ?, ?, ?)",
+                        . " VALUES(?, 'enigma', ?, ?, ?)",
                         $this->rc->user->ID, $filename, $mtime, $data);
                 }
                 else {
                     $result = $db->query(
                         "UPDATE $table SET `mtime` = ?, `data` = ? WHERE `file_id` = ?",
-                        $mtime, $data, $records[$filename]['file_id']);
+                        $mtime, $data, $existing['file_id']);
                 }
 
                 if ($db->is_error($result)) {
@@ -688,6 +703,46 @@ class enigma_driver_gnupg extends enigma_driver
                 }
             }
         }
+
+        // Delete removed files from database
+        foreach (array_keys($records) as $filename) {
+            $file   = $this->homedir . '/' . $filename;
+            $result = $db->query("DELETE FROM $table WHERE `user_id` = ? AND `context` = ? AND `filename` = ?",
+                $this->rc->user->ID, 'enigma', $filename);
+
+            if ($db->is_error($result)) {
+                rcube::raise_error(array(
+                        'code' => 605, 'line' => __LINE__, 'file' => __FILE__,
+                        'message' => "Enigma: Failed to delete $file from database."
+                    ), true, false);
+
+                break;
+            }
+
+            if ($this->debug) {
+                $this->debug("SYNC: Removed file: $file");
+            }
+        }
+    }
+
+    /**
+     * Returns list of homedir files to backup
+     */
+    protected function db_files_list()
+    {
+        $files = array();
+
+        foreach ($this->db_files as $file) {
+            if (file_exists($this->homedir . '/' . $file)) {
+                $files[] = $file;
+            }
+        }
+
+        foreach (glob($this->homedir . '/private-keys-v1.d/*.key') as $file) {
+            $files[] = ltrim(substr($file, strlen($this->homedir)), '/');
+        }
+
+        return $files;
     }
 
     /**