dbbackup: fixup, update README.md, add requirements.txt

Author: SimonVrouwe

dbbackup/README.md

@@ -1,10 +1,47 @@
-# dbbackup plugin
+## Summary plugin
 
-STILL WORK-IN-PROGRESS ** EXPERIMENTAL **
+This plugin keeps a synchronized backup of c-lightning's (CL) sqlite3 database.
+It uses the db_write hook so that every commit (write) to CL's database is first
+written to the backup. This allows recovery of any committed-to channel state,
+including HTLCs. This plugin does not backup the seed and is not a complete
+node-backup.
 
-# testing
-The tests utilize c-lightning's pytest framework. To run the tests, you should symlink
-this repositories (dbbackup) directory into c-lightning repo's /test directory.
+## Installation
 
-Then go into c-lighting repo directory and to run dbbackup's two tests:
-`DEVELOPER=1 py.test tests/ -s -v -k test_dbbackup_`
+** DO NOT ** copy or link this repository's `/dbbackup` directory into
+`~/.lightning/plugins`, because it also contains executable python scripts
+in its `/tests` sub-directory.
+
+Instead copy or link the single `dbbackup.py` file into
+`~/.lightning/plugins/dbbackup` or start CL with the option:
+
+`--plugin=/path/to/this/repo/plugins/dbbackup/dbbackup.py`
+
+## Options:
+
+* `--db-backup-file`: path of the backup file
+
+## Usage
+
+If the given [`db-backup-file`] doesn't exist yet, it will be created from a
+copy of CL's database.
+
+During startup, any existing backup file is checked to match CL's current
+database. If that check fails or initialization fails for other reasons, it will
+shutdown CL and log `**BROKEN**`. If the plugin fails in writing to the backup
+file, it will trigger CL to crash.
+
+The backup file is created with rw permission for the owner, it contains
+sensitive information, so be a bit careful. When plugin complains about mismatch
+between backup and original db, please investigate what caused it before
+recovering.
+
+To recover: shutdown CL and copy the backup to `~/.lighting/lightningd.sqlite3`
+File permissions may need to be restored.
+
+## Testing
+
+The tests uses c-lightning's pytest framework. To run the tests, you can
+link or copy this repository's `/dbbackup` directory into c-lightning repo's
+`/test` directory. Then cd into the c-lighting repo directory and to run
+test_dbbackup_* tests, run: `DEVELOPER=1 py.test tests/ -s -v -k test_dbbackup_`

dbbackup/dbbackup.py

@@ -1,12 +1,11 @@
 #!/usr/bin/env python3
-"""This plugin creates and updates a backup of c-lightnings .sqlite3 database.
-"""
 from lightning import Plugin
 import os
 import shutil
 import sqlite3
 from stat import S_IRUSR, S_IWUSR
 
+
 plugin = Plugin()
 plugin.sqlite_pre_init_cmds = []
 plugin.initted = False
@@ -38,8 +37,7 @@ def init(configuration, options, plugin):
         db = os.path.join(configuration['lightning-dir'], 'lightningd.sqlite3')
         backup = plugin.get_option('db-backup-file')
 
-        # If backup exist, replay pre_init_cmds on a temporary copy and compare that
-        # with the original database
+        # If backup exist, replay pre_init_cmds on a temporary copy
         if os.path.isfile(backup):
             plugin.log('Found existing db-backup-file: {} comparing...'.format(backup))
             backup_copy = shutil.copy(backup, backup + '.tmp')
@@ -48,7 +46,11 @@ def init(configuration, options, plugin):
             for c in plugin.sqlite_pre_init_cmds:
                 db1.execute(c)
 
-            if [x for x in db1.iterdump()] == [x for x in db2.iterdump()]:
+            # If it then matches orignal db, replace backup with copy ... else abort
+            dbs_match = [x for x in db1.iterdump()] == [x for x in db2.iterdump()]
+            db1.close()
+            db2.close()
+            if dbs_match:
                 os.rename(backup_copy, backup)
                 plugin.log("Existing db-backup-file OK and successfully synced")
             else:
@@ -57,8 +59,6 @@ def init(configuration, options, plugin):
                 os.remove(backup_copy)
                 plugin.rpc.stop()   # stop lightningd
 
-            db1.close()
-            db2.close()
         else:
             new_backup_file(db, backup)
 

dbbackup/requirements.txt

@@ -0,0 +1 @@
+pylightning>=0.0.7.3

dbbackup/tests/test_dbbackup.py

@@ -1,10 +1,15 @@
 from fixtures import *  # noqa: F401,F403
 from flaky import flaky  # noqa: F401
 from lightning import RpcError
+from utils import DEVELOPER, wait_for
+
+import os
+import pytest
+import re
+import shutil
 import signal
 import time
 import unittest
-from utils import wait_for
 
 
 # Crashing or shutting-down a node raises unpredictable errors/exceptions, thus @flaky
@@ -62,12 +67,12 @@ def test_dbbackup_recover(node_factory, executor):
     # l3 is our unfortunate, may_reconnect=False prevents reconnect-attempts,
     # but incoming or manual connections still work
     db_backup = os.path.join(node_factory.directory, "l3_lightningd.sqlite3-backup")
-    opts=[{'may_reconnect': True},
-          {'may_reconnect': False, 'may_fail': True},
-          {'may_reconnect': False, 'may_fail': True,
-           'plugin': 'tests/dbbackup/dbbackup.py',
-           'db-backup-file': db_backup,
-           'disconnect': ['@WIRE_UPDATE_FULFILL_HTLC']}]
+    opts = [{'may_reconnect': True},
+            {'may_reconnect': False, 'may_fail': True},
+            {'may_reconnect': False, 'may_fail': True,
+             'plugin': 'tests/dbbackup/dbbackup.py',
+             'db-backup-file': db_backup,
+             'disconnect': ['@WIRE_UPDATE_FULFILL_HTLC']}]
 
     # l3 looses its database with a beneficial HTLC in flight
     l1, l2, l3 = node_factory.line_graph(3, opts=opts, wait_for_announce=True)
@@ -85,10 +90,10 @@ def test_dbbackup_recover(node_factory, executor):
     os.rename(db_backup, db_orig)
     l3.daemon.opts.pop('dev-disconnect')
     l3.daemon.opts.pop('dev-no-reconnect')
-    assert l1.rpc.listsendpays(payment_hash = phash)['payments'][0]['status'] == 'pending'
+    assert l1.rpc.listsendpays(payment_hash=phash)['payments'][0]['status'] == 'pending'
     l3.start()
     l2.rpc.connect(l3.info['id'], 'localhost', l3.port)['id']
-    wait_for(lambda: l1.rpc.listsendpays(payment_hash = phash)['payments'][0]['status'] == 'complete')
+    wait_for(lambda: l1.rpc.listsendpays(payment_hash=phash)['payments'][0]['status'] == 'complete')
 
     # a HACK to get around `ValueError: 2 nodes had unexpected reconnections`
     l3.daemon.logs = [re.sub('Peer has reconnected', 'MODDED_PeerHasReconnected', l) for l in l3.daemon.logs]
@@ -150,26 +155,7 @@ def test_dbbackup_plugin_kill(node_factory, executor):
     # kill the plugin, be a bit careful extracting pid from log
     logline = l1.daemon.is_in_log('plugin-manager started\(\d+\).*dbbackup.py')
     assert logline is not None
-    pid = int(re.search(r'plugin-manager started\((\d+)\).*dbbackup.py',logline ).group(1))
+    pid = int(re.search(r'plugin-manager started\((\d+)\).*dbbackup.py', logline).group(1))
     os.kill(pid, signal.SIGTERM)
     time.sleep(1)
     assert l1.daemon.is_in_log(r'\*\*BROKEN\*\*')
-
-
-# @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1")
-# def test_dummy(node_factory, executor):
-# #     l1, l2 = node_factory.line_graph(2, wait_for_announce=True,
-# #                                      opts=[{'may_reconnect': False},
-# #                                            {'may_reconnect': False,
-# #                                             'disconnect': ['@WIRE_COMMITMENT_SIGNED']}])
-#     l1 = node_factory.get_node(allow_broken_log=True, random_hsm=True, start=False,
-#                                options={'plugin': 'tests/dbbackup/dbbackup.py',
-#                                         'db-backup-file': None})
-#
-#     # Now with an invalid path, should error and shutdown
-#     bad_path = node_factory.directory
-#     l1.daemon.opts['db-backup-file'] = bad_path
-#     with pytest.raises(ConnectionResetError):
-#         l1.start()
-#     import ipdb; ipdb.set_trace()
-#     print("\n".join(l1.daemon.logs))