From f7a1ec367e5d14ce2f95a12c4414f208e83f79c3 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sun, 28 Jun 2020 22:23:43 -0500 Subject: [PATCH 01/97] Building out smart sync --- gravity-sync.sh | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 17ef29d..bd5562c 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -367,6 +367,15 @@ function push_gs { } +function smart_gs { + md5_compare + + echo -e $primaryDBMD5 + echo -e $secondDBMD5 + echo -e $primaryCLMD5 + echo -e $secondCLMD5 +} + function restore_gs { MESSAGE="This will restore ${GRAVITY_FI} on $HOSTNAME with the previous version!" echo_warn @@ -1383,7 +1392,17 @@ function root_check { case $# in 0) - task_invalid + TASKTYPE='SYNC' + MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" + echo_good + + import_gs + validate_gs_folders + validate_ph_folders + validate_os_sshpass + + smart_gs + exit ;; 1) From 75eb9c769509c5e42b0020c3999b3902e9f73995 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 08:47:09 -0500 Subject: [PATCH 02/97] Write hashes from log out --- gravity-sync.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/gravity-sync.sh b/gravity-sync.sh index bd5562c..4388842 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -500,6 +500,15 @@ function logs_export { echo_stat echo -e $(date) "[${TASKTYPE}]" >> ${LOG_PATH}/${SYNCING_LOG} error_validate + + MESSAGE="Saving File Hashes" + echo_stat + rm -f backup/gravity-sync.md5 + echo -e ${primaryDBMD5} >> backup/gravity-sync.md5 + echo -e ${secondDBMD5} >> backup/gravity-sync.md5 + echo -e ${primaryCLMD5} >> backup/gravity-sync.md5 + echo -e ${secondCLMD5} >> backup/gravity-sync.md5 + error_validate } ### Output Sync Logs From 37fd78858db6926f6ccec465d2e84d1670b574a5 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 09:21:50 -0500 Subject: [PATCH 03/97] Import last hashes --- .gitignore | 1 + gravity-sync.sh | 28 +++++++++++++++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 410ed35..4a4bd19 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ gravity-sync.log gravity-sync.cron gravity.db.last gravity-sync.conf +gravity-sync.md5 backup/*.last backup/*.backup backup/*.push diff --git a/gravity-sync.sh b/gravity-sync.sh index 4388842..efd7d2e 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -370,10 +370,20 @@ function push_gs { function smart_gs { md5_compare + last_primaryDBMD5=$(sed "1q;d" gravity-sync.md5) + last_secondDBMD5=$(sed "2q;d" gravity-sync.md5) + last_primaryCIMD5=$(sed "3q;d" gravity-sync.md5) + last_secondCIMD5=$(sed "4q;d" gravity-sync.md5) + echo -e $primaryDBMD5 echo -e $secondDBMD5 echo -e $primaryCLMD5 echo -e $secondCLMD5 + + echo -e $last_primaryDBMD5 + echo -e $last_secondDBMD5 + echo -e $last_primaryCLMD5 + echo -e $last_secondCLMD5 } function restore_gs { @@ -496,19 +506,19 @@ function restore_gs { ## Core Logging ### Write Logs Out function logs_export { + MESSAGE="Saving File Hashes" + echo_stat + rm -f gravity-sync.md5 + echo -e ${primaryDBMD5} >> gravity-sync.md5 + echo -e ${secondDBMD5} >> gravity-sync.md5 + echo -e ${primaryCLMD5} >> gravity-sync.md5 + echo -e ${secondCLMD5} >> gravity-sync.md5 + error_validate + MESSAGE="Logging Successful ${TASKTYPE}" echo_stat echo -e $(date) "[${TASKTYPE}]" >> ${LOG_PATH}/${SYNCING_LOG} error_validate - - MESSAGE="Saving File Hashes" - echo_stat - rm -f backup/gravity-sync.md5 - echo -e ${primaryDBMD5} >> backup/gravity-sync.md5 - echo -e ${secondDBMD5} >> backup/gravity-sync.md5 - echo -e ${primaryCLMD5} >> backup/gravity-sync.md5 - echo -e ${secondCLMD5} >> backup/gravity-sync.md5 - error_validate } ### Output Sync Logs From 339d5a456f866aedce83c4530511802d1b28f397 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 09:25:14 -0500 Subject: [PATCH 04/97] Mark hashes --- gravity-sync.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index efd7d2e..5ec7278 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -375,15 +375,15 @@ function smart_gs { last_primaryCIMD5=$(sed "3q;d" gravity-sync.md5) last_secondCIMD5=$(sed "4q;d" gravity-sync.md5) - echo -e $primaryDBMD5 - echo -e $secondDBMD5 - echo -e $primaryCLMD5 - echo -e $secondCLMD5 + echo -e $primaryDBMD5 " 1" + echo -e $secondDBMD5 " 2" + echo -e $primaryCLMD5 " 3" + echo -e $secondCLMD5 " 4" - echo -e $last_primaryDBMD5 - echo -e $last_secondDBMD5 - echo -e $last_primaryCLMD5 - echo -e $last_secondCLMD5 + echo -e $last_primaryDBMD5 " 5" + echo -e $last_secondDBMD5 " 6" + echo -e $last_primaryCLMD5 " 7" + echo -e $last_secondCLMD5 " 8" } function restore_gs { From 06338d6a778ac4171242f73eafd56c56ab2f1397 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 12:57:13 -0500 Subject: [PATCH 05/97] Comparison logic --- gravity-sync.sh | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 5ec7278..1282f3f 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -372,18 +372,34 @@ function smart_gs { last_primaryDBMD5=$(sed "1q;d" gravity-sync.md5) last_secondDBMD5=$(sed "2q;d" gravity-sync.md5) - last_primaryCIMD5=$(sed "3q;d" gravity-sync.md5) - last_secondCIMD5=$(sed "4q;d" gravity-sync.md5) + last_primaryCLMD5=$(sed "3q;d" gravity-sync.md5) + last_secondCLMD5=$(sed "4q;d" gravity-sync.md5) - echo -e $primaryDBMD5 " 1" - echo -e $secondDBMD5 " 2" - echo -e $primaryCLMD5 " 3" - echo -e $secondCLMD5 " 4" + echo -e ${primaryDBMD5} " 1" + echo -e ${secondDBMD5} " 2" + echo -e ${primaryCLMD5} " 3" + echo -e ${secondCLMD5} " 4" - echo -e $last_primaryDBMD5 " 5" - echo -e $last_secondDBMD5 " 6" - echo -e $last_primaryCLMD5 " 7" - echo -e $last_secondCLMD5 " 8" + echo -e ${last_primaryDBMD5} " 5" + echo -e ${last_secondDBMD5} " 6" + echo -e ${last_primaryCLMD5} " 7" + echo -e ${last_secondCLMD5} " 8" + + if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] + then + echo "Primary DB has changed" + elif [ "${primaryDBMD5}" == "${last_primaryDBMD5}" ] + then + echo "Secondary DB has changed" + fi + + if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] + then + echo "Primary CL has changed" + elif [ "${primaryCLMD5}" == "${last_primaryCLMD5}" ] + then + echo "Secondary CL has changed" + fi } function restore_gs { From c229d5e12ebffe674381916d4f6dcd848e707b8a Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 13:04:38 -0500 Subject: [PATCH 06/97] No changes detected --- gravity-sync.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 1282f3f..dc760be 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -388,17 +388,21 @@ function smart_gs { if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] then echo "Primary DB has changed" - elif [ "${primaryDBMD5}" == "${last_primaryDBMD5}" ] + elif [ "${secondDBMD5}" != "${last_secondDBMD5}" ] then echo "Secondary DB has changed" + else + echo "No DB changes" fi if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] then echo "Primary CL has changed" - elif [ "${primaryCLMD5}" == "${last_primaryCLMD5}" ] + elif [ "${secondCLMD5}" != "${last_secondCLMD5}" ] then echo "Secondary CL has changed" + else + echo "No CL changes" fi } From afc33c54290f6bdce90460f14d97f8d68dfc5fe2 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 13:41:54 -0500 Subject: [PATCH 07/97] Break up push and pull into more subfunctions --- gravity-sync.sh | 101 ++++++++++++++++++++++++------------------------ 1 file changed, 50 insertions(+), 51 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index dc760be..53e8de8 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -135,10 +135,8 @@ function update_gs { } # Gravity Core Functions -## Pull Function -function pull_gs { - md5_compare - +## Pull Gravity +function pull_gs_grav { MESSAGE="Backing Up ${GRAVITY_FI} on $HOSTNAME" echo_stat cp ${PIHOLE_DIR}/${GRAVITY_FI} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${GRAVITY_FI}.backup >/dev/null 2>&1 @@ -158,45 +156,10 @@ function pull_gs { MESSAGE="Validating Settings of ${GRAVITY_FI}" echo_stat - - GRAVDB_OWN=$(ls -ld ${PIHOLE_DIR}/${GRAVITY_FI} | awk '{print $3 $4}') - if [ "$GRAVDB_OWN" != "piholepihole" ] - then - MESSAGE="Validating Ownership on ${GRAVITY_FI}" - echo_fail - - MESSAGE="Attempting to Compensate" - echo_warn - - MESSAGE="Setting Ownership on ${GRAVITY_FI}" - echo_stat - sudo chown pihole:pihole ${PIHOLE_DIR}/${GRAVITY_FI} >/dev/null 2>&1 - error_validate - - MESSAGE="Continuing Validation of ${GRAVITY_FI}" - echo_stat - fi - - GRAVDB_RWE=$(namei -m ${PIHOLE_DIR}/${GRAVITY_FI} | grep -v f: | grep ${GRAVITY_FI} | awk '{print $1}') - if [ "$GRAVDB_RWE" != "-rw-rw-r--" ] - then - MESSAGE="Validating Permissions on ${GRAVITY_FI}" - echo_fail - - MESSAGE="Attempting to Compensate" - echo_warn - - MESSAGE="Setting Permissions on ${GRAVITY_FI}" - echo_stat - sudo chmod 664 ${PIHOLE_DIR}/${GRAVITY_FI} >/dev/null 2>&1 - error_validate - - MESSAGE="Continuing Validation of ${GRAVITY_FI}" - echo_stat - fi - - echo_good +} +## Pull Custom +function pull_gs_cust { if [ "$SKIP_CUSTOM" != '1' ] then if [ "$REMOTE_CUSTOM_DNS" == "1" ] @@ -263,7 +226,10 @@ function pull_gs { echo_good fi fi +} +## Pull Reload +function pull_gs_reload { MESSAGE="Isolating Regeneration Pathways" echo_info sleep 1 @@ -277,17 +243,22 @@ function pull_gs { echo_stat ${PIHOLE_BIN} restartdns >/dev/null 2>&1 error_validate +} + +## Pull Function +function pull_gs { + md5_compare + + pull_gs_grav + pull_gs_cust + pull_gs_reload logs_export exit_withchange } -## Push Function -function push_gs { - md5_compare - - intent_validate - +## Push Gravity +function push_gs_grav { MESSAGE="Backing Up ${GRAVITY_FI} from ${REMOTE_HOST}" echo_stat RSYNC_REPATH="rsync" @@ -313,7 +284,10 @@ function push_gs { CMD_TIMEOUT='15' CMD_REQUESTED="sudo chown pihole:pihole ${PIHOLE_DIR}/${GRAVITY_FI}" create_sshcmd +} +## Push Custom +function push_gs_cust { if [ "$SKIP_CUSTOM" != '1' ] then if [ "$REMOTE_CUSTOM_DNS" == "1" ] @@ -345,28 +319,42 @@ function push_gs { create_sshcmd fi fi +} +## Push Reload +function push_gs_reload { MESSAGE="Inverting Tachyon Pulses" echo_info sleep 1 - MESSAGE="Updating FTLDNS Configuration" + MESSAGE="Updating Remote FTLDNS Configuration" echo_stat CMD_TIMEOUT='15' CMD_REQUESTED="${PIHOLE_BIN} restartdns reloadlists" create_sshcmd - MESSAGE="Reloading FTLDNS Services" + MESSAGE="Reloading Remote FTLDNS Services" echo_stat CMD_TIMEOUT='15' CMD_REQUESTED="${PIHOLE_BIN} restartdns" create_sshcmd +} + +## Push Function +function push_gs { + md5_compare + + intent_validate + + push_gs_grav + push_gs_cust + push_gs_reload logs_export exit_withchange - } +## Smart Sync Function function smart_gs { md5_compare @@ -385,12 +373,19 @@ function smart_gs { echo -e ${last_primaryCLMD5} " 7" echo -e ${last_secondCLMD5} " 8" + DB_SMARTPULL="0" + DB_SMARTPUSH="0" + CL_SMARTPULL="0" + CL_SMARTPUSH="0" + if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] then echo "Primary DB has changed" + DB_SMARTPULL="1" elif [ "${secondDBMD5}" != "${last_secondDBMD5}" ] then echo "Secondary DB has changed" + DB_SMARTPUSH="1" else echo "No DB changes" fi @@ -398,12 +393,16 @@ function smart_gs { if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] then echo "Primary CL has changed" + CL_SMARTPULL="1" elif [ "${secondCLMD5}" != "${last_secondCLMD5}" ] then echo "Secondary CL has changed" + CL_SMARTPUSH="1" else echo "No CL changes" fi + + } function restore_gs { From 570d8fc86d7fbf32ff5e9fcd2f8ebe59c86ad5ab Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 13:48:54 -0500 Subject: [PATCH 08/97] Oops --- gravity-sync.sh | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/gravity-sync.sh b/gravity-sync.sh index 53e8de8..494f479 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -156,6 +156,44 @@ function pull_gs_grav { MESSAGE="Validating Settings of ${GRAVITY_FI}" echo_stat + + GRAVDB_OWN=$(ls -ld ${PIHOLE_DIR}/${GRAVITY_FI} | awk '{print $3 $4}') + if [ "$GRAVDB_OWN" != "piholepihole" ] + then + MESSAGE="Validating Ownership on ${GRAVITY_FI}" + echo_fail + + MESSAGE="Attempting to Compensate" + echo_warn + + MESSAGE="Setting Ownership on ${GRAVITY_FI}" + echo_stat + sudo chown pihole:pihole ${PIHOLE_DIR}/${GRAVITY_FI} >/dev/null 2>&1 + error_validate + + MESSAGE="Continuing Validation of ${GRAVITY_FI}" + echo_stat + fi + + GRAVDB_RWE=$(namei -m ${PIHOLE_DIR}/${GRAVITY_FI} | grep -v f: | grep ${GRAVITY_FI} | awk '{print $1}') + if [ "$GRAVDB_RWE" != "-rw-rw-r--" ] + then + MESSAGE="Validating Permissions on ${GRAVITY_FI}" + echo_fail + + MESSAGE="Attempting to Compensate" + echo_warn + + MESSAGE="Setting Permissions on ${GRAVITY_FI}" + echo_stat + sudo chmod 664 ${PIHOLE_DIR}/${GRAVITY_FI} >/dev/null 2>&1 + error_validate + + MESSAGE="Continuing Validation of ${GRAVITY_FI}" + echo_stat + fi + + echo_good } ## Pull Custom From 9abf85c3f7af1495e973cc525447f2046a82995a Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 14:03:47 -0500 Subject: [PATCH 09/97] HISTORY_MD5 --- ADVANCED.md | 4 ++-- VERSION | 2 +- gravity-sync.sh | 11 ++++++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index 62dd9cd..e3f3ea8 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -14,8 +14,8 @@ Download the latest release from [GitHub](https://github.com/vmstan/gravity-sync ```bash cd ~ -wget https://github.com/vmstan/gravity-sync/archive/v1.8.3.zip -unzip v1.8.3.zip -d gravity-sync +wget https://github.com/vmstan/gravity-sync/archive/v2.0.0.zip +unzip v2.0.0.zip -d gravity-sync cd gravity-sync ``` diff --git a/VERSION b/VERSION index fe4e75f..359a5b9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.8.3 \ No newline at end of file +2.0.0 \ No newline at end of file diff --git a/gravity-sync.sh b/gravity-sync.sh index 494f479..bece31b 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -3,7 +3,7 @@ SCRIPT_START=$SECONDS # GRAVITY SYNC BY VMSTAN ##################### PROGRAM='Gravity Sync' -VERSION='1.8.3' +VERSION='2.0.0' # Execute from the home folder of the user who owns it (ex: 'cd ~/gravity-sync') # For documentation or downloading updates visit https://github.com/vmstan/gravity-sync @@ -24,6 +24,7 @@ BACKUP_FOLD='backup' # must exist as subdirectory in LOCAL_FOLDR LOG_PATH="$HOME/${LOCAL_FOLDR}" # replace in gravity-sync.conf to overwrite SYNCING_LOG='gravity-sync.log' # replace in gravity-sync.conf to overwrite CRONJOB_LOG='gravity-sync.cron' # replace in gravity-sync.conf to overwrite +HISTORY_MD5='gravity-sync.md5' # replace in gravity-sync.conf to overwrite # Interaction Customization VERIFY_PASS='0' # replace in gravity-sync.conf to overwrite @@ -396,10 +397,10 @@ function push_gs { function smart_gs { md5_compare - last_primaryDBMD5=$(sed "1q;d" gravity-sync.md5) - last_secondDBMD5=$(sed "2q;d" gravity-sync.md5) - last_primaryCLMD5=$(sed "3q;d" gravity-sync.md5) - last_secondCLMD5=$(sed "4q;d" gravity-sync.md5) + last_primaryDBMD5=$(sed "1q;d" ${HISTORY_MD5}) + last_secondDBMD5=$(sed "2q;d" ${HISTORY_MD5}) + last_primaryCLMD5=$(sed "3q;d" ${HISTORY_MD5}) + last_secondCLMD5=$(sed "4q;d" ${HISTORY_MD5}) echo -e ${primaryDBMD5} " 1" echo -e ${secondDBMD5} " 2" From 5cee345ee938ca51255d1eadbae41bbe711f045a Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 14:42:53 -0500 Subject: [PATCH 10/97] Automate update --- CHANGELOG.md | 12 ++++++++++++ gravity-sync.sh | 31 ++++++------------------------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4db75da..75ad9f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # The Changelog +## 2.0 +### The Smart Release + +**Features** +For 2.0, the `push` and `pull` functions have been broken up for each component part (`gravity.db` and `custom.list`), and Gravity Sync will now detect not only if each component has changed since the last sync but also what direction they need to go. + +Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull and a push of each part and then restart the correct components on each server. + +This allows you to be more flexible in where you make your configuration changes to block/allow lists, but it's still suggested to identify a primary server to make these changes on. In the event there are configuration changes to the same element (`custom.list` changes at both sides) then the primary will still be the authoritative source and overwrite the secondary copy. + +Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. + ## 1.8 ### The Logical Release diff --git a/gravity-sync.sh b/gravity-sync.sh index bece31b..73fbfed 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -402,41 +402,22 @@ function smart_gs { last_primaryCLMD5=$(sed "3q;d" ${HISTORY_MD5}) last_secondCLMD5=$(sed "4q;d" ${HISTORY_MD5}) - echo -e ${primaryDBMD5} " 1" - echo -e ${secondDBMD5} " 2" - echo -e ${primaryCLMD5} " 3" - echo -e ${secondCLMD5} " 4" - - echo -e ${last_primaryDBMD5} " 5" - echo -e ${last_secondDBMD5} " 6" - echo -e ${last_primaryCLMD5} " 7" - echo -e ${last_secondCLMD5} " 8" - - DB_SMARTPULL="0" - DB_SMARTPUSH="0" - CL_SMARTPULL="0" - CL_SMARTPUSH="0" - if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] then - echo "Primary DB has changed" - DB_SMARTPULL="1" + pull_gs_grav elif [ "${secondDBMD5}" != "${last_secondDBMD5}" ] then - echo "Secondary DB has changed" - DB_SMARTPUSH="1" + push_gs_grav else echo "No DB changes" fi if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] then - echo "Primary CL has changed" - CL_SMARTPULL="1" + pull_gs_cust elif [ "${secondCLMD5}" != "${last_secondCLMD5}" ] then - echo "Secondary CL has changed" - CL_SMARTPUSH="1" + push_gs_cust else echo "No CL changes" fi @@ -1293,7 +1274,7 @@ function task_automate { MESSAGE="Saving New Automation" echo_stat - (crontab -l 2>/dev/null; echo "*/${INPUT_AUTO_FREQ} * * * * ${BASH_PATH} $HOME/${LOCAL_FOLDR}/${GS_FILENAME} pull > ${LOG_PATH}/${CRONJOB_LOG}") | crontab - + (crontab -l 2>/dev/null; echo "*/${INPUT_AUTO_FREQ} * * * * ${BASH_PATH} $HOME/${LOCAL_FOLDR}/${GS_FILENAME} > ${LOG_PATH}/${CRONJOB_LOG}") | crontab - error_validate fi exit_withchange @@ -1305,7 +1286,7 @@ function clear_cron { echo_stat crontab -l > cronjob-old.tmp - sed '/.sh pull/d' cronjob-old.tmp > cronjob-new.tmp + sed '/${GS_FILENAME}/d' cronjob-old.tmp > cronjob-new.tmp crontab cronjob-new.tmp 2>/dev/null error_validate rm cronjob-old.tmp From adb63debc13f0cd8d1d857c59699b0d924323291 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 15:49:16 -0500 Subject: [PATCH 11/97] Full sync logic --- CHANGELOG.md | 6 +++--- gravity-sync.sh | 40 +++++++++++++++++++++++++++++++++++----- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75ad9f9..c17a8e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,11 @@ **Features** For 2.0, the `push` and `pull` functions have been broken up for each component part (`gravity.db` and `custom.list`), and Gravity Sync will now detect not only if each component has changed since the last sync but also what direction they need to go. -Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull and a push of each part and then restart the correct components on each server. +Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull and then push for each part, and then restart the correct components on each server. It will also only perform a sync of each component if there are changes within each type to replicate. -This allows you to be more flexible in where you make your configuration changes to block/allow lists, but it's still suggested to identify a primary server to make these changes on. In the event there are configuration changes to the same element (`custom.list` changes at both sides) then the primary will still be the authoritative source and overwrite the secondary copy. +This allows you to be more flexible in where you make your configuration changes to block/allow lists and local DNS settings being made on either the primary or secondary, but it's best practice to continue making changes on the primary server where possible. In the event there are configuration changes to the same element (`custom.list` changes at both sides) then the primary will still be the authoritative source and overwrite the secondary copy. -Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. +Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. You can also continue to use `pull` in your automation if you choose. New installs will use the smart Sync function by default. Running `./gravity-sync.sh` without any arguments will now run the smart Sync function. ## 1.8 ### The Logical Release diff --git a/gravity-sync.sh b/gravity-sync.sh index 73fbfed..be26c32 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -405,24 +405,39 @@ function smart_gs { if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] then pull_gs_grav + PULLRESTART="1" elif [ "${secondDBMD5}" != "${last_secondDBMD5}" ] then push_gs_grav + PUSHRESTART="1" else - echo "No DB changes" + # echo "No DB changes" fi if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] then pull_gs_cust + PULLRESTART="1" elif [ "${secondCLMD5}" != "${last_secondCLMD5}" ] then push_gs_cust + PUSHRESTART="1" else - echo "No CL changes" + # echo "No CL changes" fi + if [ "$PULLRESTART" == "1" ] + then + pull_gs_reload + fi + if [ "$PUSHRESTART" == "1" ] + then + push_gs_reload + fi + + logs_export + exit_withchange } function restore_gs { @@ -1178,9 +1193,10 @@ function list_gs_arguments { echo -e " ${YELLOW}version${NC} Display installed version of ${PROGRAM}" echo -e "" echo -e "Replication Options:" - echo -e " ${YELLOW}pull${NC} Sync remote ${GRAVITY_FI} to this server" - echo -e " ${YELLOW}push${NC} Force changes made on this server back" - echo -e " ${YELLOW}restore${NC} Restore ${GRAVITY_FI} on this server" + echo -e " ${YELLOW}sync${NC} Detect changes on each side and bring them together" + echo -e " ${YELLOW}pull${NC} Force remote configuration changes to this server" + echo -e " ${YELLOW}push${NC} Force local configuration made on this server back" + echo -e " ${YELLOW}restore${NC} Restore the ${GRAVITY_FI} on this server" echo -e " ${YELLOW}compare${NC} Just check for differences" echo -e "" echo -e "Debug Options:" @@ -1465,6 +1481,20 @@ case $# in 1) case $1 in + sync) + TASKTYPE='SYNC' + MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" + echo_good + + import_gs + validate_gs_folders + validate_ph_folders + validate_os_sshpass + + smart_gs + exit + ;; + pull) TASKTYPE='PULL' MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" From fa270aff677033b9c749dd36fde53b2fee76cd60 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 15:50:58 -0500 Subject: [PATCH 12/97] Validate changes --- gravity-sync.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gravity-sync.sh b/gravity-sync.sh index be26c32..31e768d 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -436,6 +436,8 @@ function smart_gs { push_gs_reload fi + md5_compare + logs_export exit_withchange } From 0e72da570f4da25717d393b137c3cbee821ade08 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 16:06:28 -0500 Subject: [PATCH 13/97] Recompare --- gravity-sync.sh | 84 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 4 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 31e768d..b0bafd6 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -410,8 +410,6 @@ function smart_gs { then push_gs_grav PUSHRESTART="1" - else - # echo "No DB changes" fi if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] @@ -422,8 +420,6 @@ function smart_gs { then push_gs_cust PUSHRESTART="1" - else - # echo "No CL changes" fi if [ "$PULLRESTART" == "1" ] @@ -1019,6 +1015,86 @@ function md5_compare { fi } +function md5_recheck { + MESSAGE="Performing Replicator Diagnostics" + echo_info + + HASHMARK='0' + + MESSAGE="Reanalyzing ${GRAVITY_FI} on ${REMOTE_HOST}" + echo_stat + primaryDBMD5=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "md5sum ${PIHOLE_DIR}/${GRAVITY_FI}" | sed 's/\s.*$//') + error_validate + + MESSAGE="Reanalyzing ${GRAVITY_FI} on $HOSTNAME" + echo_stat + secondDBMD5=$(md5sum ${PIHOLE_DIR}/${GRAVITY_FI} | sed 's/\s.*$//') + error_validate + + if [ "$primaryDBMD5" == "$secondDBMD5" ] + then + HASHMARK=$((HASHMARK+0)) + else + MESSAGE="Differenced ${GRAVITY_FI} Detected" + echo_warn + HASHMARK=$((HASHMARK+1)) + fi + + if [ "$SKIP_CUSTOM" != '1' ] + then + if [ -f ${PIHOLE_DIR}/${CUSTOM_DNS} ] + then + if ${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} test -e ${PIHOLE_DIR}/${CUSTOM_DNS} + then + REMOTE_CUSTOM_DNS="1" + MESSAGE="Reanalyzing ${CUSTOM_DNS} on ${REMOTE_HOST}" + echo_stat + + primaryCLMD5=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "md5sum ${PIHOLE_DIR}/${CUSTOM_DNS} | sed 's/\s.*$//'") + error_validate + + MESSAGE="Reanalyzing ${CUSTOM_DNS} on $HOSTNAME" + echo_stat + secondCLMD5=$(md5sum ${PIHOLE_DIR}/${CUSTOM_DNS} | sed 's/\s.*$//') + error_validate + + if [ "$primaryCLMD5" == "$secondCLMD5" ] + then + # MESSAGE="${CUSTOM_DNS} Identical" + # echo_info + HASHMARK=$((HASHMARK+0)) + else + MESSAGE="Differenced ${CUSTOM_DNS} Detected" + echo_warn + HASHMARK=$((HASHMARK+1)) + fi + else + MESSAGE="No ${CUSTOM_DNS} Detected on ${REMOTE_HOST}" + echo_info + fi + else + if ${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} test -e ${PIHOLE_DIR}/${CUSTOM_DNS} + then + REMOTE_CUSTOM_DNS="1" + MESSAGE="${REMOTE_HOST} has ${CUSTOM_DNS}" + HASHMARK=$((HASHMARK+1)) + echo_info + fi + MESSAGE="No ${CUSTOM_DNS} Detected on $HOSTNAME" + echo_info + fi + fi + + if [ "$HASHMARK" != "0" ] + then + MESSAGE="Replication Checks Failed" + echo_warn + else + MESSAGE="Replication Was Successful" + echo_info + fi +} + ## Validate Intent function intent_validate { if [ "$VERIFY_PASS" == "0" ] From 3114f72e4c511ec1ab70e2d67a8504bde2f37c47 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 16:08:06 -0500 Subject: [PATCH 14/97] recheck --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index b0bafd6..c4ce2e6 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -432,7 +432,7 @@ function smart_gs { push_gs_reload fi - md5_compare + md5_recheckr logs_export exit_withchange From c0b594b2bc323347043e40dc2eaef24ee83a12ae Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 16:08:27 -0500 Subject: [PATCH 15/97] No r --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index c4ce2e6..6ee523c 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -432,7 +432,7 @@ function smart_gs { push_gs_reload fi - md5_recheckr + md5_recheck logs_export exit_withchange From 1ef5fec67653ee53d140bb37729c9b64e4728e72 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 14:26:45 -0500 Subject: [PATCH 16/97] building both sides check --- gravity-sync.sh | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 6ee523c..d72e1b1 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -402,14 +402,40 @@ function smart_gs { last_primaryCLMD5=$(sed "3q;d" ${HISTORY_MD5}) last_secondCLMD5=$(sed "4q;d" ${HISTORY_MD5}) + PRIDBCHANGE="0" + SECDBCHANGE="0" + PRICLCHANGE="0" + SECICLCHANGE="0" + if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] then - pull_gs_grav - PULLRESTART="1" - elif [ "${secondDBMD5}" != "${last_secondDBMD5}" ] + PRIDBCHANGE="1" + # pull_gs_grav + # PULLRESTART="1" + fi + + if [ "${secondDBMD5}" != "${last_secondDBMD5}" ] then - push_gs_grav - PUSHRESTART="1" + SECDBCHANGE="1" + # push_gs_grav + # PUSHRESTART="1" + fi + + if [ "${PRIDBCHANGE}" == "${SECDBCHANGE}" ] + then + if [ "${PRIDBCHANGE}" != "0" ] + then + echo "Both sides have changed" + fi + else + then + if [ "${PRIDBCHANGE}" != "0" ] + then + echo "Primary has changed" + elif [ "${SECDBCHANGE}" != "0" ] + then + echo "Secondary has changed" + fi fi if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] From c604697d5455eaaa01435d546bb8768d85fd5bdc Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 14:29:38 -0500 Subject: [PATCH 17/97] logic insert --- gravity-sync.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index d72e1b1..a48a043 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -426,15 +426,17 @@ function smart_gs { if [ "${PRIDBCHANGE}" != "0" ] then echo "Both sides have changed" + exit fi else - then if [ "${PRIDBCHANGE}" != "0" ] then - echo "Primary has changed" + pull_gs_grav + PULLRESTART="1" elif [ "${SECDBCHANGE}" != "0" ] then - echo "Secondary has changed" + push_gs_grav + PUSHRESTART="1" fi fi From d3966191f01df8835ea8846eb0b39dc77c1b1b16 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 14:55:52 -0500 Subject: [PATCH 18/97] checks dates --- gravity-sync.sh | 73 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index a48a043..089e848 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -410,23 +410,37 @@ function smart_gs { if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] then PRIDBCHANGE="1" - # pull_gs_grav - # PULLRESTART="1" fi if [ "${secondDBMD5}" != "${last_secondDBMD5}" ] then SECDBCHANGE="1" - # push_gs_grav - # PUSHRESTART="1" fi if [ "${PRIDBCHANGE}" == "${SECDBCHANGE}" ] then if [ "${PRIDBCHANGE}" != "0" ] then - echo "Both sides have changed" - exit + MESSAGE="Both Sides Have Changed" + echo_warn + + PRIDBDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${GRAVITY_FI}") + SECDBDATE=$(stat -c %Y ${PIHOLE_DIR}/${GRAVITY_FI}) + + if [ "${PRIDBDATE}" -gt "$SECDBDATE" ] + then + MESSAGE="Primary was Last Changed" + echo_info + + pull_gs_grav + PULLRESTART="1" + else + MESSAGE="Secondary was Last Changed" + echo_info + + push_gs_grav + PUSHRESTART="1" + fi fi else if [ "${PRIDBCHANGE}" != "0" ] @@ -442,12 +456,49 @@ function smart_gs { if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] then - pull_gs_cust - PULLRESTART="1" - elif [ "${secondCLMD5}" != "${last_secondCLMD5}" ] + PRICLCHANGE="1" + fi + + if [ "${secondCLMD5}" != "${last_secondCLMD5}" ] then - push_gs_cust - PUSHRESTART="1" + SECCLCHANGE="1" + fi + + if [ "${PRICLCHANGE}" == "${SECCLCHANGE}" ] + then + if [ "${PRICLCHANGE}" != "0" ] + then + MESSAGE="Both Sides Have Changed" + echo_warn + + PRICLDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${CUSTOM_DNS}") + SECCLDATE=$(stat -c %Y ${PIHOLE_DIR}/${CUSTOM_DNS}) + + if [ "${PRICLDATE}" -gt "$SECCLDATE" ] + then + MESSAGE="Primary was Last Changed" + echo_info + + pull_gs_cust + PULLRESTART="1" + else + MESSAGE="Secondary was Last Changed" + echo_info + + push_gs_cust + PUSHRESTART="1" + fi + fi + else + if [ "${PRICLCHANGE}" != "0" ] + then + pull_gs_cust + PULLRESTART="1" + elif [ "${SECCLCHANGE}" != "0" ] + then + push_gs_cust + PUSHRESTART="1" + fi fi if [ "$PULLRESTART" == "1" ] From c7a6ba4f692bacc7d60d5667fa64cbaee708f0a7 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 15:00:08 -0500 Subject: [PATCH 19/97] extra letter --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 089e848..a64bd6c 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -405,7 +405,7 @@ function smart_gs { PRIDBCHANGE="0" SECDBCHANGE="0" PRICLCHANGE="0" - SECICLCHANGE="0" + SECCLCHANGE="0" if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] then From d5ededce123c916ff07f94e3483fd4f7a3e957e9 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 15:15:42 -0500 Subject: [PATCH 20/97] checks for history_md5 existing --- ADVANCED.md | 2 +- CHANGELOG.md | 2 +- gravity-sync.sh | 56 ++++++++++++++++++++++++++++++++++--------------- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index e3f3ea8..c9c408a 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -173,5 +173,5 @@ If you prefer to still use cron but modify your settings by hand, using the entr ```bash crontab -e -*/30 * * * * /bin/bash /home/USER/gravity-sync/gravity-sync.sh pull > /home/USER/gravity-sync/gravity-sync.cron +*/30 * * * * /bin/bash /home/USER/gravity-sync/gravity-sync.sh > /home/USER/gravity-sync/gravity-sync.cron ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index c17a8e9..22197e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ For 2.0, the `push` and `pull` functions have been broken up for each component Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull and then push for each part, and then restart the correct components on each server. It will also only perform a sync of each component if there are changes within each type to replicate. -This allows you to be more flexible in where you make your configuration changes to block/allow lists and local DNS settings being made on either the primary or secondary, but it's best practice to continue making changes on the primary server where possible. In the event there are configuration changes to the same element (`custom.list` changes at both sides) then the primary will still be the authoritative source and overwrite the secondary copy. +This allows you to be more flexible in where you make your configuration changes to block/allow lists and local DNS settings being made on either the primary or secondary, but it's best practice to continue making changes on the primary server where possible. In the event there are configuration changes to the same element (`custom.list` changes at both sides) then Gravity Sync will attempt to determine based on timestamps on what side the last changed happened, in which case the latest changes will be considered authoritative and overwrite the other side. Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. You can also continue to use `pull` in your automation if you choose. New installs will use the smart Sync function by default. Running `./gravity-sync.sh` without any arguments will now run the smart Sync function. diff --git a/gravity-sync.sh b/gravity-sync.sh index a64bd6c..9f6096c 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -397,10 +397,18 @@ function push_gs { function smart_gs { md5_compare - last_primaryDBMD5=$(sed "1q;d" ${HISTORY_MD5}) - last_secondDBMD5=$(sed "2q;d" ${HISTORY_MD5}) - last_primaryCLMD5=$(sed "3q;d" ${HISTORY_MD5}) - last_secondCLMD5=$(sed "4q;d" ${HISTORY_MD5}) + if [ -f "${LOG_PATH}/${HISTORY_MD5}" ] + then + last_primaryDBMD5=$(sed "1q;d" ${LOG_PATH}/${HISTORY_MD5}) + last_secondDBMD5=$(sed "2q;d" ${LOG_PATH}/${HISTORY_MD5}) + last_primaryCLMD5=$(sed "3q;d" ${LOG_PATH}/${HISTORY_MD5}) + last_secondCLMD5=$(sed "4q;d" ${LOG_PATH}/{HISTORY_MD5}) + else + last_primaryDBMD5="0" + last_secondDBMD5="0" + last_primaryCLMD5="0" + last_secondCLMD5="0" + fi PRIDBCHANGE="0" SECDBCHANGE="0" @@ -421,7 +429,7 @@ function smart_gs { then if [ "${PRIDBCHANGE}" != "0" ] then - MESSAGE="Both Sides Have Changed" + MESSAGE="Both Sides Gravity Database Changed" echo_warn PRIDBDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${GRAVITY_FI}") @@ -429,13 +437,13 @@ function smart_gs { if [ "${PRIDBDATE}" -gt "$SECDBDATE" ] then - MESSAGE="Primary was Last Changed" + MESSAGE="Primary Gravity Database Last Changed" echo_info pull_gs_grav PULLRESTART="1" else - MESSAGE="Secondary was Last Changed" + MESSAGE="Secondary Gravity Database Last Changed" echo_info push_gs_grav @@ -468,7 +476,7 @@ function smart_gs { then if [ "${PRICLCHANGE}" != "0" ] then - MESSAGE="Both Sides Have Changed" + MESSAGE="Both Sides Custom DNS Changed" echo_warn PRICLDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${CUSTOM_DNS}") @@ -476,13 +484,13 @@ function smart_gs { if [ "${PRICLDATE}" -gt "$SECCLDATE" ] then - MESSAGE="Primary was Last Changed" + MESSAGE="Primary Custom DNS Last Changed" echo_info pull_gs_cust PULLRESTART="1" else - MESSAGE="Secondary was Last Changed" + MESSAGE="Secondary Custom DNS Last Changed" echo_info push_gs_cust @@ -639,11 +647,11 @@ function restore_gs { function logs_export { MESSAGE="Saving File Hashes" echo_stat - rm -f gravity-sync.md5 - echo -e ${primaryDBMD5} >> gravity-sync.md5 - echo -e ${secondDBMD5} >> gravity-sync.md5 - echo -e ${primaryCLMD5} >> gravity-sync.md5 - echo -e ${secondCLMD5} >> gravity-sync.md5 + rm -f ${LOG_PATH}/${HISTORY_MD5} + echo -e ${primaryDBMD5} >> ${LOG_PATH}/${HISTORY_MD5} + echo -e ${secondDBMD5} >> ${LOG_PATH}/${HISTORY_MD5} + echo -e ${primaryCLMD5} >> ${LOG_PATH}/${HISTORY_MD5} + echo -e ${secondCLMD5} >> ${LOG_PATH}/${HISTORY_MD5} error_validate MESSAGE="Logging Successful ${TASKTYPE}" @@ -1623,7 +1631,7 @@ function root_check { case $# in 0) - TASKTYPE='SYNC' + TASKTYPE='SMART' MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good @@ -1639,7 +1647,21 @@ case $# in 1) case $1 in sync) - TASKTYPE='SYNC' + TASKTYPE='SMART' + MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" + echo_good + + import_gs + validate_gs_folders + validate_ph_folders + validate_os_sshpass + + smart_gs + exit + ;; + + smart) + TASKTYPE='SMART' MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good From 95dd3d56347d304cd5ee38bcc7035e06fe075510 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 15:19:59 -0500 Subject: [PATCH 21/97] missing $ --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 9f6096c..a968bb6 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -402,7 +402,7 @@ function smart_gs { last_primaryDBMD5=$(sed "1q;d" ${LOG_PATH}/${HISTORY_MD5}) last_secondDBMD5=$(sed "2q;d" ${LOG_PATH}/${HISTORY_MD5}) last_primaryCLMD5=$(sed "3q;d" ${LOG_PATH}/${HISTORY_MD5}) - last_secondCLMD5=$(sed "4q;d" ${LOG_PATH}/{HISTORY_MD5}) + last_secondCLMD5=$(sed "4q;d" ${LOG_PATH}/{$HISTORY_MD5}) else last_primaryDBMD5="0" last_secondDBMD5="0" From 6fe6a467948360e3a811938e43d112d61fc403fc Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 15:21:58 -0500 Subject: [PATCH 22/97] BAH --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index a968bb6..c180e9b 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -402,7 +402,7 @@ function smart_gs { last_primaryDBMD5=$(sed "1q;d" ${LOG_PATH}/${HISTORY_MD5}) last_secondDBMD5=$(sed "2q;d" ${LOG_PATH}/${HISTORY_MD5}) last_primaryCLMD5=$(sed "3q;d" ${LOG_PATH}/${HISTORY_MD5}) - last_secondCLMD5=$(sed "4q;d" ${LOG_PATH}/{$HISTORY_MD5}) + last_secondCLMD5=$(sed "4q;d" ${LOG_PATH}/${HISTORY_MD5}) else last_primaryDBMD5="0" last_secondDBMD5="0" From 583c7c84005e5f9b3061c70f2ce08f3f92d910d3 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 15:37:20 -0500 Subject: [PATCH 23/97] 2.0.0 --- ADVANCED.md | 7 ++++++- CHANGELOG.md | 10 ++++++---- gravity-sync.conf.example | 1 + gravity-sync.sh | 14 ++++++++------ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index c9c408a..92a825d 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -97,7 +97,7 @@ Gravity Sync will place logs in the same folder as the script (identified as .cr Default setting in Gravity Sync is `$HOME/${LOCAL_FOLDR}` #### `SYNCING_LOG=''` -Gravity Sync will write a timestamp for any completed pull, push or restore job to this file. If you want to change the name of this file, you will also need to adjust the LOG_PATH variable above, otherwise your file will be remove during an `update` operations. +Gravity Sync will write a timestamp for any completed sync, pull, push or restore job to this file. If you want to change the name of this file, you will also need to adjust the LOG_PATH variable above, otherwise your file will be remove during an `update` operations. Default setting in Gravity Sync is `gravity-sync.log` @@ -108,6 +108,11 @@ This will have an impact to both the `./gravity-sync.sh automate` function and t Default setting in Gravity Sync is `gravity-sync.cron` +#### `HISTORY_MD5=''` +Gravity Sync will log the file hashes of the previous `smart` task to this file. If you want to change the name of this file, you will also need to adjust the LOG_PATH variable above, otherwise your file will be removed during an `update` operations. + +Default setting in Gravity Sync is `gravity-sync.md5` + #### `VERIFY_PASS=''` Gravity Sync will prompt to verify user interactivity during push, restore, or config operations (that overwrite an existing configuration) with the intention that it prevents someone from accidentally automating in the wrong direction or overwriting data intentionally. If you'd like to automate a push function, or just don't like to be asked twice to do something distructive, then you can opt-out. diff --git a/CHANGELOG.md b/CHANGELOG.md index 22197e2..c57efdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,15 @@ ### The Smart Release **Features** -For 2.0, the `push` and `pull` functions have been broken up for each component part (`gravity.db` and `custom.list`), and Gravity Sync will now detect not only if each component has changed since the last sync but also what direction they need to go. +In this release, Gravity Sync will now detect not only if each component (`gravity.db` and `custom.list`) has changed since the last sync, but also what direction they need to go. It will then initate a `push` and/or `pull` specific to each piece. -Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull and then push for each part, and then restart the correct components on each server. It will also only perform a sync of each component if there are changes within each type to replicate. +Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull of the `gravity.db` then push `custom.list` and finally restart the correct components on each server. It will also now only perform a sync of each component if there are changes within each type to replicate. So if you only make a small change to your Local DNS settings, it doesn't kickoff the larger `gravity.db` replication. -This allows you to be more flexible in where you make your configuration changes to block/allow lists and local DNS settings being made on either the primary or secondary, but it's best practice to continue making changes on the primary server where possible. In the event there are configuration changes to the same element (`custom.list` changes at both sides) then Gravity Sync will attempt to determine based on timestamps on what side the last changed happened, in which case the latest changes will be considered authoritative and overwrite the other side. +The default command for Gravity Sync is now just `./gravity-sync.sh` -- but you can also run `./gravity-sync.sh smart` if you feel like it, and it'll do the same thing. -Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. You can also continue to use `pull` in your automation if you choose. New installs will use the smart Sync function by default. Running `./gravity-sync.sh` without any arguments will now run the smart Sync function. +This allows you to be more flexible in where you make your configuration changes to block/allow lists and local DNS settings being made on either the primary or secondary, but it's best practice to continue making changes on one side where possible. In the event there are configuration changes to the same element (example, `custom.list` changes at both sides) then Gravity Sync will attempt to determine based on timestamps on what side the last changed happened, in which case the latest changes will be considered authoritative and overwrite the other side. Gravity Sync does not merge the contents of the files when changes happen, it simply overwrites the entire content. + +New installs will use the `smart` function by default. Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. This is not required. The previous `./gravity-sync.sh pull` and `./gravity-sync.sh push` commands continue to function as they did previously, with no intention to break this functionality. ## 1.8 ### The Logical Release diff --git a/gravity-sync.conf.example b/gravity-sync.conf.example index 8388f89..c125d38 100644 --- a/gravity-sync.conf.example +++ b/gravity-sync.conf.example @@ -27,6 +27,7 @@ REMOTE_PASS='' # LOG_PATH='' # SYNCING_LOG='' # CRONJOB_LOG='' +# HISTORY_MD5='' # VERIFY_PASS='' # SKIP_CUSTOM='' diff --git a/gravity-sync.sh b/gravity-sync.sh index c180e9b..689171a 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -429,7 +429,7 @@ function smart_gs { then if [ "${PRIDBCHANGE}" != "0" ] then - MESSAGE="Both Sides Gravity Database Changed" + MESSAGE="Both ${GRAVITY_FI} Changed" echo_warn PRIDBDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${GRAVITY_FI}") @@ -437,13 +437,13 @@ function smart_gs { if [ "${PRIDBDATE}" -gt "$SECDBDATE" ] then - MESSAGE="Primary Gravity Database Last Changed" + MESSAGE="Primary ${GRAVITY_FI} Last Changed" echo_info pull_gs_grav PULLRESTART="1" else - MESSAGE="Secondary Gravity Database Last Changed" + MESSAGE="Secondary ${GRAVITY_FI} Last Changed" echo_info push_gs_grav @@ -476,7 +476,7 @@ function smart_gs { then if [ "${PRICLCHANGE}" != "0" ] then - MESSAGE="Both Sides Custom DNS Changed" + MESSAGE="Both ${CUSTOM_DNS} Changed" echo_warn PRICLDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${CUSTOM_DNS}") @@ -484,13 +484,13 @@ function smart_gs { if [ "${PRICLDATE}" -gt "$SECCLDATE" ] then - MESSAGE="Primary Custom DNS Last Changed" + MESSAGE="Primary ${CUSTOM_DNS} Last Changed" echo_info pull_gs_cust PULLRESTART="1" else - MESSAGE="Secondary Custom DNS Last Changed" + MESSAGE="Secondary ${CUSTOM_DNS} Last Changed" echo_info push_gs_cust @@ -668,6 +668,8 @@ function logs_gs { echo_info echo -e "========================================================" + echo -e "Recent Complete ${YELLOW}SMART${NC} Executions" + tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep SMART echo -e "Recent Complete ${YELLOW}PULL${NC} Executions" tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep PULL echo -e "Recent Complete ${YELLOW}PUSH${NC} Executions" From fb92a236e535517eaf89d72d105eeb90e4ef6830 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 15:50:38 -0500 Subject: [PATCH 24/97] 2.0.0 README --- README.md | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index df7245a..6b4afc6 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,12 @@ Gravity Sync **requires** Pi-hole 5.0 or higher be already installed on your ser - Gravity Sync leverages SSH and RSYNC to do the leavy lifting between your Pi-hole nodes. OpenSSH is reccomended but if you're using a ultra-lightweight Pi distrbution (such as DietPi) that uses Dropbear by default, it should work as well. Other SSH client/server combonations are not supported at this time. ### Pi-hole Architecture -You will need to designate one Pi-Hole as primary and at least one as secondary. The primary Pi-hole is where you'll make all your configuration changes through the Web UI, doing things such as; manual whitelisting, adding blocklists, device/group management, configuring custom/local network DNS, and other changing other list settings. The secondary Pi-hole(s) are where you will install and configure Gravity Sync. - -Gravity Sync will pull the `gravity.db` and `custom.list` files of the primary Pi-hole to the secondary. It will also bring over the downloaded blocklist files after a `pihole -g` command is run to update blocklists on the primary, so you do not need to reach out to all your blocklist hosts for updates after syncing on the secondary. +You will want to designate one Pi-Hole as primary and at least one as secondary. The primary Pi-hole is where you'll make most of your configuration changes through the Web UI, doing things such as; manual allow-listing, adding blocklists, device/group management, configuring custom/local network DNS, and other changing other list settings. The secondary Pi-hole(s) are where you will install and configure Gravity Sync. The designation of primary and secondary is purely at your discretion. The doesn't matter if you're using an HA process like keepalived to present a single DNS IP address to clients, or handing out two DNS resolvers via DHCP. Generally it is expected that the two (or more) Pi-hole(s) will be at the same phyiscal location, or at least on the same internal networks. It should also be possible to to replicate to a secondary Pi-hole across networks, either over a VPN or open-Internet, with the approprate firewall/NAT configuration. +Starting with version 2.0, Gravity Sync will sync the `gravity.db` and `custom.list` files on each Pi-hole with each other. (Previous versions only pulled data one way.) + ## Installation Login to your *secondary* Pi-hole, and run: @@ -59,8 +59,30 @@ Now, test Gravity Sync. You can run a comparison between primary and secondary d Assuming Gravity Sync runs successfully, it will indicate if there are changes pending between the two databases. If not, make a subtle change to a whitelist/blacklist on your primary Pi-hole, such as changing a description field or disabling a whitelist item, and then running `./gravity-sync.sh compare` again to validate your installation is working correctly. +### Smart Sync + +The default command for Gravity Sync is simple. + +``` +./gravity-sync.sh` +``` + +But you can also run `./gravity-sync.sh smart` if you feel like it, and it'll do the same thing. + +It will perform some checks to help insure success and then stop before making changes if it detects an issue. It will also perform the same `compare` function outlined above, and if there are no changes pending, it will exit without making an attempt to copy data. + +**Example:** If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull of the `gravity.db` then push `custom.list` and finally restart the correct components on each server. It will also now only perform a sync of each component if there are changes within each type to replicate. So if you only make a small change to your Local DNS settings, it doesn't kickoff the larger `gravity.db` replication. + +This allows you to be more flexible in where you make your configuration changes to block/allow lists and local DNS settings being made on either the primary or secondary, but it's best practice to continue making changes on one side where possible. In the event there are configuration changes to the same element (example, `custom.list` changes at both sides) then Gravity Sync will attempt to determine based on timestamps on what side the last changed happened, in which case the latest changes will be considered authoritative and overwrite the other side. Gravity Sync does not merge the contents of the files when changes happen, it simply overwrites the entire content. + +If the execution completes, you will now have overwritten your running `gravity.db` and `custom.list` on the secondary Pi-hole after creating a copy of the running files (with `.backup` appended) in the `backup` subfolder located with your script. Gravity Sync will also keep a copy of the last sync'd files from the primary (in the `backup` folder appended with `.pull` or `.push`) for future use. + +Finally, a file called `gravity-sync.log` will be created in the `gravity-sync` folder along side the script with the date the script was last executed appended to the bottom. + +You can check for successful pull attempts by running: `./gravity-sync.sh logs` + ### The Pull Function -The Gravity Sync Pull, is the standard method of sync operation, and will not prompt for user input after execution. It will perform some checks to help insure success and then stop before making changes if it detects an issue. It will also perform the same `compare` function outlined above, and if there are no changes pending, it will exit without making an attempt to copy data. +The Gravity Sync Pull, prior to version 2.0, was the standard method of sync operation, and will not prompt for user input after execution. ```bash ./gravity-sync.sh pull @@ -68,10 +90,6 @@ The Gravity Sync Pull, is the standard method of sync operation, and will not pr If the execution completes, you will now have overwritten your running `gravity.db` and `custom.list` on the secondary Pi-hole after creating a copy of the running files (with `.backup` appended) in the `backup` subfolder located with your script. Gravity Sync will also keep a copy of the last sync'd files from the primary (in the `backup` folder appended with `.pull`) for future use. -Finally, a file called `gravity-sync.log` will be created in the `gravity-sync` folder along side the script with the date the script was last executed appended to the bottom. - -You can check for successful pull attempts by running: `./gravity-sync.sh logs` - ### The Push Function Gravity Sync includes the ability to `push` from the secondary Pi-hole back to the primary. This would be useful in a situation where your primary Pi-hole is down for an extended period of time, and you have made list changes on the secondary Pi-hole that you want to force back to the primary, when it comes online. From 2f56c02f76ef2b016388bcbcf4f69f4180cad04c Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 15:51:48 -0500 Subject: [PATCH 25/97] ` --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b4afc6..b6695bc 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Assuming Gravity Sync runs successfully, it will indicate if there are changes p The default command for Gravity Sync is simple. ``` -./gravity-sync.sh` +./gravity-sync.sh ``` But you can also run `./gravity-sync.sh smart` if you feel like it, and it'll do the same thing. From f8d0b9b2c699751d0c68067e6107ee76031647d4 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 15:52:05 -0500 Subject: [PATCH 26/97] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b4afc6..b6695bc 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Assuming Gravity Sync runs successfully, it will indicate if there are changes p The default command for Gravity Sync is simple. ``` -./gravity-sync.sh` +./gravity-sync.sh ``` But you can also run `./gravity-sync.sh smart` if you feel like it, and it'll do the same thing. From 717cad56cbb254554d9c8bb12d646f4da617d6c4 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 20:05:15 -0500 Subject: [PATCH 27/97] Update CHANGELOG.md --- CHANGELOG.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c57efdf..38f3784 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,6 @@ ## 2.0 ### The Smart Release -**Features** In this release, Gravity Sync will now detect not only if each component (`gravity.db` and `custom.list`) has changed since the last sync, but also what direction they need to go. It will then initate a `push` and/or `pull` specific to each piece. Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull of the `gravity.db` then push `custom.list` and finally restart the correct components on each server. It will also now only perform a sync of each component if there are changes within each type to replicate. So if you only make a small change to your Local DNS settings, it doesn't kickoff the larger `gravity.db` replication. @@ -17,7 +16,6 @@ New installs will use the `smart` function by default. Existing users who want t ## 1.8 ### The Logical Release -**Features** There is nothing really sexy here, but a lot of changes under the covers to improve reliablity between different SSH client types. A lot of the logic and functions are more consistent and cleaner. In some cultures, fewer bugs and more reliablity are considered features. Much of this will continue through the 1.8.x line. - SSH/RSYNC connection logic rewritten to be specific to client options between OpenSSH, OpenSSH w/ SSHPASS, and Dropbear. @@ -242,4 +240,4 @@ echo 'Reloading configuration of HA secondary FTLDNS from new gravity.db' pihole restartdns reload-lists ``` -For real, that's it. 6 lines, and could probably have be done with less. \ No newline at end of file +For real, that's it. 6 lines, and could probably have be done with less. From 624e89ae6c0fb380f904324113959c6bdd0bddcd Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 20:06:17 -0500 Subject: [PATCH 28/97] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38f3784..99c86fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ In this release, Gravity Sync will now detect not only if each component (`gravity.db` and `custom.list`) has changed since the last sync, but also what direction they need to go. It will then initate a `push` and/or `pull` specific to each piece. -Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull of the `gravity.db` then push `custom.list` and finally restart the correct components on each server. It will also now only perform a sync of each component if there are changes within each type to replicate. So if you only make a small change to your Local DNS settings, it doesn't kickoff the larger `gravity.db` replication. +**Example:** If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull of the `gravity.db` then push `custom.list` and finally restart the correct components on each server. It will also now only perform a sync of each component if there are changes within each type to replicate. So if you only make a small change to your Local DNS settings, it doesn't kickoff the larger `gravity.db` replication. The default command for Gravity Sync is now just `./gravity-sync.sh` -- but you can also run `./gravity-sync.sh smart` if you feel like it, and it'll do the same thing. From d46b2f5a9a59453959b0622f3e6a52c79b038fa7 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Wed, 8 Jul 2020 09:32:20 -0500 Subject: [PATCH 29/97] " --- gravity-sync.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 689171a..44f72aa 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -3,7 +3,7 @@ SCRIPT_START=$SECONDS # GRAVITY SYNC BY VMSTAN ##################### PROGRAM='Gravity Sync' -VERSION='2.0.0' +VERSION='2.0.1' # Execute from the home folder of the user who owns it (ex: 'cd ~/gravity-sync') # For documentation or downloading updates visit https://github.com/vmstan/gravity-sync @@ -1469,7 +1469,7 @@ function clear_cron { echo_stat crontab -l > cronjob-old.tmp - sed '/${GS_FILENAME}/d' cronjob-old.tmp > cronjob-new.tmp + sed "/${GS_FILENAME}/d" cronjob-old.tmp > cronjob-new.tmp crontab cronjob-new.tmp 2>/dev/null error_validate rm cronjob-old.tmp From aecca3e7f09d6ee6f63bda655ea32092887ac33b Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Wed, 8 Jul 2020 09:33:25 -0500 Subject: [PATCH 30/97] 2.0.1 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b6695bc..29ab571 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Now, test Gravity Sync. You can run a comparison between primary and secondary d Assuming Gravity Sync runs successfully, it will indicate if there are changes pending between the two databases. If not, make a subtle change to a whitelist/blacklist on your primary Pi-hole, such as changing a description field or disabling a whitelist item, and then running `./gravity-sync.sh compare` again to validate your installation is working correctly. -### Smart Sync +### The Smart Sync The default command for Gravity Sync is simple. From ea8f196669210fada2524e73ce69e63ed4cc958b Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Wed, 8 Jul 2020 09:34:18 -0500 Subject: [PATCH 31/97] 2.0.1 --- ADVANCED.md | 4 ++-- VERSION | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index 92a825d..7a52bb1 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -14,8 +14,8 @@ Download the latest release from [GitHub](https://github.com/vmstan/gravity-sync ```bash cd ~ -wget https://github.com/vmstan/gravity-sync/archive/v2.0.0.zip -unzip v2.0.0.zip -d gravity-sync +wget https://github.com/vmstan/gravity-sync/archive/v2.0.1.zip +unzip v2.0.1.zip -d gravity-sync cd gravity-sync ``` diff --git a/VERSION b/VERSION index 359a5b9..10bf840 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.0 \ No newline at end of file +2.0.1 \ No newline at end of file From 6626ec93a0d3ce53c1062b0b5349471d5c25dcc6 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Wed, 8 Jul 2020 09:35:31 -0500 Subject: [PATCH 32/97] 2.0.1 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99c86fc..20b3c1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ This allows you to be more flexible in where you make your configuration changes New installs will use the `smart` function by default. Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. This is not required. The previous `./gravity-sync.sh pull` and `./gravity-sync.sh push` commands continue to function as they did previously, with no intention to break this functionality. +#### 2.0.1 +- Fixes bug that caused existing crontab entry not to be removed when switching from `pull` to Smart Sync. + ## 1.8 ### The Logical Release From c8955d0ecfd304893bb080bb59974e60fcdff509 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Wed, 8 Jul 2020 09:48:24 -0500 Subject: [PATCH 33/97] Correct output of argument listing --- ADVANCED.md | 4 +- CHANGELOG.md | 166 ++++++++++++++++++++++++------------------------ VERSION | 2 +- gravity-sync.sh | 2 +- 4 files changed, 88 insertions(+), 86 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index 7a52bb1..eecbb94 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -14,8 +14,8 @@ Download the latest release from [GitHub](https://github.com/vmstan/gravity-sync ```bash cd ~ -wget https://github.com/vmstan/gravity-sync/archive/v2.0.1.zip -unzip v2.0.1.zip -d gravity-sync +wget https://github.com/vmstan/gravity-sync/archive/v2.0.2.zip +unzip v2.0.2.zip -d gravity-sync cd gravity-sync ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 20b3c1d..bbb6aca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ New installs will use the `smart` function by default. Existing users who want t #### 2.0.1 - Fixes bug that caused existing crontab entry not to be removed when switching from `pull` to Smart Sync. +#### 2.0.2 +- Correct output of `smart` function when script is run without proper function requested. + ## 1.8 ### The Logical Release @@ -27,18 +30,18 @@ There is nothing really sexy here, but a lot of changes under the covers to impr - Adds custom port specification to ssh-copy-id and dropbearkey commands during configuration generation. - Generally better error handling of configuration options. +#### 1.8.1 +- Detects if script is running as the root user or via `sudo ./gravity-sync.sh` and exits on error. [#34](https://github.com/vmstan/gravity-sync/issues/34) + +#### 1.8.2 +- Corrects issue where `custom.list` file would not replicate if the file didn't exist locally, and there were no other changes to replicate. [#39](https://github.com/vmstan/gravity-sync/issues/39) + #### 1.8.3 - Simplified method for input of automation frequency when running `./gravity-sync.sh automate` function. - Now removes existing automation task from crontab, if it exists, when re-running `automate` function. - Automation can be disabled by setting frequency to `0` when prompted. - Adds `dev` tag to `./gravity-sync.sh version` output for users running off the development branch. -#### 1.8.2 -- Corrects issue where `custom.list` file would not replicate if the file didn't exist locally, and there were no other changes to replicate. [#39](https://github.com/vmstan/gravity-sync/issues/39) - -#### 1.8.1 -- Detects if script is running as the root user or via `sudo ./gravity-sync.sh` and exits on error. [#34](https://github.com/vmstan/gravity-sync/issues/34) - ## 1.7 ### The Andrew Release @@ -50,35 +53,24 @@ There is nothing really sexy here, but a lot of changes under the covers to impr **Known Issues** - No new Star Trek references. -#### 1.7.7 -- `config` function will attempt to ping remote host to validate network connection, can by bypassed by adding `PING_AVOID='1'` to your `gravity-sync.conf` file. -- Changes some [INFO] messages to [WARN] where approprate. -- Adds aliases for more Gravity Sync functions. -- Shows current version on each script execution. -- Adds time output to Aborting message (exit without change.) -- Includes parsing of functions in time calculation. -- Checks for existance of Pi-hole binaries during validation. -- Less chatty about each step of configuration validation if it completes. -- Less chatty about replication validation if it's not necessary. -- Less chatty about file validation if no changes are required. -- When applying `update` in DEV mode, the Git branch used will be shown. -- Validates log export operation. +#### 1.7.1 +- There is a changelog file now. I'm mentioning it in the changelog file. So meta. +- `./gravity-sync.sh version` will check for and alert you for new versions. -#### 1.7.6 -- Detects `dbclient` install as alternative to OpenSSH Client. -- Attempts to install OpenSSH Client if not found, and Dropbear is not alternative. -- Fix bug with `dropbearkey` not finding .ssh folder. -- Numerous fixes to accomidate DietPi in general. -- Fixes issue where `compare` function would show changes where actually none existed. -- [WARN] header is now purple all the way across, consistent with [INFO] as of 1.7.4. -- Fixes issue where `custom.list` would only pull if the file already existed on the secondary Pi-hole. -- One new Star Trek reference. +#### 1.7.2 +This update changes the way that beta/development updates are applied. To continue receving the development branch, create an empty file in the `gravity-sync` folder called `dev` and afterwards the standard `./gravity-sync.sh update` function will apply the correct updates. +``` +cd gravity-sync +touch dev +./gravity-sync.sh update +``` +Delete the `dev` file and update again to revert back to the stable/master branch. -#### 1.7.5 -- No code changes! -- Primary README now only reflect "The Easy Way" to install and configure Gravity Sync -- "The Less Easy Way" are now part of [ADVANCED.md](https://github.com/vmstan/gravity-sync/blob/master/ADVANCED.md) -- All advanced configuration options are outlined in [ADVANCED.md](https://github.com/vmstan/gravity-sync/blob/master/ADVANCED.md) +#### 1.7.3 +- Cleaning up output of argument listing + +**Deprecation** +- Removes `beta` function for applying development branch updates. #### 1.7.4 - `./gravity-sync.sh dev` will now toggle dev flag on/off. No `touch` required, although it still works that way under the covers. Improvement of methods added in 1.7.2. @@ -93,24 +85,35 @@ There is nothing really sexy here, but a lot of changes under the covers to impr - Detects absence of `ssh-keygen` utility on host OS and will use `dropbearkey` as an alternative (DietPi) - Changelog polarity reversed after heated discussions with marketing team. -#### 1.7.3 -- Cleaning up output of argument listing +#### 1.7.5 +- No code changes! +- Primary README now only reflect "The Easy Way" to install and configure Gravity Sync +- "The Less Easy Way" are now part of [ADVANCED.md](https://github.com/vmstan/gravity-sync/blob/master/ADVANCED.md) +- All advanced configuration options are outlined in [ADVANCED.md](https://github.com/vmstan/gravity-sync/blob/master/ADVANCED.md) -#### 1.7.2 -This update changes the way that beta/development updates are applied. To continue receving the development branch, create an empty file in the `gravity-sync` folder called `dev` and afterwards the standard `./gravity-sync.sh update` function will apply the correct updates. -``` -cd gravity-sync -touch dev -./gravity-sync.sh update -``` -Delete the `dev` file and update again to revert back to the stable/master branch. +#### 1.7.6 +- Detects `dbclient` install as alternative to OpenSSH Client. +- Attempts to install OpenSSH Client if not found, and Dropbear is not alternative. +- Fix bug with `dropbearkey` not finding .ssh folder. +- Numerous fixes to accomidate DietPi in general. +- Fixes issue where `compare` function would show changes where actually none existed. +- [WARN] header is now purple all the way across, consistent with [INFO] as of 1.7.4. +- Fixes issue where `custom.list` would only pull if the file already existed on the secondary Pi-hole. +- One new Star Trek reference. -**Deprecation** -- Removes `beta` function for applying development branch updates. - -#### 1.7.1 -- There is a changelog file now. I'm mentioning it in the changelog file. So meta. -- `./gravity-sync.sh version` will check for and alert you for new versions. +#### 1.7.7 +- `config` function will attempt to ping remote host to validate network connection, can by bypassed by adding `PING_AVOID='1'` to your `gravity-sync.conf` file. +- Changes some [INFO] messages to [WARN] where approprate. +- Adds aliases for more Gravity Sync functions. +- Shows current version on each script execution. +- Adds time output to Aborting message (exit without change.) +- Includes parsing of functions in time calculation. +- Checks for existance of Pi-hole binaries during validation. +- Less chatty about each step of configuration validation if it completes. +- Less chatty about replication validation if it's not necessary. +- Less chatty about file validation if no changes are required. +- When applying `update` in DEV mode, the Git branch used will be shown. +- Validates log export operation. ## 1.6 ### The Restorative Release @@ -146,14 +149,14 @@ Delete the `dev` file and update again to revert back to the stable/master branc - All new and exciting code comments. - No new Star Trek references. -#### 1.4.3 -- Bug fixes around not properly utilizing custom SSH keyfile. +#### 1.4.1 +- Adds variables for custom log locations to `gravity-sync.conf`, see `.example` file for listing. #### 1.4.2 - Will prompt to create new `gravity-sync.conf` file when run without an existing configuration. -#### 1.4.1 -- Adds variables for custom log locations to `gravity-sync.conf`, see `.example` file for listing. +#### 1.4.3 +- Bug fixes around not properly utilizing custom SSH keyfile. ## 1.3 ### The Comparison Release @@ -165,22 +168,21 @@ Delete the `dev` file and update again to revert back to the stable/master branc - Additional debugging options such as checking last cronjob output via `./gravity-sync.sh cron` if configured. - Much more consistency in how running commands are processed in interactive mode. -#### 1.3.4 -- Moves backup of local database before initiating remote pull. -- Validates file ownership and permissions before attempting to rewrite. -- Added two Star Trek references. - -#### 1.3.3 -- Corrected Pihole bin path issue that cause automated sync not to reload services. - -#### 1.3.2 -- MUCH cleaner output, same great features. - #### 1.3.1 - Changes [GOOD] to [DONE] in execution output. - Better validation of initial SSH connection. - Support for password based authentication using SSHPASS. +#### 1.3.2 +- MUCH cleaner output, same great features. + +#### 1.3.3 +- Corrected Pihole bin path issue that cause automated sync not to reload services. + +#### 1.3.4 +- Moves backup of local database before initiating remote pull. +- Validates file ownership and permissions before attempting to rewrite. +- Added two Star Trek references. ## 1.2 ### The Functional Release @@ -188,23 +190,23 @@ Delete the `dev` file and update again to revert back to the stable/master branc - Does not look for permission to update when run. - Cleanup and expand comments. -#### 1.2.5 -- Push function now does a backup, on the secondary PH, of the primary database, before pushing. +#### 1.2.1 +- Improved logging functions. -#### 1.2.4 -- Changes `~` to `$HOME`. -- Fixes bug that prevented sync from working when run via crontab. +#### 1.2.2 +- Different style for status updates. #### 1.2.3 - Uses a dedicated backup folder for `.backup` and `.last` files. - Copies db instead of moving to rename and then replacing to be more reliable. - Even cleaner label status. -#### 1.2.2 -- Different style for status updates. +#### 1.2.4 +- Changes `~` to `$HOME`. +- Fixes bug that prevented sync from working when run via crontab. -#### 1.2.1 -- Improved logging functions. +#### 1.2.5 +- Push function now does a backup, on the secondary PH, of the primary database, before pushing. ## 1.1 ### The Pushy Release @@ -212,22 +214,22 @@ Delete the `dev` file and update again to revert back to the stable/master branc - Seperated main purpose of script into `pull` argument. - Allow process to reverse back using `push` argument. -#### 1.1.6 -- Code easier to read with proper tabs. +#### 1.1.2 +- First release since move from being just a Gist. +- Just relearning how to use GitHub, minor bug fixes. -#### 1.1.5 -- Added ability to view logs with `./gravity-sync.sh logs`. +#### 1.1.3 +- Now includes example an configuration file. #### 1.1.4 - Added update script. - Added version check. -#### 1.1.3 -- Now includes example an configuration file. +#### 1.1.5 +- Added ability to view logs with `./gravity-sync.sh logs`. -#### 1.1.2 -- First release since move from being just a Gist. -- Just relearning how to use GitHub, minor bug fixes. +#### 1.1.6 +- Code easier to read with proper tabs. ## 1.0 ### The Initial Release diff --git a/VERSION b/VERSION index 10bf840..f93ea0c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.1 \ No newline at end of file +2.0.2 \ No newline at end of file diff --git a/gravity-sync.sh b/gravity-sync.sh index 44f72aa..59e456b 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1360,7 +1360,7 @@ function list_gs_arguments { echo -e " ${YELLOW}version${NC} Display installed version of ${PROGRAM}" echo -e "" echo -e "Replication Options:" - echo -e " ${YELLOW}sync${NC} Detect changes on each side and bring them together" + echo -e " ${YELLOW}smart${NC} Detect changes on each side and bring them together" echo -e " ${YELLOW}pull${NC} Force remote configuration changes to this server" echo -e " ${YELLOW}push${NC} Force local configuration made on this server back" echo -e " ${YELLOW}restore${NC} Restore the ${GRAVITY_FI} on this server" From c4704a2d459312f58f88d989995bf59269f165e9 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Wed, 8 Jul 2020 11:23:02 -0500 Subject: [PATCH 34/97] Chris was right --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbb6aca..feb295b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ New installs will use the `smart` function by default. Existing users who want t #### 2.0.2 - Correct output of `smart` function when script is run without proper function requested. +- Decided marketing team was correct about display of versions in `CHANGELOG.md` -- sorry Chris. ## 1.8 ### The Logical Release From 0536e36eef93e1581c7431025e4bc9f4374ae5c6 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Wed, 8 Jul 2020 16:10:00 -0500 Subject: [PATCH 35/97] Reference arch --- ADVANCED.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/ADVANCED.md b/ADVANCED.md index eecbb94..1d6a44f 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -180,3 +180,39 @@ If you prefer to still use cron but modify your settings by hand, using the entr crontab -e */30 * * * * /bin/bash /home/USER/gravity-sync/gravity-sync.sh > /home/USER/gravity-sync/gravity-sync.cron ``` + +## Reference Architectures + +There are three reference architectures that I'll outline. All of them require an external DHCP server (such as a router, or dedicated DHCP server) handing out the DNS address(es) for your Pi-holes. Use of the integrated DHCP function in Pi-hole when using Gravity Sync is discouraged, although I'm sure there are ways to make it work. **Gravity Sync does not manage any DHCP settings.** + +### Easy Peasy +![Easy Peasy](https://user-images.githubusercontent.com/3002053/86969050-c1131400-c132-11ea-9328-dfda772dc280.png) + +This design requires the least amount of overhead, or additional software/network configuration beyond Pi-hole and Gravity Sync. + +1. Client requests an IP address from a DHCP server on the network and receives it along with DNS and gateway information back. Two DNS servers (Pi-hole) are returned to the client. +2. Client queries one of the two DNS servers, and Pi-hole does it's thing. + +You can make changes to your blocklist, exceptions, etc, on either Pi-hole and they will be sync'd to the other within the timeframe you establish (here, 15 minutes.) + +### Stay Alive +![Stay Alive](https://user-images.githubusercontent.com/3002053/86969056-c40e0480-c132-11ea-8db4-15c26ab8999d.png) + +The downside in the above design is you have two places where your clients are logging lookup requests to. Gravity Sync will let you change filter settings in either location, but if you're doing it often things may get overwritten. One way to get around this is by using keepalived and present a single virtual IP address of the two Pi-hole, to clients in an active/passive mode. The two nodes will check their own status, and each other, and hand the VIP around if there are issues. + +1. Client requests an IP address from a DHCP server on the network and receives it along with DNS and gateway information back. One DNS server (VIP) is returned to the client. +2. The VIP managed by the keepalived service will determine which Pi-hole responds. You make your configuration changes to the active VIP address. +3. Client queries the single DNS servers, and Pi-hole does it's thing. + +You make your configuration changes to the active VIP address and they will be sync'd to the other within the timeframe you establish (here, 15 minutes.) + +### Crazy Town +![Crazy Town](https://user-images.githubusercontent.com/3002053/86969028-b8bad900-c132-11ea-83f1-88b4888af32c.png) + +For those who really love Pi-hole and Gravity Sync. Combining the best of both worlds. + +1. Client requests an IP address from a DHCP server on the network and receives it along with DNS and gateway information back. Two DNS servers (VIPs) are returned to the client. +2. The VIPs are managed by the keepalived service on each side and will determine which of two Pi-hole responds. You can make your configuration changes to the active VIP address on either side. +3. Client queries one of the two VIP servers, and the responding Pi-hole does it's thing. + +Here we use `./gravity-sync pull` on the secondary Pi-hole at each side, and ofset the update intervals from the main sync. \ No newline at end of file From ab6a90d118688377770ad7514be719a32c3981f2 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Wed, 8 Jul 2020 16:10:32 -0500 Subject: [PATCH 36/97] More references --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index feb295b..b5df980 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ New installs will use the `smart` function by default. Existing users who want t #### 2.0.2 - Correct output of `smart` function when script is run without proper function requested. - Decided marketing team was correct about display of versions in `CHANGELOG.md` -- sorry Chris. +- Adds reference architectures to the `ADVANCED.md` file. ## 1.8 ### The Logical Release From 393924a836d159391c15ca4a81966cd6b434509f Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Wed, 8 Jul 2020 16:12:31 -0500 Subject: [PATCH 37/97] easy --- ADVANCED.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index 1d6a44f..f7bb9d7 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -193,12 +193,12 @@ This design requires the least amount of overhead, or additional software/networ 1. Client requests an IP address from a DHCP server on the network and receives it along with DNS and gateway information back. Two DNS servers (Pi-hole) are returned to the client. 2. Client queries one of the two DNS servers, and Pi-hole does it's thing. -You can make changes to your blocklist, exceptions, etc, on either Pi-hole and they will be sync'd to the other within the timeframe you establish (here, 15 minutes.) +You can make changes to your blocklist, exceptions, etc, on either Pi-hole and they will be sync'd to the other within the timeframe you establish (here, 15 minutes.) The downside in the above design is you have two places where your clients are logging lookup requests to. Gravity Sync will let you change filter settings in either location, but if you're doing it often things may get overwritten. ### Stay Alive ![Stay Alive](https://user-images.githubusercontent.com/3002053/86969056-c40e0480-c132-11ea-8db4-15c26ab8999d.png) -The downside in the above design is you have two places where your clients are logging lookup requests to. Gravity Sync will let you change filter settings in either location, but if you're doing it often things may get overwritten. One way to get around this is by using keepalived and present a single virtual IP address of the two Pi-hole, to clients in an active/passive mode. The two nodes will check their own status, and each other, and hand the VIP around if there are issues. +One way to get around having logging in two places is by using keepalived and present a single virtual IP address of the two Pi-hole, to clients in an active/passive mode. The two nodes will check their own status, and each other, and hand the VIP around if there are issues. 1. Client requests an IP address from a DHCP server on the network and receives it along with DNS and gateway information back. One DNS server (VIP) is returned to the client. 2. The VIP managed by the keepalived service will determine which Pi-hole responds. You make your configuration changes to the active VIP address. From 4d824c54ebf1588e7e6981700d6f774967259c7f Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Wed, 8 Jul 2020 20:48:53 -0500 Subject: [PATCH 38/97] Check for rsync on remote host --- ADVANCED.md | 2 +- gravity-sync.sh | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/ADVANCED.md b/ADVANCED.md index f7bb9d7..6f117b2 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -215,4 +215,4 @@ For those who really love Pi-hole and Gravity Sync. Combining the best of both w 2. The VIPs are managed by the keepalived service on each side and will determine which of two Pi-hole responds. You can make your configuration changes to the active VIP address on either side. 3. Client queries one of the two VIP servers, and the responding Pi-hole does it's thing. -Here we use `./gravity-sync pull` on the secondary Pi-hole at each side, and ofset the update intervals from the main sync. \ No newline at end of file +Here we use `./gravity-sync pull` on the secondary Pi-hole at each side, and off-set the update intervals from the main sync. \ No newline at end of file diff --git a/gravity-sync.sh b/gravity-sync.sh index 59e456b..a50da97 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1013,6 +1013,23 @@ function detect_ssh { fi } +function detect_remotersync { + MESSAGE="Validating RSYNC Installed on ${REMOTE_HOST}" + echo_stat + + REMOTERSYNC=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "hash rsync") + + if [ "${REMOTERSYNC}" == "-bash: hash: rsync: not found" + then + echo_fail + MESSAGE="Please install RSYNC on ${REMOTE_HOST}" + echo_info + exit_nochange + else + echo_good + fi +} + ## Error Validation function error_validate { if [ "$?" != "0" ] @@ -1304,6 +1321,8 @@ function config_generate { echo_info validate_os_sshpass + + detect_remotersync exit_withchange } From dc75b2b7ae14bf252c3af6a3e32be173f98a1c99 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Wed, 8 Jul 2020 20:53:20 -0500 Subject: [PATCH 39/97] bracket --- gravity-sync.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index a50da97..fa41d40 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1019,12 +1019,11 @@ function detect_remotersync { REMOTERSYNC=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "hash rsync") - if [ "${REMOTERSYNC}" == "-bash: hash: rsync: not found" + if [ "${REMOTERSYNC}" == "-bash: hash: rsync: not found" ] then echo_fail MESSAGE="Please install RSYNC on ${REMOTE_HOST}" - echo_info - exit_nochange + echo_warn else echo_good fi From e10c202601d0d3a7c92701513874a87ed7eefd90 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Wed, 8 Jul 2020 20:55:59 -0500 Subject: [PATCH 40/97] Thing down flip it and reverse it --- gravity-sync.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index fa41d40..a274475 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1017,15 +1017,15 @@ function detect_remotersync { MESSAGE="Validating RSYNC Installed on ${REMOTE_HOST}" echo_stat - REMOTERSYNC=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "hash rsync") + REMOTERSYNC=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "hash rsync") >/dev/null 2>&1 - if [ "${REMOTERSYNC}" == "-bash: hash: rsync: not found" ] + if [ "${REMOTERSYNC}" == "" ] then + echo_good + else echo_fail MESSAGE="Please install RSYNC on ${REMOTE_HOST}" echo_warn - else - echo_good fi } From 49a9198a51c243bedd8c4e22536bed15d560e015 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Wed, 8 Jul 2020 21:00:03 -0500 Subject: [PATCH 41/97] Sed --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index a274475..7799201 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1017,7 +1017,7 @@ function detect_remotersync { MESSAGE="Validating RSYNC Installed on ${REMOTE_HOST}" echo_stat - REMOTERSYNC=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "hash rsync") >/dev/null 2>&1 + REMOTERSYNC=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "hash rsync" | sed 's/\s.*$//') if [ "${REMOTERSYNC}" == "" ] then From 34f47c1267acb73b747ca0de493d3a3b2f3c9c7d Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Wed, 8 Jul 2020 21:18:13 -0500 Subject: [PATCH 42/97] rewrite check --- gravity-sync.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 7799201..596c8cb 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1017,9 +1017,9 @@ function detect_remotersync { MESSAGE="Validating RSYNC Installed on ${REMOTE_HOST}" echo_stat - REMOTERSYNC=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "hash rsync" | sed 's/\s.*$//') + ${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "hash rsync" 2>/dev/null - if [ "${REMOTERSYNC}" == "" ] + if [ "$?" == "-bash: 0: command not found" ] then echo_good else From c13ca47f0657cbeb603f1869ea5699b9bbc22235 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Wed, 8 Jul 2020 21:23:15 -0500 Subject: [PATCH 43/97] 0 --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 596c8cb..23a2c38 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1019,7 +1019,7 @@ function detect_remotersync { ${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "hash rsync" 2>/dev/null - if [ "$?" == "-bash: 0: command not found" ] + if [ "$?" == "0" ] then echo_good else From e725ab7f5d86e38e9755768c994d4ff316d4b35e Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 9 Jul 2020 10:02:51 -0500 Subject: [PATCH 44/97] New rsync validation method --- gravity-sync.sh | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 23a2c38..715d732 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1017,16 +1017,22 @@ function detect_remotersync { MESSAGE="Validating RSYNC Installed on ${REMOTE_HOST}" echo_stat - ${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "hash rsync" 2>/dev/null + CMD_TIMEOUT='15' + CMD_REQUESTED="touch ~/gs.test" + create_sshcmd - if [ "$?" == "0" ] - then - echo_good - else - echo_fail - MESSAGE="Please install RSYNC on ${REMOTE_HOST}" - echo_warn - fi + RSYNC_REPATH="rsync" + RSYNC_SOURCE="${REMOTE_USER}@${REMOTE_HOST}:~/gs.test" + RSYNC_TARGET="$HOME/${LOCAL_FOLDR}/gs.test" + create_rsynccmd + + rm $HOME/${LOCAL_FOLDR}/gs.test + + CMD_TIMEOUT='15' + CMD_REQUESTED="rm ~/gs.test" + create_sshcmd + + error_validate } ## Error Validation From 270b354e80c57f9ab3b03926adbc1e8df8f6afa7 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 9 Jul 2020 10:06:55 -0500 Subject: [PATCH 45/97] Messages --- gravity-sync.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 715d732..1098ebb 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1014,25 +1014,32 @@ function detect_ssh { } function detect_remotersync { - MESSAGE="Validating RSYNC Installed on ${REMOTE_HOST}" + MESSAGE="Validating RSYNC Ability to ${REMOTE_HOST}" + echo_info + + MESSAGE="Creating Test File on ${REMOTE_HOST}" echo_stat CMD_TIMEOUT='15' CMD_REQUESTED="touch ~/gs.test" create_sshcmd + MESSAGE="Pulling Test File to $HOSTNAME" + echo_stat + RSYNC_REPATH="rsync" RSYNC_SOURCE="${REMOTE_USER}@${REMOTE_HOST}:~/gs.test" RSYNC_TARGET="$HOME/${LOCAL_FOLDR}/gs.test" create_rsynccmd + MESSAGE="Cleaning Up Test Files" + echo_stat + rm $HOME/${LOCAL_FOLDR}/gs.test CMD_TIMEOUT='15' CMD_REQUESTED="rm ~/gs.test" create_sshcmd - - error_validate } ## Error Validation From 201807bc4d6f6465d8461c94ef47cd15b5b6050c Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 9 Jul 2020 10:10:25 -0500 Subject: [PATCH 46/97] Warn instead of info --- gravity-sync.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 1098ebb..708d546 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1014,9 +1014,6 @@ function detect_ssh { } function detect_remotersync { - MESSAGE="Validating RSYNC Ability to ${REMOTE_HOST}" - echo_info - MESSAGE="Creating Test File on ${REMOTE_HOST}" echo_stat @@ -1024,6 +1021,9 @@ function detect_remotersync { CMD_REQUESTED="touch ~/gs.test" create_sshcmd + MESSAGE="If this fails make sure RSYNC is installed on ${REMOTE_HOST}" + echo_warn + MESSAGE="Pulling Test File to $HOSTNAME" echo_stat From 23a6ceed3a9780bc6a9705885ce067da0a152797 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 9 Jul 2020 10:42:09 -0500 Subject: [PATCH 47/97] 2.0.2 --- ADVANCED.md | 44 +++++++++++++++++++++++--- CHANGELOG.md | 4 ++- README.md | 82 ++++++++++++++++--------------------------------- gravity-sync.sh | 4 +-- 4 files changed, 72 insertions(+), 62 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index 6f117b2..2b019e4 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -76,6 +76,39 @@ Gravity Sync will validate that the `sshpass` utility is installed on your syste Save. Keep calm, carry on. +### The Pull Function +The Gravity Sync Pull, prior to version 2.0, was the standard method of sync operation, and will not prompt for user input after execution. + +```bash +./gravity-sync.sh pull +``` + +If the execution completes, you will now have overwritten your running `gravity.db` and `custom.list` on the secondary Pi-hole after creating a copy of the running files (with `.backup` appended) in the `backup` subfolder located with your script. Gravity Sync will also keep a copy of the last sync'd files from the primary (in the `backup` folder appended with `.pull`) for future use. + +### The Push Function +Gravity Sync includes the ability to `push` from the secondary Pi-hole back to the primary. This would be useful in a situation where your primary Pi-hole is down for an extended period of time, and you have made list changes on the secondary Pi-hole that you want to force back to the primary, when it comes online. + +```bash +./gravity-sync.sh push +``` + +Before executing, this will make a copy of the remote database under `backup/gravity.db.push` and `backup/custom.list.push` then sync the local configuration to the primary Pi-hole. + +This function purposefuly asks for user interaction to avoid being accidentally automated. + +- If your script prompts for a password on the remote system, make sure that your remote user account is setup not to require passwords in the sudoers file. + +### The Restore Function +Gravity Sync can also `restore` the database on the secondary Pi-hole in the event you've overwritten it accidentally. This might happen in the above scenario where you've had your primary Pi-hole down for an extended period, made changes to the secondary, but perhaps didn't get a chance to perform a `push` of the changes back to the primary, before your automated sync ran. + +```bash +./gravity-sync.sh restore +``` + +This will copy your last `gravity.db.backup` and `custom.list.backup` to the running copy on the secondary Pi-hole. + +This function purposefuly asks for user interaction to avoid being accidentally automated. + ### Hidden Figures There are a series of advanced configuration options that you may need to change to better adapt Gravity Sync to your environment. They are referenced at the end of the `gravity-sync.conf` file. It is suggested that you make any necessary variable changes to this file, as they will superceed the ones located in the core script. If you want to revert back to the Gravity Sync default for any of these settings, just apply a `#` to the beginning of the line to comment it out. @@ -182,11 +215,12 @@ crontab -e ``` ## Reference Architectures +The designation of primary and secondary is purely at your discretion. The doesn't matter if you're using an HA process like keepalived to present a single DNS IP address to clients, or handing out two DNS resolvers via DHCP. Generally it is expected that the two (or more) Pi-hole(s) will be at the same phyiscal location, or at least on the same internal networks. It should also be possible to to replicate to a secondary Pi-hole across networks, either over a VPN or open-Internet, with the approprate firewall/NAT configuration. There are three reference architectures that I'll outline. All of them require an external DHCP server (such as a router, or dedicated DHCP server) handing out the DNS address(es) for your Pi-holes. Use of the integrated DHCP function in Pi-hole when using Gravity Sync is discouraged, although I'm sure there are ways to make it work. **Gravity Sync does not manage any DHCP settings.** ### Easy Peasy -![Easy Peasy](https://user-images.githubusercontent.com/3002053/86969050-c1131400-c132-11ea-9328-dfda772dc280.png) +![Easy Peasy](https://user-images.githubusercontent.com/3002053/87058413-ac378e80-c1cd-11ea-9f21-376170e69ff3.png) This design requires the least amount of overhead, or additional software/network configuration beyond Pi-hole and Gravity Sync. @@ -196,7 +230,7 @@ This design requires the least amount of overhead, or additional software/networ You can make changes to your blocklist, exceptions, etc, on either Pi-hole and they will be sync'd to the other within the timeframe you establish (here, 15 minutes.) The downside in the above design is you have two places where your clients are logging lookup requests to. Gravity Sync will let you change filter settings in either location, but if you're doing it often things may get overwritten. ### Stay Alive -![Stay Alive](https://user-images.githubusercontent.com/3002053/86969056-c40e0480-c132-11ea-8db4-15c26ab8999d.png) +![Stay Alive](https://user-images.githubusercontent.com/3002053/87058415-acd02500-c1cd-11ea-8884-6579a2d5eedc.png) One way to get around having logging in two places is by using keepalived and present a single virtual IP address of the two Pi-hole, to clients in an active/passive mode. The two nodes will check their own status, and each other, and hand the VIP around if there are issues. @@ -207,7 +241,7 @@ One way to get around having logging in two places is by using keepalived and pr You make your configuration changes to the active VIP address and they will be sync'd to the other within the timeframe you establish (here, 15 minutes.) ### Crazy Town -![Crazy Town](https://user-images.githubusercontent.com/3002053/86969028-b8bad900-c132-11ea-83f1-88b4888af32c.png) +![Crazy Town](https://user-images.githubusercontent.com/3002053/87058406-aa6dcb00-c1cd-11ea-8f64-59c529b00166.png) For those who really love Pi-hole and Gravity Sync. Combining the best of both worlds. @@ -215,4 +249,6 @@ For those who really love Pi-hole and Gravity Sync. Combining the best of both w 2. The VIPs are managed by the keepalived service on each side and will determine which of two Pi-hole responds. You can make your configuration changes to the active VIP address on either side. 3. Client queries one of the two VIP servers, and the responding Pi-hole does it's thing. -Here we use `./gravity-sync pull` on the secondary Pi-hole at each side, and off-set the update intervals from the main sync. \ No newline at end of file +Here we use `./gravity-sync pull` on the secondary Pi-hole at each side, and off-set the update intervals from the main sync. + +(I call this crazy, but this is what I use at home.) \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index b5df980..400d621 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,12 +14,14 @@ This allows you to be more flexible in where you make your configuration changes New installs will use the `smart` function by default. Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. This is not required. The previous `./gravity-sync.sh pull` and `./gravity-sync.sh push` commands continue to function as they did previously, with no intention to break this functionality. #### 2.0.1 -- Fixes bug that caused existing crontab entry not to be removed when switching from `pull` to Smart Sync. +- Fixes bug that caused existing crontab entry not to be removed when switching from `pull` to Smart Sync. [#50](https://github.com/vmstan/gravity-sync/issues/50) #### 2.0.2 - Correct output of `smart` function when script is run without proper function requested. - Decided marketing team was correct about display of versions in `CHANGELOG.md` -- sorry Chris. - Adds reference architectures to the `ADVANCED.md` file. +- Checks for RSYNC functionality to remote host during `./gravity-sync.sh configure` and prompts to install. [#53](https://github.com/vmstan/gravity-sync/issues/53) +- Move much of the previous `README.md` to `ADVANCED.md` file. ## 1.8 ### The Logical Release diff --git a/README.md b/README.md index 29ab571..3105c58 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,30 @@ # Gravity Sync -## Background -What is better than a [Pi-hole](https://github.com/pi-hole/pi-hole) blocking ads via DNS on your network? That's right, Two Pi-hole! But if you have more than one Pi-hole in your network you'll want a simple way to keep the list configurations identical between the two. +## Features +What is better than a [Pi-hole](https://github.com/pi-hole/pi-hole) blocking ads via DNS on your network? That's right, Two Pi-hole! (Redundency is key in any network infrastucture.) But if you have more than one Pi-hole in your network you'll want a simple way to keep the list configurations and local DNS settings identical between the two. That's where Gravity Sync comes in. -That's Gravity Sync. +Gravity Sync will: +- Sync that the allow/blocklist configurations stored in `gravity.db` between two Pi-hole +- Sync the local DNS settings stored in `custom.list` between two Pi-hole. +- Provide an easy way to keep this happening in the background. -At it's core, Gravity Sync is maybe a handful of core bash commands, that uses rsync to reach out to a remote host, copy the running `gravity.db` file that contains the Pi-hole blocklist, as well as the `custom.list` file that contains local DNS enteries, and then replaces the copy on the local system. What Gravity Sync provides is an easy way to keep this happening in the background. Ideally you set it and forget it and in the long term, it would be awesome if the Pi-hole team made this entire script unncessary. +Ideally you set up Gravity Sync and forget about it -- and in the long term, it would be awesome if the Pi-hole team made this entire script unncessary. -Gravity Sync will **not** overwrite device specific settings such as device network configuration, admin/API passwords/keys, upstream DNS resolvers, etc. It will also **not** keep DHCP settings or device leases synchronized. +### Limitations +Gravity Sync will **not**: +- Overwrite individual Pi-hole specific settings such as the device's network configuration, admin/API passwords/keys, upstream DNS resolvers, etc. +- Keep DHCP settings or device leases synchronized. -## Prerequisites -### System Requirements -Gravity Sync **requires** Pi-hole 5.0 or higher be already installed on your server. -- Gravity Sync is regularly tested during development with on any of the Linux distrobutions that Pi-hole is [certified to run on](https://docs.pi-hole.net/main/prerequesites/#supported-operating-systems). As Gravity Sync is just an (admittedly) long bash script, it will likely work on other Linux distributions that have the the necessary components. -- Gravity Sync uses the `SUDO` command to elevate permissions for itself, on both Pi-hole systems, during execution. You will need to make sure that you have passwordless SUDO enabled for the accounts on both the primary and secondary Pi-hole that will be performing the work. Most of the pre-built images available for the Raspberry Pi already have this configured, but if you have your Pi-hole running in a virtual machine built from a generic ISO, you may need to [adjust this manually](https://linuxize.com/post/how-to-run-sudo-command-without-password/). -- Gravity Sync has not been tested with Docker deployments of Pi-hole, and is not expected to work there without major modifications. You will need Pi-hole setup with a "traditional" install directly in the base operating system. There are likely other methods of sharing the `gravity.db` file between multiple Docker instances that are better suited to a container environment. -- Gravity Sync leverages SSH and RSYNC to do the leavy lifting between your Pi-hole nodes. OpenSSH is reccomended but if you're using a ultra-lightweight Pi distrbution (such as DietPi) that uses Dropbear by default, it should work as well. Other SSH client/server combonations are not supported at this time. +It is suggested that you use an external DHCP server on your network (such as your router) when using multiple Pi-hole. + +### Requirements +- Pi-hole 5.0 (or higher) be installed on at least two systems, using any of the Linux distros that Pi-hole is [certified to run on](https://docs.pi-hole.net/main/prerequesites/#supported-operating-systems). Docker deployments of Pi-hole are not supported. +- You will need to make sure that you have passwordless `SUDO` enabled for the accounts on both the primary and secondary Pi-hole that will be performing the work. Most of the pre-built images available for the Raspberry Pi already have this configured, but if you have your Pi-hole running in a virtual machine built from a generic ISO, you may need to [adjust this manually](https://linuxize.com/post/how-to-run-sudo-command-without-password/). +- Make sure `SSH` and `RSYNC` are installed on both the primary and secondary Pi-hole prior to installation. This is what does the leavy lifting between your Pi-hole nodes. OpenSSH is reccomended but if you're using a ultra-lightweight Pi distrbution (such as DietPi) that uses Dropbear by default, it should work as well. Other SSH client/server combonations are not supported at this time. ### Pi-hole Architecture You will want to designate one Pi-Hole as primary and at least one as secondary. The primary Pi-hole is where you'll make most of your configuration changes through the Web UI, doing things such as; manual allow-listing, adding blocklists, device/group management, configuring custom/local network DNS, and other changing other list settings. The secondary Pi-hole(s) are where you will install and configure Gravity Sync. -The designation of primary and secondary is purely at your discretion. The doesn't matter if you're using an HA process like keepalived to present a single DNS IP address to clients, or handing out two DNS resolvers via DHCP. Generally it is expected that the two (or more) Pi-hole(s) will be at the same phyiscal location, or at least on the same internal networks. It should also be possible to to replicate to a secondary Pi-hole across networks, either over a VPN or open-Internet, with the approprate firewall/NAT configuration. +For more information and for reference architectures, please [refer to this document](https://github.com/vmstan/gravity-sync/blob/master/ADVANCED.md#reference-architectures) Starting with version 2.0, Gravity Sync will sync the `gravity.db` and `custom.list` files on each Pi-hole with each other. (Previous versions only pulled data one way.) @@ -42,11 +47,11 @@ After you install Gravity Sync to your server you will need to create a configur ``` This will guide you through the process of: -- Specifying the IP or DNS name of your primary Pi-hole -- Specifying the SSH username to connect to your primary Pi-hole -- Selecting the SSH authentication mechanism (key-pair or password) -- Configuring your key-pair and applying it to your primary Pi-hole -- Testing your authentication method +- Specifying the IP or DNS name of your primary Pi-hole. +- Specifying the SSH username to connect to your primary Pi-hole. +- Selecting the SSH authentication mechanism (key-pair or password.) +- Configuring your key-pair and applying it to your primary Pi-hole. +- Testing your authentication method, and testing RSYNC to the primary. After you've completed your configuration, proceed to the Execution phase. @@ -57,9 +62,9 @@ Now, test Gravity Sync. You can run a comparison between primary and secondary d ./gravity-sync.sh compare ``` -Assuming Gravity Sync runs successfully, it will indicate if there are changes pending between the two databases. If not, make a subtle change to a whitelist/blacklist on your primary Pi-hole, such as changing a description field or disabling a whitelist item, and then running `./gravity-sync.sh compare` again to validate your installation is working correctly. +Assuming Gravity Sync runs successfully, it will indicate if there are changes pending between the two databases. If not, make a subtle change to a allowlist/blocklist on your primary Pi-hole, such as changing a description field or disabling a allowlist item, and then running `./gravity-sync.sh compare` again to validate your installation is working correctly. -### The Smart Sync +### The Sync The default command for Gravity Sync is simple. @@ -69,7 +74,7 @@ The default command for Gravity Sync is simple. But you can also run `./gravity-sync.sh smart` if you feel like it, and it'll do the same thing. -It will perform some checks to help insure success and then stop before making changes if it detects an issue. It will also perform the same `compare` function outlined above, and if there are no changes pending, it will exit without making an attempt to copy data. +Gravity Sync will perform some checks to help insure success and then stop before making changes if it detects an issue. It will also perform the same `compare` function outlined above, and if there are no changes pending, it will exit without making an attempt to copy data. **Example:** If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull of the `gravity.db` then push `custom.list` and finally restart the correct components on each server. It will also now only perform a sync of each component if there are changes within each type to replicate. So if you only make a small change to your Local DNS settings, it doesn't kickoff the larger `gravity.db` replication. @@ -81,39 +86,6 @@ Finally, a file called `gravity-sync.log` will be created in the `gravity-sync` You can check for successful pull attempts by running: `./gravity-sync.sh logs` -### The Pull Function -The Gravity Sync Pull, prior to version 2.0, was the standard method of sync operation, and will not prompt for user input after execution. - -```bash -./gravity-sync.sh pull -``` - -If the execution completes, you will now have overwritten your running `gravity.db` and `custom.list` on the secondary Pi-hole after creating a copy of the running files (with `.backup` appended) in the `backup` subfolder located with your script. Gravity Sync will also keep a copy of the last sync'd files from the primary (in the `backup` folder appended with `.pull`) for future use. - -### The Push Function -Gravity Sync includes the ability to `push` from the secondary Pi-hole back to the primary. This would be useful in a situation where your primary Pi-hole is down for an extended period of time, and you have made list changes on the secondary Pi-hole that you want to force back to the primary, when it comes online. - -```bash -./gravity-sync.sh push -``` - -Before executing, this will make a copy of the remote database under `backup/gravity.db.push` and `backup/custom.list.push` then sync the local configuration to the primary Pi-hole. - -This function purposefuly asks for user interaction to avoid being accidentally automated. - -- If your script prompts for a password on the remote system, make sure that your remote user account is setup not to require passwords in the sudoers file. - -### The Restore Function -Gravity Sync can also `restore` the database on the secondary Pi-hole in the event you've overwritten it accidentally. This might happen in the above scenario where you've had your primary Pi-hole down for an extended period, made changes to the secondary, but perhaps didn't get a chance to perform a `push` of the changes back to the primary, before your automated sync ran. - -```bash -./gravity-sync.sh restore -``` - -This will copy your last `gravity.db.backup` and `custom.list.backup` to the running copy on the secondary Pi-hole. - -This function purposefuly asks for user interaction to avoid being accidentally automated. - ## Updates If you'd like to know what version of the script you have running the built in version checker. It will notify you if there are updates available. @@ -156,4 +128,4 @@ Keep in mind if your cron task has never run, you will not see any valid output You can verify your existing automation entry by running `crontab -l` and see it listed at the bottom of the crontab file. If you decide to remove or change your frequency (as of version 1.8.3) you can run `./gravity-sync.sh automate` again and pick a new timing, including setting it to 0 to disable automation. ## Advanced Installation -Please review the [Advanced Installation](https://github.com/vmstan/gravity-sync/blob/master/ADVANCED.md) guide for assistance. +Please review the [Advanced Installation](https://github.com/vmstan/gravity-sync/blob/master/ADVANCED.md) guide for more assistance. diff --git a/gravity-sync.sh b/gravity-sync.sh index 708d546..1611444 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -3,7 +3,7 @@ SCRIPT_START=$SECONDS # GRAVITY SYNC BY VMSTAN ##################### PROGRAM='Gravity Sync' -VERSION='2.0.1' +VERSION='2.0.2' # Execute from the home folder of the user who owns it (ex: 'cd ~/gravity-sync') # For documentation or downloading updates visit https://github.com/vmstan/gravity-sync @@ -1021,7 +1021,7 @@ function detect_remotersync { CMD_REQUESTED="touch ~/gs.test" create_sshcmd - MESSAGE="If this fails make sure RSYNC is installed on ${REMOTE_HOST}" + MESSAGE="If pull test fails insure RSYNC is installed on ${REMOTE_HOST}" echo_warn MESSAGE="Pulling Test File to $HOSTNAME" From b629f57e3a7b3b9d834624aa72a6e09a30fe6fd1 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 9 Jul 2020 19:43:34 -0500 Subject: [PATCH 48/97] disclaimer --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3105c58..49628d8 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ What is better than a [Pi-hole](https://github.com/pi-hole/pi-hole) blocking ads via DNS on your network? That's right, Two Pi-hole! (Redundency is key in any network infrastucture.) But if you have more than one Pi-hole in your network you'll want a simple way to keep the list configurations and local DNS settings identical between the two. That's where Gravity Sync comes in. Gravity Sync will: -- Sync that the allow/blocklist configurations stored in `gravity.db` between two Pi-hole +- Sync the allow/blocklist configurations stored in `gravity.db` between two Pi-hole - Sync the local DNS settings stored in `custom.list` between two Pi-hole. - Provide an easy way to keep this happening in the background. @@ -16,7 +16,10 @@ Gravity Sync will **not**: It is suggested that you use an external DHCP server on your network (such as your router) when using multiple Pi-hole. -### Requirements +### Disclaimer +Gravity Sync is not developed by or affiliated with the Pi-hole project. This is a community effort that seeks to implement replication, which is currently not a part of the core Pi-hole product. The code has been well tested across multiple user environments but there always is an element of risk involved with running any arbitrary software you find on the Internet. + +## Requirements - Pi-hole 5.0 (or higher) be installed on at least two systems, using any of the Linux distros that Pi-hole is [certified to run on](https://docs.pi-hole.net/main/prerequesites/#supported-operating-systems). Docker deployments of Pi-hole are not supported. - You will need to make sure that you have passwordless `SUDO` enabled for the accounts on both the primary and secondary Pi-hole that will be performing the work. Most of the pre-built images available for the Raspberry Pi already have this configured, but if you have your Pi-hole running in a virtual machine built from a generic ISO, you may need to [adjust this manually](https://linuxize.com/post/how-to-run-sudo-command-without-password/). - Make sure `SSH` and `RSYNC` are installed on both the primary and secondary Pi-hole prior to installation. This is what does the leavy lifting between your Pi-hole nodes. OpenSSH is reccomended but if you're using a ultra-lightweight Pi distrbution (such as DietPi) that uses Dropbear by default, it should work as well. Other SSH client/server combonations are not supported at this time. From 248536417b0379feae6b17ffb85b1c53bb726f49 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Fri, 10 Jul 2020 21:43:05 -0500 Subject: [PATCH 49/97] backup --- ADVANCED.md | 4 ++-- CHANGELOG.md | 5 +++++ VERSION | 2 +- gravity-sync.sh | 15 ++++++++++++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index 2b019e4..4ce0455 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -14,8 +14,8 @@ Download the latest release from [GitHub](https://github.com/vmstan/gravity-sync ```bash cd ~ -wget https://github.com/vmstan/gravity-sync/archive/v2.0.2.zip -unzip v2.0.2.zip -d gravity-sync +wget https://github.com/vmstan/gravity-sync/archive/v2.1.0.zip +unzip v2.1.0.zip -d gravity-sync cd gravity-sync ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 400d621..f5e3efd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # The Changelog +## 2.1 +### The Backup Release + +A new function `./gravity-sync.sh backup` will now perform a sqlite operated backup of the `gravity.db` on the local Pi-hole. + ## 2.0 ### The Smart Release diff --git a/VERSION b/VERSION index f93ea0c..50aea0e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.2 \ No newline at end of file +2.1.0 \ No newline at end of file diff --git a/gravity-sync.sh b/gravity-sync.sh index 1611444..f7911d4 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -3,7 +3,7 @@ SCRIPT_START=$SECONDS # GRAVITY SYNC BY VMSTAN ##################### PROGRAM='Gravity Sync' -VERSION='2.0.2' +VERSION='2.1.0' # Execute from the home folder of the user who owns it (ex: 'cd ~/gravity-sync') # For documentation or downloading updates visit https://github.com/vmstan/gravity-sync @@ -1605,6 +1605,15 @@ function task_invalid { list_gs_arguments } +## Backup Task +function task_backup { + TASKTYPE='BACKUP' + MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" + echo_good + + sqlite3 ${PIHOLE_DIR}/${GRAVITY_FI} ".backup '$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/$(date +%F)-${GRAVITY_FI}'" +} + # Echo Stack ## Informative function echo_info { @@ -1800,6 +1809,10 @@ case $# in task_automate ;; + backup) + task_backup + ;; + *) task_invalid ;; From 54ba8dab347302f95903025a0dde4c71b27ec958 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Fri, 10 Jul 2020 21:47:40 -0500 Subject: [PATCH 50/97] date format --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index f7911d4..03b943c 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1611,7 +1611,7 @@ function task_backup { MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - sqlite3 ${PIHOLE_DIR}/${GRAVITY_FI} ".backup '$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/$(date +%F)-${GRAVITY_FI}'" + sqlite3 ${PIHOLE_DIR}/${GRAVITY_FI} ".backup '$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/$(date +%F-%H-%M+%S)-${GRAVITY_FI}.backup'" } # Echo Stack From 043928237f2c4c5135b30de514bc839cdbcbe524 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Fri, 10 Jul 2020 21:57:04 -0500 Subject: [PATCH 51/97] backup status --- gravity-sync.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 03b943c..d2629bb 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1611,7 +1611,19 @@ function task_backup { MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - sqlite3 ${PIHOLE_DIR}/${GRAVITY_FI} ".backup '$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/$(date +%F-%H-%M+%S)-${GRAVITY_FI}.backup'" + MESSAGE="Performing SQLITE3 Backup of ${GRAVITY_FI}" + echo_stat + + sqlite3 ${PIHOLE_DIR}/${GRAVITY_FI} ".backup '$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/$(date +%F-%H%M%S)-${GRAVITY_FI}.backup'" + error_validate + + MESSAGE="Backing Up ${CUSTOM_DNS}" + echo_stat + + cp ${PIHOLE_DIR}/${CUSTOM_DNS} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/$(date +%F-%H%M%S)-${GRAVITY_FI}.backup + error_validate + + exit_withchange } # Echo Stack From bc08ed3fa6ea41e4177b5f3f9940e7261bac151f Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Fri, 10 Jul 2020 22:13:49 -0500 Subject: [PATCH 52/97] backup retention variable --- ADVANCED.md | 5 +++++ gravity-sync.conf.example | 2 ++ gravity-sync.sh | 13 +++++++++---- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index 4ce0455..b9201c9 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -169,6 +169,11 @@ The `./gravity-sync.sh config` function will attempt to ping the remote host to Default setting in Gravity Sync is 0, change to 1 to skip this network test. +#### `BACKUP_RETAIN=''` +The `./gravity-sync.sh backup` function will retain a defined number of previous `gravity.db` and `custom.list` backups. + +Default setting in Gravity Sync is 7, adjust as resired. + ## Execution If you are just straight up unable to run the `gravity-sync.sh` file, make sure it's marked as an executable by Linux. diff --git a/gravity-sync.conf.example b/gravity-sync.conf.example index c125d38..1e55653 100644 --- a/gravity-sync.conf.example +++ b/gravity-sync.conf.example @@ -33,3 +33,5 @@ REMOTE_PASS='' # SKIP_CUSTOM='' # DATE_OUTPUT='' # PING_AVOID='' + +# BACKUP_RETAIN='' \ No newline at end of file diff --git a/gravity-sync.sh b/gravity-sync.sh index d2629bb..3980f7b 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -32,6 +32,9 @@ SKIP_CUSTOM='0' # replace in gravity-sync.conf to overwrite DATE_OUTPUT='0' # replace in gravity-sync.conf to overwrite PING_AVOID='0' # replace in gravity-sync.conf to overwrite +# Backup Customization +BACKUP_RETAIN='7' # replace in gravity-sync.conf to overwrite + # Pi-hole Folder/File Locations PIHOLE_DIR='/etc/pihole' # default Pi-hole data directory GRAVITY_FI='gravity.db' # default Pi-hole database file @@ -1611,16 +1614,18 @@ function task_backup { MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - MESSAGE="Performing SQLITE3 Backup of ${GRAVITY_FI}" + BACKUPTIMESTAMP=$(date +%F-%H%M%S) + + MESSAGE="Performing Backup of ${GRAVITY_FI}" echo_stat - sqlite3 ${PIHOLE_DIR}/${GRAVITY_FI} ".backup '$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/$(date +%F-%H%M%S)-${GRAVITY_FI}.backup'" + sqlite3 ${PIHOLE_DIR}/${GRAVITY_FI} ".backup '$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${BACKUPTIMESTAMP}-${GRAVITY_FI}.backup'" error_validate - MESSAGE="Backing Up ${CUSTOM_DNS}" + MESSAGE="Performing Backup Up ${CUSTOM_DNS}" echo_stat - cp ${PIHOLE_DIR}/${CUSTOM_DNS} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/$(date +%F-%H%M%S)-${GRAVITY_FI}.backup + cp ${PIHOLE_DIR}/${CUSTOM_DNS} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${BACKUPTIMESTAMP}-${CUSTOM_DNS}.backup error_validate exit_withchange From 693ca75715d084f97ca0a825f7d9524f89e72ae1 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Fri, 10 Jul 2020 22:19:28 -0500 Subject: [PATCH 53/97] Backup cleanup --- ADVANCED.md | 2 +- gravity-sync.sh | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ADVANCED.md b/ADVANCED.md index b9201c9..318f5a6 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -170,7 +170,7 @@ The `./gravity-sync.sh config` function will attempt to ping the remote host to Default setting in Gravity Sync is 0, change to 1 to skip this network test. #### `BACKUP_RETAIN=''` -The `./gravity-sync.sh backup` function will retain a defined number of previous `gravity.db` and `custom.list` backups. +The `./gravity-sync.sh backup` function will retain a defined number of days worth of `gravity.db` and `custom.list` backups. Default setting in Gravity Sync is 7, adjust as resired. diff --git a/gravity-sync.sh b/gravity-sync.sh index 3980f7b..ebfa19b 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1627,6 +1627,12 @@ function task_backup { cp ${PIHOLE_DIR}/${CUSTOM_DNS} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${BACKUPTIMESTAMP}-${CUSTOM_DNS}.backup error_validate + + MESSAGE="Cleaning Up Old Backups" + echo_stat + + find $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/$(date +%Y)*.backup -mtime +${BACKUP_RETAIN} -type f -delete + error_validate exit_withchange } From 2c31da31e43c185ca0ce19dee2e16409b9e76421 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Fri, 10 Jul 2020 22:32:32 -0500 Subject: [PATCH 54/97] automate backups --- gravity-sync.sh | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index ebfa19b..bc6d074 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1461,6 +1461,9 @@ function task_automate { CRON_EXIST='1' fi + MESSAGE="Configuring Hourly Smart Sync" + echo_info + MESSAGE="Sync Frequency in Minutes (1-30) or 0 to Disable" echo_need read INPUT_AUTO_FREQ @@ -1476,12 +1479,11 @@ function task_automate { then clear_cron - MESSAGE="Automation Disabled" - echo_info + MESSAGE="Sync Automation Disabled" + echo_warn else - MESSAGE="No Automation Scheduled" - echo_info - exit_nochange + MESSAGE="No Sync Automation Scheduled" + echo_warn fi else if [ $CRON_EXIST == 1 ] @@ -1489,11 +1491,35 @@ function task_automate { clear_cron fi - MESSAGE="Saving New Automation" + MESSAGE="Saving New Sync Automation" echo_stat - (crontab -l 2>/dev/null; echo "*/${INPUT_AUTO_FREQ} * * * * ${BASH_PATH} $HOME/${LOCAL_FOLDR}/${GS_FILENAME} > ${LOG_PATH}/${CRONJOB_LOG}") | crontab - + (crontab -l 2>/dev/null; echo "*/${INPUT_AUTO_FREQ} * * * * ${BASH_PATH} $HOME/${LOCAL_FOLDR}/${GS_FILENAME} smart > ${LOG_PATH}/${CRONJOB_LOG}") | crontab - error_validate fi + + MESSAGE="Configuring Daily Backup Frequency" + echo_info + + MESSAGE="Hour of Day to Backup (1-24) or 0 to Disable" + echo_need + read INPUT_AUTO_BACKUP + + if [ $INPUT_AUTO_BACKUP -gt 24 ] + then + MESSAGE="Invalid Frequency Range" + echo_fail + exit_nochange + elif [ $INPUT_AUTO_BACKUP-lt 1 ] + then + MESSAGE="No Backup Automation Scheduled" + echo_warn + else + MESSAGE="Saving New Backup Automation" + echo_stat + (crontab -l 2>/dev/null; echo "* ${INPUT_AUTO_FREQ} * * * ${BASH_PATH} $HOME/${LOCAL_FOLDR}/${GS_FILENAME} backup >/dev/null 2>&1") | crontab - + error_validate + fi + exit_withchange } From 958aa017f86569055176cfcd8bd6f8e3e7a3b648 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Fri, 10 Jul 2020 22:33:31 -0500 Subject: [PATCH 55/97] wrong variable --- gravity-sync.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index bc6d074..a01a72f 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1509,14 +1509,14 @@ function task_automate { MESSAGE="Invalid Frequency Range" echo_fail exit_nochange - elif [ $INPUT_AUTO_BACKUP-lt 1 ] + elif [ $INPUT_AUTO_BACKUP -lt 1 ] then MESSAGE="No Backup Automation Scheduled" echo_warn else MESSAGE="Saving New Backup Automation" echo_stat - (crontab -l 2>/dev/null; echo "* ${INPUT_AUTO_FREQ} * * * ${BASH_PATH} $HOME/${LOCAL_FOLDR}/${GS_FILENAME} backup >/dev/null 2>&1") | crontab - + (crontab -l 2>/dev/null; echo "* ${INPUT_AUTO_BACKUP} * * * ${BASH_PATH} $HOME/${LOCAL_FOLDR}/${GS_FILENAME} backup >/dev/null 2>&1") | crontab - error_validate fi From a284d3b4d28826c1c96816b052b20c9bdf8269bb Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Fri, 10 Jul 2020 22:58:43 -0500 Subject: [PATCH 56/97] restore function --- CHANGELOG.md | 4 +++- gravity-sync.sh | 40 +++++++++++++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5e3efd..888f972 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,9 @@ ## 2.1 ### The Backup Release -A new function `./gravity-sync.sh backup` will now perform a sqlite operated backup of the `gravity.db` on the local Pi-hole. +A new function `./gravity-sync.sh backup` will now perform a sqlite operated backup of the `gravity.db` on the local Pi-hole. This can be run at any time you wish, but can also be automated by the `./gravity-sync.sh automate` function to run once a day. New and existing users will be prompted to configure both during this task. If can also disable both using the automate function, or just automate one or the other, by setting the value to `0` during setup. + +By default, 7 days worth of backups will be retained in the `backup` folder. You can adjust the retention length by changing the `BACKUP_RETAIN` function in your `gravity-sync.conf` file. See the `ADVANCED.md` file for more information on setting these custom configuration options. ## 2.0 ### The Smart Release diff --git a/gravity-sync.sh b/gravity-sync.sh index a01a72f..2931326 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -529,14 +529,36 @@ function smart_gs { } function restore_gs { - MESSAGE="This will restore ${GRAVITY_FI} on $HOSTNAME with the previous version!" + MESSAGE="This will restore your settings on $HOSTNAME with a previous version!" echo_warn + MESSAGE="PREVIOUS BACKUPS" + echo_info + ls $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD} | grep $(date +%Y) | grep ${GRAVITY_FI} | colrm 18 + + MESSAGE="Enter the date you want to restore from" + echo_need + read INPUT_BACKUP_DATE + + if [ -f $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_BACKUP_DATE}-${GRAVITY_FI}.backup ] + then + MESSAGE="Backup File Located" + echo_info + else + MESSAGE="Invalid Requested" + fi + intent_validate + MESSAGE="Stopping Pi-hole Services" + echo_stat + + sudo service pihole-FTL stop >/dev/null 2>&1 + error_validate + MESSAGE="Restoring ${GRAVITY_FI} on $HOSTNAME" echo_stat - sudo cp $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${GRAVITY_FI}.backup ${PIHOLE_DIR}/${GRAVITY_FI} >/dev/null 2>&1 + sudo cp $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_BACKUP_DATE}-${GRAVITY_FI}.backup ${PIHOLE_DIR}/${GRAVITY_FI} >/dev/null 2>&1 error_validate MESSAGE="Validating Ownership on ${GRAVITY_FI}" @@ -579,11 +601,11 @@ function restore_gs { if [ "$SKIP_CUSTOM" != '1' ] then - if [ -f $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${CUSTOM_DNS}.backup ] + if [ -f $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_BACKUP_DATE}-${CUSTOM_DNS}.backup ] then MESSAGE="Restoring ${CUSTOM_DNS} on $HOSTNAME" echo_stat - sudo cp $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${CUSTOM_DNS}.backup ${PIHOLE_DIR}/${CUSTOM_DNS} >/dev/null 2>&1 + sudo cp $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_BACKUP_DATE}-${CUSTOM_DNS}.backup ${PIHOLE_DIR}/${CUSTOM_DNS} >/dev/null 2>&1 error_validate MESSAGE="Validating Ownership on ${CUSTOM_DNS}" @@ -629,17 +651,17 @@ function restore_gs { MESSAGE="Evacuating Saucer Section" echo_info sleep 1 + + MESSAGE="Restarting FTLDNS Services" + echo_stat + sudo service pihole-FTL start >/dev/null 2>&1 + error_validate MESSAGE="Updating FTLDNS Configuration" echo_stat ${PIHOLE_BIN} restartdns reloadlists >/dev/null 2>&1 error_validate - MESSAGE="Reloading FTLDNS Services" - echo_stat - ${PIHOLE_BIN} restartdns >/dev/null 2>&1 - error_validate - logs_export exit_withchange } From 87f1a0fd3770aebc7a669821455718f03ee90f37 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Fri, 10 Jul 2020 23:15:28 -0500 Subject: [PATCH 57/97] Push back --- CHANGELOG.md | 2 ++ gravity-sync.sh | 20 ++++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 888f972..62dafa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ A new function `./gravity-sync.sh backup` will now perform a sqlite operated bac By default, 7 days worth of backups will be retained in the `backup` folder. You can adjust the retention length by changing the `BACKUP_RETAIN` function in your `gravity-sync.conf` file. See the `ADVANCED.md` file for more information on setting these custom configuration options. +There are also enhancements to the `./gravity-sync.sh restore` function, where as previously this task would only restore the previous copy of the database that is made during sync operations, now this will ask you to select a previous backup copy (by date) and will use that file to restore. This will stop the Pi-hole services on the local server while the task is completed. You will now also be prompted to perform a push operation of the restored database to the primary Pi-hole server. + ## 2.0 ### The Smart Release diff --git a/gravity-sync.sh b/gravity-sync.sh index 2931326..0ef8f7b 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -662,8 +662,24 @@ function restore_gs { ${PIHOLE_BIN} restartdns reloadlists >/dev/null 2>&1 error_validate - logs_export - exit_withchange + + MESSAGE="Do you want to push the restored configuration to the primary Pi-hole? (yes/no)" + echo_need + read PUSH_TO_PRIMARY + + if [ "${PUSH_TO_PRIMARY}" == "Yes" ] || [ "${PUSH_TO_PRIMARY}" == "yes" ] || [ "${PUSH_TO_PRIMARY}" == "Y" ] || [ "${PUSH_TO_PRIMARY}" == "y" ] + then + push_gs + elif [ "${PUSH_TO_PRIMARY}" == "No" ] || [ "${PUSH_TO_PRIMARY}" == "no" ] || [ "${PUSH_TO_PRIMARY}" == "N" ] || [ "${PUSH_TO_PRIMARY}" == "n" ] + logs_export + exit_withchange + else + MESSAGE="Invalid Selection - Defaulting No" + echo_warn + + logs_export + exit_withchange + fi } # Logging Functions From 63cf87c1dc2c9d084612a5b43c3e5814099c1996 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Fri, 10 Jul 2020 23:15:51 -0500 Subject: [PATCH 58/97] then --- gravity-sync.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/gravity-sync.sh b/gravity-sync.sh index 0ef8f7b..9d45125 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -671,6 +671,7 @@ function restore_gs { then push_gs elif [ "${PUSH_TO_PRIMARY}" == "No" ] || [ "${PUSH_TO_PRIMARY}" == "no" ] || [ "${PUSH_TO_PRIMARY}" == "N" ] || [ "${PUSH_TO_PRIMARY}" == "n" ] + then logs_export exit_withchange else From 2f57ab6d695584283d5a2cc2d7cfa96cbb618054 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Fri, 10 Jul 2020 23:34:54 -0500 Subject: [PATCH 59/97] warn for dbclient --- CHANGELOG.md | 14 ++++++++++++-- gravity-sync.sh | 16 +++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62dafa3..750e6a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,11 +3,21 @@ ## 2.1 ### The Backup Release -A new function `./gravity-sync.sh backup` will now perform a sqlite operated backup of the `gravity.db` on the local Pi-hole. This can be run at any time you wish, but can also be automated by the `./gravity-sync.sh automate` function to run once a day. New and existing users will be prompted to configure both during this task. If can also disable both using the automate function, or just automate one or the other, by setting the value to `0` during setup. +A new function `./gravity-sync.sh backup` will now perform a `SQLITE3` operated backup of the `gravity.db` on the local Pi-hole. This can be run at any time you wish, but can also be automated by the `./gravity-sync.sh automate` function to run once a day. New and existing users will be prompted to configure both during this task. If can also disable both using the automate function, or just automate one or the other, by setting the value to `0` during setup. + +New users will automatically have their local settings backed up after completion of the initial setup, before the first run of any sync tasks. By default, 7 days worth of backups will be retained in the `backup` folder. You can adjust the retention length by changing the `BACKUP_RETAIN` function in your `gravity-sync.conf` file. See the `ADVANCED.md` file for more information on setting these custom configuration options. -There are also enhancements to the `./gravity-sync.sh restore` function, where as previously this task would only restore the previous copy of the database that is made during sync operations, now this will ask you to select a previous backup copy (by date) and will use that file to restore. This will stop the Pi-hole services on the local server while the task is completed. You will now also be prompted to perform a push operation of the restored database to the primary Pi-hole server. +There are also enhancements to the `./gravity-sync.sh restore` function, where as previously this task would only restore the previous copy of the database that is made during sync operations, now this will ask you to select a previous backup copy (by date) and will use that file to restore. This will stop the Pi-hole services on the local server while the task is completed. After a successful restoration, you will now also be prompted to perform a `push` operation of the restored database to the primary Pi-hole server. + +It's suggested to make sure your local restore was successful before completing the `restore` operation with the `push` job. + +#### Deprecation + +Support for the the Dropbear SSH client/server (which was added in 1.7.6) will be removed in an upcoming version of Gravity Sync. If you are using this instead of OpenSSH (common with DietPi) please reconfigure your installation to use OpenSSH. You will want to delete your existing `~/.ssh/id_rsa` and `~/.ssh/id_rsa.pub` files and run `./gravity-sync.sh configure` again to generate a new key and copy it to the primary Pi-hole. + +The `./gravity-sync.sh update` and `version` functions will look for the `dbclient` binary on the local system and warn users to change. ## 2.0 ### The Smart Release diff --git a/gravity-sync.sh b/gravity-sync.sh index 9d45125..4cf0328 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1377,6 +1377,8 @@ function config_generate { validate_os_sshpass detect_remotersync + + task_backup exit_withchange } @@ -1473,13 +1475,23 @@ function show_version { else if [ "$GITVERSION" != "$VERSION" ] then - MESSAGE="Upgrade Available: ${PURPLE}${GITVERSION}${NC}" + MESSAGE="Update Available: ${PURPLE}${GITVERSION}${NC}" else MESSAGE="Latest Version: ${GREEN}${GITVERSION}${NC}" fi fi echo_info echo -e "========================================================" + + dbclient_warning +} + +function dbclient_warning { + if hash dbclient 2>/dev/null + then + MESSAGE="Dropbear support has been deprecated - please convert to OpenSSH" + echo_warn + fi } # Task Stack @@ -1622,6 +1634,8 @@ function task_update { TASKTYPE='UPDATE' MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good + + dbclient_warning update_gs } From ac7a897f10d60985cb9cd99c444a238582ba6d68 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 08:50:32 -0500 Subject: [PATCH 60/97] 2.0.1 --- CHANGELOG.md | 2 +- gravity-sync.sh | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 750e6a7..b91cd9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ It's suggested to make sure your local restore was successful before completing Support for the the Dropbear SSH client/server (which was added in 1.7.6) will be removed in an upcoming version of Gravity Sync. If you are using this instead of OpenSSH (common with DietPi) please reconfigure your installation to use OpenSSH. You will want to delete your existing `~/.ssh/id_rsa` and `~/.ssh/id_rsa.pub` files and run `./gravity-sync.sh configure` again to generate a new key and copy it to the primary Pi-hole. -The `./gravity-sync.sh update` and `version` functions will look for the `dbclient` binary on the local system and warn users to change. +The `./gravity-sync.sh update` and `version` functions will look for the `dbclient` binary on the local system and warn users about the upcoming changes. ## 2.0 ### The Smart Release diff --git a/gravity-sync.sh b/gravity-sync.sh index 4cf0328..e211fb6 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1701,11 +1701,17 @@ function task_backup { sqlite3 ${PIHOLE_DIR}/${GRAVITY_FI} ".backup '$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${BACKUPTIMESTAMP}-${GRAVITY_FI}.backup'" error_validate - MESSAGE="Performing Backup Up ${CUSTOM_DNS}" - echo_stat + if [ "$SKIP_CUSTOM" != '1' ] + then + if [ -f ${PIHOLE_DIR}/${CUSTOM_DNS} ] + then + MESSAGE="Performing Backup Up ${CUSTOM_DNS}" + echo_stat - cp ${PIHOLE_DIR}/${CUSTOM_DNS} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${BACKUPTIMESTAMP}-${CUSTOM_DNS}.backup - error_validate + cp ${PIHOLE_DIR}/${CUSTOM_DNS} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${BACKUPTIMESTAMP}-${CUSTOM_DNS}.backup + error_validate + fi + fi MESSAGE="Cleaning Up Old Backups" echo_stat From 4eacab6fa7bf9cda8b40d23ec11f74d247449715 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 08:53:04 -0500 Subject: [PATCH 61/97] 2.1.1 --- ADVANCED.md | 4 ++-- CHANGELOG.md | 3 +++ VERSION | 2 +- gravity-sync.sh | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index 318f5a6..edf8fab 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -14,8 +14,8 @@ Download the latest release from [GitHub](https://github.com/vmstan/gravity-sync ```bash cd ~ -wget https://github.com/vmstan/gravity-sync/archive/v2.1.0.zip -unzip v2.1.0.zip -d gravity-sync +wget https://github.com/vmstan/gravity-sync/archive/v2.1.1.zip +unzip v2.1.1.zip -d gravity-sync cd gravity-sync ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index b91cd9c..aeaa724 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ Support for the the Dropbear SSH client/server (which was added in 1.7.6) will b The `./gravity-sync.sh update` and `version` functions will look for the `dbclient` binary on the local system and warn users about the upcoming changes. +#### 2.1.1 +- Last release was incorrectly published without logic to ignore `custom.list` if request or not used. + ## 2.0 ### The Smart Release diff --git a/VERSION b/VERSION index 50aea0e..7c32728 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.0 \ No newline at end of file +2.1.1 \ No newline at end of file diff --git a/gravity-sync.sh b/gravity-sync.sh index e211fb6..5e6d141 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -3,7 +3,7 @@ SCRIPT_START=$SECONDS # GRAVITY SYNC BY VMSTAN ##################### PROGRAM='Gravity Sync' -VERSION='2.1.0' +VERSION='2.1.1' # Execute from the home folder of the user who owns it (ex: 'cd ~/gravity-sync') # For documentation or downloading updates visit https://github.com/vmstan/gravity-sync From b1312e043f004e3c58d5d8bf7003abb5ec3d85a1 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 08:53:45 -0500 Subject: [PATCH 62/97] 2.1.1 (#57) * 2.1.1 Co-authored-by: Michael Stanclift --- ADVANCED.md | 4 ++-- CHANGELOG.md | 5 ++++- VERSION | 2 +- gravity-sync.sh | 16 +++++++++++----- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index 318f5a6..edf8fab 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -14,8 +14,8 @@ Download the latest release from [GitHub](https://github.com/vmstan/gravity-sync ```bash cd ~ -wget https://github.com/vmstan/gravity-sync/archive/v2.1.0.zip -unzip v2.1.0.zip -d gravity-sync +wget https://github.com/vmstan/gravity-sync/archive/v2.1.1.zip +unzip v2.1.1.zip -d gravity-sync cd gravity-sync ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 750e6a7..aeaa724 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,10 @@ It's suggested to make sure your local restore was successful before completing Support for the the Dropbear SSH client/server (which was added in 1.7.6) will be removed in an upcoming version of Gravity Sync. If you are using this instead of OpenSSH (common with DietPi) please reconfigure your installation to use OpenSSH. You will want to delete your existing `~/.ssh/id_rsa` and `~/.ssh/id_rsa.pub` files and run `./gravity-sync.sh configure` again to generate a new key and copy it to the primary Pi-hole. -The `./gravity-sync.sh update` and `version` functions will look for the `dbclient` binary on the local system and warn users to change. +The `./gravity-sync.sh update` and `version` functions will look for the `dbclient` binary on the local system and warn users about the upcoming changes. + +#### 2.1.1 +- Last release was incorrectly published without logic to ignore `custom.list` if request or not used. ## 2.0 ### The Smart Release diff --git a/VERSION b/VERSION index 50aea0e..7c32728 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.0 \ No newline at end of file +2.1.1 \ No newline at end of file diff --git a/gravity-sync.sh b/gravity-sync.sh index 4cf0328..5e6d141 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -3,7 +3,7 @@ SCRIPT_START=$SECONDS # GRAVITY SYNC BY VMSTAN ##################### PROGRAM='Gravity Sync' -VERSION='2.1.0' +VERSION='2.1.1' # Execute from the home folder of the user who owns it (ex: 'cd ~/gravity-sync') # For documentation or downloading updates visit https://github.com/vmstan/gravity-sync @@ -1701,11 +1701,17 @@ function task_backup { sqlite3 ${PIHOLE_DIR}/${GRAVITY_FI} ".backup '$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${BACKUPTIMESTAMP}-${GRAVITY_FI}.backup'" error_validate - MESSAGE="Performing Backup Up ${CUSTOM_DNS}" - echo_stat + if [ "$SKIP_CUSTOM" != '1' ] + then + if [ -f ${PIHOLE_DIR}/${CUSTOM_DNS} ] + then + MESSAGE="Performing Backup Up ${CUSTOM_DNS}" + echo_stat - cp ${PIHOLE_DIR}/${CUSTOM_DNS} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${BACKUPTIMESTAMP}-${CUSTOM_DNS}.backup - error_validate + cp ${PIHOLE_DIR}/${CUSTOM_DNS} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${BACKUPTIMESTAMP}-${CUSTOM_DNS}.backup + error_validate + fi + fi MESSAGE="Cleaning Up Old Backups" echo_stat From 39aea0d602d9025e60bab4bd56c154665b699550 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 10:04:40 -0500 Subject: [PATCH 63/97] 2.1.2 --- ADVANCED.md | 4 ++-- CHANGELOG.md | 3 +++ VERSION | 2 +- gravity-sync.sh | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index edf8fab..0e46f01 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -14,8 +14,8 @@ Download the latest release from [GitHub](https://github.com/vmstan/gravity-sync ```bash cd ~ -wget https://github.com/vmstan/gravity-sync/archive/v2.1.1.zip -unzip v2.1.1.zip -d gravity-sync +wget https://github.com/vmstan/gravity-sync/archive/v2.1.2.zip +unzip v2.1.2.zip -d gravity-sync cd gravity-sync ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index aeaa724..2a10d90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ The `./gravity-sync.sh update` and `version` functions will look for the `dbclie #### 2.1.1 - Last release was incorrectly published without logic to ignore `custom.list` if request or not used. +#### 2.1.2 +- Corrects a bug in `backup` automation that causes the backup to run every minute during the hour selected. + ## 2.0 ### The Smart Release diff --git a/VERSION b/VERSION index 7c32728..8f9174b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.1 \ No newline at end of file +2.1.2 \ No newline at end of file diff --git a/gravity-sync.sh b/gravity-sync.sh index 5e6d141..b0980ea 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -3,7 +3,7 @@ SCRIPT_START=$SECONDS # GRAVITY SYNC BY VMSTAN ##################### PROGRAM='Gravity Sync' -VERSION='2.1.1' +VERSION='2.1.2' # Execute from the home folder of the user who owns it (ex: 'cd ~/gravity-sync') # For documentation or downloading updates visit https://github.com/vmstan/gravity-sync @@ -1567,7 +1567,7 @@ function task_automate { else MESSAGE="Saving New Backup Automation" echo_stat - (crontab -l 2>/dev/null; echo "* ${INPUT_AUTO_BACKUP} * * * ${BASH_PATH} $HOME/${LOCAL_FOLDR}/${GS_FILENAME} backup >/dev/null 2>&1") | crontab - + (crontab -l 2>/dev/null; echo "0 ${INPUT_AUTO_BACKUP} * * * ${BASH_PATH} $HOME/${LOCAL_FOLDR}/${GS_FILENAME} backup >/dev/null 2>&1") | crontab - error_validate fi From e14588add54c319f56677fd45c19df8c27cc6549 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 14:42:52 -0500 Subject: [PATCH 64/97] breakup into functions --- ADVANCED.md | 4 ++-- CHANGELOG.md | 3 +++ VERSION | 2 +- gravity-sync.sh | 33 +++++++++++++++++++++++++-------- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index 0e46f01..3c63b5a 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -14,8 +14,8 @@ Download the latest release from [GitHub](https://github.com/vmstan/gravity-sync ```bash cd ~ -wget https://github.com/vmstan/gravity-sync/archive/v2.1.2.zip -unzip v2.1.2.zip -d gravity-sync +wget https://github.com/vmstan/gravity-sync/archive/v2.1.5.zip +unzip v2.1.5.zip -d gravity-sync cd gravity-sync ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a10d90..7a813c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,9 @@ The `./gravity-sync.sh update` and `version` functions will look for the `dbclie #### 2.1.2 - Corrects a bug in `backup` automation that causes the backup to run every minute during the hour selected. +#### 2.1.5 +Skipping a few digits because what does it really matter? + ## 2.0 ### The Smart Release diff --git a/VERSION b/VERSION index 8f9174b..c5864dc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.2 \ No newline at end of file +2.1.5 \ No newline at end of file diff --git a/gravity-sync.sh b/gravity-sync.sh index b0980ea..ea27732 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -3,7 +3,7 @@ SCRIPT_START=$SECONDS # GRAVITY SYNC BY VMSTAN ##################### PROGRAM='Gravity Sync' -VERSION='2.1.2' +VERSION='2.1.5' # Execute from the home folder of the user who owns it (ex: 'cd ~/gravity-sync') # For documentation or downloading updates visit https://github.com/vmstan/gravity-sync @@ -141,10 +141,13 @@ function update_gs { # Gravity Core Functions ## Pull Gravity function pull_gs_grav { - MESSAGE="Backing Up ${GRAVITY_FI} on $HOSTNAME" - echo_stat - cp ${PIHOLE_DIR}/${GRAVITY_FI} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${GRAVITY_FI}.backup >/dev/null 2>&1 - error_validate + + backup_local_gravity + + # MESSAGE="Backing Up ${GRAVITY_FI} on $HOSTNAME" + # echo_stat + # cp ${PIHOLE_DIR}/${GRAVITY_FI} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${GRAVITY_FI}.backup >/dev/null 2>&1 + # error_validate MESSAGE="Pulling ${GRAVITY_FI} from ${REMOTE_HOST}" echo_stat @@ -291,6 +294,7 @@ function pull_gs_reload { function pull_gs { md5_compare + backup_settime pull_gs_grav pull_gs_cust pull_gs_reload @@ -1693,14 +1697,27 @@ function task_backup { MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - BACKUPTIMESTAMP=$(date +%F-%H%M%S) + backup_settime + backup_local_gravity + backup_local_custom + backup_cleanup + + exit_withchange +} +function backup_settime { + BACKUPTIMESTAMP=$(date +%F-%H%M%S) +} + +function backup_local_gravity { MESSAGE="Performing Backup of ${GRAVITY_FI}" echo_stat sqlite3 ${PIHOLE_DIR}/${GRAVITY_FI} ".backup '$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${BACKUPTIMESTAMP}-${GRAVITY_FI}.backup'" error_validate +} +function backup_local_custom { if [ "$SKIP_CUSTOM" != '1' ] then if [ -f ${PIHOLE_DIR}/${CUSTOM_DNS} ] @@ -1712,14 +1729,14 @@ function task_backup { error_validate fi fi +} +function backup_cleanup { MESSAGE="Cleaning Up Old Backups" echo_stat find $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/$(date +%Y)*.backup -mtime +${BACKUP_RETAIN} -type f -delete error_validate - - exit_withchange } # Echo Stack From 9b7d536977c4a4ed83d48df2d16090d9085acc5d Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 14:55:03 -0500 Subject: [PATCH 65/97] Updates --- gravity-sync.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index ea27732..190d1fd 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -205,17 +205,19 @@ function pull_gs_grav { ## Pull Custom function pull_gs_cust { + backup_local_custom + if [ "$SKIP_CUSTOM" != '1' ] then if [ "$REMOTE_CUSTOM_DNS" == "1" ] then - if [ -f ${PIHOLE_DIR}/${CUSTOM_DNS} ] - then - MESSAGE="Backing Up ${CUSTOM_DNS} on $HOSTNAME" - echo_stat - cp ${PIHOLE_DIR}/${CUSTOM_DNS} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${CUSTOM_DNS}.backup >/dev/null 2>&1 - error_validate - fi + # if [ -f ${PIHOLE_DIR}/${CUSTOM_DNS} ] + # then + # MESSAGE="Backing Up ${CUSTOM_DNS} on $HOSTNAME" + # echo_stat + # cp ${PIHOLE_DIR}/${CUSTOM_DNS} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${CUSTOM_DNS}.backup >/dev/null 2>&1 + # error_validate + # fi MESSAGE="Pulling ${CUSTOM_DNS} from ${REMOTE_HOST}" echo_stat @@ -403,6 +405,7 @@ function push_gs { ## Smart Sync Function function smart_gs { md5_compare + backup_settime if [ -f "${LOG_PATH}/${HISTORY_MD5}" ] then From 78f97716a857eb9cd170860d58c798e21ca3fcc5 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 15:17:27 -0500 Subject: [PATCH 66/97] backup remote grav --- gravity-sync.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/gravity-sync.sh b/gravity-sync.sh index 190d1fd..8f2585b 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -148,6 +148,8 @@ function pull_gs_grav { # echo_stat # cp ${PIHOLE_DIR}/${GRAVITY_FI} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${GRAVITY_FI}.backup >/dev/null 2>&1 # error_validate + + MESSAGE="Pulling ${GRAVITY_FI} from ${REMOTE_HOST}" echo_stat @@ -205,6 +207,7 @@ function pull_gs_grav { ## Pull Custom function pull_gs_cust { + backup_local_custom if [ "$SKIP_CUSTOM" != '1' ] @@ -307,6 +310,8 @@ function pull_gs { ## Push Gravity function push_gs_grav { + backup_remote_gravity + MESSAGE="Backing Up ${GRAVITY_FI} from ${REMOTE_HOST}" echo_stat RSYNC_REPATH="rsync" @@ -1734,6 +1739,16 @@ function backup_local_custom { fi } +function backup_remote_gravity { + CMD_TIMEOUT='15' + CMD_REQUESTED="sqlite3 ${PIHOLE_DIR}/${GRAVITY_FI} \".backup '${PIHOLE_DIR}/${GRAVITY_FI}.backup'\"" + create_sshcmd +} + +function backup_remote_custom { + +} + function backup_cleanup { MESSAGE="Cleaning Up Old Backups" echo_stat From 624ab9aab70fbc86da8bf80a5b165110fbabddb2 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 15:33:35 -0500 Subject: [PATCH 67/97] New backup and copy method --- gravity-sync.sh | 44 ++++++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 8f2585b..e4fb55d 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -143,6 +143,7 @@ function update_gs { function pull_gs_grav { backup_local_gravity + backup_remote_gravity # MESSAGE="Backing Up ${GRAVITY_FI} on $HOSTNAME" # echo_stat @@ -154,7 +155,7 @@ function pull_gs_grav { MESSAGE="Pulling ${GRAVITY_FI} from ${REMOTE_HOST}" echo_stat RSYNC_REPATH="rsync" - RSYNC_SOURCE="${REMOTE_USER}@${REMOTE_HOST}:${PIHOLE_DIR}/${GRAVITY_FI}" + RSYNC_SOURCE="${REMOTE_USER}@${REMOTE_HOST}:${PIHOLE_DIR}/${GRAVITY_FI}.backup" RSYNC_TARGET="$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${GRAVITY_FI}.pull" create_rsynccmd @@ -209,6 +210,7 @@ function pull_gs_grav { function pull_gs_cust { backup_local_custom + backup_remote_custom if [ "$SKIP_CUSTOM" != '1' ] then @@ -225,7 +227,7 @@ function pull_gs_cust { MESSAGE="Pulling ${CUSTOM_DNS} from ${REMOTE_HOST}" echo_stat RSYNC_REPATH="rsync" - RSYNC_SOURCE="${REMOTE_USER}@${REMOTE_HOST}:${PIHOLE_DIR}/${CUSTOM_DNS}" + RSYNC_SOURCE="${REMOTE_USER}@${REMOTE_HOST}:${PIHOLE_DIR}/${CUSTOM_DNS}.backup" RSYNC_TARGET="$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${CUSTOM_DNS}.pull" create_rsynccmd @@ -311,18 +313,19 @@ function pull_gs { ## Push Gravity function push_gs_grav { backup_remote_gravity + backup_local_gravity - MESSAGE="Backing Up ${GRAVITY_FI} from ${REMOTE_HOST}" + MESSAGE="Copying ${GRAVITY_FI} from ${REMOTE_HOST}" echo_stat RSYNC_REPATH="rsync" - RSYNC_SOURCE="${REMOTE_USER}@${REMOTE_HOST}:${PIHOLE_DIR}/${GRAVITY_FI}" + RSYNC_SOURCE="${REMOTE_USER}@${REMOTE_HOST}:${PIHOLE_DIR}/${GRAVITY_FI}.backup" RSYNC_TARGET="$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${GRAVITY_FI}.push" create_rsynccmd MESSAGE="Pushing ${GRAVITY_FI} to ${REMOTE_HOST}" echo_stat RSYNC_REPATH="sudo rsync" - RSYNC_SOURCE="${PIHOLE_DIR}/${GRAVITY_FI}" + RSYNC_SOURCE="$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${BACKUPTIMESTAMP}-${GRAVITY_FI}.backup" RSYNC_TARGET="${REMOTE_USER}@${REMOTE_HOST}:${PIHOLE_DIR}/${GRAVITY_FI}" create_rsynccmd @@ -341,21 +344,24 @@ function push_gs_grav { ## Push Custom function push_gs_cust { + backup_remote_custom + backup_local_custom + if [ "$SKIP_CUSTOM" != '1' ] then if [ "$REMOTE_CUSTOM_DNS" == "1" ] then - MESSAGE="Backing Up ${CUSTOM_DNS} from ${REMOTE_HOST}" + MESSAGE="Copying ${CUSTOM_DNS} from ${REMOTE_HOST}" echo_stat RSYNC_REPATH="rsync" - RSYNC_SOURCE="${REMOTE_USER}@${REMOTE_HOST}:${PIHOLE_DIR}/${CUSTOM_DNS}" + RSYNC_SOURCE="${REMOTE_USER}@${REMOTE_HOST}:${PIHOLE_DIR}/${CUSTOM_DNS}.backup" RSYNC_TARGET="$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${CUSTOM_DNS}.push" create_rsynccmd MESSAGE="Pushing ${CUSTOM_DNS} to ${REMOTE_HOST}" echo_stat RSYNC_REPATH="sudo rsync" - RSYNC_SOURCE="${PIHOLE_DIR}/${CUSTOM_DNS}" + RSYNC_SOURCE="$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${BACKUPTIMESTAMP}-${CUSTOM_DNS}.backup" RSYNC_TARGET="${REMOTE_USER}@${REMOTE_HOST}:${PIHOLE_DIR}/${CUSTOM_DNS}" create_rsynccmd @@ -1718,7 +1724,7 @@ function backup_settime { } function backup_local_gravity { - MESSAGE="Performing Backup of ${GRAVITY_FI}" + MESSAGE="Performing Backup of Local ${GRAVITY_FI}" echo_stat sqlite3 ${PIHOLE_DIR}/${GRAVITY_FI} ".backup '$HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${BACKUPTIMESTAMP}-${GRAVITY_FI}.backup'" @@ -1730,7 +1736,7 @@ function backup_local_custom { then if [ -f ${PIHOLE_DIR}/${CUSTOM_DNS} ] then - MESSAGE="Performing Backup Up ${CUSTOM_DNS}" + MESSAGE="Performing Backup Up Local ${CUSTOM_DNS}" echo_stat cp ${PIHOLE_DIR}/${CUSTOM_DNS} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${BACKUPTIMESTAMP}-${CUSTOM_DNS}.backup @@ -1740,13 +1746,27 @@ function backup_local_custom { } function backup_remote_gravity { + MESSAGE="Performing Backup of Remote ${GRAVITY_FI}" + echo_stat + CMD_TIMEOUT='15' - CMD_REQUESTED="sqlite3 ${PIHOLE_DIR}/${GRAVITY_FI} \".backup '${PIHOLE_DIR}/${GRAVITY_FI}.backup'\"" + CMD_REQUESTED="sudo sqlite3 ${PIHOLE_DIR}/${GRAVITY_FI} \".backup '${PIHOLE_DIR}/${GRAVITY_FI}.backup'\"" create_sshcmd } function backup_remote_custom { - + if [ "$SKIP_CUSTOM" != '1' ] + then + if [ -f ${PIHOLE_DIR}/${CUSTOM_DNS} ] + then + MESSAGE="Performing Backup of Remote ${CUSTOM_DNS}" + echo_stat + + CMD_TIMEOUT='15' + CMD_REQUESTED="sudo cp ${PIHOLE_DIR}/${CUSTOM_DNS} ${PIHOLE_DIR}/${CUSTOM_DNS}.backup'\"" + create_sshcmd + fi + fi } function backup_cleanup { From a158a2d0426f69941575178887ae07d22fd4ec19 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 18:34:22 -0500 Subject: [PATCH 68/97] betamode --- gravity-sync.sh | 50 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/gravity-sync.sh b/gravity-sync.sh index e4fb55d..04d6ad9 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1634,6 +1634,17 @@ function task_devmode { echo_stat rm -f $HOME/${LOCAL_FOLDR}/dev error_validate + elif [ -f $HOME/${LOCAL_FOLDR}/beta ] + then + MESSAGE="Disabling BETA" + echo_stat + rm -f $HOME/${LOCAL_FOLDR}/beta + error_validate + + MESSAGE="Enabling ${TASKTYPE}" + echo_stat + touch $HOME/${LOCAL_FOLDR}/dev + error_validate else MESSAGE="Enabling ${TASKTYPE}" echo_stat @@ -1647,6 +1658,42 @@ function task_devmode { exit_withchange } +## Devmode Task +function task_betamode { + TASKTYPE='BETA' + MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" + echo_good + + if [ -f $HOME/${LOCAL_FOLDR}/beta ] + then + MESSAGE="Disabling ${TASKTYPE}" + echo_stat + rm -f $HOME/${LOCAL_FOLDR}/beta + error_validate + elif [ -f $HOME/${LOCAL_FOLDR}/dev ] + then + MESSAGE="Disabling DEV" + echo_stat + rm -f $HOME/${LOCAL_FOLDR}/dev + error_validate + + MESSAGE="Enabling ${TASKTYPE}" + echo_stat + touch $HOME/${LOCAL_FOLDR}/beta + error_validate + else + MESSAGE="Enabling ${TASKTYPE}" + echo_stat + touch $HOME/${LOCAL_FOLDR}/beta + error_validate + fi + + MESSAGE="Run UPDATE to apply changes" + echo_info + + exit_withchange +} + ## Update Task function task_update { TASKTYPE='UPDATE' @@ -1936,6 +1983,9 @@ case $# in task_devmode ;; + beta) + task_betamode + devmode) task_devmode ;; From 668475c103bf5854ac1722690543fbc0bfea280e Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 18:42:22 -0500 Subject: [PATCH 69/97] beta mode in changelog --- CHANGELOG.md | 3 +++ gravity-sync.sh | 1 + 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a813c6..7a505a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,9 @@ The `./gravity-sync.sh update` and `version` functions will look for the `dbclie #### 2.1.5 Skipping a few digits because what does it really matter? +- Implements a new beta branch, and with it a new `./gravity-sync.sh beta` function to enable it. This will hopefully allow new features and such to be added for test users who can adopt them and provide feedback before rolling out to the main update branch. +- Uses new SQLITE3 backup methodology introduced in 2.1, for all push/pull sync operations. + ## 2.0 ### The Smart Release diff --git a/gravity-sync.sh b/gravity-sync.sh index 04d6ad9..b4e3737 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1985,6 +1985,7 @@ case $# in beta) task_betamode + ;; devmode) task_devmode From 0617525e64ad7e07b77c0f2c441c646643f60007 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 18:46:29 -0500 Subject: [PATCH 70/97] extra quotes --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index b4e3737..bbc57ed 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1810,7 +1810,7 @@ function backup_remote_custom { echo_stat CMD_TIMEOUT='15' - CMD_REQUESTED="sudo cp ${PIHOLE_DIR}/${CUSTOM_DNS} ${PIHOLE_DIR}/${CUSTOM_DNS}.backup'\"" + CMD_REQUESTED="sudo cp ${PIHOLE_DIR}/${CUSTOM_DNS} ${PIHOLE_DIR}/${CUSTOM_DNS}.backup" create_sshcmd fi fi From ac8e6389983aec64643b5051efb085eb4c7489f3 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 20:48:51 -0500 Subject: [PATCH 71/97] adjust md5 hash checking --- gravity-sync.sh | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index bbc57ed..2a76be5 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -299,12 +299,14 @@ function pull_gs_reload { ## Pull Function function pull_gs { + previous_md5 md5_compare backup_settime pull_gs_grav pull_gs_cust pull_gs_reload + md5_recheck logs_export exit_withchange @@ -401,6 +403,7 @@ function push_gs_reload { ## Push Function function push_gs { + previous_md5 md5_compare intent_validate @@ -413,11 +416,7 @@ function push_gs { exit_withchange } -## Smart Sync Function -function smart_gs { - md5_compare - backup_settime - +function previous_md5 { if [ -f "${LOG_PATH}/${HISTORY_MD5}" ] then last_primaryDBMD5=$(sed "1q;d" ${LOG_PATH}/${HISTORY_MD5}) @@ -430,6 +429,13 @@ function smart_gs { last_primaryCLMD5="0" last_secondCLMD5="0" fi +} + +## Smart Sync Function +function smart_gs { + previous_md5 + md5_compare + backup_settime PRIDBCHANGE="0" SECDBCHANGE="0" @@ -1115,6 +1121,11 @@ function error_validate { ## Validate Sync Required function md5_compare { + # last_primaryDBMD5="0" + # last_secondDBMD5="0" + # last_primaryCLMD5="0" + # last_secondCLMD5="0" + HASHMARK='0' MESSAGE="Analyzing ${GRAVITY_FI} on ${REMOTE_HOST}" @@ -1127,7 +1138,7 @@ function md5_compare { secondDBMD5=$(md5sum ${PIHOLE_DIR}/${GRAVITY_FI} | sed 's/\s.*$//') error_validate - if [ "$primaryDBMD5" == "$secondDBMD5" ] + if [ "$primaryDBMD5" == "$last_primaryDBMD5" ] && [ "$secondDBMD5" == "$last_secondDBMD5" ] then HASHMARK=$((HASHMARK+0)) else @@ -1154,7 +1165,7 @@ function md5_compare { secondCLMD5=$(md5sum ${PIHOLE_DIR}/${CUSTOM_DNS} | sed 's/\s.*$//') error_validate - if [ "$primaryCLMD5" == "$secondCLMD5" ] + if [ "$primaryCLMD5" == "$last_primaryCLMD5" ] && [ "$secondCLMD5" == "$last_secondCLMD5" ] then # MESSAGE="${CUSTOM_DNS} Identical" # echo_info From 985ce623557bfcd76eaeb80fdbaeb3eb0cb70270 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 20:50:59 -0500 Subject: [PATCH 72/97] settime --- gravity-sync.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gravity-sync.sh b/gravity-sync.sh index 2a76be5..0279805 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -405,6 +405,7 @@ function push_gs_reload { function push_gs { previous_md5 md5_compare + backup_settime intent_validate @@ -412,6 +413,7 @@ function push_gs { push_gs_cust push_gs_reload + md5_recheck logs_export exit_withchange } From 90ea491d9ec0f7d221632818ec0af5ef5af5bfed Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 21:01:21 -0500 Subject: [PATCH 73/97] comment out warning on recheck --- gravity-sync.sh | 50 ++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 0279805..59e4dab 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1222,14 +1222,14 @@ function md5_recheck { secondDBMD5=$(md5sum ${PIHOLE_DIR}/${GRAVITY_FI} | sed 's/\s.*$//') error_validate - if [ "$primaryDBMD5" == "$secondDBMD5" ] - then - HASHMARK=$((HASHMARK+0)) - else - MESSAGE="Differenced ${GRAVITY_FI} Detected" - echo_warn - HASHMARK=$((HASHMARK+1)) - fi + # if [ "$primaryDBMD5" == "$secondDBMD5" ] + # then + # HASHMARK=$((HASHMARK+0)) + # else + # MESSAGE="Differenced ${GRAVITY_FI} Detected" + # echo_warn + # HASHMARK=$((HASHMARK+1)) + # fi if [ "$SKIP_CUSTOM" != '1' ] then @@ -1249,16 +1249,16 @@ function md5_recheck { secondCLMD5=$(md5sum ${PIHOLE_DIR}/${CUSTOM_DNS} | sed 's/\s.*$//') error_validate - if [ "$primaryCLMD5" == "$secondCLMD5" ] - then + # if [ "$primaryCLMD5" == "$secondCLMD5" ] + # then # MESSAGE="${CUSTOM_DNS} Identical" # echo_info - HASHMARK=$((HASHMARK+0)) - else - MESSAGE="Differenced ${CUSTOM_DNS} Detected" - echo_warn - HASHMARK=$((HASHMARK+1)) - fi + # HASHMARK=$((HASHMARK+0)) + # else + # MESSAGE="Differenced ${CUSTOM_DNS} Detected" + # echo_warn + # HASHMARK=$((HASHMARK+1)) + # fi else MESSAGE="No ${CUSTOM_DNS} Detected on ${REMOTE_HOST}" echo_info @@ -1268,7 +1268,7 @@ function md5_recheck { then REMOTE_CUSTOM_DNS="1" MESSAGE="${REMOTE_HOST} has ${CUSTOM_DNS}" - HASHMARK=$((HASHMARK+1)) + # HASHMARK=$((HASHMARK+1)) echo_info fi MESSAGE="No ${CUSTOM_DNS} Detected on $HOSTNAME" @@ -1276,14 +1276,14 @@ function md5_recheck { fi fi - if [ "$HASHMARK" != "0" ] - then - MESSAGE="Replication Checks Failed" - echo_warn - else - MESSAGE="Replication Was Successful" - echo_info - fi + # if [ "$HASHMARK" != "0" ] + # then + # MESSAGE="Replication Checks Failed" + # echo_warn + # else + # MESSAGE="Replication Was Successful" + # echo_info + # fi } ## Validate Intent From bc756641295f54a145423dcb876de8c64c6fabb0 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 21:06:06 -0500 Subject: [PATCH 74/97] load previous md5 for compare --- gravity-sync.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 59e4dab..be52d0b 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1747,7 +1747,8 @@ function task_compare { validate_gs_folders validate_ph_folders validate_os_sshpass - + + previous_md5 md5_compare } From 8e78bca575299674791b64419ac80a69868ab01d Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 21:12:21 -0500 Subject: [PATCH 75/97] beta updater --- gravity-sync.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index be52d0b..7e702f7 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -108,6 +108,9 @@ function update_gs { if [ -f "$HOME/${LOCAL_FOLDR}/dev" ] then BRANCH='development' + elif if [ -f "$HOME/${LOCAL_FOLDR}/beta" ] + then + BRANCH='beta' else BRANCH='master' fi @@ -1492,6 +1495,9 @@ function show_version { if [ -f $HOME/${LOCAL_FOLDR}/dev ] then DEVVERSION="dev" + elif [ -f $HOME/${LOCAL_FOLDR}/beta ] + then + DEVVERSION="beta" else DEVVERSION="" fi @@ -1513,8 +1519,6 @@ function show_version { fi echo_info echo -e "========================================================" - - dbclient_warning } function dbclient_warning { From ad1f758384af59e4c48bc42e8b27856c4afccb41 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sat, 11 Jul 2020 21:15:01 -0500 Subject: [PATCH 76/97] if --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 7e702f7..fd22e34 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -108,7 +108,7 @@ function update_gs { if [ -f "$HOME/${LOCAL_FOLDR}/dev" ] then BRANCH='development' - elif if [ -f "$HOME/${LOCAL_FOLDR}/beta" ] + elif [ -f "$HOME/${LOCAL_FOLDR}/beta" ] then BRANCH='beta' else From d4276042b5a8f31e97b98a0916eb2a59007e21e1 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sun, 12 Jul 2020 09:20:58 -0500 Subject: [PATCH 77/97] select different files for restore --- CHANGELOG.md | 1 + gravity-sync.sh | 38 +++++++++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a505a3..46e5ffb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ Skipping a few digits because what does it really matter? - Implements a new beta branch, and with it a new `./gravity-sync.sh beta` function to enable it. This will hopefully allow new features and such to be added for test users who can adopt them and provide feedback before rolling out to the main update branch. - Uses new SQLITE3 backup methodology introduced in 2.1, for all push/pull sync operations. +- `./gravity-sync.sh restore` lets you select a different `gravity.db` and `custom.list` for restoration. ## 2.0 ### The Smart Release diff --git a/gravity-sync.sh b/gravity-sync.sh index fd22e34..14c3609 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -565,18 +565,46 @@ function restore_gs { echo_info ls $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD} | grep $(date +%Y) | grep ${GRAVITY_FI} | colrm 18 - MESSAGE="Enter the date you want to restore from" + MESSAGE="Enter the backup you want to restore ${GRAVITY_FI} from" echo_need read INPUT_BACKUP_DATE if [ -f $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_BACKUP_DATE}-${GRAVITY_FI}.backup ] then - MESSAGE="Backup File Located" + MESSAGE="Backup File Selected" echo_info else - MESSAGE="Invalid Requested" + MESSAGE="Invalid Request" + echo_info fi + if [ "$SKIP_CUSTOM" != '1' ] + then + + if [ -f ${PIHOLE_DIR}/${CUSTOM_DNS} ] + then + ls $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD} | grep $(date +%Y) | grep ${CUSTOM_DNS} | colrm 18 + + MESSAGE="Enter the backup you want to restore ${CUSTOM_DNS} from" + echo_need + read INPUT_DNSBACKUP_DATE + + if [ -f $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_DNSBACKUP_DATE}-${CUSTOM_DNS}.backup ] + then + MESSAGE="Backup File Selected" + echo_info + else + MESSAGE="Invalid Request" + echo_info + fi + fi + fi + + MESSAGE="${GRAVITY_FI} from ${INPUT_BACKUP_DATE} Selected" + echo_info + MESSAGE="${CUSTOM_DNS} from ${INPUT_DNSBACKUP_DATE} Selected" + echo_info + intent_validate MESSAGE="Stopping Pi-hole Services" @@ -630,11 +658,11 @@ function restore_gs { if [ "$SKIP_CUSTOM" != '1' ] then - if [ -f $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_BACKUP_DATE}-${CUSTOM_DNS}.backup ] + if [ -f $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_DNSBACKUP_DATE}-${CUSTOM_DNS}.backup ] then MESSAGE="Restoring ${CUSTOM_DNS} on $HOSTNAME" echo_stat - sudo cp $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_BACKUP_DATE}-${CUSTOM_DNS}.backup ${PIHOLE_DIR}/${CUSTOM_DNS} >/dev/null 2>&1 + sudo cp $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_DNSBACKUP_DATE}-${CUSTOM_DNS}.backup ${PIHOLE_DIR}/${CUSTOM_DNS} >/dev/null 2>&1 error_validate MESSAGE="Validating Ownership on ${CUSTOM_DNS}" From dd5e7a63b5db105169b34b767abb5948d7d0292a Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sun, 12 Jul 2020 09:24:45 -0500 Subject: [PATCH 78/97] star trek reference --- CHANGELOG.md | 1 + gravity-sync.sh | 15 ++++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46e5ffb..56abe28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ Skipping a few digits because what does it really matter? - Implements a new beta branch, and with it a new `./gravity-sync.sh beta` function to enable it. This will hopefully allow new features and such to be added for test users who can adopt them and provide feedback before rolling out to the main update branch. - Uses new SQLITE3 backup methodology introduced in 2.1, for all push/pull sync operations. - `./gravity-sync.sh restore` lets you select a different `gravity.db` and `custom.list` for restoration. +- One new Star Trek reference. ## 2.0 ### The Smart Release diff --git a/gravity-sync.sh b/gravity-sync.sh index 14c3609..bc7e85f 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -561,21 +561,22 @@ function restore_gs { MESSAGE="This will restore your settings on $HOSTNAME with a previous version!" echo_warn - MESSAGE="PREVIOUS BACKUPS" + MESSAGE="PREVIOUS BACKUPS AVAILABLE FOR RESTORATION" echo_info ls $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD} | grep $(date +%Y) | grep ${GRAVITY_FI} | colrm 18 - MESSAGE="Enter the backup you want to restore ${GRAVITY_FI} from" + MESSAGE="Select backup date to restore ${GRAVITY_FI} from" echo_need read INPUT_BACKUP_DATE if [ -f $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_BACKUP_DATE}-${GRAVITY_FI}.backup ] then MESSAGE="Backup File Selected" - echo_info else MESSAGE="Invalid Request" echo_info + + exit_nochange fi if [ "$SKIP_CUSTOM" != '1' ] @@ -585,17 +586,18 @@ function restore_gs { then ls $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD} | grep $(date +%Y) | grep ${CUSTOM_DNS} | colrm 18 - MESSAGE="Enter the backup you want to restore ${CUSTOM_DNS} from" + MESSAGE="Select backup date to restore ${CUSTOM_DNS} from" echo_need read INPUT_DNSBACKUP_DATE if [ -f $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_DNSBACKUP_DATE}-${CUSTOM_DNS}.backup ] then MESSAGE="Backup File Selected" - echo_info else MESSAGE="Invalid Request" echo_info + + exit_nochange fi fi fi @@ -607,6 +609,9 @@ function restore_gs { intent_validate + MESSAGE="Making Time Warp Calculations" + echo_info + MESSAGE="Stopping Pi-hole Services" echo_stat From e2acade06300165258c6a65f964ee8a490e21ff9 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sun, 12 Jul 2020 10:26:07 -0500 Subject: [PATCH 79/97] Log backup jobs --- CHANGELOG.md | 1 + gravity-sync.sh | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56abe28..94996e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ Skipping a few digits because what does it really matter? - Uses new SQLITE3 backup methodology introduced in 2.1, for all push/pull sync operations. - `./gravity-sync.sh restore` lets you select a different `gravity.db` and `custom.list` for restoration. - One new Star Trek reference. +- `./gravity-sync.sh restore` now shows recent complete Backup executions. ## 2.0 ### The Smart Release diff --git a/gravity-sync.sh b/gravity-sync.sh index bc7e85f..4632284 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -749,6 +749,9 @@ function restore_gs { ## Core Logging ### Write Logs Out function logs_export { + + if [ "${TASKTYPE}" != "BACKUP" ] + then MESSAGE="Saving File Hashes" echo_stat rm -f ${LOG_PATH}/${HISTORY_MD5} @@ -757,6 +760,7 @@ function logs_export { echo -e ${primaryCLMD5} >> ${LOG_PATH}/${HISTORY_MD5} echo -e ${secondCLMD5} >> ${LOG_PATH}/${HISTORY_MD5} error_validate + fi MESSAGE="Logging Successful ${TASKTYPE}" echo_stat @@ -778,6 +782,8 @@ function logs_gs { tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep PULL echo -e "Recent Complete ${YELLOW}PUSH${NC} Executions" tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep PUSH + echo -e "Recent Complete ${YELLOW}BACKUP${NC} Executions" + tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep BACKUP echo -e "Recent Complete ${YELLOW}RESTORE${NC} Executions" tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep RESTORE echo -e "========================================================" @@ -1814,6 +1820,7 @@ function task_backup { backup_local_custom backup_cleanup + logs_export exit_withchange } From fb23d48296e2973b36e144f67a567f41f47fab83 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sun, 12 Jul 2020 10:27:15 -0500 Subject: [PATCH 80/97] color restore --- gravity-sync.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 4632284..fbb4a63 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -602,9 +602,9 @@ function restore_gs { fi fi - MESSAGE="${GRAVITY_FI} from ${INPUT_BACKUP_DATE} Selected" + MESSAGE="${GRAVITY_FI} from ${CYAN}${INPUT_BACKUP_DATE}${NC} Selected" echo_info - MESSAGE="${CUSTOM_DNS} from ${INPUT_DNSBACKUP_DATE} Selected" + MESSAGE="${CUSTOM_DNS} from ${CYAN}${INPUT_DNSBACKUP_DATE}${NC} Selected" echo_info intent_validate From a994652b2bb4319d1c89bb89a01f3b6340eeeba1 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 16 Jul 2020 11:37:06 -0500 Subject: [PATCH 81/97] ping avoid and custom ssh prompts --- ADVANCED.md | 4 ++-- CHANGELOG.md | 3 +++ VERSION | 2 +- gravity-sync.sh | 39 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index 3c63b5a..2f6d3ae 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -14,8 +14,8 @@ Download the latest release from [GitHub](https://github.com/vmstan/gravity-sync ```bash cd ~ -wget https://github.com/vmstan/gravity-sync/archive/v2.1.5.zip -unzip v2.1.5.zip -d gravity-sync +wget https://github.com/vmstan/gravity-sync/archive/v2.1.6.zip +unzip v2.1.6.zip -d gravity-sync cd gravity-sync ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 94996e4..2c1ec65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,9 @@ Skipping a few digits because what does it really matter? - One new Star Trek reference. - `./gravity-sync.sh restore` now shows recent complete Backup executions. +#### 2.1.6 +- Adds prompts during `./gravity-sync.sh configure` to allow custom SSH port and enable PING avoidance. + ## 2.0 ### The Smart Release diff --git a/VERSION b/VERSION index c5864dc..b6da512 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.5 \ No newline at end of file +2.1.6 \ No newline at end of file diff --git a/gravity-sync.sh b/gravity-sync.sh index fbb4a63..8c8afed 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -3,7 +3,7 @@ SCRIPT_START=$SECONDS # GRAVITY SYNC BY VMSTAN ##################### PROGRAM='Gravity Sync' -VERSION='2.1.5' +VERSION='2.1.6' # Execute from the home folder of the user who owns it (ex: 'cd ~/gravity-sync') # For documentation or downloading updates visit https://github.com/vmstan/gravity-sync @@ -602,9 +602,9 @@ function restore_gs { fi fi - MESSAGE="${GRAVITY_FI} from ${CYAN}${INPUT_BACKUP_DATE}${NC} Selected" + MESSAGE="${GRAVITY_FI} from ${INPUT_BACKUP_DATE} Selected" echo_info - MESSAGE="${CUSTOM_DNS} from ${CYAN}${INPUT_DNSBACKUP_DATE}${NC} Selected" + MESSAGE="${CUSTOM_DNS} from ${INPUT_DNSBACKUP_DATE} Selected" echo_info intent_validate @@ -1373,13 +1373,46 @@ function config_generate { echo_stat cp $HOME/${LOCAL_FOLDR}/${CONFIG_FILE}.example $HOME/${LOCAL_FOLDR}/${CONFIG_FILE} error_validate + + MESSAGE="Environment Customization" + echo_info + + MESSAGE="Are you using a custom SSH Port? (Leave blank for default '22')" + echo_need + read INPUT_SSH_PORT + + if [ "${INPUT_SSH_PORT}" != "" ] || [ "${INPUT_SSH_PORT}" != "22" ] + then + MESSAGE="Saving Custom SSH Port to ${CONFIG_FILE}" + echo_stat + sed -i "/# SSH_PORT=''/c\SSH_PORT='${INPUT_SSH_PORT}'" $HOME/${LOCAL_FOLDR}/${CONFIG_FILE} + error_validate + fi + + MESSAGE="Perform PING tests between Pi-holes? (Leave blank for default 'Yes')" + echo_need + read INPUT_PING_AVOID + + if [ "${INPUT_PING_AVOID}" != "" ] || [ "${INPUT_PING_AVOID}" != "Yes" ] || [ "${INPUT_PING_AVOID}" != "yes" ] || [ "${INPUT_PING_AVOID}" != "Y" ] || [ "${INPUT_PING_AVOID}" != "y" ] + then + MESSAGE="Saving Ping Avoidance to ${CONFIG_FILE}" + echo_stat + sed -i "/# PING_AVOID=''/c\PING_AVOID='1'" $HOME/${LOCAL_FOLDR}/${CONFIG_FILE} + error_validate + PING_AVOID=1 + fi + MESSAGE="Standard Settings" + echo_info + MESSAGE="IP or DNS of Primary Pi-hole" echo_need read INPUT_REMOTE_HOST if [ "${PING_AVOID}" != "1" ] then + + MESSAGE="Testing Network Connection (PING)" echo_stat ping -c 3 ${INPUT_REMOTE_HOST} >/dev/null 2>&1 From 1092db143e1765a45742a623dd261c04f0d457af Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 16 Jul 2020 11:44:20 -0500 Subject: [PATCH 82/97] z variable --- gravity-sync.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 8c8afed..438ddc0 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1377,11 +1377,11 @@ function config_generate { MESSAGE="Environment Customization" echo_info - MESSAGE="Are you using a custom SSH Port? (Leave blank for default '22')" + MESSAGE="Enter a custom SSH port if required (Leave blank for default '22')" echo_need read INPUT_SSH_PORT - if [ "${INPUT_SSH_PORT}" != "" ] || [ "${INPUT_SSH_PORT}" != "22" ] + if [ -z "${INPUT_SSH_PORT}" ] || [ "${INPUT_SSH_PORT}" != "22" ] then MESSAGE="Saving Custom SSH Port to ${CONFIG_FILE}" echo_stat @@ -1393,7 +1393,7 @@ function config_generate { echo_need read INPUT_PING_AVOID - if [ "${INPUT_PING_AVOID}" != "" ] || [ "${INPUT_PING_AVOID}" != "Yes" ] || [ "${INPUT_PING_AVOID}" != "yes" ] || [ "${INPUT_PING_AVOID}" != "Y" ] || [ "${INPUT_PING_AVOID}" != "y" ] + if [ -z "${INPUT_PING_AVOID}" ] || [ "${INPUT_PING_AVOID}" != "Yes" ] || [ "${INPUT_PING_AVOID}" != "yes" ] || [ "${INPUT_PING_AVOID}" != "Y" ] || [ "${INPUT_PING_AVOID}" != "y" ] then MESSAGE="Saving Ping Avoidance to ${CONFIG_FILE}" echo_stat From 719d2bf0777c1dba3b0f6f4f87cb38f455e4afee Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 16 Jul 2020 11:49:12 -0500 Subject: [PATCH 83/97] or --- gravity-sync.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 438ddc0..7368896 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1381,7 +1381,7 @@ function config_generate { echo_need read INPUT_SSH_PORT - if [ -z "${INPUT_SSH_PORT}" ] || [ "${INPUT_SSH_PORT}" != "22" ] + if [[ -z "${INPUT_SSH_PORT}" || "${INPUT_SSH_PORT}" != "22" ]] then MESSAGE="Saving Custom SSH Port to ${CONFIG_FILE}" echo_stat @@ -1393,7 +1393,7 @@ function config_generate { echo_need read INPUT_PING_AVOID - if [ -z "${INPUT_PING_AVOID}" ] || [ "${INPUT_PING_AVOID}" != "Yes" ] || [ "${INPUT_PING_AVOID}" != "yes" ] || [ "${INPUT_PING_AVOID}" != "Y" ] || [ "${INPUT_PING_AVOID}" != "y" ] + if [[ -z "${INPUT_PING_AVOID}" || [ "${INPUT_PING_AVOID}" != "Yes" || "${INPUT_PING_AVOID}" != "yes" || "${INPUT_PING_AVOID}" != "Y" || [ "${INPUT_PING_AVOID}" != "y" ]] then MESSAGE="Saving Ping Avoidance to ${CONFIG_FILE}" echo_stat From 86805ab02dd4f658a0b3593e5c3b090c8939d8c5 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 16 Jul 2020 11:50:45 -0500 Subject: [PATCH 84/97] [] --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 7368896..3c4b746 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1393,7 +1393,7 @@ function config_generate { echo_need read INPUT_PING_AVOID - if [[ -z "${INPUT_PING_AVOID}" || [ "${INPUT_PING_AVOID}" != "Yes" || "${INPUT_PING_AVOID}" != "yes" || "${INPUT_PING_AVOID}" != "Y" || [ "${INPUT_PING_AVOID}" != "y" ]] + if [[ -z "${INPUT_PING_AVOID}" || "${INPUT_PING_AVOID}" != "Yes" || "${INPUT_PING_AVOID}" != "yes" || "${INPUT_PING_AVOID}" != "Y" || "${INPUT_PING_AVOID}" != "y" ]] then MESSAGE="Saving Ping Avoidance to ${CONFIG_FILE}" echo_stat From c73b3c9dfc0aafc26f06952a1e712aba12716d4f Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 16 Jul 2020 11:57:02 -0500 Subject: [PATCH 85/97] new method for defaults --- gravity-sync.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 3c4b746..5a5add2 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1380,8 +1380,9 @@ function config_generate { MESSAGE="Enter a custom SSH port if required (Leave blank for default '22')" echo_need read INPUT_SSH_PORT + INPUT_SSH_PORT="${INPUT_SSH_PORT:-22}" - if [[ -z "${INPUT_SSH_PORT}" || "${INPUT_SSH_PORT}" != "22" ]] + if [ "${INPUT_SSH_PORT}" != "22" ] then MESSAGE="Saving Custom SSH Port to ${CONFIG_FILE}" echo_stat @@ -1392,8 +1393,9 @@ function config_generate { MESSAGE="Perform PING tests between Pi-holes? (Leave blank for default 'Yes')" echo_need read INPUT_PING_AVOID + INPUT_SSH_PORT="${INPUT_SSH_PORT:-Y}" - if [[ -z "${INPUT_PING_AVOID}" || "${INPUT_PING_AVOID}" != "Yes" || "${INPUT_PING_AVOID}" != "yes" || "${INPUT_PING_AVOID}" != "Y" || "${INPUT_PING_AVOID}" != "y" ]] + if [[ "${INPUT_PING_AVOID}" != "Yes" || "${INPUT_PING_AVOID}" != "yes" || "${INPUT_PING_AVOID}" != "Y" || "${INPUT_PING_AVOID}" != "y" ]] then MESSAGE="Saving Ping Avoidance to ${CONFIG_FILE}" echo_stat From 8c5dadb5601aca0aa2ccda1eb031e97407a66506 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 16 Jul 2020 11:59:02 -0500 Subject: [PATCH 86/97] variables matter --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 5a5add2..e22d4b8 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1393,7 +1393,7 @@ function config_generate { MESSAGE="Perform PING tests between Pi-holes? (Leave blank for default 'Yes')" echo_need read INPUT_PING_AVOID - INPUT_SSH_PORT="${INPUT_SSH_PORT:-Y}" + INPUT_PING_AVOID="${INPUT_PING_AVOID:-Y}" if [[ "${INPUT_PING_AVOID}" != "Yes" || "${INPUT_PING_AVOID}" != "yes" || "${INPUT_PING_AVOID}" != "Y" || "${INPUT_PING_AVOID}" != "y" ]] then From 2d7fa277ad6c6cc3c266c878ee23f87766ad5753 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 16 Jul 2020 12:02:48 -0500 Subject: [PATCH 87/97] or this? --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index e22d4b8..47ec631 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1395,7 +1395,7 @@ function config_generate { read INPUT_PING_AVOID INPUT_PING_AVOID="${INPUT_PING_AVOID:-Y}" - if [[ "${INPUT_PING_AVOID}" != "Yes" || "${INPUT_PING_AVOID}" != "yes" || "${INPUT_PING_AVOID}" != "Y" || "${INPUT_PING_AVOID}" != "y" ]] + if [ "${INPUT_PING_AVOID}" != "Yes" ] || [ "${INPUT_PING_AVOID}" != "yes" ] || [ "${INPUT_PING_AVOID}" != "Y" ] || [ "${INPUT_PING_AVOID}" != "y" ] then MESSAGE="Saving Ping Avoidance to ${CONFIG_FILE}" echo_stat From 70da7aad1470d8d0375fb33d215eebbf2e3d198c Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 16 Jul 2020 12:04:34 -0500 Subject: [PATCH 88/97] making it specific --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 47ec631..06bf34a 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1395,7 +1395,7 @@ function config_generate { read INPUT_PING_AVOID INPUT_PING_AVOID="${INPUT_PING_AVOID:-Y}" - if [ "${INPUT_PING_AVOID}" != "Yes" ] || [ "${INPUT_PING_AVOID}" != "yes" ] || [ "${INPUT_PING_AVOID}" != "Y" ] || [ "${INPUT_PING_AVOID}" != "y" ] + if [ "${INPUT_PING_AVOID}" != "Y" ] then MESSAGE="Saving Ping Avoidance to ${CONFIG_FILE}" echo_stat From 6cb842a12bd11f6ce037f47ac637b6f68f75dddf Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 16 Jul 2020 12:25:07 -0500 Subject: [PATCH 89/97] 2.1.6 beta 1 (#65) * select different files for restore * star trek reference * Log backup jobs * color restore * ping avoid and custom ssh prompts * z variable * or * [] * new method for defaults * variables matter * or this? * making it specific Co-authored-by: Michael Stanclift --- ADVANCED.md | 4 +-- CHANGELOG.md | 6 ++++ VERSION | 2 +- gravity-sync.sh | 91 ++++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 92 insertions(+), 11 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index 3c63b5a..2f6d3ae 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -14,8 +14,8 @@ Download the latest release from [GitHub](https://github.com/vmstan/gravity-sync ```bash cd ~ -wget https://github.com/vmstan/gravity-sync/archive/v2.1.5.zip -unzip v2.1.5.zip -d gravity-sync +wget https://github.com/vmstan/gravity-sync/archive/v2.1.6.zip +unzip v2.1.6.zip -d gravity-sync cd gravity-sync ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a505a3..2c1ec65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,12 @@ Skipping a few digits because what does it really matter? - Implements a new beta branch, and with it a new `./gravity-sync.sh beta` function to enable it. This will hopefully allow new features and such to be added for test users who can adopt them and provide feedback before rolling out to the main update branch. - Uses new SQLITE3 backup methodology introduced in 2.1, for all push/pull sync operations. +- `./gravity-sync.sh restore` lets you select a different `gravity.db` and `custom.list` for restoration. +- One new Star Trek reference. +- `./gravity-sync.sh restore` now shows recent complete Backup executions. + +#### 2.1.6 +- Adds prompts during `./gravity-sync.sh configure` to allow custom SSH port and enable PING avoidance. ## 2.0 ### The Smart Release diff --git a/VERSION b/VERSION index c5864dc..b6da512 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.5 \ No newline at end of file +2.1.6 \ No newline at end of file diff --git a/gravity-sync.sh b/gravity-sync.sh index fd22e34..06bf34a 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -3,7 +3,7 @@ SCRIPT_START=$SECONDS # GRAVITY SYNC BY VMSTAN ##################### PROGRAM='Gravity Sync' -VERSION='2.1.5' +VERSION='2.1.6' # Execute from the home folder of the user who owns it (ex: 'cd ~/gravity-sync') # For documentation or downloading updates visit https://github.com/vmstan/gravity-sync @@ -561,24 +561,57 @@ function restore_gs { MESSAGE="This will restore your settings on $HOSTNAME with a previous version!" echo_warn - MESSAGE="PREVIOUS BACKUPS" + MESSAGE="PREVIOUS BACKUPS AVAILABLE FOR RESTORATION" echo_info ls $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD} | grep $(date +%Y) | grep ${GRAVITY_FI} | colrm 18 - MESSAGE="Enter the date you want to restore from" + MESSAGE="Select backup date to restore ${GRAVITY_FI} from" echo_need read INPUT_BACKUP_DATE if [ -f $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_BACKUP_DATE}-${GRAVITY_FI}.backup ] then - MESSAGE="Backup File Located" - echo_info + MESSAGE="Backup File Selected" else - MESSAGE="Invalid Requested" + MESSAGE="Invalid Request" + echo_info + + exit_nochange fi + if [ "$SKIP_CUSTOM" != '1' ] + then + + if [ -f ${PIHOLE_DIR}/${CUSTOM_DNS} ] + then + ls $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD} | grep $(date +%Y) | grep ${CUSTOM_DNS} | colrm 18 + + MESSAGE="Select backup date to restore ${CUSTOM_DNS} from" + echo_need + read INPUT_DNSBACKUP_DATE + + if [ -f $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_DNSBACKUP_DATE}-${CUSTOM_DNS}.backup ] + then + MESSAGE="Backup File Selected" + else + MESSAGE="Invalid Request" + echo_info + + exit_nochange + fi + fi + fi + + MESSAGE="${GRAVITY_FI} from ${INPUT_BACKUP_DATE} Selected" + echo_info + MESSAGE="${CUSTOM_DNS} from ${INPUT_DNSBACKUP_DATE} Selected" + echo_info + intent_validate + MESSAGE="Making Time Warp Calculations" + echo_info + MESSAGE="Stopping Pi-hole Services" echo_stat @@ -630,11 +663,11 @@ function restore_gs { if [ "$SKIP_CUSTOM" != '1' ] then - if [ -f $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_BACKUP_DATE}-${CUSTOM_DNS}.backup ] + if [ -f $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_DNSBACKUP_DATE}-${CUSTOM_DNS}.backup ] then MESSAGE="Restoring ${CUSTOM_DNS} on $HOSTNAME" echo_stat - sudo cp $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_BACKUP_DATE}-${CUSTOM_DNS}.backup ${PIHOLE_DIR}/${CUSTOM_DNS} >/dev/null 2>&1 + sudo cp $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${INPUT_DNSBACKUP_DATE}-${CUSTOM_DNS}.backup ${PIHOLE_DIR}/${CUSTOM_DNS} >/dev/null 2>&1 error_validate MESSAGE="Validating Ownership on ${CUSTOM_DNS}" @@ -716,6 +749,9 @@ function restore_gs { ## Core Logging ### Write Logs Out function logs_export { + + if [ "${TASKTYPE}" != "BACKUP" ] + then MESSAGE="Saving File Hashes" echo_stat rm -f ${LOG_PATH}/${HISTORY_MD5} @@ -724,6 +760,7 @@ function logs_export { echo -e ${primaryCLMD5} >> ${LOG_PATH}/${HISTORY_MD5} echo -e ${secondCLMD5} >> ${LOG_PATH}/${HISTORY_MD5} error_validate + fi MESSAGE="Logging Successful ${TASKTYPE}" echo_stat @@ -745,6 +782,8 @@ function logs_gs { tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep PULL echo -e "Recent Complete ${YELLOW}PUSH${NC} Executions" tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep PUSH + echo -e "Recent Complete ${YELLOW}BACKUP${NC} Executions" + tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep BACKUP echo -e "Recent Complete ${YELLOW}RESTORE${NC} Executions" tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep RESTORE echo -e "========================================================" @@ -1334,13 +1373,48 @@ function config_generate { echo_stat cp $HOME/${LOCAL_FOLDR}/${CONFIG_FILE}.example $HOME/${LOCAL_FOLDR}/${CONFIG_FILE} error_validate + + MESSAGE="Environment Customization" + echo_info + + MESSAGE="Enter a custom SSH port if required (Leave blank for default '22')" + echo_need + read INPUT_SSH_PORT + INPUT_SSH_PORT="${INPUT_SSH_PORT:-22}" + + if [ "${INPUT_SSH_PORT}" != "22" ] + then + MESSAGE="Saving Custom SSH Port to ${CONFIG_FILE}" + echo_stat + sed -i "/# SSH_PORT=''/c\SSH_PORT='${INPUT_SSH_PORT}'" $HOME/${LOCAL_FOLDR}/${CONFIG_FILE} + error_validate + fi + + MESSAGE="Perform PING tests between Pi-holes? (Leave blank for default 'Yes')" + echo_need + read INPUT_PING_AVOID + INPUT_PING_AVOID="${INPUT_PING_AVOID:-Y}" + + if [ "${INPUT_PING_AVOID}" != "Y" ] + then + MESSAGE="Saving Ping Avoidance to ${CONFIG_FILE}" + echo_stat + sed -i "/# PING_AVOID=''/c\PING_AVOID='1'" $HOME/${LOCAL_FOLDR}/${CONFIG_FILE} + error_validate + PING_AVOID=1 + fi + MESSAGE="Standard Settings" + echo_info + MESSAGE="IP or DNS of Primary Pi-hole" echo_need read INPUT_REMOTE_HOST if [ "${PING_AVOID}" != "1" ] then + + MESSAGE="Testing Network Connection (PING)" echo_stat ping -c 3 ${INPUT_REMOTE_HOST} >/dev/null 2>&1 @@ -1781,6 +1855,7 @@ function task_backup { backup_local_custom backup_cleanup + logs_export exit_withchange } From f6cdbc40ec9d60c3352cf5af74847a05597877c3 Mon Sep 17 00:00:00 2001 From: fbourqui Date: Thu, 16 Jul 2020 20:09:44 +0200 Subject: [PATCH 90/97] adv conf: add ROOT_CHECK_AVOID for container. (#64) * adv conf: add ROOT_CHECK_AVOID for container. import_gs only once at start * allow parameters for automation option Co-authored-by: Michael Stanclift --- ADVANCED.md | 18 ++++++++++- gravity-sync.conf.example | 1 + gravity-sync.sh | 65 ++++++++++++++++++++++++++------------- 3 files changed, 61 insertions(+), 23 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index 2f6d3ae..e6d9374 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -169,6 +169,11 @@ The `./gravity-sync.sh config` function will attempt to ping the remote host to Default setting in Gravity Sync is 0, change to 1 to skip this network test. +#### `ROOT_CHECK_AVOID=''` +The `./gravity-sync.sh` check that it's deployed with it's own user, but for container deployment it's a nuisance, to install a user and gave it passwordless sudo power, witch result with the same security risk as running as root. + +Default setting in Gravity Sync is 0, change to 1 to skip this root user test. + #### `BACKUP_RETAIN=''` The `./gravity-sync.sh backup` function will retain a defined number of days worth of `gravity.db` and `custom.list` backups. @@ -212,11 +217,22 @@ If your code is still not updating after this, reinstallation is suggested rathe ## Automation There are many automation methods available to run scripts on a regular basis of a Linux system. The one built into all of them is cron, but if you'd like to utilize something different then the principles are still the same. +To automate the deployment of automation option, you can call it with 2 parameters: +first interval in minutes to run sync [0-30], +then the hour to run backup [0-24] +Note: 0 will disable the cron entry. + +for example: +gravity-sync.sh automate 15 23 + +will sync every 15 minutes and backup at 23:00. + If you prefer to still use cron but modify your settings by hand, using the entry below will cause the entry to run at the top and bottom of every hour (1:00 PM, 1:30 PM, 2:00 PM, etc) but you are free to dial this back or be more agressive if you feel the need. ```bash crontab -e -*/30 * * * * /bin/bash /home/USER/gravity-sync/gravity-sync.sh > /home/USER/gravity-sync/gravity-sync.cron +*/15 * * * * /bin/bash /home/USER/gravity-sync/gravity-sync.sh > /home/USER/gravity-sync/gravity-sync.cron +0 23 * * * /bin/bash /home/USER//gravity-sync/gravity-sync.sh backup >/dev/null 2>&1 ``` ## Reference Architectures diff --git a/gravity-sync.conf.example b/gravity-sync.conf.example index 1e55653..c340201 100644 --- a/gravity-sync.conf.example +++ b/gravity-sync.conf.example @@ -33,5 +33,6 @@ REMOTE_PASS='' # SKIP_CUSTOM='' # DATE_OUTPUT='' # PING_AVOID='' +# ROOT_CHECK_AVOID='' # BACKUP_RETAIN='' \ No newline at end of file diff --git a/gravity-sync.sh b/gravity-sync.sh index 06bf34a..18d1c4c 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -31,6 +31,7 @@ VERIFY_PASS='0' # replace in gravity-sync.conf to overwrite SKIP_CUSTOM='0' # replace in gravity-sync.conf to overwrite DATE_OUTPUT='0' # replace in gravity-sync.conf to overwrite PING_AVOID='0' # replace in gravity-sync.conf to overwrite +ROOT_CHECK_AVOID='0' # replace in gravity-sync.conf to overwrite # Backup Customization BACKUP_RETAIN='7' # replace in gravity-sync.conf to overwrite @@ -770,7 +771,6 @@ function logs_export { ### Output Sync Logs function logs_gs { - import_gs MESSAGE="Tailing ${LOG_PATH}/${SYNCING_LOG}" echo_info @@ -794,7 +794,6 @@ function logs_gs { ## Crontab Logs ### Core Crontab Logs function show_crontab { - import_gs MESSAGE="Replaying Last Cronjob" echo_stat @@ -1610,8 +1609,6 @@ function task_automate { MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - import_gs - CRON_EXIST='0' CRON_CHECK=$(crontab -l | grep -q "${GS_FILENAME}" && echo '1' || echo '0') if [ ${CRON_CHECK} == 1 ] @@ -1624,9 +1621,14 @@ function task_automate { MESSAGE="Configuring Hourly Smart Sync" echo_info - MESSAGE="Sync Frequency in Minutes (1-30) or 0 to Disable" - echo_need - read INPUT_AUTO_FREQ + if [[ $1 =~ ^[0-9][0-9]?$ ]] + then + INPUT_AUTO_FREQ=$1 + else + MESSAGE="Sync Frequency in Minutes (1-30) or 0 to Disable" + echo_need + read INPUT_AUTO_FREQ + fi if [ $INPUT_AUTO_FREQ -gt 30 ] then @@ -1660,9 +1662,14 @@ function task_automate { MESSAGE="Configuring Daily Backup Frequency" echo_info - MESSAGE="Hour of Day to Backup (1-24) or 0 to Disable" - echo_need - read INPUT_AUTO_BACKUP + if [[ $2 =~ ^[0-9][0-9]?$ ]] + then + INPUT_AUTO_BACKUP=$2 + else + MESSAGE="Hour of Day to Backup (1-24) or 0 to Disable" + echo_need + read INPUT_AUTO_BACKUP + fi if [ $INPUT_AUTO_BACKUP -gt 24 ] then @@ -1820,8 +1827,7 @@ function task_compare { TASKTYPE='COMPARE' MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - - import_gs + validate_gs_folders validate_ph_folders validate_os_sshpass @@ -1970,8 +1976,11 @@ function root_check { MESSAGE="Evaluating Arguments" echo_stat - - root_check + import_gs + if [ "${ROOT_CHECK_AVOID}" != "1" ] + then + root_check + fi case $# in @@ -1980,7 +1989,6 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - import_gs validate_gs_folders validate_ph_folders validate_os_sshpass @@ -1996,7 +2004,6 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - import_gs validate_gs_folders validate_ph_folders validate_os_sshpass @@ -2010,7 +2017,6 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - import_gs validate_gs_folders validate_ph_folders validate_os_sshpass @@ -2023,8 +2029,7 @@ case $# in TASKTYPE='PULL' MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - - import_gs + validate_gs_folders validate_ph_folders validate_os_sshpass @@ -2037,8 +2042,7 @@ case $# in TASKTYPE='PUSH' MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - - import_gs + validate_gs_folders validate_ph_folders validate_os_sshpass @@ -2052,7 +2056,6 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - import_gs validate_gs_folders validate_ph_folders @@ -2125,6 +2128,24 @@ case $# in ;; esac ;; + + 2) + case $1 in + automate) + task_automate + ;; + + esac + ;; + + 3) + case $1 in + automate) + task_automate $2 $3 + ;; + + esac + ;; *) task_invalid From 9a8e49f07c8d644d4fd6efa28a53627104fb6e0a Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 16 Jul 2020 13:15:38 -0500 Subject: [PATCH 91/97] Post PR cleanup --- ADVANCED.md | 25 ++++++++++++++----------- CHANGELOG.md | 2 ++ gravity-sync.sh | 2 ++ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index e6d9374..ca4a138 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -170,7 +170,7 @@ The `./gravity-sync.sh config` function will attempt to ping the remote host to Default setting in Gravity Sync is 0, change to 1 to skip this network test. #### `ROOT_CHECK_AVOID=''` -The `./gravity-sync.sh` check that it's deployed with it's own user, but for container deployment it's a nuisance, to install a user and gave it passwordless sudo power, witch result with the same security risk as running as root. +At execution, Gravity Sync will check that it's deployed with it's own user (not running as root), but for a container deployment this is not necessary. Default setting in Gravity Sync is 0, change to 1 to skip this root user test. @@ -217,16 +217,6 @@ If your code is still not updating after this, reinstallation is suggested rathe ## Automation There are many automation methods available to run scripts on a regular basis of a Linux system. The one built into all of them is cron, but if you'd like to utilize something different then the principles are still the same. -To automate the deployment of automation option, you can call it with 2 parameters: -first interval in minutes to run sync [0-30], -then the hour to run backup [0-24] -Note: 0 will disable the cron entry. - -for example: -gravity-sync.sh automate 15 23 - -will sync every 15 minutes and backup at 23:00. - If you prefer to still use cron but modify your settings by hand, using the entry below will cause the entry to run at the top and bottom of every hour (1:00 PM, 1:30 PM, 2:00 PM, etc) but you are free to dial this back or be more agressive if you feel the need. ```bash @@ -235,6 +225,19 @@ crontab -e 0 23 * * * /bin/bash /home/USER//gravity-sync/gravity-sync.sh backup >/dev/null 2>&1 ``` +### Automating Automation + +To automate the deployment of automation option you can call it with 2 parameters: +- First interval in minutes to run sync [0-30], +- Second the hour to run backup [0-24] + +(0 will disable the cron entry) + +For example: +`./gravity-sync.sh automate 15 23` + +Will configure automation of the sync function every 15 minutes and of a backup at 23:00. + ## Reference Architectures The designation of primary and secondary is purely at your discretion. The doesn't matter if you're using an HA process like keepalived to present a single DNS IP address to clients, or handing out two DNS resolvers via DHCP. Generally it is expected that the two (or more) Pi-hole(s) will be at the same phyiscal location, or at least on the same internal networks. It should also be possible to to replicate to a secondary Pi-hole across networks, either over a VPN or open-Internet, with the approprate firewall/NAT configuration. diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c1ec65..26ab0b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,8 @@ Skipping a few digits because what does it really matter? #### 2.1.6 - Adds prompts during `./gravity-sync.sh configure` to allow custom SSH port and enable PING avoidance. +- Adds `ROOT_CHECK_AVOID` variable to advanced configuration options, to help facilitate running Gravity Sync with container installations of Pi-hole. (PR [#64](https://github.com/vmstan/gravity-sync/pull/64)) +- Adds ## 2.0 ### The Smart Release diff --git a/gravity-sync.sh b/gravity-sync.sh index 18d1c4c..1d5f0e1 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -1976,7 +1976,9 @@ function root_check { MESSAGE="Evaluating Arguments" echo_stat + import_gs + if [ "${ROOT_CHECK_AVOID}" != "1" ] then root_check From 5fc12be73b83419fdce5c5d714706556d6ee9643 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 16 Jul 2020 13:18:51 -0500 Subject: [PATCH 92/97] Thanks for fbourqui --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26ab0b3..405ebe8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,9 @@ Skipping a few digits because what does it really matter? #### 2.1.6 - Adds prompts during `./gravity-sync.sh configure` to allow custom SSH port and enable PING avoidance. - Adds `ROOT_CHECK_AVOID` variable to advanced configuration options, to help facilitate running Gravity Sync with container installations of Pi-hole. (PR [#64](https://github.com/vmstan/gravity-sync/pull/64)) -- Adds +- Adds the ability to automate automation. :mind_blown_emoji: Please see the ADVANCED.md document for more information. (PR [#64](https://github.com/vmstan/gravity-sync/pull/64)) + +(Thanks to [@fbourqui](https://github.com/fbourqui) for this contributions to this release.) ## 2.0 ### The Smart Release From 606336377394b3143799f3de4878a8dfbf4aaf9b Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 16 Jul 2020 13:37:27 -0500 Subject: [PATCH 93/97] readjusts import_gs --- gravity-sync.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 1d5f0e1..9b5f4e5 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -771,6 +771,7 @@ function logs_export { ### Output Sync Logs function logs_gs { + import_gs MESSAGE="Tailing ${LOG_PATH}/${SYNCING_LOG}" echo_info @@ -794,6 +795,7 @@ function logs_gs { ## Crontab Logs ### Core Crontab Logs function show_crontab { + import_gs MESSAGE="Replaying Last Cronjob" echo_stat @@ -1609,6 +1611,8 @@ function task_automate { MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good + import_gs + CRON_EXIST='0' CRON_CHECK=$(crontab -l | grep -q "${GS_FILENAME}" && echo '1' || echo '0') if [ ${CRON_CHECK} == 1 ] @@ -1828,6 +1832,8 @@ function task_compare { MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good + import_gs + validate_gs_folders validate_ph_folders validate_os_sshpass @@ -1976,8 +1982,6 @@ function root_check { MESSAGE="Evaluating Arguments" echo_stat - - import_gs if [ "${ROOT_CHECK_AVOID}" != "1" ] then @@ -1991,6 +1995,7 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good + import_gs validate_gs_folders validate_ph_folders validate_os_sshpass @@ -2006,6 +2011,7 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good + import_gs validate_gs_folders validate_ph_folders validate_os_sshpass @@ -2019,6 +2025,7 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good + import_gs validate_gs_folders validate_ph_folders validate_os_sshpass @@ -2032,6 +2039,7 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good + import_gs validate_gs_folders validate_ph_folders validate_os_sshpass @@ -2045,6 +2053,7 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good + import_gs validate_gs_folders validate_ph_folders validate_os_sshpass @@ -2058,6 +2067,7 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good + import_gs validate_gs_folders validate_ph_folders From ba034a586cfb9aed6b799e77bf137281fc18a20a Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 16 Jul 2020 13:42:42 -0500 Subject: [PATCH 94/97] 2.1.6 beta 2 (#67) * select different files for restore * star trek reference * Log backup jobs * color restore * ping avoid and custom ssh prompts * z variable * or * [] * new method for defaults * variables matter * or this? * making it specific * adv conf: add ROOT_CHECK_AVOID for container. (#64) * adv conf: add ROOT_CHECK_AVOID for container. import_gs only once at start * allow parameters for automation option Co-authored-by: Michael Stanclift * Post PR cleanup * Thanks for fbourqui * readjusts import_gs Co-authored-by: Michael Stanclift Co-authored-by: fbourqui --- ADVANCED.md | 21 +++++++++++++++- CHANGELOG.md | 4 +++ gravity-sync.conf.example | 1 + gravity-sync.sh | 53 +++++++++++++++++++++++++++++++-------- 4 files changed, 68 insertions(+), 11 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index 2f6d3ae..ca4a138 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -169,6 +169,11 @@ The `./gravity-sync.sh config` function will attempt to ping the remote host to Default setting in Gravity Sync is 0, change to 1 to skip this network test. +#### `ROOT_CHECK_AVOID=''` +At execution, Gravity Sync will check that it's deployed with it's own user (not running as root), but for a container deployment this is not necessary. + +Default setting in Gravity Sync is 0, change to 1 to skip this root user test. + #### `BACKUP_RETAIN=''` The `./gravity-sync.sh backup` function will retain a defined number of days worth of `gravity.db` and `custom.list` backups. @@ -216,9 +221,23 @@ If you prefer to still use cron but modify your settings by hand, using the entr ```bash crontab -e -*/30 * * * * /bin/bash /home/USER/gravity-sync/gravity-sync.sh > /home/USER/gravity-sync/gravity-sync.cron +*/15 * * * * /bin/bash /home/USER/gravity-sync/gravity-sync.sh > /home/USER/gravity-sync/gravity-sync.cron +0 23 * * * /bin/bash /home/USER//gravity-sync/gravity-sync.sh backup >/dev/null 2>&1 ``` +### Automating Automation + +To automate the deployment of automation option you can call it with 2 parameters: +- First interval in minutes to run sync [0-30], +- Second the hour to run backup [0-24] + +(0 will disable the cron entry) + +For example: +`./gravity-sync.sh automate 15 23` + +Will configure automation of the sync function every 15 minutes and of a backup at 23:00. + ## Reference Architectures The designation of primary and secondary is purely at your discretion. The doesn't matter if you're using an HA process like keepalived to present a single DNS IP address to clients, or handing out two DNS resolvers via DHCP. Generally it is expected that the two (or more) Pi-hole(s) will be at the same phyiscal location, or at least on the same internal networks. It should also be possible to to replicate to a secondary Pi-hole across networks, either over a VPN or open-Internet, with the approprate firewall/NAT configuration. diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c1ec65..405ebe8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,10 @@ Skipping a few digits because what does it really matter? #### 2.1.6 - Adds prompts during `./gravity-sync.sh configure` to allow custom SSH port and enable PING avoidance. +- Adds `ROOT_CHECK_AVOID` variable to advanced configuration options, to help facilitate running Gravity Sync with container installations of Pi-hole. (PR [#64](https://github.com/vmstan/gravity-sync/pull/64)) +- Adds the ability to automate automation. :mind_blown_emoji: Please see the ADVANCED.md document for more information. (PR [#64](https://github.com/vmstan/gravity-sync/pull/64)) + +(Thanks to [@fbourqui](https://github.com/fbourqui) for this contributions to this release.) ## 2.0 ### The Smart Release diff --git a/gravity-sync.conf.example b/gravity-sync.conf.example index 1e55653..c340201 100644 --- a/gravity-sync.conf.example +++ b/gravity-sync.conf.example @@ -33,5 +33,6 @@ REMOTE_PASS='' # SKIP_CUSTOM='' # DATE_OUTPUT='' # PING_AVOID='' +# ROOT_CHECK_AVOID='' # BACKUP_RETAIN='' \ No newline at end of file diff --git a/gravity-sync.sh b/gravity-sync.sh index 06bf34a..9b5f4e5 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -31,6 +31,7 @@ VERIFY_PASS='0' # replace in gravity-sync.conf to overwrite SKIP_CUSTOM='0' # replace in gravity-sync.conf to overwrite DATE_OUTPUT='0' # replace in gravity-sync.conf to overwrite PING_AVOID='0' # replace in gravity-sync.conf to overwrite +ROOT_CHECK_AVOID='0' # replace in gravity-sync.conf to overwrite # Backup Customization BACKUP_RETAIN='7' # replace in gravity-sync.conf to overwrite @@ -1624,9 +1625,14 @@ function task_automate { MESSAGE="Configuring Hourly Smart Sync" echo_info - MESSAGE="Sync Frequency in Minutes (1-30) or 0 to Disable" - echo_need - read INPUT_AUTO_FREQ + if [[ $1 =~ ^[0-9][0-9]?$ ]] + then + INPUT_AUTO_FREQ=$1 + else + MESSAGE="Sync Frequency in Minutes (1-30) or 0 to Disable" + echo_need + read INPUT_AUTO_FREQ + fi if [ $INPUT_AUTO_FREQ -gt 30 ] then @@ -1660,9 +1666,14 @@ function task_automate { MESSAGE="Configuring Daily Backup Frequency" echo_info - MESSAGE="Hour of Day to Backup (1-24) or 0 to Disable" - echo_need - read INPUT_AUTO_BACKUP + if [[ $2 =~ ^[0-9][0-9]?$ ]] + then + INPUT_AUTO_BACKUP=$2 + else + MESSAGE="Hour of Day to Backup (1-24) or 0 to Disable" + echo_need + read INPUT_AUTO_BACKUP + fi if [ $INPUT_AUTO_BACKUP -gt 24 ] then @@ -1820,8 +1831,9 @@ function task_compare { TASKTYPE='COMPARE' MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - + import_gs + validate_gs_folders validate_ph_folders validate_os_sshpass @@ -1971,7 +1983,10 @@ function root_check { MESSAGE="Evaluating Arguments" echo_stat - root_check + if [ "${ROOT_CHECK_AVOID}" != "1" ] + then + root_check + fi case $# in @@ -2023,7 +2038,7 @@ case $# in TASKTYPE='PULL' MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - + import_gs validate_gs_folders validate_ph_folders @@ -2037,7 +2052,7 @@ case $# in TASKTYPE='PUSH' MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - + import_gs validate_gs_folders validate_ph_folders @@ -2125,6 +2140,24 @@ case $# in ;; esac ;; + + 2) + case $1 in + automate) + task_automate + ;; + + esac + ;; + + 3) + case $1 in + automate) + task_automate $2 $3 + ;; + + esac + ;; *) task_invalid From b0f1c6d4c77c2d9d7a6ab5cc928c393843360b05 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 16 Jul 2020 13:55:22 -0500 Subject: [PATCH 95/97] advanced md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 405ebe8..0211e67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,7 @@ Skipping a few digits because what does it really matter? #### 2.1.6 - Adds prompts during `./gravity-sync.sh configure` to allow custom SSH port and enable PING avoidance. - Adds `ROOT_CHECK_AVOID` variable to advanced configuration options, to help facilitate running Gravity Sync with container installations of Pi-hole. (PR [#64](https://github.com/vmstan/gravity-sync/pull/64)) -- Adds the ability to automate automation. :mind_blown_emoji: Please see the ADVANCED.md document for more information. (PR [#64](https://github.com/vmstan/gravity-sync/pull/64)) +- Adds the ability to automate automation. :mind_blown_emoji: Please see the [ADVANCED.md](https://github.com/vmstan/gravity-sync/blob/master/ADVANCED.md) document for more information. (PR [#64](https://github.com/vmstan/gravity-sync/pull/64)) (Thanks to [@fbourqui](https://github.com/fbourqui) for this contributions to this release.) From 1afc7cd521a41b20c247cffa79cd4ed47c085c25 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Thu, 16 Jul 2020 22:02:46 -0500 Subject: [PATCH 96/97] rebase --- ADVANCED.md | 4 ++-- CHANGELOG.md | 3 +++ VERSION | 2 +- gravity-sync.sh | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index ca4a138..2cc369c 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -14,8 +14,8 @@ Download the latest release from [GitHub](https://github.com/vmstan/gravity-sync ```bash cd ~ -wget https://github.com/vmstan/gravity-sync/archive/v2.1.6.zip -unzip v2.1.6.zip -d gravity-sync +wget https://github.com/vmstan/gravity-sync/archive/v2.1.7.zip +unzip v2.1.7.zip -d gravity-sync cd gravity-sync ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 0211e67..cc456ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,9 @@ Skipping a few digits because what does it really matter? (Thanks to [@fbourqui](https://github.com/fbourqui) for this contributions to this release.) +#### 2.1.7 +- Git Rebase + ## 2.0 ### The Smart Release diff --git a/VERSION b/VERSION index b6da512..9671f9a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.6 \ No newline at end of file +2.1.7 \ No newline at end of file diff --git a/gravity-sync.sh b/gravity-sync.sh index 9b5f4e5..bace2f8 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -3,7 +3,7 @@ SCRIPT_START=$SECONDS # GRAVITY SYNC BY VMSTAN ##################### PROGRAM='Gravity Sync' -VERSION='2.1.6' +VERSION='2.1.7' # Execute from the home folder of the user who owns it (ex: 'cd ~/gravity-sync') # For documentation or downloading updates visit https://github.com/vmstan/gravity-sync From fb2ae3872314acb2f1ab15cad3bfaae671a09a98 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Fri, 17 Jul 2020 09:45:08 -0500 Subject: [PATCH 97/97] 2.1.7 (#70) * import config file before executing * show_target function * git Co-authored-by: Michael Stanclift --- CHANGELOG.md | 4 ++++ gravity-sync.sh | 43 +++++++++++++++++++++++++++++-------------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 405ebe8..cf1f2ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,10 @@ Skipping a few digits because what does it really matter? (Thanks to [@fbourqui](https://github.com/fbourqui) for this contributions to this release.) +#### 2.1.7 +- Adjusts placement of configuration import to fully implement `ROOT_CHECK_AVOID` variable. +- Someday I'll understand all these git error messages. + ## 2.0 ### The Smart Release diff --git a/gravity-sync.sh b/gravity-sync.sh index bace2f8..435788e 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -86,10 +86,10 @@ function import_gs { source $HOME/${LOCAL_FOLDR}/${CONFIG_FILE} error_validate - MESSAGE="Targeting ${REMOTE_USER}@${REMOTE_HOST}" - echo_info + # MESSAGE="Targeting ${REMOTE_USER}@${REMOTE_HOST}" + # echo_info - detect_ssh + # detect_ssh else echo_fail @@ -101,6 +101,13 @@ function import_gs { fi } +function show_target { + MESSAGE="Targeting ${REMOTE_USER}@${REMOTE_HOST}" + echo_info + + detect_ssh +} + # GS Update Functions ## Master Branch function update_gs { @@ -771,7 +778,7 @@ function logs_export { ### Output Sync Logs function logs_gs { - import_gs + # import_gs MESSAGE="Tailing ${LOG_PATH}/${SYNCING_LOG}" echo_info @@ -795,7 +802,7 @@ function logs_gs { ## Crontab Logs ### Core Crontab Logs function show_crontab { - import_gs + # import_gs MESSAGE="Replaying Last Cronjob" echo_stat @@ -1611,7 +1618,7 @@ function task_automate { MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - import_gs + # import_gs CRON_EXIST='0' CRON_CHECK=$(crontab -l | grep -q "${GS_FILENAME}" && echo '1' || echo '0') @@ -1832,8 +1839,8 @@ function task_compare { MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - import_gs - + # import_gs + show_target validate_gs_folders validate_ph_folders validate_os_sshpass @@ -1980,6 +1987,8 @@ function root_check { MESSAGE="${PROGRAM} ${VERSION} Executing" echo_info + import_gs + MESSAGE="Evaluating Arguments" echo_stat @@ -1995,7 +2004,8 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - import_gs + # import_gs + show_target validate_gs_folders validate_ph_folders validate_os_sshpass @@ -2011,7 +2021,8 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - import_gs + # import_gs + show_target validate_gs_folders validate_ph_folders validate_os_sshpass @@ -2025,7 +2036,8 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - import_gs + # import_gs + show_target validate_gs_folders validate_ph_folders validate_os_sshpass @@ -2039,7 +2051,8 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - import_gs + # import_gs + show_target validate_gs_folders validate_ph_folders validate_os_sshpass @@ -2053,7 +2066,8 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - import_gs + # import_gs + show_target validate_gs_folders validate_ph_folders validate_os_sshpass @@ -2067,7 +2081,8 @@ case $# in MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good - import_gs + # import_gs + show_target validate_gs_folders validate_ph_folders