diff --git a/src/Core/Main.vala b/src/Core/Main.vala index 0cd3cb7..28f9423 100644 --- a/src/Core/Main.vala +++ b/src/Core/Main.vala @@ -120,8 +120,6 @@ public class Main : GLib.Object{ public int startup_delay_interval_mins = 10; public int retain_snapshots_max_days = 200; - - public int64 snapshot_location_free_space = 0; public const uint64 MIN_FREE_SPACE = 1 * GB; public static uint64 first_snapshot_size = 0; @@ -163,6 +161,7 @@ public class Main : GLib.Object{ public RsyncTask task; public DeleteFileTask delete_file_task; + public RsyncSpaceCheckTask space_check_task; public Gee.HashMap current_system_users; public string users_with_encrypted_home = ""; @@ -1246,24 +1245,57 @@ public class Main : GLib.Object{ } } - if (!repo.available() || !repo.has_space()){ + if (!repo.available()) { log_error(repo.status_message); log_error(repo.status_details); exit_app(); } - // create new snapshot ----------------------- + if (repo.mount_path.length == 0){ + log_error("Backup location not mounted"); + exit_app(); + } + + // create new snapshot ----------------------- Snapshot new_snapshot = null; - if (btrfs_mode){ + if (btrfs_mode) + { new_snapshot = create_snapshot_with_btrfs(tag, dt_created); } - else{ + else + if (first_snapshot_size > 0 && repo.device.free_bytes < first_snapshot_size) + { + // Perform a dry-run of the intended backup and make sure we'll have enough room + uint64 needed = get_space_needed_for_rsync_snapshot(dt_created); + bool enough = repo.has_space(needed); + + var message = "Space required for snapshot: %lld (%s). Space available: %lu (%s)" + .printf(needed, format_file_size(needed), repo.device.free_bytes, repo.device.free); + log_msg(message); + if (!enough) + { + message = "Not enough disk space! Additional required: %lld (%s)".printf(needed - repo.device.free_bytes, format_file_size(needed - repo.device.free_bytes)); + log_msg(message); + } + + if (enough) + { + new_snapshot = create_snapshot_with_rsync(tag, dt_created); + } + else + { + return false; + } + } + else + { + // this is the initial snapshot (which means a size check has already been made) or the target + // drive's available space is more than the the original (full) snapshot's size. new_snapshot = create_snapshot_with_rsync(tag, dt_created); } - // finish ------------------------------ - + var dt_end = new DateTime.now_local(); TimeSpan elapsed = dt_end.difference(dt_begin); long seconds = (long)(elapsed * 1.0 / TimeSpan.SECOND); @@ -1287,8 +1319,109 @@ public class Main : GLib.Object{ return status; } - private Snapshot? create_snapshot_with_rsync(string tag, DateTime dt_created){ + private uint64 get_space_needed_for_rsync_snapshot(DateTime dt_created) { + log_msg(string.nfill(78, '-')); + log_msg("Checking if target drive has enough free space for a snapshot (RSYNC)"); + log_msg("Target device: %s, mount path: %s".printf(repo.device.device, repo.mount_path)); + + string time_stamp = dt_created.format("%Y-%m-%d_%H-%M-%S"); + string snapshot_dir = repo.snapshots_path; + string snapshot_name = time_stamp; + string snapshot_path = path_combine(snapshot_dir, snapshot_name); + dir_create(snapshot_path); + string localhost_path = path_combine(snapshot_path, "localhost"); + dir_create(localhost_path); + + string sys_uuid = (sys_root == null) ? "" : sys_root.uuid; + + Snapshot snapshot_to_link = null; + + // check if a snapshot was restored recently and use it for linking --------- + + try{ + + string ctl_path = path_combine(snapshot_dir, ".sync-restore"); + var f = File.new_for_path(ctl_path); + + if (f.query_exists()){ + + // read snapshot name from file + string snap_path = file_read(ctl_path); + string snap_name = file_basename(snap_path); + + // find the snapshot that was restored + foreach(var bak in repo.snapshots){ + if ((bak.name == snap_name) && (bak.sys_uuid == sys_uuid)){ + // use for linking + snapshot_to_link = bak; + // delete the restore-control-file + f.delete(); + break; + } + } + } + } + catch(Error e){ + log_error (e.message); + return 0; + } + + // get latest snapshot to link if not set ------- + + if (snapshot_to_link == null){ + snapshot_to_link = repo.get_latest_snapshot("", sys_uuid); + } + + string link_from_path = ""; + if (snapshot_to_link != null){ + log_msg("Linking from snapshot: %s".printf(snapshot_to_link.name)); + link_from_path = "%s/localhost/".printf(snapshot_to_link.path); + } + + // save exclude list ---------------- + + bool ok = save_exclude_list_for_backup(snapshot_path); + + string exclude_from_file = path_combine(snapshot_path, "exclude.list"); + + if (!ok){ + log_error("Failed to save exclude list"); + return 0; + } + + progress_text = _("Calculating disk space required..."); + log_msg(progress_text); + + space_check_task = new RsyncSpaceCheckTask(); + + space_check_task.source_path = ""; + space_check_task.dest_path = snapshot_path + "/localhost/"; + space_check_task.link_from_path = link_from_path; + space_check_task.exclude_from_file = exclude_from_file; + space_check_task.prg_count_total = Main.first_snapshot_count; + + space_check_task.relative = true; + space_check_task.verbose = true; + space_check_task.delete_extra = true; + space_check_task.delete_excluded = true; + space_check_task.delete_after = false; + + space_check_task.execute(); + + while (space_check_task.status == AppStatus.RUNNING){ + sleep(1000); + gtk_do_events(); + + stdout.flush(); + } + + var total_size = space_check_task.total_size; + space_check_task = null; + return total_size; + } + + private Snapshot? create_snapshot_with_rsync(string tag, DateTime dt_created){ log_msg(string.nfill(78, '-')); if (first_snapshot_size == 0){ diff --git a/src/Core/SnapshotRepo.vala b/src/Core/SnapshotRepo.vala index 4879107..12351b2 100755 --- a/src/Core/SnapshotRepo.vala +++ b/src/Core/SnapshotRepo.vala @@ -43,6 +43,7 @@ public class SnapshotRepo : GLib.Object{ public string status_message = ""; public string status_details = ""; public SnapshotLocationStatus status_code; + public bool last_snapshot_failed_space = false; // private private Gtk.Window? parent_window = null; @@ -420,13 +421,19 @@ public class SnapshotRepo : GLib.Object{ log_debug("SnapshotRepo: check_status()"); - status_code = SnapshotLocationStatus.HAS_SNAPSHOTS_HAS_SPACE; - status_message = ""; - status_details = ""; + if (!last_snapshot_failed_space) + { + status_code = SnapshotLocationStatus.HAS_SNAPSHOTS_HAS_SPACE; + status_message = ""; + status_details = ""; + } if (available()){ has_snapshots(); - has_space(); + if (!last_snapshot_failed_space) + { + has_space(); + } } if ((App != null) && (App.app_mode.length == 0)){ @@ -448,6 +455,7 @@ public class SnapshotRepo : GLib.Object{ log_debug(""); } + last_snapshot_failed_space = false; log_debug("SnapshotRepo: check_status(): exit"); } @@ -517,9 +525,8 @@ public class SnapshotRepo : GLib.Object{ return (snapshots.size > 0); } - public bool has_space(){ - - log_debug("SnapshotRepo: has_space()"); + public bool has_space(uint64 needed = 0) { + log_debug("SnapshotRepo: has_space() - %llu required (%s)".printf(needed, format_file_size(needed))); if ((device != null) && (device.device.length > 0)){ device.query_disk_space(); @@ -532,15 +539,14 @@ public class SnapshotRepo : GLib.Object{ if (snapshots.size > 0){ // has snapshots, check minimum space - //log_debug("has snapshots"); - - if (device.free_bytes < Main.MIN_FREE_SPACE){ + if (device.free_bytes < (needed > 0 ? needed : Main.MIN_FREE_SPACE)) { status_message = _("Not enough disk space"); - status_message += " (< %s)".printf(format_file_size(Main.MIN_FREE_SPACE, false, "", true, 0)); + status_message += " (< %s)".printf(format_file_size((needed > 0 ? needed : Main.MIN_FREE_SPACE), false, "", true, 0)); status_details = _("Select another device or free up some space"); status_code = SnapshotLocationStatus.HAS_SNAPSHOTS_NO_SPACE; + last_snapshot_failed_space = true; return false; } else{ @@ -550,6 +556,7 @@ public class SnapshotRepo : GLib.Object{ status_details = _("%d snapshots, %s free").printf( snapshots.size, format_file_size(device.free_bytes)); + last_snapshot_failed_space = false; status_code = SnapshotLocationStatus.HAS_SNAPSHOTS_HAS_SPACE; return true; } @@ -608,7 +615,7 @@ public class SnapshotRepo : GLib.Object{ public void auto_remove(){ log_debug("SnapshotRepo: auto_remove()"); - + last_snapshot_failed_space = false; DateTime now = new DateTime.now_local(); DateTime dt_limit; int count_limit; diff --git a/src/Gtk/BackupBox.vala b/src/Gtk/BackupBox.vala index 4926503..daa7cef 100755 --- a/src/Gtk/BackupBox.vala +++ b/src/Gtk/BackupBox.vala @@ -37,7 +37,7 @@ using TeeJee.System; using TeeJee.Misc; class BackupBox : Gtk.Box{ - + private Gtk.Box details_box; private Gtk.Spinner spinner; public Gtk.Label lbl_msg; public Gtk.Label lbl_status; @@ -77,24 +77,27 @@ class BackupBox : Gtk.Box{ Gtk.SizeGroup sg_label = null; Gtk.SizeGroup sg_value = null; - var label = add_label(this, _("File and directory counts:"), true); + details_box = new Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=6); + add(details_box); + + var label = add_label(details_box, _("File and directory counts:"), true); label.margin_bottom = 6; label.margin_top = 12; - lbl_unchanged = add_count_label(this, _("No Change"), ref sg_label, ref sg_value); - lbl_created = add_count_label(this, _("Created"), ref sg_label, ref sg_value); - lbl_deleted = add_count_label(this, _("Deleted"), ref sg_label, ref sg_value); - lbl_modified = add_count_label(this, _("Changed"), ref sg_label, ref sg_value, 12); + lbl_unchanged = add_count_label(details_box, _("No Change"), ref sg_label, ref sg_value); + lbl_created = add_count_label(details_box, _("Created"), ref sg_label, ref sg_value); + lbl_deleted = add_count_label(details_box, _("Deleted"), ref sg_label, ref sg_value); + lbl_modified = add_count_label(details_box, _("Changed"), ref sg_label, ref sg_value, 12); - label = add_label(this, _("Changed items:"), true); + label = add_label(details_box, _("Changed items:"), true); label.margin_bottom = 6; - lbl_checksum = add_count_label(this, _("Checksum"), ref sg_label, ref sg_value); - lbl_size = add_count_label(this, _("Size"), ref sg_label, ref sg_value); - lbl_timestamp = add_count_label(this, _("Timestamp"), ref sg_label, ref sg_value); - lbl_permissions = add_count_label(this, _("Permissions"), ref sg_label, ref sg_value); - lbl_owner = add_count_label(this, _("Owner"), ref sg_label, ref sg_value); - lbl_group = add_count_label(this, _("Group"), ref sg_label, ref sg_value, 24); + lbl_checksum = add_count_label(details_box, _("Checksum"), ref sg_label, ref sg_value); + lbl_size = add_count_label(details_box, _("Size"), ref sg_label, ref sg_value); + lbl_timestamp = add_count_label(details_box, _("Timestamp"), ref sg_label, ref sg_value); + lbl_permissions = add_count_label(details_box, _("Permissions"), ref sg_label, ref sg_value); + lbl_owner = add_count_label(details_box, _("Owner"), ref sg_label, ref sg_value); + lbl_group = add_count_label(details_box, _("Group"), ref sg_label, ref sg_value, 24); lbl_deleted.sensitive = false; @@ -208,10 +211,30 @@ class BackupBox : Gtk.Box{ string status_line = ""; string last_status_line = ""; int remaining_counter = 10; - - while (thread_is_running){ - status_line = escape_html(App.task.status_line); + while (thread_is_running){ + string task_status_line; + double fraction; + string task_stat_time_remaining; + + bool checking = App.space_check_task != null; + + details_box.visible = !checking; + + if (checking) + { + task_status_line = App.space_check_task.status_line; + fraction = App.space_check_task.progress; + task_stat_time_remaining = App.space_check_task.stat_time_remaining; + } + else + { + task_status_line = App.task.status_line; + fraction = App.task.progress; + task_stat_time_remaining = App.task.stat_time_remaining; + } + + status_line = escape_html(task_status_line); if (status_line != last_status_line){ lbl_status.label = status_line; last_status_line = status_line; @@ -225,13 +248,11 @@ class BackupBox : Gtk.Box{ } } - double fraction = App.task.progress; - // time remaining remaining_counter--; if (remaining_counter == 0){ lbl_remaining.label = - App.task.stat_time_remaining + " " + _("remaining"); + task_stat_time_remaining + " " + _("remaining"); remaining_counter = 10; } @@ -245,16 +266,19 @@ class BackupBox : Gtk.Box{ lbl_msg.label = escape_html(App.progress_text); - lbl_unchanged.label = "%'d".printf(App.task.count_unchanged); - lbl_created.label = "%'d".printf(App.task.count_created); - lbl_deleted.label = "%'d".printf(App.task.count_deleted); - lbl_modified.label = "%'d".printf(App.task.count_modified); - lbl_checksum.label = "%'d".printf(App.task.count_checksum); - lbl_size.label = "%'d".printf(App.task.count_size); - lbl_timestamp.label = "%'d".printf(App.task.count_timestamp); - lbl_permissions.label = "%'d".printf(App.task.count_permissions); - lbl_owner.label = "%'d".printf(App.task.count_owner); - lbl_group.label = "%'d".printf(App.task.count_group); + if (!checking) + { + lbl_unchanged.label = "%'d".printf(App.task.count_unchanged); + lbl_created.label = "%'d".printf(App.task.count_created); + lbl_deleted.label = "%'d".printf(App.task.count_deleted); + lbl_modified.label = "%'d".printf(App.task.count_modified); + lbl_checksum.label = "%'d".printf(App.task.count_checksum); + lbl_size.label = "%'d".printf(App.task.count_size); + lbl_timestamp.label = "%'d".printf(App.task.count_timestamp); + lbl_permissions.label = "%'d".printf(App.task.count_permissions); + lbl_owner.label = "%'d".printf(App.task.count_owner); + lbl_group.label = "%'d".printf(App.task.count_group); + } gtk_do_events(); diff --git a/src/Gtk/BackupWindow.vala b/src/Gtk/BackupWindow.vala index f81f454..2741848 100644 --- a/src/Gtk/BackupWindow.vala +++ b/src/Gtk/BackupWindow.vala @@ -315,7 +315,17 @@ class BackupWindow : Gtk.Window{ break; case Tabs.BACKUP_FINISH: backup_finish_box.update_message(success); - wait_and_close_window(1000, this); + if (App.repo.status_code == SnapshotLocationStatus.HAS_SNAPSHOTS_NO_SPACE) + { + this.hide(); + gtk_messagebox(App.repo.status_message, App.repo.status_details, this, true); + this.destroy(); + } + else + { + backup_finish_box.update_message(success); + wait_and_close_window(1000, this); + } break; } } diff --git a/src/Gtk/MainWindow.vala b/src/Gtk/MainWindow.vala index b142b69..9ff6663 100644 --- a/src/Gtk/MainWindow.vala +++ b/src/Gtk/MainWindow.vala @@ -504,7 +504,7 @@ class MainWindow : Gtk.Window{ // check snapshot device ----------- - if (!App.repo.available() || !App.repo.has_space()){ + if (!App.repo.available()){ gtk_messagebox(App.repo.status_message, App.repo.status_details, this, true); // allow user to continue after showing message } diff --git a/src/Utility/RsyncSpaceCheckTask.vala b/src/Utility/RsyncSpaceCheckTask.vala new file mode 100644 index 0000000..13c4262 --- /dev/null +++ b/src/Utility/RsyncSpaceCheckTask.vala @@ -0,0 +1,209 @@ +/* + * RsyncTask.vala + * + * Copyright 2012-2018 Tony George + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + * + * + */ + +using TeeJee.Logging; +using TeeJee.FileSystem; +using TeeJee.JsonHelper; +using TeeJee.ProcessHelper; +using TeeJee.System; +using TeeJee.Misc; + +public class RsyncSpaceCheckTask : AsyncTask{ + + // settings + public bool delete_extra = true; + public bool delete_after = false; + public bool delete_excluded = false; + public bool relative = false; + + public string exclude_from_file = ""; + public string link_from_path = ""; + public string source_path = ""; + public string dest_path = ""; + public bool verbose = true; + public bool io_nice = true; + public bool dry_run = false; + + // regex + private Regex sent_bytes_regex; + + // status + public int64 status_line_count = 0; + public int64 total_size = 0; + + public RsyncSpaceCheckTask(){ + init_regular_expressions(); + } + + private void init_regular_expressions(){ + if (sent_bytes_regex != null){ + return; // already initialized + } + + try { + sent_bytes_regex = new Regex("""sent ([0-9,]+)[ \t]+bytes[ \t]+received"""); + } + catch (Error e) { + log_error (e.message); + } + } + + public void prepare() { + string script_text = build_script(); + + log_debug(script_text); + + save_bash_script_temp(script_text, script_file); + log_debug("RsyncSpaceCheckTask:prepare(): saved: %s".printf(script_file)); + + total_size = 0; + status_line_count = 0; + } + + private string build_script() { + var cmd = "export LC_ALL=C.UTF-8\n"; + + cmd += "rsync -aii"; + + cmd += " --recursive"; + + if (verbose){ + cmd += " --verbose"; + } + else{ + cmd += " --quiet"; + } + + if (delete_extra){ + cmd += " --delete"; + } + + if (delete_after){ + cmd += " --delete-after"; + } + + cmd += " --force"; // allow deletion of non-empty directories + + cmd += " --stats"; + + cmd += " --sparse"; + + if (delete_excluded){ + cmd += " --delete-excluded"; + } + + cmd += " --dry-run"; + + if (link_from_path.length > 0){ + if (!link_from_path.has_suffix("/")){ + link_from_path = "%s/".printf(link_from_path); + } + + cmd += " --link-dest='%s'".printf(escape_single_quote(link_from_path)); + } + + if (exclude_from_file.length > 0){ + cmd += " --exclude-from='%s'".printf(escape_single_quote(exclude_from_file)); + + if (delete_extra && delete_excluded){ + cmd += " --delete-excluded"; + } + } + + source_path = remove_trailing_slash(source_path); + + dest_path = remove_trailing_slash(dest_path); + + cmd += " '%s/'".printf(escape_single_quote(source_path)); + + cmd += " '%s/'".printf(escape_single_quote(dest_path)); + + return cmd; + } + + // execution ---------------------------- + + public void execute() { + + log_debug("RsyncSpaceCheckTask:execute()"); + + prepare(); + begin(); + } + + public override void parse_stdout_line(string out_line){ + if (is_terminated) { + return; + } + + update_progress_parse_console_output(out_line); + } + + public override void parse_stderr_line(string err_line){ + if (is_terminated) { + return; + } + + update_progress_parse_console_output(err_line); + } + + public bool update_progress_parse_console_output (string line) { + if ((line == null) || (line.length == 0)) { + return true; + } + + status_line_count++; + + if (prg_count_total > 0){ + prg_count = status_line_count; + progress = (prg_count * 1.0) / prg_count_total; + } + + MatchInfo match; + + if (sent_bytes_regex.match(line, 0, out match)) { + total_size = int64.parse(match.fetch(1).replace(",","")); + } + else{ + //log_debug("not-matched: %s".printf(line)); + } + + return true; + } + + protected override void finish_task(){ + if ((status != AppStatus.CANCELLED) && (status != AppStatus.PASSWORD_REQUIRED)) { + status = AppStatus.FINISHED; + } + } + + public int read_status(){ + var status_file = working_dir + "/status"; + var f = File.new_for_path(status_file); + if (f.query_exists()){ + var txt = file_read(status_file); + return int.parse(txt); + } + return -1; + } +}