- djm@cvs.openbsd.org 2013/07/25 00:56:52

     [sftp-client.c sftp-client.h sftp.1 sftp.c]
     sftp support for resuming partial downloads; patch mostly by Loganaden
     Velvindron/AfriNIC with some tweaks by me; feedback and ok dtucker@

5 files changed, 133 insertions, 56 deletions

diff --git a/ChangeLog b/ChangeLog
index f799663fd..9552a9b81 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -15,6 +15,10 @@
      [ssh.c]
      daemonise backgrounded (ControlPersist'ed) multiplexing master to ensure
      it is fully detached from its controlling terminal. based on debugging
+   - djm@cvs.openbsd.org 2013/07/25 00:56:52
+     [sftp-client.c sftp-client.h sftp.1 sftp.c]
+     sftp support for resuming partial downloads; patch mostly by Loganaden
+     Velvindron/AfriNIC with some tweaks by me; feedback and ok dtucker@
 
 20130720
  - (djm) OpenBSD CVS Sync
diff --git a/sftp-client.c b/sftp-client.c
index ab035c713..cb4efd3ea 100644
--- a/sftp-client.c
+++ b/sftp-client.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: sftp-client.c,v 1.100 2013/06/01 22:34:50 dtucker Exp $ */
+/* $OpenBSD: sftp-client.c,v 1.101 2013/07/25 00:56:51 djm Exp $ */
 /*
  * Copyright (c) 2001-2004 Damien Miller <djm@openbsd.org>
  *
@@ -112,7 +112,7 @@ send_msg(struct sftp_conn *conn, Buffer *m)
         iov[1].iov_len = buffer_len(m);
 
         if (atomiciov6(writev, conn->fd_out, iov, 2,
-            conn->limit_kbps > 0 ? sftpio : NULL, &conn->bwlimit_out) != 
+            conn->limit_kbps > 0 ? sftpio : NULL, &conn->bwlimit_out) !=
             buffer_len(m) + sizeof(mlen))
                 fatal("Couldn't send packet: %s", strerror(errno));
 
@@ -988,16 +988,17 @@ send_read_request(struct sftp_conn *conn, u_int id, u_int64_t offset,
 
 int
 do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
-    Attrib *a, int pflag)
+    Attrib *a, int pflag, int resume)
 {
         Attrib junk;
         Buffer msg;
         char *handle;
-        int local_fd, status = 0, write_error;
-        int read_error, write_errno;
-        u_int64_t offset, size;
+        int local_fd = -1, status = 0, write_error;
+        int read_error, write_errno, reordered = 0;
+        u_int64_t offset = 0, size, highwater;
         u_int handle_len, mode, type, id, buflen, num_req, max_req;
         off_t progress_counter;
+        struct stat st;
         struct request {
                 u_int id;
                 u_int len;
@@ -1050,21 +1051,36 @@ do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
                 return(-1);
         }
 
-        local_fd = open(local_path, O_WRONLY | O_CREAT | O_TRUNC,
+        local_fd = open(local_path, O_WRONLY | O_CREAT | (resume ? : O_TRUNC),
             mode | S_IWUSR);
         if (local_fd == -1) {
                 error("Couldn't open local file \"%s\" for writing: %s",
                     local_path, strerror(errno));
-                do_close(conn, handle, handle_len);
-                buffer_free(&msg);
-                free(handle);
-                return(-1);
+                goto fail;
+        }
+        offset = highwater = 0;
+        if (resume) {
+                if (fstat(local_fd, &st) == -1) {
+                        error("Unable to stat local file \"%s\": %s",
+                            local_path, strerror(errno));
+                        goto fail;
+                }
+                if ((size_t)st.st_size > size) {
+                        error("Unable to resume download of \"%s\": "
+                            "local file is larger than remote", local_path);
+ fail:
+                        do_close(conn, handle, handle_len);
+                        buffer_free(&msg);
+                        free(handle);
+                        return -1;
+                }
+                offset = highwater = st.st_size;
         }
 
         /* Read from remote and write to local */
-        write_error = read_error = write_errno = num_req = offset = 0;
+        write_error = read_error = write_errno = num_req = 0;
         max_req = 1;
-        progress_counter = 0;
+        progress_counter = offset;
 
         if (showprogress && size != 0)
                 start_progress_meter(remote_path, size, &progress_counter);
@@ -1139,6 +1155,10 @@ do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
                                 write_error = 1;
                                 max_req = 0;
                         }
+                        else if (!reordered && req->offset <= highwater)
+                                highwater = req->offset + len;
+                        else if (!reordered && req->offset > highwater)
+                                reordered = 1;
                         progress_counter += len;
                         free(data);
 
@@ -1187,7 +1207,15 @@ do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
         /* Sanity check */
         if (TAILQ_FIRST(&requests) != NULL)
                 fatal("Transfer complete, but requests still in queue");
-
+        /* Truncate at highest contiguous point to avoid holes on interrupt */
+        if (read_error || write_error || interrupted) {
+                if (reordered && resume) {
+                        error("Unable to resume download of \"%s\": "
+                            "server reordered requests", local_path);
+                }
+                debug("truncating at %llu", (unsigned long long)highwater);
+                ftruncate(local_fd, highwater);
+        }
         if (read_error) {
                 error("Couldn't read from remote file \"%s\" : %s",
                     remote_path, fx2txt(status));
@@ -1199,7 +1227,8 @@ do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
                 do_close(conn, handle, handle_len);
         } else {
                 status = do_close(conn, handle, handle_len);
-
+                if (interrupted)
+                        status = -1;
                 /* Override umask and utimes if asked */
 #ifdef HAVE_FCHMOD
                 if (pflag && fchmod(local_fd, mode) == -1)
@@ -1227,7 +1256,7 @@ do_download(struct sftp_conn *conn, char *remote_path, char *local_path,
 
 static int
 download_dir_internal(struct sftp_conn *conn, char *src, char *dst,
-    Attrib *dirattrib, int pflag, int printflag, int depth)
+    Attrib *dirattrib, int pflag, int printflag, int depth, int resume)
 {
         int i, ret = 0;
         SFTP_DIRENT **dir_entries;
@@ -1280,11 +1309,11 @@ download_dir_internal(struct sftp_conn *conn, char *src, char *dst,
                                 continue;
                         if (download_dir_internal(conn, new_src, new_dst,
                             &(dir_entries[i]->a), pflag, printflag,
-                            depth + 1) == -1)
+                            depth + 1, resume) == -1)
                                 ret = -1;
                 } else if (S_ISREG(dir_entries[i]->a.perm) ) {
                         if (do_download(conn, new_src, new_dst,
-                            &(dir_entries[i]->a), pflag) == -1) {
+                            &(dir_entries[i]->a), pflag, resume) == -1) {
                                 error("Download of file %s to %s failed",
                                     new_src, new_dst);
                                 ret = -1;
@@ -1317,7 +1346,7 @@ download_dir_internal(struct sftp_conn *conn, char *src, char *dst,
 
 int
 download_dir(struct sftp_conn *conn, char *src, char *dst,
-    Attrib *dirattrib, int pflag, int printflag)
+    Attrib *dirattrib, int pflag, int printflag, int resume)
 {
         char *src_canon;
         int ret;
@@ -1328,7 +1357,7 @@ download_dir(struct sftp_conn *conn, char *src, char *dst,
         }
 
         ret = download_dir_internal(conn, src_canon, dst,
-            dirattrib, pflag, printflag, 0);
+            dirattrib, pflag, printflag, 0, resume);
         free(src_canon);
         return ret;
 }
@@ -1553,7 +1582,7 @@ upload_dir_internal(struct sftp_conn *conn, char *src, char *dst,
         a.perm &= 01777;
         if (!pflag)
                 a.flags &= ~SSH2_FILEXFER_ATTR_ACMODTIME;
-        
+
         status = do_mkdir(conn, dst, &a, 0);
         /*
          * we lack a portable status for errno EEXIST,
@@ -1563,7 +1592,7 @@ upload_dir_internal(struct sftp_conn *conn, char *src, char *dst,
         if (status != SSH2_FX_OK) {
                 if (status != SSH2_FX_FAILURE)
                         return -1;
-                if (do_stat(conn, dst, 0) == NULL) 
+                if (do_stat(conn, dst, 0) == NULL)
                         return -1;
         }
 
@@ -1571,7 +1600,7 @@ upload_dir_internal(struct sftp_conn *conn, char *src, char *dst,
                 error("Failed to open dir \"%s\": %s", src, strerror(errno));
                 return -1;
         }
-        
+
         while (((dp = readdir(dirp)) != NULL) && !interrupted) {
                 if (dp->d_ino == 0)
                         continue;
diff --git a/sftp-client.h b/sftp-client.h
index aef54ef49..111a998c8 100644
--- a/sftp-client.h
+++ b/sftp-client.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: sftp-client.h,v 1.20 2010/12/04 00:18:01 djm Exp $ */
+/* $OpenBSD: sftp-client.h,v 1.21 2013/07/25 00:56:51 djm Exp $ */
 
 /*
  * Copyright (c) 2001-2004 Damien Miller <djm@openbsd.org>
@@ -106,13 +106,13 @@ int do_symlink(struct sftp_conn *, char *, char *);
  * Download 'remote_path' to 'local_path'. Preserve permissions and times
  * if 'pflag' is set
  */
-int do_download(struct sftp_conn *, char *, char *, Attrib *, int);
+int do_download(struct sftp_conn *, char *, char *, Attrib *, int, int);
 
 /*
  * Recursively download 'remote_directory' to 'local_directory'. Preserve 
  * times if 'pflag' is set
  */
-int download_dir(struct sftp_conn *, char *, char *, Attrib *, int, int);
+int download_dir(struct sftp_conn *, char *, char *, Attrib *, int, int, int);
 
 /*
  * Upload 'local_path' to 'remote_path'. Preserve permissions and times
diff --git a/sftp.1 b/sftp.1
index bcb472144..2577fe875 100644
--- a/sftp.1
+++ b/sftp.1
@@ -1,4 +1,4 @@
-.\" $OpenBSD: sftp.1,v 1.91 2011/09/05 05:56:13 djm Exp $
+.\" $OpenBSD: sftp.1,v 1.92 2013/07/25 00:56:51 djm Exp $
 .\"
 .\" Copyright (c) 2001 Damien Miller.  All rights reserved.
 .\"
@@ -22,7 +22,7 @@
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .\"
-.Dd $Mdocdate: September 5 2011 $
+.Dd $Mdocdate: July 25 2013 $
 .Dt SFTP 1
 .Os
 .Sh NAME
@@ -129,7 +129,7 @@ may be used to indicate standard input.
 .Nm
 will abort if any of the following
 commands fail:
-.Ic get , put , rename , ln ,
+.Ic get , put , reget , rename , ln ,
 .Ic rm , mkdir , chdir , ls ,
 .Ic lchdir , chmod , chown ,
 .Ic chgrp , lpwd , df , symlink ,
@@ -343,7 +343,7 @@ extension.
 Quit
 .Nm sftp .
 .It Xo Ic get
-.Op Fl Ppr
+.Op Fl aPpr
 .Ar remote-path
 .Op Ar local-path
 .Xc
@@ -363,6 +363,14 @@ is specified, then
 .Ar local-path
 must specify a directory.
 .Pp
+If the
+.Fl a
+flag is specified, then attempt to resume partial transfers of existing files.
+Note that resumption assumes that any partial copy of the local file matches
+the remote copy.
+If the remote file differs from the partial local copy then the resultant file
+is likely to be corrupt.
+.Pp
 If either the
 .Fl P
 or
@@ -503,6 +511,18 @@ Display remote working directory.
 .It Ic quit
 Quit
 .Nm sftp .
+.It Xo Ic reget
+.Op Fl Ppr
+.Ar remote-path
+.Op Ar local-path
+.Xc
+Resume download of
+.Ar remote-path .
+Equivalent to
+.Ic get
+with the
+.Fl a
+flag set.
 .It Ic rename Ar oldpath Ar newpath
 Rename remote file from
 .Ar oldpath
diff --git a/sftp.c b/sftp.c
index f0daaefa4..969328de4 100644
--- a/sftp.c
+++ b/sftp.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: sftp.c,v 1.147 2013/07/12 00:20:00 djm Exp $ */
+/* $OpenBSD: sftp.c,v 1.148 2013/07/25 00:56:52 djm Exp $ */
 /*
  * Copyright (c) 2001-2004 Damien Miller <djm@openbsd.org>
  *
@@ -88,6 +88,9 @@ int showprogress = 1;
 /* When this option is set, we always recursively download/upload directories */
 int global_rflag = 0;
 
+/* When this option is set, we resume download if possible */
+int global_aflag = 0;
+
 /* When this option is set, the file transfers will always preserve times */
 int global_pflag = 0;
 
@@ -151,6 +154,7 @@ extern char *__progname;
 #define I_SYMLINK       21
 #define I_VERSION       22
 #define I_PROGRESS      23
+#define I_REGET         26
 
 struct CMD {
         const char *c;
@@ -190,6 +194,7 @@ static const struct CMD cmds[] = {
         { "put",        I_PUT,          LOCAL   },
         { "pwd",        I_PWD,          REMOTE  },
         { "quit",       I_QUIT,         NOARGS  },
+        { "reget",      I_REGET,        REMOTE  },
         { "rename",     I_RENAME,       REMOTE  },
         { "rm",         I_RM,           REMOTE  },
         { "rmdir",      I_RMDIR,        REMOTE  },
@@ -239,6 +244,7 @@ help(void)
             "                                   filesystem containing 'path'\n"
             "exit                               Quit sftp\n"
             "get [-Ppr] remote [local]          Download file\n"
+            "reget remote [local]               Resume download file\n"
             "help                               Display this help text\n"
             "lcd path                           Change local directory to 'path'\n"
             "lls [ls-options [path]]            Display local directory listing\n"
@@ -350,8 +356,8 @@ make_absolute(char *p, char *pwd)
 }
 
 static int
-parse_getput_flags(const char *cmd, char **argv, int argc, int *pflag,
-    int *rflag)
+parse_getput_flags(const char *cmd, char **argv, int argc,
+    int *aflag, int *pflag, int *rflag)
 {
         extern int opterr, optind, optopt, optreset;
         int ch;
@@ -359,9 +365,12 @@ parse_getput_flags(const char *cmd, char **argv, int argc, int *pflag,
         optind = optreset = 1;
         opterr = 0;
 
-        *rflag = *pflag = 0;
-        while ((ch = getopt(argc, argv, "PpRr")) != -1) {
+        *aflag = *rflag = *pflag = 0;
+        while ((ch = getopt(argc, argv, "aPpRr")) != -1) {
                 switch (ch) {
+                case 'a':
+                        *aflag = 1;
+                        break;
                 case 'p':
                 case 'P':
                         *pflag = 1;
@@ -519,7 +528,7 @@ pathname_is_dir(char *pathname)
 
 static int
 process_get(struct sftp_conn *conn, char *src, char *dst, char *pwd,
-    int pflag, int rflag)
+    int pflag, int rflag, int resume)
 {
         char *abs_src = NULL;
         char *abs_dst = NULL;
@@ -571,15 +580,18 @@ process_get(struct sftp_conn *conn, char *src, char *dst, char *pwd,
                 }
                 free(tmp);
 
-                if (!quiet)
+                resume |= global_aflag;
+                if (!quiet && resume)
+                        printf("Resuming %s to %s\n", g.gl_pathv[i], abs_dst);
+                else if (!quiet && !resume)
                         printf("Fetching %s to %s\n", g.gl_pathv[i], abs_dst);
                 if (pathname_is_dir(g.gl_pathv[i]) && (rflag || global_rflag)) {
-                        if (download_dir(conn, g.gl_pathv[i], abs_dst, NULL, 
-                            pflag || global_pflag, 1) == -1)
+                        if (download_dir(conn, g.gl_pathv[i], abs_dst, NULL,
+                            pflag || global_pflag, 1, resume) == -1)
                                 err = -1;
                 } else {
                         if (do_download(conn, g.gl_pathv[i], abs_dst, NULL,
-                            pflag || global_pflag) == -1)
+                            pflag || global_pflag, resume) == -1)
                                 err = -1;
                 }
                 free(abs_dst);
@@ -1118,8 +1130,9 @@ makeargv(const char *arg, int *argcp, int sloppy, char *lastquote,
 }
 
 static int
-parse_args(const char **cpp, int *pflag, int *rflag, int *lflag, int *iflag,
-    int *hflag, int *sflag, unsigned long *n_arg, char **path1, char **path2)
+parse_args(const char **cpp, int *aflag, int *hflag, int *iflag, int *lflag,
+    int *pflag, int *rflag, int *sflag, unsigned long *n_arg,
+    char **path1, char **path2)
 {
         const char *cmd, *cp = *cpp;
         char *cp2, **argv;
@@ -1163,14 +1176,15 @@ parse_args(const char **cpp, int *pflag, int *rflag, int *lflag, int *iflag,
         }
 
         /* Get arguments and parse flags */
-        *lflag = *pflag = *rflag = *hflag = *n_arg = 0;
+        *aflag = *lflag = *pflag = *rflag = *hflag = *n_arg = 0;
         *path1 = *path2 = NULL;
         optidx = 1;
         switch (cmdnum) {
         case I_GET:
+        case I_REGET:
         case I_PUT:
                 if ((optidx = parse_getput_flags(cmd, argv, argc,
-                    pflag, rflag)) == -1)
+                    aflag, pflag, rflag)) == -1)
                         return -1;
                 /* Get first pathname (mandatory) */
                 if (argc - optidx < 1) {
@@ -1185,6 +1199,11 @@ parse_args(const char **cpp, int *pflag, int *rflag, int *lflag, int *iflag,
                         /* Destination is not globbed */
                         undo_glob_escape(*path2);
                 }
+                if (*aflag && cmdnum == I_PUT) {
+                        /* XXX implement resume for uploads */
+                        error("Resume is not supported for uploads");
+                        return -1;
+                }
                 break;
         case I_LINK:
                 if ((optidx = parse_link_flags(cmd, argv, argc, sflag)) == -1)
@@ -1293,7 +1312,8 @@ parse_dispatch_command(struct sftp_conn *conn, const char *cmd, char **pwd,
     int err_abort)
 {
         char *path1, *path2, *tmp;
-        int pflag = 0, rflag = 0, lflag = 0, iflag = 0, hflag = 0, sflag = 0;
+        int aflag = 0, hflag = 0, iflag = 0, lflag = 0, pflag = 0;
+        int rflag = 0, sflag = 0;
         int cmdnum, i;
         unsigned long n_arg = 0;
         Attrib a, *aa;
@@ -1302,9 +1322,8 @@ parse_dispatch_command(struct sftp_conn *conn, const char *cmd, char **pwd,
         glob_t g;
 
         path1 = path2 = NULL;
-        cmdnum = parse_args(&cmd, &pflag, &rflag, &lflag, &iflag, &hflag,
-            &sflag, &n_arg, &path1, &path2);
-
+        cmdnum = parse_args(&cmd, &aflag, &hflag, &iflag, &lflag, &pflag,
+            &rflag, &sflag, &n_arg, &path1, &path2);
         if (iflag != 0)
                 err_abort = 0;
 
@@ -1319,8 +1338,12 @@ parse_dispatch_command(struct sftp_conn *conn, const char *cmd, char **pwd,
                 /* Unrecognized command */
                 err = -1;
                 break;
+        case I_REGET:
+                aflag = 1;
+                /* FALLTHROUGH */
         case I_GET:
-                err = process_get(conn, path1, path2, *pwd, pflag, rflag);
+                err = process_get(conn, path1, path2, *pwd, pflag,
+                    rflag, aflag);
                 break;
         case I_PUT:
                 err = process_put(conn, path1, path2, *pwd, pflag, rflag);
@@ -1949,12 +1972,10 @@ interactive_loop(struct sftp_conn *conn, char *file1, char *file2)
                         }
                 } else {
                         /* XXX this is wrong wrt quoting */
-                        if (file2 == NULL)
-                                snprintf(cmd, sizeof cmd, "get %s", dir);
-                        else
-                                snprintf(cmd, sizeof cmd, "get %s %s", dir,
-                                    file2);
-
+                        snprintf(cmd, sizeof cmd, "get%s %s%s%s",
+                            global_aflag ? " -a" : "", dir,
+                            file2 == NULL ? "" : " ",
+                            file2 == NULL ? "" : file2);
                         err = parse_dispatch_command(conn, cmd,
                             &remote_path, 1);
                         free(dir);
@@ -2143,7 +2164,7 @@ main(int argc, char **argv)
         infile = stdin;
 
         while ((ch = getopt(argc, argv,
-            "1246hpqrvCc:D:i:l:o:s:S:b:B:F:P:R:")) != -1) {
+            "1246ahpqrvCc:D:i:l:o:s:S:b:B:F:P:R:")) != -1) {
                 switch (ch) {
                 /* Passed through to ssh(1) */
                 case '4':
@@ -2183,6 +2204,9 @@ main(int argc, char **argv)
                 case '2':
                         sshver = 2;
                         break;
+                case 'a':
+                        global_aflag = 1;
+                        break;
                 case 'B':
                         copy_buffer_len = strtol(optarg, &cp, 10);
                         if (copy_buffer_len == 0 || *cp != '\0')