Sparse file support: create an empty sparse file for in-place restore

integration_tests/sparse_test.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Testing backup and restore of sparse files
+
+. ./test_functions.sh
+
+fixture
+
+pushd ${TEST_REPO}
+${DUPLICACY} init integration-tests $TEST_STORAGE -c 1m
+
+for i in `seq 1 10`; do
+    dd if=/dev/urandom of=file3 bs=1000 count=1000 seek=$((100000 * $i))
+done
+
+ls -lsh file3
+
+${DUPLICACY} backup 
+${DUPLICACY} check --files -stats
+
+rm file1 file3
+
+${DUPLICACY} restore -r 1
+
+ls -lsh file3
+
+popd

src/duplicacy_backupmanager.go

@@ -1150,33 +1150,36 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
     var offset int64
 
     existingFile, err = os.Open(fullPath)
-    if err != nil && !os.IsNotExist(err) {
+    if err != nil {
-        LOG_TRACE("DOWNLOAD_OPEN", "Can't open the existing file: %v", err)
+        if os.IsNotExist(err) {
-    }
+            if inPlace && entry.Size > 100 * 1024 * 1024 {
-
+                // Create an empty sparse file
-    fileHash := ""
+                existingFile, err = os.OpenFile(fullPath, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0600)
-    if existingFile != nil {
+                if err != nil {
-        // Break existing file into chunks.
+                    LOG_ERROR("DOWNLOAD_CREATE", "Failed to create the file %s for in-place writing", fullPath)
-        chunkMaker.ForEachChunk(
+                    return false
-            existingFile,
+                }
-            func (chunk *Chunk, final bool) {
+                _, err = existingFile.Seek(entry.Size - 1, 0)
-                hash := chunk.GetHash()
+                if err != nil {
-                chunkSize := chunk.GetLength()
+                    LOG_ERROR("DOWNLOAD_CREATE", "Failed to resize the initial file %s for in-place writing", fullPath)
-                existingChunks = append(existingChunks, hash)
+                    return false
-                existingLengths = append(existingLengths, chunkSize)
+                }
-                offsetMap[hash] = offset
+                _, err = existingFile.Write([]byte("\x00"))
-                lengthMap[hash] = chunkSize
+                if err != nil {
-                offset += int64(chunkSize)
+                    LOG_ERROR("DOWNLOAD_CREATE", "Failed to initialize the sparse file %s for in-place writing", fullPath)
-            },
+                    return false
-            func (fileSize int64, hash string) (io.Reader, bool) {
+                }
-                fileHash = hash
+                existingFile.Close()
-                return nil, false
+                existingFile, err = os.Open(fullPath)
-            })
+                if err != nil {
-        if fileHash == entry.Hash && fileHash != "" {
+                    LOG_ERROR("DOWNLOAD_OPEN", "Can't reopen the initial file just created: %v", err)
-            LOG_TRACE("DOWNLOAD_SKIP", "File %s unchanged (by hash)", entry.Path)
+                    return false
-            return false
+                }
+            }
+        } else {
+            LOG_TRACE("DOWNLOAD_OPEN", "Can't open the existing file: %v", err)
         }
-
+    } else {
         if !overwrite {
             LOG_ERROR("DOWNLOAD_OVERWRITE",
                       "File %s already exists.  Please specify the -overwrite option to continue", entry.Path)
@@ -1184,6 +1187,79 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
         }
     }
 
+    fileHash := ""
+    if existingFile != nil {
+
+        if inPlace {
+            // In inplace mode, we only consider chunks in the existing file with the same offsets, so we
+            // break the original file at offsets retrieved from the backup
+            buffer := make([]byte, 64 * 1024)
+            err = nil
+            for i := entry.StartChunk; i <= entry.EndChunk; i++ {
+                hasher := manager.config.NewKeyedHasher(manager.config.HashKey)
+                chunkSize := chunkDownloader.taskList[i].chunkLength
+                if i == entry.StartChunk {
+                    chunkSize -= entry.StartOffset
+                } else if i == entry.EndChunk {
+                    chunkSize = entry.EndOffset
+                }
+                count := 0
+                for count < chunkSize {
+                    n := chunkSize - count
+                    if n > cap(buffer) {
+                        n = cap(buffer)
+                    }
+                    n, err := existingFile.Read(buffer[:n])
+                    if n > 0 {
+                        hasher.Write(buffer[:n])
+                        count += n
+                    }
+                    if err == io.EOF {
+                        break
+                    }
+                    if err != nil {
+                        LOG_ERROR("DOWNLOAD_SPLIT", "Failed to read existing file: %v", err)
+                        return false
+                    }
+                }
+                if count > 0 {
+                    hash := string(hasher.Sum(nil))
+                    existingChunks = append(existingChunks, hash)
+                    existingLengths = append(existingLengths, chunkSize)
+                    offsetMap[hash] = offset
+                    lengthMap[hash] = chunkSize
+                    offset += int64(chunkSize)
+                }
+
+                if err == io.EOF {
+                    break
+                }
+            }
+        } else {
+            // If it is not inplace, we want to reuse any chunks in the existing file regardless their offets, so
+            // we run the chunk maker to split the original file.
+            chunkMaker.ForEachChunk(
+                existingFile,
+                func (chunk *Chunk, final bool) {
+                    hash := chunk.GetHash()
+                    chunkSize := chunk.GetLength()
+                    existingChunks = append(existingChunks, hash)
+                    existingLengths = append(existingLengths, chunkSize)
+                    offsetMap[hash] = offset
+                    lengthMap[hash] = chunkSize
+                    offset += int64(chunkSize)
+                },
+                func (fileSize int64, hash string) (io.Reader, bool) {
+                    fileHash = hash
+                    return nil, false
+                })
+            if fileHash == entry.Hash && fileHash != "" {
+                LOG_TRACE("DOWNLOAD_SKIP", "File %s unchanged (by hash)", entry.Path)
+                return false
+            }
+        }
+    }
+
     for i := entry.StartChunk; i <= entry.EndChunk; i++ {
         if _, found := offsetMap[chunkDownloader.taskList[i].chunkHash]; !found {
             chunkDownloader.taskList[i].needed = true