Implement copy_to and move_to for S3_File (#9054)

- Closes #8833
- Tests for copying between S3 and `Enso_File` will only be added once we implement Enso_File writing.

CHANGELOG.md

@@ -616,6 +616,7 @@
 - [Added `Table.to_xml`.][8979]
 - [Implemented Write support for `S3_File`.][8921]
 - [Separate `Group_By` from `columns` into new argument on `aggregate`.][9027]
+- [Allow `copy_to` and `move_to` to work between local and S3 files.][9054]
 
 [debug-shortcuts]:
   https://github.com/enso-org/enso/blob/develop/app/gui/docs/product/shortcuts.md#debug
@@ -888,6 +889,7 @@
 [8979]: https://github.com/enso-org/enso/pull/8979
 [8921]: https://github.com/enso-org/enso/pull/8921
 [9027]: https://github.com/enso-org/enso/pull/9027
+[9054]: https://github.com/enso-org/enso/pull/9054
 
 #### Enso Compiler
 

distribution/lib/Standard/AWS/0.0.0-dev/src/Internal/Request_Body.enso

@@ -1,10 +1,11 @@
 private
 from Standard.Base import all
 from Standard.Base.System.File import file_as_java
+import Standard.Base.Errors.File_Error.File_Error
 
 polyglot java import software.amazon.awssdk.core.sync.RequestBody
 
 ## PRIVATE
-from_local_file (file : File) =
+from_local_file (file : File) = File_Error.handle_java_exceptions file <|
     java_file = file_as_java file
     RequestBody.fromFile java_file

distribution/lib/Standard/AWS/0.0.0-dev/src/Internal/S3_File_Write_Strategy.enso

@@ -6,10 +6,12 @@ import Standard.Base.Errors.File_Error.File_Error
 from Standard.Base.System.File.Generic.File_Write_Strategy import File_Write_Strategy, default_overwrite, default_append, default_raise_error, generic_remote_write_with_local_file
 
 import project.Errors.S3_Error
+import project.S3.S3
+import project.S3.S3_File.S3_File
 
 ## PRIVATE
 instance =
-    File_Write_Strategy.Value default_overwrite default_append default_raise_error s3_backup create_dry_run_file remote_write_with_local_file
+    File_Write_Strategy.Value default_overwrite default_append default_raise_error s3_backup create_dry_run_file remote_write_with_local_file copy_from_local
 
 ## PRIVATE
 create_dry_run_file file copy_original =
@@ -47,6 +49,12 @@ s3_backup file action = recover_errors <|
     with_failure_handler revert_backup <|
         file.with_output_stream [File_Access.Write, File_Access.Truncate_Existing] action
 
+## PRIVATE
+copy_from_local (source : File) (destination : S3_File) (replace_existing : Boolean) =
+    if replace_existing.not && destination.exists then Error.throw (File_Error.Already_Exists destination) else
+        S3.upload_file source destination.bucket destination.prefix destination.credentials . if_not_error <|
+            destination
+
 ## PRIVATE
 with_failure_handler ~failure_action ~action =
     panic_handler caught_panic =

distribution/lib/Standard/AWS/0.0.0-dev/src/S3/S3.enso

@@ -10,6 +10,7 @@ import project.Errors.More_Records_Available
 import project.Errors.S3_Bucket_Not_Found
 import project.Errors.S3_Key_Not_Found
 import project.Errors.S3_Error
+import project.Internal.Request_Body
 
 polyglot java import java.io.IOException
 polyglot java import org.enso.aws.ClientBuilder
@@ -142,15 +143,20 @@ get_object bucket key credentials:(AWS_Credential | Nothing)=Nothing = handle_s3
 put_object (bucket : Text) (key : Text) credentials:(AWS_Credential | Nothing)=Nothing request_body = handle_s3_errors bucket=bucket key=key <|
     client = make_client credentials
     request = PutObjectRequest.builder.bucket bucket . key key . build
-    client.putObject request request_body
+    client.putObject request request_body . if_not_error Nothing
-    Nothing
 
 ## PRIVATE
+upload_file (local_file : File) (bucket : Text) (key : Text) credentials:(AWS_Credential | Nothing)=Nothing = handle_s3_errors bucket=bucket key=key <|
+    request_body = Request_Body.from_local_file local_file
+    put_object bucket key credentials request_body
+
+## PRIVATE
+   Deletes the object.
+   It will not raise any errors if the object does not exist.
 delete_object (bucket : Text) (key : Text) credentials:(AWS_Credential | Nothing)=Nothing = handle_s3_errors bucket=bucket key=key <|
     client = make_client credentials
     request = DeleteObjectRequest.builder . bucket bucket . key key . build
-    client.deleteObject request
+    client.deleteObject request . if_not_error Nothing
-    Nothing
 
 ## PRIVATE
 copy_object (source_bucket : Text) (source_key : Text) (target_bucket : Text) (target_key : Text) credentials:(AWS_Credential | Nothing)=Nothing = handle_s3_errors bucket=source_bucket key=source_key <|
@@ -161,8 +167,7 @@ copy_object (source_bucket : Text) (source_key : Text) (target_bucket : Text) (t
         . sourceBucket source_bucket
         . sourceKey source_key
         . build
-    client.copyObject request
+    client.copyObject request . if_not_error Nothing
-    Nothing
 
 ## PRIVATE
    Splits a S3 URI into bucket and key.

distribution/lib/Standard/AWS/0.0.0-dev/src/S3/S3_File.enso

@@ -11,11 +11,11 @@ import Standard.Base.System.File.Generic.Writable_File.Writable_File
 import Standard.Base.System.Input_Stream.Input_Stream
 import Standard.Base.System.Output_Stream.Output_Stream
 from Standard.Base.System.File import find_extension_from_name
+from Standard.Base.System.File.Generic.File_Write_Strategy import generic_copy
 
 import project.AWS_Credential.AWS_Credential
 import project.Errors.S3_Error
 import project.Errors.S3_Key_Not_Found
-import project.Internal.Request_Body
 import project.Internal.S3_File_Write_Strategy
 import project.S3.S3
 
@@ -49,7 +49,7 @@ type S3_File
     exists self = if self.bucket == "" then True else
         if self.prefix == "" then translate_file_errors self <| S3.head self.bucket "" self.credentials . is_error . not else
             pair = translate_file_errors self <| S3.read_bucket self.bucket self.prefix self.credentials max_count=1
-            pair.second.length > 0
+            pair.second.contains self.prefix
 
     ## GROUP Standard.Base.Metadata
        Checks if this is a folder.
@@ -98,8 +98,7 @@ type S3_File
                             result = tmp_file.with_output_stream [File_Access.Write] action
                             # Only proceed if the write succeeded
                             result.if_not_error <|
-                                request_body = Request_Body.from_local_file tmp_file
+                                (translate_file_errors self <| S3.upload_file tmp_file self.bucket self.prefix self.credentials) . if_not_error <|
-                                (translate_file_errors self <| S3.put_object self.bucket self.prefix self.credentials request_body) . if_not_error <|
                                     result
 
 
@@ -172,12 +171,17 @@ type S3_File
     read_text self (encoding=Encoding.utf_8) (on_problems=Problem_Behavior.Report_Warning) =
         self.read (Plain_Text encoding) on_problems
 
-    ## UNSTABLE
+    ## Deletes the object.
-       Deletes the object.
     delete : Nothing
     delete self = if self.is_directory then Error.throw (S3_Error.Error "Deleting S3 folders is currently not implemented." self.uri) else
+        if self.exists.not then Error.throw (File_Error.Not_Found self) else
+            self.delete_if_exists
+
+    ## Deletes the file if it had existed.
+    delete_if_exists : Nothing
+    delete_if_exists self = if self.is_directory then Error.throw (S3_Error.Error "Deleting S3 folders is currently not implemented." self.uri) else
         if Context.Output.is_enabled.not then Error.throw (Forbidden_Operation.Error "Deleting an S3_File is forbidden as the Output context is disabled.") else
-            translate_file_errors self <| S3.delete_object self.bucket self.prefix self.credentials . if_not_error <| Nothing
+            translate_file_errors self <| S3.delete_object self.bucket self.prefix self.credentials . if_not_error Nothing
 
     ## Copies the file to the specified destination.
 
@@ -185,8 +189,8 @@ type S3_File
        - destination: the destination to move the file to.
        - replace_existing: specifies if the operation should proceed if the
          destination file already exists. Defaults to `False`.
-    copy_to : S3_File -> Boolean -> Any ! File_Error
+    copy_to : Writable_File -> Boolean -> Any ! File_Error
-    copy_to self (destination : Writable_File) replace_existing=False =
+    copy_to self (destination : Writable_File) (replace_existing : Boolean = False) =
         if self.is_directory then Error.throw (S3_Error.Error "Copying S3 folders is currently not implemented." self.uri) else
             if Context.Output.is_enabled.not then Error.throw (Forbidden_Operation.Error "Copying an S3_File is forbidden as the Output context is disabled.") else
                 case destination.file of
@@ -194,13 +198,27 @@ type S3_File
                     s3_destination : S3_File ->
                         if replace_existing.not && s3_destination.exists then Error.throw (File_Error.Already_Exists destination) else
                             translate_file_errors self <| S3.copy_object self.bucket self.prefix s3_destination.bucket s3_destination.prefix self.credentials . if_not_error <| s3_destination
+                    _ -> generic_copy self destination.file replace_existing
 
-                    # Generic implementation using streams
+    ## Moves the file to the specified destination.
-                    _ ->
+
-                        self.with_input_stream [File_Access.Read] input_stream->
+       ! S3 Move is a Copy and Delete
-                            open_settings = if replace_existing then [File_Access.Write, File_Access.Create, File_Access.Truncate_Existing] else [File_Access.Write, File_Access.Create_New]
+
-                            destination.with_output_stream open_settings output_stream->
+         Since S3 does not support moving files, this operation is implemented
-                                output_stream.write_stream input_stream
+         as a copy followed by delete. Keep in mind that the space usage of the
+         file will briefly be doubled and that the operation may not be as fast
+         as a local move often is.
+
+       Arguments:
+       - destination: the destination to move the file to.
+       - replace_existing: specifies if the operation should proceed if the
+         destination file already exists. Defaults to `False`.
+    move_to : Writable_File -> Boolean -> Nothing ! File_Error
+    move_to self (destination : Writable_File) (replace_existing : Boolean = False) =
+        if Context.Output.is_enabled.not then Error.throw (Forbidden_Operation.Error "File moving is forbidden as the Output context is disabled.") else
+            r = self.copy_to destination replace_existing=replace_existing
+            r.if_not_error <|
+                self.delete.if_not_error r
 
     ## GROUP Standard.Base.Operators
        Join two path segments together.

distribution/lib/Standard/Base/0.0.0-dev/src/Data/Enso_Cloud/Enso_File.enso

@@ -10,6 +10,7 @@ import project.Data.Text.Text_Sub_Range.Text_Sub_Range
 import project.Data.Time.Date_Time.Date_Time
 import project.Data.Vector.Vector
 import project.Error.Error
+import project.Errors.Common.Forbidden_Operation
 import project.Errors.Common.Not_Found
 import project.Errors.File_Error.File_Error
 import project.Errors.Illegal_Argument.Illegal_Argument
@@ -20,12 +21,15 @@ import project.Network.HTTP.HTTP
 import project.Network.HTTP.HTTP_Method.HTTP_Method
 import project.Network.HTTP.Request_Body.Request_Body
 import project.Nothing.Nothing
+import project.Runtime.Context
 import project.System.File.File_Access.File_Access
+import project.System.File.Generic.Writable_File.Writable_File
 import project.System.File_Format_Metadata.File_Format_Metadata
 import project.System.Input_Stream.Input_Stream
 import project.System.Output_Stream.Output_Stream
 from project.Data.Boolean import Boolean, False, True
 from project.Data.Text.Extensions import all
+from project.System.File.Generic.File_Write_Strategy import generic_copy
 from project.System.File_Format import Auto_Detect, Bytes, File_Format, Plain_Text_Format
 
 type Enso_File
@@ -253,7 +257,37 @@ type Enso_File
         uri = if self.is_directory then Utils.directory_api + "/" + self.id else self.internal_uri
         auth_header = Utils.authorization_header
         response = HTTP.post uri Request_Body.Empty HTTP_Method.Delete [auth_header]
-        response.if_not_error <| Nothing
+        response.if_not_error Nothing
+
+    ## Deletes the file if it had existed.
+    delete_if_exists : Nothing
+    delete_if_exists self =
+        r = self.delete
+        r.catch File_Error err-> case err of
+            File_Error.Not_Found _ -> Nothing
+            _ -> r
+
+    ## Copies the file to the specified destination.
+
+       Arguments:
+       - destination: the destination to move the file to.
+       - replace_existing: specifies if the operation should proceed if the
+         destination file already exists. Defaults to `False`.
+    copy_to : Writable_File -> Boolean -> Any ! File_Error
+    copy_to self (destination : Writable_File) (replace_existing : Boolean = False) =
+        generic_copy self destination.file replace_existing
+
+    ## Moves the file to the specified destination.
+
+       Arguments:
+       - destination: the destination to move the file to.
+       - replace_existing: specifies if the operation should proceed if the
+         destination file already exists. Defaults to `False`.
+    move_to : Writable_File -> Boolean -> Nothing ! File_Error
+    move_to self (destination : Writable_File) (replace_existing : Boolean = False) =
+        _ = [destination, replace_existing]
+        if Context.Output.is_enabled.not then Error.throw (Forbidden_Operation.Error "File moving is forbidden as the Output context is disabled.") else
+            Unimplemented.throw "Enso_File.move_to is not implemented"
 
     ## UNSTABLE
        GROUP Operators

distribution/lib/Standard/Base/0.0.0-dev/src/Errors/File_Error.enso

@@ -12,6 +12,7 @@ import project.System.File_Format_Metadata.File_Format_Metadata
 
 polyglot java import java.io.FileNotFoundException
 polyglot java import java.io.IOException
+polyglot java import java.io.UncheckedIOException
 polyglot java import java.nio.file.AccessDeniedException
 polyglot java import java.nio.file.FileAlreadyExistsException
 polyglot java import java.nio.file.NoSuchFileException
@@ -71,8 +72,13 @@ type File_Error
 
        Utility method for running an action with Java exceptions mapping.
     handle_java_exceptions (file : File | Nothing) ~action =
-        Panic.catch IOException action caught_panic->
+        handle_io_exception caught_panic =
             File_Error.wrap_io_exception file caught_panic.payload
+        handle_unchecked_io_exception caught_panic =
+            File_Error.wrap_io_exception file caught_panic.payload.getCause
+        Panic.catch IOException handler=handle_io_exception <|
+            Panic.catch UncheckedIOException handler=handle_unchecked_io_exception <|
+                action
 
     ## PRIVATE
        Raises an error indicating that the user does not have permission to

distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso

@@ -636,14 +636,12 @@ type File
        - destination: the destination to move the file to.
        - replace_existing: specifies if the operation should proceed if the
          destination file already exists. Defaults to `False`.
-    copy_to : File -> Boolean -> Nothing ! File_Error
+    copy_to : Writable_File -> Boolean -> Nothing ! File_Error
-    copy_to self destination replace_existing=False =
+    copy_to self (destination : Writable_File) (replace_existing : Boolean = False) =
         if Context.Output.is_enabled.not then Error.throw (Forbidden_Operation.Error "File copying is forbidden as the Output context is disabled.") else
-            File_Error.handle_java_exceptions self <| case replace_existing of
+            case destination.file of
-                True ->
+                _ : File -> local_file_copy self destination.file replace_existing
-                  copy_options = [StandardCopyOption.REPLACE_EXISTING]
+                _ -> destination.copy_from_local self replace_existing
-                  self.copy_builtin destination copy_options
-                False -> self.copy_builtin destination []
 
     ## Moves the file to the specified destination.
 
@@ -651,14 +649,15 @@ type File
        - destination: the destination to move the file to.
        - replace_existing: specifies if the operation should proceed if the
          destination file already exists. Defaults to `False`.
-    move_to : File -> Boolean -> Nothing ! File_Error
+    move_to : Writable_File -> Boolean -> Nothing ! File_Error
-    move_to self destination replace_existing=False =
+    move_to self (destination : Writable_File) (replace_existing : Boolean = False) =
         if Context.Output.is_enabled.not then Error.throw (Forbidden_Operation.Error "File moving is forbidden as the Output context is disabled.") else
-            File_Error.handle_java_exceptions self <| case replace_existing of
+            case destination.file of
-                True ->
+                _ : File -> local_file_move self destination.file replace_existing
-                  copy_options = [StandardCopyOption.REPLACE_EXISTING]
+                _ ->
-                  self.move_builtin destination copy_options
+                    r = destination.copy_from_local self replace_existing
-                False -> self.move_builtin destination []
+                    r.if_not_error <|
+                        self.delete . if_not_error r
 
     ## Deletes the file if it exists on disk.
 
@@ -843,3 +842,17 @@ File_Like.from (that : File) = File_Like.Value that
 ## PRIVATE
 Writable_File.from (that : File) =
     Writable_File.Value that.absolute.normalize Local_File_Write_Strategy.instance
+
+## PRIVATE
+local_file_copy : File -> File -> Boolean -> Nothing ! File_Error
+local_file_copy source destination replace_existing =
+    File_Error.handle_java_exceptions source <|
+        copy_options = if replace_existing then [StandardCopyOption.REPLACE_EXISTING] else []
+        source.copy_builtin destination copy_options
+
+## PRIVATE
+local_file_move : File -> File -> Boolean -> Nothing ! File_Error
+local_file_move source destination replace_existing =
+    File_Error.handle_java_exceptions source <|
+        copy_options = if replace_existing then [StandardCopyOption.REPLACE_EXISTING] else []
+        source.move_builtin destination copy_options

distribution/lib/Standard/Base/0.0.0-dev/src/System/File/Generic/File_Like.enso

@@ -1,4 +1,5 @@
 import project.Data.Text.Text
+import project.System.File.File
 
 ## PRIVATE
    A generic interface for file-like objects.
@@ -19,3 +20,6 @@ type File_Like
 
     ## PRIVATE
     to_display_text self -> Text = self.underlying.to_display_text
+
+## PRIVATE
+File_Like.from (that : Text) = File_Like.from (File.new that)

distribution/lib/Standard/Base/0.0.0-dev/src/System/File/Generic/File_Write_Strategy.enso

@@ -31,7 +31,12 @@ type File_Write_Strategy
        A remote file is downloaded to a temporary file and the provided action
        is called with that local temporary file. Then that file is uploaded to
        replace the remote file.
-    Value write_overwriting write_appending write_raising_error write_backing_up create_dry_run_file write_with_local_file
+
+       The `copy_from_local` action creates the file on a given backend from a
+       local file. It can be used to implement more efficient upload strategies
+       than ones based on just writing to an output stream.
+       The default `generic_copy` implementation can always be used here.
+    Value write_overwriting write_appending write_raising_error write_backing_up create_dry_run_file write_with_local_file copy_from_local
 
     ## PRIVATE
        Writes to a file according to the provided existing file behaviour.
@@ -128,3 +133,12 @@ dry_run_behavior file behavior:Existing_File_Behavior -> Dry_Run_File_Settings =
         Dry_Run_File_Settings.Value Existing_File_Behavior.Overwrite copy_original=False
     Existing_File_Behavior.Append ->
         Dry_Run_File_Settings.Value Existing_File_Behavior.Append copy_original=True
+
+## PRIVATE
+   Generic `copy` implementation between two backends.
+   The files only need to support `with_input_stream` and `with_output_stream`.
+generic_copy source destination replace_existing =
+    source.with_input_stream [File_Access.Read] input_stream->
+        options = if replace_existing then [File_Access.Write, File_Access.Create, File_Access.Truncate_Existing] else [File_Access.Write, File_Access.Create_New]
+        destination.with_output_stream options output_stream->
+            output_stream.write_stream input_stream . if_not_error destination

distribution/lib/Standard/Base/0.0.0-dev/src/System/File/Generic/Writable_File.enso

@@ -54,6 +54,13 @@ type Writable_File
     write_requiring_local_file self (existing_file_behavior : Existing_File_Behavior) (action : File -> Any) -> Any =
         self.write_strategy.write_with_local_file self.file existing_file_behavior action
 
+    ## PRIVATE
+       Writes a local file to this `Writable_File` destination.
+       This is used by `File.copy_to` and `File.move_to` to possibly implement
+       the upload more efficiently (avoiding duplicated temporary files).
+    copy_from_local self (source : File) (replace_existing : Boolean) =
+        self.write_strategy.copy_from_local source self.file replace_existing
+
     ## PRIVATE
     with_output_stream self (open_options : Vector) action =
         self.file.with_output_stream open_options action
@@ -69,10 +76,7 @@ type Writable_File
     to_display_text self -> Text = self.file.to_display_text
 
 ## PRIVATE
-Writable_File.from (that : Text) =
+Writable_File.from (that : Text) = Writable_File.from (File.new that)
-    ## Currently this only works for local filesystem paths
-       TODO We should extend it to also support custom paths like S3, through a ServiceProvider solution
-    Writable_File.from (File.new that)
 
 ## PRIVATE
    If a conversion to `File_Format_Metadata` is needed, we delegate to the underlying file.

distribution/lib/Standard/Base/0.0.0-dev/src/System/File/Local_File_Write_Strategy.enso

@@ -6,6 +6,7 @@ import project.Errors.Illegal_State.Illegal_State
 import project.Nothing.Nothing
 import project.Panic.Caught_Panic
 import project.Panic.Panic
+import project.System.File.File
 import project.System.File.File_Access.File_Access
 import project.System.Output_Stream.Output_Stream
 from project.Data.Boolean import Boolean, False, True
@@ -13,12 +14,15 @@ from project.System.File.Generic.File_Write_Strategy import File_Write_Strategy,
 
 ## PRIVATE
 instance =
-    File_Write_Strategy.Value default_overwrite default_append default_raise_error moving_backup create_dry_run_file write_with_local_file
+    File_Write_Strategy.Value default_overwrite default_append default_raise_error moving_backup create_dry_run_file write_with_local_file copy_local_from_local
 
 ## PRIVATE
 create_dry_run_file file copy_original =
     file.create_dry_run_file copy_original
 
+## PRIVATE
+copy_local_from_local (source : File) (destination : File) = source.copy_to destination
+
 ## PRIVATE
    A `Backup` strategy that does the following:
    1. If the file does not exist, we write to it.

test/AWS_Tests/src/Main.enso

@@ -4,7 +4,7 @@ from Standard.Test import all
 
 import project.S3_Spec
 
-main =
+main filter=Nothing =
     suite = Test.build suite_builder->
         S3_Spec.add_specs suite_builder
-    suite.run_with_filter
+    suite.run_with_filter filter

test/AWS_Tests/src/S3_Spec.enso

@@ -156,6 +156,10 @@ add_specs suite_builder =
             r.should_be_a Vector
             r.at 0 . get "name" . should_equal "Green St Green"
 
+        group_builder.specify "should work with Data.read" <|
+            r = Data.read "s3://"+bucket_name+"/examples/folder 2/hello.txt"
+            r.should_equal "Hello WORLD!"
+
         group_builder.specify "should be able to read a file as bytes or stream" <|
             bytes = hello_txt.read_bytes
             bytes.should_equal "Hello WORLD!".utf_8
@@ -211,17 +215,18 @@ add_specs suite_builder =
             # AWS S3 does not record creation time, only last modified time.
             hello_txt.creation_time . should_fail_with S3_Error
 
+    writable_root = S3_File.new "s3://"+writable_bucket_name+"/"
+    my_writable_dir = writable_root / "test-run-"+(Date_Time.now.format "yyyy-MM-dd_HHmmss.fV")+"/"
+    delete_on_panic file ~action =
+        handler caught_panic =
+            file.delete
+            Panic.throw caught_panic
+        Panic.catch Any action handler
+    delete_afterwards file ~action =
+        Panic.with_finalizer file.delete action
+
     suite_builder.group "S3_File writing" pending=api_pending group_builder->
-        writable_root = S3_File.new "s3://"+writable_bucket_name+"/"
-        my_writable_dir = writable_root / "test-run-"+(Date_Time.now.format "yyyy-MM-dd_HHmmss.fV")+"/"
         assert my_writable_dir.is_directory
-        delete_on_panic file ~action =
-            handler caught_panic =
-                file.delete
-                Panic.throw caught_panic
-            Panic.catch Any action handler
-        delete_afterwards file ~action =
-            Panic.with_finalizer file.delete action
 
         group_builder.specify "should be able to write and delete a new file" <|
             new_file = my_writable_dir / "new_file1.txt"
@@ -268,43 +273,6 @@ add_specs suite_builder =
                 r.catch.to_display_text . should_contain "already exists"
                 r.catch.to_display_text . should_contain "new_file-exists.txt"
 
-        group_builder.specify "should be able to copy a file" <|
-            base_file = my_writable_dir / "new_file-for-copy.txt"
-            "Hello".write base_file . should_succeed
-            delete_afterwards base_file <|
-                base_file.read . should_equal "Hello"
-
-                dest_file = my_writable_dir / "new_file-the-copy-2.txt"
-                base_file.copy_to dest_file . should_succeed
-                delete_afterwards dest_file <|
-                    dest_file.read . should_equal "Hello"
-
-        group_builder.specify "will fail if source file does not exist" <|
-            base_file = my_writable_dir / "nonexistent-src-file.txt"
-            dest_file = my_writable_dir / "nonexistent-dest-file.txt"
-            r = base_file.copy_to dest_file
-            r.should_fail_with File_Error
-            r.catch.should_be_a File_Error.Not_Found
-            dest_file.exists . should_be_false
-
-        group_builder.specify "respects replace_existing setting in copy_to" <|
-            base_file = my_writable_dir / "new_file-for-copy-2.txt"
-            "Hello".write base_file . should_succeed
-            delete_afterwards base_file <|
-                dest_file = my_writable_dir / "new_file-dest.txt"
-                "World".write dest_file . should_succeed
-                delete_afterwards dest_file <|
-                    r1 = base_file.copy_to dest_file replace_existing=False
-                    r1.should_fail_with File_Error
-                    r1.catch.should_be_a File_Error.Already_Exists
-                    r1.catch.to_display_text . should_contain "already exists"
-
-                    dest_file.read . should_equal "World"
-
-                    # Now allow the overwrite:
-                    r2 = base_file.copy_to dest_file replace_existing=True
-                    r2.should_equal dest_file
-
         group_builder.specify "should be able to write a raw stream" <|
             new_file = my_writable_dir / "new_file-stream.txt"
             r = new_file.with_output_stream [File_Access.Write] stream->
@@ -348,7 +316,7 @@ add_specs suite_builder =
             bak_file.exists.should_be_false
 
             "version1".write my_file . should_succeed
-            delete_afterwards my_file <|
+            delete_on_panic my_file <|
                 my_file.read . should_equal "version1"
                 bak_file.exists . should_be_false
 
@@ -371,6 +339,14 @@ add_specs suite_builder =
                     parent_dir = my_file.parent
                     parent_dir.list . should_contain_the_same_elements_as [my_file, bak_file]
 
+                    # If the original file is deleted and the backup file remains, the original file should _not_ count as existing (this used to fail).
+                    my_file.delete
+                    bak_file.exists . should_be_true
+                    my_file.exists . should_be_false
+                    files = my_file.parent.list
+                    files . should_contain bak_file
+                    files . should_not_contain my_file
+
         group_builder.specify "should fail cleanly if Auto_Detect fails to detect a format" <|
             weird_ext = my_writable_dir / "weird-ext.unknown"
             "Hello".write weird_ext . should_succeed
@@ -432,14 +408,94 @@ add_specs suite_builder =
 
         group_builder.specify "should fail to delete a file if the Output context is not enabled" <|
             Context.Output.with_disabled <|
-                new_file = my_writable_dir / "new_file-ctx.txt"
+                hello_txt = S3_File.new "s3://"+bucket_name+"/examples/folder 2/hello.txt"
-                new_file.delete . should_fail_with Forbidden_Operation
+                hello_txt.delete . should_fail_with Forbidden_Operation
 
-        group_builder.specify "does not raise an exception if the file being deleted did not exist in the first place" <|
+        group_builder.specify "may fail with Not_Found if the file to delete does not exist, even if the Output Context is disabled" <|
+            Context.Output.with_disabled <|
+                new_file = my_writable_dir / "nonexistent-file.txt"
+                r = new_file.delete
+                r.should_fail_with File_Error
+                r.catch.should_be_a File_Error.Not_Found
+
+        group_builder.specify "does not raise an exception if the file being `delete_if_exists` did not exist in the first place" <|
             new_file = my_writable_dir / "nonexistent-file.txt"
-            new_file.delete . should_succeed
+            new_file.delete_if_exists . should_succeed
 
-main =
+        group_builder.specify "fails if the file being deleted did not exist" <|
+            new_file = my_writable_dir / "nonexistent-file2.txt"
+            r = new_file.delete
+            r.should_fail_with File_Error
+            r.catch.should_be_a File_Error.Not_Found
+
+    sources = [my_writable_dir / "source1.txt", File.create_temporary_file "source2" ".txt"]
+    destinations = [my_writable_dir / "destination1.txt", File.create_temporary_file "destination2" ".txt"]
+    sources.each source_file-> destinations.each destination_file-> if source_file.is_a File && destination_file.is_a File then Nothing else
+        src_typ = Meta.type_of source_file . to_display_text
+        dest_typ = Meta.type_of destination_file . to_display_text
+        suite_builder.group "("+src_typ+" -> "+dest_typ+") copying/moving" pending=api_pending group_builder->
+            group_builder.teardown <|
+                source_file.delete_if_exists
+                destination_file.delete_if_exists
+
+            group_builder.specify "should be able to copy files" <|
+                "Hello".write source_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed
+                destination_file.delete_if_exists
+
+                source_file.copy_to destination_file . should_succeed
+                destination_file.read . should_equal "Hello"
+                source_file.exists . should_be_true
+
+            group_builder.specify "should be able to move files" <|
+                "Hello".write source_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed
+                destination_file.delete_if_exists
+
+                source_file.move_to destination_file . should_succeed
+                destination_file.read . should_equal "Hello"
+                source_file.exists . should_be_false
+
+            group_builder.specify "should fail if the source file does not exist" <|
+                source_file.delete_if_exists
+                destination_file.delete_if_exists
+
+                r = source_file.copy_to destination_file
+                r.should_fail_with File_Error
+                r.catch.should_be_a File_Error.Not_Found
+
+                r2 = source_file.move_to destination_file
+                r2.should_fail_with File_Error
+                r2.catch.should_be_a File_Error.Not_Found
+
+                destination_file.exists . should_be_false
+
+            group_builder.specify "should fail to copy/move a file if it exists and replace_existing=False" <|
+                "Hello".write source_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed
+                "World".write destination_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed
+
+                r = source_file.copy_to destination_file
+                r.should_fail_with File_Error
+                r.catch.should_be_a File_Error.Already_Exists
+
+                r2 = source_file.move_to destination_file
+                r2.should_fail_with File_Error
+                r2.catch.should_be_a File_Error.Already_Exists
+
+                destination_file.read . should_equal "World"
+
+            group_builder.specify "should overwrite existing destination in copy/move if replace_existing=True" <|
+                "Hello".write source_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed
+                "World".write destination_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed
+
+                source_file.copy_to destination_file replace_existing=True . should_succeed
+                destination_file.read . should_equal "Hello"
+                source_file.exists . should_be_true
+
+                "FooBar".write source_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed
+                source_file.move_to destination_file replace_existing=True . should_succeed
+                destination_file.read . should_equal "FooBar"
+                source_file.exists . should_be_false
+
+main filter=Nothing =
     suite = Test.build suite_builder->
         add_specs suite_builder
-    suite.run_with_filter
+    suite.run_with_filter filter