mirror of
https://github.com/enso-org/enso.git
synced 2024-12-23 10:42:05 +03:00
Implement copy_to
and move_to
for S3_File
(#9054)
- Closes #8833 - Tests for copying between S3 and `Enso_File` will only be added once we implement Enso_File writing.
This commit is contained in:
parent
f2d2f73e89
commit
642d5a691e
@ -616,6 +616,7 @@
|
||||
- [Added `Table.to_xml`.][8979]
|
||||
- [Implemented Write support for `S3_File`.][8921]
|
||||
- [Separate `Group_By` from `columns` into new argument on `aggregate`.][9027]
|
||||
- [Allow `copy_to` and `move_to` to work between local and S3 files.][9054]
|
||||
|
||||
[debug-shortcuts]:
|
||||
https://github.com/enso-org/enso/blob/develop/app/gui/docs/product/shortcuts.md#debug
|
||||
@ -888,6 +889,7 @@
|
||||
[8979]: https://github.com/enso-org/enso/pull/8979
|
||||
[8921]: https://github.com/enso-org/enso/pull/8921
|
||||
[9027]: https://github.com/enso-org/enso/pull/9027
|
||||
[9054]: https://github.com/enso-org/enso/pull/9054
|
||||
|
||||
#### Enso Compiler
|
||||
|
||||
|
@ -1,10 +1,11 @@
|
||||
private
|
||||
from Standard.Base import all
|
||||
from Standard.Base.System.File import file_as_java
|
||||
import Standard.Base.Errors.File_Error.File_Error
|
||||
|
||||
polyglot java import software.amazon.awssdk.core.sync.RequestBody
|
||||
|
||||
## PRIVATE
|
||||
from_local_file (file : File) =
|
||||
from_local_file (file : File) = File_Error.handle_java_exceptions file <|
|
||||
java_file = file_as_java file
|
||||
RequestBody.fromFile java_file
|
||||
|
@ -6,10 +6,12 @@ import Standard.Base.Errors.File_Error.File_Error
|
||||
from Standard.Base.System.File.Generic.File_Write_Strategy import File_Write_Strategy, default_overwrite, default_append, default_raise_error, generic_remote_write_with_local_file
|
||||
|
||||
import project.Errors.S3_Error
|
||||
import project.S3.S3
|
||||
import project.S3.S3_File.S3_File
|
||||
|
||||
## PRIVATE
|
||||
instance =
|
||||
File_Write_Strategy.Value default_overwrite default_append default_raise_error s3_backup create_dry_run_file remote_write_with_local_file
|
||||
File_Write_Strategy.Value default_overwrite default_append default_raise_error s3_backup create_dry_run_file remote_write_with_local_file copy_from_local
|
||||
|
||||
## PRIVATE
|
||||
create_dry_run_file file copy_original =
|
||||
@ -47,6 +49,12 @@ s3_backup file action = recover_errors <|
|
||||
with_failure_handler revert_backup <|
|
||||
file.with_output_stream [File_Access.Write, File_Access.Truncate_Existing] action
|
||||
|
||||
## PRIVATE
|
||||
copy_from_local (source : File) (destination : S3_File) (replace_existing : Boolean) =
|
||||
if replace_existing.not && destination.exists then Error.throw (File_Error.Already_Exists destination) else
|
||||
S3.upload_file source destination.bucket destination.prefix destination.credentials . if_not_error <|
|
||||
destination
|
||||
|
||||
## PRIVATE
|
||||
with_failure_handler ~failure_action ~action =
|
||||
panic_handler caught_panic =
|
||||
|
@ -10,6 +10,7 @@ import project.Errors.More_Records_Available
|
||||
import project.Errors.S3_Bucket_Not_Found
|
||||
import project.Errors.S3_Key_Not_Found
|
||||
import project.Errors.S3_Error
|
||||
import project.Internal.Request_Body
|
||||
|
||||
polyglot java import java.io.IOException
|
||||
polyglot java import org.enso.aws.ClientBuilder
|
||||
@ -142,15 +143,20 @@ get_object bucket key credentials:(AWS_Credential | Nothing)=Nothing = handle_s3
|
||||
put_object (bucket : Text) (key : Text) credentials:(AWS_Credential | Nothing)=Nothing request_body = handle_s3_errors bucket=bucket key=key <|
|
||||
client = make_client credentials
|
||||
request = PutObjectRequest.builder.bucket bucket . key key . build
|
||||
client.putObject request request_body
|
||||
Nothing
|
||||
client.putObject request request_body . if_not_error Nothing
|
||||
|
||||
## PRIVATE
|
||||
upload_file (local_file : File) (bucket : Text) (key : Text) credentials:(AWS_Credential | Nothing)=Nothing = handle_s3_errors bucket=bucket key=key <|
|
||||
request_body = Request_Body.from_local_file local_file
|
||||
put_object bucket key credentials request_body
|
||||
|
||||
## PRIVATE
|
||||
Deletes the object.
|
||||
It will not raise any errors if the object does not exist.
|
||||
delete_object (bucket : Text) (key : Text) credentials:(AWS_Credential | Nothing)=Nothing = handle_s3_errors bucket=bucket key=key <|
|
||||
client = make_client credentials
|
||||
request = DeleteObjectRequest.builder . bucket bucket . key key . build
|
||||
client.deleteObject request
|
||||
Nothing
|
||||
client.deleteObject request . if_not_error Nothing
|
||||
|
||||
## PRIVATE
|
||||
copy_object (source_bucket : Text) (source_key : Text) (target_bucket : Text) (target_key : Text) credentials:(AWS_Credential | Nothing)=Nothing = handle_s3_errors bucket=source_bucket key=source_key <|
|
||||
@ -161,8 +167,7 @@ copy_object (source_bucket : Text) (source_key : Text) (target_bucket : Text) (t
|
||||
. sourceBucket source_bucket
|
||||
. sourceKey source_key
|
||||
. build
|
||||
client.copyObject request
|
||||
Nothing
|
||||
client.copyObject request . if_not_error Nothing
|
||||
|
||||
## PRIVATE
|
||||
Splits a S3 URI into bucket and key.
|
||||
|
@ -11,11 +11,11 @@ import Standard.Base.System.File.Generic.Writable_File.Writable_File
|
||||
import Standard.Base.System.Input_Stream.Input_Stream
|
||||
import Standard.Base.System.Output_Stream.Output_Stream
|
||||
from Standard.Base.System.File import find_extension_from_name
|
||||
from Standard.Base.System.File.Generic.File_Write_Strategy import generic_copy
|
||||
|
||||
import project.AWS_Credential.AWS_Credential
|
||||
import project.Errors.S3_Error
|
||||
import project.Errors.S3_Key_Not_Found
|
||||
import project.Internal.Request_Body
|
||||
import project.Internal.S3_File_Write_Strategy
|
||||
import project.S3.S3
|
||||
|
||||
@ -49,7 +49,7 @@ type S3_File
|
||||
exists self = if self.bucket == "" then True else
|
||||
if self.prefix == "" then translate_file_errors self <| S3.head self.bucket "" self.credentials . is_error . not else
|
||||
pair = translate_file_errors self <| S3.read_bucket self.bucket self.prefix self.credentials max_count=1
|
||||
pair.second.length > 0
|
||||
pair.second.contains self.prefix
|
||||
|
||||
## GROUP Standard.Base.Metadata
|
||||
Checks if this is a folder.
|
||||
@ -98,8 +98,7 @@ type S3_File
|
||||
result = tmp_file.with_output_stream [File_Access.Write] action
|
||||
# Only proceed if the write succeeded
|
||||
result.if_not_error <|
|
||||
request_body = Request_Body.from_local_file tmp_file
|
||||
(translate_file_errors self <| S3.put_object self.bucket self.prefix self.credentials request_body) . if_not_error <|
|
||||
(translate_file_errors self <| S3.upload_file tmp_file self.bucket self.prefix self.credentials) . if_not_error <|
|
||||
result
|
||||
|
||||
|
||||
@ -172,12 +171,17 @@ type S3_File
|
||||
read_text self (encoding=Encoding.utf_8) (on_problems=Problem_Behavior.Report_Warning) =
|
||||
self.read (Plain_Text encoding) on_problems
|
||||
|
||||
## UNSTABLE
|
||||
Deletes the object.
|
||||
## Deletes the object.
|
||||
delete : Nothing
|
||||
delete self = if self.is_directory then Error.throw (S3_Error.Error "Deleting S3 folders is currently not implemented." self.uri) else
|
||||
if self.exists.not then Error.throw (File_Error.Not_Found self) else
|
||||
self.delete_if_exists
|
||||
|
||||
## Deletes the file if it had existed.
|
||||
delete_if_exists : Nothing
|
||||
delete_if_exists self = if self.is_directory then Error.throw (S3_Error.Error "Deleting S3 folders is currently not implemented." self.uri) else
|
||||
if Context.Output.is_enabled.not then Error.throw (Forbidden_Operation.Error "Deleting an S3_File is forbidden as the Output context is disabled.") else
|
||||
translate_file_errors self <| S3.delete_object self.bucket self.prefix self.credentials . if_not_error <| Nothing
|
||||
translate_file_errors self <| S3.delete_object self.bucket self.prefix self.credentials . if_not_error Nothing
|
||||
|
||||
## Copies the file to the specified destination.
|
||||
|
||||
@ -185,8 +189,8 @@ type S3_File
|
||||
- destination: the destination to move the file to.
|
||||
- replace_existing: specifies if the operation should proceed if the
|
||||
destination file already exists. Defaults to `False`.
|
||||
copy_to : S3_File -> Boolean -> Any ! File_Error
|
||||
copy_to self (destination : Writable_File) replace_existing=False =
|
||||
copy_to : Writable_File -> Boolean -> Any ! File_Error
|
||||
copy_to self (destination : Writable_File) (replace_existing : Boolean = False) =
|
||||
if self.is_directory then Error.throw (S3_Error.Error "Copying S3 folders is currently not implemented." self.uri) else
|
||||
if Context.Output.is_enabled.not then Error.throw (Forbidden_Operation.Error "Copying an S3_File is forbidden as the Output context is disabled.") else
|
||||
case destination.file of
|
||||
@ -194,13 +198,27 @@ type S3_File
|
||||
s3_destination : S3_File ->
|
||||
if replace_existing.not && s3_destination.exists then Error.throw (File_Error.Already_Exists destination) else
|
||||
translate_file_errors self <| S3.copy_object self.bucket self.prefix s3_destination.bucket s3_destination.prefix self.credentials . if_not_error <| s3_destination
|
||||
_ -> generic_copy self destination.file replace_existing
|
||||
|
||||
# Generic implementation using streams
|
||||
_ ->
|
||||
self.with_input_stream [File_Access.Read] input_stream->
|
||||
open_settings = if replace_existing then [File_Access.Write, File_Access.Create, File_Access.Truncate_Existing] else [File_Access.Write, File_Access.Create_New]
|
||||
destination.with_output_stream open_settings output_stream->
|
||||
output_stream.write_stream input_stream
|
||||
## Moves the file to the specified destination.
|
||||
|
||||
! S3 Move is a Copy and Delete
|
||||
|
||||
Since S3 does not support moving files, this operation is implemented
|
||||
as a copy followed by delete. Keep in mind that the space usage of the
|
||||
file will briefly be doubled and that the operation may not be as fast
|
||||
as a local move often is.
|
||||
|
||||
Arguments:
|
||||
- destination: the destination to move the file to.
|
||||
- replace_existing: specifies if the operation should proceed if the
|
||||
destination file already exists. Defaults to `False`.
|
||||
move_to : Writable_File -> Boolean -> Nothing ! File_Error
|
||||
move_to self (destination : Writable_File) (replace_existing : Boolean = False) =
|
||||
if Context.Output.is_enabled.not then Error.throw (Forbidden_Operation.Error "File moving is forbidden as the Output context is disabled.") else
|
||||
r = self.copy_to destination replace_existing=replace_existing
|
||||
r.if_not_error <|
|
||||
self.delete.if_not_error r
|
||||
|
||||
## GROUP Standard.Base.Operators
|
||||
Join two path segments together.
|
||||
|
@ -10,6 +10,7 @@ import project.Data.Text.Text_Sub_Range.Text_Sub_Range
|
||||
import project.Data.Time.Date_Time.Date_Time
|
||||
import project.Data.Vector.Vector
|
||||
import project.Error.Error
|
||||
import project.Errors.Common.Forbidden_Operation
|
||||
import project.Errors.Common.Not_Found
|
||||
import project.Errors.File_Error.File_Error
|
||||
import project.Errors.Illegal_Argument.Illegal_Argument
|
||||
@ -20,12 +21,15 @@ import project.Network.HTTP.HTTP
|
||||
import project.Network.HTTP.HTTP_Method.HTTP_Method
|
||||
import project.Network.HTTP.Request_Body.Request_Body
|
||||
import project.Nothing.Nothing
|
||||
import project.Runtime.Context
|
||||
import project.System.File.File_Access.File_Access
|
||||
import project.System.File.Generic.Writable_File.Writable_File
|
||||
import project.System.File_Format_Metadata.File_Format_Metadata
|
||||
import project.System.Input_Stream.Input_Stream
|
||||
import project.System.Output_Stream.Output_Stream
|
||||
from project.Data.Boolean import Boolean, False, True
|
||||
from project.Data.Text.Extensions import all
|
||||
from project.System.File.Generic.File_Write_Strategy import generic_copy
|
||||
from project.System.File_Format import Auto_Detect, Bytes, File_Format, Plain_Text_Format
|
||||
|
||||
type Enso_File
|
||||
@ -253,7 +257,37 @@ type Enso_File
|
||||
uri = if self.is_directory then Utils.directory_api + "/" + self.id else self.internal_uri
|
||||
auth_header = Utils.authorization_header
|
||||
response = HTTP.post uri Request_Body.Empty HTTP_Method.Delete [auth_header]
|
||||
response.if_not_error <| Nothing
|
||||
response.if_not_error Nothing
|
||||
|
||||
## Deletes the file if it had existed.
|
||||
delete_if_exists : Nothing
|
||||
delete_if_exists self =
|
||||
r = self.delete
|
||||
r.catch File_Error err-> case err of
|
||||
File_Error.Not_Found _ -> Nothing
|
||||
_ -> r
|
||||
|
||||
## Copies the file to the specified destination.
|
||||
|
||||
Arguments:
|
||||
- destination: the destination to move the file to.
|
||||
- replace_existing: specifies if the operation should proceed if the
|
||||
destination file already exists. Defaults to `False`.
|
||||
copy_to : Writable_File -> Boolean -> Any ! File_Error
|
||||
copy_to self (destination : Writable_File) (replace_existing : Boolean = False) =
|
||||
generic_copy self destination.file replace_existing
|
||||
|
||||
## Moves the file to the specified destination.
|
||||
|
||||
Arguments:
|
||||
- destination: the destination to move the file to.
|
||||
- replace_existing: specifies if the operation should proceed if the
|
||||
destination file already exists. Defaults to `False`.
|
||||
move_to : Writable_File -> Boolean -> Nothing ! File_Error
|
||||
move_to self (destination : Writable_File) (replace_existing : Boolean = False) =
|
||||
_ = [destination, replace_existing]
|
||||
if Context.Output.is_enabled.not then Error.throw (Forbidden_Operation.Error "File moving is forbidden as the Output context is disabled.") else
|
||||
Unimplemented.throw "Enso_File.move_to is not implemented"
|
||||
|
||||
## UNSTABLE
|
||||
GROUP Operators
|
||||
|
@ -12,6 +12,7 @@ import project.System.File_Format_Metadata.File_Format_Metadata
|
||||
|
||||
polyglot java import java.io.FileNotFoundException
|
||||
polyglot java import java.io.IOException
|
||||
polyglot java import java.io.UncheckedIOException
|
||||
polyglot java import java.nio.file.AccessDeniedException
|
||||
polyglot java import java.nio.file.FileAlreadyExistsException
|
||||
polyglot java import java.nio.file.NoSuchFileException
|
||||
@ -71,8 +72,13 @@ type File_Error
|
||||
|
||||
Utility method for running an action with Java exceptions mapping.
|
||||
handle_java_exceptions (file : File | Nothing) ~action =
|
||||
Panic.catch IOException action caught_panic->
|
||||
handle_io_exception caught_panic =
|
||||
File_Error.wrap_io_exception file caught_panic.payload
|
||||
handle_unchecked_io_exception caught_panic =
|
||||
File_Error.wrap_io_exception file caught_panic.payload.getCause
|
||||
Panic.catch IOException handler=handle_io_exception <|
|
||||
Panic.catch UncheckedIOException handler=handle_unchecked_io_exception <|
|
||||
action
|
||||
|
||||
## PRIVATE
|
||||
Raises an error indicating that the user does not have permission to
|
||||
|
@ -636,14 +636,12 @@ type File
|
||||
- destination: the destination to move the file to.
|
||||
- replace_existing: specifies if the operation should proceed if the
|
||||
destination file already exists. Defaults to `False`.
|
||||
copy_to : File -> Boolean -> Nothing ! File_Error
|
||||
copy_to self destination replace_existing=False =
|
||||
copy_to : Writable_File -> Boolean -> Nothing ! File_Error
|
||||
copy_to self (destination : Writable_File) (replace_existing : Boolean = False) =
|
||||
if Context.Output.is_enabled.not then Error.throw (Forbidden_Operation.Error "File copying is forbidden as the Output context is disabled.") else
|
||||
File_Error.handle_java_exceptions self <| case replace_existing of
|
||||
True ->
|
||||
copy_options = [StandardCopyOption.REPLACE_EXISTING]
|
||||
self.copy_builtin destination copy_options
|
||||
False -> self.copy_builtin destination []
|
||||
case destination.file of
|
||||
_ : File -> local_file_copy self destination.file replace_existing
|
||||
_ -> destination.copy_from_local self replace_existing
|
||||
|
||||
## Moves the file to the specified destination.
|
||||
|
||||
@ -651,14 +649,15 @@ type File
|
||||
- destination: the destination to move the file to.
|
||||
- replace_existing: specifies if the operation should proceed if the
|
||||
destination file already exists. Defaults to `False`.
|
||||
move_to : File -> Boolean -> Nothing ! File_Error
|
||||
move_to self destination replace_existing=False =
|
||||
move_to : Writable_File -> Boolean -> Nothing ! File_Error
|
||||
move_to self (destination : Writable_File) (replace_existing : Boolean = False) =
|
||||
if Context.Output.is_enabled.not then Error.throw (Forbidden_Operation.Error "File moving is forbidden as the Output context is disabled.") else
|
||||
File_Error.handle_java_exceptions self <| case replace_existing of
|
||||
True ->
|
||||
copy_options = [StandardCopyOption.REPLACE_EXISTING]
|
||||
self.move_builtin destination copy_options
|
||||
False -> self.move_builtin destination []
|
||||
case destination.file of
|
||||
_ : File -> local_file_move self destination.file replace_existing
|
||||
_ ->
|
||||
r = destination.copy_from_local self replace_existing
|
||||
r.if_not_error <|
|
||||
self.delete . if_not_error r
|
||||
|
||||
## Deletes the file if it exists on disk.
|
||||
|
||||
@ -843,3 +842,17 @@ File_Like.from (that : File) = File_Like.Value that
|
||||
## PRIVATE
|
||||
Writable_File.from (that : File) =
|
||||
Writable_File.Value that.absolute.normalize Local_File_Write_Strategy.instance
|
||||
|
||||
## PRIVATE
|
||||
local_file_copy : File -> File -> Boolean -> Nothing ! File_Error
|
||||
local_file_copy source destination replace_existing =
|
||||
File_Error.handle_java_exceptions source <|
|
||||
copy_options = if replace_existing then [StandardCopyOption.REPLACE_EXISTING] else []
|
||||
source.copy_builtin destination copy_options
|
||||
|
||||
## PRIVATE
|
||||
local_file_move : File -> File -> Boolean -> Nothing ! File_Error
|
||||
local_file_move source destination replace_existing =
|
||||
File_Error.handle_java_exceptions source <|
|
||||
copy_options = if replace_existing then [StandardCopyOption.REPLACE_EXISTING] else []
|
||||
source.move_builtin destination copy_options
|
||||
|
@ -1,4 +1,5 @@
|
||||
import project.Data.Text.Text
|
||||
import project.System.File.File
|
||||
|
||||
## PRIVATE
|
||||
A generic interface for file-like objects.
|
||||
@ -19,3 +20,6 @@ type File_Like
|
||||
|
||||
## PRIVATE
|
||||
to_display_text self -> Text = self.underlying.to_display_text
|
||||
|
||||
## PRIVATE
|
||||
File_Like.from (that : Text) = File_Like.from (File.new that)
|
||||
|
@ -31,7 +31,12 @@ type File_Write_Strategy
|
||||
A remote file is downloaded to a temporary file and the provided action
|
||||
is called with that local temporary file. Then that file is uploaded to
|
||||
replace the remote file.
|
||||
Value write_overwriting write_appending write_raising_error write_backing_up create_dry_run_file write_with_local_file
|
||||
|
||||
The `copy_from_local` action creates the file on a given backend from a
|
||||
local file. It can be used to implement more efficient upload strategies
|
||||
than ones based on just writing to an output stream.
|
||||
The default `generic_copy` implementation can always be used here.
|
||||
Value write_overwriting write_appending write_raising_error write_backing_up create_dry_run_file write_with_local_file copy_from_local
|
||||
|
||||
## PRIVATE
|
||||
Writes to a file according to the provided existing file behaviour.
|
||||
@ -128,3 +133,12 @@ dry_run_behavior file behavior:Existing_File_Behavior -> Dry_Run_File_Settings =
|
||||
Dry_Run_File_Settings.Value Existing_File_Behavior.Overwrite copy_original=False
|
||||
Existing_File_Behavior.Append ->
|
||||
Dry_Run_File_Settings.Value Existing_File_Behavior.Append copy_original=True
|
||||
|
||||
## PRIVATE
|
||||
Generic `copy` implementation between two backends.
|
||||
The files only need to support `with_input_stream` and `with_output_stream`.
|
||||
generic_copy source destination replace_existing =
|
||||
source.with_input_stream [File_Access.Read] input_stream->
|
||||
options = if replace_existing then [File_Access.Write, File_Access.Create, File_Access.Truncate_Existing] else [File_Access.Write, File_Access.Create_New]
|
||||
destination.with_output_stream options output_stream->
|
||||
output_stream.write_stream input_stream . if_not_error destination
|
||||
|
@ -54,6 +54,13 @@ type Writable_File
|
||||
write_requiring_local_file self (existing_file_behavior : Existing_File_Behavior) (action : File -> Any) -> Any =
|
||||
self.write_strategy.write_with_local_file self.file existing_file_behavior action
|
||||
|
||||
## PRIVATE
|
||||
Writes a local file to this `Writable_File` destination.
|
||||
This is used by `File.copy_to` and `File.move_to` to possibly implement
|
||||
the upload more efficiently (avoiding duplicated temporary files).
|
||||
copy_from_local self (source : File) (replace_existing : Boolean) =
|
||||
self.write_strategy.copy_from_local source self.file replace_existing
|
||||
|
||||
## PRIVATE
|
||||
with_output_stream self (open_options : Vector) action =
|
||||
self.file.with_output_stream open_options action
|
||||
@ -69,10 +76,7 @@ type Writable_File
|
||||
to_display_text self -> Text = self.file.to_display_text
|
||||
|
||||
## PRIVATE
|
||||
Writable_File.from (that : Text) =
|
||||
## Currently this only works for local filesystem paths
|
||||
TODO We should extend it to also support custom paths like S3, through a ServiceProvider solution
|
||||
Writable_File.from (File.new that)
|
||||
Writable_File.from (that : Text) = Writable_File.from (File.new that)
|
||||
|
||||
## PRIVATE
|
||||
If a conversion to `File_Format_Metadata` is needed, we delegate to the underlying file.
|
||||
|
@ -6,6 +6,7 @@ import project.Errors.Illegal_State.Illegal_State
|
||||
import project.Nothing.Nothing
|
||||
import project.Panic.Caught_Panic
|
||||
import project.Panic.Panic
|
||||
import project.System.File.File
|
||||
import project.System.File.File_Access.File_Access
|
||||
import project.System.Output_Stream.Output_Stream
|
||||
from project.Data.Boolean import Boolean, False, True
|
||||
@ -13,12 +14,15 @@ from project.System.File.Generic.File_Write_Strategy import File_Write_Strategy,
|
||||
|
||||
## PRIVATE
|
||||
instance =
|
||||
File_Write_Strategy.Value default_overwrite default_append default_raise_error moving_backup create_dry_run_file write_with_local_file
|
||||
File_Write_Strategy.Value default_overwrite default_append default_raise_error moving_backup create_dry_run_file write_with_local_file copy_local_from_local
|
||||
|
||||
## PRIVATE
|
||||
create_dry_run_file file copy_original =
|
||||
file.create_dry_run_file copy_original
|
||||
|
||||
## PRIVATE
|
||||
copy_local_from_local (source : File) (destination : File) = source.copy_to destination
|
||||
|
||||
## PRIVATE
|
||||
A `Backup` strategy that does the following:
|
||||
1. If the file does not exist, we write to it.
|
||||
|
@ -4,7 +4,7 @@ from Standard.Test import all
|
||||
|
||||
import project.S3_Spec
|
||||
|
||||
main =
|
||||
main filter=Nothing =
|
||||
suite = Test.build suite_builder->
|
||||
S3_Spec.add_specs suite_builder
|
||||
suite.run_with_filter
|
||||
suite.run_with_filter filter
|
||||
|
@ -156,6 +156,10 @@ add_specs suite_builder =
|
||||
r.should_be_a Vector
|
||||
r.at 0 . get "name" . should_equal "Green St Green"
|
||||
|
||||
group_builder.specify "should work with Data.read" <|
|
||||
r = Data.read "s3://"+bucket_name+"/examples/folder 2/hello.txt"
|
||||
r.should_equal "Hello WORLD!"
|
||||
|
||||
group_builder.specify "should be able to read a file as bytes or stream" <|
|
||||
bytes = hello_txt.read_bytes
|
||||
bytes.should_equal "Hello WORLD!".utf_8
|
||||
@ -211,17 +215,18 @@ add_specs suite_builder =
|
||||
# AWS S3 does not record creation time, only last modified time.
|
||||
hello_txt.creation_time . should_fail_with S3_Error
|
||||
|
||||
writable_root = S3_File.new "s3://"+writable_bucket_name+"/"
|
||||
my_writable_dir = writable_root / "test-run-"+(Date_Time.now.format "yyyy-MM-dd_HHmmss.fV")+"/"
|
||||
delete_on_panic file ~action =
|
||||
handler caught_panic =
|
||||
file.delete
|
||||
Panic.throw caught_panic
|
||||
Panic.catch Any action handler
|
||||
delete_afterwards file ~action =
|
||||
Panic.with_finalizer file.delete action
|
||||
|
||||
suite_builder.group "S3_File writing" pending=api_pending group_builder->
|
||||
writable_root = S3_File.new "s3://"+writable_bucket_name+"/"
|
||||
my_writable_dir = writable_root / "test-run-"+(Date_Time.now.format "yyyy-MM-dd_HHmmss.fV")+"/"
|
||||
assert my_writable_dir.is_directory
|
||||
delete_on_panic file ~action =
|
||||
handler caught_panic =
|
||||
file.delete
|
||||
Panic.throw caught_panic
|
||||
Panic.catch Any action handler
|
||||
delete_afterwards file ~action =
|
||||
Panic.with_finalizer file.delete action
|
||||
|
||||
group_builder.specify "should be able to write and delete a new file" <|
|
||||
new_file = my_writable_dir / "new_file1.txt"
|
||||
@ -268,43 +273,6 @@ add_specs suite_builder =
|
||||
r.catch.to_display_text . should_contain "already exists"
|
||||
r.catch.to_display_text . should_contain "new_file-exists.txt"
|
||||
|
||||
group_builder.specify "should be able to copy a file" <|
|
||||
base_file = my_writable_dir / "new_file-for-copy.txt"
|
||||
"Hello".write base_file . should_succeed
|
||||
delete_afterwards base_file <|
|
||||
base_file.read . should_equal "Hello"
|
||||
|
||||
dest_file = my_writable_dir / "new_file-the-copy-2.txt"
|
||||
base_file.copy_to dest_file . should_succeed
|
||||
delete_afterwards dest_file <|
|
||||
dest_file.read . should_equal "Hello"
|
||||
|
||||
group_builder.specify "will fail if source file does not exist" <|
|
||||
base_file = my_writable_dir / "nonexistent-src-file.txt"
|
||||
dest_file = my_writable_dir / "nonexistent-dest-file.txt"
|
||||
r = base_file.copy_to dest_file
|
||||
r.should_fail_with File_Error
|
||||
r.catch.should_be_a File_Error.Not_Found
|
||||
dest_file.exists . should_be_false
|
||||
|
||||
group_builder.specify "respects replace_existing setting in copy_to" <|
|
||||
base_file = my_writable_dir / "new_file-for-copy-2.txt"
|
||||
"Hello".write base_file . should_succeed
|
||||
delete_afterwards base_file <|
|
||||
dest_file = my_writable_dir / "new_file-dest.txt"
|
||||
"World".write dest_file . should_succeed
|
||||
delete_afterwards dest_file <|
|
||||
r1 = base_file.copy_to dest_file replace_existing=False
|
||||
r1.should_fail_with File_Error
|
||||
r1.catch.should_be_a File_Error.Already_Exists
|
||||
r1.catch.to_display_text . should_contain "already exists"
|
||||
|
||||
dest_file.read . should_equal "World"
|
||||
|
||||
# Now allow the overwrite:
|
||||
r2 = base_file.copy_to dest_file replace_existing=True
|
||||
r2.should_equal dest_file
|
||||
|
||||
group_builder.specify "should be able to write a raw stream" <|
|
||||
new_file = my_writable_dir / "new_file-stream.txt"
|
||||
r = new_file.with_output_stream [File_Access.Write] stream->
|
||||
@ -348,7 +316,7 @@ add_specs suite_builder =
|
||||
bak_file.exists.should_be_false
|
||||
|
||||
"version1".write my_file . should_succeed
|
||||
delete_afterwards my_file <|
|
||||
delete_on_panic my_file <|
|
||||
my_file.read . should_equal "version1"
|
||||
bak_file.exists . should_be_false
|
||||
|
||||
@ -371,6 +339,14 @@ add_specs suite_builder =
|
||||
parent_dir = my_file.parent
|
||||
parent_dir.list . should_contain_the_same_elements_as [my_file, bak_file]
|
||||
|
||||
# If the original file is deleted and the backup file remains, the original file should _not_ count as existing (this used to fail).
|
||||
my_file.delete
|
||||
bak_file.exists . should_be_true
|
||||
my_file.exists . should_be_false
|
||||
files = my_file.parent.list
|
||||
files . should_contain bak_file
|
||||
files . should_not_contain my_file
|
||||
|
||||
group_builder.specify "should fail cleanly if Auto_Detect fails to detect a format" <|
|
||||
weird_ext = my_writable_dir / "weird-ext.unknown"
|
||||
"Hello".write weird_ext . should_succeed
|
||||
@ -432,14 +408,94 @@ add_specs suite_builder =
|
||||
|
||||
group_builder.specify "should fail to delete a file if the Output context is not enabled" <|
|
||||
Context.Output.with_disabled <|
|
||||
new_file = my_writable_dir / "new_file-ctx.txt"
|
||||
new_file.delete . should_fail_with Forbidden_Operation
|
||||
hello_txt = S3_File.new "s3://"+bucket_name+"/examples/folder 2/hello.txt"
|
||||
hello_txt.delete . should_fail_with Forbidden_Operation
|
||||
|
||||
group_builder.specify "does not raise an exception if the file being deleted did not exist in the first place" <|
|
||||
group_builder.specify "may fail with Not_Found if the file to delete does not exist, even if the Output Context is disabled" <|
|
||||
Context.Output.with_disabled <|
|
||||
new_file = my_writable_dir / "nonexistent-file.txt"
|
||||
r = new_file.delete
|
||||
r.should_fail_with File_Error
|
||||
r.catch.should_be_a File_Error.Not_Found
|
||||
|
||||
group_builder.specify "does not raise an exception if the file being `delete_if_exists` did not exist in the first place" <|
|
||||
new_file = my_writable_dir / "nonexistent-file.txt"
|
||||
new_file.delete . should_succeed
|
||||
new_file.delete_if_exists . should_succeed
|
||||
|
||||
main =
|
||||
group_builder.specify "fails if the file being deleted did not exist" <|
|
||||
new_file = my_writable_dir / "nonexistent-file2.txt"
|
||||
r = new_file.delete
|
||||
r.should_fail_with File_Error
|
||||
r.catch.should_be_a File_Error.Not_Found
|
||||
|
||||
sources = [my_writable_dir / "source1.txt", File.create_temporary_file "source2" ".txt"]
|
||||
destinations = [my_writable_dir / "destination1.txt", File.create_temporary_file "destination2" ".txt"]
|
||||
sources.each source_file-> destinations.each destination_file-> if source_file.is_a File && destination_file.is_a File then Nothing else
|
||||
src_typ = Meta.type_of source_file . to_display_text
|
||||
dest_typ = Meta.type_of destination_file . to_display_text
|
||||
suite_builder.group "("+src_typ+" -> "+dest_typ+") copying/moving" pending=api_pending group_builder->
|
||||
group_builder.teardown <|
|
||||
source_file.delete_if_exists
|
||||
destination_file.delete_if_exists
|
||||
|
||||
group_builder.specify "should be able to copy files" <|
|
||||
"Hello".write source_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed
|
||||
destination_file.delete_if_exists
|
||||
|
||||
source_file.copy_to destination_file . should_succeed
|
||||
destination_file.read . should_equal "Hello"
|
||||
source_file.exists . should_be_true
|
||||
|
||||
group_builder.specify "should be able to move files" <|
|
||||
"Hello".write source_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed
|
||||
destination_file.delete_if_exists
|
||||
|
||||
source_file.move_to destination_file . should_succeed
|
||||
destination_file.read . should_equal "Hello"
|
||||
source_file.exists . should_be_false
|
||||
|
||||
group_builder.specify "should fail if the source file does not exist" <|
|
||||
source_file.delete_if_exists
|
||||
destination_file.delete_if_exists
|
||||
|
||||
r = source_file.copy_to destination_file
|
||||
r.should_fail_with File_Error
|
||||
r.catch.should_be_a File_Error.Not_Found
|
||||
|
||||
r2 = source_file.move_to destination_file
|
||||
r2.should_fail_with File_Error
|
||||
r2.catch.should_be_a File_Error.Not_Found
|
||||
|
||||
destination_file.exists . should_be_false
|
||||
|
||||
group_builder.specify "should fail to copy/move a file if it exists and replace_existing=False" <|
|
||||
"Hello".write source_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed
|
||||
"World".write destination_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed
|
||||
|
||||
r = source_file.copy_to destination_file
|
||||
r.should_fail_with File_Error
|
||||
r.catch.should_be_a File_Error.Already_Exists
|
||||
|
||||
r2 = source_file.move_to destination_file
|
||||
r2.should_fail_with File_Error
|
||||
r2.catch.should_be_a File_Error.Already_Exists
|
||||
|
||||
destination_file.read . should_equal "World"
|
||||
|
||||
group_builder.specify "should overwrite existing destination in copy/move if replace_existing=True" <|
|
||||
"Hello".write source_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed
|
||||
"World".write destination_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed
|
||||
|
||||
source_file.copy_to destination_file replace_existing=True . should_succeed
|
||||
destination_file.read . should_equal "Hello"
|
||||
source_file.exists . should_be_true
|
||||
|
||||
"FooBar".write source_file on_existing_file=Existing_File_Behavior.Overwrite . should_succeed
|
||||
source_file.move_to destination_file replace_existing=True . should_succeed
|
||||
destination_file.read . should_equal "FooBar"
|
||||
source_file.exists . should_be_false
|
||||
|
||||
main filter=Nothing =
|
||||
suite = Test.build suite_builder->
|
||||
add_specs suite_builder
|
||||
suite.run_with_filter
|
||||
suite.run_with_filter filter
|
||||
|
Loading…
Reference in New Issue
Block a user