chia-blockchain/chia/util/files.py
Jeff c4e14f5c78
Peer db new serialization (#9079)
* Serialize/deserialize peer data alongside existing sqlite implementation (to be removed)

* Simplified AddressManagerStore. No longer uses sqlite and is no longer async.

* Removed aiosqlite usage from AddressManagerStore.
Added PeerStoreResolver class to determine the appropriate location for "peers.dat"
Updated initial-config.yaml to include "peers_file_path" default, replacing "peer_db_path" (similar change for "wallet_peers_path")

* Minor comment changes/additions

* Added migration from sqlite peer db.
Made AddressManagerStore's serialization async as it was right at the edge of blocking for too long.

* Minor tweaks to checking for migration

* Removed AddressManagerSQLiteStore class scaffolding

* makePeerDataSerialization now returns bytes instead of a PeerDataSerialization object

* Async file I/O for write_file_async using aiofiles
Added more tests

* Separate out the synchronous part of move_file

* Renamed write_file to files since we're opening up the capabilities a bit

* Update references to write_file

* Renamed test_write_file to test_files

* Tests covering move_file and move_file_async

* Minor refinements to behavior and tests

* Use aiofiles for reading peers.dat

* Added missing mypy typing info for aiofiles. Also added types-PyYAML to dev_dependencies so that `mypy chia tests` doesn't require running with --install-types

* Add types-aiofiles to the linting workflow

* Directory perms can now be passed into write_file_async.
Added an explicit f.flush() followed by os.fsync() after writing the temp file contents.
2021-11-19 11:12:58 -08:00

90 lines
2.9 KiB
Python

import asyncio
import logging
import os
import shutil
from aiofiles import tempfile # type: ignore
from pathlib import Path
from typing import Union
log = logging.getLogger(__name__)
def move_file(src: Path, dst: Path):
"""
Attempts to move the file at src to dst, falling back to a copy if the move fails.
"""
dir_perms: int = 0o700
# Create the parent directory if necessary
os.makedirs(dst.parent, mode=dir_perms, exist_ok=True)
try:
# Attempt an atomic move first
os.replace(os.fspath(src), os.fspath(dst))
except Exception as e:
log.debug(f"Failed to move {src} to {dst} using os.replace, reattempting with shutil.move: {e}")
try:
# If that fails, use the more robust shutil.move(), though it may internally initiate a copy
shutil.move(os.fspath(src), os.fspath(dst))
except Exception:
log.exception(f"Failed to move {src} to {dst} using shutil.move")
raise
async def move_file_async(src: Path, dst: Path, *, reattempts: int = 6, reattempt_delay: float = 0.5):
"""
Attempts to move the file at src to dst, making multiple attempts if the move fails.
"""
remaining_attempts: int = reattempts
while True:
try:
move_file(src, dst)
except Exception:
if remaining_attempts > 0:
log.debug(f"Failed to move {src} to {dst}, retrying in {reattempt_delay} seconds")
remaining_attempts -= 1
await asyncio.sleep(reattempt_delay)
else:
break
else:
break
if not dst.exists():
raise FileNotFoundError(f"Failed to move {src} to {dst}")
else:
log.debug(f"Moved {src} to {dst}")
async def write_file_async(file_path: Path, data: Union[str, bytes], *, file_mode: int = 0o600, dir_mode: int = 0o700):
"""
Writes the provided data to a temporary file and then moves it to the final destination.
"""
# Create the parent directory if necessary
os.makedirs(file_path.parent, mode=dir_mode, exist_ok=True)
mode: str = "w+" if type(data) == str else "w+b"
temp_file_path: Path
async with tempfile.NamedTemporaryFile(dir=file_path.parent, mode=mode, delete=False) as f:
temp_file_path = f.name
await f.write(data)
await f.flush()
os.fsync(f.fileno())
try:
await move_file_async(temp_file_path, file_path)
except Exception:
log.exception(f"Failed to move temp file {temp_file_path} to {file_path}")
else:
os.chmod(file_path, file_mode)
finally:
# We expect the file replace/move to have succeeded, but cleanup the temp file just in case
try:
if Path(temp_file_path).exists():
os.remove(temp_file_path)
except Exception:
log.exception(f"Failed to remove temp file {temp_file_path}")