mirror of
https://github.com/Chia-Network/chia-blockchain.git
synced 2024-11-11 01:28:17 +03:00
c4e14f5c78
* Serialize/deserialize peer data alongside existing sqlite implementation (to be removed) * Simplified AddressManagerStore. No longer uses sqlite and is no longer async. * Removed aiosqlite usage from AddressManagerStore. Added PeerStoreResolver class to determine the appropriate location for "peers.dat" Updated initial-config.yaml to include "peers_file_path" default, replacing "peer_db_path" (similar change for "wallet_peers_path") * Minor comment changes/additions * Added migration from sqlite peer db. Made AddressManagerStore's serialization async as it was right at the edge of blocking for too long. * Minor tweaks to checking for migration * Removed AddressManagerSQLiteStore class scaffolding * makePeerDataSerialization now returns bytes instead of a PeerDataSerialization object * Async file I/O for write_file_async using aiofiles Added more tests * Separate out the synchronous part of move_file * Renamed write_file to files since we're opening up the capabilities a bit * Update references to write_file * Renamed test_write_file to test_files * Tests covering move_file and move_file_async * Minor refinements to behavior and tests * Use aiofiles for reading peers.dat * Added missing mypy typing info for aiofiles. Also added types-PyYAML to dev_dependencies so that `mypy chia tests` doesn't require running with --install-types * Add types-aiofiles to the linting workflow * Directory perms can now be passed into write_file_async. Added an explicit f.flush() followed by os.fsync() after writing the temp file contents.
90 lines
2.9 KiB
Python
90 lines
2.9 KiB
Python
import asyncio
|
|
import logging
|
|
import os
|
|
import shutil
|
|
|
|
from aiofiles import tempfile # type: ignore
|
|
from pathlib import Path
|
|
from typing import Union
|
|
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def move_file(src: Path, dst: Path):
|
|
"""
|
|
Attempts to move the file at src to dst, falling back to a copy if the move fails.
|
|
"""
|
|
|
|
dir_perms: int = 0o700
|
|
# Create the parent directory if necessary
|
|
os.makedirs(dst.parent, mode=dir_perms, exist_ok=True)
|
|
|
|
try:
|
|
# Attempt an atomic move first
|
|
os.replace(os.fspath(src), os.fspath(dst))
|
|
except Exception as e:
|
|
log.debug(f"Failed to move {src} to {dst} using os.replace, reattempting with shutil.move: {e}")
|
|
try:
|
|
# If that fails, use the more robust shutil.move(), though it may internally initiate a copy
|
|
shutil.move(os.fspath(src), os.fspath(dst))
|
|
except Exception:
|
|
log.exception(f"Failed to move {src} to {dst} using shutil.move")
|
|
raise
|
|
|
|
|
|
async def move_file_async(src: Path, dst: Path, *, reattempts: int = 6, reattempt_delay: float = 0.5):
|
|
"""
|
|
Attempts to move the file at src to dst, making multiple attempts if the move fails.
|
|
"""
|
|
|
|
remaining_attempts: int = reattempts
|
|
while True:
|
|
try:
|
|
move_file(src, dst)
|
|
except Exception:
|
|
if remaining_attempts > 0:
|
|
log.debug(f"Failed to move {src} to {dst}, retrying in {reattempt_delay} seconds")
|
|
remaining_attempts -= 1
|
|
await asyncio.sleep(reattempt_delay)
|
|
else:
|
|
break
|
|
else:
|
|
break
|
|
|
|
if not dst.exists():
|
|
raise FileNotFoundError(f"Failed to move {src} to {dst}")
|
|
else:
|
|
log.debug(f"Moved {src} to {dst}")
|
|
|
|
|
|
async def write_file_async(file_path: Path, data: Union[str, bytes], *, file_mode: int = 0o600, dir_mode: int = 0o700):
|
|
"""
|
|
Writes the provided data to a temporary file and then moves it to the final destination.
|
|
"""
|
|
|
|
# Create the parent directory if necessary
|
|
os.makedirs(file_path.parent, mode=dir_mode, exist_ok=True)
|
|
|
|
mode: str = "w+" if type(data) == str else "w+b"
|
|
temp_file_path: Path
|
|
async with tempfile.NamedTemporaryFile(dir=file_path.parent, mode=mode, delete=False) as f:
|
|
temp_file_path = f.name
|
|
await f.write(data)
|
|
await f.flush()
|
|
os.fsync(f.fileno())
|
|
|
|
try:
|
|
await move_file_async(temp_file_path, file_path)
|
|
except Exception:
|
|
log.exception(f"Failed to move temp file {temp_file_path} to {file_path}")
|
|
else:
|
|
os.chmod(file_path, file_mode)
|
|
finally:
|
|
# We expect the file replace/move to have succeeded, but cleanup the temp file just in case
|
|
try:
|
|
if Path(temp_file_path).exists():
|
|
os.remove(temp_file_path)
|
|
except Exception:
|
|
log.exception(f"Failed to remove temp file {temp_file_path}")
|