Add proxy support to Data Layer download client (#14281)

* add proxy

* fix format document

* format document

* fix whitespace

* fix end of file
This commit is contained in:
Chida82 2023-01-06 21:18:36 +01:00 committed by GitHub
parent 1796206f25
commit 5fd468df15
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 7 additions and 1 deletions

View File

@ -376,6 +376,7 @@ class DataLayer:
try:
timeout = self.config.get("client_timeout", 15)
proxy_url = self.config.get("proxy_url", None)
success = await insert_from_delta_file(
self.data_store,
tree_id,
@ -385,6 +386,7 @@ class DataLayer:
self.server_files_location,
timeout,
self.log,
proxy_url,
)
if success:
self.log.info(

View File

@ -136,6 +136,7 @@ async def insert_from_delta_file(
client_foldername: Path,
timeout: int,
log: logging.Logger,
proxy_url: str,
) -> bool:
for root_hash in root_hashes:
timestamp = int(time.time())
@ -144,7 +145,7 @@ async def insert_from_delta_file(
try:
async with aiohttp.ClientSession() as session:
async with session.get(server_info.url + "/" + filename, timeout=timeout) as resp:
async with session.get(server_info.url + "/" + filename, timeout=timeout, proxy=proxy_url) as resp:
resp.raise_for_status()
size = int(resp.headers.get("content-length", 0))
log.debug(f"Downloading delta file {filename}. Size {size} bytes.")

View File

@ -580,6 +580,9 @@ data_layer:
server_files_location: "data_layer/db/server_files_location_CHALLENGE"
# The timeout for the client to download a file from a server
client_timeout: 15
# If you need use a proxy for download data you can use this setting sample
# proxy_url: http://localhost:8888
# Data for running a data layer server.
host_ip: 0.0.0.0
host_port: 8575