2022-01-06 01:42:23 +03:00
|
|
|
# Portions Copyright (c) Meta Platforms, Inc. and affiliates.
|
2019-11-01 22:33:28 +03:00
|
|
|
#
|
|
|
|
# This software may be used and distributed according to the terms of the
|
|
|
|
# GNU General Public License version 2.
|
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
# This library is free software; you can redistribute it and/or
|
|
|
|
# modify it under the terms of the GNU Lesser General Public
|
|
|
|
# License as published by the Free Software Foundation; either
|
|
|
|
# version 2.1 of the License, or (at your option) any later version.
|
|
|
|
#
|
|
|
|
# This library is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
# Lesser General Public License for more details.
|
|
|
|
#
|
|
|
|
# You should have received a copy of the GNU Lesser General Public
|
2012-01-06 19:27:13 +04:00
|
|
|
# License along with this library; if not, see
|
|
|
|
# <http://www.gnu.org/licenses/>.
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
# This file is part of urlgrabber, a high-level cross-protocol url-grabber
|
|
|
|
# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
|
|
|
|
|
2007-01-08 18:18:29 +03:00
|
|
|
# Modified by Benoit Boissinot:
|
|
|
|
# - fix for digest auth (inspired from urllib2.py @ Python v2.4)
|
2008-04-05 00:36:40 +04:00
|
|
|
# Modified by Dirkjan Ochtman:
|
|
|
|
# - import md5 function from a local util module
|
2009-11-02 19:03:22 +03:00
|
|
|
# Modified by Augie Fackler:
|
|
|
|
# - add safesend method and use it to prevent broken pipe errors
|
|
|
|
# on large POST requests
|
2007-01-08 18:18:29 +03:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
"""An HTTP handler for urllib2 that supports HTTP 1.1 and keepalive.
|
|
|
|
|
|
|
|
>>> import urllib2
|
|
|
|
>>> from keepalive import HTTPHandler
|
|
|
|
>>> keepalive_handler = HTTPHandler()
|
2016-04-07 02:22:12 +03:00
|
|
|
>>> opener = urlreq.buildopener(keepalive_handler)
|
|
|
|
>>> urlreq.installopener(opener)
|
2006-07-12 19:28:00 +04:00
|
|
|
>>>
|
2016-04-07 02:22:12 +03:00
|
|
|
>>> fo = urlreq.urlopen('http://www.python.org')
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
If a connection to a given host is requested, and all of the existing
|
|
|
|
connections are still in use, another connection will be opened. If
|
|
|
|
the handler tries to use an existing connection but it fails in some
|
|
|
|
way, it will be closed and removed from the pool.
|
|
|
|
|
|
|
|
To remove the handler, simply re-run build_opener with no arguments, and
|
|
|
|
install that opener.
|
|
|
|
|
|
|
|
You can explicitly close connections by using the close_connection()
|
|
|
|
method of the returned file-like object (described below) or you can
|
|
|
|
use the handler methods:
|
|
|
|
|
|
|
|
close_connection(host)
|
|
|
|
close_all()
|
|
|
|
open_connections()
|
|
|
|
|
|
|
|
NOTE: using the close_connection and close_all methods of the handler
|
|
|
|
should be done with care when using multiple threads.
|
|
|
|
* there is nothing that prevents another thread from creating new
|
|
|
|
connections immediately after connections are closed
|
|
|
|
* no checks are done to prevent in-use connections from being closed
|
|
|
|
|
|
|
|
>>> keepalive_handler.close_all()
|
|
|
|
|
|
|
|
EXTRA ATTRIBUTES AND METHODS
|
|
|
|
|
|
|
|
Upon a status of 200, the object returned has a few additional
|
|
|
|
attributes and methods, which should not be used if you want to
|
|
|
|
remain consistent with the normal urllib2-returned objects:
|
|
|
|
|
|
|
|
close_connection() - close the connection to the host
|
|
|
|
readlines() - you know, readlines()
|
2012-08-16 00:38:42 +04:00
|
|
|
status - the return status (i.e. 404)
|
|
|
|
reason - english translation of status (i.e. 'File not found')
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
If you want the best of both worlds, use this inside an
|
|
|
|
AttributeError-catching try:
|
|
|
|
|
|
|
|
>>> try: status = fo.status
|
|
|
|
>>> except AttributeError: status = None
|
|
|
|
|
|
|
|
Unfortunately, these are ONLY there if status == 200, so it's not
|
|
|
|
easy to distinguish between non-200 responses. The reason is that
|
|
|
|
urllib2 tries to do clever things with error codes 301, 302, 401,
|
|
|
|
and 407, and it wraps the object upon return.
|
|
|
|
"""
|
|
|
|
|
2006-06-16 17:00:37 +04:00
|
|
|
# $Id: keepalive.py,v 1.14 2006/04/04 21:00:32 mstenner Exp $
|
2006-06-15 23:57:59 +04:00
|
|
|
|
2015-06-22 09:14:54 +03:00
|
|
|
from __future__ import absolute_import, print_function
|
2015-12-23 03:28:28 +03:00
|
|
|
|
2019-12-05 04:02:25 +03:00
|
|
|
import email
|
2009-11-02 19:03:22 +03:00
|
|
|
import errno
|
2016-06-10 07:12:33 +03:00
|
|
|
import hashlib
|
2006-06-15 23:57:59 +04:00
|
|
|
import socket
|
2015-12-23 03:28:28 +03:00
|
|
|
import sys
|
2016-07-01 16:47:16 +03:00
|
|
|
import threading
|
2019-12-05 04:02:25 +03:00
|
|
|
from typing import List
|
2016-04-07 02:22:12 +03:00
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
from . import pycompat, urllibcompat, util
|
2017-09-25 12:05:16 +03:00
|
|
|
from .i18n import _
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2016-04-07 02:22:12 +03:00
|
|
|
|
2020-04-23 22:07:36 +03:00
|
|
|
# pyre-fixme[11]: Annotation `client` is not defined as a type.
|
2016-06-28 13:31:53 +03:00
|
|
|
httplib = util.httplib
|
2016-04-07 02:22:12 +03:00
|
|
|
urlerr = util.urlerr
|
|
|
|
urlreq = util.urlreq
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
DEBUG = None
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2011-06-29 17:00:00 +04:00
|
|
|
class ConnectionManager(object):
|
2006-06-15 23:57:59 +04:00
|
|
|
"""
|
|
|
|
The connection manager must be able to:
|
|
|
|
* keep track of all existing
|
2020-10-15 06:19:13 +03:00
|
|
|
"""
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
def __init__(self):
|
2016-07-01 16:47:16 +03:00
|
|
|
self._lock = threading.Lock()
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
self._hostmap = {} # map hosts to a list of connections
|
|
|
|
self._connmap = {} # map connections to host
|
|
|
|
self._readymap = {} # map connection to ready state
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
def add(self, host, connection, ready):
|
|
|
|
self._lock.acquire()
|
|
|
|
try:
|
2012-05-12 18:00:57 +04:00
|
|
|
if host not in self._hostmap:
|
2010-01-25 09:05:27 +03:00
|
|
|
self._hostmap[host] = []
|
2006-06-15 23:57:59 +04:00
|
|
|
self._hostmap[host].append(connection)
|
|
|
|
self._connmap[connection] = host
|
|
|
|
self._readymap[connection] = ready
|
|
|
|
finally:
|
|
|
|
self._lock.release()
|
|
|
|
|
|
|
|
def remove(self, connection):
|
|
|
|
self._lock.acquire()
|
|
|
|
try:
|
|
|
|
try:
|
|
|
|
host = self._connmap[connection]
|
|
|
|
except KeyError:
|
|
|
|
pass
|
|
|
|
else:
|
|
|
|
del self._connmap[connection]
|
|
|
|
del self._readymap[connection]
|
|
|
|
self._hostmap[host].remove(connection)
|
2017-09-29 18:49:20 +03:00
|
|
|
if not self._hostmap[host]:
|
|
|
|
del self._hostmap[host]
|
2006-06-15 23:57:59 +04:00
|
|
|
finally:
|
|
|
|
self._lock.release()
|
|
|
|
|
|
|
|
def set_ready(self, connection, ready):
|
2010-01-25 09:05:27 +03:00
|
|
|
try:
|
|
|
|
self._readymap[connection] = ready
|
|
|
|
except KeyError:
|
|
|
|
pass
|
2006-07-12 19:28:00 +04:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
def get_ready_conn(self, host):
|
|
|
|
conn = None
|
|
|
|
self._lock.acquire()
|
|
|
|
try:
|
2008-01-20 16:39:25 +03:00
|
|
|
if host in self._hostmap:
|
2006-06-15 23:57:59 +04:00
|
|
|
for c in self._hostmap[host]:
|
|
|
|
if self._readymap[c]:
|
|
|
|
self._readymap[c] = 0
|
|
|
|
conn = c
|
|
|
|
break
|
|
|
|
finally:
|
|
|
|
self._lock.release()
|
|
|
|
return conn
|
|
|
|
|
|
|
|
def get_all(self, host=None):
|
|
|
|
if host:
|
|
|
|
return list(self._hostmap.get(host, []))
|
|
|
|
else:
|
|
|
|
return dict(self._hostmap)
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2011-06-29 17:00:00 +04:00
|
|
|
class KeepAliveHandler(object):
|
2006-06-15 23:57:59 +04:00
|
|
|
def __init__(self):
|
|
|
|
self._cm = ConnectionManager()
|
2006-07-12 19:28:00 +04:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
#### Connection Management
|
|
|
|
def open_connections(self):
|
|
|
|
"""return a list of connected hosts and the number of connections
|
|
|
|
to each. [('foo.com:80', 2), ('bar.org', 1)]"""
|
|
|
|
return [(host, len(li)) for (host, li) in self._cm.get_all().items()]
|
|
|
|
|
|
|
|
def close_connection(self, host):
|
|
|
|
"""close connection(s) to <host>
|
|
|
|
host is the host:port spec, as in 'www.cnn.com:8080' as passed in.
|
|
|
|
no error occurs if there is no connection to that host."""
|
|
|
|
for h in self._cm.get_all(host):
|
|
|
|
self._cm.remove(h)
|
|
|
|
h.close()
|
2006-07-12 19:28:00 +04:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
def close_all(self):
|
|
|
|
"""close all open connections"""
|
2020-01-29 03:25:29 +03:00
|
|
|
for host, conns in pycompat.iteritems(self._cm.get_all()):
|
2006-06-15 23:57:59 +04:00
|
|
|
for h in conns:
|
|
|
|
self._cm.remove(h)
|
|
|
|
h.close()
|
2006-07-12 19:28:00 +04:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
def _request_closed(self, request, host, connection):
|
2012-07-26 04:54:13 +04:00
|
|
|
"""tells us that this request is now closed and that the
|
2006-06-15 23:57:59 +04:00
|
|
|
connection is ready for another request"""
|
|
|
|
self._cm.set_ready(connection, 1)
|
|
|
|
|
|
|
|
def _remove_connection(self, host, connection, close=0):
|
2010-01-25 09:05:27 +03:00
|
|
|
if close:
|
|
|
|
connection.close()
|
2006-06-15 23:57:59 +04:00
|
|
|
self._cm.remove(connection)
|
2006-07-12 19:28:00 +04:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
#### Transaction Execution
|
|
|
|
def http_open(self, req):
|
|
|
|
return self.do_open(HTTPConnection, req)
|
|
|
|
|
|
|
|
def do_open(self, http_class, req):
|
2017-10-01 19:14:21 +03:00
|
|
|
host = urllibcompat.gethost(req)
|
2006-06-15 23:57:59 +04:00
|
|
|
if not host:
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
raise urlerr.urlerror("no host given")
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
try:
|
|
|
|
h = self._cm.get_ready_conn(host)
|
|
|
|
while h:
|
|
|
|
r = self._reuse_connection(h, req, host)
|
|
|
|
|
|
|
|
# if this response is non-None, then it worked and we're
|
|
|
|
# done. Break out, skipping the else block.
|
2010-01-25 09:05:27 +03:00
|
|
|
if r:
|
|
|
|
break
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
# connection is bad - possibly closed by server
|
|
|
|
# discard it and ask for the next free connection
|
|
|
|
h.close()
|
|
|
|
self._cm.remove(h)
|
|
|
|
h = self._cm.get_ready_conn(host)
|
|
|
|
else:
|
|
|
|
# no (working) free connections were found. Create a new one.
|
|
|
|
h = http_class(host)
|
2010-01-25 09:05:27 +03:00
|
|
|
if DEBUG:
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
DEBUG.info("creating new connection to %s (%d)", host, id(h))
|
2006-06-15 23:57:59 +04:00
|
|
|
self._cm.add(host, h, 0)
|
|
|
|
self._start_transaction(h, req)
|
|
|
|
r = h.getresponse()
|
2017-09-25 12:05:16 +03:00
|
|
|
# The string form of BadStatusLine is the status line. Add some context
|
|
|
|
# to make the error message slightly more useful.
|
|
|
|
except httplib.BadStatusLine as err:
|
2020-02-01 04:57:24 +03:00
|
|
|
raise urlerr.urlerror(_("bad HTTP status line: %s") % err.line)
|
2015-06-24 08:20:08 +03:00
|
|
|
except (socket.error, httplib.HTTPException) as err:
|
2016-04-07 02:22:12 +03:00
|
|
|
raise urlerr.urlerror(err)
|
2006-07-12 19:28:00 +04:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
# if not a persistent connection, don't try to reuse it
|
2010-01-25 09:05:27 +03:00
|
|
|
if r.will_close:
|
|
|
|
self._cm.remove(h)
|
2006-06-15 23:57:59 +04:00
|
|
|
|
2010-01-25 09:05:27 +03:00
|
|
|
if DEBUG:
|
|
|
|
DEBUG.info("STATUS: %s, %s", r.status, r.reason)
|
2006-06-15 23:57:59 +04:00
|
|
|
r._handler = self
|
|
|
|
r._host = host
|
|
|
|
r._url = req.get_full_url()
|
|
|
|
r._connection = h
|
|
|
|
r.code = r.status
|
2006-06-16 17:00:37 +04:00
|
|
|
r.headers = r.msg
|
|
|
|
r.msg = r.reason
|
2006-07-12 19:28:00 +04:00
|
|
|
|
2016-11-22 06:17:45 +03:00
|
|
|
return r
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
def _reuse_connection(self, h, req, host):
|
|
|
|
"""start the transaction with a re-used connection
|
|
|
|
return a response object (r) upon success or None on failure.
|
|
|
|
This DOES not close or remove bad connections in cases where
|
|
|
|
it returns. However, if an unexpected exception occurs, it
|
|
|
|
will close and remove the connection before re-raising.
|
|
|
|
"""
|
|
|
|
try:
|
|
|
|
self._start_transaction(h, req)
|
|
|
|
r = h.getresponse()
|
|
|
|
# note: just because we got something back doesn't mean it
|
|
|
|
# worked. We'll check the version below, too.
|
|
|
|
except (socket.error, httplib.HTTPException):
|
|
|
|
r = None
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
except: # re-raises
|
2006-06-15 23:57:59 +04:00
|
|
|
# adding this block just in case we've missed
|
|
|
|
# something we will still raise the exception, but
|
|
|
|
# lets try and close the connection and remove it
|
|
|
|
# first. We previously got into a nasty loop
|
|
|
|
# where an exception was uncaught, and so the
|
|
|
|
# connection stayed open. On the next try, the
|
2012-08-16 00:38:42 +04:00
|
|
|
# same exception was raised, etc. The trade-off is
|
2006-06-15 23:57:59 +04:00
|
|
|
# that it's now possible this call will raise
|
|
|
|
# a DIFFERENT exception
|
2010-01-25 09:05:27 +03:00
|
|
|
if DEBUG:
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
DEBUG.error(
|
|
|
|
"unexpected exception - closing " "connection to %s (%d)",
|
|
|
|
host,
|
|
|
|
id(h),
|
|
|
|
)
|
2006-06-15 23:57:59 +04:00
|
|
|
self._cm.remove(h)
|
|
|
|
h.close()
|
|
|
|
raise
|
2006-07-12 19:28:00 +04:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
if r is None or r.version == 9:
|
|
|
|
# httplib falls back to assuming HTTP 0.9 if it gets a
|
|
|
|
# bad header back. This is most likely to happen if
|
|
|
|
# the socket has been closed by the server since we
|
|
|
|
# last used the connection.
|
2010-01-25 09:05:27 +03:00
|
|
|
if DEBUG:
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
DEBUG.info("failed to re-use connection to %s (%d)", host, id(h))
|
2006-06-15 23:57:59 +04:00
|
|
|
r = None
|
|
|
|
else:
|
2010-01-25 09:05:27 +03:00
|
|
|
if DEBUG:
|
|
|
|
DEBUG.info("re-using connection to %s (%d)", host, id(h))
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
return r
|
|
|
|
|
|
|
|
def _start_transaction(self, h, req):
|
2009-04-28 00:15:36 +04:00
|
|
|
# What follows mostly reimplements HTTPConnection.request()
|
2017-04-14 04:04:38 +03:00
|
|
|
# except it adds self.parent.addheaders in the mix and sends headers
|
|
|
|
# in a deterministic order (to make testing easier).
|
2018-08-10 07:07:17 +03:00
|
|
|
headers = util.altsortdict(self.parent.addheaders)
|
2017-04-14 04:04:38 +03:00
|
|
|
headers.update(sorted(req.headers.items()))
|
|
|
|
headers.update(sorted(req.unredirected_hdrs.items()))
|
2018-08-10 07:07:17 +03:00
|
|
|
headers = util.altsortdict((n.lower(), v) for n, v in headers.items())
|
2009-04-28 00:15:36 +04:00
|
|
|
skipheaders = {}
|
keepalive: send message body with the headers
Summary:
On the LFS Server, we've noted that LFS batch requests from Mercurial have
about ~40ms latency, but that the same request sent through curl doesn't.
Those requests, when sent through curl, complete in < 1ms, so thats a lot of overhead.
After adding more logging in the LFS server (D18636703), it turned out that that is
because the body of the request shows up 40ms after the request itself.
After capturing the traffic on the backend (D18689109), it turned out that
Mercurial is sending the headers and the response body in 2 separate TCP
packets.
The reason for that is delayed TCP acks. What happens is that when we call
`h.endheaders()`, that calls `send(2)`, which in turns send the packet with the
headers. Later, when we call `h.send(data)`, we make another `send(2)` syscall,
but that doesn't go over the wire immediately: because of Nagle's algorithm, we
wait for an ACK for the first packet we send before sending a new one, and
because of delayed ACKs, the ACK takes 40ms for this ACK to show up (which is
over 40 times the roundtrip latency!).
This diff passes the data to `h.endheaders()`, which in Python's httplib will
concatenate the headers and the data into a single `send(2)` syscall if the
data isn't a file-like object (which it isn't for batch requests), and
otherwise fall back to doing exactly what we're doing right now.
This will result in everything in a single packet if we're fetching a single
file (and AFAICT, without further delay if we're fetching enough files that the
request doesn't fit in a single packet).
An alternative approach would be to set `TCP_NODELAY` on our socket here, but
for now this seems less intrusive and just as effective, so I've opted for that
approach.
Reviewed By: farnz
Differential Revision: D18811419
fbshipit-source-id: 4cf5719e5eed90e5dd994e6c8861aceb69373d89
2019-12-04 23:08:57 +03:00
|
|
|
data = None
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
for n in ("host", "accept-encoding"):
|
2009-04-28 00:15:36 +04:00
|
|
|
if n in headers:
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
skipheaders["skip_" + n.replace("-", "_")] = 1
|
2006-06-15 23:57:59 +04:00
|
|
|
try:
|
2017-10-01 19:14:21 +03:00
|
|
|
if urllibcompat.hasdata(req):
|
|
|
|
data = urllibcompat.getdata(req)
|
2017-02-13 23:12:17 +03:00
|
|
|
h.putrequest(
|
2020-02-01 03:33:39 +03:00
|
|
|
req.get_method(), urllibcompat.getselector(req), **skipheaders
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
)
|
|
|
|
if "content-type" not in headers:
|
|
|
|
h.putheader("Content-type", "application/x-www-form-urlencoded")
|
|
|
|
if "content-length" not in headers:
|
|
|
|
h.putheader("Content-length", "%d" % len(data))
|
2009-04-24 00:40:10 +04:00
|
|
|
else:
|
2017-02-13 23:12:17 +03:00
|
|
|
h.putrequest(
|
2020-02-01 03:33:39 +03:00
|
|
|
req.get_method(), urllibcompat.getselector(req), **skipheaders
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
)
|
2016-02-28 11:00:13 +03:00
|
|
|
except socket.error as err:
|
2016-04-07 02:22:12 +03:00
|
|
|
raise urlerr.urlerror(err)
|
2009-04-28 00:15:36 +04:00
|
|
|
for k, v in headers.items():
|
2009-04-24 00:40:10 +04:00
|
|
|
h.putheader(k, v)
|
keepalive: send message body with the headers
Summary:
On the LFS Server, we've noted that LFS batch requests from Mercurial have
about ~40ms latency, but that the same request sent through curl doesn't.
Those requests, when sent through curl, complete in < 1ms, so thats a lot of overhead.
After adding more logging in the LFS server (D18636703), it turned out that that is
because the body of the request shows up 40ms after the request itself.
After capturing the traffic on the backend (D18689109), it turned out that
Mercurial is sending the headers and the response body in 2 separate TCP
packets.
The reason for that is delayed TCP acks. What happens is that when we call
`h.endheaders()`, that calls `send(2)`, which in turns send the packet with the
headers. Later, when we call `h.send(data)`, we make another `send(2)` syscall,
but that doesn't go over the wire immediately: because of Nagle's algorithm, we
wait for an ACK for the first packet we send before sending a new one, and
because of delayed ACKs, the ACK takes 40ms for this ACK to show up (which is
over 40 times the roundtrip latency!).
This diff passes the data to `h.endheaders()`, which in Python's httplib will
concatenate the headers and the data into a single `send(2)` syscall if the
data isn't a file-like object (which it isn't for batch requests), and
otherwise fall back to doing exactly what we're doing right now.
This will result in everything in a single packet if we're fetching a single
file (and AFAICT, without further delay if we're fetching enough files that the
request doesn't fit in a single packet).
An alternative approach would be to set `TCP_NODELAY` on our socket here, but
for now this seems less intrusive and just as effective, so I've opted for that
approach.
Reviewed By: farnz
Differential Revision: D18811419
fbshipit-source-id: 4cf5719e5eed90e5dd994e6c8861aceb69373d89
2019-12-04 23:08:57 +03:00
|
|
|
h.endheaders(message_body=data)
|
2009-04-24 00:40:10 +04:00
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2019-12-04 21:53:06 +03:00
|
|
|
# pyre-fixme[11]: Annotation `httphandler` is not defined as a type.
|
2016-04-07 02:22:12 +03:00
|
|
|
class HTTPHandler(KeepAliveHandler, urlreq.httphandler):
|
2008-02-04 02:03:46 +03:00
|
|
|
pass
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
class HTTPResponse(httplib.HTTPResponse):
|
|
|
|
# we need to subclass HTTPResponse in order to
|
|
|
|
# 1) add readline() and readlines() methods
|
|
|
|
# 2) add close_connection() methods
|
|
|
|
# 3) add info() and geturl() methods
|
|
|
|
|
|
|
|
# in order to add readline(), read must be modified to deal with a
|
|
|
|
# buffer. example: readline must read a buffer and then spit back
|
|
|
|
# one line at a time. The only real alternative is to read one
|
|
|
|
# BYTE at a time (ick). Once something has been read, it can't be
|
|
|
|
# put back (ok, maybe it can, but that's even uglier than this),
|
|
|
|
# so if you THEN do a normal read, you must first take stuff from
|
|
|
|
# the buffer.
|
|
|
|
|
2012-08-16 00:38:42 +04:00
|
|
|
# the read method wraps the original to accommodate buffering,
|
2006-06-15 23:57:59 +04:00
|
|
|
# although read() never adds to the buffer.
|
|
|
|
# Both readline and readlines have been stolen with almost no
|
|
|
|
# modification from socket.py
|
2006-07-12 19:28:00 +04:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
def __init__(self, sock, debuglevel=0, strict=0, method=None):
|
2017-10-01 19:15:53 +03:00
|
|
|
extrakw = {}
|
2019-12-05 04:02:25 +03:00
|
|
|
if sys.version_info[0] < 3:
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
extrakw[r"strict"] = True
|
|
|
|
extrakw[r"buffering"] = True
|
|
|
|
httplib.HTTPResponse.__init__(
|
|
|
|
self, sock, debuglevel=debuglevel, method=method, **extrakw
|
|
|
|
)
|
2006-06-15 23:57:59 +04:00
|
|
|
self.fileno = sock.fileno
|
|
|
|
self.code = None
|
2019-12-05 04:02:25 +03:00
|
|
|
self._rbuf = b""
|
2006-06-15 23:57:59 +04:00
|
|
|
self._rbufsize = 8096
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
self._handler = None # inserted by the handler later
|
|
|
|
self._host = None # (same)
|
2019-12-05 04:02:25 +03:00
|
|
|
self._url = "" # (same)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
self._connection = None # (same)
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
_raw_read = httplib.HTTPResponse.read
|
|
|
|
|
2022-02-10 01:26:09 +03:00
|
|
|
def close(self) -> None:
|
2019-12-05 04:02:25 +03:00
|
|
|
httplib.HTTPResponse.close(self)
|
|
|
|
|
|
|
|
handler = self._handler
|
|
|
|
if handler:
|
|
|
|
handler._request_closed(self, self._host, self._connection)
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
def close_connection(self):
|
|
|
|
self._handler._remove_connection(self._host, self._connection, close=1)
|
|
|
|
self.close()
|
2006-07-12 19:28:00 +04:00
|
|
|
|
2022-02-10 01:26:09 +03:00
|
|
|
def info(self) -> "email.message.Message":
|
2006-06-16 17:00:37 +04:00
|
|
|
return self.headers
|
2006-06-15 23:57:59 +04:00
|
|
|
|
2022-02-10 01:26:09 +03:00
|
|
|
def geturl(self) -> str:
|
2006-06-15 23:57:59 +04:00
|
|
|
return self._url
|
|
|
|
|
2022-02-10 01:26:09 +03:00
|
|
|
def read(self, amt=None) -> bytes:
|
2006-06-15 23:57:59 +04:00
|
|
|
# the _rbuf test is only in this first if for speed. It's not
|
|
|
|
# logically necessary
|
2019-12-05 04:02:25 +03:00
|
|
|
if amt:
|
|
|
|
if self._rbuf:
|
|
|
|
L = len(self._rbuf)
|
|
|
|
if amt > L:
|
|
|
|
amt -= L
|
|
|
|
else:
|
|
|
|
s = self._rbuf[:amt]
|
|
|
|
self._rbuf = self._rbuf[amt:]
|
|
|
|
return s
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
s = self._rbuf + self._raw_read(amt)
|
2019-12-05 04:02:25 +03:00
|
|
|
self._rbuf = b""
|
2006-06-15 23:57:59 +04:00
|
|
|
return s
|
|
|
|
|
2009-02-17 02:37:23 +03:00
|
|
|
# stolen from Python SVN #68532 to fix issue1088
|
|
|
|
def _read_chunked(self, amt):
|
|
|
|
chunk_left = self.chunk_left
|
2015-10-08 01:33:52 +03:00
|
|
|
parts = []
|
2009-02-17 02:37:23 +03:00
|
|
|
|
|
|
|
while True:
|
|
|
|
if chunk_left is None:
|
|
|
|
line = self.fp.readline()
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
i = line.find(";")
|
2009-02-17 02:37:23 +03:00
|
|
|
if i >= 0:
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
line = line[:i] # strip chunk-extensions
|
2009-02-17 02:37:23 +03:00
|
|
|
try:
|
|
|
|
chunk_left = int(line, 16)
|
|
|
|
except ValueError:
|
2012-08-16 00:38:42 +04:00
|
|
|
# close the connection as protocol synchronization is
|
2009-02-17 02:37:23 +03:00
|
|
|
# probably lost
|
|
|
|
self.close()
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
raise httplib.IncompleteRead("".join(parts))
|
2009-02-17 02:37:23 +03:00
|
|
|
if chunk_left == 0:
|
|
|
|
break
|
|
|
|
if amt is None:
|
2015-10-08 01:33:52 +03:00
|
|
|
parts.append(self._safe_read(chunk_left))
|
2009-02-17 02:37:23 +03:00
|
|
|
elif amt < chunk_left:
|
2015-10-08 01:33:52 +03:00
|
|
|
parts.append(self._safe_read(amt))
|
2009-02-17 02:37:23 +03:00
|
|
|
self.chunk_left = chunk_left - amt
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
return "".join(parts)
|
2009-02-17 02:37:23 +03:00
|
|
|
elif amt == chunk_left:
|
2015-10-08 01:33:52 +03:00
|
|
|
parts.append(self._safe_read(amt))
|
2009-02-17 02:37:23 +03:00
|
|
|
self._safe_read(2) # toss the CRLF at the end of the chunk
|
|
|
|
self.chunk_left = None
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
return "".join(parts)
|
2009-02-17 02:37:23 +03:00
|
|
|
else:
|
2015-10-08 01:33:52 +03:00
|
|
|
parts.append(self._safe_read(chunk_left))
|
2009-02-17 02:37:23 +03:00
|
|
|
amt -= chunk_left
|
|
|
|
|
|
|
|
# we read the whole chunk, get another
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
self._safe_read(2) # toss the CRLF at the end of the chunk
|
2009-02-17 02:37:23 +03:00
|
|
|
chunk_left = None
|
|
|
|
|
|
|
|
# read and discard trailer up to the CRLF terminator
|
|
|
|
### note: we shouldn't have any trailers!
|
|
|
|
while True:
|
|
|
|
line = self.fp.readline()
|
|
|
|
if not line:
|
|
|
|
# a vanishingly small number of sites EOF without
|
|
|
|
# sending the trailer
|
|
|
|
break
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
if line == "\r\n":
|
2009-02-17 02:37:23 +03:00
|
|
|
break
|
|
|
|
|
|
|
|
# we read everything; close the "file"
|
|
|
|
self.close()
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
return "".join(parts)
|
2009-02-17 02:37:23 +03:00
|
|
|
|
2022-05-05 05:32:11 +03:00
|
|
|
# pyre-fixme[14]: `readline` overrides method defined in `HTTPResponse`
|
|
|
|
# inconsistently.
|
2022-02-10 01:26:09 +03:00
|
|
|
def readline(self, size: int = -1) -> bytes:
|
2015-10-08 02:02:45 +03:00
|
|
|
# Fast path for a line is already available in read buffer.
|
2019-12-05 04:02:25 +03:00
|
|
|
i = self._rbuf.find(b"\n")
|
2015-10-08 02:02:45 +03:00
|
|
|
if i >= 0:
|
|
|
|
i += 1
|
|
|
|
line = self._rbuf[:i]
|
|
|
|
self._rbuf = self._rbuf[i:]
|
|
|
|
return line
|
|
|
|
|
|
|
|
# No newline in local buffer. Read until we find one.
|
|
|
|
chunks = [self._rbuf]
|
|
|
|
i = -1
|
|
|
|
readsize = self._rbufsize
|
|
|
|
while True:
|
|
|
|
new = self._raw_read(readsize)
|
2010-01-25 09:05:27 +03:00
|
|
|
if not new:
|
|
|
|
break
|
2015-10-08 02:02:45 +03:00
|
|
|
|
|
|
|
chunks.append(new)
|
2019-12-05 04:02:25 +03:00
|
|
|
i = new.find(b"\n")
|
2010-01-25 09:05:27 +03:00
|
|
|
if i >= 0:
|
2015-10-08 02:02:45 +03:00
|
|
|
break
|
|
|
|
|
|
|
|
# We either have exhausted the stream or have a newline in chunks[-1].
|
|
|
|
|
|
|
|
# EOF
|
|
|
|
if i == -1:
|
2019-12-05 04:02:25 +03:00
|
|
|
self._rbuf = b""
|
|
|
|
return b"".join(chunks)
|
2015-10-08 02:02:45 +03:00
|
|
|
|
|
|
|
i += 1
|
|
|
|
self._rbuf = chunks[-1][i:]
|
|
|
|
chunks[-1] = chunks[-1][:i]
|
2019-12-05 04:02:25 +03:00
|
|
|
return b"".join(chunks)
|
2006-06-15 23:57:59 +04:00
|
|
|
|
2023-02-02 07:39:41 +03:00
|
|
|
# pyre-fixme[14]: `readlines` overrides method defined in `IOBase` inconsistently.
|
2022-02-10 01:26:09 +03:00
|
|
|
def readlines(self, hint: int = 0) -> "List[bytes]":
|
2006-06-15 23:57:59 +04:00
|
|
|
total = 0
|
|
|
|
list = []
|
2011-06-01 14:38:46 +04:00
|
|
|
while True:
|
2006-06-15 23:57:59 +04:00
|
|
|
line = self.readline()
|
2010-01-25 09:05:27 +03:00
|
|
|
if not line:
|
|
|
|
break
|
2006-06-15 23:57:59 +04:00
|
|
|
list.append(line)
|
|
|
|
total += len(line)
|
2019-12-05 04:02:25 +03:00
|
|
|
if hint and total >= hint:
|
2006-06-15 23:57:59 +04:00
|
|
|
break
|
|
|
|
return list
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2009-11-02 19:03:22 +03:00
|
|
|
def safesend(self, str):
|
|
|
|
"""Send `str' to the server.
|
|
|
|
|
|
|
|
Shamelessly ripped off from httplib to patch a bad behavior.
|
|
|
|
"""
|
|
|
|
# _broken_pipe_resp is an attribute we set in this function
|
|
|
|
# if the socket is closed while we're sending data but
|
|
|
|
# the server sent us a response before hanging up.
|
|
|
|
# In that case, we want to pretend to send the rest of the
|
|
|
|
# outgoing data, and then let the user use getresponse()
|
|
|
|
# (which we wrap) to get this last response before
|
|
|
|
# opening a new socket.
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
if getattr(self, "_broken_pipe_resp", None) is not None:
|
2009-11-02 19:03:22 +03:00
|
|
|
return
|
|
|
|
|
|
|
|
if self.sock is None:
|
|
|
|
if self.auto_open:
|
|
|
|
self.connect()
|
|
|
|
else:
|
2012-05-12 18:00:58 +04:00
|
|
|
raise httplib.NotConnected
|
2009-11-02 19:03:22 +03:00
|
|
|
|
|
|
|
# send the data to the server. if we get a broken pipe, then close
|
|
|
|
# the socket. we want to reconnect when somebody tries to send again.
|
|
|
|
#
|
|
|
|
# NOTE: we DO propagate the error, though, because we cannot simply
|
|
|
|
# ignore the error... the caller will know if they can retry.
|
|
|
|
if self.debuglevel > 0:
|
2015-06-22 09:14:54 +03:00
|
|
|
print("send:", repr(str))
|
2009-11-02 19:03:22 +03:00
|
|
|
try:
|
2010-01-25 09:05:27 +03:00
|
|
|
blocksize = 8192
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
read = getattr(str, "read", None)
|
2011-07-26 05:36:59 +04:00
|
|
|
if read is not None:
|
2019-08-02 15:12:24 +03:00
|
|
|
# Uploading data is always expected to consume the entire file
|
|
|
|
# or buffer as we send a content-length for the full length. So
|
|
|
|
# it's safe for us to rewind the cursor back to the beginning of
|
|
|
|
# our file/buffer. This avoid the problem that when we retry a
|
|
|
|
# request that we can't read the data that we need, as we've already
|
|
|
|
# consumed all the data by reading it.
|
|
|
|
seek = getattr(str, "seek", None)
|
|
|
|
if seek is not None:
|
|
|
|
if self.debuglevel > 0:
|
|
|
|
print("sending a seek()able")
|
|
|
|
seek(0)
|
|
|
|
|
lfs/http keepalive: fix broken uploads on retry
Summary:
The KeepAlive HTTP implementation is bugged in it’s retry logic, it supports reading from a file pointer, but doesn’t support rewinding of the seek cursor when it performa a retry. So it can happen that an upload fails for whatever reason and will then ‘hang’ on the retry event.
The sequence of events that get triggered are:
- Upload file A, goes OK. Keep-Alive caches connection.
- Upload file B, fails due to (for example) failing Keep-Alive, but LFS file pointer has
been consumed for the upload and fd has been closed.
- Retry for file B starts, sets the Content-Length properly to the expected file size, but since file pointer has been consumed no data will be uploaded, causing the server to wait for the uploaded data until either client or server reaches a timeout, making it seem as our mercurial process hangs.
This is just a stop-gap measure to prevent this behavior from blocking Mercurial (LFS has retry logic). A proper solutions need to be build on top of this stop-gap measure: for upload from file pointers, we should support fseek() on the interface,
Since we expect to consume the whole file always anyways, this should bafe. This way we can seek back to the beginning on a retry.
Reviewed By: quark-zju
Differential Revision: D16604844
fbshipit-source-id: 9a0d418f00520bda499272b864ff24c99e888aa2
2019-08-02 12:37:38 +03:00
|
|
|
expectedlen = len(str)
|
|
|
|
totalwritten = 0
|
2010-01-25 09:05:27 +03:00
|
|
|
if self.debuglevel > 0:
|
2015-06-22 09:14:54 +03:00
|
|
|
print("sending a read()able")
|
2011-07-26 05:36:59 +04:00
|
|
|
data = read(blocksize)
|
2009-11-02 19:03:22 +03:00
|
|
|
while data:
|
lfs/http keepalive: fix broken uploads on retry
Summary:
The KeepAlive HTTP implementation is bugged in it’s retry logic, it supports reading from a file pointer, but doesn’t support rewinding of the seek cursor when it performa a retry. So it can happen that an upload fails for whatever reason and will then ‘hang’ on the retry event.
The sequence of events that get triggered are:
- Upload file A, goes OK. Keep-Alive caches connection.
- Upload file B, fails due to (for example) failing Keep-Alive, but LFS file pointer has
been consumed for the upload and fd has been closed.
- Retry for file B starts, sets the Content-Length properly to the expected file size, but since file pointer has been consumed no data will be uploaded, causing the server to wait for the uploaded data until either client or server reaches a timeout, making it seem as our mercurial process hangs.
This is just a stop-gap measure to prevent this behavior from blocking Mercurial (LFS has retry logic). A proper solutions need to be build on top of this stop-gap measure: for upload from file pointers, we should support fseek() on the interface,
Since we expect to consume the whole file always anyways, this should bafe. This way we can seek back to the beginning on a retry.
Reviewed By: quark-zju
Differential Revision: D16604844
fbshipit-source-id: 9a0d418f00520bda499272b864ff24c99e888aa2
2019-08-02 12:37:38 +03:00
|
|
|
totalwritten += len(data)
|
2009-11-02 19:03:22 +03:00
|
|
|
self.sock.sendall(data)
|
2011-07-26 05:36:59 +04:00
|
|
|
data = read(blocksize)
|
lfs/http keepalive: fix broken uploads on retry
Summary:
The KeepAlive HTTP implementation is bugged in it’s retry logic, it supports reading from a file pointer, but doesn’t support rewinding of the seek cursor when it performa a retry. So it can happen that an upload fails for whatever reason and will then ‘hang’ on the retry event.
The sequence of events that get triggered are:
- Upload file A, goes OK. Keep-Alive caches connection.
- Upload file B, fails due to (for example) failing Keep-Alive, but LFS file pointer has
been consumed for the upload and fd has been closed.
- Retry for file B starts, sets the Content-Length properly to the expected file size, but since file pointer has been consumed no data will be uploaded, causing the server to wait for the uploaded data until either client or server reaches a timeout, making it seem as our mercurial process hangs.
This is just a stop-gap measure to prevent this behavior from blocking Mercurial (LFS has retry logic). A proper solutions need to be build on top of this stop-gap measure: for upload from file pointers, we should support fseek() on the interface,
Since we expect to consume the whole file always anyways, this should bafe. This way we can seek back to the beginning on a retry.
Reviewed By: quark-zju
Differential Revision: D16604844
fbshipit-source-id: 9a0d418f00520bda499272b864ff24c99e888aa2
2019-08-02 12:37:38 +03:00
|
|
|
|
2019-08-02 15:12:24 +03:00
|
|
|
# This is a safety check to avoid waiting for a server response in
|
|
|
|
# the case where we haven't written the amount of data that the
|
|
|
|
# server is expecting. Earlier in the stack we set the Content-Length
|
|
|
|
# to the length of 'str'. It's possible a server could wait indefenitely
|
|
|
|
# in some cases where less data has been written then expected, therefor
|
|
|
|
# always ensure we have actually written what we are expecting.
|
lfs/http keepalive: fix broken uploads on retry
Summary:
The KeepAlive HTTP implementation is bugged in it’s retry logic, it supports reading from a file pointer, but doesn’t support rewinding of the seek cursor when it performa a retry. So it can happen that an upload fails for whatever reason and will then ‘hang’ on the retry event.
The sequence of events that get triggered are:
- Upload file A, goes OK. Keep-Alive caches connection.
- Upload file B, fails due to (for example) failing Keep-Alive, but LFS file pointer has
been consumed for the upload and fd has been closed.
- Retry for file B starts, sets the Content-Length properly to the expected file size, but since file pointer has been consumed no data will be uploaded, causing the server to wait for the uploaded data until either client or server reaches a timeout, making it seem as our mercurial process hangs.
This is just a stop-gap measure to prevent this behavior from blocking Mercurial (LFS has retry logic). A proper solutions need to be build on top of this stop-gap measure: for upload from file pointers, we should support fseek() on the interface,
Since we expect to consume the whole file always anyways, this should bafe. This way we can seek back to the beginning on a retry.
Reviewed By: quark-zju
Differential Revision: D16604844
fbshipit-source-id: 9a0d418f00520bda499272b864ff24c99e888aa2
2019-08-02 12:37:38 +03:00
|
|
|
if expectedlen != totalwritten:
|
|
|
|
raise urlerr.urlerror(
|
|
|
|
"couldn't read {} bytes, only got {}".format(
|
|
|
|
expectedlen, totalwritten
|
|
|
|
)
|
|
|
|
)
|
2009-11-02 19:03:22 +03:00
|
|
|
else:
|
|
|
|
self.sock.sendall(str)
|
2015-06-24 08:20:08 +03:00
|
|
|
except socket.error as v:
|
2009-11-02 19:03:22 +03:00
|
|
|
reraise = True
|
2020-08-25 15:20:01 +03:00
|
|
|
if v.errno == errno.EPIPE: # Broken pipe
|
2009-11-02 19:03:22 +03:00
|
|
|
if self._HTTPConnection__state == httplib._CS_REQ_SENT:
|
|
|
|
self._broken_pipe_resp = None
|
|
|
|
self._broken_pipe_resp = self.getresponse()
|
|
|
|
reraise = False
|
|
|
|
self.close()
|
|
|
|
if reraise:
|
|
|
|
raise
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2009-11-02 19:03:22 +03:00
|
|
|
def wrapgetresponse(cls):
|
2020-10-15 06:19:13 +03:00
|
|
|
"""Wraps getresponse in cls with a broken-pipe sane version."""
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2009-11-02 19:03:22 +03:00
|
|
|
def safegetresponse(self):
|
|
|
|
# In safesend() we might set the _broken_pipe_resp
|
|
|
|
# attribute, in which case the socket has already
|
|
|
|
# been closed and we just need to give them the response
|
|
|
|
# back. Otherwise, we use the normal response path.
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
r = getattr(self, "_broken_pipe_resp", None)
|
2009-11-02 19:03:22 +03:00
|
|
|
if r is not None:
|
|
|
|
return r
|
|
|
|
return cls.getresponse(self)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2009-11-02 19:03:22 +03:00
|
|
|
safegetresponse.__doc__ = cls.getresponse.__doc__
|
|
|
|
return safegetresponse
|
2006-06-15 23:57:59 +04:00
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
class HTTPConnection(httplib.HTTPConnection):
|
|
|
|
# use the modified response class
|
|
|
|
response_class = HTTPResponse
|
2009-11-02 19:03:22 +03:00
|
|
|
send = safesend
|
|
|
|
getresponse = wrapgetresponse(httplib.HTTPConnection)
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
#########################################################################
|
|
|
|
##### TEST FUNCTIONS
|
|
|
|
#########################################################################
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
def continuity(url):
|
2016-06-10 07:12:33 +03:00
|
|
|
md5 = hashlib.md5
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
format = "%25s: %s"
|
2006-07-12 19:28:00 +04:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
# first fetch the file with the normal http handler
|
2016-04-07 02:22:12 +03:00
|
|
|
opener = urlreq.buildopener()
|
|
|
|
urlreq.installopener(opener)
|
|
|
|
fo = urlreq.urlopen(url)
|
2006-06-15 23:57:59 +04:00
|
|
|
foo = fo.read()
|
|
|
|
fo.close()
|
2014-09-24 10:52:40 +04:00
|
|
|
m = md5(foo)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
print(format % ("normal urllib", m.hexdigest()))
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
# now install the keepalive handler and try again
|
2016-04-07 02:22:12 +03:00
|
|
|
opener = urlreq.buildopener(HTTPHandler())
|
|
|
|
urlreq.installopener(opener)
|
2006-06-15 23:57:59 +04:00
|
|
|
|
2016-04-07 02:22:12 +03:00
|
|
|
fo = urlreq.urlopen(url)
|
2006-06-15 23:57:59 +04:00
|
|
|
foo = fo.read()
|
|
|
|
fo.close()
|
2014-09-24 10:52:40 +04:00
|
|
|
m = md5(foo)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
print(format % ("keepalive read", m.hexdigest()))
|
2006-06-15 23:57:59 +04:00
|
|
|
|
2016-04-07 02:22:12 +03:00
|
|
|
fo = urlreq.urlopen(url)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
foo = ""
|
2011-06-01 14:38:46 +04:00
|
|
|
while True:
|
2006-06-15 23:57:59 +04:00
|
|
|
f = fo.readline()
|
2010-01-25 09:05:27 +03:00
|
|
|
if f:
|
|
|
|
foo = foo + f
|
2017-09-29 18:49:20 +03:00
|
|
|
else:
|
|
|
|
break
|
2006-06-15 23:57:59 +04:00
|
|
|
fo.close()
|
2014-09-24 10:52:40 +04:00
|
|
|
m = md5(foo)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
print(format % ("keepalive readline", m.hexdigest()))
|
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
def comp(N, url):
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
print(" making %i connections to:\n %s" % (N, url))
|
2006-06-15 23:57:59 +04:00
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
util.stdout.write(" first using the normal urllib handlers")
|
2006-06-15 23:57:59 +04:00
|
|
|
# first use normal opener
|
2016-04-07 02:22:12 +03:00
|
|
|
opener = urlreq.buildopener()
|
|
|
|
urlreq.installopener(opener)
|
2006-06-15 23:57:59 +04:00
|
|
|
t1 = fetch(N, url)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
print(" TIME: %.3f s" % t1)
|
2006-06-15 23:57:59 +04:00
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
util.stdout.write(" now using the keepalive handler ")
|
2006-06-15 23:57:59 +04:00
|
|
|
# now install the keepalive handler and try again
|
2016-04-07 02:22:12 +03:00
|
|
|
opener = urlreq.buildopener(HTTPHandler())
|
|
|
|
urlreq.installopener(opener)
|
2006-06-15 23:57:59 +04:00
|
|
|
t2 = fetch(N, url)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
print(" TIME: %.3f s" % t2)
|
|
|
|
print(" improvement factor: %.2f" % (t1 / t2))
|
|
|
|
|
2006-07-12 19:28:00 +04:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
def fetch(N, url, delay=0):
|
|
|
|
import time
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
lens = []
|
|
|
|
starttime = time.time()
|
|
|
|
for i in range(N):
|
2010-01-25 09:05:27 +03:00
|
|
|
if delay and i > 0:
|
|
|
|
time.sleep(delay)
|
2016-04-07 02:22:12 +03:00
|
|
|
fo = urlreq.urlopen(url)
|
2006-06-15 23:57:59 +04:00
|
|
|
foo = fo.read()
|
|
|
|
fo.close()
|
|
|
|
lens.append(len(foo))
|
|
|
|
diff = time.time() - starttime
|
|
|
|
|
|
|
|
j = 0
|
|
|
|
for i in lens[1:]:
|
|
|
|
j = j + 1
|
|
|
|
if not i == lens[0]:
|
2015-06-22 09:14:54 +03:00
|
|
|
print("WARNING: inconsistent length on read %i: %i" % (j, i))
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
return diff
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
def test_timeout(url):
|
|
|
|
global DEBUG
|
|
|
|
dbbackup = DEBUG
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2011-06-29 17:00:00 +04:00
|
|
|
class FakeLogger(object):
|
2010-01-25 09:05:27 +03:00
|
|
|
def debug(self, msg, *args):
|
2015-06-22 09:14:54 +03:00
|
|
|
print(msg % args)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
info = warning = error = debug
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
DEBUG = FakeLogger()
|
2015-06-22 09:14:54 +03:00
|
|
|
print(" fetching the file to establish a connection")
|
2016-04-07 02:22:12 +03:00
|
|
|
fo = urlreq.urlopen(url)
|
2006-06-15 23:57:59 +04:00
|
|
|
data1 = fo.read()
|
|
|
|
fo.close()
|
2006-07-12 19:28:00 +04:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
i = 20
|
2015-06-22 09:14:54 +03:00
|
|
|
print(" waiting %i seconds for the server to close the connection" % i)
|
2006-06-15 23:57:59 +04:00
|
|
|
while i > 0:
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
util.stdout.write("\r %2i" % i)
|
2016-10-20 17:53:36 +03:00
|
|
|
util.stdout.flush()
|
2006-06-15 23:57:59 +04:00
|
|
|
time.sleep(1)
|
|
|
|
i -= 1
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
util.stderr.write("\r")
|
2006-06-15 23:57:59 +04:00
|
|
|
|
2015-06-22 09:14:54 +03:00
|
|
|
print(" fetching the file a second time")
|
2016-04-07 02:22:12 +03:00
|
|
|
fo = urlreq.urlopen(url)
|
2006-06-15 23:57:59 +04:00
|
|
|
data2 = fo.read()
|
|
|
|
fo.close()
|
|
|
|
|
|
|
|
if data1 == data2:
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
print(" data are identical")
|
2006-06-15 23:57:59 +04:00
|
|
|
else:
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
print(" ERROR: DATA DIFFER")
|
2006-06-15 23:57:59 +04:00
|
|
|
|
|
|
|
DEBUG = dbbackup
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
def test(url, N=10):
|
2015-06-22 09:14:54 +03:00
|
|
|
print("performing continuity test (making sure stuff isn't corrupted)")
|
2006-06-15 23:57:59 +04:00
|
|
|
continuity(url)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
print("")
|
2015-06-22 09:14:54 +03:00
|
|
|
print("performing speed comparison")
|
2006-06-15 23:57:59 +04:00
|
|
|
comp(N, url)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
print("")
|
2015-06-22 09:14:54 +03:00
|
|
|
print("performing dropped-connection check")
|
2006-06-15 23:57:59 +04:00
|
|
|
test_timeout(url)
|
2006-07-12 19:28:00 +04:00
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
|
|
|
if __name__ == "__main__":
|
2006-06-15 23:57:59 +04:00
|
|
|
import time
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2006-06-15 23:57:59 +04:00
|
|
|
try:
|
|
|
|
N = int(sys.argv[1])
|
|
|
|
url = sys.argv[2]
|
2012-05-12 18:02:45 +04:00
|
|
|
except (IndexError, ValueError):
|
2015-06-22 09:14:54 +03:00
|
|
|
print("%s <integer> <url>" % sys.argv[0])
|
2006-06-15 23:57:59 +04:00
|
|
|
else:
|
|
|
|
test(url, N)
|