diff --git a/trunk/PKGBUILD b/trunk/PKGBUILD index 220d9a4..7211be3 100644 --- a/trunk/PKGBUILD +++ b/trunk/PKGBUILD @@ -3,8 +3,8 @@ _pkgname=aiohttp pkgname=python-aiohttp -_gitcommit=2f655a59d0daedfa2a794996c4355b576c98ecc8 -pkgver=3.7.3 +_gitcommit=0a26acc1de9e1b0244456b7881ec16ba8bb64fc3 +pkgver=3.7.4 pkgrel=1 pkgdesc='HTTP client/server for asyncio' url='https://aiohttp.readthedocs.io' @@ -14,17 +14,21 @@ depends=('python' 'python-chardet' 'python-multidict' 'python-async-timeout' 'python-yarl' 'python-attrs') makedepends=('cython' 'python-setuptools' 'git') checkdepends=('python-pytest' 'python-pytest-runner' 'python-pytest-mock' - 'python-pytest-timeout' 'python-async_generator' 'python-brotlipy' + 'python-pytest-timeout' 'python-async_generator' 'python-brotli' 'python-pytest-xdist' 'python-pytest-forked' 'python-pytest-cov' 'python-trustme' 'python-freezegun' 'gunicorn' 'python-re-assert') optdepends=('gunicorn: to deploy using Gunicorn' 'python-aiodns: for fast DNS resolving' 'python-cchardet: for faster encoding detection' - 'python-brotlipy: for Brotli transfer-encodings support') + 'python-brotli: for Brotli transfer-encodings support') source=(${pkgname}::"git+https://github.com/aio-libs/aiohttp#commit=${_gitcommit}" - git+https://github.com/nodejs/http-parser) + git+https://github.com/nodejs/http-parser + python-aiohttp-release-resources-pytest.patch + python-aiohttp-brotli.patch) sha512sums=('SKIP' - 'SKIP') + 'SKIP' + '42db1eb1173f34351a76fcd0be28dbfa1f18be5da4bc0e75adfb4be666e26acc9fbca11a83506f4eee729122110f98512133cdc0a46615f75ee2846645f4fb7a' + '3bfc6511d0a1a54e20c5b10457041621960da869a752a0b751a424db357d4153578ff1a5e9268f27e4badb2ac01f1c76d23f0058f76b9dd44063f56a046712d8') pkgver() { cd ${pkgname} @@ -36,6 +40,8 @@ prepare() { git submodule init git config submodule."vendor/http-parser".url "${srcdir}/http-parser" git submodule update --recursive + patch -p1 -i ../python-aiohttp-release-resources-pytest.patch + patch -p1 -i ../python-aiohttp-brotli.patch sed 's|.install-cython ||' -i Makefile } diff --git a/trunk/python-aiohttp-brotli.patch b/trunk/python-aiohttp-brotli.patch new file mode 100644 index 0000000..2792c84 --- /dev/null +++ b/trunk/python-aiohttp-brotli.patch @@ -0,0 +1,110 @@ +commit 1739805a29f6e5097630854f1002304e170a6045 +Author: Andrew Svetlov +Date: Fri Dec 11 10:37:45 2020 +0200 + + Backport #3803: Replace brotlipy with Brotli (#5335) + + (cherry picked from commit 506d07548a15c4301affa0c8b8e23fd7826eb977) + +diff --git a/CHANGES/3803.feature b/CHANGES/3803.feature +new file mode 100644 +index 000000000..b2a465619 +--- /dev/null ++++ b/CHANGES/3803.feature +@@ -0,0 +1 @@ ++Use Brotli instead of brotlipy +diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py +index 71ba815ae..854e6796e 100644 +--- a/aiohttp/http_parser.py ++++ b/aiohttp/http_parser.py +@@ -812,6 +812,8 @@ class HttpPayloadParser: + class DeflateBuffer: + """DeflateStream decompress stream and feed data into specified stream.""" + ++ decompressor: Any ++ + def __init__(self, out: StreamReader, encoding: Optional[str]) -> None: + self.out = out + self.size = 0 +@@ -822,9 +824,27 @@ class DeflateBuffer: + if not HAS_BROTLI: # pragma: no cover + raise ContentEncodingError( + "Can not decode content-encoding: brotli (br). " +- "Please install `brotlipy`" ++ "Please install `Brotli`" + ) +- self.decompressor = brotli.Decompressor() ++ ++ class BrotliDecoder: ++ # Supports both 'brotlipy' and 'Brotli' packages ++ # since they share an import name. The top branches ++ # are for 'brotlipy' and bottom branches for 'Brotli' ++ def __init__(self) -> None: ++ self._obj = brotli.Decompressor() ++ ++ def decompress(self, data: bytes) -> bytes: ++ if hasattr(self._obj, "decompress"): ++ return self._obj.decompress(data) ++ return self._obj.process(data) ++ ++ def flush(self) -> bytes: ++ if hasattr(self._obj, "flush"): ++ return self._obj.flush() ++ return b"" ++ ++ self.decompressor = BrotliDecoder() + else: + zlib_mode = 16 + zlib.MAX_WBITS if encoding == "gzip" else zlib.MAX_WBITS + self.decompressor = zlib.decompressobj(wbits=zlib_mode) +diff --git a/docs/client_quickstart.rst b/docs/client_quickstart.rst +index fe770243e..e96dca453 100644 +--- a/docs/client_quickstart.rst ++++ b/docs/client_quickstart.rst +@@ -174,7 +174,7 @@ The ``gzip`` and ``deflate`` transfer-encodings are automatically + decoded for you. + + You can enable ``brotli`` transfer-encodings support, +-just install `brotlipy `_. ++just install `brotli `_. + + JSON Request + ============ +diff --git a/docs/index.rst b/docs/index.rst +index 13fe723b4..4091c0019 100644 +--- a/docs/index.rst ++++ b/docs/index.rst +@@ -52,7 +52,7 @@ Installing speedups altogether + ------------------------------ + + The following will get you ``aiohttp`` along with :term:`chardet`, +-:term:`aiodns` and ``brotlipy`` in one bundle. No need to type ++:term:`aiodns` and ``Brotli`` in one bundle. No need to type + separate commands anymore! + + .. code-block:: bash +diff --git a/requirements/base.txt b/requirements/base.txt +index ffd04d12a..859407200 100644 +--- a/requirements/base.txt ++++ b/requirements/base.txt +@@ -4,7 +4,7 @@ aiodns==2.0.0; sys_platform=="linux" or sys_platform=="darwin" and python_versio + async-generator==1.10 + async-timeout==3.0.1 + attrs==20.3.0 +-brotlipy==0.7.0 ++brotli==1.0.7 + cchardet==2.1.7 + chardet==4.0.0 + gunicorn==20.0.4 +diff --git a/setup.py b/setup.py +index 428df5d4e..1c6b1cfad 100644 +--- a/setup.py ++++ b/setup.py +@@ -137,7 +137,7 @@ args = dict( + extras_require={ + "speedups": [ + "aiodns", +- "brotlipy", ++ "Brotli", + "cchardet", + ], + }, diff --git a/trunk/python-aiohttp-release-resources-pytest.patch b/trunk/python-aiohttp-release-resources-pytest.patch new file mode 100644 index 0000000..98596b6 --- /dev/null +++ b/trunk/python-aiohttp-release-resources-pytest.patch @@ -0,0 +1,482 @@ +commit eda9eddfd1b797c23fd07751db331d37e9d347e7 +Author: Sviatoslav Sydorenko +Date: Wed Feb 24 12:25:25 2021 +0100 + + Merge branch 'bugfixes/release-resources-pytest' + + This change makes sure to release most of the hanging resources + in the lib and tests. They were detected by pytest v6.2+. + + PR #5494 + + (cherry picked from commit 8c82ba11b9e38851d75476d261a1442402cc7592) + (cherry picked from commit fe5e684a8f81f3a8db69a8c063641b7d1da94f61) + +diff --git a/CHANGES/5494.bugfix b/CHANGES/5494.bugfix +new file mode 100644 +index 000000000..449b6bdf3 +--- /dev/null ++++ b/CHANGES/5494.bugfix +@@ -0,0 +1,4 @@ ++Fixed the multipart POST requests processing to always release file ++descriptors for the ``tempfile.Temporaryfile``-created ++``_io.BufferedRandom`` instances of files sent within multipart request ++bodies via HTTP POST requests. +diff --git a/CHANGES/5494.misc b/CHANGES/5494.misc +new file mode 100644 +index 000000000..3d83a77a0 +--- /dev/null ++++ b/CHANGES/5494.misc +@@ -0,0 +1,3 @@ ++Made sure to always close most of file descriptors and release other ++resouces in tests. Started ignoring ``ResourceWarning``s in pytest for ++warnings that are hard to track. +diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py +index f11e7be44..9f9092059 100644 +--- a/aiohttp/web_request.py ++++ b/aiohttp/web_request.py +@@ -661,6 +661,7 @@ class BaseRequest(MutableMapping[str, Any], HeadersMixin): + tmp.write(chunk) + size += len(chunk) + if 0 < max_size < size: ++ tmp.close() + raise HTTPRequestEntityTooLarge( + max_size=max_size, actual_size=size + ) +diff --git a/requirements/test.txt b/requirements/test.txt +index 3085dd588..0a9e8ffcf 100644 +--- a/requirements/test.txt ++++ b/requirements/test.txt +@@ -5,7 +5,7 @@ cryptography==3.2.1; platform_machine!="i686" and python_version<"3.9" # no 32-b + freezegun==1.0.0 + mypy==0.790; implementation_name=="cpython" + mypy-extensions==0.4.3; implementation_name=="cpython" +-pytest==6.1.2 ++pytest==6.2.2 + pytest-cov==2.10.1 + pytest-mock==3.3.1 + re-assert==1.1.0 +diff --git a/setup.cfg b/setup.cfg +index df8fbc315..63e7282cd 100644 +--- a/setup.cfg ++++ b/setup.cfg +@@ -39,6 +39,15 @@ addopts = --cov=aiohttp -v -rxXs --durations 10 + filterwarnings = + error + ignore:module 'ssl' has no attribute 'OP_NO_COMPRESSION'. The Python interpreter is compiled against OpenSSL < 1.0.0. Ref. https.//docs.python.org/3/library/ssl.html#ssl.OP_NO_COMPRESSION:UserWarning ++ ignore:Exception ignored in. :pytest.PytestUnraisableExceptionWarning:_pytest.unraisableexception ++ ++ # aiohttp 3.x: ++ ignore:Exception ignored in. :pytest.PytestUnraisableExceptionWarning:_pytest.unraisableexception ++ ignore:Exception ignored in. None: + resp = await req.send(conn) + assert "CONTENT-TYPE" not in req.headers + resp.close() ++ await req.close() + + + async def test_content_type_auto_header_form(loop, conn) -> None: +@@ -715,6 +716,7 @@ async def test_pass_falsy_data_file(loop, tmpdir) -> None: + ) + assert req.headers.get("CONTENT-LENGTH", None) is not None + await req.close() ++ testfile.close() + + + # Elasticsearch API requires to send request body with GET-requests +diff --git a/tests/test_client_response.py b/tests/test_client_response.py +index 55aae9708..cec7aa320 100644 +--- a/tests/test_client_response.py ++++ b/tests/test_client_response.py +@@ -46,6 +46,7 @@ async def test_http_processing_error(session) -> None: + await response.start(connection) + + assert info.value.request_info is request_info ++ response.close() + + + def test_del(session) -> None: +diff --git a/tests/test_client_session.py b/tests/test_client_session.py +index 298dac9f2..9e9e28cf2 100644 +--- a/tests/test_client_session.py ++++ b/tests/test_client_session.py +@@ -5,6 +5,7 @@ import json + import sys + from http.cookies import SimpleCookie + from io import BytesIO ++from typing import Any + from unittest import mock + + import pytest +@@ -30,7 +31,7 @@ def connector(loop): + proto = mock.Mock() + conn._conns["a"] = [(proto, 123)] + yield conn +- conn.close() ++ loop.run_until_complete(conn.close()) + + + @pytest.fixture +@@ -292,7 +293,7 @@ async def test_connector(create_session, loop, mocker) -> None: + + await session.close() + assert connector.close.called +- connector.close() ++ await connector.close() + + + async def test_create_connector(create_session, loop, mocker) -> None: +@@ -327,7 +328,7 @@ def test_connector_loop(loop) -> None: + ) + + +-def test_detach(session) -> None: ++def test_detach(loop: Any, session: Any) -> None: + conn = session.connector + try: + assert not conn.closed +@@ -336,7 +337,7 @@ def test_detach(session) -> None: + assert session.closed + assert not conn.closed + finally: +- conn.close() ++ loop.run_until_complete(conn.close()) + + + async def test_request_closed_session(session) -> None: +@@ -514,6 +515,7 @@ async def test_cookie_jar_usage(loop, aiohttp_client) -> None: + async def test_session_default_version(loop) -> None: + session = aiohttp.ClientSession(loop=loop) + assert session.version == aiohttp.HttpVersion11 ++ await session.close() + + + async def test_session_loop(loop) -> None: +@@ -632,6 +634,8 @@ async def test_request_tracing_exception() -> None: + ) + assert not on_request_end.called + ++ await session.close() ++ + + async def test_request_tracing_interpose_headers(loop, aiohttp_client) -> None: + async def handler(request): +@@ -674,6 +678,7 @@ async def test_client_session_custom_attr(loop) -> None: + session = ClientSession(loop=loop) + with pytest.warns(DeprecationWarning): + session.custom = None ++ await session.close() + + + async def test_client_session_timeout_args(loop) -> None: +@@ -698,21 +703,25 @@ async def test_client_session_timeout_args(loop) -> None: + async def test_client_session_timeout_default_args(loop) -> None: + session1 = ClientSession() + assert session1.timeout == client.DEFAULT_TIMEOUT ++ await session1.close() + + + async def test_client_session_timeout_argument() -> None: + session = ClientSession(timeout=500) + assert session.timeout == 500 ++ await session.close() + + + async def test_requote_redirect_url_default() -> None: + session = ClientSession() + assert session.requote_redirect_url ++ await session.close() + + + async def test_requote_redirect_url_default_disable() -> None: + session = ClientSession(requote_redirect_url=False) + assert not session.requote_redirect_url ++ await session.close() + + + async def test_requote_redirect_setter() -> None: +@@ -721,3 +730,4 @@ async def test_requote_redirect_setter() -> None: + with pytest.warns(DeprecationWarning): + session.requote_redirect_url = False + assert not session.requote_redirect_url ++ await session.close() +diff --git a/tests/test_connector.py b/tests/test_connector.py +index 09841923e..f9034f6db 100644 +--- a/tests/test_connector.py ++++ b/tests/test_connector.py +@@ -657,7 +657,7 @@ async def test_tcp_connector_multiple_hosts_errors(loop) -> None: + + conn._loop.create_connection = create_connection + +- await conn.connect(req, [], ClientTimeout()) ++ established_connection = await conn.connect(req, [], ClientTimeout()) + assert ips == ips_tried + + assert os_error +@@ -666,6 +666,8 @@ async def test_tcp_connector_multiple_hosts_errors(loop) -> None: + assert fingerprint_error + assert connected + ++ established_connection.close() ++ + + async def test_tcp_connector_resolve_host(loop) -> None: + conn = aiohttp.TCPConnector(loop=loop, use_dns_cache=True) +@@ -1595,6 +1597,8 @@ async def test_connect_with_limit_cancelled(loop) -> None: + await asyncio.wait_for(conn.connect(req, None, ClientTimeout()), 0.01) + connection.close() + ++ await conn.close() ++ + + async def test_connect_with_capacity_release_waiters(loop) -> None: + def check_with_exc(err): +@@ -2260,3 +2264,5 @@ async def test_connector_does_not_remove_needed_waiters(loop, key) -> None: + await_connection_and_check_waiters(), + allow_connection_and_add_dummy_waiter(), + ) ++ ++ await connector.close() +diff --git a/tests/test_proxy.py b/tests/test_proxy.py +index 3b1bf0c05..541783153 100644 +--- a/tests/test_proxy.py ++++ b/tests/test_proxy.py +@@ -72,6 +72,8 @@ class TestProxy(unittest.TestCase): + ssl=None, + ) + ++ conn.close() ++ + @mock.patch("aiohttp.connector.ClientRequest") + def test_proxy_headers(self, ClientRequestMock) -> None: + req = ClientRequest( +@@ -112,6 +114,8 @@ class TestProxy(unittest.TestCase): + ssl=None, + ) + ++ conn.close() ++ + def test_proxy_auth(self) -> None: + with self.assertRaises(ValueError) as ctx: + ClientRequest( +diff --git a/tests/test_run_app.py b/tests/test_run_app.py +index d2ba2262a..deeb483f7 100644 +--- a/tests/test_run_app.py ++++ b/tests/test_run_app.py +@@ -630,27 +630,29 @@ web.run_app(app, host=()) + def test_sigint() -> None: + skip_if_on_windows() + +- proc = subprocess.Popen( +- [sys.executable, "-u", "-c", _script_test_signal], stdout=subprocess.PIPE +- ) +- for line in proc.stdout: +- if line.startswith(b"======== Running on"): +- break +- proc.send_signal(signal.SIGINT) +- assert proc.wait() == 0 ++ with subprocess.Popen( ++ [sys.executable, "-u", "-c", _script_test_signal], ++ stdout=subprocess.PIPE, ++ ) as proc: ++ for line in proc.stdout: ++ if line.startswith(b"======== Running on"): ++ break ++ proc.send_signal(signal.SIGINT) ++ assert proc.wait() == 0 + + + def test_sigterm() -> None: + skip_if_on_windows() + +- proc = subprocess.Popen( +- [sys.executable, "-u", "-c", _script_test_signal], stdout=subprocess.PIPE +- ) +- for line in proc.stdout: +- if line.startswith(b"======== Running on"): +- break +- proc.terminate() +- assert proc.wait() == 0 ++ with subprocess.Popen( ++ [sys.executable, "-u", "-c", _script_test_signal], ++ stdout=subprocess.PIPE, ++ ) as proc: ++ for line in proc.stdout: ++ if line.startswith(b"======== Running on"): ++ break ++ proc.terminate() ++ assert proc.wait() == 0 + + + def test_startup_cleanup_signals_even_on_failure(patched_loop) -> None: +diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py +index a28fcd4f5..7810401e8 100644 +--- a/tests/test_web_functional.py ++++ b/tests/test_web_functional.py +@@ -4,6 +4,7 @@ import json + import pathlib + import socket + import zlib ++from typing import Any + from unittest import mock + + import pytest +@@ -324,7 +325,8 @@ async def test_post_single_file(aiohttp_client) -> None: + + fname = here / "data.unknown_mime_type" + +- resp = await client.post("/", data=[fname.open("rb")]) ++ with fname.open("rb") as fd: ++ resp = await client.post("/", data=[fd]) + assert 200 == resp.status + + +@@ -874,13 +876,16 @@ async def test_response_with_streamer_no_params(aiohttp_client, fname) -> None: + assert resp.headers.get("Content-Length") == str(len(resp_data)) + + +-async def test_response_with_file(aiohttp_client, fname) -> None: ++async def test_response_with_file(aiohttp_client: Any, fname: Any) -> None: ++ outer_file_descriptor = None + + with fname.open("rb") as f: + data = f.read() + + async def handler(request): +- return web.Response(body=fname.open("rb")) ++ nonlocal outer_file_descriptor ++ outer_file_descriptor = fname.open("rb") ++ return web.Response(body=outer_file_descriptor) + + app = web.Application() + app.router.add_get("/", handler) +@@ -901,15 +906,21 @@ async def test_response_with_file(aiohttp_client, fname) -> None: + assert resp.headers.get("Content-Length") == str(len(resp_data)) + assert resp.headers.get("Content-Disposition") == expected_content_disposition + ++ outer_file_descriptor.close() + +-async def test_response_with_file_ctype(aiohttp_client, fname) -> None: ++ ++async def test_response_with_file_ctype(aiohttp_client: Any, fname: Any) -> None: ++ outer_file_descriptor = None + + with fname.open("rb") as f: + data = f.read() + + async def handler(request): ++ nonlocal outer_file_descriptor ++ outer_file_descriptor = fname.open("rb") ++ + return web.Response( +- body=fname.open("rb"), headers={"content-type": "text/binary"} ++ body=outer_file_descriptor, headers={"content-type": "text/binary"} + ) + + app = web.Application() +@@ -927,14 +938,19 @@ async def test_response_with_file_ctype(aiohttp_client, fname) -> None: + assert resp.headers.get("Content-Length") == str(len(resp_data)) + assert resp.headers.get("Content-Disposition") == expected_content_disposition + ++ outer_file_descriptor.close() ++ + +-async def test_response_with_payload_disp(aiohttp_client, fname) -> None: ++async def test_response_with_payload_disp(aiohttp_client: Any, fname: Any) -> None: ++ outer_file_descriptor = None + + with fname.open("rb") as f: + data = f.read() + + async def handler(request): +- pl = aiohttp.get_payload(fname.open("rb")) ++ nonlocal outer_file_descriptor ++ outer_file_descriptor = fname.open("rb") ++ pl = aiohttp.get_payload(outer_file_descriptor) + pl.set_content_disposition("inline", filename="test.txt") + return web.Response(body=pl, headers={"content-type": "text/binary"}) + +@@ -953,6 +969,8 @@ async def test_response_with_payload_disp(aiohttp_client, fname) -> None: + == "inline; filename=\"test.txt\"; filename*=utf-8''test.txt" + ) + ++ outer_file_descriptor.close() ++ + + async def test_response_with_payload_stringio(aiohttp_client, fname) -> None: + async def handler(request): +@@ -1565,6 +1583,7 @@ async def test_post_max_client_size(aiohttp_client) -> None: + assert ( + "Maximum request body size 10 exceeded, " "actual body size 1024" in resp_text + ) ++ data["file"].close() + + + async def test_post_max_client_size_for_file(aiohttp_client) -> None: +@@ -1618,11 +1637,12 @@ async def test_response_with_bodypart_named(aiohttp_client, tmpdir) -> None: + + f = tmpdir.join("foobar.txt") + f.write_text("test", encoding="utf8") +- data = {"file": open(str(f), "rb")} +- resp = await client.post("/", data=data) ++ with open(str(f), "rb") as fd: ++ data = {"file": fd} ++ resp = await client.post("/", data=data) + +- assert 200 == resp.status +- body = await resp.read() ++ assert 200 == resp.status ++ body = await resp.read() + assert body == b"test" + + disp = multipart.parse_content_disposition(resp.headers["content-disposition"]) +@@ -1700,12 +1720,15 @@ async def test_response_context_manager(aiohttp_server) -> None: + app = web.Application() + app.router.add_route("GET", "/", handler) + server = await aiohttp_server(app) +- resp = await aiohttp.ClientSession().get(server.make_url("/")) ++ session = aiohttp.ClientSession() ++ resp = await session.get(server.make_url("/")) + async with resp: + assert resp.status == 200 + assert resp.connection is None + assert resp.connection is None + ++ await session.close() ++ + + async def test_response_context_manager_error(aiohttp_server) -> None: + async def handler(request): +@@ -1726,6 +1749,8 @@ async def test_response_context_manager_error(aiohttp_server) -> None: + + assert len(session._connector._conns) == 1 + ++ await session.close() ++ + + async def aiohttp_client_api_context_manager(aiohttp_server): + async def handler(request): +diff --git a/tests/test_web_urldispatcher.py b/tests/test_web_urldispatcher.py +index 0ba2e7c20..b45a27bd3 100644 +--- a/tests/test_web_urldispatcher.py ++++ b/tests/test_web_urldispatcher.py +@@ -152,6 +152,7 @@ async def test_access_to_the_file_with_spaces( + r = await client.get(url) + assert r.status == 200 + assert (await r.text()) == data ++ await r.release() + + + async def test_access_non_existing_resource(tmp_dir_path, aiohttp_client) -> None: