From 4a9e23d532965e1cb2ff00e0fbae1a3d9b493af3 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 18 Jan 2020 21:11:35 +0200 Subject: [PATCH] [3.7] Fix python 3.8 tests under Windows (#4513) * Add Python 3.8 to build matrix * fix tests * flake fix * fixes * fix mypy * one more assert * Move selector_loop fixture to conftest.py * Add CHANGELOG * Fix tests on Python 3.8.1 Co-authored-by: hh-h . (cherry picked from commit ec493d6cb5381d9d6867494edc28c6ff20bc8a6e) Co-authored-by: Andrew Svetlov --- CHANGES/4513.feature | 1 + aiohttp/payload_streamer.py | 6 +- tests/conftest.py | 19 +++++ tests/test_client_request.py | 100 ------------------------- tests/test_client_session.py | 14 +++- tests/test_connector.py | 61 ++++++++------- tests/test_multipart.py | 26 ++++--- tests/test_web_functional.py | 10 ++- tests/test_web_runner.py | 6 +- tests/test_web_urldispatcher.py | 3 +- tests/test_web_websocket_functional.py | 16 ++-- 13 files changed, 143 insertions(+), 160 deletions(-) create mode 100644 CHANGES/4513.feature diff --git a/CHANGES/4513.feature b/CHANGES/4513.feature new file mode 100644 index 000000000..e68f516e3 --- /dev/null +++ b/CHANGES/4513.feature @@ -0,0 +1 @@ +Pass tests on Python 3.8 for Windows. diff --git a/aiohttp/payload_streamer.py b/aiohttp/payload_streamer.py index e76bf430a..132270437 100644 --- a/aiohttp/payload_streamer.py +++ b/aiohttp/payload_streamer.py @@ -21,7 +21,7 @@ async def file_sender(writer, file_name=None): """ -import asyncio +import types import warnings from typing import Any, Awaitable, Callable, Dict, Tuple @@ -37,12 +37,12 @@ def __init__(self, coro: Callable[..., Awaitable[None]], args: Tuple[Any, ...], kwargs: Dict[str, Any]) -> None: - self.coro = asyncio.coroutine(coro) + self.coro = types.coroutine(coro) self.args = args self.kwargs = kwargs async def __call__(self, writer: AbstractStreamWriter) -> None: - await self.coro(writer, *self.args, **self.kwargs) + await self.coro(writer, *self.args, **self.kwargs) # type: ignore class streamer: diff --git a/tests/conftest.py b/tests/conftest.py index d1cec0b73..171c97f78 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,16 @@ +import asyncio import hashlib import pathlib import shutil import ssl +import sys import tempfile import uuid import pytest +from aiohttp.test_utils import loop_context + try: import trustme TRUSTME = True @@ -85,3 +89,18 @@ def tls_certificate_fingerprint_sha256(tls_certificate_pem_bytes): def pipe_name(): name = r'\\.\pipe\{}'.format(uuid.uuid4().hex) return name +@pytest.fixture +def selector_loop(): + if sys.version_info < (3, 7): + policy = asyncio.get_event_loop_policy() + policy._loop_factory = asyncio.SelectorEventLoop # type: ignore + else: + if sys.version_info >= (3, 8): + policy = asyncio.WindowsSelectorEventLoopPolicy() # type: ignore + else: + policy = asyncio.DefaultEventLoopPolicy() + asyncio.set_event_loop_policy(policy) + + with loop_context(policy.new_event_loop) as _loop: + asyncio.set_event_loop(_loop) + yield _loop diff --git a/tests/test_client_request.py b/tests/test_client_request.py index e3a7d1e32..bc75fcd2e 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -964,33 +964,6 @@ async def throw_exc(): await req.close() -async def test_data_stream_exc_deprecated(loop, conn) -> None: - fut = loop.create_future() - - with pytest.warns(DeprecationWarning): - @aiohttp.streamer - async def gen(writer): - await writer.write(b'binary data') - await fut - - req = ClientRequest( - 'POST', URL('http://python.org/'), data=gen(), loop=loop) - assert req.chunked - assert req.headers['TRANSFER-ENCODING'] == 'chunked' - - async def throw_exc(): - await asyncio.sleep(0.01, loop=loop) - fut.set_exception(ValueError) - - loop.create_task(throw_exc()) - - await req.send(conn) - await req._writer - # assert conn.close.called - assert conn.protocol.set_exception.called - await req.close() - - async def test_data_stream_exc_chain(loop, conn) -> None: fut = loop.create_future() @@ -1020,36 +993,6 @@ async def throw_exc(): await req.close() -async def test_data_stream_exc_chain_deprecated(loop, conn) -> None: - fut = loop.create_future() - - with pytest.warns(DeprecationWarning): - @aiohttp.streamer - async def gen(writer): - await fut - - req = ClientRequest('POST', URL('http://python.org/'), - data=gen(), loop=loop) - - inner_exc = ValueError() - - async def throw_exc(): - await asyncio.sleep(0.01, loop=loop) - fut.set_exception(inner_exc) - - loop.create_task(throw_exc()) - - await req.send(conn) - await req._writer - # assert connection.close.called - assert conn.protocol.set_exception.called - outer_exc = conn.protocol.set_exception.call_args[0][0] - assert isinstance(outer_exc, ValueError) - assert inner_exc is outer_exc - assert inner_exc is outer_exc - await req.close() - - async def test_data_stream_continue(loop, buf, conn) -> None: @async_generator async def gen(): @@ -1075,33 +1018,6 @@ async def coro(): resp.close() -async def test_data_stream_continue_deprecated(loop, buf, conn) -> None: - with pytest.warns(DeprecationWarning): - @aiohttp.streamer - async def gen(writer): - await writer.write(b'binary data') - await writer.write(b' result') - await writer.write_eof() - - req = ClientRequest( - 'POST', URL('http://python.org/'), data=gen(), - expect100=True, loop=loop) - assert req.chunked - - async def coro(): - await asyncio.sleep(0.0001, loop=loop) - req._continue.set_result(1) - - loop.create_task(coro()) - - resp = await req.send(conn) - await req._writer - assert buf.split(b'\r\n\r\n', 1)[1] == \ - b'b\r\nbinary data\r\n7\r\n result\r\n0\r\n\r\n' - await req.close() - resp.close() - - async def test_data_continue(loop, buf, conn) -> None: req = ClientRequest( 'POST', URL('http://python.org/'), data=b'data', @@ -1136,22 +1052,6 @@ async def gen(): resp.close() -async def test_close_deprecated(loop, buf, conn) -> None: - with pytest.warns(DeprecationWarning): - @aiohttp.streamer - async def gen(writer): - await asyncio.sleep(0.00001, loop=loop) - await writer.write(b'result') - - req = ClientRequest( - 'POST', URL('http://python.org/'), data=gen(), loop=loop) - resp = await req.send(conn) - await req.close() - assert buf.split(b'\r\n\r\n', 1)[1] == b'6\r\nresult\r\n0\r\n\r\n' - await req.close() - resp.close() - - async def test_custom_response_class(loop, conn) -> None: class CustomResponse(ClientResponse): def read(self, decode=False): diff --git a/tests/test_client_session.py b/tests/test_client_session.py index 5b348ab58..07c2bd00e 100644 --- a/tests/test_client_session.py +++ b/tests/test_client_session.py @@ -3,6 +3,7 @@ import gc import json import re +import sys from http.cookies import SimpleCookie from io import BytesIO from unittest import mock @@ -608,7 +609,8 @@ async def on_response_chunk_received(session, context, params): {'ok': True}).encode('utf8') -async def test_request_tracing_exception(loop) -> None: +async def test_request_tracing_exception() -> None: + loop = asyncio.get_event_loop() on_request_end = mock.Mock(side_effect=make_mocked_coro(mock.Mock())) on_request_exception = mock.Mock( side_effect=make_mocked_coro(mock.Mock()) @@ -620,9 +622,13 @@ async def test_request_tracing_exception(loop) -> None: with mock.patch("aiohttp.client.TCPConnector.connect") as connect_patched: error = Exception() - f = loop.create_future() - f.set_exception(error) - connect_patched.return_value = f + if sys.version_info >= (3, 8, 1): + connect_patched.side_effect = error + else: + loop = asyncio.get_event_loop() + f = loop.create_future() + f.set_exception(error) + connect_patched.return_value = f session = aiohttp.ClientSession( loop=loop, diff --git a/tests/test_connector.py b/tests/test_connector.py index 6e1e41d29..d854890dd 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -193,7 +193,7 @@ async def test_del_with_scheduled_cleanup(loop) -> None: # obviously doesn't deletion because loop has a strong # reference to connector's instance method, isn't it? del conn - await asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01) gc.collect() assert not conns_impl @@ -647,7 +647,7 @@ async def test_tcp_connector_resolve_host(loop) -> None: def dns_response(loop): async def coro(): # simulates a network operation - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) return ["127.0.0.1"] return coro @@ -709,7 +709,7 @@ async def test_tcp_connector_dns_throttle_requests(loop, dns_response) -> None: m_resolver().resolve.return_value = dns_response() loop.create_task(conn._resolve_host('localhost', 8080)) loop.create_task(conn._resolve_host('localhost', 8080)) - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) m_resolver().resolve.assert_called_once_with( 'localhost', 8080, @@ -729,7 +729,7 @@ async def test_tcp_connector_dns_throttle_requests_exception_spread( m_resolver().resolve.side_effect = e r1 = loop.create_task(conn._resolve_host('localhost', 8080)) r2 = loop.create_task(conn._resolve_host('localhost', 8080)) - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert r1.exception() == e assert r2.exception() == e @@ -748,7 +748,7 @@ async def test_tcp_connector_dns_throttle_requests_cancelled_when_close( loop.create_task(conn._resolve_host('localhost', 8080)) f = loop.create_task(conn._resolve_host('localhost', 8080)) - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) conn.close() with pytest.raises(asyncio.CancelledError): @@ -938,7 +938,7 @@ async def test_tcp_connector_dns_tracing_throttle_requests( m_resolver().resolve.return_value = dns_response() loop.create_task(conn._resolve_host('localhost', 8080, traces=traces)) loop.create_task(conn._resolve_host('localhost', 8080, traces=traces)) - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) on_dns_cache_hit.assert_called_once_with( session, trace_config_ctx, @@ -1103,7 +1103,7 @@ async def test_close_during_connect(loop) -> None: conn._create_connection.return_value = fut task = loop.create_task(conn.connect(req, None, ClientTimeout())) - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) conn.close() fut.set_result(proto) @@ -1423,10 +1423,10 @@ async def f(): task = loop.create_task(f()) - await asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01) assert not acquired connection1.release() - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert acquired await task conn.close() @@ -1486,7 +1486,7 @@ async def f(): connection2.release() task = asyncio.ensure_future(f(), loop=loop) - await asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01) connection1.release() await task conn.close() @@ -1557,10 +1557,10 @@ async def f(): task = loop.create_task(f()) - await asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01) assert not acquired connection1.release() - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert acquired await task conn.close() @@ -1589,10 +1589,10 @@ async def f(): task = loop.create_task(f()) - await asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01) assert not acquired connection1.release() - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert acquired await task conn.close() @@ -1623,7 +1623,7 @@ async def f(): task = loop.create_task(f()) - await asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01) assert acquired connection1.release() await task @@ -1653,7 +1653,7 @@ async def test_connect_with_limit_cancelled(loop) -> None: with pytest.raises(asyncio.TimeoutError): # limit exhausted await asyncio.wait_for(conn.connect(req, None, ClientTimeout()), - 0.01, loop=loop) + 0.01) connection.close() @@ -1695,7 +1695,7 @@ async def test_connect_with_limit_concurrent(loop) -> None: async def create_connection(req, traces, timeout): nonlocal num_connections num_connections += 1 - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) # Make a new transport mock each time because acquired # transports are stored in a set. Reusing the same object @@ -1724,13 +1724,13 @@ async def f(start=True): num_requests += 1 if not start: connection = await conn.connect(req, None, ClientTimeout()) - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) connection.release() tasks = [ loop.create_task(f(start=False)) for i in range(start_requests) ] - await asyncio.wait(tasks, loop=loop) + await asyncio.wait(tasks) await f() conn.close() @@ -1749,11 +1749,11 @@ async def test_connect_waiters_cleanup(loop) -> None: t = loop.create_task(conn.connect(req, None, ClientTimeout())) - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert conn._waiters.keys() t.cancel() - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert not conn._waiters.keys() @@ -1768,7 +1768,7 @@ async def test_connect_waiters_cleanup_key_error(loop) -> None: t = loop.create_task(conn.connect(req, None, ClientTimeout())) - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert conn._waiters.keys() # we delete the entry explicitly before the @@ -1776,7 +1776,7 @@ async def test_connect_waiters_cleanup_key_error(loop) -> None: # must expect a none failure termination conn._waiters.clear() t.cancel() - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert not conn._waiters.keys() == [] @@ -1875,7 +1875,7 @@ async def create_connection(req, traces, timeout): t1 = loop.create_task(conn.connect(req, None, ClientTimeout())) t2 = loop.create_task(conn.connect(req, None, ClientTimeout())) - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert not t1.done() assert not t2.done() assert len(conn._acquired_per_host[key]) == 1 @@ -1908,7 +1908,7 @@ async def create_connection(req, traces=None): conn._acquired.add(proto) conn2 = loop.create_task(conn.connect(req, None, ClientTimeout())) - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) conn2.cancel() with pytest.raises(asyncio.CancelledError): @@ -1943,7 +1943,7 @@ async def create_connection(req, traces, timeout): t1 = loop.create_task(conn.connect(req, None, ClientTimeout())) t2 = loop.create_task(conn.connect(req, None, ClientTimeout())) t3 = loop.create_task(conn.connect(req, None, ClientTimeout())) - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) assert not t1.done() assert not t2.done() assert len(conn._acquired_per_host[key]) == 1 @@ -2006,9 +2006,12 @@ async def test_unix_connector_permission(loop) -> None: @pytest.mark.skipif(platform.system() != "Windows", reason="Proactor Event loop present only in Windows") -async def test_named_pipe_connector_wrong_loop(loop, pipe_name) -> None: +async def test_named_pipe_connector_wrong_loop( + selector_loop, + pipe_name +) -> None: with pytest.raises(RuntimeError): - aiohttp.NamedPipeConnector(pipe_name, loop=loop) + aiohttp.NamedPipeConnector(pipe_name, loop=asyncio.get_event_loop()) @pytest.mark.skipif(platform.system() != "Windows", @@ -2245,7 +2248,7 @@ def test_not_expired_ttl(self) -> None: async def test_expired_ttl(self, loop) -> None: dns_cache_table = _DNSCacheTable(ttl=0.01) dns_cache_table.add('localhost', ['127.0.0.1']) - await asyncio.sleep(0.02, loop=loop) + await asyncio.sleep(0.02) assert dns_cache_table.expired('localhost') def test_next_addrs(self, dns_cache_table) -> None: diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 074dd23dc..af07bbe2d 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -1,6 +1,6 @@ -import asyncio import io import json +import sys import zlib from unittest import mock @@ -158,13 +158,15 @@ async def test_read_chunk_without_content_length(self) -> None: assert c3 == b'' async def test_read_incomplete_chunk(self) -> None: - loop = asyncio.get_event_loop() stream = Stream(b'') - def prepare(data): - f = loop.create_future() - f.set_result(data) - return f + if sys.version_info >= (3, 8, 1): + # Workaround for a weird behavior of patch.object + def prepare(data): + return data + else: + async def prepare(data): + return data with mock.patch.object(stream, 'read', side_effect=[ prepare(b'Hello, '), @@ -200,13 +202,15 @@ async def test_read_incomplete_body_chunked(self) -> None: assert b'Hello, World!\r\n-' == result async def test_read_boundary_with_incomplete_chunk(self) -> None: - loop = asyncio.get_event_loop() stream = Stream(b'') - def prepare(data): - f = loop.create_future() - f.set_result(data) - return f + if sys.version_info >= (3, 8, 1): + # Workaround for weird 3.8.1 patch.object() behavior + def prepare(data): + return data + else: + async def prepare(data): + return data with mock.patch.object(stream, 'read', side_effect=[ prepare(b'Hello, World'), diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index ceedb1b98..cce8eb7cd 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -1509,7 +1509,10 @@ async def handler(request): assert 413 == resp.status resp_text = await resp.text() assert 'Maximum request body size 1048576 exceeded, ' \ - 'actual body size 1048591' in resp_text + 'actual body size' in resp_text + # Maximum request body size X exceeded, actual body size X + body_size = int(resp_text.split()[-1]) + assert body_size >= max_size async def test_app_max_client_size_adjusted(aiohttp_client) -> None: @@ -1535,7 +1538,10 @@ async def handler(request): assert 413 == resp.status resp_text = await resp.text() assert 'Maximum request body size 2097152 exceeded, ' \ - 'actual body size 2097166' in resp_text + 'actual body size' in resp_text + # Maximum request body size X exceeded, actual body size X + body_size = int(resp_text.split()[-1]) + assert body_size >= custom_max_size async def test_app_max_client_size_none(aiohttp_client) -> None: diff --git a/tests/test_web_runner.py b/tests/test_web_runner.py index f4c91de7f..7ffaf2f59 100644 --- a/tests/test_web_runner.py +++ b/tests/test_web_runner.py @@ -118,7 +118,11 @@ async def test_addresses(make_runner, shorttmpdir) -> None: @pytest.mark.skipif(platform.system() != "Windows", reason="Proactor Event loop present only in Windows") -async def test_named_pipe_runner_wrong_loop(app, pipe_name) -> None: +async def test_named_pipe_runner_wrong_loop( + app, + selector_loop, + pipe_name +) -> None: runner = web.AppRunner(app) await runner.setup() with pytest.raises(RuntimeError): diff --git a/tests/test_web_urldispatcher.py b/tests/test_web_urldispatcher.py index 30a5beac8..63f9bb090 100644 --- a/tests/test_web_urldispatcher.py +++ b/tests/test_web_urldispatcher.py @@ -276,8 +276,7 @@ async def test_partially_applied_handler(aiohttp_client) -> None: async def handler(data, request): return web.Response(body=data) - with pytest.warns(DeprecationWarning): - app.router.add_route('GET', '/', functools.partial(handler, b'hello')) + app.router.add_route('GET', '/', functools.partial(handler, b'hello')) client = await aiohttp_client(app) r = await client.get('/') diff --git a/tests/test_web_websocket_functional.py b/tests/test_web_websocket_functional.py index 7ad984045..59f5e1101 100644 --- a/tests/test_web_websocket_functional.py +++ b/tests/test_web_websocket_functional.py @@ -273,7 +273,7 @@ async def handler(request): # The server closes here. Then the client sends bogus messages with an # internval shorter than server-side close timeout, to make the server # hanging indefinitely. - await asyncio.sleep(0.08, loop=loop) + await asyncio.sleep(0.08) msg = await ws._reader.read() assert msg.type == WSMsgType.CLOSE await ws.send_str('hang') @@ -281,16 +281,16 @@ async def handler(request): # i am not sure what do we test here # under uvloop this code raises RuntimeError try: - await asyncio.sleep(0.08, loop=loop) + await asyncio.sleep(0.08) await ws.send_str('hang') - await asyncio.sleep(0.08, loop=loop) + await asyncio.sleep(0.08) await ws.send_str('hang') - await asyncio.sleep(0.08, loop=loop) + await asyncio.sleep(0.08) await ws.send_str('hang') except RuntimeError: pass - await asyncio.sleep(0.08, loop=loop) + await asyncio.sleep(0.08) assert (await aborted) assert elapsed < 0.25, \ @@ -316,7 +316,7 @@ async def handler(request): msg = await ws.receive() assert msg.type == WSMsgType.CLOSING - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) msg = await ws.receive() assert msg.type == WSMsgType.CLOSED @@ -335,7 +335,7 @@ async def handler(request): msg = await ws.receive() assert msg.type == WSMsgType.CLOSE - await asyncio.sleep(0, loop=loop) + await asyncio.sleep(0) msg = await ws.receive() assert msg.type == WSMsgType.CLOSED @@ -713,7 +713,7 @@ async def handler(request): app.router.add_route('GET', '/', handler) server = await aiohttp_server(app) - async with aiohttp.ClientSession(loop=loop) as sm: + async with aiohttp.ClientSession() as sm: async with sm.ws_connect(server.make_url('/')) as resp: items = ['q1', 'q2', 'q3']