From deeaac49e267285158264643799624623f4a7b29 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 22 Mar 2022 12:04:36 +0200 Subject: [PATCH] bpo-40280: Skip socket, fork, subprocess tests on Emscripten (GH-31986) - Add requires_fork and requires_subprocess to more tests - Skip extension import tests if dlopen is not available - Don't assume that _testcapi is a shared extension - Skip a lot of socket tests that don't work on Emscripten - Skip mmap tests, mmap emulation is incomplete - venv does not work yet - Cannot get libc from executable The "entire" test suite is now passing on Emscripten with EMSDK from git head (91 suites are skipped). --- Lib/test/lock_tests.py | 2 +- Lib/test/pythoninfo.py | 3 +- Lib/test/support/__init__.py | 16 +++++++ Lib/test/test_asyncgen.py | 4 +- Lib/test/test_asynchat.py | 2 + Lib/test/test_asyncio/__init__.py | 2 + Lib/test/test_asyncore.py | 2 + Lib/test/test_contextlib_async.py | 1 + Lib/test/test_doctest.py | 7 +++- Lib/test/test_docxmlrpc.py | 3 ++ Lib/test/test_ftplib.py | 1 + Lib/test/test_httplib.py | 1 + Lib/test/test_httpservers.py | 1 + Lib/test/test_imaplib.py | 5 ++- Lib/test/test_import/__init__.py | 16 +++++-- .../test_importlib/extension/test_finder.py | 4 ++ .../test_importlib/extension/test_loader.py | 5 +++ Lib/test/test_json/test_tool.py | 1 + Lib/test/test_logging.py | 12 ++++++ Lib/test/test_mailbox.py | 2 +- Lib/test/test_mmap.py | 10 ++++- Lib/test/test_pdb.py | 2 + Lib/test/test_peg_generator/test_c_parser.py | 1 + Lib/test/test_platform.py | 1 + Lib/test/test_poll.py | 5 ++- Lib/test/test_poplib.py | 2 + Lib/test/test_pydoc.py | 3 +- Lib/test/test_robotparser.py | 3 ++ Lib/test/test_select.py | 2 + Lib/test/test_selectors.py | 4 ++ Lib/test/test_smtplib.py | 2 + Lib/test/test_socket.py | 2 + Lib/test/test_socketserver.py | 2 +- Lib/test/test_sys_settrace.py | 1 + Lib/test/test_telnetlib.py | 2 + Lib/test/test_tools/__init__.py | 4 ++ Lib/test/test_urllib2.py | 2 + Lib/test/test_urllib2_localnet.py | 3 ++ Lib/test/test_venv.py | 5 ++- Lib/test/test_wait3.py | 4 +- Lib/test/test_wait4.py | 4 +- Lib/test/test_xmlrpc.py | 2 + Lib/unittest/test/__init__.py | 5 ++- Lib/unittest/test/test_async_case.py | 2 + Lib/unittest/test/test_program.py | 1 + Lib/unittest/test/test_runner.py | 2 + Lib/unittest/test/testmock/testasync.py | 3 ++ Makefile.pre.in | 2 +- .../2022-03-19-10-25-04.bpo-40280.wBRSel.rst | 2 + configure | 41 +++++++++++++++++- configure.ac | 42 ++++++++++++++++++- setup.py | 5 +++ 52 files changed, 238 insertions(+), 23 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2022-03-19-10-25-04.bpo-40280.wBRSel.rst diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index d82629368df..f16c7ed952c 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -15,7 +15,7 @@ from test import support from test.support import threading_helper -requires_fork = unittest.skipUnless(hasattr(os, 'fork'), +requires_fork = unittest.skipUnless(support.has_fork_support, "platform doesn't support fork " "(no _at_fork_reinit method)") diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index d15a11c80b6..b00830c279e 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -6,6 +6,7 @@ import errno import re import sys import traceback +import unittest import warnings @@ -615,7 +616,7 @@ def collect_resource(info_add): def collect_test_socket(info_add): try: from test import test_socket - except ImportError: + except (ImportError, unittest.SkipTest): return # all check attributes like HAVE_SOCKET_CAN diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index fc1b86bebcd..c5666d66f47 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -42,6 +42,7 @@ __all__ = [ "requires_IEEE_754", "requires_zlib", "has_fork_support", "requires_fork", "has_subprocess_support", "requires_subprocess", + "has_socket_support", "requires_working_socket", "anticipate_failure", "load_package_tests", "detect_api_mismatch", "check__all__", "skip_if_buggy_ucrt_strfptime", "check_disallow_instantiation", "check_sanitizer", "skip_if_sanitizer", @@ -520,6 +521,21 @@ def requires_subprocess(): """Used for subprocess, os.spawn calls, fd inheritance""" return unittest.skipUnless(has_subprocess_support, "requires subprocess support") +# Emscripten's socket emulation has limitation. WASI doesn't have sockets yet. +has_socket_support = not is_emscripten and not is_wasi + +def requires_working_socket(*, module=False): + """Skip tests or modules that require working sockets + + Can be used as a function/class decorator or to skip an entire module. + """ + msg = "requires socket support" + if module: + if not has_socket_support: + raise unittest.SkipTest(msg) + else: + return unittest.skipUnless(has_socket_support, msg) + # Does strftime() support glibc extension like '%4Y'? has_strftime_extensions = False if sys.platform != "win32": diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 473bce484b4..fb22f411c2e 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -4,10 +4,12 @@ import unittest import contextlib from test.support.import_helper import import_module -from test.support import gc_collect +from test.support import gc_collect, requires_working_socket asyncio = import_module("asyncio") +requires_working_socket(module=True) + _no_default = object() diff --git a/Lib/test/test_asynchat.py b/Lib/test/test_asynchat.py index 973ac1f7d97..d28d67732e7 100644 --- a/Lib/test/test_asynchat.py +++ b/Lib/test/test_asynchat.py @@ -18,6 +18,8 @@ with warnings.catch_warnings(): import asynchat import asyncore +support.requires_working_socket(module=True) + HOST = socket_helper.HOST SERVER_QUIT = b'QUIT\n' diff --git a/Lib/test/test_asyncio/__init__.py b/Lib/test/test_asyncio/__init__.py index 5d415044d7d..ab0b5aa9489 100644 --- a/Lib/test/test_asyncio/__init__.py +++ b/Lib/test/test_asyncio/__init__.py @@ -1,7 +1,9 @@ import os +from test import support from test.support import load_package_tests from test.support import import_helper +support.requires_working_socket(module=True) # Skip tests if we don't have concurrent.futures. import_helper.import_module('concurrent.futures') diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index ecd1e120ecb..e3f31bd06d0 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -18,6 +18,8 @@ from io import BytesIO if support.PGO: raise unittest.SkipTest("test is not helpful for PGO") +support.requires_working_socket(module=True) + import warnings with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index c16c7ecd19a..462e05cc79a 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -8,6 +8,7 @@ import unittest from test.test_contextlib import TestBaseExitStack +support.requires_working_socket(module=True) def _async_test(func): """Decorator to turn an async function into a test case.""" diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 407a14c7ddf..8616aeddc1d 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -18,6 +18,11 @@ import shutil import types import contextlib + +if not support.has_subprocess_support: + raise unittest.SkipTest("test_CLI requires subprocess support.") + + # NOTE: There are some additional tests relating to interaction with # zipimport in the test_zipimport_support test module. @@ -455,7 +460,7 @@ We'll simulate a __file__ attr that ends in pyc: >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [] + [] The exact name depends on how test_doctest was invoked, so allow for leading path components. diff --git a/Lib/test/test_docxmlrpc.py b/Lib/test/test_docxmlrpc.py index 9a06be45855..89d80091850 100644 --- a/Lib/test/test_docxmlrpc.py +++ b/Lib/test/test_docxmlrpc.py @@ -4,6 +4,9 @@ import re import sys import threading import unittest +from test import support + +support.requires_working_socket(module=True) def make_request_and_skipIf(condition, reason): # If we skip the test, we have to make a request because diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 2f5cc06ca4b..2794fccfe7c 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -29,6 +29,7 @@ with warnings.catch_warnings(): import asyncore import asynchat +support.requires_working_socket(module=True) TIMEOUT = support.LOOPBACK_TIMEOUT DEFAULT_ENCODING = 'utf-8' diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 8265b8d1d6d..15dab0356f5 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -19,6 +19,7 @@ from test.support import os_helper from test.support import socket_helper from test.support import warnings_helper +support.requires_working_socket(module=True) here = os.path.dirname(__file__) # Self-signed cert file for 'localhost' diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 1cc020f6353..d20b45e8e02 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -33,6 +33,7 @@ from test import support from test.support import os_helper from test.support import threading_helper +support.requires_working_socket(module=True) class NoLogRequestHandler: def log_message(self, *args): diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 30b553746af..ff13edea2d1 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -11,7 +11,8 @@ import threading import socket from test.support import (verbose, - run_with_tz, run_with_locale, cpython_only) + run_with_tz, run_with_locale, cpython_only, + requires_working_socket) from test.support import hashlib_helper from test.support import threading_helper from test.support import warnings_helper @@ -23,6 +24,8 @@ try: except ImportError: ssl = None +support.requires_working_socket(module=True) + CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "keycert3.pem") CAFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "pycacert.pem") diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 8857fd5000b..7cca9f9d60a 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -20,7 +20,7 @@ from unittest import mock from test.support import os_helper from test.support import ( - STDLIB_DIR, is_jython, swap_attr, swap_item, cpython_only) + STDLIB_DIR, is_jython, swap_attr, swap_item, cpython_only, is_emscripten) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport) from test.support.os_helper import ( @@ -101,8 +101,17 @@ class ImportTests(unittest.TestCase): with self.assertRaises(ImportError) as cm: from _testcapi import i_dont_exist self.assertEqual(cm.exception.name, '_testcapi') - self.assertEqual(cm.exception.path, _testcapi.__file__) - self.assertRegex(str(cm.exception), r"cannot import name 'i_dont_exist' from '_testcapi' \(.*\.(so|pyd)\)") + if hasattr(_testcapi, "__file__"): + self.assertEqual(cm.exception.path, _testcapi.__file__) + self.assertRegex( + str(cm.exception), + r"cannot import name 'i_dont_exist' from '_testcapi' \(.*\.(so|pyd)\)" + ) + else: + self.assertEqual( + str(cm.exception), + "cannot import name 'i_dont_exist' from '_testcapi' (unknown location)" + ) def test_from_import_missing_attr_has_name(self): with self.assertRaises(ImportError) as cm: @@ -525,6 +534,7 @@ class FilePermissionTests(unittest.TestCase): @unittest.skipUnless(os.name == 'posix', "test meaningful only on posix systems") + @unittest.skipIf(is_emscripten, "Emscripten's umask is a stub.") def test_creation_mode(self): mask = 0o022 with temp_umask(mask), _ready_to_import() as (name, path): diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index 140f20657f7..b6663a44845 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -10,6 +10,10 @@ class FinderTests(abc.FinderTests): """Test the finder for extension modules.""" + def setUp(self): + if not self.machinery.EXTENSION_SUFFIXES: + raise unittest.SkipTest("Requires dynamic loading support.") + def find_spec(self, fullname): importer = self.machinery.FileFinder(util.EXTENSIONS.path, (self.machinery.ExtensionFileLoader, diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index e7a88a8f5e3..5080009bee3 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -12,11 +12,14 @@ import importlib.util import importlib from test.support.script_helper import assert_python_failure + class LoaderTests(abc.LoaderTests): """Test load_module() for extension modules.""" def setUp(self): + if not self.machinery.EXTENSION_SUFFIXES: + raise unittest.SkipTest("Requires dynamic loading support.") self.loader = self.machinery.ExtensionFileLoader(util.EXTENSIONS.name, util.EXTENSIONS.file_path) @@ -91,6 +94,8 @@ class MultiPhaseExtensionModuleTests(abc.LoaderTests): # Test loading extension modules with multi-phase initialization (PEP 489). def setUp(self): + if not self.machinery.EXTENSION_SUFFIXES: + raise unittest.SkipTest("Requires dynamic loading support.") self.name = '_testmultiphase' finder = self.machinery.FileFinder(None) self.spec = importlib.util.find_spec(self.name) diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 1d7fca6efb1..2b63810d539 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -10,6 +10,7 @@ from test.support import os_helper from test.support.script_helper import assert_python_ok +@support.requires_subprocess() class TestTool(unittest.TestCase): data = """ diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index be193dcdacf..5f72a6dd587 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -75,6 +75,7 @@ try: except ImportError: pass + class BaseTest(unittest.TestCase): """Base class for logging tests.""" @@ -626,6 +627,9 @@ class HandlerTest(BaseTest): os.unlink(fn) @unittest.skipIf(os.name == 'nt', 'WatchedFileHandler not appropriate for Windows.') + @unittest.skipIf( + support.is_emscripten, "Emscripten cannot fstat unlinked files." + ) def test_race(self): # Issue #14632 refers. def remove_loop(fname, tries): @@ -1058,6 +1062,7 @@ if hasattr(socket, "AF_UNIX"): # - end of server_helper section +@support.requires_working_socket() class SMTPHandlerTest(BaseTest): # bpo-14314, bpo-19665, bpo-34092: don't wait forever TIMEOUT = support.LONG_TIMEOUT @@ -1681,6 +1686,7 @@ class ConfigFileTest(BaseTest): os.unlink(fn) +@support.requires_working_socket() class SocketHandlerTest(BaseTest): """Test for SocketHandler objects.""" @@ -1795,6 +1801,7 @@ class UnixSocketHandlerTest(SocketHandlerTest): SocketHandlerTest.tearDown(self) os_helper.unlink(self.address) +@support.requires_working_socket() class DatagramHandlerTest(BaseTest): """Test for DatagramHandler.""" @@ -1876,6 +1883,7 @@ class UnixDatagramHandlerTest(DatagramHandlerTest): DatagramHandlerTest.tearDown(self) os_helper.unlink(self.address) +@support.requires_working_socket() class SysLogHandlerTest(BaseTest): """Test for SysLogHandler using UDP.""" @@ -1985,6 +1993,7 @@ class IPv6SysLogHandlerTest(SysLogHandlerTest): self.server_class.address_family = socket.AF_INET super(IPv6SysLogHandlerTest, self).tearDown() +@support.requires_working_socket() class HTTPHandlerTest(BaseTest): """Test for HTTPHandler.""" @@ -3261,6 +3270,7 @@ class ConfigDictTest(BaseTest): logging.config.stopListening() threading_helper.join_thread(t) + @support.requires_working_socket() def test_listen_config_10_ok(self): with support.captured_stdout() as output: self.setup_via_listener(json.dumps(self.config10)) @@ -3280,6 +3290,7 @@ class ConfigDictTest(BaseTest): ('ERROR', '4'), ], stream=output) + @support.requires_working_socket() def test_listen_config_1_ok(self): with support.captured_stdout() as output: self.setup_via_listener(textwrap.dedent(ConfigFileTest.config1)) @@ -3294,6 +3305,7 @@ class ConfigDictTest(BaseTest): # Original logger output is empty. self.assert_log_lines([]) + @support.requires_working_socket() def test_listen_verify(self): def verify_fail(stuff): diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 604fc4525f5..20c460e300c 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -1061,7 +1061,7 @@ class _TestMboxMMDF(_TestSingleFile): self.assertEqual(contents, f.read()) self._box = self._factory(self._path) - @unittest.skipUnless(hasattr(os, 'fork'), "Test needs fork().") + @support.requires_fork() @unittest.skipUnless(hasattr(socket, 'socketpair'), "Test needs socketpair().") def test_lock_conflict(self): # Fork off a child process that will lock the mailbox temporarily, diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 014171cbb49..213a44d56f3 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -1,4 +1,6 @@ -from test.support import (requires, _2G, _4G, gc_collect, cpython_only) +from test.support import ( + requires, _2G, _4G, gc_collect, cpython_only, is_emscripten +) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink import unittest @@ -21,6 +23,12 @@ def random_tagname(length=10): suffix = ''.join(random.choices(string.ascii_uppercase, k=length)) return f'{tagname_prefix}_{suffix}' +# Python's mmap module dup()s the file descriptor. Emscripten's FS layer +# does not materialize file changes through a dupped fd to a new mmap. +if is_emscripten: + raise unittest.SkipTest("incompatible with Emscripten's mmap emulation.") + + class MmapTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index d2bf3dc90ed..bfa2cc92d25 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -13,6 +13,7 @@ import linecache from contextlib import ExitStack, redirect_stdout from io import StringIO +from test import support from test.support import os_helper # This little helper class is essential for testing pdb under doctest. from test.test_doctest import _FakeInput @@ -1363,6 +1364,7 @@ def test_pdb_issue_43318(): """ +@support.requires_subprocess() class PdbTestCase(unittest.TestCase): def tearDown(self): os_helper.unlink(os_helper.TESTFN) diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index b761bd493f5..51a4f7d7c07 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -70,6 +70,7 @@ unittest.main() """ +@support.requires_subprocess() class TestCParser(unittest.TestCase): def setUp(self): self._backup_config_vars = dict(sysconfig._CONFIG_VARS) diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index d70ef155271..9b2cd201f3c 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -364,6 +364,7 @@ class PlatformTest(unittest.TestCase): # parent support.wait_process(pid, exitcode=0) + @unittest.skipIf(support.is_emscripten, "Does not apply to Emscripten") def test_libc_ver(self): # check that libc_ver(executable) doesn't raise an exception if os.path.isdir(sys.executable) and \ diff --git a/Lib/test/test_poll.py b/Lib/test/test_poll.py index ae3ffc77e99..7d542b5cfd7 100644 --- a/Lib/test/test_poll.py +++ b/Lib/test/test_poll.py @@ -7,7 +7,9 @@ import select import threading import time import unittest -from test.support import cpython_only, requires_subprocess +from test.support import ( + cpython_only, requires_subprocess, requires_working_socket +) from test.support import threading_helper from test.support.os_helper import TESTFN @@ -17,6 +19,7 @@ try: except AttributeError: raise unittest.SkipTest("select.poll not defined") +requires_working_socket(module=True) def find_ready_matching(ready, flag): match = [] diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 1220ca32ef8..57ccc541b81 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -22,6 +22,8 @@ with warnings.catch_warnings(): import asynchat import asyncore +test_support.requires_working_socket(module=True) + HOST = socket_helper.HOST PORT = 0 diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 7cedf76fb3a..e2dab120717 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -27,7 +27,7 @@ from test.support import os_helper from test.support.script_helper import assert_python_ok, assert_python_failure from test.support import threading_helper from test.support import (reap_children, captured_output, captured_stdout, - captured_stderr, requires_docstrings) + captured_stderr, is_emscripten, requires_docstrings) from test.support.os_helper import (TESTFN, rmtree, unlink) from test import pydoc_mod @@ -1339,6 +1339,7 @@ foo ) +@unittest.skipIf(is_emscripten, "Socket server not available on Emscripten.") class PydocServerTest(unittest.TestCase): """Tests for pydoc._start_server""" diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index b0bed431d4b..08bdf59d333 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -308,6 +308,9 @@ class RobotHandler(BaseHTTPRequestHandler): pass +@unittest.skipIf( + support.is_emscripten, "Socket server not available on Emscripten." +) class PasswordProtectedSiteTestCase(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_select.py b/Lib/test/test_select.py index 8b59321121e..ca2a9d9d24d 100644 --- a/Lib/test/test_select.py +++ b/Lib/test/test_select.py @@ -7,6 +7,8 @@ import textwrap import unittest from test import support +support.requires_working_socket(module=True) + @unittest.skipIf((sys.platform[:3]=='win'), "can't easily test on this system") class SelectTestCase(unittest.TestCase): diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py index fe6b725a4bd..c927331d438 100644 --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -19,6 +19,10 @@ except ImportError: resource = None +if support.is_emscripten: + raise unittest.SkipTest("Cannot create socketpair on Emscripten.") + + if hasattr(socket, 'socketpair'): socketpair = socket.socketpair else: diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 1a60fef8a42..e798645a5ca 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -29,6 +29,8 @@ with warnings.catch_warnings(): import asyncore import smtpd +support.requires_working_socket(module=True) + HOST = socket_helper.HOST if sys.platform == 'darwin': diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 53aa5e90fa2..97cc6260c3c 100755 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -37,6 +37,8 @@ try: except ImportError: fcntl = None +support.requires_working_socket(module=True) + HOST = socket_helper.HOST # test unicode string and carriage return MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 211321f3761..c498d3d12e2 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -28,7 +28,7 @@ HOST = socket_helper.HOST HAVE_UNIX_SOCKETS = hasattr(socket, "AF_UNIX") requires_unix_sockets = unittest.skipUnless(HAVE_UNIX_SOCKETS, 'requires Unix sockets') -HAVE_FORKING = hasattr(os, "fork") +HAVE_FORKING = test.support.has_fork_support requires_forking = unittest.skipUnless(HAVE_FORKING, 'requires forking') def signal_alarm(n): diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index fe6f7da0fd7..85d6bdf7221 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -8,6 +8,7 @@ import gc from functools import wraps import asyncio +support.requires_working_socket(module=True) class tracecontext: """Context manager that traces its enter and exit.""" diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py index 41c4fcd4195..b50df1459d1 100644 --- a/Lib/test/test_telnetlib.py +++ b/Lib/test/test_telnetlib.py @@ -8,6 +8,8 @@ from test import support from test.support import socket_helper import unittest +support.requires_working_socket(module=True) + HOST = socket_helper.HOST def server(evt, serv): diff --git a/Lib/test/test_tools/__init__.py b/Lib/test/test_tools/__init__.py index 34b0d3b8fb3..2102b9fceff 100644 --- a/Lib/test/test_tools/__init__.py +++ b/Lib/test/test_tools/__init__.py @@ -13,6 +13,10 @@ if support.check_sanitizer(address=True, memory=True): raise unittest.SkipTest("test too slow on ASAN/MSAN build") +if not support.has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") + + basepath = os.path.normpath( os.path.dirname( # os.path.dirname( # Lib diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index a4772a5b1d9..46a0ab1d83e 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -24,6 +24,8 @@ from urllib.parse import urlparse import urllib.error import http.client +support.requires_working_socket(module=True) + # XXX # Request # CacheFTPHandler (hard to write) diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index 1b2baf2f366..36314312ac0 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -8,6 +8,7 @@ import threading import unittest import hashlib +from test import support from test.support import hashlib_helper from test.support import threading_helper from test.support import warnings_helper @@ -17,6 +18,8 @@ try: except ImportError: ssl = None +support.requires_working_socket(module=True) + here = os.path.dirname(__file__) # Self-signed cert file for 'localhost' CERT_localhost = os.path.join(here, 'keycert.pem') diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index db812f21dbc..c91b7549384 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -16,7 +16,7 @@ import sys import tempfile from test.support import (captured_stdout, captured_stderr, requires_zlib, skip_if_broken_multiprocessing_synchronize, verbose, - requires_subprocess) + requires_subprocess, is_emscripten) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) import unittest import venv @@ -34,6 +34,9 @@ requireVenvCreate = unittest.skipUnless( or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') +if is_emscripten: + raise unittest.SkipTest("venv is not available on Emscripten.") + @requires_subprocess() def check_output(cmd, encoding=None): p = subprocess.Popen(cmd, diff --git a/Lib/test/test_wait3.py b/Lib/test/test_wait3.py index aa166baa400..4ec7690ac19 100644 --- a/Lib/test/test_wait3.py +++ b/Lib/test/test_wait3.py @@ -9,8 +9,8 @@ import unittest from test.fork_wait import ForkWait from test import support -if not hasattr(os, 'fork'): - raise unittest.SkipTest("os.fork not defined") +if not support.has_fork_support: + raise unittest.SkipTest("requires working os.fork()") if not hasattr(os, 'wait3'): raise unittest.SkipTest("os.wait3 not defined") diff --git a/Lib/test/test_wait4.py b/Lib/test/test_wait4.py index f8d5e13014f..24f1aaec60c 100644 --- a/Lib/test/test_wait4.py +++ b/Lib/test/test_wait4.py @@ -9,7 +9,9 @@ from test.fork_wait import ForkWait from test import support # If either of these do not exist, skip this test. -support.get_attribute(os, 'fork') +if not support.has_fork_support: + raise unittest.SkipTest("requires working os.fork()") + support.get_attribute(os, 'wait4') diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 1f06f5fdf48..6c4601ddadd 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -25,6 +25,8 @@ try: except ImportError: gzip = None +support.requires_working_socket(module=True) + alist = [{'astring': 'foo@bar.baz.spam', 'afloat': 7283.43, 'anint': 2**20, diff --git a/Lib/unittest/test/__init__.py b/Lib/unittest/test/__init__.py index cdae8a74427..143f4ab5a3d 100644 --- a/Lib/unittest/test/__init__.py +++ b/Lib/unittest/test/__init__.py @@ -11,7 +11,10 @@ def suite(): for fn in os.listdir(here): if fn.startswith("test") and fn.endswith(".py"): modname = "unittest.test." + fn[:-3] - __import__(modname) + try: + __import__(modname) + except unittest.SkipTest: + continue module = sys.modules[modname] suite.addTest(loader.loadTestsFromModule(module)) suite.addTest(loader.loadTestsFromName('unittest.test.testmock')) diff --git a/Lib/unittest/test/test_async_case.py b/Lib/unittest/test/test_async_case.py index 7dc8a6bffa0..a48140829cc 100644 --- a/Lib/unittest/test/test_async_case.py +++ b/Lib/unittest/test/test_async_case.py @@ -3,6 +3,8 @@ import contextvars import unittest from test import support +support.requires_working_socket(module=True) + class MyException(Exception): pass diff --git a/Lib/unittest/test/test_program.py b/Lib/unittest/test/test_program.py index fa9e6291caa..126497aa566 100644 --- a/Lib/unittest/test/test_program.py +++ b/Lib/unittest/test/test_program.py @@ -196,6 +196,7 @@ class FakeRunner(object): return RESULT +@support.requires_subprocess() class TestCommandLineArgs(unittest.TestCase): def setUp(self): diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py index c487e9ec4ef..18062ae5a58 100644 --- a/Lib/unittest/test/test_runner.py +++ b/Lib/unittest/test/test_runner.py @@ -3,6 +3,7 @@ import os import sys import pickle import subprocess +from test import support import unittest from unittest.case import _Outcome @@ -1139,6 +1140,7 @@ class Test_TextTestRunner(unittest.TestCase): expectedresult = (runner.stream, DESCRIPTIONS, VERBOSITY) self.assertEqual(runner._makeResult(), expectedresult) + @support.requires_subprocess() def test_warnings(self): """ Check that warnings argument of TextTestRunner correctly affects the diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/unittest/test/testmock/testasync.py index 122e6956635..1bab671acde 100644 --- a/Lib/unittest/test/testmock/testasync.py +++ b/Lib/unittest/test/testmock/testasync.py @@ -4,6 +4,9 @@ import inspect import re import unittest from contextlib import contextmanager +from test import support + +support.requires_working_socket(module=True) from asyncio import run, iscoroutinefunction from unittest import IsolatedAsyncioTestCase diff --git a/Makefile.pre.in b/Makefile.pre.in index c4034dc248c..4b9aab0a284 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2352,7 +2352,7 @@ clean-retain-profile: pycremoval -rm -f pybuilddir.txt -rm -f Lib/lib2to3/*Grammar*.pickle -rm -f _bootstrap_python - -rm -f python.html python*.js python.data + -rm -f python.html python*.js python.data python*.symbols python*.map -rm -rf $(WASM_STDLIB) -rm -f Programs/_testembed Programs/_freeze_module -rm -f Python/deepfreeze/*.[co] diff --git a/Misc/NEWS.d/next/Tests/2022-03-19-10-25-04.bpo-40280.wBRSel.rst b/Misc/NEWS.d/next/Tests/2022-03-19-10-25-04.bpo-40280.wBRSel.rst new file mode 100644 index 00000000000..2572c27a821 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-03-19-10-25-04.bpo-40280.wBRSel.rst @@ -0,0 +1,2 @@ +The test suite is now passing on the Emscripten platform. All fork, socket, +and subprocess-based tests are skipped. diff --git a/configure b/configure index 5fa6efaab4f..5ebcadd6423 100755 --- a/configure +++ b/configure @@ -7396,6 +7396,13 @@ $as_echo "$as_me: llvm-ar found via xcrun: ${LLVM_AR}" >&6;} ;; esac ;; + *emcc*) + if test "$Py_LTO_POLICY" != "default"; then + as_fn_error $? "emcc supports only default lto." "$LINENO" 5 + fi + LTOFLAGS="-flto" + LTOCFLAGS="-flto" + ;; *gcc*) if test $Py_LTO_POLICY = thin then @@ -7717,17 +7724,34 @@ then fi # WASM flags +# TODO: Add -s MAIN_MODULE=2 for dlopen() support. +# The option disables code elimination, which increases code size of main +# binary. All objects must be built with -fPIC. case $ac_sys_system/$ac_sys_emscripten_target in #( Emscripten/browser) : - LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 --preload-file \$(WASM_ASSETS_DIR)" + LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1" + LINKFORSHARED="--preload-file \$(WASM_ASSETS_DIR)" WASM_ASSETS_DIR=".\$(prefix)" WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py" + if test "$Py_DEBUG" = 'true'; then + LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1" + LINKFORSHARED="$LINKFORSHARED -gsource-map --emit-symbol-map" + else + LINKFORSHARED="$LINKFORSHARED -O2 -g0" + fi ;; #( Emscripten/node) : - LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 -s NODERAWFS=1 -s EXIT_RUNTIME=1 -s USE_PTHREADS -s PROXY_TO_PTHREAD" + LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1 -s NODERAWFS=1 -s USE_PTHREADS=1" + LINKFORSHARED="-s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1" CFLAGS_NODIST="$CFLAGS_NODIST -pthread" + if test "$Py_DEBUG" = 'true'; then + LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1" + LINKFORSHARED="$LINKFORSHARED -gseparate-dwarf --emit-symbol-map" + else + LINKFORSHARED="$LINKFORSHARED -O2 -gseparate-dwarf" + fi ;; #( WASI/*) : @@ -10403,6 +10427,10 @@ then Linux*|GNU*|QNX*|VxWorks*|Haiku*) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; + Emscripten*) + LDSHARED='$(CC) -shared -s SIDE_MODULE=1' + LDCXXSHARED='$(CXX) -shared -s SIDE_MODULE=1' + ;; FreeBSD*) if [ "`$CC -dM -E - &6; } fi +if test "$have_zlib" = "yes" -a "$ac_sys_system" = "Emscripten" -a "$ZLIB_LIBS" = "-lz"; then + ZLIB_LIBS="-s USE_ZLIB=1" +fi + if test "x$have_zlib" = xyes; then : BINASCII_CFLAGS="-DUSE_ZLIB_CRC32 $ZLIB_CFLAGS" @@ -15292,6 +15324,11 @@ $as_echo "yes" >&6; } have_bzip2=yes fi +if test "$have_bzip2" = "yes" -a "$ac_sys_system" = "Emscripten" -a "$BZIP2_LIBS" = "-lbz2"; then + BZIP2_LIBS="-s USE_BZIP2=1" +fi + + pkg_failed=no { $as_echo "$as_me:${as_lineno-$LINENO}: checking for LIBLZMA" >&5 diff --git a/configure.ac b/configure.ac index 7a37ad279d0..4afc898e81d 100644 --- a/configure.ac +++ b/configure.ac @@ -1659,6 +1659,13 @@ if test "$Py_LTO" = 'true' ; then ;; esac ;; + *emcc*) + if test "$Py_LTO_POLICY" != "default"; then + AC_MSG_ERROR([emcc supports only default lto.]) + fi + LTOFLAGS="-flto" + LTOCFLAGS="-flto" + ;; *gcc*) if test $Py_LTO_POLICY = thin then @@ -1861,15 +1868,33 @@ then fi # WASM flags +# TODO: Add -s MAIN_MODULE=2 for dlopen() support. +# The option disables code elimination, which increases code size of main +# binary. All objects must be built with -fPIC. AS_CASE([$ac_sys_system/$ac_sys_emscripten_target], [Emscripten/browser], [ - LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 --preload-file \$(WASM_ASSETS_DIR)" + LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1" + LINKFORSHARED="--preload-file \$(WASM_ASSETS_DIR)" WASM_ASSETS_DIR=".\$(prefix)" WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py" + dnl separate-dwarf does not seem to work in Chrome DevTools Support. + if test "$Py_DEBUG" = 'true'; then + LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1" + LINKFORSHARED="$LINKFORSHARED -gsource-map --emit-symbol-map" + else + LINKFORSHARED="$LINKFORSHARED -O2 -g0" + fi ], [Emscripten/node], [ - LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 -s NODERAWFS=1 -s EXIT_RUNTIME=1 -s USE_PTHREADS -s PROXY_TO_PTHREAD" + LDFLAGS_NODIST="$LDFLAGS_NODIST -s ALLOW_MEMORY_GROWTH=1 -s NODERAWFS=1 -s USE_PTHREADS=1" + LINKFORSHARED="-s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1" CFLAGS_NODIST="$CFLAGS_NODIST -pthread" + if test "$Py_DEBUG" = 'true'; then + LDFLAGS_NODIST="$LDFLAGS_NODIST -s ASSERTIONS=1" + LINKFORSHARED="$LINKFORSHARED -gseparate-dwarf --emit-symbol-map" + else + LINKFORSHARED="$LINKFORSHARED -O2 -gseparate-dwarf" + fi ], [WASI/*], [ AC_DEFINE([_WASI_EMULATED_SIGNAL], [1], [Define to 1 if you want to emulate signals on WASI]) @@ -2880,6 +2905,10 @@ then Linux*|GNU*|QNX*|VxWorks*|Haiku*) LDSHARED='$(CC) -shared' LDCXXSHARED='$(CXX) -shared';; + Emscripten*) + LDSHARED='$(CC) -shared -s SIDE_MODULE=1' + LDCXXSHARED='$(CXX) -shared -s SIDE_MODULE=1' + ;; FreeBSD*) if [[ "`$CC -dM -E - = 1.2.0], [ ], [have_zlib=no]) ]) +if test "$have_zlib" = "yes" -a "$ac_sys_system" = "Emscripten" -a "$ZLIB_LIBS" = "-lz"; then + ZLIB_LIBS="-s USE_ZLIB=1" +fi + dnl binascii can use zlib for optimized crc32. AS_VAR_IF([have_zlib], [yes], [ BINASCII_CFLAGS="-DUSE_ZLIB_CRC32 $ZLIB_CFLAGS" @@ -4372,6 +4405,11 @@ PKG_CHECK_MODULES([BZIP2], [bzip2], [have_bzip2=yes], [ ], [have_bzip2=no]) ]) +if test "$have_bzip2" = "yes" -a "$ac_sys_system" = "Emscripten" -a "$BZIP2_LIBS" = "-lbz2"; then + BZIP2_LIBS="-s USE_BZIP2=1" +fi + + PKG_CHECK_MODULES([LIBLZMA], [liblzma], [have_liblzma=yes], [ AC_CHECK_HEADERS([lzma.h], [ WITH_SAVE_ENV([ diff --git a/setup.py b/setup.py index c3cf2417bc4..070ae9822bd 100644 --- a/setup.py +++ b/setup.py @@ -84,10 +84,15 @@ CYGWIN = (HOST_PLATFORM == 'cygwin') MACOS = (HOST_PLATFORM == 'darwin') AIX = (HOST_PLATFORM.startswith('aix')) VXWORKS = ('vxworks' in HOST_PLATFORM) +EMSCRIPTEN = HOST_PLATFORM == 'emscripten-wasm32' CC = os.environ.get("CC") if not CC: CC = sysconfig.get_config_var("CC") +if EMSCRIPTEN: + # emcc is a Python script from a different Python interpreter. + os.environ.pop("PYTHONPATH", None) + SUMMARY = """ Python is an interpreted, interactive, object-oriented programming