mirrors
/
cpython
mirror of https://github.com/python/cpython.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

gh-76785: Add More Tests to test_interpreters.test_api (gh-117662) 

In addition to the increase test coverage, this is a precursor to sorting out how we handle interpreters created directly via the C-API.
This commit is contained in:
Eric Snow 2024-04-10 18:37:01 -06:00 committed by GitHub
parent 0cc71bde00
commit 993c3cca16
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 2015 additions and 421 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Include/internal/pycore_crossinterp.h
View File

@ -217,6 +217,11 @@ typedef struct _excinfo {
const char *errdisplay; const char *errdisplay;
} _PyXI_excinfo; } _PyXI_excinfo;
PyAPI_FUNC(int) _PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc);
PyAPI_FUNC(PyObject *) _PyXI_FormatExcInfo(_PyXI_excinfo *info);
PyAPI_FUNC(PyObject *) _PyXI_ExcInfoAsObject(_PyXI_excinfo *info);
PyAPI_FUNC(void) _PyXI_ClearExcInfo(_PyXI_excinfo *info);
typedef enum error_code { typedef enum error_code {
_PyXI_ERR_NO_ERROR = 0, _PyXI_ERR_NO_ERROR = 0,
@ -313,6 +318,21 @@ PyAPI_FUNC(PyObject *) _PyXI_ApplyCapturedException(_PyXI_session *session);
PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session); PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session);
/*************/
/* other API */
/*************/
// Export for _testinternalcapi shared extension
PyAPI_FUNC(PyInterpreterState *) _PyXI_NewInterpreter(
PyInterpreterConfig *config,
PyThreadState **p_tstate,
PyThreadState **p_save_tstate);
PyAPI_FUNC(void) _PyXI_EndInterpreter(
PyInterpreterState *interp,
PyThreadState *tstate,
PyThreadState **p_save_tstate);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

16
Include/internal/pycore_interp.h
View File

@ -103,11 +103,22 @@ struct _is {
int requires_idref; int requires_idref;
PyThread_type_lock id_mutex; PyThread_type_lock id_mutex;
#define _PyInterpreterState_WHENCE_NOTSET -1
#define _PyInterpreterState_WHENCE_UNKNOWN 0
#define _PyInterpreterState_WHENCE_RUNTIME 1
#define _PyInterpreterState_WHENCE_LEGACY_CAPI 2
#define _PyInterpreterState_WHENCE_CAPI 3
#define _PyInterpreterState_WHENCE_XI 4
#define _PyInterpreterState_WHENCE_MAX 4
long _whence;
/* Has been initialized to a safe state. /* Has been initialized to a safe state.
In order to be effective, this must be set to 0 during or right In order to be effective, this must be set to 0 during or right
after allocation. */ after allocation. */
int _initialized; int _initialized;
/* Has been fully initialized via pylifecycle.c. */
int _ready;
int finalizing; int finalizing;
uintptr_t last_restart_version; uintptr_t last_restart_version;
@ -305,6 +316,11 @@ PyAPI_FUNC(int) _PyInterpreterState_IDInitref(PyInterpreterState *);
PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_IDIncref(PyInterpreterState *);
PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(PyInterpreterState *);
PyAPI_FUNC(long) _PyInterpreterState_GetWhence(PyInterpreterState *interp);
extern void _PyInterpreterState_SetWhence(
PyInterpreterState *interp,
long whence);
extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp); extern const PyConfig* _PyInterpreterState_GetConfig(PyInterpreterState *interp);
// Get a copy of the current interpreter configuration. // Get a copy of the current interpreter configuration.

3
Include/internal/pycore_pystate.h
View File

@ -77,6 +77,9 @@ _Py_IsMainInterpreterFinalizing(PyInterpreterState *interp)
interp == &_PyRuntime._main_interpreter); interp == &_PyRuntime._main_interpreter);
} }
// Export for _xxsubinterpreters module.
PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *);
// Export for _xxsubinterpreters module. // Export for _xxsubinterpreters module.
PyAPI_FUNC(int) _PyInterpreterState_SetRunningMain(PyInterpreterState *); PyAPI_FUNC(int) _PyInterpreterState_SetRunningMain(PyInterpreterState *);
PyAPI_FUNC(void) _PyInterpreterState_SetNotRunningMain(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_SetNotRunningMain(PyInterpreterState *);

1
Include/internal/pycore_runtime_init.h
View File

@ -162,6 +162,7 @@ extern PyTypeObject _PyExc_MemoryError;
#define _PyInterpreterState_INIT(INTERP) \ #define _PyInterpreterState_INIT(INTERP) \
{ \ { \
.id_refcount = -1, \ .id_refcount = -1, \
._whence = _PyInterpreterState_WHENCE_NOTSET, \
.imports = IMPORTS_INIT, \ .imports = IMPORTS_INIT, \
.ceval = { \ .ceval = { \
.recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \

7
Lib/test/support/interpreters/__init__.py
View File

@ -79,18 +79,19 @@ def create():
def list_all(): def list_all():
"""Return all existing interpreters.""" """Return all existing interpreters."""
return [Interpreter(id) for id in _interpreters.list_all()] return [Interpreter(id)
for id, in _interpreters.list_all()]
def get_current(): def get_current():
"""Return the currently running interpreter.""" """Return the currently running interpreter."""
id = _interpreters.get_current() id, = _interpreters.get_current()
return Interpreter(id) return Interpreter(id)
def get_main(): def get_main():
"""Return the main interpreter.""" """Return the main interpreter."""
id = _interpreters.get_main() id, = _interpreters.get_main()
return Interpreter(id) return Interpreter(id)

111
Lib/test/test__xxinterpchannels.py
View File

@ -9,7 +9,7 @@
from test.support import import_helper from test.support import import_helper
from test.test__xxsubinterpreters import ( from test.test__xxsubinterpreters import (
interpreters, _interpreters,
_run_output, _run_output,
clean_up_interpreters, clean_up_interpreters,
) )
@ -49,14 +49,15 @@ def run_interp(id, source, **shared):
def _run_interp(id, source, shared, _mainns={}): def _run_interp(id, source, shared, _mainns={}):
source = dedent(source) source = dedent(source)
main = interpreters.get_main() main, *_ = _interpreters.get_main()
if main == id: if main == id:
if interpreters.get_current() != main: cur, *_ = _interpreters.get_current()
if cur != main:
raise RuntimeError raise RuntimeError
# XXX Run a func? # XXX Run a func?
exec(source, _mainns) exec(source, _mainns)
else: else:
interpreters.run_string(id, source, shared) _interpreters.run_string(id, source, shared)
class Interpreter(namedtuple('Interpreter', 'name id')): class Interpreter(namedtuple('Interpreter', 'name id')):
@ -71,7 +72,7 @@ def from_raw(cls, raw):
raise NotImplementedError raise NotImplementedError
def __new__(cls, name=None, id=None): def __new__(cls, name=None, id=None):
main = interpreters.get_main() main, *_ = _interpreters.get_main()
if id == main: if id == main:
if not name: if not name:
name = 'main' name = 'main'
@ -89,7 +90,7 @@ def __new__(cls, name=None, id=None):
name = 'main' name = 'main'
id = main id = main
else: else:
id = interpreters.create() id = _interpreters.create()
self = super().__new__(cls, name, id) self = super().__new__(cls, name, id)
return self return self
@ -370,7 +371,7 @@ def test_sequential_ids(self):
self.assertEqual(set(after) - set(before), {id1, id2, id3}) self.assertEqual(set(after) - set(before), {id1, id2, id3})
def test_ids_global(self): def test_ids_global(self):
id1 = interpreters.create() id1 = _interpreters.create()
out = _run_output(id1, dedent(""" out = _run_output(id1, dedent("""
import _xxinterpchannels as _channels import _xxinterpchannels as _channels
cid = _channels.create() cid = _channels.create()
@ -378,7 +379,7 @@ def test_ids_global(self):
""")) """))
cid1 = int(out.strip()) cid1 = int(out.strip())
id2 = interpreters.create() id2 = _interpreters.create()
out = _run_output(id2, dedent(""" out = _run_output(id2, dedent("""
import _xxinterpchannels as _channels import _xxinterpchannels as _channels
cid = _channels.create() cid = _channels.create()
@ -390,7 +391,7 @@ def test_ids_global(self):
def test_channel_list_interpreters_none(self): def test_channel_list_interpreters_none(self):
"""Test listing interpreters for a channel with no associations.""" """Test listing interpreters for a channel with no associations."""
# Test for channel with no associated interpreters. # Test for channel with no associated _interpreters.
cid = channels.create() cid = channels.create()
send_interps = channels.list_interpreters(cid, send=True) send_interps = channels.list_interpreters(cid, send=True)
recv_interps = channels.list_interpreters(cid, send=False) recv_interps = channels.list_interpreters(cid, send=False)
@ -398,8 +399,8 @@ def test_channel_list_interpreters_none(self):
self.assertEqual(recv_interps, []) self.assertEqual(recv_interps, [])
def test_channel_list_interpreters_basic(self): def test_channel_list_interpreters_basic(self):
"""Test basic listing channel interpreters.""" """Test basic listing channel _interpreters."""
interp0 = interpreters.get_main() interp0, *_ = _interpreters.get_main()
cid = channels.create() cid = channels.create()
channels.send(cid, "send", blocking=False) channels.send(cid, "send", blocking=False)
# Test for a channel that has one end associated to an interpreter. # Test for a channel that has one end associated to an interpreter.
@ -408,7 +409,7 @@ def test_channel_list_interpreters_basic(self):
self.assertEqual(send_interps, [interp0]) self.assertEqual(send_interps, [interp0])
self.assertEqual(recv_interps, []) self.assertEqual(recv_interps, [])
interp1 = interpreters.create() interp1 = _interpreters.create()
_run_output(interp1, dedent(f""" _run_output(interp1, dedent(f"""
import _xxinterpchannels as _channels import _xxinterpchannels as _channels
obj = _channels.recv({cid}) obj = _channels.recv({cid})
@ -421,10 +422,10 @@ def test_channel_list_interpreters_basic(self):
def test_channel_list_interpreters_multiple(self): def test_channel_list_interpreters_multiple(self):
"""Test listing interpreters for a channel with many associations.""" """Test listing interpreters for a channel with many associations."""
interp0 = interpreters.get_main() interp0, *_ = _interpreters.get_main()
interp1 = interpreters.create() interp1 = _interpreters.create()
interp2 = interpreters.create() interp2 = _interpreters.create()
interp3 = interpreters.create() interp3 = _interpreters.create()
cid = channels.create() cid = channels.create()
channels.send(cid, "send", blocking=False) channels.send(cid, "send", blocking=False)
@ -447,8 +448,8 @@ def test_channel_list_interpreters_multiple(self):
def test_channel_list_interpreters_destroyed(self): def test_channel_list_interpreters_destroyed(self):
"""Test listing channel interpreters with a destroyed interpreter.""" """Test listing channel interpreters with a destroyed interpreter."""
interp0 = interpreters.get_main() interp0, *_ = _interpreters.get_main()
interp1 = interpreters.create() interp1 = _interpreters.create()
cid = channels.create() cid = channels.create()
channels.send(cid, "send", blocking=False) channels.send(cid, "send", blocking=False)
_run_output(interp1, dedent(f""" _run_output(interp1, dedent(f"""
@ -461,7 +462,7 @@ def test_channel_list_interpreters_destroyed(self):
self.assertEqual(send_interps, [interp0]) self.assertEqual(send_interps, [interp0])
self.assertEqual(recv_interps, [interp1]) self.assertEqual(recv_interps, [interp1])
interpreters.destroy(interp1) _interpreters.destroy(interp1)
# Destroyed interpreter should not be listed. # Destroyed interpreter should not be listed.
send_interps = channels.list_interpreters(cid, send=True) send_interps = channels.list_interpreters(cid, send=True)
recv_interps = channels.list_interpreters(cid, send=False) recv_interps = channels.list_interpreters(cid, send=False)
@ -472,9 +473,9 @@ def test_channel_list_interpreters_released(self):
"""Test listing channel interpreters with a released channel.""" """Test listing channel interpreters with a released channel."""
# Set up one channel with main interpreter on the send end and two # Set up one channel with main interpreter on the send end and two
# subinterpreters on the receive end. # subinterpreters on the receive end.
interp0 = interpreters.get_main() interp0, *_ = _interpreters.get_main()
interp1 = interpreters.create() interp1 = _interpreters.create()
interp2 = interpreters.create() interp2 = _interpreters.create()
cid = channels.create() cid = channels.create()
channels.send(cid, "data", blocking=False) channels.send(cid, "data", blocking=False)
_run_output(interp1, dedent(f""" _run_output(interp1, dedent(f"""
@ -494,7 +495,7 @@ def test_channel_list_interpreters_released(self):
# Release the main interpreter from the send end. # Release the main interpreter from the send end.
channels.release(cid, send=True) channels.release(cid, send=True)
# Send end should have no associated interpreters. # Send end should have no associated _interpreters.
send_interps = channels.list_interpreters(cid, send=True) send_interps = channels.list_interpreters(cid, send=True)
recv_interps = channels.list_interpreters(cid, send=False) recv_interps = channels.list_interpreters(cid, send=False)
self.assertEqual(len(send_interps), 0) self.assertEqual(len(send_interps), 0)
@ -513,8 +514,8 @@ def test_channel_list_interpreters_released(self):
def test_channel_list_interpreters_closed(self): def test_channel_list_interpreters_closed(self):
"""Test listing channel interpreters with a closed channel.""" """Test listing channel interpreters with a closed channel."""
interp0 = interpreters.get_main() interp0, *_ = _interpreters.get_main()
interp1 = interpreters.create() interp1 = _interpreters.create()
cid = channels.create() cid = channels.create()
# Put something in the channel so that it's not empty. # Put something in the channel so that it's not empty.
channels.send(cid, "send", blocking=False) channels.send(cid, "send", blocking=False)
@ -535,8 +536,8 @@ def test_channel_list_interpreters_closed(self):
def test_channel_list_interpreters_closed_send_end(self): def test_channel_list_interpreters_closed_send_end(self):
"""Test listing channel interpreters with a channel's send end closed.""" """Test listing channel interpreters with a channel's send end closed."""
interp0 = interpreters.get_main() interp0, *_ = _interpreters.get_main()
interp1 = interpreters.create() interp1 = _interpreters.create()
cid = channels.create() cid = channels.create()
# Put something in the channel so that it's not empty. # Put something in the channel so that it's not empty.
channels.send(cid, "send", blocking=False) channels.send(cid, "send", blocking=False)
@ -589,9 +590,9 @@ def test_allowed_types(self):
def test_run_string_arg_unresolved(self): def test_run_string_arg_unresolved(self):
cid = channels.create() cid = channels.create()
interp = interpreters.create() interp = _interpreters.create()
interpreters.set___main___attrs(interp, dict(cid=cid.send)) _interpreters.set___main___attrs(interp, dict(cid=cid.send))
out = _run_output(interp, dedent(""" out = _run_output(interp, dedent("""
import _xxinterpchannels as _channels import _xxinterpchannels as _channels
print(cid.end) print(cid.end)
@ -609,7 +610,7 @@ def test_run_string_arg_unresolved(self):
def test_run_string_arg_resolved(self): def test_run_string_arg_resolved(self):
cid = channels.create() cid = channels.create()
cid = channels._channel_id(cid, _resolve=True) cid = channels._channel_id(cid, _resolve=True)
interp = interpreters.create() interp = _interpreters.create()
out = _run_output(interp, dedent(""" out = _run_output(interp, dedent("""
import _xxinterpchannels as _channels import _xxinterpchannels as _channels
@ -635,7 +636,7 @@ def test_send_recv_main(self):
self.assertIsNot(obj, orig) self.assertIsNot(obj, orig)
def test_send_recv_same_interpreter(self): def test_send_recv_same_interpreter(self):
id1 = interpreters.create() id1 = _interpreters.create()
out = _run_output(id1, dedent(""" out = _run_output(id1, dedent("""
import _xxinterpchannels as _channels import _xxinterpchannels as _channels
cid = _channels.create() cid = _channels.create()
@ -648,7 +649,7 @@ def test_send_recv_same_interpreter(self):
def test_send_recv_different_interpreters(self): def test_send_recv_different_interpreters(self):
cid = channels.create() cid = channels.create()
id1 = interpreters.create() id1 = _interpreters.create()
out = _run_output(id1, dedent(f""" out = _run_output(id1, dedent(f"""
import _xxinterpchannels as _channels import _xxinterpchannels as _channels
_channels.send({cid}, b'spam', blocking=False) _channels.send({cid}, b'spam', blocking=False)
@ -674,7 +675,7 @@ def f():
def test_send_recv_different_interpreters_and_threads(self): def test_send_recv_different_interpreters_and_threads(self):
cid = channels.create() cid = channels.create()
id1 = interpreters.create() id1 = _interpreters.create()
out = None out = None
def f(): def f():
@ -737,12 +738,12 @@ def test_recv_default(self):
def test_recv_sending_interp_destroyed(self): def test_recv_sending_interp_destroyed(self):
with self.subTest('closed'): with self.subTest('closed'):
cid1 = channels.create() cid1 = channels.create()
interp = interpreters.create() interp = _interpreters.create()
interpreters.run_string(interp, dedent(f""" _interpreters.run_string(interp, dedent(f"""
import _xxinterpchannels as _channels import _xxinterpchannels as _channels
_channels.send({cid1}, b'spam', blocking=False) _channels.send({cid1}, b'spam', blocking=False)
""")) """))
interpreters.destroy(interp) _interpreters.destroy(interp)
with self.assertRaisesRegex(RuntimeError, with self.assertRaisesRegex(RuntimeError,
f'channel {cid1} is closed'): f'channel {cid1} is closed'):
@ -750,13 +751,13 @@ def test_recv_sending_interp_destroyed(self):
del cid1 del cid1
with self.subTest('still open'): with self.subTest('still open'):
cid2 = channels.create() cid2 = channels.create()
interp = interpreters.create() interp = _interpreters.create()
interpreters.run_string(interp, dedent(f""" _interpreters.run_string(interp, dedent(f"""
import _xxinterpchannels as _channels import _xxinterpchannels as _channels
_channels.send({cid2}, b'spam', blocking=False) _channels.send({cid2}, b'spam', blocking=False)
""")) """))
channels.send(cid2, b'eggs', blocking=False) channels.send(cid2, b'eggs', blocking=False)
interpreters.destroy(interp) _interpreters.destroy(interp)
channels.recv(cid2) channels.recv(cid2)
with self.assertRaisesRegex(RuntimeError, with self.assertRaisesRegex(RuntimeError,
@ -1010,24 +1011,24 @@ def test_close_single_user(self):
def test_close_multiple_users(self): def test_close_multiple_users(self):
cid = channels.create() cid = channels.create()
id1 = interpreters.create() id1 = _interpreters.create()
id2 = interpreters.create() id2 = _interpreters.create()
interpreters.run_string(id1, dedent(f""" _interpreters.run_string(id1, dedent(f"""
import _xxinterpchannels as _channels import _xxinterpchannels as _channels
_channels.send({cid}, b'spam', blocking=False) _channels.send({cid}, b'spam', blocking=False)
""")) """))
interpreters.run_string(id2, dedent(f""" _interpreters.run_string(id2, dedent(f"""
import _xxinterpchannels as _channels import _xxinterpchannels as _channels
_channels.recv({cid}) _channels.recv({cid})
""")) """))
channels.close(cid) channels.close(cid)
excsnap = interpreters.run_string(id1, dedent(f""" excsnap = _interpreters.run_string(id1, dedent(f"""
_channels.send({cid}, b'spam') _channels.send({cid}, b'spam')
""")) """))
self.assertEqual(excsnap.type.__name__, 'ChannelClosedError') self.assertEqual(excsnap.type.__name__, 'ChannelClosedError')
excsnap = interpreters.run_string(id2, dedent(f""" excsnap = _interpreters.run_string(id2, dedent(f"""
_channels.send({cid}, b'spam') _channels.send({cid}, b'spam')
""")) """))
self.assertEqual(excsnap.type.__name__, 'ChannelClosedError') self.assertEqual(excsnap.type.__name__, 'ChannelClosedError')
@ -1154,8 +1155,8 @@ def test_close_never_used(self):
def test_close_by_unassociated_interp(self): def test_close_by_unassociated_interp(self):
cid = channels.create() cid = channels.create()
channels.send(cid, b'spam', blocking=False) channels.send(cid, b'spam', blocking=False)
interp = interpreters.create() interp = _interpreters.create()
interpreters.run_string(interp, dedent(f""" _interpreters.run_string(interp, dedent(f"""
import _xxinterpchannels as _channels import _xxinterpchannels as _channels
_channels.close({cid}, force=True) _channels.close({cid}, force=True)
""")) """))
@ -1251,9 +1252,9 @@ def test_single_user(self):
def test_multiple_users(self): def test_multiple_users(self):
cid = channels.create() cid = channels.create()
id1 = interpreters.create() id1 = _interpreters.create()
id2 = interpreters.create() id2 = _interpreters.create()
interpreters.run_string(id1, dedent(f""" _interpreters.run_string(id1, dedent(f"""
import _xxinterpchannels as _channels import _xxinterpchannels as _channels
_channels.send({cid}, b'spam', blocking=False) _channels.send({cid}, b'spam', blocking=False)
""")) """))
@ -1263,7 +1264,7 @@ def test_multiple_users(self):
_channels.release({cid}) _channels.release({cid})
print(repr(obj)) print(repr(obj))
""")) """))
interpreters.run_string(id1, dedent(f""" _interpreters.run_string(id1, dedent(f"""
_channels.release({cid}) _channels.release({cid})
""")) """))
@ -1310,8 +1311,8 @@ def test_never_used(self):
def test_by_unassociated_interp(self): def test_by_unassociated_interp(self):
cid = channels.create() cid = channels.create()
channels.send(cid, b'spam', blocking=False) channels.send(cid, b'spam', blocking=False)
interp = interpreters.create() interp = _interpreters.create()
interpreters.run_string(interp, dedent(f""" _interpreters.run_string(interp, dedent(f"""
import _xxinterpchannels as _channels import _xxinterpchannels as _channels
_channels.release({cid}) _channels.release({cid})
""")) """))
@ -1325,8 +1326,8 @@ def test_by_unassociated_interp(self):
def test_close_if_unassociated(self): def test_close_if_unassociated(self):
# XXX Something's not right with this test... # XXX Something's not right with this test...
cid = channels.create() cid = channels.create()
interp = interpreters.create() interp = _interpreters.create()
interpreters.run_string(interp, dedent(f""" _interpreters.run_string(interp, dedent(f"""
import _xxinterpchannels as _channels import _xxinterpchannels as _channels
obj = _channels.send({cid}, b'spam', blocking=False) obj = _channels.send({cid}, b'spam', blocking=False)
_channels.release({cid}) _channels.release({cid})

338
Lib/test/test__xxsubinterpreters.py
View File

@ -13,7 +13,7 @@
from test.support import script_helper from test.support import script_helper
interpreters = import_helper.import_module('_xxsubinterpreters') _interpreters = import_helper.import_module('_xxsubinterpreters')
_testinternalcapi = import_helper.import_module('_testinternalcapi') _testinternalcapi = import_helper.import_module('_testinternalcapi')
from _xxsubinterpreters import InterpreterNotFoundError from _xxsubinterpreters import InterpreterNotFoundError
@ -36,7 +36,7 @@ def _captured_script(script):
def _run_output(interp, request): def _run_output(interp, request):
script, rpipe = _captured_script(request) script, rpipe = _captured_script(request)
with rpipe: with rpipe:
interpreters.run_string(interp, script) _interpreters.run_string(interp, script)
return rpipe.read() return rpipe.read()
@ -47,7 +47,7 @@ def _wait_for_interp_to_run(interp, timeout=None):
if timeout is None: if timeout is None:
timeout = support.SHORT_TIMEOUT timeout = support.SHORT_TIMEOUT
for _ in support.sleeping_retry(timeout, error=False): for _ in support.sleeping_retry(timeout, error=False):
if interpreters.is_running(interp): if _interpreters.is_running(interp):
break break
else: else:
raise RuntimeError('interp is not running') raise RuntimeError('interp is not running')
@ -57,7 +57,7 @@ def _wait_for_interp_to_run(interp, timeout=None):
def _running(interp): def _running(interp):
r, w = os.pipe() r, w = os.pipe()
def run(): def run():
interpreters.run_string(interp, dedent(f""" _interpreters.run_string(interp, dedent(f"""
# wait for "signal" # wait for "signal"
with open({r}, encoding="utf-8") as rpipe: with open({r}, encoding="utf-8") as rpipe:
rpipe.read() rpipe.read()
@ -75,12 +75,12 @@ def run():
def clean_up_interpreters(): def clean_up_interpreters():
for id in interpreters.list_all(): for id, *_ in _interpreters.list_all():
if id == 0: # main if id == 0: # main
continue continue
try: try:
interpreters.destroy(id) _interpreters.destroy(id)
except interpreters.InterpreterError: except _interpreters.InterpreterError:
pass # already destroyed pass # already destroyed
@ -112,7 +112,7 @@ def test_default_shareables(self):
for obj in shareables: for obj in shareables:
with self.subTest(obj): with self.subTest(obj):
self.assertTrue( self.assertTrue(
interpreters.is_shareable(obj)) _interpreters.is_shareable(obj))
def test_not_shareable(self): def test_not_shareable(self):
class Cheese: class Cheese:
@ -141,7 +141,7 @@ class SubBytes(bytes):
for obj in not_shareables: for obj in not_shareables:
with self.subTest(repr(obj)): with self.subTest(repr(obj)):
self.assertFalse( self.assertFalse(
interpreters.is_shareable(obj)) _interpreters.is_shareable(obj))
class ShareableTypeTests(unittest.TestCase): class ShareableTypeTests(unittest.TestCase):
@ -230,7 +230,7 @@ class ModuleTests(TestBase):
def test_import_in_interpreter(self): def test_import_in_interpreter(self):
_run_output( _run_output(
interpreters.create(), _interpreters.create(),
'import _xxsubinterpreters as _interpreters', 'import _xxsubinterpreters as _interpreters',
) )
@ -241,45 +241,45 @@ def test_import_in_interpreter(self):
class ListAllTests(TestBase): class ListAllTests(TestBase):
def test_initial(self): def test_initial(self):
main = interpreters.get_main() main, *_ = _interpreters.get_main()
ids = interpreters.list_all() ids = [id for id, *_ in _interpreters.list_all()]
self.assertEqual(ids, [main]) self.assertEqual(ids, [main])
def test_after_creating(self): def test_after_creating(self):
main = interpreters.get_main() main, *_ = _interpreters.get_main()
first = interpreters.create() first = _interpreters.create()
second = interpreters.create() second = _interpreters.create()
ids = interpreters.list_all() ids = [id for id, *_ in _interpreters.list_all()]
self.assertEqual(ids, [main, first, second]) self.assertEqual(ids, [main, first, second])
def test_after_destroying(self): def test_after_destroying(self):
main = interpreters.get_main() main, *_ = _interpreters.get_main()
first = interpreters.create() first = _interpreters.create()
second = interpreters.create() second = _interpreters.create()
interpreters.destroy(first) _interpreters.destroy(first)
ids = interpreters.list_all() ids = [id for id, *_ in _interpreters.list_all()]
self.assertEqual(ids, [main, second]) self.assertEqual(ids, [main, second])
class GetCurrentTests(TestBase): class GetCurrentTests(TestBase):
def test_main(self): def test_main(self):
main = interpreters.get_main() main, *_ = _interpreters.get_main()
cur = interpreters.get_current() cur, *_ = _interpreters.get_current()
self.assertEqual(cur, main) self.assertEqual(cur, main)
self.assertIsInstance(cur, int) self.assertIsInstance(cur, int)
def test_subinterpreter(self): def test_subinterpreter(self):
main = interpreters.get_main() main, *_ = _interpreters.get_main()
interp = interpreters.create() interp = _interpreters.create()
out = _run_output(interp, dedent(""" out = _run_output(interp, dedent("""
import _xxsubinterpreters as _interpreters import _xxsubinterpreters as _interpreters
cur = _interpreters.get_current() cur, *_ = _interpreters.get_current()
print(cur) print(cur)
assert isinstance(cur, int) assert isinstance(cur, int)
""")) """))
cur = int(out.strip()) cur = int(out.strip())
_, expected = interpreters.list_all() _, expected = [id for id, *_ in _interpreters.list_all()]
self.assertEqual(cur, expected) self.assertEqual(cur, expected)
self.assertNotEqual(cur, main) self.assertNotEqual(cur, main)
@ -287,17 +287,17 @@ def test_subinterpreter(self):
class GetMainTests(TestBase): class GetMainTests(TestBase):
def test_from_main(self): def test_from_main(self):
[expected] = interpreters.list_all() [expected] = [id for id, *_ in _interpreters.list_all()]
main = interpreters.get_main() main, *_ = _interpreters.get_main()
self.assertEqual(main, expected) self.assertEqual(main, expected)
self.assertIsInstance(main, int) self.assertIsInstance(main, int)
def test_from_subinterpreter(self): def test_from_subinterpreter(self):
[expected] = interpreters.list_all() [expected] = [id for id, *_ in _interpreters.list_all()]
interp = interpreters.create() interp = _interpreters.create()
out = _run_output(interp, dedent(""" out = _run_output(interp, dedent("""
import _xxsubinterpreters as _interpreters import _xxsubinterpreters as _interpreters
main = _interpreters.get_main() main, *_ = _interpreters.get_main()
print(main) print(main)
assert isinstance(main, int) assert isinstance(main, int)
""")) """))
@ -308,20 +308,20 @@ def test_from_subinterpreter(self):
class IsRunningTests(TestBase): class IsRunningTests(TestBase):
def test_main(self): def test_main(self):
main = interpreters.get_main() main, *_ = _interpreters.get_main()
self.assertTrue(interpreters.is_running(main)) self.assertTrue(_interpreters.is_running(main))
@unittest.skip('Fails on FreeBSD') @unittest.skip('Fails on FreeBSD')
def test_subinterpreter(self): def test_subinterpreter(self):
interp = interpreters.create() interp = _interpreters.create()
self.assertFalse(interpreters.is_running(interp)) self.assertFalse(_interpreters.is_running(interp))
with _running(interp): with _running(interp):
self.assertTrue(interpreters.is_running(interp)) self.assertTrue(_interpreters.is_running(interp))
self.assertFalse(interpreters.is_running(interp)) self.assertFalse(_interpreters.is_running(interp))
def test_from_subinterpreter(self): def test_from_subinterpreter(self):
interp = interpreters.create() interp = _interpreters.create()
out = _run_output(interp, dedent(f""" out = _run_output(interp, dedent(f"""
import _xxsubinterpreters as _interpreters import _xxsubinterpreters as _interpreters
if _interpreters.is_running({interp}): if _interpreters.is_running({interp}):
@ -332,34 +332,35 @@ def test_from_subinterpreter(self):
self.assertEqual(out.strip(), 'True') self.assertEqual(out.strip(), 'True')
def test_already_destroyed(self): def test_already_destroyed(self):
interp = interpreters.create() interp = _interpreters.create()
interpreters.destroy(interp) _interpreters.destroy(interp)
with self.assertRaises(InterpreterNotFoundError): with self.assertRaises(InterpreterNotFoundError):
interpreters.is_running(interp) _interpreters.is_running(interp)
def test_does_not_exist(self): def test_does_not_exist(self):
with self.assertRaises(InterpreterNotFoundError): with self.assertRaises(InterpreterNotFoundError):
interpreters.is_running(1_000_000) _interpreters.is_running(1_000_000)
def test_bad_id(self): def test_bad_id(self):
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
interpreters.is_running(-1) _interpreters.is_running(-1)
class CreateTests(TestBase): class CreateTests(TestBase):
def test_in_main(self): def test_in_main(self):
id = interpreters.create() id = _interpreters.create()
self.assertIsInstance(id, int) self.assertIsInstance(id, int)
self.assertIn(id, interpreters.list_all()) after = [id for id, *_ in _interpreters.list_all()]
self.assertIn(id, after)
@unittest.skip('enable this test when working on pystate.c') @unittest.skip('enable this test when working on pystate.c')
def test_unique_id(self): def test_unique_id(self):
seen = set() seen = set()
for _ in range(100): for _ in range(100):
id = interpreters.create() id = _interpreters.create()
interpreters.destroy(id) _interpreters.destroy(id)
seen.add(id) seen.add(id)
self.assertEqual(len(seen), 100) self.assertEqual(len(seen), 100)
@ -369,7 +370,7 @@ def test_in_thread(self):
id = None id = None
def f(): def f():
nonlocal id nonlocal id
id = interpreters.create() id = _interpreters.create()
lock.acquire() lock.acquire()
lock.release() lock.release()
@ -377,11 +378,12 @@ def f():
with lock: with lock:
t.start() t.start()
t.join() t.join()
self.assertIn(id, interpreters.list_all()) after = set(id for id, *_ in _interpreters.list_all())
self.assertIn(id, after)
def test_in_subinterpreter(self): def test_in_subinterpreter(self):
main, = interpreters.list_all() main, = [id for id, *_ in _interpreters.list_all()]
id1 = interpreters.create() id1 = _interpreters.create()
out = _run_output(id1, dedent(""" out = _run_output(id1, dedent("""
import _xxsubinterpreters as _interpreters import _xxsubinterpreters as _interpreters
id = _interpreters.create() id = _interpreters.create()
@ -390,11 +392,12 @@ def test_in_subinterpreter(self):
""")) """))
id2 = int(out.strip()) id2 = int(out.strip())
self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) after = set(id for id, *_ in _interpreters.list_all())
self.assertEqual(after, {main, id1, id2})
def test_in_threaded_subinterpreter(self): def test_in_threaded_subinterpreter(self):
main, = interpreters.list_all() main, = [id for id, *_ in _interpreters.list_all()]
id1 = interpreters.create() id1 = _interpreters.create()
id2 = None id2 = None
def f(): def f():
nonlocal id2 nonlocal id2
@ -409,144 +412,155 @@ def f():
t.start() t.start()
t.join() t.join()
self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) after = set(id for id, *_ in _interpreters.list_all())
self.assertEqual(after, {main, id1, id2})
def test_after_destroy_all(self): def test_after_destroy_all(self):
before = set(interpreters.list_all()) before = set(id for id, *_ in _interpreters.list_all())
# Create 3 subinterpreters. # Create 3 subinterpreters.
ids = [] ids = []
for _ in range(3): for _ in range(3):
id = interpreters.create() id = _interpreters.create()
ids.append(id) ids.append(id)
# Now destroy them. # Now destroy them.
for id in ids: for id in ids:
interpreters.destroy(id) _interpreters.destroy(id)
# Finally, create another. # Finally, create another.
id = interpreters.create() id = _interpreters.create()
self.assertEqual(set(interpreters.list_all()), before | {id}) after = set(id for id, *_ in _interpreters.list_all())
self.assertEqual(after, before | {id})
def test_after_destroy_some(self): def test_after_destroy_some(self):
before = set(interpreters.list_all()) before = set(id for id, *_ in _interpreters.list_all())
# Create 3 subinterpreters. # Create 3 subinterpreters.
id1 = interpreters.create() id1 = _interpreters.create()
id2 = interpreters.create() id2 = _interpreters.create()
id3 = interpreters.create() id3 = _interpreters.create()
# Now destroy 2 of them. # Now destroy 2 of them.
interpreters.destroy(id1) _interpreters.destroy(id1)
interpreters.destroy(id3) _interpreters.destroy(id3)
# Finally, create another. # Finally, create another.
id = interpreters.create() id = _interpreters.create()
self.assertEqual(set(interpreters.list_all()), before | {id, id2}) after = set(id for id, *_ in _interpreters.list_all())
self.assertEqual(after, before | {id, id2})
class DestroyTests(TestBase): class DestroyTests(TestBase):
def test_one(self): def test_one(self):
id1 = interpreters.create() id1 = _interpreters.create()
id2 = interpreters.create() id2 = _interpreters.create()
id3 = interpreters.create() id3 = _interpreters.create()
self.assertIn(id2, interpreters.list_all()) before = set(id for id, *_ in _interpreters.list_all())
interpreters.destroy(id2) self.assertIn(id2, before)
self.assertNotIn(id2, interpreters.list_all())
self.assertIn(id1, interpreters.list_all()) _interpreters.destroy(id2)
self.assertIn(id3, interpreters.list_all())
after = set(id for id, *_ in _interpreters.list_all())
self.assertNotIn(id2, after)
self.assertIn(id1, after)
self.assertIn(id3, after)
def test_all(self): def test_all(self):
before = set(interpreters.list_all()) initial = set(id for id, *_ in _interpreters.list_all())
ids = set() ids = set()
for _ in range(3): for _ in range(3):
id = interpreters.create() id = _interpreters.create()
ids.add(id) ids.add(id)
self.assertEqual(set(interpreters.list_all()), before | ids) before = set(id for id, *_ in _interpreters.list_all())
self.assertEqual(before, initial | ids)
for id in ids: for id in ids:
interpreters.destroy(id) _interpreters.destroy(id)
self.assertEqual(set(interpreters.list_all()), before) after = set(id for id, *_ in _interpreters.list_all())
self.assertEqual(after, initial)
def test_main(self): def test_main(self):
main, = interpreters.list_all() main, = [id for id, *_ in _interpreters.list_all()]
with self.assertRaises(interpreters.InterpreterError): with self.assertRaises(_interpreters.InterpreterError):
interpreters.destroy(main) _interpreters.destroy(main)
def f(): def f():
with self.assertRaises(interpreters.InterpreterError): with self.assertRaises(_interpreters.InterpreterError):
interpreters.destroy(main) _interpreters.destroy(main)
t = threading.Thread(target=f) t = threading.Thread(target=f)
t.start() t.start()
t.join() t.join()
def test_already_destroyed(self): def test_already_destroyed(self):
id = interpreters.create() id = _interpreters.create()
interpreters.destroy(id) _interpreters.destroy(id)
with self.assertRaises(InterpreterNotFoundError): with self.assertRaises(InterpreterNotFoundError):
interpreters.destroy(id) _interpreters.destroy(id)
def test_does_not_exist(self): def test_does_not_exist(self):
with self.assertRaises(InterpreterNotFoundError): with self.assertRaises(InterpreterNotFoundError):
interpreters.destroy(1_000_000) _interpreters.destroy(1_000_000)
def test_bad_id(self): def test_bad_id(self):
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
interpreters.destroy(-1) _interpreters.destroy(-1)
def test_from_current(self): def test_from_current(self):
main, = interpreters.list_all() main, = [id for id, *_ in _interpreters.list_all()]
id = interpreters.create() id = _interpreters.create()
script = dedent(f""" script = dedent(f"""
import _xxsubinterpreters as _interpreters import _xxsubinterpreters as _interpreters
try: try:
_interpreters.destroy({id}) _interpreters.destroy({id})
except interpreters.InterpreterError: except _interpreters.InterpreterError:
pass pass
""") """)
interpreters.run_string(id, script) _interpreters.run_string(id, script)
self.assertEqual(set(interpreters.list_all()), {main, id}) after = set(id for id, *_ in _interpreters.list_all())
self.assertEqual(after, {main, id})
def test_from_sibling(self): def test_from_sibling(self):
main, = interpreters.list_all() main, = [id for id, *_ in _interpreters.list_all()]
id1 = interpreters.create() id1 = _interpreters.create()
id2 = interpreters.create() id2 = _interpreters.create()
script = dedent(f""" script = dedent(f"""
import _xxsubinterpreters as _interpreters import _xxsubinterpreters as _interpreters
_interpreters.destroy({id2}) _interpreters.destroy({id2})
""") """)
interpreters.run_string(id1, script) _interpreters.run_string(id1, script)
self.assertEqual(set(interpreters.list_all()), {main, id1}) after = set(id for id, *_ in _interpreters.list_all())
self.assertEqual(after, {main, id1})
def test_from_other_thread(self): def test_from_other_thread(self):
id = interpreters.create() id = _interpreters.create()
def f(): def f():
interpreters.destroy(id) _interpreters.destroy(id)
t = threading.Thread(target=f) t = threading.Thread(target=f)
t.start() t.start()
t.join() t.join()
def test_still_running(self): def test_still_running(self):
main, = interpreters.list_all() main, = [id for id, *_ in _interpreters.list_all()]
interp = interpreters.create() interp = _interpreters.create()
with _running(interp): with _running(interp):
self.assertTrue(interpreters.is_running(interp), self.assertTrue(_interpreters.is_running(interp),
msg=f"Interp {interp} should be running before destruction.") msg=f"Interp {interp} should be running before destruction.")
with self.assertRaises(interpreters.InterpreterError, with self.assertRaises(_interpreters.InterpreterError,
msg=f"Should not be able to destroy interp {interp} while it's still running."): msg=f"Should not be able to destroy interp {interp} while it's still running."):
interpreters.destroy(interp) _interpreters.destroy(interp)
self.assertTrue(interpreters.is_running(interp)) self.assertTrue(_interpreters.is_running(interp))
class RunStringTests(TestBase): class RunStringTests(TestBase):
def setUp(self): def setUp(self):
super().setUp() super().setUp()
self.id = interpreters.create() self.id = _interpreters.create()
def test_success(self): def test_success(self):
script, file = _captured_script('print("it worked!", end="")') script, file = _captured_script('print("it worked!", end="")')
with file: with file:
interpreters.run_string(self.id, script) _interpreters.run_string(self.id, script)
out = file.read() out = file.read()
self.assertEqual(out, 'it worked!') self.assertEqual(out, 'it worked!')
@ -555,7 +569,7 @@ def test_in_thread(self):
script, file = _captured_script('print("it worked!", end="")') script, file = _captured_script('print("it worked!", end="")')
with file: with file:
def f(): def f():
interpreters.run_string(self.id, script) _interpreters.run_string(self.id, script)
t = threading.Thread(target=f) t = threading.Thread(target=f)
t.start() t.start()
@ -565,7 +579,7 @@ def f():
self.assertEqual(out, 'it worked!') self.assertEqual(out, 'it worked!')
def test_create_thread(self): def test_create_thread(self):
subinterp = interpreters.create() subinterp = _interpreters.create()
script, file = _captured_script(""" script, file = _captured_script("""
import threading import threading
def f(): def f():
@ -576,7 +590,7 @@ def f():
t.join() t.join()
""") """)
with file: with file:
interpreters.run_string(subinterp, script) _interpreters.run_string(subinterp, script)
out = file.read() out = file.read()
self.assertEqual(out, 'it worked!') self.assertEqual(out, 'it worked!')
@ -584,7 +598,7 @@ def f():
def test_create_daemon_thread(self): def test_create_daemon_thread(self):
with self.subTest('isolated'): with self.subTest('isolated'):
expected = 'spam spam spam spam spam' expected = 'spam spam spam spam spam'
subinterp = interpreters.create('isolated') subinterp = _interpreters.create('isolated')
script, file = _captured_script(f""" script, file = _captured_script(f"""
import threading import threading
def f(): def f():
@ -598,13 +612,13 @@ def f():
print('{expected}', end='') print('{expected}', end='')
""") """)
with file: with file:
interpreters.run_string(subinterp, script) _interpreters.run_string(subinterp, script)
out = file.read() out = file.read()
self.assertEqual(out, expected) self.assertEqual(out, expected)
with self.subTest('not isolated'): with self.subTest('not isolated'):
subinterp = interpreters.create('legacy') subinterp = _interpreters.create('legacy')
script, file = _captured_script(""" script, file = _captured_script("""
import threading import threading
def f(): def f():
@ -615,13 +629,13 @@ def f():
t.join() t.join()
""") """)
with file: with file:
interpreters.run_string(subinterp, script) _interpreters.run_string(subinterp, script)
out = file.read() out = file.read()
self.assertEqual(out, 'it worked!') self.assertEqual(out, 'it worked!')
def test_shareable_types(self): def test_shareable_types(self):
interp = interpreters.create() interp = _interpreters.create()
objects = [ objects = [
None, None,
'spam', 'spam',
@ -630,15 +644,15 @@ def test_shareable_types(self):
] ]
for obj in objects: for obj in objects:
with self.subTest(obj): with self.subTest(obj):
interpreters.set___main___attrs(interp, dict(obj=obj)) _interpreters.set___main___attrs(interp, dict(obj=obj))
interpreters.run_string( _interpreters.run_string(
interp, interp,
f'assert(obj == {obj!r})', f'assert(obj == {obj!r})',
) )
def test_os_exec(self): def test_os_exec(self):
expected = 'spam spam spam spam spam' expected = 'spam spam spam spam spam'
subinterp = interpreters.create() subinterp = _interpreters.create()
script, file = _captured_script(f""" script, file = _captured_script(f"""
import os, sys import os, sys
try: try:
@ -647,7 +661,7 @@ def test_os_exec(self):
print('{expected}', end='') print('{expected}', end='')
""") """)
with file: with file:
interpreters.run_string(subinterp, script) _interpreters.run_string(subinterp, script)
out = file.read() out = file.read()
self.assertEqual(out, expected) self.assertEqual(out, expected)
@ -668,7 +682,7 @@ def test_fork(self):
with open('{file.name}', 'w', encoding='utf-8') as out: with open('{file.name}', 'w', encoding='utf-8') as out:
out.write('{expected}') out.write('{expected}')
""") """)
interpreters.run_string(self.id, script) _interpreters.run_string(self.id, script)
file.seek(0) file.seek(0)
content = file.read() content = file.read()
@ -676,31 +690,31 @@ def test_fork(self):
def test_already_running(self): def test_already_running(self):
with _running(self.id): with _running(self.id):
with self.assertRaises(interpreters.InterpreterError): with self.assertRaises(_interpreters.InterpreterError):
interpreters.run_string(self.id, 'print("spam")') _interpreters.run_string(self.id, 'print("spam")')
def test_does_not_exist(self): def test_does_not_exist(self):
id = 0 id = 0
while id in interpreters.list_all(): while id in set(id for id, *_ in _interpreters.list_all()):
id += 1 id += 1
with self.assertRaises(InterpreterNotFoundError): with self.assertRaises(InterpreterNotFoundError):
interpreters.run_string(id, 'print("spam")') _interpreters.run_string(id, 'print("spam")')
def test_error_id(self): def test_error_id(self):
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
interpreters.run_string(-1, 'print("spam")') _interpreters.run_string(-1, 'print("spam")')
def test_bad_id(self): def test_bad_id(self):
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
interpreters.run_string('spam', 'print("spam")') _interpreters.run_string('spam', 'print("spam")')
def test_bad_script(self): def test_bad_script(self):
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
interpreters.run_string(self.id, 10) _interpreters.run_string(self.id, 10)
def test_bytes_for_script(self): def test_bytes_for_script(self):
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
interpreters.run_string(self.id, b'print("spam")') _interpreters.run_string(self.id, b'print("spam")')
def test_with_shared(self): def test_with_shared(self):
r, w = os.pipe() r, w = os.pipe()
@ -721,8 +735,8 @@ def test_with_shared(self):
with open({w}, 'wb') as chan: with open({w}, 'wb') as chan:
pickle.dump(ns, chan) pickle.dump(ns, chan)
""") """)
interpreters.set___main___attrs(self.id, shared) _interpreters.set___main___attrs(self.id, shared)
interpreters.run_string(self.id, script) _interpreters.run_string(self.id, script)
with open(r, 'rb') as chan: with open(r, 'rb') as chan:
ns = pickle.load(chan) ns = pickle.load(chan)
@ -732,7 +746,7 @@ def test_with_shared(self):
self.assertIsNone(ns['cheddar']) self.assertIsNone(ns['cheddar'])
def test_shared_overwrites(self): def test_shared_overwrites(self):
interpreters.run_string(self.id, dedent(""" _interpreters.run_string(self.id, dedent("""
spam = 'eggs' spam = 'eggs'
ns1 = dict(vars()) ns1 = dict(vars())
del ns1['__builtins__'] del ns1['__builtins__']
@ -743,8 +757,8 @@ def test_shared_overwrites(self):
ns2 = dict(vars()) ns2 = dict(vars())
del ns2['__builtins__'] del ns2['__builtins__']
""") """)
interpreters.set___main___attrs(self.id, shared) _interpreters.set___main___attrs(self.id, shared)
interpreters.run_string(self.id, script) _interpreters.run_string(self.id, script)
r, w = os.pipe() r, w = os.pipe()
script = dedent(f""" script = dedent(f"""
@ -754,7 +768,7 @@ def test_shared_overwrites(self):
with open({w}, 'wb') as chan: with open({w}, 'wb') as chan:
pickle.dump(ns, chan) pickle.dump(ns, chan)
""") """)
interpreters.run_string(self.id, script) _interpreters.run_string(self.id, script)
with open(r, 'rb') as chan: with open(r, 'rb') as chan:
ns = pickle.load(chan) ns = pickle.load(chan)
@ -775,8 +789,8 @@ def test_shared_overwrites_default_vars(self):
with open({w}, 'wb') as chan: with open({w}, 'wb') as chan:
pickle.dump(ns, chan) pickle.dump(ns, chan)
""") """)
interpreters.set___main___attrs(self.id, shared) _interpreters.set___main___attrs(self.id, shared)
interpreters.run_string(self.id, script) _interpreters.run_string(self.id, script)
with open(r, 'rb') as chan: with open(r, 'rb') as chan:
ns = pickle.load(chan) ns = pickle.load(chan)
@ -784,7 +798,7 @@ def test_shared_overwrites_default_vars(self):
def test_main_reused(self): def test_main_reused(self):
r, w = os.pipe() r, w = os.pipe()
interpreters.run_string(self.id, dedent(f""" _interpreters.run_string(self.id, dedent(f"""
spam = True spam = True
ns = dict(vars()) ns = dict(vars())
@ -798,7 +812,7 @@ def test_main_reused(self):
ns1 = pickle.load(chan) ns1 = pickle.load(chan)
r, w = os.pipe() r, w = os.pipe()
interpreters.run_string(self.id, dedent(f""" _interpreters.run_string(self.id, dedent(f"""
eggs = False eggs = False
ns = dict(vars()) ns = dict(vars())
@ -827,7 +841,7 @@ def test_execution_namespace_is_main(self):
with open({w}, 'wb') as chan: with open({w}, 'wb') as chan:
pickle.dump(ns, chan) pickle.dump(ns, chan)
""") """)
interpreters.run_string(self.id, script) _interpreters.run_string(self.id, script)
with open(r, 'rb') as chan: with open(r, 'rb') as chan:
ns = pickle.load(chan) ns = pickle.load(chan)
@ -872,13 +886,13 @@ class RunFailedTests(TestBase):
def setUp(self): def setUp(self):
super().setUp() super().setUp()
self.id = interpreters.create() self.id = _interpreters.create()
def add_module(self, modname, text): def add_module(self, modname, text):
import tempfile import tempfile
tempdir = tempfile.mkdtemp() tempdir = tempfile.mkdtemp()
self.addCleanup(lambda: os_helper.rmtree(tempdir)) self.addCleanup(lambda: os_helper.rmtree(tempdir))
interpreters.run_string(self.id, dedent(f""" _interpreters.run_string(self.id, dedent(f"""
import sys import sys
sys.path.insert(0, {tempdir!r}) sys.path.insert(0, {tempdir!r})
""")) """))
@ -900,11 +914,11 @@ class NeverError(Exception): pass
raise NeverError # never raised raise NeverError # never raised
""").format(dedent(text)) """).format(dedent(text))
if fails: if fails:
err = interpreters.run_string(self.id, script) err = _interpreters.run_string(self.id, script)
self.assertIsNot(err, None) self.assertIsNot(err, None)
return err return err
else: else:
err = interpreters.run_string(self.id, script) err = _interpreters.run_string(self.id, script)
self.assertIs(err, None) self.assertIs(err, None)
return None return None
except: except:
@ -1029,7 +1043,7 @@ class RunFuncTests(TestBase):
def setUp(self): def setUp(self):
super().setUp() super().setUp()
self.id = interpreters.create() self.id = _interpreters.create()
def test_success(self): def test_success(self):
r, w = os.pipe() r, w = os.pipe()
@ -1039,8 +1053,8 @@ def script():
with open(w, 'w', encoding="utf-8") as spipe: with open(w, 'w', encoding="utf-8") as spipe:
with contextlib.redirect_stdout(spipe): with contextlib.redirect_stdout(spipe):
print('it worked!', end='') print('it worked!', end='')
interpreters.set___main___attrs(self.id, dict(w=w)) _interpreters.set___main___attrs(self.id, dict(w=w))
interpreters.run_func(self.id, script) _interpreters.run_func(self.id, script)
with open(r, encoding="utf-8") as outfile: with open(r, encoding="utf-8") as outfile:
out = outfile.read() out = outfile.read()
@ -1056,8 +1070,8 @@ def script():
with contextlib.redirect_stdout(spipe): with contextlib.redirect_stdout(spipe):
print('it worked!', end='') print('it worked!', end='')
def f(): def f():
interpreters.set___main___attrs(self.id, dict(w=w)) _interpreters.set___main___attrs(self.id, dict(w=w))
interpreters.run_func(self.id, script) _interpreters.run_func(self.id, script)
t = threading.Thread(target=f) t = threading.Thread(target=f)
t.start() t.start()
t.join() t.join()
@ -1077,8 +1091,8 @@ def script():
with contextlib.redirect_stdout(spipe): with contextlib.redirect_stdout(spipe):
print('it worked!', end='') print('it worked!', end='')
code = script.__code__ code = script.__code__
interpreters.set___main___attrs(self.id, dict(w=w)) _interpreters.set___main___attrs(self.id, dict(w=w))
interpreters.run_func(self.id, code) _interpreters.run_func(self.id, code)
with open(r, encoding="utf-8") as outfile: with open(r, encoding="utf-8") as outfile:
out = outfile.read() out = outfile.read()
@ -1091,7 +1105,7 @@ def script():
assert spam assert spam
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
interpreters.run_func(self.id, script) _interpreters.run_func(self.id, script)
# XXX This hasn't been fixed yet. # XXX This hasn't been fixed yet.
@unittest.expectedFailure @unittest.expectedFailure
@ -1099,38 +1113,38 @@ def test_return_value(self):
def script(): def script():
return 'spam' return 'spam'
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
interpreters.run_func(self.id, script) _interpreters.run_func(self.id, script)
def test_args(self): def test_args(self):
with self.subTest('args'): with self.subTest('args'):
def script(a, b=0): def script(a, b=0):
assert a == b assert a == b
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
interpreters.run_func(self.id, script) _interpreters.run_func(self.id, script)
with self.subTest('*args'): with self.subTest('*args'):
def script(*args): def script(*args):
assert not args assert not args
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
interpreters.run_func(self.id, script) _interpreters.run_func(self.id, script)
with self.subTest('**kwargs'): with self.subTest('**kwargs'):
def script(**kwargs): def script(**kwargs):
assert not kwargs assert not kwargs
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
interpreters.run_func(self.id, script) _interpreters.run_func(self.id, script)
with self.subTest('kwonly'): with self.subTest('kwonly'):
def script(*, spam=True): def script(*, spam=True):
assert spam assert spam
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
interpreters.run_func(self.id, script) _interpreters.run_func(self.id, script)
with self.subTest('posonly'): with self.subTest('posonly'):
def script(spam, /): def script(spam, /):
assert spam assert spam
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
interpreters.run_func(self.id, script) _interpreters.run_func(self.id, script)
if __name__ == '__main__': if __name__ == '__main__':

8
Lib/test/test_capi/test_misc.py
View File

@ -2065,7 +2065,7 @@ def test_configured_settings(self):
_testinternalcapi.get_interp_settings() _testinternalcapi.get_interp_settings()
raise NotImplementedError('unreachable') raise NotImplementedError('unreachable')
''') ''')
with self.assertRaises(RuntimeError): with self.assertRaises(_interpreters.InterpreterError):
support.run_in_subinterp_with_config(script, **kwargs) support.run_in_subinterp_with_config(script, **kwargs)
@unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module") @unittest.skipIf(_testsinglephase is None, "test requires _testsinglephase module")
@ -2403,7 +2403,7 @@ def check(config):
continue continue
if match(config, invalid): if match(config, invalid):
with self.subTest(f'invalid: {config}'): with self.subTest(f'invalid: {config}'):
with self.assertRaises(RuntimeError): with self.assertRaises(_interpreters.InterpreterError):
check(config) check(config)
elif match(config, questionable): elif match(config, questionable):
with self.subTest(f'questionable: {config}'): with self.subTest(f'questionable: {config}'):
@ -2427,7 +2427,7 @@ def new_interp(config):
with self.subTest('main'): with self.subTest('main'):
expected = _interpreters.new_config('legacy') expected = _interpreters.new_config('legacy')
expected.gil = 'own' expected.gil = 'own'
interpid = _interpreters.get_main() interpid, *_ = _interpreters.get_main()
config = _interpreters.get_config(interpid) config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected) self.assert_ns_equal(config, expected)
@ -2579,7 +2579,7 @@ def test_linked_lifecycle_does_not_exist(self):
def test_linked_lifecycle_initial(self): def test_linked_lifecycle_initial(self):
is_linked = _testinternalcapi.interpreter_refcount_linked is_linked = _testinternalcapi.interpreter_refcount_linked
get_refcount = _testinternalcapi.get_interpreter_refcount get_refcount, _, _ = self.get_refcount_helpers()
# A new interpreter will start out not linked, with a refcount of 0. # A new interpreter will start out not linked, with a refcount of 0.
interpid = self.new_interpreter() interpid = self.new_interpreter()

540
Lib/test/test_interpreters/test_api.py
View File

@ -1,6 +1,7 @@
import os import os
import pickle import pickle
from textwrap import dedent import sys
from textwrap import dedent, indent
import threading import threading
import types import types
import unittest import unittest
@ -10,8 +11,13 @@
# Raise SkipTest if subinterpreters not supported. # Raise SkipTest if subinterpreters not supported.
_interpreters = import_helper.import_module('_xxsubinterpreters') _interpreters = import_helper.import_module('_xxsubinterpreters')
from test.support import interpreters from test.support import interpreters
from test.support.interpreters import InterpreterNotFoundError from test.support.interpreters import (
from .utils import _captured_script, _run_output, _running, TestBase InterpreterError, InterpreterNotFoundError, ExecutionFailed,
)
from .utils import (
_captured_script, _run_output, _running, TestBase,
requires_test_modules, _testinternalcapi,
)
class ModuleTests(TestBase): class ModuleTests(TestBase):
@ -157,6 +163,20 @@ def test_idempotent(self):
id2 = id(interp) id2 = id(interp)
self.assertNotEqual(id1, id2) self.assertNotEqual(id1, id2)
@requires_test_modules
def test_created_with_capi(self):
last = 0
for id, *_ in _interpreters.list_all():
last = max(last, id)
expected = _testinternalcapi.next_interpreter_id()
text = self.run_temp_from_capi(f"""
import {interpreters.__name__} as interpreters
interp = interpreters.get_current()
print(interp.id)
""")
interpid = eval(text)
self.assertEqual(interpid, expected)
class ListAllTests(TestBase): class ListAllTests(TestBase):
@ -199,6 +219,33 @@ def test_idempotent(self):
for interp1, interp2 in zip(actual, expected): for interp1, interp2 in zip(actual, expected):
self.assertIs(interp1, interp2) self.assertIs(interp1, interp2)
def test_created_with_capi(self):
mainid, *_ = _interpreters.get_main()
interpid1 = _interpreters.create()
interpid2 = _interpreters.create()
interpid3 = _interpreters.create()
interpid4 = interpid3 + 1
interpid5 = interpid4 + 1
expected = [
(mainid,),
(interpid1,),
(interpid2,),
(interpid3,),
(interpid4,),
(interpid5,),
]
expected2 = expected[:-2]
text = self.run_temp_from_capi(f"""
import {interpreters.__name__} as interpreters
interp = interpreters.create()
print(
[(i.id,) for i in interpreters.list_all()])
""")
res = eval(text)
res2 = [(i.id,) for i in interpreters.list_all()]
self.assertEqual(res, expected)
self.assertEqual(res2, expected2)
class InterpreterObjectTests(TestBase): class InterpreterObjectTests(TestBase):
@ -276,6 +323,7 @@ def test_main(self):
main = interpreters.get_main() main = interpreters.get_main()
self.assertTrue(main.is_running()) self.assertTrue(main.is_running())
# XXX Is this still true?
@unittest.skip('Fails on FreeBSD') @unittest.skip('Fails on FreeBSD')
def test_subinterpreter(self): def test_subinterpreter(self):
interp = interpreters.create() interp = interpreters.create()
@ -337,6 +385,55 @@ def task():
interp.exec('t.join()') interp.exec('t.join()')
self.assertEqual(os.read(r_interp, 1), FINISHED) self.assertEqual(os.read(r_interp, 1), FINISHED)
def test_created_with_capi(self):
script = dedent(f"""
import {interpreters.__name__} as interpreters
interp = interpreters.get_current()
print(interp.is_running())
""")
def parse_results(text):
self.assertNotEqual(text, "")
try:
return eval(text)
except Exception:
raise Exception(repr(text))
with self.subTest('running __main__ (from self)'):
with self.interpreter_from_capi() as interpid:
text = self.run_from_capi(interpid, script, main=True)
running = parse_results(text)
self.assertTrue(running)
with self.subTest('running, but not __main__ (from self)'):
text = self.run_temp_from_capi(script)
running = parse_results(text)
self.assertFalse(running)
with self.subTest('running __main__ (from other)'):
with self.interpreter_obj_from_capi() as (interp, interpid):
before = interp.is_running()
with self.running_from_capi(interpid, main=True):
during = interp.is_running()
after = interp.is_running()
self.assertFalse(before)
self.assertTrue(during)
self.assertFalse(after)
with self.subTest('running, but not __main__ (from other)'):
with self.interpreter_obj_from_capi() as (interp, interpid):
before = interp.is_running()
with self.running_from_capi(interpid, main=False):
during = interp.is_running()
after = interp.is_running()
self.assertFalse(before)
self.assertFalse(during)
self.assertFalse(after)
with self.subTest('not running (from other)'):
with self.interpreter_obj_from_capi() as (interp, _):
running = interp.is_running()
self.assertFalse(running)
class TestInterpreterClose(TestBase): class TestInterpreterClose(TestBase):
@ -364,11 +461,11 @@ def test_all(self):
def test_main(self): def test_main(self):
main, = interpreters.list_all() main, = interpreters.list_all()
with self.assertRaises(interpreters.InterpreterError): with self.assertRaises(InterpreterError):
main.close() main.close()
def f(): def f():
with self.assertRaises(interpreters.InterpreterError): with self.assertRaises(InterpreterError):
main.close() main.close()
t = threading.Thread(target=f) t = threading.Thread(target=f)
@ -419,12 +516,13 @@ def f():
t.start() t.start()
t.join() t.join()
# XXX Is this still true?
@unittest.skip('Fails on FreeBSD') @unittest.skip('Fails on FreeBSD')
def test_still_running(self): def test_still_running(self):
main, = interpreters.list_all() main, = interpreters.list_all()
interp = interpreters.create() interp = interpreters.create()
with _running(interp): with _running(interp):
with self.assertRaises(interpreters.InterpreterError): with self.assertRaises(InterpreterError):
interp.close() interp.close()
self.assertTrue(interp.is_running()) self.assertTrue(interp.is_running())
@ -459,6 +557,52 @@ def task():
self.assertEqual(os.read(r_interp, 1), FINISHED) self.assertEqual(os.read(r_interp, 1), FINISHED)
def test_created_with_capi(self):
script = dedent(f"""
import {interpreters.__name__} as interpreters
interp = interpreters.get_current()
interp.close()
""")
with self.subTest('running __main__ (from self)'):
with self.interpreter_from_capi() as interpid:
with self.assertRaisesRegex(ExecutionFailed,
'InterpreterError.*current'):
self.run_from_capi(interpid, script, main=True)
with self.subTest('running, but not __main__ (from self)'):
with self.assertRaisesRegex(ExecutionFailed,
'InterpreterError.*current'):
self.run_temp_from_capi(script)
with self.subTest('running __main__ (from other)'):
with self.interpreter_obj_from_capi() as (interp, interpid):
with self.running_from_capi(interpid, main=True):
with self.assertRaisesRegex(InterpreterError, 'running'):
interp.close()
# Make sure it wssn't closed.
self.assertTrue(
interp.is_running())
# The rest must be skipped until we deal with running threads when
# interp.close() is called.
return
with self.subTest('running, but not __main__ (from other)'):
with self.interpreter_obj_from_capi() as (interp, interpid):
with self.running_from_capi(interpid, main=False):
with self.assertRaisesRegex(InterpreterError, 'not managed'):
interp.close()
# Make sure it wssn't closed.
self.assertFalse(interp.is_running())
with self.subTest('not running (from other)'):
with self.interpreter_obj_from_capi() as (interp, _):
with self.assertRaisesRegex(InterpreterError, 'not managed'):
interp.close()
# Make sure it wssn't closed.
self.assertFalse(interp.is_running())
class TestInterpreterPrepareMain(TestBase): class TestInterpreterPrepareMain(TestBase):
@ -511,26 +655,45 @@ def test_not_shareable(self):
interp.prepare_main(spam={'spam': 'eggs', 'foo': 'bar'}) interp.prepare_main(spam={'spam': 'eggs', 'foo': 'bar'})
# Make sure neither was actually bound. # Make sure neither was actually bound.
with self.assertRaises(interpreters.ExecutionFailed): with self.assertRaises(ExecutionFailed):
interp.exec('print(foo)') interp.exec('print(foo)')
with self.assertRaises(interpreters.ExecutionFailed): with self.assertRaises(ExecutionFailed):
interp.exec('print(spam)') interp.exec('print(spam)')
def test_running(self):
interp = interpreters.create()
interp.prepare_main({'spam': True})
with self.running(interp):
with self.assertRaisesRegex(InterpreterError, 'running'):
interp.prepare_main({'spam': False})
interp.exec('assert spam is True')
@requires_test_modules
def test_created_with_capi(self):
with self.interpreter_from_capi() as interpid:
interp = interpreters.Interpreter(interpid)
interp.prepare_main({'spam': True})
rc = _testinternalcapi.exec_interpreter(interpid,
'assert spam is True')
assert rc == 0, rc
class TestInterpreterExec(TestBase): class TestInterpreterExec(TestBase):
def test_success(self): def test_success(self):
interp = interpreters.create() interp = interpreters.create()
script, file = _captured_script('print("it worked!", end="")') script, results = _captured_script('print("it worked!", end="")')
with file: with results:
interp.exec(script) interp.exec(script)
out = file.read() results = results.final()
results.raise_if_failed()
out = results.stdout
self.assertEqual(out, 'it worked!') self.assertEqual(out, 'it worked!')
def test_failure(self): def test_failure(self):
interp = interpreters.create() interp = interpreters.create()
with self.assertRaises(interpreters.ExecutionFailed): with self.assertRaises(ExecutionFailed):
interp.exec('raise Exception') interp.exec('raise Exception')
def test_display_preserved_exception(self): def test_display_preserved_exception(self):
@ -583,15 +746,17 @@ def script():
def test_in_thread(self): def test_in_thread(self):
interp = interpreters.create() interp = interpreters.create()
script, file = _captured_script('print("it worked!", end="")') script, results = _captured_script('print("it worked!", end="")')
with file: with results:
def f(): def f():
interp.exec(script) interp.exec(script)
t = threading.Thread(target=f) t = threading.Thread(target=f)
t.start() t.start()
t.join() t.join()
out = file.read() results = results.final()
results.raise_if_failed()
out = results.stdout
self.assertEqual(out, 'it worked!') self.assertEqual(out, 'it worked!')
@ -618,6 +783,7 @@ def test_fork(self):
content = file.read() content = file.read()
self.assertEqual(content, expected) self.assertEqual(content, expected)
# XXX Is this still true?
@unittest.skip('Fails on FreeBSD') @unittest.skip('Fails on FreeBSD')
def test_already_running(self): def test_already_running(self):
interp = interpreters.create() interp = interpreters.create()
@ -666,6 +832,11 @@ def task():
self.assertEqual(os.read(r_interp, 1), RAN) self.assertEqual(os.read(r_interp, 1), RAN)
self.assertEqual(os.read(r_interp, 1), FINISHED) self.assertEqual(os.read(r_interp, 1), FINISHED)
def test_created_with_capi(self):
with self.interpreter_obj_from_capi() as (interp, _):
with self.assertRaisesRegex(ExecutionFailed, 'it worked'):
interp.exec('raise Exception("it worked!")')
# test_xxsubinterpreters covers the remaining # test_xxsubinterpreters covers the remaining
# Interpreter.exec() behavior. # Interpreter.exec() behavior.
@ -830,7 +1001,7 @@ def test_call(self):
raise Exception((args, kwargs)) raise Exception((args, kwargs))
interp.call(callable) interp.call(callable)
with self.assertRaises(interpreters.ExecutionFailed): with self.assertRaises(ExecutionFailed):
interp.call(call_func_failure) interp.call(call_func_failure)
def test_call_in_thread(self): def test_call_in_thread(self):
@ -942,6 +1113,14 @@ class LowLevelTests(TestBase):
# encountered by the high-level module, thus they # encountered by the high-level module, thus they
# mostly shouldn't matter as much. # mostly shouldn't matter as much.
def interp_exists(self, interpid):
try:
_interpreters.is_running(interpid)
except InterpreterNotFoundError:
return False
else:
return True
def test_new_config(self): def test_new_config(self):
# This test overlaps with # This test overlaps with
# test.test_capi.test_misc.InterpreterConfigTests. # test.test_capi.test_misc.InterpreterConfigTests.
@ -1064,46 +1243,107 @@ def test_new_config(self):
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
_interpreters.new_config(gil=value) _interpreters.new_config(gil=value)
def test_get_config(self): def test_get_main(self):
# This test overlaps with interpid, = _interpreters.get_main()
# test.test_capi.test_misc.InterpreterConfigTests. self.assertEqual(interpid, 0)
def test_get_current(self):
with self.subTest('main'):
main, *_ = _interpreters.get_main()
interpid, = _interpreters.get_current()
self.assertEqual(interpid, main)
script = f"""
import {_interpreters.__name__} as _interpreters
interpid, = _interpreters.get_current()
print(interpid)
"""
def parse_stdout(text):
parts = text.split()
assert len(parts) == 1, parts
interpid, = parts
interpid = int(interpid)
return interpid,
with self.subTest('from _interpreters'):
orig = _interpreters.create()
text = self.run_and_capture(orig, script)
interpid, = parse_stdout(text)
self.assertEqual(interpid, orig)
with self.subTest('from C-API'):
last = 0
for id, *_ in _interpreters.list_all():
last = max(last, id)
expected = last + 1
text = self.run_temp_from_capi(script)
interpid, = parse_stdout(text)
self.assertEqual(interpid, expected)
def test_list_all(self):
mainid, *_ = _interpreters.get_main()
interpid1 = _interpreters.create()
interpid2 = _interpreters.create()
interpid3 = _interpreters.create()
expected = [
(mainid,),
(interpid1,),
(interpid2,),
(interpid3,),
]
with self.subTest('main'): with self.subTest('main'):
expected = _interpreters.new_config('legacy') res = _interpreters.list_all()
expected.gil = 'own' self.assertEqual(res, expected)
interpid = _interpreters.get_main()
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
with self.subTest('isolated'): with self.subTest('from _interpreters'):
expected = _interpreters.new_config('isolated') text = self.run_and_capture(interpid2, f"""
interpid = _interpreters.create('isolated') import {_interpreters.__name__} as _interpreters
config = _interpreters.get_config(interpid) print(
self.assert_ns_equal(config, expected) _interpreters.list_all())
""")
with self.subTest('legacy'): res = eval(text)
expected = _interpreters.new_config('legacy') self.assertEqual(res, expected)
interpid = _interpreters.create('legacy')
config = _interpreters.get_config(interpid) with self.subTest('from C-API'):
self.assert_ns_equal(config, expected) interpid4 = interpid3 + 1
interpid5 = interpid4 + 1
expected2 = expected + [
(interpid4,),
(interpid5,),
]
expected3 = expected + [
(interpid5,),
]
text = self.run_temp_from_capi(f"""
import {_interpreters.__name__} as _interpreters
_interpreters.create()
print(
_interpreters.list_all())
""")
res2 = eval(text)
res3 = _interpreters.list_all()
self.assertEqual(res2, expected2)
self.assertEqual(res3, expected3)
def test_create(self): def test_create(self):
isolated = _interpreters.new_config('isolated') isolated = _interpreters.new_config('isolated')
legacy = _interpreters.new_config('legacy') legacy = _interpreters.new_config('legacy')
default = isolated default = isolated
with self.subTest('no arg'): with self.subTest('no args'):
interpid = _interpreters.create() interpid = _interpreters.create()
config = _interpreters.get_config(interpid) config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, default) self.assert_ns_equal(config, default)
with self.subTest('arg: None'): with self.subTest('config: None'):
interpid = _interpreters.create(None) interpid = _interpreters.create(None)
config = _interpreters.get_config(interpid) config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, default) self.assert_ns_equal(config, default)
with self.subTest('arg: \'empty\''): with self.subTest('config: \'empty\''):
with self.assertRaises(interpreters.InterpreterError): with self.assertRaises(InterpreterError):
# The "empty" config isn't viable on its own. # The "empty" config isn't viable on its own.
_interpreters.create('empty') _interpreters.create('empty')
@ -1138,6 +1378,230 @@ def test_create(self):
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
_interpreters.create(orig) _interpreters.create(orig)
@requires_test_modules
def test_destroy(self):
with self.subTest('from _interpreters'):
interpid = _interpreters.create()
before = [id for id, *_ in _interpreters.list_all()]
_interpreters.destroy(interpid)
after = [id for id, *_ in _interpreters.list_all()]
self.assertIn(interpid, before)
self.assertNotIn(interpid, after)
self.assertFalse(
self.interp_exists(interpid))
with self.subTest('main'):
interpid, *_ = _interpreters.get_main()
with self.assertRaises(InterpreterError):
# It is the current interpreter.
_interpreters.destroy(interpid)
with self.subTest('from C-API'):
interpid = _testinternalcapi.create_interpreter()
_interpreters.destroy(interpid)
self.assertFalse(
self.interp_exists(interpid))
def test_get_config(self):
# This test overlaps with
# test.test_capi.test_misc.InterpreterConfigTests.
with self.subTest('main'):
expected = _interpreters.new_config('legacy')
expected.gil = 'own'
interpid, *_ = _interpreters.get_main()
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
with self.subTest('main'):
expected = _interpreters.new_config('legacy')
expected.gil = 'own'
interpid, *_ = _interpreters.get_main()
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
with self.subTest('isolated'):
expected = _interpreters.new_config('isolated')
interpid = _interpreters.create('isolated')
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
with self.subTest('legacy'):
expected = _interpreters.new_config('legacy')
interpid = _interpreters.create('legacy')
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, expected)
with self.subTest('from C-API'):
orig = _interpreters.new_config('isolated')
with self.interpreter_from_capi(orig) as interpid:
config = _interpreters.get_config(interpid)
self.assert_ns_equal(config, orig)
@requires_test_modules
def test_whence(self):
with self.subTest('main'):
interpid, *_ = _interpreters.get_main()
whence = _interpreters.whence(interpid)
self.assertEqual(whence, _interpreters.WHENCE_RUNTIME)
with self.subTest('stdlib'):
interpid = _interpreters.create()
whence = _interpreters.whence(interpid)
self.assertEqual(whence, _interpreters.WHENCE_XI)
for orig, name in {
# XXX Also check WHENCE_UNKNOWN.
_interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API',
_interpreters.WHENCE_CAPI: 'C-API',
_interpreters.WHENCE_XI: 'cross-interpreter C-API',
}.items():
with self.subTest(f'from C-API ({orig}: {name})'):
with self.interpreter_from_capi(whence=orig) as interpid:
whence = _interpreters.whence(interpid)
self.assertEqual(whence, orig)
with self.subTest('from C-API, running'):
text = self.run_temp_from_capi(dedent(f"""
import {_interpreters.__name__} as _interpreters
interpid, *_ = _interpreters.get_current()
print(_interpreters.whence(interpid))
"""),
config=True)
whence = eval(text)
self.assertEqual(whence, _interpreters.WHENCE_CAPI)
with self.subTest('from legacy C-API, running'):
...
text = self.run_temp_from_capi(dedent(f"""
import {_interpreters.__name__} as _interpreters
interpid, *_ = _interpreters.get_current()
print(_interpreters.whence(interpid))
"""),
config=False)
whence = eval(text)
self.assertEqual(whence, _interpreters.WHENCE_LEGACY_CAPI)
def test_is_running(self):
with self.subTest('main'):
interpid, *_ = _interpreters.get_main()
running = _interpreters.is_running(interpid)
self.assertTrue(running)
with self.subTest('from _interpreters (running)'):
interpid = _interpreters.create()
with self.running(interpid):
running = _interpreters.is_running(interpid)
self.assertTrue(running)
with self.subTest('from _interpreters (not running)'):
interpid = _interpreters.create()
running = _interpreters.is_running(interpid)
self.assertFalse(running)
with self.subTest('from C-API (running __main__)'):
with self.interpreter_from_capi() as interpid:
with self.running_from_capi(interpid, main=True):
running = _interpreters.is_running(interpid)
self.assertTrue(running)
with self.subTest('from C-API (running, but not __main__)'):
with self.interpreter_from_capi() as interpid:
with self.running_from_capi(interpid, main=False):
running = _interpreters.is_running(interpid)
self.assertFalse(running)
with self.subTest('from C-API (not running)'):
with self.interpreter_from_capi() as interpid:
running = _interpreters.is_running(interpid)
self.assertFalse(running)
def test_exec(self):
with self.subTest('run script'):
interpid = _interpreters.create()
script, results = _captured_script('print("it worked!", end="")')
with results:
exc = _interpreters.exec(interpid, script)
results = results.final()
results.raise_if_failed()
out = results.stdout
self.assertEqual(out, 'it worked!')
with self.subTest('uncaught exception'):
interpid = _interpreters.create()
script, results = _captured_script("""
raise Exception('uh-oh!')
print("it worked!", end="")
""")
with results:
exc = _interpreters.exec(interpid, script)
out = results.stdout()
self.assertEqual(out, '')
self.assert_ns_equal(exc, types.SimpleNamespace(
type=types.SimpleNamespace(
__name__='Exception',
__qualname__='Exception',
__module__='builtins',
),
msg='uh-oh!',
# We check these in other tests.
formatted=exc.formatted,
errdisplay=exc.errdisplay,
))
with self.subTest('from C-API'):
with self.interpreter_from_capi() as interpid:
exc = _interpreters.exec(interpid, 'raise Exception("it worked!")')
self.assertIsNot(exc, None)
self.assertEqual(exc.msg, 'it worked!')
def test_call(self):
with self.subTest('no args'):
interpid = _interpreters.create()
exc = _interpreters.call(interpid, call_func_return_shareable)
self.assertIs(exc, None)
with self.subTest('uncaught exception'):
interpid = _interpreters.create()
exc = _interpreters.call(interpid, call_func_failure)
self.assertEqual(exc, types.SimpleNamespace(
type=types.SimpleNamespace(
__name__='Exception',
__qualname__='Exception',
__module__='builtins',
),
msg='spam!',
# We check these in other tests.
formatted=exc.formatted,
errdisplay=exc.errdisplay,
))
def test_set___main___attrs(self):
with self.subTest('from _interpreters'):
interpid = _interpreters.create()
before1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'')
before2 = _interpreters.exec(interpid, 'assert ham == 42')
self.assertEqual(before1.type.__name__, 'NameError')
self.assertEqual(before2.type.__name__, 'NameError')
_interpreters.set___main___attrs(interpid, dict(
spam='eggs',
ham=42,
))
after1 = _interpreters.exec(interpid, 'assert spam == \'eggs\'')
after2 = _interpreters.exec(interpid, 'assert ham == 42')
after3 = _interpreters.exec(interpid, 'assert spam == 42')
self.assertIs(after1, None)
self.assertIs(after2, None)
self.assertEqual(after3.type.__name__, 'AssertionError')
with self.subTest('from C-API'):
with self.interpreter_from_capi() as interpid:
_interpreters.set___main___attrs(interpid, {'spam': True})
exc = _interpreters.exec(interpid, 'assert spam is True')
self.assertIsNone(exc)
if __name__ == '__main__': if __name__ == '__main__':
# Test needs to be a package, so we can do relative imports. # Test needs to be a package, so we can do relative imports.

523
Lib/test/test_interpreters/utils.py
View File

@ -1,30 +1,344 @@
from collections import namedtuple
import contextlib import contextlib
import json
import io
import os import os
import os.path import os.path
import pickle
import queue
#import select
import subprocess import subprocess
import sys import sys
import tempfile import tempfile
from textwrap import dedent from textwrap import dedent, indent
import threading import threading
import types import types
import unittest import unittest
import warnings
from test import support from test import support
from test.support import os_helper from test.support import os_helper
from test.support import import_helper
_interpreters = import_helper.import_module('_xxsubinterpreters')
from test.support import interpreters from test.support import interpreters
def _captured_script(script): try:
r, w = os.pipe() import _testinternalcapi
indented = script.replace('\n', '\n ') import _testcapi
wrapped = dedent(f""" except ImportError:
import contextlib _testinternalcapi = None
with open({w}, 'w', encoding='utf-8') as spipe: _testcapi = None
with contextlib.redirect_stdout(spipe):
def requires_test_modules(func):
return unittest.skipIf(_testinternalcapi is None, "test requires _testinternalcapi module")(func)
def _dump_script(text):
lines = text.splitlines()
print()
print('-' * 20)
for i, line in enumerate(lines, 1):
print(f' {i:>{len(str(len(lines)))}} {line}')
print('-' * 20)
def _close_file(file):
try:
if hasattr(file, 'close'):
file.close()
else:
os.close(file)
except OSError as exc:
if exc.errno != 9:
raise # re-raise
# It was closed already.
def pack_exception(exc=None):
captured = _interpreters.capture_exception(exc)
data = dict(captured.__dict__)
data['type'] = dict(captured.type.__dict__)
return json.dumps(data)
def unpack_exception(packed):
try:
data = json.loads(packed)
except json.decoder.JSONDecodeError:
warnings.warn('incomplete exception data', RuntimeWarning)
print(packed if isinstance(packed, str) else packed.decode('utf-8'))
return None
exc = types.SimpleNamespace(**data)
exc.type = types.SimpleNamespace(**exc.type)
return exc;
class CapturingResults:
STDIO = dedent("""\
with open({w_pipe}, 'wb', buffering=0) as _spipe_{stream}:
_captured_std{stream} = io.StringIO()
with contextlib.redirect_std{stream}(_captured_std{stream}):
#########################
# begin wrapped script
{indented} {indented}
""")
return wrapped, open(r, encoding='utf-8') # end wrapped script
#########################
text = _captured_std{stream}.getvalue()
_spipe_{stream}.write(text.encode('utf-8'))
""")[:-1]
EXC = dedent("""\
with open({w_pipe}, 'wb', buffering=0) as _spipe_exc:
try:
#########################
# begin wrapped script
{indented}
# end wrapped script
#########################
except Exception as exc:
text = _interp_utils.pack_exception(exc)
_spipe_exc.write(text.encode('utf-8'))
""")[:-1]
@classmethod
def wrap_script(cls, script, *, stdout=True, stderr=False, exc=False):
script = dedent(script).strip(os.linesep)
imports = [
f'import {__name__} as _interp_utils',
]
wrapped = script
# Handle exc.
if exc:
exc = os.pipe()
r_exc, w_exc = exc
indented = wrapped.replace('\n', '\n ')
wrapped = cls.EXC.format(
w_pipe=w_exc,
indented=indented,
)
else:
exc = None
# Handle stdout.
if stdout:
imports.extend([
'import contextlib, io',
])
stdout = os.pipe()
r_out, w_out = stdout
indented = wrapped.replace('\n', '\n ')
wrapped = cls.STDIO.format(
w_pipe=w_out,
indented=indented,
stream='out',
)
else:
stdout = None
# Handle stderr.
if stderr == 'stdout':
stderr = None
elif stderr:
if not stdout:
imports.extend([
'import contextlib, io',
])
stderr = os.pipe()
r_err, w_err = stderr
indented = wrapped.replace('\n', '\n ')
wrapped = cls.STDIO.format(
w_pipe=w_err,
indented=indented,
stream='err',
)
else:
stderr = None
if wrapped == script:
raise NotImplementedError
else:
for line in imports:
wrapped = f'{line}{os.linesep}{wrapped}'
results = cls(stdout, stderr, exc)
return wrapped, results
def __init__(self, out, err, exc):
self._rf_out = None
self._rf_err = None
self._rf_exc = None
self._w_out = None
self._w_err = None
self._w_exc = None
if out is not None:
r_out, w_out = out
self._rf_out = open(r_out, 'rb', buffering=0)
self._w_out = w_out
if err is not None:
r_err, w_err = err
self._rf_err = open(r_err, 'rb', buffering=0)
self._w_err = w_err
if exc is not None:
r_exc, w_exc = exc
self._rf_exc = open(r_exc, 'rb', buffering=0)
self._w_exc = w_exc
self._buf_out = b''
self._buf_err = b''
self._buf_exc = b''
self._exc = None
self._closed = False
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
@property
def closed(self):
return self._closed
def close(self):
if self._closed:
return
self._closed = True
if self._w_out is not None:
_close_file(self._w_out)
self._w_out = None
if self._w_err is not None:
_close_file(self._w_err)
self._w_err = None
if self._w_exc is not None:
_close_file(self._w_exc)
self._w_exc = None
self._capture()
if self._rf_out is not None:
_close_file(self._rf_out)
self._rf_out = None
if self._rf_err is not None:
_close_file(self._rf_err)
self._rf_err = None
if self._rf_exc is not None:
_close_file(self._rf_exc)
self._rf_exc = None
def _capture(self):
# Ideally this is called only after the script finishes
# (and thus has closed the write end of the pipe.
if self._rf_out is not None:
chunk = self._rf_out.read(100)
while chunk:
self._buf_out += chunk
chunk = self._rf_out.read(100)
if self._rf_err is not None:
chunk = self._rf_err.read(100)
while chunk:
self._buf_err += chunk
chunk = self._rf_err.read(100)
if self._rf_exc is not None:
chunk = self._rf_exc.read(100)
while chunk:
self._buf_exc += chunk
chunk = self._rf_exc.read(100)
def _unpack_stdout(self):
return self._buf_out.decode('utf-8')
def _unpack_stderr(self):
return self._buf_err.decode('utf-8')
def _unpack_exc(self):
if self._exc is not None:
return self._exc
if not self._buf_exc:
return None
self._exc = unpack_exception(self._buf_exc)
return self._exc
def stdout(self):
if self.closed:
return self.final().stdout
self._capture()
return self._unpack_stdout()
def stderr(self):
if self.closed:
return self.final().stderr
self._capture()
return self._unpack_stderr()
def exc(self):
if self.closed:
return self.final().exc
self._capture()
return self._unpack_exc()
def final(self, *, force=False):
try:
return self._final
except AttributeError:
if not self._closed:
if not force:
raise Exception('no final results available yet')
else:
return CapturedResults.Proxy(self)
self._final = CapturedResults(
self._unpack_stdout(),
self._unpack_stderr(),
self._unpack_exc(),
)
return self._final
class CapturedResults(namedtuple('CapturedResults', 'stdout stderr exc')):
class Proxy:
def __init__(self, capturing):
self._capturing = capturing
def _finish(self):
if self._capturing is None:
return
self._final = self._capturing.final()
self._capturing = None
def __iter__(self):
self._finish()
yield from self._final
def __len__(self):
self._finish()
return len(self._final)
def __getattr__(self, name):
self._finish()
if name.startswith('_'):
raise AttributeError(name)
return getattr(self._final, name)
def raise_if_failed(self):
if self.exc is not None:
raise interpreters.ExecutionFailed(self.exc)
def _captured_script(script, *, stdout=True, stderr=False, exc=False):
return CapturingResults.wrap_script(
script,
stdout=stdout,
stderr=stderr,
exc=exc,
)
def clean_up_interpreters(): def clean_up_interpreters():
@ -33,17 +347,17 @@ def clean_up_interpreters():
continue continue
try: try:
interp.close() interp.close()
except RuntimeError: except _interpreters.InterpreterError:
pass # already destroyed pass # already destroyed
def _run_output(interp, request, init=None): def _run_output(interp, request, init=None):
script, rpipe = _captured_script(request) script, results = _captured_script(request)
with rpipe: with results:
if init: if init:
interp.prepare_main(init) interp.prepare_main(init)
interp.exec(script) interp.exec(script)
return rpipe.read() return results.stdout()
@contextlib.contextmanager @contextlib.contextmanager
@ -175,3 +489,184 @@ def assert_ns_equal(self, ns1, ns2, msg=None):
diff = f'namespace({diff})' diff = f'namespace({diff})'
standardMsg = self._truncateMessage(standardMsg, diff) standardMsg = self._truncateMessage(standardMsg, diff)
self.fail(self._formatMessage(msg, standardMsg)) self.fail(self._formatMessage(msg, standardMsg))
def _run_string(self, interp, script):
wrapped, results = _captured_script(script, exc=False)
#_dump_script(wrapped)
with results:
if isinstance(interp, interpreters.Interpreter):
interp.exec(script)
else:
err = _interpreters.run_string(interp, wrapped)
if err is not None:
return None, err
return results.stdout(), None
def run_and_capture(self, interp, script):
text, err = self._run_string(interp, script)
if err is not None:
raise interpreters.ExecutionFailed(err)
else:
return text
@requires_test_modules
@contextlib.contextmanager
def interpreter_from_capi(self, config=None, whence=None):
if config is False:
if whence is None:
whence = _interpreters.WHENCE_LEGACY_CAPI
else:
assert whence in (_interpreters.WHENCE_LEGACY_CAPI,
_interpreters.WHENCE_UNKNOWN), repr(whence)
config = None
elif config is True:
config = _interpreters.new_config('default')
elif config is None:
if whence not in (
_interpreters.WHENCE_LEGACY_CAPI,
_interpreters.WHENCE_UNKNOWN,
):
config = _interpreters.new_config('legacy')
elif isinstance(config, str):
config = _interpreters.new_config(config)
if whence is None:
whence = _interpreters.WHENCE_XI
interpid = _testinternalcapi.create_interpreter(config, whence=whence)
try:
yield interpid
finally:
try:
_testinternalcapi.destroy_interpreter(interpid)
except _interpreters.InterpreterNotFoundError:
pass
@contextlib.contextmanager
def interpreter_obj_from_capi(self, config='legacy'):
with self.interpreter_from_capi(config) as interpid:
yield interpreters.Interpreter(interpid), interpid
@contextlib.contextmanager
def capturing(self, script):
wrapped, capturing = _captured_script(script, stdout=True, exc=True)
#_dump_script(wrapped)
with capturing:
yield wrapped, capturing.final(force=True)
@requires_test_modules
def run_from_capi(self, interpid, script, *, main=False):
with self.capturing(script) as (wrapped, results):
rc = _testinternalcapi.exec_interpreter(interpid, wrapped, main=main)
assert rc == 0, rc
results.raise_if_failed()
return results.stdout
@contextlib.contextmanager
def _running(self, run_interp, exec_interp):
token = b'\0'
r_in, w_in = self.pipe()
r_out, w_out = self.pipe()
def close():
_close_file(r_in)
_close_file(w_in)
_close_file(r_out)
_close_file(w_out)
# Start running (and wait).
script = dedent(f"""
import os
try:
# handshake
token = os.read({r_in}, 1)
os.write({w_out}, token)
# Wait for the "done" message.
os.read({r_in}, 1)
except BrokenPipeError:
pass
except OSError as exc:
if exc.errno != 9:
raise # re-raise
# It was closed already.
""")
failed = None
def run():
nonlocal failed
try:
run_interp(script)
except Exception as exc:
failed = exc
close()
t = threading.Thread(target=run)
t.start()
# handshake
try:
os.write(w_in, token)
token2 = os.read(r_out, 1)
assert token2 == token, (token2, token)
except OSError:
t.join()
if failed is not None:
raise failed
# CM __exit__()
try:
try:
yield
finally:
# Send "done".
os.write(w_in, b'\0')
finally:
close()
t.join()
if failed is not None:
raise failed
@contextlib.contextmanager
def running(self, interp):
if isinstance(interp, int):
interpid = interp
def exec_interp(script):
exc = _interpreters.exec(interpid, script)
assert exc is None, exc
run_interp = exec_interp
else:
def run_interp(script):
text = self.run_and_capture(interp, script)
assert text == '', repr(text)
def exec_interp(script):
interp.exec(script)
with self._running(run_interp, exec_interp):
yield
@requires_test_modules
@contextlib.contextmanager
def running_from_capi(self, interpid, *, main=False):
def run_interp(script):
text = self.run_from_capi(interpid, script, main=main)
assert text == '', repr(text)
def exec_interp(script):
rc = _testinternalcapi.exec_interpreter(interpid, script)
assert rc == 0, rc
with self._running(run_interp, exec_interp):
yield
@requires_test_modules
def run_temp_from_capi(self, script, config='legacy'):
if config is False:
# Force using Py_NewInterpreter().
run_in_interp = (lambda s, c: _testcapi.run_in_subinterp(s))
config = None
else:
run_in_interp = _testinternalcapi.run_in_subinterp_with_config
if config is True:
config = 'default'
if isinstance(config, str):
config = _interpreters.new_config(config)
with self.capturing(script) as (wrapped, results):
rc = run_in_interp(wrapped, config)
assert rc == 0, rc
results.raise_if_failed()
return results.stdout

17
Modules/_interpreters_common.h
View File

@ -19,20 +19,3 @@ clear_xid_class(PyTypeObject *cls)
return _PyCrossInterpreterData_UnregisterClass(cls); return _PyCrossInterpreterData_UnregisterClass(cls);
} }
#endif #endif
#ifdef RETURNS_INTERPID_OBJECT
static PyObject *
get_interpid_obj(PyInterpreterState *interp)
{
if (_PyInterpreterState_IDInitref(interp) != 0) {
return NULL;
};
int64_t id = PyInterpreterState_GetID(interp);
if (id < 0) {
return NULL;
}
assert(id < LLONG_MAX);
return PyLong_FromLongLong(id);
}
#endif

308
Modules/_testinternalcapi.c
View File

@ -1369,56 +1369,284 @@ dict_getitem_knownhash(PyObject *self, PyObject *args)
} }
/* To run some code in a sub-interpreter. */ static int
_init_interp_config_from_object(PyInterpreterConfig *config, PyObject *obj)
{
if (obj == NULL) {
*config = (PyInterpreterConfig)_PyInterpreterConfig_INIT;
return 0;
}
PyObject *dict = PyObject_GetAttrString(obj, "__dict__");
if (dict == NULL) {
PyErr_Format(PyExc_TypeError, "bad config %R", obj);
return -1;
}
int res = _PyInterpreterConfig_InitFromDict(config, dict);
Py_DECREF(dict);
if (res < 0) {
return -1;
}
return 0;
}
static PyInterpreterState *
_new_interpreter(PyInterpreterConfig *config, long whence)
{
if (whence == _PyInterpreterState_WHENCE_XI) {
return _PyXI_NewInterpreter(config, NULL, NULL);
}
PyObject *exc = NULL;
PyInterpreterState *interp = NULL;
if (whence == _PyInterpreterState_WHENCE_UNKNOWN) {
assert(config == NULL);
interp = PyInterpreterState_New();
}
else if (whence == _PyInterpreterState_WHENCE_CAPI
|| whence == _PyInterpreterState_WHENCE_LEGACY_CAPI)
{
PyThreadState *tstate = NULL;
PyThreadState *save_tstate = PyThreadState_Swap(NULL);
if (whence == _PyInterpreterState_WHENCE_LEGACY_CAPI) {
assert(config == NULL);
tstate = Py_NewInterpreter();
PyThreadState_Swap(save_tstate);
}
else {
PyStatus status = Py_NewInterpreterFromConfig(&tstate, config);
PyThreadState_Swap(save_tstate);
if (PyStatus_Exception(status)) {
assert(tstate == NULL);
_PyErr_SetFromPyStatus(status);
exc = PyErr_GetRaisedException();
}
}
if (tstate != NULL) {
interp = PyThreadState_GetInterpreter(tstate);
// Throw away the initial tstate.
PyThreadState_Swap(tstate);
PyThreadState_Clear(tstate);
PyThreadState_Swap(save_tstate);
PyThreadState_Delete(tstate);
}
}
else {
PyErr_Format(PyExc_ValueError,
"unsupported whence %ld", whence);
return NULL;
}
if (interp == NULL) {
PyErr_SetString(PyExc_InterpreterError,
"sub-interpreter creation failed");
if (exc != NULL) {
_PyErr_ChainExceptions1(exc);
}
}
return interp;
}
// This exists mostly for testing the _interpreters module, as an
// alternative to _interpreters.create()
static PyObject *
create_interpreter(PyObject *self, PyObject *args, PyObject *kwargs)
{
static char *kwlist[] = {"config", "whence", NULL};
PyObject *configobj = NULL;
long whence = _PyInterpreterState_WHENCE_XI;
if (!PyArg_ParseTupleAndKeywords(args, kwargs,
"|O$l:create_interpreter", kwlist,
&configobj, &whence))
{
return NULL;
}
if (configobj == Py_None) {
configobj = NULL;
}
// Resolve the config.
PyInterpreterConfig *config = NULL;
PyInterpreterConfig _config;
if (whence == _PyInterpreterState_WHENCE_UNKNOWN
|| whence == _PyInterpreterState_WHENCE_LEGACY_CAPI)
{
if (configobj != NULL) {
PyErr_SetString(PyExc_ValueError, "got unexpected config");
return NULL;
}
}
else {
config = &_config;
if (_init_interp_config_from_object(config, configobj) < 0) {
return NULL;
}
}
// Create the interpreter.
PyInterpreterState *interp = _new_interpreter(config, whence);
if (interp == NULL) {
return NULL;
}
// Return the ID.
PyObject *idobj = _PyInterpreterState_GetIDObject(interp);
if (idobj == NULL) {
_PyXI_EndInterpreter(interp, NULL, NULL);
return NULL;
}
return idobj;
}
// This exists mostly for testing the _interpreters module, as an
// alternative to _interpreters.destroy()
static PyObject *
destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs)
{
static char *kwlist[] = {"id", NULL};
PyObject *idobj = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwargs,
"O:destroy_interpreter", kwlist,
&idobj))
{
return NULL;
}
PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj);
if (interp == NULL) {
return NULL;
}
_PyXI_EndInterpreter(interp, NULL, NULL);
Py_RETURN_NONE;
}
// This exists mostly for testing the _interpreters module, as an
// alternative to _interpreters.destroy()
static PyObject *
exec_interpreter(PyObject *self, PyObject *args, PyObject *kwargs)
{
static char *kwlist[] = {"id", "code", "main", NULL};
PyObject *idobj;
const char *code;
int runningmain = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwargs,
"Os|$p:exec_interpreter", kwlist,
&idobj, &code, &runningmain))
{
return NULL;
}
PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj);
if (interp == NULL) {
return NULL;
}
PyObject *res = NULL;
PyThreadState *tstate = PyThreadState_New(interp);
_PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_EXEC);
PyThreadState *save_tstate = PyThreadState_Swap(tstate);
if (runningmain) {
if (_PyInterpreterState_SetRunningMain(interp) < 0) {
goto finally;
}
}
/* only initialise 'cflags.cf_flags' to test backwards compatibility */
PyCompilerFlags cflags = {0};
int r = PyRun_SimpleStringFlags(code, &cflags);
if (PyErr_Occurred()) {
PyErr_PrintEx(0);
}
if (runningmain) {
_PyInterpreterState_SetNotRunningMain(interp);
}
res = PyLong_FromLong(r);
finally:
PyThreadState_Clear(tstate);
PyThreadState_Swap(save_tstate);
PyThreadState_Delete(tstate);
return res;
}
/* To run some code in a sub-interpreter.
Generally you can use test.support.interpreters,
but we keep this helper as a distinct implementation.
That's especially important for testing test.support.interpreters.
*/
static PyObject * static PyObject *
run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
{ {
const char *code; const char *code;
PyObject *configobj; PyObject *configobj;
static char *kwlist[] = {"code", "config", NULL}; int xi = 0;
static char *kwlist[] = {"code", "config", "xi", NULL};
if (!PyArg_ParseTupleAndKeywords(args, kwargs, if (!PyArg_ParseTupleAndKeywords(args, kwargs,
"sO:run_in_subinterp_with_config", kwlist, "sO|$p:run_in_subinterp_with_config", kwlist,
&code, &configobj)) &code, &configobj, &xi))
{ {
return NULL; return NULL;
} }
PyInterpreterConfig config; PyInterpreterConfig config;
PyObject *dict = PyObject_GetAttrString(configobj, "__dict__"); if (_init_interp_config_from_object(&config, configobj) < 0) {
if (dict == NULL) {
PyErr_Format(PyExc_TypeError, "bad config %R", configobj);
return NULL;
}
int res = _PyInterpreterConfig_InitFromDict(&config, dict);
Py_DECREF(dict);
if (res < 0) {
return NULL; return NULL;
} }
PyThreadState *mainstate = PyThreadState_Get();
PyThreadState_Swap(NULL);
PyThreadState *substate;
PyStatus status = Py_NewInterpreterFromConfig(&substate, &config);
if (PyStatus_Exception(status)) {
/* Since no new thread state was created, there is no exception to
propagate; raise a fresh one after swapping in the old thread
state. */
PyThreadState_Swap(mainstate);
_PyErr_SetFromPyStatus(status);
PyObject *exc = PyErr_GetRaisedException();
PyErr_SetString(PyExc_RuntimeError, "sub-interpreter creation failed");
_PyErr_ChainExceptions1(exc);
return NULL;
}
assert(substate != NULL);
/* only initialise 'cflags.cf_flags' to test backwards compatibility */ /* only initialise 'cflags.cf_flags' to test backwards compatibility */
PyCompilerFlags cflags = {0}; PyCompilerFlags cflags = {0};
int r = PyRun_SimpleStringFlags(code, &cflags);
Py_EndInterpreter(substate);
PyThreadState_Swap(mainstate); int r;
if (xi) {
PyThreadState *save_tstate;
PyThreadState *tstate;
/* Create an interpreter, staying switched to it. */
PyInterpreterState *interp = \
_PyXI_NewInterpreter(&config, &tstate, &save_tstate);
if (interp == NULL) {
return NULL;
}
/* Exec the code in the new interpreter. */
r = PyRun_SimpleStringFlags(code, &cflags);
/* clean up post-exec. */
_PyXI_EndInterpreter(interp, tstate, &save_tstate);
}
else {
PyThreadState *substate;
PyThreadState *mainstate = PyThreadState_Swap(NULL);
/* Create an interpreter, staying switched to it. */
PyStatus status = Py_NewInterpreterFromConfig(&substate, &config);
if (PyStatus_Exception(status)) {
/* Since no new thread state was created, there is no exception to
propagate; raise a fresh one after swapping in the old thread
state. */
PyThreadState_Swap(mainstate);
_PyErr_SetFromPyStatus(status);
PyObject *exc = PyErr_GetRaisedException();
PyErr_SetString(PyExc_InterpreterError,
"sub-interpreter creation failed");
_PyErr_ChainExceptions1(exc);
return NULL;
}
/* Exec the code in the new interpreter. */
r = PyRun_SimpleStringFlags(code, &cflags);
/* clean up post-exec. */
Py_EndInterpreter(substate);
PyThreadState_Swap(mainstate);
}
return PyLong_FromLong(r); return PyLong_FromLong(r);
} }
@ -1434,6 +1662,13 @@ normalize_interp_id(PyObject *self, PyObject *idobj)
return PyLong_FromLongLong(interpid); return PyLong_FromLongLong(interpid);
} }
static PyObject *
next_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored))
{
int64_t interpid = _PyRuntime.interpreters.next_id;
return PyLong_FromLongLong(interpid);
}
static PyObject * static PyObject *
unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored)) unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored))
{ {
@ -1751,10 +1986,17 @@ static PyMethodDef module_functions[] = {
{"get_object_dict_values", get_object_dict_values, METH_O}, {"get_object_dict_values", get_object_dict_values, METH_O},
{"hamt", new_hamt, METH_NOARGS}, {"hamt", new_hamt, METH_NOARGS},
{"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS},
{"create_interpreter", _PyCFunction_CAST(create_interpreter),
METH_VARARGS | METH_KEYWORDS},
{"destroy_interpreter", _PyCFunction_CAST(destroy_interpreter),
METH_VARARGS | METH_KEYWORDS},
{"exec_interpreter", _PyCFunction_CAST(exec_interpreter),
METH_VARARGS | METH_KEYWORDS},
{"run_in_subinterp_with_config", {"run_in_subinterp_with_config",
_PyCFunction_CAST(run_in_subinterp_with_config), _PyCFunction_CAST(run_in_subinterp_with_config),
METH_VARARGS | METH_KEYWORDS}, METH_VARARGS | METH_KEYWORDS},
{"normalize_interp_id", normalize_interp_id, METH_O}, {"normalize_interp_id", normalize_interp_id, METH_O},
{"next_interpreter_id", next_interpreter_id, METH_NOARGS},
{"unused_interpreter_id", unused_interpreter_id, METH_NOARGS}, {"unused_interpreter_id", unused_interpreter_id, METH_NOARGS},
{"interpreter_exists", interpreter_exists, METH_O}, {"interpreter_exists", interpreter_exists, METH_O},
{"get_interpreter_refcount", get_interpreter_refcount, METH_O}, {"get_interpreter_refcount", get_interpreter_refcount, METH_O},

5
Modules/_xxinterpchannelsmodule.c
View File

@ -8,6 +8,7 @@
#include "Python.h" #include "Python.h"
#include "pycore_crossinterp.h" // struct _xid #include "pycore_crossinterp.h" // struct _xid
#include "pycore_interp.h" // _PyInterpreterState_LookUpID() #include "pycore_interp.h" // _PyInterpreterState_LookUpID()
#include "pycore_pystate.h" // _PyInterpreterState_GetIDObject()
#ifdef MS_WINDOWS #ifdef MS_WINDOWS
#define WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN
@ -17,9 +18,7 @@
#endif #endif
#define REGISTERS_HEAP_TYPES #define REGISTERS_HEAP_TYPES
#define RETURNS_INTERPID_OBJECT
#include "_interpreters_common.h" #include "_interpreters_common.h"
#undef RETURNS_INTERPID_OBJECT
#undef REGISTERS_HEAP_TYPES #undef REGISTERS_HEAP_TYPES
@ -2909,7 +2908,7 @@ channelsmod_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds)
goto except; goto except;
} }
if (res) { if (res) {
interpid_obj = get_interpid_obj(interp); interpid_obj = _PyInterpreterState_GetIDObject(interp);
if (interpid_obj == NULL) { if (interpid_obj == NULL) {
goto except; goto except;
} }

225
Modules/_xxsubinterpretersmodule.c
View File

@ -20,9 +20,7 @@
#include "marshal.h" // PyMarshal_ReadObjectFromString() #include "marshal.h" // PyMarshal_ReadObjectFromString()
#define RETURNS_INTERPID_OBJECT
#include "_interpreters_common.h" #include "_interpreters_common.h"
#undef RETURNS_INTERPID_OBJECT
#define MODULE_NAME _xxsubinterpreters #define MODULE_NAME _xxsubinterpreters
@ -425,59 +423,6 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config)
} }
static PyInterpreterState *
new_interpreter(PyInterpreterConfig *config, PyObject **p_idobj, PyThreadState **p_tstate)
{
PyThreadState *save_tstate = PyThreadState_Get();
assert(save_tstate != NULL);
PyThreadState *tstate = NULL;
// XXX Possible GILState issues?
PyStatus status = Py_NewInterpreterFromConfig(&tstate, config);
PyThreadState_Swap(save_tstate);
if (PyStatus_Exception(status)) {
/* Since no new thread state was created, there is no exception to
propagate; raise a fresh one after swapping in the old thread
state. */
_PyErr_SetFromPyStatus(status);
return NULL;
}
assert(tstate != NULL);
PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate);
if (_PyInterpreterState_IDInitref(interp) < 0) {
goto error;
}
if (p_idobj != NULL) {
// We create the object using the original interpreter.
PyObject *idobj = get_interpid_obj(interp);
if (idobj == NULL) {
goto error;
}
*p_idobj = idobj;
}
if (p_tstate != NULL) {
*p_tstate = tstate;
}
else {
PyThreadState_Swap(tstate);
PyThreadState_Clear(tstate);
PyThreadState_Swap(save_tstate);
PyThreadState_Delete(tstate);
}
return interp;
error:
// XXX Possible GILState issues?
save_tstate = PyThreadState_Swap(tstate);
Py_EndInterpreter(tstate);
PyThreadState_Swap(save_tstate);
return NULL;
}
static int static int
_run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags)
{ {
@ -546,6 +491,19 @@ _run_in_interpreter(PyInterpreterState *interp,
/* module level code ********************************************************/ /* module level code ********************************************************/
static PyObject *
get_summary(PyInterpreterState *interp)
{
PyObject *idobj = _PyInterpreterState_GetIDObject(interp);
if (idobj == NULL) {
return NULL;
}
PyObject *res = PyTuple_Pack(1, idobj);
Py_DECREF(idobj);
return res;
}
static PyObject * static PyObject *
interp_new_config(PyObject *self, PyObject *args, PyObject *kwds) interp_new_config(PyObject *self, PyObject *args, PyObject *kwds)
{ {
@ -606,8 +564,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds)
return NULL; return NULL;
} }
PyObject *idobj = NULL; PyInterpreterState *interp = _PyXI_NewInterpreter(&config, NULL, NULL);
PyInterpreterState *interp = new_interpreter(&config, &idobj, NULL);
if (interp == NULL) { if (interp == NULL) {
// XXX Move the chained exception to interpreters.create()? // XXX Move the chained exception to interpreters.create()?
PyObject *exc = PyErr_GetRaisedException(); PyObject *exc = PyErr_GetRaisedException();
@ -617,6 +574,12 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds)
return NULL; return NULL;
} }
PyObject *idobj = _PyInterpreterState_GetIDObject(interp);
if (idobj == NULL) {
_PyXI_EndInterpreter(interp, NULL, NULL);
return NULL;
}
if (reqrefs) { if (reqrefs) {
// Decref to 0 will destroy the interpreter. // Decref to 0 will destroy the interpreter.
_PyInterpreterState_RequireIDRef(interp, 1); _PyInterpreterState_RequireIDRef(interp, 1);
@ -678,12 +641,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds)
} }
// Destroy the interpreter. // Destroy the interpreter.
PyThreadState *tstate = PyThreadState_New(interp); _PyXI_EndInterpreter(interp, NULL, NULL);
_PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_INTERP);
// XXX Possible GILState issues?
PyThreadState *save_tstate = PyThreadState_Swap(tstate);
Py_EndInterpreter(tstate);
PyThreadState_Swap(save_tstate);
Py_RETURN_NONE; Py_RETURN_NONE;
} }
@ -700,7 +658,7 @@ So does an unrecognized ID.");
static PyObject * static PyObject *
interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
{ {
PyObject *ids, *id; PyObject *ids;
PyInterpreterState *interp; PyInterpreterState *interp;
ids = PyList_New(0); ids = PyList_New(0);
@ -710,14 +668,14 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
interp = PyInterpreterState_Head(); interp = PyInterpreterState_Head();
while (interp != NULL) { while (interp != NULL) {
id = get_interpid_obj(interp); PyObject *item = get_summary(interp);
if (id == NULL) { if (item == NULL) {
Py_DECREF(ids); Py_DECREF(ids);
return NULL; return NULL;
} }
// insert at front of list // insert at front of list
int res = PyList_Insert(ids, 0, id); int res = PyList_Insert(ids, 0, item);
Py_DECREF(id); Py_DECREF(item);
if (res < 0) { if (res < 0) {
Py_DECREF(ids); Py_DECREF(ids);
return NULL; return NULL;
@ -730,7 +688,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
} }
PyDoc_STRVAR(list_all_doc, PyDoc_STRVAR(list_all_doc,
"list_all() -> [ID]\n\ "list_all() -> [(ID,)]\n\
\n\ \n\
Return a list containing the ID of every existing interpreter."); Return a list containing the ID of every existing interpreter.");
@ -742,11 +700,11 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored))
if (interp == NULL) { if (interp == NULL) {
return NULL; return NULL;
} }
return get_interpid_obj(interp); return get_summary(interp);
} }
PyDoc_STRVAR(get_current_doc, PyDoc_STRVAR(get_current_doc,
"get_current() -> ID\n\ "get_current() -> (ID,)\n\
\n\ \n\
Return the ID of current interpreter."); Return the ID of current interpreter.");
@ -754,13 +712,12 @@ Return the ID of current interpreter.");
static PyObject * static PyObject *
interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored)) interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored))
{ {
// Currently, 0 is always the main interpreter. PyInterpreterState *interp = _PyInterpreterState_Main();
int64_t id = 0; return get_summary(interp);
return PyLong_FromLongLong(id);
} }
PyDoc_STRVAR(get_main_doc, PyDoc_STRVAR(get_main_doc,
"get_main() -> ID\n\ "get_main() -> (ID,)\n\
\n\ \n\
Return the ID of main interpreter."); Return the ID of main interpreter.");
@ -1194,6 +1151,32 @@ PyDoc_STRVAR(get_config_doc,
Return a representation of the config used to initialize the interpreter."); Return a representation of the config used to initialize the interpreter.");
static PyObject *
interp_whence(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"id", NULL};
PyObject *id;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O:whence", kwlist, &id))
{
return NULL;
}
PyInterpreterState *interp = look_up_interp(id);
if (interp == NULL) {
return NULL;
}
long whence = _PyInterpreterState_GetWhence(interp);
return PyLong_FromLong(whence);
}
PyDoc_STRVAR(whence_doc,
"whence(id) -> int\n\
\n\
Return an identifier for where the interpreter was created.");
static PyObject * static PyObject *
interp_incref(PyObject *self, PyObject *args, PyObject *kwds) interp_incref(PyObject *self, PyObject *args, PyObject *kwds)
{ {
@ -1242,9 +1225,78 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds)
} }
static PyObject *
capture_exception(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"exc", NULL};
PyObject *exc_arg = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"|O:capture_exception", kwlist,
&exc_arg))
{
return NULL;
}
PyObject *exc = exc_arg;
if (exc == NULL || exc == Py_None) {
exc = PyErr_GetRaisedException();
if (exc == NULL) {
Py_RETURN_NONE;
}
}
else if (!PyExceptionInstance_Check(exc)) {
PyErr_Format(PyExc_TypeError, "expected exception, got %R", exc);
return NULL;
}
PyObject *captured = NULL;
_PyXI_excinfo info = {0};
if (_PyXI_InitExcInfo(&info, exc) < 0) {
goto finally;
}
captured = _PyXI_ExcInfoAsObject(&info);
if (captured == NULL) {
goto finally;
}
PyObject *formatted = _PyXI_FormatExcInfo(&info);
if (formatted == NULL) {
Py_CLEAR(captured);
goto finally;
}
int res = PyObject_SetAttrString(captured, "formatted", formatted);
Py_DECREF(formatted);
if (res < 0) {
Py_CLEAR(captured);
goto finally;
}
finally:
_PyXI_ClearExcInfo(&info);
if (exc != exc_arg) {
if (PyErr_Occurred()) {
PyErr_SetRaisedException(exc);
}
else {
_PyErr_ChainExceptions1(exc);
}
}
return captured;
}
PyDoc_STRVAR(capture_exception_doc,
"capture_exception(exc=None) -> types.SimpleNamespace\n\
\n\
Return a snapshot of an exception. If \"exc\" is None\n\
then the current exception, if any, is used (but not cleared).\n\
\n\
The returned snapshot is the same as what _interpreters.exec() returns.");
static PyMethodDef module_functions[] = { static PyMethodDef module_functions[] = {
{"new_config", _PyCFunction_CAST(interp_new_config), {"new_config", _PyCFunction_CAST(interp_new_config),
METH_VARARGS | METH_KEYWORDS, new_config_doc}, METH_VARARGS | METH_KEYWORDS, new_config_doc},
{"create", _PyCFunction_CAST(interp_create), {"create", _PyCFunction_CAST(interp_create),
METH_VARARGS | METH_KEYWORDS, create_doc}, METH_VARARGS | METH_KEYWORDS, create_doc},
{"destroy", _PyCFunction_CAST(interp_destroy), {"destroy", _PyCFunction_CAST(interp_destroy),
@ -1260,6 +1312,8 @@ static PyMethodDef module_functions[] = {
METH_VARARGS | METH_KEYWORDS, is_running_doc}, METH_VARARGS | METH_KEYWORDS, is_running_doc},
{"get_config", _PyCFunction_CAST(interp_get_config), {"get_config", _PyCFunction_CAST(interp_get_config),
METH_VARARGS | METH_KEYWORDS, get_config_doc}, METH_VARARGS | METH_KEYWORDS, get_config_doc},
{"whence", _PyCFunction_CAST(interp_whence),
METH_VARARGS | METH_KEYWORDS, whence_doc},
{"exec", _PyCFunction_CAST(interp_exec), {"exec", _PyCFunction_CAST(interp_exec),
METH_VARARGS | METH_KEYWORDS, exec_doc}, METH_VARARGS | METH_KEYWORDS, exec_doc},
{"call", _PyCFunction_CAST(interp_call), {"call", _PyCFunction_CAST(interp_call),
@ -1271,14 +1325,18 @@ static PyMethodDef module_functions[] = {
{"set___main___attrs", _PyCFunction_CAST(interp_set___main___attrs), {"set___main___attrs", _PyCFunction_CAST(interp_set___main___attrs),
METH_VARARGS, set___main___attrs_doc}, METH_VARARGS, set___main___attrs_doc},
{"is_shareable", _PyCFunction_CAST(object_is_shareable),
METH_VARARGS | METH_KEYWORDS, is_shareable_doc},
{"incref", _PyCFunction_CAST(interp_incref), {"incref", _PyCFunction_CAST(interp_incref),
METH_VARARGS | METH_KEYWORDS, NULL}, METH_VARARGS | METH_KEYWORDS, NULL},
{"decref", _PyCFunction_CAST(interp_decref), {"decref", _PyCFunction_CAST(interp_decref),
METH_VARARGS | METH_KEYWORDS, NULL}, METH_VARARGS | METH_KEYWORDS, NULL},
{"is_shareable", _PyCFunction_CAST(object_is_shareable),
METH_VARARGS | METH_KEYWORDS, is_shareable_doc},
{"capture_exception", _PyCFunction_CAST(capture_exception),
METH_VARARGS | METH_KEYWORDS, capture_exception_doc},
{NULL, NULL} /* sentinel */ {NULL, NULL} /* sentinel */
}; };
@ -1295,6 +1353,19 @@ module_exec(PyObject *mod)
PyInterpreterState *interp = PyInterpreterState_Get(); PyInterpreterState *interp = PyInterpreterState_Get();
module_state *state = get_module_state(mod); module_state *state = get_module_state(mod);
#define ADD_WHENCE(NAME) \
if (PyModule_AddIntConstant(mod, "WHENCE_" #NAME, \
_PyInterpreterState_WHENCE_##NAME) < 0) \
{ \
goto error; \
}
ADD_WHENCE(UNKNOWN)
ADD_WHENCE(RUNTIME)
ADD_WHENCE(LEGACY_CAPI)
ADD_WHENCE(CAPI)
ADD_WHENCE(XI)
#undef ADD_WHENCE
// exceptions // exceptions
if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterError) < 0) { if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterError) < 0) {
goto error; goto error;

233
Python/crossinterp.c
View File

@ -468,7 +468,7 @@ _release_xid_data(_PyCrossInterpreterData *data, int rawfree)
/***********************/ /***********************/
static int static int
_excinfo_init_type(struct _excinfo_type *info, PyObject *exc) _excinfo_init_type_from_exception(struct _excinfo_type *info, PyObject *exc)
{ {
/* Note that this copies directly rather than into an intermediate /* Note that this copies directly rather than into an intermediate
struct and does not clear on error. If we need that then we struct and does not clear on error. If we need that then we
@ -504,7 +504,7 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc)
} }
info->qualname = _copy_string_obj_raw(strobj, NULL); info->qualname = _copy_string_obj_raw(strobj, NULL);
Py_DECREF(strobj); Py_DECREF(strobj);
if (info->name == NULL) { if (info->qualname == NULL) {
return -1; return -1;
} }
@ -515,10 +515,51 @@ _excinfo_init_type(struct _excinfo_type *info, PyObject *exc)
} }
info->module = _copy_string_obj_raw(strobj, NULL); info->module = _copy_string_obj_raw(strobj, NULL);
Py_DECREF(strobj); Py_DECREF(strobj);
if (info->module == NULL) {
return -1;
}
return 0;
}
static int
_excinfo_init_type_from_object(struct _excinfo_type *info, PyObject *exctype)
{
PyObject *strobj = NULL;
// __name__
strobj = PyObject_GetAttrString(exctype, "__name__");
if (strobj == NULL) {
return -1;
}
info->name = _copy_string_obj_raw(strobj, NULL);
Py_DECREF(strobj);
if (info->name == NULL) { if (info->name == NULL) {
return -1; return -1;
} }
// __qualname__
strobj = PyObject_GetAttrString(exctype, "__qualname__");
if (strobj == NULL) {
return -1;
}
info->qualname = _copy_string_obj_raw(strobj, NULL);
Py_DECREF(strobj);
if (info->qualname == NULL) {
return -1;
}
// __module__
strobj = PyObject_GetAttrString(exctype, "__module__");
if (strobj == NULL) {
return -1;
}
info->module = _copy_string_obj_raw(strobj, NULL);
Py_DECREF(strobj);
if (info->module == NULL) {
return -1;
}
return 0; return 0;
} }
@ -584,7 +625,7 @@ _PyXI_excinfo_Clear(_PyXI_excinfo *info)
*info = (_PyXI_excinfo){{NULL}}; *info = (_PyXI_excinfo){{NULL}};
} }
static PyObject * PyObject *
_PyXI_excinfo_format(_PyXI_excinfo *info) _PyXI_excinfo_format(_PyXI_excinfo *info)
{ {
const char *module, *qualname; const char *module, *qualname;
@ -627,7 +668,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc)
} }
const char *failure = NULL; const char *failure = NULL;
if (_excinfo_init_type(&info->type, exc) < 0) { if (_excinfo_init_type_from_exception(&info->type, exc) < 0) {
failure = "error while initializing exception type snapshot"; failure = "error while initializing exception type snapshot";
goto error; goto error;
} }
@ -672,6 +713,57 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc)
return failure; return failure;
} }
static const char *
_PyXI_excinfo_InitFromObject(_PyXI_excinfo *info, PyObject *obj)
{
const char *failure = NULL;
PyObject *exctype = PyObject_GetAttrString(obj, "type");
if (exctype == NULL) {
failure = "exception snapshot missing 'type' attribute";
goto error;
}
int res = _excinfo_init_type_from_object(&info->type, exctype);
Py_DECREF(exctype);
if (res < 0) {
failure = "error while initializing exception type snapshot";
goto error;
}
// Extract the exception message.
PyObject *msgobj = PyObject_GetAttrString(obj, "msg");
if (msgobj == NULL) {
failure = "exception snapshot missing 'msg' attribute";
goto error;
}
info->msg = _copy_string_obj_raw(msgobj, NULL);
Py_DECREF(msgobj);
if (info->msg == NULL) {
failure = "error while copying exception message";
goto error;
}
// Pickle a traceback.TracebackException.
PyObject *errdisplay = PyObject_GetAttrString(obj, "errdisplay");
if (errdisplay == NULL) {
failure = "exception snapshot missing 'errdisplay' attribute";
goto error;
}
info->errdisplay = _copy_string_obj_raw(errdisplay, NULL);
Py_DECREF(errdisplay);
if (info->errdisplay == NULL) {
failure = "error while copying exception error display";
goto error;
}
return NULL;
error:
assert(failure != NULL);
_PyXI_excinfo_Clear(info);
return failure;
}
static void static void
_PyXI_excinfo_Apply(_PyXI_excinfo *info, PyObject *exctype) _PyXI_excinfo_Apply(_PyXI_excinfo *info, PyObject *exctype)
{ {
@ -825,6 +917,47 @@ _PyXI_excinfo_AsObject(_PyXI_excinfo *info)
} }
int
_PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc)
{
assert(!PyErr_Occurred());
if (exc == NULL || exc == Py_None) {
PyErr_SetString(PyExc_ValueError, "missing exc");
return -1;
}
const char *failure;
if (PyExceptionInstance_Check(exc) || PyExceptionClass_Check(exc)) {
failure = _PyXI_excinfo_InitFromException(info, exc);
}
else {
failure = _PyXI_excinfo_InitFromObject(info, exc);
}
if (failure != NULL) {
PyErr_SetString(PyExc_Exception, failure);
return -1;
}
return 0;
}
PyObject *
_PyXI_FormatExcInfo(_PyXI_excinfo *info)
{
return _PyXI_excinfo_format(info);
}
PyObject *
_PyXI_ExcInfoAsObject(_PyXI_excinfo *info)
{
return _PyXI_excinfo_AsObject(info);
}
void
_PyXI_ClearExcInfo(_PyXI_excinfo *info)
{
_PyXI_excinfo_Clear(info);
}
/***************************/ /***************************/
/* short-term data sharing */ /* short-term data sharing */
/***************************/ /***************************/
@ -1682,3 +1815,95 @@ _PyXI_FiniTypes(PyInterpreterState *interp)
{ {
fini_exceptions(interp); fini_exceptions(interp);
} }
/*************/
/* other API */
/*************/
PyInterpreterState *
_PyXI_NewInterpreter(PyInterpreterConfig *config,
PyThreadState **p_tstate, PyThreadState **p_save_tstate)
{
PyThreadState *save_tstate = PyThreadState_Swap(NULL);
assert(save_tstate != NULL);
PyThreadState *tstate;
PyStatus status = Py_NewInterpreterFromConfig(&tstate, config);
if (PyStatus_Exception(status)) {
// Since no new thread state was created, there is no exception
// to propagate; raise a fresh one after swapping back in the
// old thread state.
PyThreadState_Swap(save_tstate);
_PyErr_SetFromPyStatus(status);
PyObject *exc = PyErr_GetRaisedException();
PyErr_SetString(PyExc_InterpreterError,
"sub-interpreter creation failed");
_PyErr_ChainExceptions1(exc);
return NULL;
}
assert(tstate != NULL);
PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate);
_PyInterpreterState_SetWhence(interp, _PyInterpreterState_WHENCE_XI);
if (p_tstate != NULL) {
// We leave the new thread state as the current one.
*p_tstate = tstate;
}
else {
// Throw away the initial tstate.
PyThreadState_Clear(tstate);
PyThreadState_Swap(save_tstate);
PyThreadState_Delete(tstate);
save_tstate = NULL;
}
if (p_save_tstate != NULL) {
*p_save_tstate = save_tstate;
}
return interp;
}
void
_PyXI_EndInterpreter(PyInterpreterState *interp,
PyThreadState *tstate, PyThreadState **p_save_tstate)
{
PyThreadState *save_tstate = NULL;
PyThreadState *cur_tstate = PyThreadState_GET();
if (tstate == NULL) {
if (PyThreadState_GetInterpreter(cur_tstate) == interp) {
tstate = cur_tstate;
}
else {
tstate = PyThreadState_New(interp);
_PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_INTERP);
assert(tstate != NULL);
save_tstate = PyThreadState_Swap(tstate);
}
}
else {
assert(PyThreadState_GetInterpreter(tstate) == interp);
if (tstate != cur_tstate) {
assert(PyThreadState_GetInterpreter(cur_tstate) != interp);
save_tstate = PyThreadState_Swap(tstate);
}
}
long whence = _PyInterpreterState_GetWhence(interp);
assert(whence != _PyInterpreterState_WHENCE_RUNTIME);
if (whence == _PyInterpreterState_WHENCE_UNKNOWN) {
assert(!interp->_ready);
PyThreadState *tstate = PyThreadState_New(interp);
save_tstate = PyThreadState_Swap(tstate);
_PyInterpreterState_Clear(tstate);
PyInterpreterState_Delete(interp);
}
else {
Py_EndInterpreter(tstate);
}
if (p_save_tstate != NULL) {
save_tstate = *p_save_tstate;
}
PyThreadState_Swap(save_tstate);
}

12
Python/crossinterp_exceptions.h
View File

@ -6,9 +6,9 @@ static PyTypeObject _PyExc_InterpreterError = {
.tp_name = "interpreters.InterpreterError", .tp_name = "interpreters.InterpreterError",
.tp_doc = PyDoc_STR("A cross-interpreter operation failed"), .tp_doc = PyDoc_STR("A cross-interpreter operation failed"),
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
//.tp_traverse = ((PyTypeObject *)PyExc_BaseException)->tp_traverse, //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse,
//.tp_clear = ((PyTypeObject *)PyExc_BaseException)->tp_clear, //.tp_clear = ((PyTypeObject *)PyExc_Exception)->tp_clear,
//.tp_base = (PyTypeObject *)PyExc_BaseException, //.tp_base = (PyTypeObject *)PyExc_Exception,
}; };
PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError;
@ -19,8 +19,8 @@ static PyTypeObject _PyExc_InterpreterNotFoundError = {
.tp_name = "interpreters.InterpreterNotFoundError", .tp_name = "interpreters.InterpreterNotFoundError",
.tp_doc = PyDoc_STR("An interpreter was not found"), .tp_doc = PyDoc_STR("An interpreter was not found"),
.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
//.tp_traverse = ((PyTypeObject *)PyExc_BaseException)->tp_traverse, //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse,
//.tp_clear = ((PyTypeObject *)PyExc_BaseException)->tp_clear, //.tp_clear = ((PyTypeObject *)PyExc_Exception)->tp_clear,
.tp_base = &_PyExc_InterpreterError, .tp_base = &_PyExc_InterpreterError,
}; };
PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFoundError; PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFoundError;
@ -61,7 +61,7 @@ _get_not_shareable_error_type(PyInterpreterState *interp)
static int static int
init_exceptions(PyInterpreterState *interp) init_exceptions(PyInterpreterState *interp)
{ {
PyTypeObject *base = (PyTypeObject *)PyExc_BaseException; PyTypeObject *base = (PyTypeObject *)PyExc_Exception;
// builtin static types // builtin static types

14
Python/pylifecycle.c
View File

@ -477,6 +477,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime,
if (interp == NULL) { if (interp == NULL) {
return _PyStatus_ERR("can't make main interpreter"); return _PyStatus_ERR("can't make main interpreter");
} }
assert(interp->_ready);
status = _PyConfig_Write(config, runtime); status = _PyConfig_Write(config, runtime);
if (_PyStatus_EXCEPTION(status)) { if (_PyStatus_EXCEPTION(status)) {
@ -631,6 +632,8 @@ pycore_create_interpreter(_PyRuntimeState *runtime,
} }
assert(interp != NULL); assert(interp != NULL);
assert(_Py_IsMainInterpreter(interp)); assert(_Py_IsMainInterpreter(interp));
_PyInterpreterState_SetWhence(interp, _PyInterpreterState_WHENCE_RUNTIME);
interp->_ready = 1;
status = _PyConfig_Copy(&interp->config, src_config); status = _PyConfig_Copy(&interp->config, src_config);
if (_PyStatus_EXCEPTION(status)) { if (_PyStatus_EXCEPTION(status)) {
@ -2120,7 +2123,8 @@ Py_Finalize(void)
*/ */
static PyStatus static PyStatus
new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) new_interpreter(PyThreadState **tstate_p,
const PyInterpreterConfig *config, long whence)
{ {
PyStatus status; PyStatus status;
@ -2143,6 +2147,8 @@ new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config)
*tstate_p = NULL; *tstate_p = NULL;
return _PyStatus_OK(); return _PyStatus_OK();
} }
_PyInterpreterState_SetWhence(interp, whence);
interp->_ready = 1;
// XXX Might new_interpreter() have been called without the GIL held? // XXX Might new_interpreter() have been called without the GIL held?
PyThreadState *save_tstate = _PyThreadState_GET(); PyThreadState *save_tstate = _PyThreadState_GET();
@ -2231,15 +2237,17 @@ PyStatus
Py_NewInterpreterFromConfig(PyThreadState **tstate_p, Py_NewInterpreterFromConfig(PyThreadState **tstate_p,
const PyInterpreterConfig *config) const PyInterpreterConfig *config)
{ {
return new_interpreter(tstate_p, config); long whence = _PyInterpreterState_WHENCE_CAPI;
return new_interpreter(tstate_p, config, whence);
} }
PyThreadState * PyThreadState *
Py_NewInterpreter(void) Py_NewInterpreter(void)
{ {
PyThreadState *tstate = NULL; PyThreadState *tstate = NULL;
long whence = _PyInterpreterState_WHENCE_LEGACY_CAPI;
const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT;
PyStatus status = new_interpreter(&tstate, &config); PyStatus status = new_interpreter(&tstate, &config, whence);
if (_PyStatus_EXCEPTION(status)) { if (_PyStatus_EXCEPTION(status)) {
Py_ExitStatusException(status); Py_ExitStatusException(status);
} }

55
Python/pystate.c
View File

@ -583,6 +583,8 @@ free_interpreter(PyInterpreterState *interp)
} }
} }
static inline int check_interpreter_whence(long);
/* Get the interpreter state to a minimal consistent state. /* Get the interpreter state to a minimal consistent state.
Further init happens in pylifecycle.c before it can be used. Further init happens in pylifecycle.c before it can be used.
All fields not initialized here are expected to be zeroed out, All fields not initialized here are expected to be zeroed out,
@ -605,12 +607,17 @@ free_interpreter(PyInterpreterState *interp)
static PyStatus static PyStatus
init_interpreter(PyInterpreterState *interp, init_interpreter(PyInterpreterState *interp,
_PyRuntimeState *runtime, int64_t id, _PyRuntimeState *runtime, int64_t id,
PyInterpreterState *next) PyInterpreterState *next,
long whence)
{ {
if (interp->_initialized) { if (interp->_initialized) {
return _PyStatus_ERR("interpreter already initialized"); return _PyStatus_ERR("interpreter already initialized");
} }
assert(interp->_whence == _PyInterpreterState_WHENCE_NOTSET);
assert(check_interpreter_whence(whence) == 0);
interp->_whence = whence;
assert(runtime != NULL); assert(runtime != NULL);
interp->runtime = runtime; interp->runtime = runtime;
@ -718,8 +725,9 @@ _PyInterpreterState_New(PyThreadState *tstate, PyInterpreterState **pinterp)
} }
interpreters->head = interp; interpreters->head = interp;
long whence = _PyInterpreterState_WHENCE_UNKNOWN;
status = init_interpreter(interp, runtime, status = init_interpreter(interp, runtime,
id, old_head); id, old_head, whence);
if (_PyStatus_EXCEPTION(status)) { if (_PyStatus_EXCEPTION(status)) {
goto error; goto error;
} }
@ -1103,6 +1111,34 @@ _PyInterpreterState_ReinitRunningMain(PyThreadState *tstate)
// accessors // accessors
//---------- //----------
static inline int
check_interpreter_whence(long whence)
{
if(whence < 0) {
return -1;
}
if (whence > _PyInterpreterState_WHENCE_MAX) {
return -1;
}
return 0;
}
long
_PyInterpreterState_GetWhence(PyInterpreterState *interp)
{
assert(check_interpreter_whence(interp->_whence) == 0);
return interp->_whence;
}
void
_PyInterpreterState_SetWhence(PyInterpreterState *interp, long whence)
{
assert(interp->_whence != _PyInterpreterState_WHENCE_NOTSET);
assert(check_interpreter_whence(whence) == 0);
interp->_whence = whence;
}
PyObject * PyObject *
PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp) PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp)
{ {
@ -1114,6 +1150,7 @@ PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp)
return PyMapping_GetItemString(modules, "__main__"); return PyMapping_GetItemString(modules, "__main__");
} }
PyObject * PyObject *
PyInterpreterState_GetDict(PyInterpreterState *interp) PyInterpreterState_GetDict(PyInterpreterState *interp)
{ {
@ -1176,6 +1213,20 @@ PyInterpreterState_GetID(PyInterpreterState *interp)
return interp->id; return interp->id;
} }
PyObject *
_PyInterpreterState_GetIDObject(PyInterpreterState *interp)
{
if (_PyInterpreterState_IDInitref(interp) != 0) {
return NULL;
};
int64_t interpid = interp->id;
if (interpid < 0) {
return NULL;
}
assert(interpid < LLONG_MAX);
return PyLong_FromLongLong(interpid);
}
int int
_PyInterpreterState_IDInitref(PyInterpreterState *interp) _PyInterpreterState_IDInitref(PyInterpreterState *interp)