GH-118093: Add tier two support for LOAD_ATTR_PROPERTY (GH-122283)

This commit is contained in:
Brandt Bucher 2024-07-25 10:45:28 -07:00 committed by GitHub
parent 5e686ff57d
commit 5f6001130f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 168 additions and 99 deletions

View File

@ -733,7 +733,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) {
case LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES:
return 1;
case LOAD_ATTR_PROPERTY:
return 1;
return 0;
case LOAD_ATTR_SLOT:
return 1 + (oparg & 1);
case LOAD_ATTR_WITH_HINT:
@ -1109,7 +1109,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = {
[LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG },
[LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
@ -1305,6 +1305,7 @@ _PyOpcode_macro_expansion[256] = {
[LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } },
[LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } },
[LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } },
[LOAD_ATTR_PROPERTY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_PROPERTY_FRAME, 4, 5 }, { _SAVE_RETURN_OFFSET, 7, 9 }, { _PUSH_FRAME, 0, 0 } } },
[LOAD_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 } } },
[LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } },
[LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, 0, 0 } } },

View File

@ -182,36 +182,36 @@ extern "C" {
#define _LOAD_ATTR_MODULE 402
#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 403
#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 404
#define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY
#define _LOAD_ATTR_SLOT 405
#define _LOAD_ATTR_SLOT_0 406
#define _LOAD_ATTR_SLOT_1 407
#define _LOAD_ATTR_WITH_HINT 408
#define _LOAD_ATTR_PROPERTY_FRAME 405
#define _LOAD_ATTR_SLOT 406
#define _LOAD_ATTR_SLOT_0 407
#define _LOAD_ATTR_SLOT_1 408
#define _LOAD_ATTR_WITH_HINT 409
#define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS
#define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT
#define _LOAD_CONST LOAD_CONST
#define _LOAD_CONST_INLINE 409
#define _LOAD_CONST_INLINE_BORROW 410
#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 411
#define _LOAD_CONST_INLINE_WITH_NULL 412
#define _LOAD_CONST_INLINE 410
#define _LOAD_CONST_INLINE_BORROW 411
#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 412
#define _LOAD_CONST_INLINE_WITH_NULL 413
#define _LOAD_DEREF LOAD_DEREF
#define _LOAD_FAST 413
#define _LOAD_FAST_0 414
#define _LOAD_FAST_1 415
#define _LOAD_FAST_2 416
#define _LOAD_FAST_3 417
#define _LOAD_FAST_4 418
#define _LOAD_FAST_5 419
#define _LOAD_FAST_6 420
#define _LOAD_FAST_7 421
#define _LOAD_FAST 414
#define _LOAD_FAST_0 415
#define _LOAD_FAST_1 416
#define _LOAD_FAST_2 417
#define _LOAD_FAST_3 418
#define _LOAD_FAST_4 419
#define _LOAD_FAST_5 420
#define _LOAD_FAST_6 421
#define _LOAD_FAST_7 422
#define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR
#define _LOAD_FAST_CHECK LOAD_FAST_CHECK
#define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST
#define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF
#define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS
#define _LOAD_GLOBAL 422
#define _LOAD_GLOBAL_BUILTINS 423
#define _LOAD_GLOBAL_MODULE 424
#define _LOAD_GLOBAL 423
#define _LOAD_GLOBAL_BUILTINS 424
#define _LOAD_GLOBAL_MODULE 425
#define _LOAD_LOCALS LOAD_LOCALS
#define _LOAD_NAME LOAD_NAME
#define _LOAD_SPECIAL LOAD_SPECIAL
@ -226,51 +226,51 @@ extern "C" {
#define _MATCH_SEQUENCE MATCH_SEQUENCE
#define _NOP NOP
#define _POP_EXCEPT POP_EXCEPT
#define _POP_JUMP_IF_FALSE 425
#define _POP_JUMP_IF_TRUE 426
#define _POP_JUMP_IF_FALSE 426
#define _POP_JUMP_IF_TRUE 427
#define _POP_TOP POP_TOP
#define _POP_TOP_LOAD_CONST_INLINE_BORROW 427
#define _POP_TOP_LOAD_CONST_INLINE_BORROW 428
#define _PUSH_EXC_INFO PUSH_EXC_INFO
#define _PUSH_FRAME 428
#define _PUSH_FRAME 429
#define _PUSH_NULL PUSH_NULL
#define _PY_FRAME_GENERAL 429
#define _REPLACE_WITH_TRUE 430
#define _PY_FRAME_GENERAL 430
#define _REPLACE_WITH_TRUE 431
#define _RESUME_CHECK RESUME_CHECK
#define _RETURN_GENERATOR RETURN_GENERATOR
#define _RETURN_VALUE RETURN_VALUE
#define _SAVE_RETURN_OFFSET 431
#define _SEND 432
#define _SEND_GEN_FRAME 433
#define _SAVE_RETURN_OFFSET 432
#define _SEND 433
#define _SEND_GEN_FRAME 434
#define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS
#define _SET_ADD SET_ADD
#define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE
#define _SET_UPDATE SET_UPDATE
#define _START_EXECUTOR 434
#define _STORE_ATTR 435
#define _STORE_ATTR_INSTANCE_VALUE 436
#define _STORE_ATTR_SLOT 437
#define _STORE_ATTR_WITH_HINT 438
#define _START_EXECUTOR 435
#define _STORE_ATTR 436
#define _STORE_ATTR_INSTANCE_VALUE 437
#define _STORE_ATTR_SLOT 438
#define _STORE_ATTR_WITH_HINT 439
#define _STORE_DEREF STORE_DEREF
#define _STORE_FAST 439
#define _STORE_FAST_0 440
#define _STORE_FAST_1 441
#define _STORE_FAST_2 442
#define _STORE_FAST_3 443
#define _STORE_FAST_4 444
#define _STORE_FAST_5 445
#define _STORE_FAST_6 446
#define _STORE_FAST_7 447
#define _STORE_FAST 440
#define _STORE_FAST_0 441
#define _STORE_FAST_1 442
#define _STORE_FAST_2 443
#define _STORE_FAST_3 444
#define _STORE_FAST_4 445
#define _STORE_FAST_5 446
#define _STORE_FAST_6 447
#define _STORE_FAST_7 448
#define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST
#define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST
#define _STORE_GLOBAL STORE_GLOBAL
#define _STORE_NAME STORE_NAME
#define _STORE_SLICE STORE_SLICE
#define _STORE_SUBSCR 448
#define _STORE_SUBSCR 449
#define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT
#define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT
#define _SWAP SWAP
#define _TIER2_RESUME_CHECK 449
#define _TO_BOOL 450
#define _TIER2_RESUME_CHECK 450
#define _TO_BOOL 451
#define _TO_BOOL_BOOL TO_BOOL_BOOL
#define _TO_BOOL_INT TO_BOOL_INT
#define _TO_BOOL_LIST TO_BOOL_LIST
@ -280,13 +280,13 @@ extern "C" {
#define _UNARY_NEGATIVE UNARY_NEGATIVE
#define _UNARY_NOT UNARY_NOT
#define _UNPACK_EX UNPACK_EX
#define _UNPACK_SEQUENCE 451
#define _UNPACK_SEQUENCE 452
#define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST
#define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE
#define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE
#define _WITH_EXCEPT_START WITH_EXCEPT_START
#define _YIELD_VALUE YIELD_VALUE
#define MAX_UOP_ID 451
#define MAX_UOP_ID 452
#ifdef __cplusplus
}

View File

@ -151,6 +151,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_LOAD_ATTR_CLASS_0] = 0,
[_LOAD_ATTR_CLASS_1] = 0,
[_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG,
[_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG,
[_GUARD_DORV_NO_DICT] = HAS_DEOPT_FLAG,
[_STORE_ATTR_INSTANCE_VALUE] = 0,
[_STORE_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG,
@ -420,6 +421,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
[_LOAD_ATTR_MODULE] = "_LOAD_ATTR_MODULE",
[_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT",
[_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES",
[_LOAD_ATTR_PROPERTY_FRAME] = "_LOAD_ATTR_PROPERTY_FRAME",
[_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT",
[_LOAD_ATTR_SLOT_0] = "_LOAD_ATTR_SLOT_0",
[_LOAD_ATTR_SLOT_1] = "_LOAD_ATTR_SLOT_1",
@ -788,6 +790,8 @@ int _PyUop_num_popped(int opcode, int oparg)
return 1;
case _LOAD_ATTR_CLASS:
return 1;
case _LOAD_ATTR_PROPERTY_FRAME:
return 1;
case _GUARD_DORV_NO_DICT:
return 1;
case _STORE_ATTR_INSTANCE_VALUE:

View File

@ -2243,32 +2243,30 @@ dummy_func(
unused/2 +
_LOAD_ATTR_CLASS;
inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused, unused if (0))) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
op(_LOAD_ATTR_PROPERTY_FRAME, (fget/4, owner -- new_frame: _PyInterpreterFrame *)) {
assert((oparg & 1) == 0);
DEOPT_IF(tstate->interp->eval_frame);
PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
DEOPT_IF(cls->tp_version_tag != type_version);
assert(Py_IS_TYPE(fget, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)fget;
assert(func_version != 0);
DEOPT_IF(f->func_version != func_version);
PyCodeObject *code = (PyCodeObject *)f->func_code;
assert(code->co_argcount == 1);
DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED);
DEOPT_IF(code->co_kwonlyargcount);
DEOPT_IF(code->co_argcount != 1);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize));
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(fget);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1);
// Manipulate stack directly because we exit with DISPATCH_INLINED().
STACK_SHRINK(1);
new_frame = _PyFrame_PushUnchecked(tstate, f, 1);
new_frame->localsplus[0] = owner;
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
}
macro(LOAD_ATTR_PROPERTY) =
unused/1 +
_CHECK_PEP_523 +
_GUARD_TYPE_VERSION +
unused/2 +
_LOAD_ATTR_PROPERTY_FRAME +
_SAVE_RETURN_OFFSET +
_PUSH_FRAME;
inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused, unused if (0))) {
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);

View File

@ -2507,7 +2507,39 @@
/* _LOAD_ATTR_CLASS is split on (oparg & 1) */
/* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _LOAD_ATTR_PROPERTY_FRAME: {
_PyStackRef owner;
_PyInterpreterFrame *new_frame;
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
PyObject *fget = (PyObject *)CURRENT_OPERAND();
assert((oparg & 1) == 0);
assert(Py_IS_TYPE(fget, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)fget;
PyCodeObject *code = (PyCodeObject *)f->func_code;
if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (code->co_kwonlyargcount) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (code->co_argcount != 1) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) {
UOP_STAT_INC(uopcode, miss);
JUMP_TO_JUMP_TARGET();
}
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(fget);
new_frame = _PyFrame_PushUnchecked(tstate, f, 1);
new_frame->localsplus[0] = owner;
stack_pointer[-1].bits = (uintptr_t)new_frame;
break;
}
/* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */

View File

@ -4429,32 +4429,63 @@
INSTRUCTION_STATS(LOAD_ATTR_PROPERTY);
static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
_PyStackRef owner;
_PyInterpreterFrame *new_frame;
/* Skip 1 cache entry */
// _CHECK_PEP_523
{
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
}
// _GUARD_TYPE_VERSION
owner = stack_pointer[-1];
uint32_t type_version = read_u32(&this_instr[2].cache);
uint32_t func_version = read_u32(&this_instr[4].cache);
PyObject *fget = read_obj(&this_instr[6].cache);
PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner);
assert((oparg & 1) == 0);
DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR);
PyTypeObject *cls = Py_TYPE(owner_o);
assert(type_version != 0);
DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR);
assert(Py_IS_TYPE(fget, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)fget;
assert(func_version != 0);
DEOPT_IF(f->func_version != func_version, LOAD_ATTR);
PyCodeObject *code = (PyCodeObject *)f->func_code;
assert(code->co_argcount == 1);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(fget);
_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1);
// Manipulate stack directly because we exit with DISPATCH_INLINED().
STACK_SHRINK(1);
new_frame->localsplus[0] = owner;
frame->return_offset = (uint16_t)(next_instr - this_instr);
DISPATCH_INLINED(new_frame);
{
uint32_t type_version = read_u32(&this_instr[2].cache);
PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner));
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR);
}
/* Skip 2 cache entries */
// _LOAD_ATTR_PROPERTY_FRAME
{
PyObject *fget = read_obj(&this_instr[6].cache);
assert((oparg & 1) == 0);
assert(Py_IS_TYPE(fget, &PyFunction_Type));
PyFunctionObject *f = (PyFunctionObject *)fget;
PyCodeObject *code = (PyCodeObject *)f->func_code;
DEOPT_IF((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED, LOAD_ATTR);
DEOPT_IF(code->co_kwonlyargcount, LOAD_ATTR);
DEOPT_IF(code->co_argcount != 1, LOAD_ATTR);
DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(fget);
new_frame = _PyFrame_PushUnchecked(tstate, f, 1);
new_frame->localsplus[0] = owner;
}
// _SAVE_RETURN_OFFSET
{
#if TIER_ONE
frame->return_offset = (uint16_t)(next_instr - this_instr);
#endif
#if TIER_TWO
frame->return_offset = oparg;
#endif
}
// _PUSH_FRAME
{
// Write it out explicitly because it's subtly different.
// Eventually this should be the only occurrence of this code.
assert(tstate->interp->eval_frame == NULL);
stack_pointer += -1;
assert(WITHIN_STACK_BOUNDS());
_PyFrame_SetStackPointer(frame, stack_pointer);
new_frame->previous = frame;
CALL_STAT_INC(inlined_py_calls);
frame = tstate->current_frame = new_frame;
tstate->py_recursion_remaining--;
LOAD_SP();
LOAD_IP(0);
LLTRACE_RESUME_FRAME();
}
DISPATCH();
}
TARGET(LOAD_ATTR_SLOT) {

View File

@ -797,7 +797,10 @@ top: // Jump here after _PUSH_FRAME or likely branches
if (uop == _PUSH_FRAME) {
assert(i + 1 == nuops);
if (opcode == FOR_ITER_GEN || opcode == SEND_GEN) {
if (opcode == FOR_ITER_GEN ||
opcode == LOAD_ATTR_PROPERTY ||
opcode == SEND_GEN)
{
DPRINTF(2, "Bailing due to dynamic target\n");
ADD_TO_TRACE(uop, oparg, 0, target);
ADD_TO_TRACE(_DYNAMIC_EXIT, 0, 0, 0);

View File

@ -1160,7 +1160,12 @@
break;
}
/* _LOAD_ATTR_PROPERTY is not a viable micro-op for tier 2 */
case _LOAD_ATTR_PROPERTY_FRAME: {
_PyInterpreterFrame *new_frame;
new_frame = sym_new_not_null(ctx);
stack_pointer[-1] = (_Py_UopsSymbol *)new_frame;
break;
}
/* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */

View File

@ -963,15 +963,10 @@ _Py_Specialize_LoadAttr(_PyStackRef owner_st, _Py_CODEUNIT *instr, PyObject *nam
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
goto fail;
}
uint32_t version = function_get_version(fget, LOAD_ATTR);
if (version == 0) {
goto fail;
}
if (_PyInterpreterState_GET()->eval_frame) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER);
goto fail;
}
write_u32(lm_cache->keys_version, version);
assert(type->tp_version_tag != 0);
write_u32(lm_cache->type_version, type->tp_version_tag);
/* borrowed */