Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(59)

Side by Side Diff: Objects/abstract.c

Issue 29259: Add tp_fastcall to PyTypeObject: support FASTCALL calling convention for all callable objects
Patch Set: Created 2 years, 12 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
OLDNEW
1 /* Abstract Object Interface (many thanks to Jim Fulton) */ 1 /* Abstract Object Interface (many thanks to Jim Fulton) */
2 2
3 #include "Python.h" 3 #include "Python.h"
4 #include <ctype.h> 4 #include <ctype.h>
5 #include "structmember.h" /* we need the offsetof() macro from there */ 5 #include "structmember.h" /* we need the offsetof() macro from there */
6 #include "longintrepr.h" 6 #include "longintrepr.h"
7 7
8 8
9 9
10 /* Shorthands to return certain errors */ 10 /* Shorthands to return certain errors */
(...skipping 2231 matching lines...) Expand 10 before | Expand all | Expand 10 after
2242 call = callable->ob_type->tp_call; 2242 call = callable->ob_type->tp_call;
2243 if (call == NULL) { 2243 if (call == NULL) {
2244 PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", 2244 PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable",
2245 callable->ob_type->tp_name); 2245 callable->ob_type->tp_name);
2246 return NULL; 2246 return NULL;
2247 } 2247 }
2248 2248
2249 if (Py_EnterRecursiveCall(" while calling a Python object")) 2249 if (Py_EnterRecursiveCall(" while calling a Python object"))
2250 return NULL; 2250 return NULL;
2251 2251
2252 result = (*call)(callable, args, kwargs); 2252 if (PyType_HasFeature(Py_TYPE(callable), Py_TPFLAGS_HAVE_FASTCALL)
2253 && Py_TYPE(callable)->tp_fastcall) {
2254 result = _Py_RawFastCallDict(callable,
2255 Py_TYPE(callable)->tp_fastcall,
2256 &PyTuple_GET_ITEM(args, 0),
2257 PyTuple_GET_SIZE(args),
2258 kwargs);
2259 }
2260 else {
2261 result = (*call)(callable, args, kwargs);
2262 }
2253 2263
2254 Py_LeaveRecursiveCall(); 2264 Py_LeaveRecursiveCall();
2255 2265
2256 return _Py_CheckFunctionResult(callable, result, NULL); 2266 return _Py_CheckFunctionResult(callable, result, NULL);
2257 } 2267 }
2258 2268
2259 /* Issue #29234: Inlining _PyStack_AsTuple() into callers increases their 2269 /* Issue #29234: Inlining _PyStack_AsTuple() into callers increases their
2260 stack consumption, Disable inlining to optimize the stack consumption. */ 2270 stack consumption, Disable inlining to optimize the stack consumption. */
2261 PyObject* _Py_NO_INLINE 2271 PyObject* _Py_NO_INLINE
2262 _PyStack_AsTuple(PyObject **stack, Py_ssize_t nargs) 2272 _PyStack_AsTuple(PyObject **stack, Py_ssize_t nargs)
2263 { 2273 {
2264 PyObject *args; 2274 PyObject *args;
2265 Py_ssize_t i; 2275 Py_ssize_t i;
2266 2276
2267 args = PyTuple_New(nargs); 2277 args = PyTuple_New(nargs);
2268 if (args == NULL) { 2278 if (args == NULL) {
2269 return NULL; 2279 return NULL;
2270 } 2280 }
2271 2281
2272 for (i=0; i < nargs; i++) { 2282 for (i=0; i < nargs; i++) {
2273 PyObject *item = stack[i]; 2283 PyObject *item = stack[i];
2274 Py_INCREF(item); 2284 Py_INCREF(item);
2275 PyTuple_SET_ITEM(args, i, item); 2285 PyTuple_SET_ITEM(args, i, item);
2276 } 2286 }
2277 return args; 2287 return args;
2288 }
2289
2290 int _Py_NO_INLINE
haypo 2017/01/26 03:35:21 Not sure if _Py_NO_INLINE is needed here.
2291 _PyStack_AsTupleAndDict(PyObject **stack, Py_ssize_t nargs, PyObject *kwnames,
2292 PyObject **p_args, PyObject **p_kwargs)
2293 {
2294 PyObject *args, *kwargs;
2295
2296 args = _PyStack_AsTuple(stack, nargs);
2297 if (args == NULL) {
2298 return -1;
2299 }
2300
2301 if (kwnames != NULL && PyTuple_GET_SIZE(kwnames) != 0) {
2302 kwargs = _PyStack_AsDict(stack + nargs, kwnames);
2303 if (kwargs == NULL) {
2304 Py_DECREF(args);
2305 return -1;
2306 }
2307 }
2308 else {
2309 kwargs = NULL;
2310 }
2311
2312 *p_args = args;
2313 *p_kwargs = kwargs;
2314 return 0;
2278 } 2315 }
2279 2316
2280 PyObject* 2317 PyObject*
2281 _PyStack_AsTupleSlice(PyObject **stack, Py_ssize_t nargs, 2318 _PyStack_AsTupleSlice(PyObject **stack, Py_ssize_t nargs,
2282 Py_ssize_t start, Py_ssize_t end) 2319 Py_ssize_t start, Py_ssize_t end)
2283 { 2320 {
2284 PyObject *args; 2321 PyObject *args;
2285 Py_ssize_t i; 2322 Py_ssize_t i;
2286 2323
2287 assert(0 <= start); 2324 assert(0 <= start);
(...skipping 32 matching lines...) Expand 10 before | Expand all | Expand 10 after
2320 2357
2321 if (Py_EnterRecursiveCall(" while calling a Python object")) { 2358 if (Py_EnterRecursiveCall(" while calling a Python object")) {
2322 return NULL; 2359 return NULL;
2323 } 2360 }
2324 2361
2325 if (PyFunction_Check(callable)) { 2362 if (PyFunction_Check(callable)) {
2326 result = _PyFunction_FastCallDict(callable, args, nargs, kwargs); 2363 result = _PyFunction_FastCallDict(callable, args, nargs, kwargs);
2327 } 2364 }
2328 else if (PyCFunction_Check(callable)) { 2365 else if (PyCFunction_Check(callable)) {
2329 result = _PyCFunction_FastCallDict(callable, args, nargs, kwargs); 2366 result = _PyCFunction_FastCallDict(callable, args, nargs, kwargs);
2367 }
2368 else if (PyType_HasFeature(Py_TYPE(callable), Py_TPFLAGS_HAVE_FASTCALL)
2369 && Py_TYPE(callable)->tp_fastcall) {
2370 fastternaryfunc fastcall = Py_TYPE(callable)->tp_fastcall;
2371
2372 result = _Py_RawFastCallDict(callable, fastcall, args, nargs, kwargs);
2373 result = _Py_CheckFunctionResult(callable, result, NULL);
2330 } 2374 }
2331 else { 2375 else {
2332 PyObject *tuple; 2376 PyObject *tuple;
2333 2377
2334 /* Slow-path: build a temporary tuple */ 2378 /* Slow-path: build a temporary tuple */
2335 call = callable->ob_type->tp_call; 2379 call = Py_TYPE(callable)->tp_call;
2336 if (call == NULL) { 2380 if (call == NULL) {
2337 PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", 2381 PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable",
2338 callable->ob_type->tp_name); 2382 callable->ob_type->tp_name);
2339 goto exit; 2383 goto exit;
2340 } 2384 }
2341 2385
2342 tuple = _PyStack_AsTuple(args, nargs); 2386 tuple = _PyStack_AsTuple(args, nargs);
2343 if (tuple == NULL) { 2387 if (tuple == NULL) {
2344 goto exit; 2388 goto exit;
2345 } 2389 }
(...skipping 11 matching lines...) Expand all
2357 } 2401 }
2358 2402
2359 /* Positional arguments are obj followed by args: 2403 /* Positional arguments are obj followed by args:
2360 call callable(obj, *args, **kwargs) */ 2404 call callable(obj, *args, **kwargs) */
2361 PyObject * 2405 PyObject *
2362 _PyObject_Call_Prepend(PyObject *callable, 2406 _PyObject_Call_Prepend(PyObject *callable,
2363 PyObject *obj, PyObject *args, PyObject *kwargs) 2407 PyObject *obj, PyObject *args, PyObject *kwargs)
2364 { 2408 {
2365 PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; 2409 PyObject *small_stack[_PY_FASTCALL_SMALL_STACK];
2366 PyObject **stack; 2410 PyObject **stack;
2367 Py_ssize_t argcount; 2411 Py_ssize_t nargs;
2368 PyObject *result; 2412 PyObject *result;
2369 2413
2370 assert(PyTuple_Check(args)); 2414 assert(PyTuple_Check(args));
2371 2415
2372 argcount = PyTuple_GET_SIZE(args); 2416 nargs = PyTuple_GET_SIZE(args);
2373 if (argcount + 1 <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { 2417 if (nargs + 1 <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) {
2374 stack = small_stack; 2418 stack = small_stack;
2375 } 2419 }
2376 else { 2420 else {
2377 stack = PyMem_Malloc((argcount + 1) * sizeof(PyObject *)); 2421 stack = PyMem_Malloc((nargs + 1) * sizeof(PyObject *));
2378 if (stack == NULL) { 2422 if (stack == NULL) {
2379 PyErr_NoMemory(); 2423 PyErr_NoMemory();
2380 return NULL; 2424 return NULL;
2381 } 2425 }
2382 } 2426 }
2383 2427
2384 /* use borrowed references */ 2428 /* use borrowed references */
2385 stack[0] = obj; 2429 stack[0] = obj;
2386 memcpy(&stack[1], 2430 memcpy(&stack[1],
2387 &PyTuple_GET_ITEM(args, 0), 2431 &PyTuple_GET_ITEM(args, 0),
2388 argcount * sizeof(PyObject *)); 2432 nargs * sizeof(PyObject *));
2389 2433
2390 result = _PyObject_FastCallDict(callable, 2434 result = _PyObject_FastCallDict(callable,
2391 stack, argcount + 1, 2435 stack, nargs + 1,
2392 kwargs); 2436 kwargs);
2393 if (stack != small_stack) { 2437 if (stack != small_stack) {
2394 PyMem_Free(stack); 2438 PyMem_Free(stack);
2395 } 2439 }
2396 return result; 2440 return result;
2441 }
2442
2443 PyObject *
2444 _PyObject_FastCall_Prepend(PyObject *func, PyObject *obj, PyObject **args,
2445 Py_ssize_t nargs, PyObject *kwnames)
2446 {
2447 PyObject *small_stack[_PY_FASTCALL_SMALL_STACK];
2448 PyObject **stack;
2449 PyObject *result;
2450 Py_ssize_t alloc;
2451
2452 alloc = nargs + 1;
2453 if (kwnames != NULL) {
2454 alloc += PyTuple_GET_SIZE(kwnames);
2455 }
2456
2457 if (alloc <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) {
2458 stack = small_stack;
2459 }
2460 else {
2461 stack = PyMem_Malloc(alloc * sizeof(PyObject *));
2462 if (stack == NULL) {
2463 PyErr_NoMemory();
2464 return NULL;
2465 }
2466 }
2467
2468 /* use borrowed references */
2469 stack[0] = obj;
2470 memcpy(&stack[1], args, (alloc - 1) * sizeof(PyObject *));
2471
2472 result = _PyObject_FastCallKeywords(func,
2473 stack, nargs + 1,
2474 kwnames);
2475 if (stack != small_stack) {
2476 PyMem_Free(stack);
2477 }
2478
2479 return result;
2397 } 2480 }
2398 2481
2399 PyObject * 2482 PyObject *
2400 _PyStack_AsDict(PyObject **values, PyObject *kwnames) 2483 _PyStack_AsDict(PyObject **values, PyObject *kwnames)
2401 { 2484 {
2402 Py_ssize_t nkwargs; 2485 Py_ssize_t nkwargs;
2403 PyObject *kwdict; 2486 PyObject *kwdict;
2404 Py_ssize_t i; 2487 Py_ssize_t i;
2405 2488
2406 assert(kwnames != NULL); 2489 assert(kwnames != NULL);
(...skipping 22 matching lines...) Expand all
2429 PyObject **stack, **kwstack; 2512 PyObject **stack, **kwstack;
2430 Py_ssize_t nkwargs; 2513 Py_ssize_t nkwargs;
2431 Py_ssize_t pos, i; 2514 Py_ssize_t pos, i;
2432 PyObject *key, *value; 2515 PyObject *key, *value;
2433 PyObject *kwnames; 2516 PyObject *kwnames;
2434 2517
2435 assert(nargs >= 0); 2518 assert(nargs >= 0);
2436 assert(kwargs == NULL || PyDict_CheckExact(kwargs)); 2519 assert(kwargs == NULL || PyDict_CheckExact(kwargs));
2437 2520
2438 if (kwargs == NULL || (nkwargs = PyDict_GET_SIZE(kwargs)) == 0) { 2521 if (kwargs == NULL || (nkwargs = PyDict_GET_SIZE(kwargs)) == 0) {
2522 /* Note: args can be NULL */
2439 *p_stack = args; 2523 *p_stack = args;
2440 *p_kwnames = NULL; 2524 *p_kwnames = NULL;
2441 return 0; 2525 return 0;
2442 } 2526 }
2443 2527
2444 if ((size_t)nargs > PY_SSIZE_T_MAX / sizeof(stack[0]) - (size_t)nkwargs) { 2528 if ((size_t)nargs > PY_SSIZE_T_MAX / sizeof(stack[0]) - (size_t)nkwargs) {
2445 PyErr_NoMemory(); 2529 PyErr_NoMemory();
2446 return -1; 2530 return -1;
2447 } 2531 }
2448 2532
2449 stack = PyMem_Malloc((nargs + nkwargs) * sizeof(stack[0])); 2533 stack = PyMem_Malloc((nargs + nkwargs) * sizeof(stack[0]));
2450 if (stack == NULL) { 2534 if (stack == NULL) {
2451 PyErr_NoMemory(); 2535 PyErr_NoMemory();
2452 return -1; 2536 return -1;
2453 } 2537 }
2454 2538
2455 kwnames = PyTuple_New(nkwargs); 2539 kwnames = PyTuple_New(nkwargs);
2456 if (kwnames == NULL) { 2540 if (kwnames == NULL) {
2457 PyMem_Free(stack); 2541 PyMem_Free(stack);
2458 return -1; 2542 return -1;
2459 } 2543 }
2460 2544
2461 /* Copy position arguments (borrowed references) */ 2545 /* args is NULL if nargs==0: don't call memcpy(stack, NULL, 0) */
2462 memcpy(stack, args, nargs * sizeof(stack[0])); 2546 if (nargs) {
2547 /* Copy position arguments (borrowed references) */
2548 memcpy(stack, args, nargs * sizeof(stack[0]));
2549 }
2463 2550
2464 kwstack = stack + nargs; 2551 kwstack = stack + nargs;
2465 pos = i = 0; 2552 pos = i = 0;
2466 /* This loop doesn't support lookup function mutating the dictionary 2553 /* This loop doesn't support lookup function mutating the dictionary
2467 to change its size. It's a deliberate choice for speed, this function is 2554 to change its size. It's a deliberate choice for speed, this function is
2468 called in the performance critical hot code. */ 2555 called in the performance critical hot code. */
2469 while (PyDict_Next(kwargs, &pos, &key, &value)) { 2556 while (PyDict_Next(kwargs, &pos, &key, &value)) {
2470 Py_INCREF(key); 2557 Py_INCREF(key);
2471 PyTuple_SET_ITEM(kwnames, i, key); 2558 PyTuple_SET_ITEM(kwnames, i, key);
2472 /* The stack contains borrowed references */ 2559 /* The stack contains borrowed references */
2473 kwstack[i] = value; 2560 kwstack[i] = value;
2474 i++; 2561 i++;
2475 } 2562 }
2476 2563
2477 *p_stack = stack; 2564 *p_stack = stack;
2478 *p_kwnames = kwnames; 2565 *p_kwnames = kwnames;
2479 return 0; 2566 return 0;
2567 }
2568
2569 PyObject *
2570 _Py_RawFastCallDict(PyObject *self, fastternaryfunc fastcall,
2571 PyObject **args, Py_ssize_t nargs, PyObject *kwargs)
2572 {
2573 PyObject **stack;
2574 PyObject *kwnames;
2575 PyObject *result;
2576
2577 assert(!PyErr_Occurred());
2578 assert(fastcall != NULL);
2579 assert(kwargs == NULL || PyDict_Check(kwargs));
2580
2581 if (_PyStack_UnpackDict(args, nargs, kwargs, &stack, &kwnames) < 0) {
2582 return NULL;
2583 }
2584
2585 result = fastcall(self, stack, nargs, kwnames);
2586 if (stack != args) {
2587 PyMem_Free(stack);
2588 }
2589 Py_XDECREF(kwnames);
2590 return result;
2480 } 2591 }
2481 2592
2482 PyObject * 2593 PyObject *
2483 _PyObject_FastCallKeywords(PyObject *callable, PyObject **stack, Py_ssize_t narg s, 2594 _PyObject_FastCallKeywords(PyObject *callable, PyObject **stack, Py_ssize_t narg s,
2484 PyObject *kwnames) 2595 PyObject *kwnames)
2485 { 2596 {
2486 /* _PyObject_FastCallKeywords() must not be called with an exception set, 2597 /* _PyObject_FastCallKeywords() must not be called with an exception set,
2487 because it can clear it (directly or indirectly) and so the 2598 because it can clear it (directly or indirectly) and so the
2488 caller loses its exception */ 2599 caller loses its exception */
2489 assert(!PyErr_Occurred()); 2600 assert(!PyErr_Occurred());
2490 2601
2491 assert(nargs >= 0); 2602 assert(nargs >= 0);
2492 assert(kwnames == NULL || PyTuple_CheckExact(kwnames)); 2603 assert(kwnames == NULL || PyTuple_CheckExact(kwnames));
2493 2604
2494 /* kwnames must only contains str strings, no subclass, and all keys must 2605 /* kwnames must only contains str strings, no subclass, and all keys must
2495 be unique: these checks are implemented in Python/ceval.c and 2606 be unique: these checks are implemented in Python/ceval.c and
2496 _PyArg_ParseStackAndKeywords(). */ 2607 _PyArg_ParseStackAndKeywords(). */
2497 2608
2498 if (PyFunction_Check(callable)) { 2609 if (PyFunction_Check(callable)) {
2610 /* FIXME: should we use Py_EnterRecursiveCall() here? */
haypo 2017/01/26 03:35:21 The FIXME can be removed, the issue is now tracked
2499 return _PyFunction_FastCallKeywords(callable, stack, nargs, kwnames); 2611 return _PyFunction_FastCallKeywords(callable, stack, nargs, kwnames);
2500 } 2612 }
2501 if (PyCFunction_Check(callable)) { 2613 if (PyCFunction_Check(callable)) {
2502 return _PyCFunction_FastCallKeywords(callable, stack, nargs, kwnames); 2614 return _PyCFunction_FastCallKeywords(callable, stack, nargs, kwnames);
2503 } 2615 }
2616 else if (PyType_HasFeature(Py_TYPE(callable), Py_TPFLAGS_HAVE_FASTCALL)
2617 && Py_TYPE(callable)->tp_fastcall) {
2618 fastternaryfunc fastcall = Py_TYPE(callable)->tp_fastcall;
2619 PyObject *result;
2620
2621 if (Py_EnterRecursiveCall(" while calling a Python object")) {
2622 return NULL;
2623 }
2624
2625 result = fastcall(callable, stack, nargs, kwnames);
2626 result = _Py_CheckFunctionResult(callable, result, NULL);
2627
2628 Py_LeaveRecursiveCall();
2629
2630 return result;
2631 }
2504 else { 2632 else {
2505 /* Slow-path: build a temporary tuple for positional arguments and a 2633 /* Slow-path: build a temporary tuple for positional arguments and a
2506 temporary dictionary for keyword arguments (if any) */ 2634 temporary dictionary for keyword arguments (if any) */
2507 2635
2508 ternaryfunc call; 2636 ternaryfunc call;
2509 PyObject *argtuple; 2637 PyObject *argtuple;
2510 PyObject *kwdict, *result; 2638 PyObject *kwdict, *result;
2511 Py_ssize_t nkwargs; 2639 Py_ssize_t nkwargs;
2512 2640
2513 result = NULL; 2641 result = NULL;
2514 nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); 2642 nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames);
2515 assert((nargs == 0 && nkwargs == 0) || stack != NULL); 2643 assert((nargs == 0 && nkwargs == 0) || stack != NULL);
2516 2644
2517 if (Py_EnterRecursiveCall(" while calling a Python object")) { 2645 if (Py_EnterRecursiveCall(" while calling a Python object")) {
2518 return NULL; 2646 return NULL;
2519 } 2647 }
2520 2648
2521 call = callable->ob_type->tp_call; 2649 call = Py_TYPE(callable)->tp_call;
2522 if (call == NULL) { 2650 if (call == NULL) {
2523 PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", 2651 PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable",
2524 callable->ob_type->tp_name); 2652 callable->ob_type->tp_name);
2525 goto exit; 2653 goto exit;
2526 } 2654 }
2527 2655
2528 argtuple = _PyStack_AsTuple(stack, nargs); 2656 argtuple = _PyStack_AsTuple(stack, nargs);
2529 if (argtuple == NULL) { 2657 if (argtuple == NULL) {
2530 goto exit; 2658 goto exit;
2531 } 2659 }
(...skipping 720 matching lines...) Expand 10 before | Expand all | Expand 10 after
3252 /* Free's a NULL terminated char** array of C strings. */ 3380 /* Free's a NULL terminated char** array of C strings. */
3253 void 3381 void
3254 _Py_FreeCharPArray(char *const array[]) 3382 _Py_FreeCharPArray(char *const array[])
3255 { 3383 {
3256 Py_ssize_t i; 3384 Py_ssize_t i;
3257 for (i = 0; array[i] != NULL; ++i) { 3385 for (i = 0; array[i] != NULL; ++i) {
3258 PyMem_Free(array[i]); 3386 PyMem_Free(array[i]);
3259 } 3387 }
3260 PyMem_Free((void*)array); 3388 PyMem_Free((void*)array);
3261 } 3389 }
OLDNEW
« no previous file with comments | « Modules/_operator.c ('k') | Objects/classobject.c » ('j') | Objects/descrobject.c » ('J')

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+