Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(1)

Side by Side Diff: Objects/abstract.c

Issue 29259: Add tp_fastcall to PyTypeObject: support FASTCALL calling convention for all callable objects
Patch Set: Created 3 years, 2 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « Modules/_operator.c ('k') | Objects/classobject.c » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 /* Abstract Object Interface (many thanks to Jim Fulton) */ 1 /* Abstract Object Interface (many thanks to Jim Fulton) */
2 2
3 #include "Python.h" 3 #include "Python.h"
4 #include <ctype.h> 4 #include <ctype.h>
5 #include "structmember.h" /* we need the offsetof() macro from there */ 5 #include "structmember.h" /* we need the offsetof() macro from there */
6 #include "longintrepr.h" 6 #include "longintrepr.h"
7 7
8 8
9 9
10 /* Shorthands to return certain errors */ 10 /* Shorthands to return certain errors */
(...skipping 2231 matching lines...) Expand 10 before | Expand all | Expand 10 after
2242 call = callable->ob_type->tp_call; 2242 call = callable->ob_type->tp_call;
2243 if (call == NULL) { 2243 if (call == NULL) {
2244 PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", 2244 PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable",
2245 callable->ob_type->tp_name); 2245 callable->ob_type->tp_name);
2246 return NULL; 2246 return NULL;
2247 } 2247 }
2248 2248
2249 if (Py_EnterRecursiveCall(" while calling a Python object")) 2249 if (Py_EnterRecursiveCall(" while calling a Python object"))
2250 return NULL; 2250 return NULL;
2251 2251
2252 result = (*call)(callable, args, kwargs); 2252 if (PyType_HasFeature(Py_TYPE(callable), Py_TPFLAGS_HAVE_FASTCALL)
2253 && Py_TYPE(callable)->tp_fastcall) {
2254 result = _Py_RawFastCallDict(callable,
2255 Py_TYPE(callable)->tp_fastcall,
2256 &PyTuple_GET_ITEM(args, 0),
2257 PyTuple_GET_SIZE(args),
2258 kwargs);
2259 }
2260 else {
2261 result = (*call)(callable, args, kwargs);
2262 }
2253 2263
2254 Py_LeaveRecursiveCall(); 2264 Py_LeaveRecursiveCall();
2255 2265
2256 return _Py_CheckFunctionResult(callable, result, NULL); 2266 return _Py_CheckFunctionResult(callable, result, NULL);
2257 } 2267 }
2258 2268
2259 /* Issue #29234: Inlining _PyStack_AsTuple() into callers increases their 2269 /* Issue #29234: Inlining _PyStack_AsTuple() into callers increases their
2260 stack consumption, Disable inlining to optimize the stack consumption. */ 2270 stack consumption, Disable inlining to optimize the stack consumption. */
2261 PyObject* _Py_NO_INLINE 2271 PyObject* _Py_NO_INLINE
2262 _PyStack_AsTuple(PyObject **stack, Py_ssize_t nargs) 2272 _PyStack_AsTuple(PyObject **stack, Py_ssize_t nargs)
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after
2320 2330
2321 if (Py_EnterRecursiveCall(" while calling a Python object")) { 2331 if (Py_EnterRecursiveCall(" while calling a Python object")) {
2322 return NULL; 2332 return NULL;
2323 } 2333 }
2324 2334
2325 if (PyFunction_Check(callable)) { 2335 if (PyFunction_Check(callable)) {
2326 result = _PyFunction_FastCallDict(callable, args, nargs, kwargs); 2336 result = _PyFunction_FastCallDict(callable, args, nargs, kwargs);
2327 } 2337 }
2328 else if (PyCFunction_Check(callable)) { 2338 else if (PyCFunction_Check(callable)) {
2329 result = _PyCFunction_FastCallDict(callable, args, nargs, kwargs); 2339 result = _PyCFunction_FastCallDict(callable, args, nargs, kwargs);
2340 }
2341 else if (PyType_HasFeature(Py_TYPE(callable), Py_TPFLAGS_HAVE_FASTCALL)
2342 && Py_TYPE(callable)->tp_fastcall) {
2343 fastternaryfunc fastcall = Py_TYPE(callable)->tp_fastcall;
2344
2345 result = _Py_RawFastCallDict(callable, fastcall, args, nargs, kwargs);
2346 result = _Py_CheckFunctionResult(callable, result, NULL);
2330 } 2347 }
2331 else { 2348 else {
2332 PyObject *tuple; 2349 PyObject *tuple;
2333 2350
2334 /* Slow-path: build a temporary tuple */ 2351 /* Slow-path: build a temporary tuple */
2335 call = callable->ob_type->tp_call; 2352 call = Py_TYPE(callable)->tp_call;
2336 if (call == NULL) { 2353 if (call == NULL) {
2337 PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", 2354 PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable",
2338 callable->ob_type->tp_name); 2355 callable->ob_type->tp_name);
2339 goto exit; 2356 goto exit;
2340 } 2357 }
2341 2358
2342 tuple = _PyStack_AsTuple(args, nargs); 2359 tuple = _PyStack_AsTuple(args, nargs);
2343 if (tuple == NULL) { 2360 if (tuple == NULL) {
2344 goto exit; 2361 goto exit;
2345 } 2362 }
(...skipping 11 matching lines...) Expand all
2357 } 2374 }
2358 2375
2359 /* Positional arguments are obj followed by args: 2376 /* Positional arguments are obj followed by args:
2360 call callable(obj, *args, **kwargs) */ 2377 call callable(obj, *args, **kwargs) */
2361 PyObject * 2378 PyObject *
2362 _PyObject_Call_Prepend(PyObject *callable, 2379 _PyObject_Call_Prepend(PyObject *callable,
2363 PyObject *obj, PyObject *args, PyObject *kwargs) 2380 PyObject *obj, PyObject *args, PyObject *kwargs)
2364 { 2381 {
2365 PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; 2382 PyObject *small_stack[_PY_FASTCALL_SMALL_STACK];
2366 PyObject **stack; 2383 PyObject **stack;
2367 Py_ssize_t argcount; 2384 Py_ssize_t nargs;
2368 PyObject *result; 2385 PyObject *result;
2369 2386
2370 assert(PyTuple_Check(args)); 2387 assert(PyTuple_Check(args));
2371 2388
2372 argcount = PyTuple_GET_SIZE(args); 2389 nargs = PyTuple_GET_SIZE(args);
2373 if (argcount + 1 <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { 2390 if (nargs + 1 <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) {
2374 stack = small_stack; 2391 stack = small_stack;
2375 } 2392 }
2376 else { 2393 else {
2377 stack = PyMem_Malloc((argcount + 1) * sizeof(PyObject *)); 2394 stack = PyMem_Malloc((nargs + 1) * sizeof(PyObject *));
2378 if (stack == NULL) { 2395 if (stack == NULL) {
2379 PyErr_NoMemory(); 2396 PyErr_NoMemory();
2380 return NULL; 2397 return NULL;
2381 } 2398 }
2382 } 2399 }
2383 2400
2384 /* use borrowed references */ 2401 /* use borrowed references */
2385 stack[0] = obj; 2402 stack[0] = obj;
2386 memcpy(&stack[1], 2403 memcpy(&stack[1],
2387 &PyTuple_GET_ITEM(args, 0), 2404 &PyTuple_GET_ITEM(args, 0),
2388 argcount * sizeof(PyObject *)); 2405 nargs * sizeof(PyObject *));
2389 2406
2390 result = _PyObject_FastCallDict(callable, 2407 result = _PyObject_FastCallDict(callable,
2391 stack, argcount + 1, 2408 stack, nargs + 1,
2392 kwargs); 2409 kwargs);
2393 if (stack != small_stack) { 2410 if (stack != small_stack) {
2394 PyMem_Free(stack); 2411 PyMem_Free(stack);
2395 } 2412 }
2396 return result; 2413 return result;
2414 }
2415
2416 PyObject *
2417 _PyObject_FastCall_Prepend(PyObject *func, PyObject *obj, PyObject **args,
2418 Py_ssize_t nargs, PyObject *kwnames)
2419 {
2420 PyObject *small_stack[_PY_FASTCALL_SMALL_STACK];
2421 PyObject **stack;
2422 PyObject *result;
2423 Py_ssize_t alloc;
2424
2425 alloc = nargs + 1;
2426 if (kwnames != NULL) {
2427 alloc += PyTuple_GET_SIZE(kwnames);
2428 }
2429
2430 if (alloc <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) {
2431 stack = small_stack;
2432 }
2433 else {
2434 stack = PyMem_Malloc(alloc * sizeof(PyObject *));
2435 if (stack == NULL) {
2436 PyErr_NoMemory();
2437 return NULL;
2438 }
2439 }
2440
2441 /* use borrowed references */
2442 stack[0] = obj;
2443 memcpy(&stack[1], args, (alloc - 1) * sizeof(PyObject *));
2444
2445 result = _PyObject_FastCallKeywords(func,
2446 stack, nargs + 1,
2447 kwnames);
2448 if (stack != small_stack) {
2449 PyMem_Free(stack);
2450 }
2451
2452 result = _Py_CheckFunctionResult(func, result, NULL);
2453 return result;
2397 } 2454 }
2398 2455
2399 PyObject * 2456 PyObject *
2400 _PyStack_AsDict(PyObject **values, PyObject *kwnames) 2457 _PyStack_AsDict(PyObject **values, PyObject *kwnames)
2401 { 2458 {
2402 Py_ssize_t nkwargs = PyTuple_GET_SIZE(kwnames); 2459 Py_ssize_t nkwargs = PyTuple_GET_SIZE(kwnames);
2403 PyObject *kwdict; 2460 PyObject *kwdict;
2404 Py_ssize_t i; 2461 Py_ssize_t i;
2405 2462
2406 kwdict = _PyDict_NewPresized(nkwargs); 2463 kwdict = _PyDict_NewPresized(nkwargs);
2407 if (kwdict == NULL) { 2464 if (kwdict == NULL) {
2408 return NULL; 2465 return NULL;
2409 } 2466 }
2410 2467
2411 for (i = 0; i < nkwargs; i++) { 2468 for (i = 0; i < nkwargs; i++) {
2412 PyObject *key = PyTuple_GET_ITEM(kwnames, i); 2469 PyObject *key = PyTuple_GET_ITEM(kwnames, i);
2413 PyObject *value = *values++; 2470 PyObject *value = *values++;
2414 assert(PyUnicode_CheckExact(key)); 2471 assert(PyUnicode_CheckExact(key));
2415 assert(PyDict_GetItem(kwdict, key) == NULL); 2472 assert(PyDict_GetItem(kwdict, key) == NULL);
2416 if (PyDict_SetItem(kwdict, key, value)) { 2473 if (PyDict_SetItem(kwdict, key, value)) {
2417 Py_DECREF(kwdict); 2474 Py_DECREF(kwdict);
2418 return NULL; 2475 return NULL;
2419 } 2476 }
2420 } 2477 }
2421 return kwdict; 2478 return kwdict;
2422 } 2479 }
2423 2480
2424 int 2481 static int
2425 _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, 2482 _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs,
2426 PyObject ***p_stack, PyObject **p_kwnames, PyObject *func) 2483 PyObject ***p_stack, PyObject **p_kwnames)
2427 { 2484 {
2428 PyObject **stack, **kwstack; 2485 PyObject **stack, **kwstack;
2429 Py_ssize_t nkwargs; 2486 Py_ssize_t nkwargs;
2430 Py_ssize_t pos, i; 2487 Py_ssize_t pos, i;
2431 PyObject *key, *value; 2488 PyObject *key, *value;
2432 PyObject *kwnames; 2489 PyObject *kwnames;
2433 2490
2434 assert(nargs >= 0); 2491 assert(nargs >= 0);
2435 assert(kwargs == NULL || PyDict_CheckExact(kwargs)); 2492 assert(kwargs == NULL || PyDict_CheckExact(kwargs));
2436 2493
2437 if (kwargs == NULL || (nkwargs = PyDict_GET_SIZE(kwargs)) == 0) { 2494 if (kwargs == NULL || (nkwargs = PyDict_GET_SIZE(kwargs)) == 0) {
2495 /* Note: args can be NULL */
2438 *p_stack = args; 2496 *p_stack = args;
2439 *p_kwnames = NULL; 2497 *p_kwnames = NULL;
2440 return 0; 2498 return 0;
2441 } 2499 }
2442 2500
2443 if ((size_t)nargs > PY_SSIZE_T_MAX / sizeof(stack[0]) - (size_t)nkwargs) { 2501 if ((size_t)nargs > PY_SSIZE_T_MAX / sizeof(stack[0]) - (size_t)nkwargs) {
2444 PyErr_NoMemory(); 2502 PyErr_NoMemory();
2445 return -1; 2503 return -1;
2446 } 2504 }
2447 2505
2448 stack = PyMem_Malloc((nargs + nkwargs) * sizeof(stack[0])); 2506 stack = PyMem_Malloc((nargs + nkwargs) * sizeof(stack[0]));
2449 if (stack == NULL) { 2507 if (stack == NULL) {
2450 PyErr_NoMemory(); 2508 PyErr_NoMemory();
2451 return -1; 2509 return -1;
2452 } 2510 }
2453 2511
2454 kwnames = PyTuple_New(nkwargs); 2512 kwnames = PyTuple_New(nkwargs);
2455 if (kwnames == NULL) { 2513 if (kwnames == NULL) {
2456 PyMem_Free(stack); 2514 PyMem_Free(stack);
2457 return -1; 2515 return -1;
2458 } 2516 }
2459 2517
2460 /* Copy position arguments (borrowed references) */ 2518 /* args is NULL if nargs==0: don't call memcpy(stack, NULL, 0) */
2461 memcpy(stack, args, nargs * sizeof(stack[0])); 2519 if (nargs) {
2520 /* Copy position arguments (borrowed references) */
2521 memcpy(stack, args, nargs * sizeof(stack[0]));
2522 }
2462 2523
2463 kwstack = stack + nargs; 2524 kwstack = stack + nargs;
2464 pos = i = 0; 2525 pos = i = 0;
2465 /* This loop doesn't support lookup function mutating the dictionary 2526 /* This loop doesn't support lookup function mutating the dictionary
2466 to change its size. It's a deliberate choice for speed, this function is 2527 to change its size. It's a deliberate choice for speed, this function is
2467 called in the performance critical hot code. */ 2528 called in the performance critical hot code. */
2468 while (PyDict_Next(kwargs, &pos, &key, &value)) { 2529 while (PyDict_Next(kwargs, &pos, &key, &value)) {
2469 Py_INCREF(key); 2530 Py_INCREF(key);
2470 PyTuple_SET_ITEM(kwnames, i, key); 2531 PyTuple_SET_ITEM(kwnames, i, key);
2471 /* The stack contains borrowed references */ 2532 /* The stack contains borrowed references */
2472 kwstack[i] = value; 2533 kwstack[i] = value;
2473 i++; 2534 i++;
2474 } 2535 }
2475 2536
2476 *p_stack = stack; 2537 *p_stack = stack;
2477 *p_kwnames = kwnames; 2538 *p_kwnames = kwnames;
2478 return 0; 2539 return 0;
2540 }
2541
2542 PyObject *
2543 _Py_RawFastCallDict(PyObject *self, fastternaryfunc fastcall,
2544 PyObject **args, Py_ssize_t nargs, PyObject *kwargs)
2545 {
2546 PyObject **stack;
2547 PyObject *kwnames;
2548 PyObject *result;
2549
2550 assert(!PyErr_Occurred());
2551
2552 if (_PyStack_UnpackDict(args, nargs, kwargs, &stack, &kwnames) < 0) {
2553 return NULL;
2554 }
2555
2556 result = fastcall(self, stack, nargs, kwnames);
2557 if (stack != args) {
2558 PyMem_Free(stack);
2559 }
2560 Py_XDECREF(kwnames);
2561
2562 return result;
2479 } 2563 }
2480 2564
2481 PyObject * 2565 PyObject *
2482 _PyObject_FastCallKeywords(PyObject *callable, PyObject **stack, Py_ssize_t narg s, 2566 _PyObject_FastCallKeywords(PyObject *callable, PyObject **stack, Py_ssize_t narg s,
2483 PyObject *kwnames) 2567 PyObject *kwnames)
2484 { 2568 {
2485 assert(nargs >= 0); 2569 assert(nargs >= 0);
2486 assert(kwnames == NULL || PyTuple_CheckExact(kwnames)); 2570 assert(kwnames == NULL || PyTuple_CheckExact(kwnames));
2487 2571
2488 /* kwnames must only contains str strings, no subclass, and all keys must 2572 /* kwnames must only contains str strings, no subclass, and all keys must
2489 be unique: these checks are implemented in Python/ceval.c and 2573 be unique: these checks are implemented in Python/ceval.c and
2490 _PyArg_ParseStackAndKeywords(). */ 2574 _PyArg_ParseStackAndKeywords(). */
2491 2575
2492 if (PyFunction_Check(callable)) { 2576 if (PyFunction_Check(callable)) {
2493 return _PyFunction_FastCallKeywords(callable, stack, nargs, kwnames); 2577 return _PyFunction_FastCallKeywords(callable, stack, nargs, kwnames);
2494 } 2578 }
2495 if (PyCFunction_Check(callable)) { 2579 if (PyCFunction_Check(callable)) {
2496 return _PyCFunction_FastCallKeywords(callable, stack, nargs, kwnames); 2580 return _PyCFunction_FastCallKeywords(callable, stack, nargs, kwnames);
2581 }
2582 else if (PyType_HasFeature(Py_TYPE(callable), Py_TPFLAGS_HAVE_FASTCALL)
2583 && Py_TYPE(callable)->tp_fastcall) {
2584 fastternaryfunc fastcall = Py_TYPE(callable)->tp_fastcall;
2585 PyObject *result;
2586
2587 if (Py_EnterRecursiveCall(" while calling a Python object")) {
2588 return NULL;
2589 }
2590
2591 result = fastcall(callable, stack, nargs, kwnames);
2592 result = _Py_CheckFunctionResult(callable, result, NULL);
2593
2594 Py_LeaveRecursiveCall();
2595
2596 return result;
2497 } 2597 }
2498 else { 2598 else {
2499 /* Slow-path: build a temporary tuple for positional arguments and a 2599 /* Slow-path: build a temporary tuple for positional arguments and a
2500 temporary dictionary for keyword arguments (if any) */ 2600 temporary dictionary for keyword arguments (if any) */
2501 2601
2502 ternaryfunc call; 2602 ternaryfunc call;
2503 PyObject *argtuple; 2603 PyObject *argtuple;
2504 PyObject *kwdict, *result; 2604 PyObject *kwdict, *result;
2505 Py_ssize_t nkwargs; 2605 Py_ssize_t nkwargs;
2506 2606
2507 result = NULL; 2607 result = NULL;
2508 nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); 2608 nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames);
2509 assert((nargs == 0 && nkwargs == 0) || stack != NULL); 2609 assert((nargs == 0 && nkwargs == 0) || stack != NULL);
2510 2610
2511 if (Py_EnterRecursiveCall(" while calling a Python object")) { 2611 if (Py_EnterRecursiveCall(" while calling a Python object")) {
2512 return NULL; 2612 return NULL;
2513 } 2613 }
2514 2614
2515 call = callable->ob_type->tp_call; 2615 call = Py_TYPE(callable)->tp_call;
2516 if (call == NULL) { 2616 if (call == NULL) {
2517 PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", 2617 PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable",
2518 callable->ob_type->tp_name); 2618 callable->ob_type->tp_name);
2519 goto exit; 2619 goto exit;
2520 } 2620 }
2521 2621
2522 argtuple = _PyStack_AsTuple(stack, nargs); 2622 argtuple = _PyStack_AsTuple(stack, nargs);
2523 if (argtuple == NULL) { 2623 if (argtuple == NULL) {
2524 goto exit; 2624 goto exit;
2525 } 2625 }
2526 2626
2527 if (nkwargs > 0) { 2627 if (nkwargs > 0) {
2528 kwdict = _PyStack_AsDict(stack + nargs, kwnames); 2628 kwdict = _PyStack_AsDict(stack + nargs, kwnames);
2529 if (kwdict == NULL) { 2629 if (kwdict == NULL) {
2530 Py_DECREF(argtuple); 2630 Py_DECREF(argtuple);
2531 goto exit; 2631 goto exit;
2532 } 2632 }
2533 } 2633 }
2534 else { 2634 else {
2535 kwdict = NULL; 2635 kwdict = NULL;
2536 } 2636 }
2537 2637
2538 result = (*call)(callable, argtuple, kwdict); 2638 result = (*call)(callable, argtuple, kwdict);
2539 Py_DECREF(argtuple); 2639 Py_DECREF(argtuple);
2540 Py_XDECREF(kwdict); 2640 Py_XDECREF(kwdict);
2641
2642 result = _Py_CheckFunctionResult(callable, result, NULL);
2541 2643
2542 exit: 2644 exit:
2543 Py_LeaveRecursiveCall(); 2645 Py_LeaveRecursiveCall();
2544 return result; 2646 return result;
2545 } 2647 }
2546 } 2648 }
2547 2649
2548 static PyObject * 2650 static PyObject *
2549 _PyObject_CallFunctionVa(PyObject *callable, const char *format, 2651 _PyObject_CallFunctionVa(PyObject *callable, const char *format,
2550 va_list va, int is_size_t) 2652 va_list va, int is_size_t)
(...skipping 693 matching lines...) Expand 10 before | Expand all | Expand 10 after
3244 /* Free's a NULL terminated char** array of C strings. */ 3346 /* Free's a NULL terminated char** array of C strings. */
3245 void 3347 void
3246 _Py_FreeCharPArray(char *const array[]) 3348 _Py_FreeCharPArray(char *const array[])
3247 { 3349 {
3248 Py_ssize_t i; 3350 Py_ssize_t i;
3249 for (i = 0; array[i] != NULL; ++i) { 3351 for (i = 0; array[i] != NULL; ++i) {
3250 PyMem_Free(array[i]); 3352 PyMem_Free(array[i]);
3251 } 3353 }
3252 PyMem_Free((void*)array); 3354 PyMem_Free((void*)array);
3253 } 3355 }
OLDNEW
« no previous file with comments | « Modules/_operator.c ('k') | Objects/classobject.c » ('j') | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+