Index: Python/ceval.c =================================================================== --- Python/ceval.c (Revision 53930) +++ Python/ceval.c (Arbeitskopie) @@ -1242,6 +1242,18 @@ } break; + case SET_ADD: + w = POP(); + v = POP(); + err = PySet_Add(v, w); + Py_DECREF(v); + Py_DECREF(w); + if (err == 0) { + PREDICT(JUMP_ABSOLUTE); + continue; + } + break; + case INPLACE_POWER: w = POP(); v = TOP(); Index: Python/graminit.c =================================================================== --- Python/graminit.c (Revision 53930) +++ Python/graminit.c (Arbeitskopie) @@ -1043,33 +1043,36 @@ {1, arcs_46_3}, {2, arcs_46_4}, }; -static arc arcs_47_0[1] = { - {106, 1}, +static arc arcs_47_0[2] = { + {105, 1}, + {106, 2}, }; static arc arcs_47_1[2] = { - {28, 2}, + {90, 3}, {0, 1}, }; static arc arcs_47_2[1] = { - {106, 3}, + {0, 2}, }; -static arc arcs_47_3[2] = { - {28, 4}, - {0, 3}, +static arc arcs_47_3[1] = { + {105, 4}, }; -static arc arcs_47_4[2] = { - {106, 3}, - {0, 4}, +static arc arcs_47_4[1] = { + {92, 5}, }; -static state states_47[5] = { - {1, arcs_47_0}, +static arc arcs_47_5[1] = { + {22, 2}, +}; +static state states_47[6] = { + {2, arcs_47_0}, {2, arcs_47_1}, {1, arcs_47_2}, - {2, arcs_47_3}, - {2, arcs_47_4}, + {1, arcs_47_3}, + {1, arcs_47_4}, + {1, arcs_47_5}, }; static arc arcs_48_0[2] = { - {107, 1}, + {105, 1}, {108, 1}, }; static arc arcs_48_1[1] = { @@ -1090,7 +1093,7 @@ {23, 3}, }; static arc arcs_49_3[1] = { - {106, 4}, + {22, 4}, }; static arc arcs_49_4[1] = { {0, 4}, @@ -1102,39 +1105,34 @@ {1, arcs_49_3}, {1, arcs_49_4}, }; -static arc arcs_50_0[2] = { - {107, 1}, - {110, 2}, +static arc arcs_50_0[1] = { + {109, 1}, }; static arc arcs_50_1[2] = { - {90, 3}, - {0, 1}, + {33, 2}, + {23, 3}, }; static arc arcs_50_2[1] = { - {0, 2}, + {23, 3}, }; static arc arcs_50_3[1] = { {107, 4}, }; static arc arcs_50_4[1] = { - {92, 5}, + {0, 4}, }; -static arc arcs_50_5[1] = { - {22, 2}, -}; -static state states_50[6] = { - {2, arcs_50_0}, +static state states_50[5] = { + {1, arcs_50_0}, {2, arcs_50_1}, {1, arcs_50_2}, {1, arcs_50_3}, {1, arcs_50_4}, - {1, arcs_50_5}, }; static arc arcs_51_0[1] = { - {111, 1}, + {110, 1}, }; static arc arcs_51_1[2] = { - {112, 0}, + {111, 0}, {0, 1}, }; static state states_51[2] = { @@ -1142,10 +1140,10 @@ {2, arcs_51_1}, }; static arc arcs_52_0[1] = { - {113, 1}, + {112, 1}, }; static arc arcs_52_1[2] = { - {114, 0}, + {113, 0}, {0, 1}, }; static state states_52[2] = { @@ -1153,11 +1151,11 @@ {2, arcs_52_1}, }; static arc arcs_53_0[2] = { - {115, 1}, - {116, 2}, + {114, 1}, + {115, 2}, }; static arc arcs_53_1[1] = { - {113, 2}, + {112, 2}, }; static arc arcs_53_2[1] = { {0, 2}, @@ -1171,7 +1169,7 @@ {101, 1}, }; static arc arcs_54_1[2] = { - {117, 0}, + {116, 0}, {0, 1}, }; static state states_54[2] = { @@ -1179,15 +1177,15 @@ {2, arcs_54_1}, }; static arc arcs_55_0[9] = { + {117, 1}, {118, 1}, {119, 1}, {120, 1}, {121, 1}, {122, 1}, - {123, 1}, {95, 1}, - {115, 2}, - {124, 3}, + {114, 2}, + {123, 3}, }; static arc arcs_55_1[1] = { {0, 1}, @@ -1196,7 +1194,7 @@ {95, 1}, }; static arc arcs_55_3[2] = { - {115, 1}, + {114, 1}, {0, 3}, }; static state states_55[4] = { @@ -1206,10 +1204,10 @@ {2, arcs_55_3}, }; static arc arcs_56_0[1] = { - {125, 1}, + {124, 1}, }; static arc arcs_56_1[2] = { - {126, 0}, + {125, 0}, {0, 1}, }; static state states_56[2] = { @@ -1217,10 +1215,10 @@ {2, arcs_56_1}, }; static arc arcs_57_0[1] = { - {127, 1}, + {126, 1}, }; static arc arcs_57_1[2] = { - {128, 0}, + {127, 0}, {0, 1}, }; static state states_57[2] = { @@ -1228,10 +1226,10 @@ {2, arcs_57_1}, }; static arc arcs_58_0[1] = { - {129, 1}, + {128, 1}, }; static arc arcs_58_1[2] = { - {130, 0}, + {129, 0}, {0, 1}, }; static state states_58[2] = { @@ -1239,11 +1237,11 @@ {2, arcs_58_1}, }; static arc arcs_59_0[1] = { - {131, 1}, + {130, 1}, }; static arc arcs_59_1[3] = { + {131, 0}, {132, 0}, - {133, 0}, {0, 1}, }; static state states_59[2] = { @@ -1251,11 +1249,11 @@ {3, arcs_59_1}, }; static arc arcs_60_0[1] = { - {134, 1}, + {133, 1}, }; static arc arcs_60_1[3] = { + {134, 0}, {135, 0}, - {136, 0}, {0, 1}, }; static state states_60[2] = { @@ -1263,13 +1261,13 @@ {3, arcs_60_1}, }; static arc arcs_61_0[1] = { - {137, 1}, + {136, 1}, }; static arc arcs_61_1[5] = { {29, 0}, + {137, 0}, {138, 0}, {139, 0}, - {140, 0}, {0, 1}, }; static state states_61[2] = { @@ -1277,13 +1275,13 @@ {5, arcs_61_1}, }; static arc arcs_62_0[4] = { + {134, 1}, {135, 1}, - {136, 1}, - {141, 1}, - {142, 2}, + {140, 1}, + {141, 2}, }; static arc arcs_62_1[1] = { - {137, 2}, + {136, 2}, }; static arc arcs_62_2[1] = { {0, 2}, @@ -1294,15 +1292,15 @@ {1, arcs_62_2}, }; static arc arcs_63_0[1] = { - {143, 1}, + {142, 1}, }; static arc arcs_63_1[3] = { - {144, 1}, + {143, 1}, {31, 2}, {0, 1}, }; static arc arcs_63_2[1] = { - {137, 3}, + {136, 3}, }; static arc arcs_63_3[1] = { {0, 3}, @@ -1315,31 +1313,31 @@ }; static arc arcs_64_0[7] = { {13, 1}, - {146, 2}, - {149, 3}, + {145, 2}, + {147, 3}, {19, 4}, - {152, 4}, - {153, 5}, + {150, 4}, + {151, 5}, {77, 6}, }; static arc arcs_64_1[3] = { {47, 7}, - {145, 7}, + {144, 7}, {15, 4}, }; static arc arcs_64_2[2] = { - {147, 8}, - {148, 4}, + {144, 8}, + {146, 4}, }; static arc arcs_64_3[2] = { - {150, 9}, - {151, 4}, + {148, 9}, + {149, 4}, }; static arc arcs_64_4[1] = { {0, 4}, }; static arc arcs_64_5[2] = { - {153, 5}, + {151, 5}, {0, 5}, }; static arc arcs_64_6[1] = { @@ -1349,10 +1347,10 @@ {15, 4}, }; static arc arcs_64_8[1] = { - {148, 4}, + {146, 4}, }; static arc arcs_64_9[1] = { - {151, 4}, + {149, 4}, }; static arc arcs_64_10[1] = { {77, 4}, @@ -1374,7 +1372,7 @@ {22, 1}, }; static arc arcs_65_1[3] = { - {154, 2}, + {152, 2}, {28, 3}, {0, 1}, }; @@ -1396,472 +1394,371 @@ {2, arcs_65_3}, {2, arcs_65_4}, }; -static arc arcs_66_0[1] = { - {22, 1}, -}; -static arc arcs_66_1[3] = { - {155, 2}, - {28, 3}, - {0, 1}, -}; -static arc arcs_66_2[1] = { - {0, 2}, -}; -static arc arcs_66_3[2] = { - {22, 4}, - {0, 3}, -}; -static arc arcs_66_4[2] = { - {28, 3}, - {0, 4}, -}; -static state states_66[5] = { - {1, arcs_66_0}, - {3, arcs_66_1}, - {1, arcs_66_2}, - {2, arcs_66_3}, - {2, arcs_66_4}, -}; -static arc arcs_67_0[1] = { - {109, 1}, -}; -static arc arcs_67_1[2] = { - {33, 2}, - {23, 3}, -}; -static arc arcs_67_2[1] = { - {23, 3}, -}; -static arc arcs_67_3[1] = { - {22, 4}, -}; -static arc arcs_67_4[1] = { - {0, 4}, -}; -static state states_67[5] = { - {1, arcs_67_0}, - {2, arcs_67_1}, - {1, arcs_67_2}, - {1, arcs_67_3}, - {1, arcs_67_4}, -}; -static arc arcs_68_0[3] = { +static arc arcs_66_0[3] = { {13, 1}, - {146, 2}, + {145, 2}, {77, 3}, }; -static arc arcs_68_1[2] = { +static arc arcs_66_1[2] = { {14, 4}, {15, 5}, }; -static arc arcs_68_2[1] = { - {156, 6}, +static arc arcs_66_2[1] = { + {153, 6}, }; -static arc arcs_68_3[1] = { +static arc arcs_66_3[1] = { {19, 5}, }; -static arc arcs_68_4[1] = { +static arc arcs_66_4[1] = { {15, 5}, }; -static arc arcs_68_5[1] = { +static arc arcs_66_5[1] = { {0, 5}, }; -static arc arcs_68_6[1] = { - {148, 5}, +static arc arcs_66_6[1] = { + {146, 5}, }; -static state states_68[7] = { - {3, arcs_68_0}, - {2, arcs_68_1}, - {1, arcs_68_2}, - {1, arcs_68_3}, - {1, arcs_68_4}, - {1, arcs_68_5}, - {1, arcs_68_6}, +static state states_66[7] = { + {3, arcs_66_0}, + {2, arcs_66_1}, + {1, arcs_66_2}, + {1, arcs_66_3}, + {1, arcs_66_4}, + {1, arcs_66_5}, + {1, arcs_66_6}, }; -static arc arcs_69_0[1] = { - {157, 1}, +static arc arcs_67_0[1] = { + {154, 1}, }; -static arc arcs_69_1[2] = { +static arc arcs_67_1[2] = { {28, 2}, {0, 1}, }; -static arc arcs_69_2[2] = { - {157, 1}, +static arc arcs_67_2[2] = { + {154, 1}, {0, 2}, }; -static state states_69[3] = { - {1, arcs_69_0}, - {2, arcs_69_1}, - {2, arcs_69_2}, +static state states_67[3] = { + {1, arcs_67_0}, + {2, arcs_67_1}, + {2, arcs_67_2}, }; -static arc arcs_70_0[2] = { +static arc arcs_68_0[2] = { {22, 1}, {23, 2}, }; -static arc arcs_70_1[2] = { +static arc arcs_68_1[2] = { {23, 2}, {0, 1}, }; -static arc arcs_70_2[3] = { +static arc arcs_68_2[3] = { {22, 3}, - {158, 4}, + {155, 4}, {0, 2}, }; -static arc arcs_70_3[2] = { - {158, 4}, +static arc arcs_68_3[2] = { + {155, 4}, {0, 3}, }; -static arc arcs_70_4[1] = { +static arc arcs_68_4[1] = { {0, 4}, }; -static state states_70[5] = { - {2, arcs_70_0}, - {2, arcs_70_1}, - {3, arcs_70_2}, - {2, arcs_70_3}, - {1, arcs_70_4}, +static state states_68[5] = { + {2, arcs_68_0}, + {2, arcs_68_1}, + {3, arcs_68_2}, + {2, arcs_68_3}, + {1, arcs_68_4}, }; -static arc arcs_71_0[1] = { +static arc arcs_69_0[1] = { {23, 1}, }; -static arc arcs_71_1[2] = { +static arc arcs_69_1[2] = { {22, 2}, {0, 1}, }; -static arc arcs_71_2[1] = { +static arc arcs_69_2[1] = { {0, 2}, }; -static state states_71[3] = { - {1, arcs_71_0}, - {2, arcs_71_1}, - {1, arcs_71_2}, +static state states_69[3] = { + {1, arcs_69_0}, + {2, arcs_69_1}, + {1, arcs_69_2}, }; -static arc arcs_72_0[1] = { +static arc arcs_70_0[1] = { {101, 1}, }; -static arc arcs_72_1[2] = { +static arc arcs_70_1[2] = { {28, 2}, {0, 1}, }; -static arc arcs_72_2[2] = { +static arc arcs_70_2[2] = { {101, 1}, {0, 2}, }; -static state states_72[3] = { - {1, arcs_72_0}, - {2, arcs_72_1}, - {2, arcs_72_2}, +static state states_70[3] = { + {1, arcs_70_0}, + {2, arcs_70_1}, + {2, arcs_70_2}, }; -static arc arcs_73_0[1] = { +static arc arcs_71_0[1] = { {22, 1}, }; -static arc arcs_73_1[2] = { +static arc arcs_71_1[2] = { {28, 2}, {0, 1}, }; -static arc arcs_73_2[2] = { +static arc arcs_71_2[2] = { {22, 1}, {0, 2}, }; -static state states_73[3] = { - {1, arcs_73_0}, - {2, arcs_73_1}, - {2, arcs_73_2}, +static state states_71[3] = { + {1, arcs_71_0}, + {2, arcs_71_1}, + {2, arcs_71_2}, }; -static arc arcs_74_0[1] = { +static arc arcs_72_0[1] = { {22, 1}, }; -static arc arcs_74_1[3] = { +static arc arcs_72_1[4] = { {23, 2}, - {28, 3}, + {152, 3}, + {28, 4}, {0, 1}, }; -static arc arcs_74_2[1] = { - {22, 4}, -}; -static arc arcs_74_3[2] = { +static arc arcs_72_2[1] = { {22, 5}, +}; +static arc arcs_72_3[1] = { {0, 3}, }; -static arc arcs_74_4[2] = { - {28, 6}, +static arc arcs_72_4[2] = { + {22, 6}, {0, 4}, }; -static arc arcs_74_5[2] = { - {28, 3}, +static arc arcs_72_5[2] = { + {28, 7}, {0, 5}, }; -static arc arcs_74_6[2] = { - {22, 7}, +static arc arcs_72_6[2] = { + {28, 4}, {0, 6}, }; -static arc arcs_74_7[1] = { +static arc arcs_72_7[2] = { + {22, 8}, + {0, 7}, +}; +static arc arcs_72_8[1] = { {23, 2}, }; -static state states_74[8] = { - {1, arcs_74_0}, - {3, arcs_74_1}, - {1, arcs_74_2}, - {2, arcs_74_3}, - {2, arcs_74_4}, - {2, arcs_74_5}, - {2, arcs_74_6}, - {1, arcs_74_7}, +static state states_72[9] = { + {1, arcs_72_0}, + {4, arcs_72_1}, + {1, arcs_72_2}, + {1, arcs_72_3}, + {2, arcs_72_4}, + {2, arcs_72_5}, + {2, arcs_72_6}, + {2, arcs_72_7}, + {1, arcs_72_8}, }; -static arc arcs_75_0[1] = { - {159, 1}, +static arc arcs_73_0[1] = { + {156, 1}, }; -static arc arcs_75_1[1] = { +static arc arcs_73_1[1] = { {19, 2}, }; -static arc arcs_75_2[2] = { +static arc arcs_73_2[2] = { {13, 3}, {23, 4}, }; -static arc arcs_75_3[2] = { +static arc arcs_73_3[2] = { {9, 5}, {15, 6}, }; -static arc arcs_75_4[1] = { +static arc arcs_73_4[1] = { {24, 7}, }; -static arc arcs_75_5[1] = { +static arc arcs_73_5[1] = { {15, 6}, }; -static arc arcs_75_6[1] = { +static arc arcs_73_6[1] = { {23, 4}, }; -static arc arcs_75_7[1] = { +static arc arcs_73_7[1] = { {0, 7}, }; -static state states_75[8] = { - {1, arcs_75_0}, - {1, arcs_75_1}, - {2, arcs_75_2}, - {2, arcs_75_3}, - {1, arcs_75_4}, - {1, arcs_75_5}, - {1, arcs_75_6}, - {1, arcs_75_7}, +static state states_73[8] = { + {1, arcs_73_0}, + {1, arcs_73_1}, + {2, arcs_73_2}, + {2, arcs_73_3}, + {1, arcs_73_4}, + {1, arcs_73_5}, + {1, arcs_73_6}, + {1, arcs_73_7}, }; -static arc arcs_76_0[3] = { - {160, 1}, +static arc arcs_74_0[3] = { + {157, 1}, {29, 2}, {31, 3}, }; -static arc arcs_76_1[2] = { +static arc arcs_74_1[2] = { {28, 4}, {0, 1}, }; -static arc arcs_76_2[1] = { +static arc arcs_74_2[1] = { {22, 5}, }; -static arc arcs_76_3[1] = { +static arc arcs_74_3[1] = { {22, 6}, }; -static arc arcs_76_4[4] = { - {160, 1}, +static arc arcs_74_4[4] = { + {157, 1}, {29, 2}, {31, 3}, {0, 4}, }; -static arc arcs_76_5[2] = { +static arc arcs_74_5[2] = { {28, 7}, {0, 5}, }; -static arc arcs_76_6[1] = { +static arc arcs_74_6[1] = { {0, 6}, }; -static arc arcs_76_7[1] = { +static arc arcs_74_7[1] = { {31, 3}, }; -static state states_76[8] = { - {3, arcs_76_0}, - {2, arcs_76_1}, - {1, arcs_76_2}, - {1, arcs_76_3}, - {4, arcs_76_4}, - {2, arcs_76_5}, - {1, arcs_76_6}, - {1, arcs_76_7}, +static state states_74[8] = { + {3, arcs_74_0}, + {2, arcs_74_1}, + {1, arcs_74_2}, + {1, arcs_74_3}, + {4, arcs_74_4}, + {2, arcs_74_5}, + {1, arcs_74_6}, + {1, arcs_74_7}, }; -static arc arcs_77_0[1] = { +static arc arcs_75_0[1] = { {22, 1}, }; -static arc arcs_77_1[3] = { - {155, 2}, +static arc arcs_75_1[3] = { + {152, 2}, {27, 3}, {0, 1}, }; -static arc arcs_77_2[1] = { +static arc arcs_75_2[1] = { {0, 2}, }; -static arc arcs_77_3[1] = { +static arc arcs_75_3[1] = { {22, 2}, }; -static state states_77[4] = { - {1, arcs_77_0}, - {3, arcs_77_1}, - {1, arcs_77_2}, - {1, arcs_77_3}, +static state states_75[4] = { + {1, arcs_75_0}, + {3, arcs_75_1}, + {1, arcs_75_2}, + {1, arcs_75_3}, }; -static arc arcs_78_0[2] = { - {154, 1}, - {162, 1}, +static arc arcs_76_0[2] = { + {152, 1}, + {159, 1}, }; -static arc arcs_78_1[1] = { +static arc arcs_76_1[1] = { {0, 1}, }; -static state states_78[2] = { - {2, arcs_78_0}, - {1, arcs_78_1}, +static state states_76[2] = { + {2, arcs_76_0}, + {1, arcs_76_1}, }; -static arc arcs_79_0[1] = { +static arc arcs_77_0[1] = { {94, 1}, }; -static arc arcs_79_1[1] = { +static arc arcs_77_1[1] = { {61, 2}, }; -static arc arcs_79_2[1] = { +static arc arcs_77_2[1] = { {95, 3}, }; -static arc arcs_79_3[1] = { +static arc arcs_77_3[1] = { {105, 4}, }; -static arc arcs_79_4[2] = { - {161, 5}, +static arc arcs_77_4[2] = { + {158, 5}, {0, 4}, }; -static arc arcs_79_5[1] = { +static arc arcs_77_5[1] = { {0, 5}, }; -static state states_79[6] = { - {1, arcs_79_0}, - {1, arcs_79_1}, - {1, arcs_79_2}, - {1, arcs_79_3}, - {2, arcs_79_4}, - {1, arcs_79_5}, +static state states_77[6] = { + {1, arcs_77_0}, + {1, arcs_77_1}, + {1, arcs_77_2}, + {1, arcs_77_3}, + {2, arcs_77_4}, + {1, arcs_77_5}, }; -static arc arcs_80_0[1] = { +static arc arcs_78_0[1] = { {90, 1}, }; -static arc arcs_80_1[1] = { - {106, 2}, +static arc arcs_78_1[1] = { + {107, 2}, }; -static arc arcs_80_2[2] = { - {161, 3}, +static arc arcs_78_2[2] = { + {158, 3}, {0, 2}, }; -static arc arcs_80_3[1] = { +static arc arcs_78_3[1] = { {0, 3}, }; -static state states_80[4] = { - {1, arcs_80_0}, - {1, arcs_80_1}, - {2, arcs_80_2}, - {1, arcs_80_3}, +static state states_78[4] = { + {1, arcs_78_0}, + {1, arcs_78_1}, + {2, arcs_78_2}, + {1, arcs_78_3}, }; -static arc arcs_81_0[2] = { - {155, 1}, - {164, 1}, -}; -static arc arcs_81_1[1] = { - {0, 1}, -}; -static state states_81[2] = { - {2, arcs_81_0}, - {1, arcs_81_1}, -}; -static arc arcs_82_0[1] = { - {94, 1}, -}; -static arc arcs_82_1[1] = { - {61, 2}, -}; -static arc arcs_82_2[1] = { - {95, 3}, -}; -static arc arcs_82_3[1] = { - {107, 4}, -}; -static arc arcs_82_4[2] = { - {163, 5}, - {0, 4}, -}; -static arc arcs_82_5[1] = { - {0, 5}, -}; -static state states_82[6] = { - {1, arcs_82_0}, - {1, arcs_82_1}, - {1, arcs_82_2}, - {1, arcs_82_3}, - {2, arcs_82_4}, - {1, arcs_82_5}, -}; -static arc arcs_83_0[1] = { - {90, 1}, -}; -static arc arcs_83_1[1] = { - {106, 2}, -}; -static arc arcs_83_2[2] = { - {163, 3}, - {0, 2}, -}; -static arc arcs_83_3[1] = { - {0, 3}, -}; -static state states_83[4] = { - {1, arcs_83_0}, - {1, arcs_83_1}, - {2, arcs_83_2}, - {1, arcs_83_3}, -}; -static arc arcs_84_0[1] = { +static arc arcs_79_0[1] = { {22, 1}, }; -static arc arcs_84_1[2] = { +static arc arcs_79_1[2] = { {28, 0}, {0, 1}, }; -static state states_84[2] = { - {1, arcs_84_0}, - {2, arcs_84_1}, +static state states_79[2] = { + {1, arcs_79_0}, + {2, arcs_79_1}, }; -static arc arcs_85_0[1] = { +static arc arcs_80_0[1] = { {19, 1}, }; -static arc arcs_85_1[1] = { +static arc arcs_80_1[1] = { {0, 1}, }; -static state states_85[2] = { - {1, arcs_85_0}, - {1, arcs_85_1}, +static state states_80[2] = { + {1, arcs_80_0}, + {1, arcs_80_1}, }; -static arc arcs_86_0[1] = { - {167, 1}, +static arc arcs_81_0[1] = { + {162, 1}, }; -static arc arcs_86_1[2] = { +static arc arcs_81_1[2] = { {9, 2}, {0, 1}, }; -static arc arcs_86_2[1] = { +static arc arcs_81_2[1] = { {0, 2}, }; -static state states_86[3] = { - {1, arcs_86_0}, - {2, arcs_86_1}, - {1, arcs_86_2}, +static state states_81[3] = { + {1, arcs_81_0}, + {2, arcs_81_1}, + {1, arcs_81_2}, }; -static dfa dfas[87] = { +static dfa dfas[82] = { {256, "single_input", 0, 3, states_0, - "\004\050\014\000\000\000\000\120\360\064\014\144\011\040\010\000\200\041\044\203\200"}, + "\004\050\014\000\000\000\000\120\360\064\014\144\011\040\004\000\300\020\312\020\004"}, {257, "file_input", 0, 2, states_1, - "\204\050\014\000\000\000\000\120\360\064\014\144\011\040\010\000\200\041\044\203\200"}, + "\204\050\014\000\000\000\000\120\360\064\014\144\011\040\004\000\300\020\312\020\004"}, {258, "eval_input", 0, 3, states_2, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\003\000"}, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\004\000\300\020\312\000\000"}, {259, "decorator", 0, 7, states_3, "\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {260, "decorators", 0, 2, states_4, @@ -1887,13 +1784,13 @@ {270, "vfplist", 0, 3, states_14, "\000\040\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {271, "stmt", 0, 2, states_15, - "\000\050\014\000\000\000\000\120\360\064\014\144\011\040\010\000\200\041\044\203\200"}, + "\000\050\014\000\000\000\000\120\360\064\014\144\011\040\004\000\300\020\312\020\004"}, {272, "simple_stmt", 0, 4, states_16, - "\000\040\010\000\000\000\000\120\360\064\014\000\000\040\010\000\200\041\044\003\200"}, + "\000\040\010\000\000\000\000\120\360\064\014\000\000\040\004\000\300\020\312\000\004"}, {273, "small_stmt", 0, 2, states_17, - "\000\040\010\000\000\000\000\120\360\064\014\000\000\040\010\000\200\041\044\003\200"}, + "\000\040\010\000\000\000\000\120\360\064\014\000\000\040\004\000\300\020\312\000\004"}, {274, "expr_stmt", 0, 6, states_18, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\003\000"}, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\004\000\300\020\312\000\000"}, {275, "augassign", 0, 2, states_19, "\000\000\000\000\000\000\377\017\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {276, "del_stmt", 0, 3, states_20, @@ -1901,7 +1798,7 @@ {277, "pass_stmt", 0, 2, states_21, "\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {278, "flow_stmt", 0, 2, states_22, - "\000\000\000\000\000\000\000\000\360\000\000\000\000\000\000\000\000\000\000\000\200"}, + "\000\000\000\000\000\000\000\000\360\000\000\000\000\000\000\000\000\000\000\000\004"}, {279, "break_stmt", 0, 2, states_23, "\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000"}, {280, "continue_stmt", 0, 2, states_24, @@ -1909,7 +1806,7 @@ {281, "return_stmt", 0, 3, states_25, "\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000"}, {282, "yield_stmt", 0, 2, states_26, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200"}, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\004"}, {283, "raise_stmt", 0, 7, states_27, "\000\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000"}, {284, "import_stmt", 0, 2, states_28, @@ -1933,7 +1830,7 @@ {293, "assert_stmt", 0, 5, states_37, "\000\000\000\000\000\000\000\000\000\000\010\000\000\000\000\000\000\000\000\000\000"}, {294, "compound_stmt", 0, 2, states_38, - "\000\010\004\000\000\000\000\000\000\000\000\144\011\000\000\000\000\000\000\200\000"}, + "\000\010\004\000\000\000\000\000\000\000\000\144\011\000\000\000\000\000\000\020\000"}, {295, "if_stmt", 0, 8, states_39, "\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000"}, {296, "while_stmt", 0, 8, states_40, @@ -1949,89 +1846,79 @@ {301, "except_clause", 0, 5, states_45, "\000\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000"}, {302, "suite", 0, 5, states_46, - "\004\040\010\000\000\000\000\120\360\064\014\000\000\040\010\000\200\041\044\003\200"}, - {303, "testlist_safe", 0, 5, states_47, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\003\000"}, - {304, "old_test", 0, 2, states_48, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\003\000"}, - {305, "old_lambdef", 0, 5, states_49, + "\004\040\010\000\000\000\000\120\360\064\014\000\000\040\004\000\300\020\312\000\004"}, + {303, "test", 0, 6, states_47, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\004\000\300\020\312\000\000"}, + {304, "test_nocond", 0, 2, states_48, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\004\000\300\020\312\000\000"}, + {305, "lambdef", 0, 5, states_49, "\000\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000"}, - {306, "test", 0, 6, states_50, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\003\000"}, + {306, "lambdef_nocond", 0, 5, states_50, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000"}, {307, "or_test", 0, 2, states_51, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\010\000\200\041\044\003\000"}, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\004\000\300\020\312\000\000"}, {308, "and_test", 0, 2, states_52, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\010\000\200\041\044\003\000"}, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\004\000\300\020\312\000\000"}, {309, "not_test", 0, 3, states_53, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\010\000\200\041\044\003\000"}, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\004\000\300\020\312\000\000"}, {310, "comparison", 0, 2, states_54, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\200\041\044\003\000"}, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\300\020\312\000\000"}, {311, "comp_op", 0, 4, states_55, - "\000\000\000\000\000\000\000\000\000\000\000\200\000\000\310\037\000\000\000\000\000"}, + "\000\000\000\000\000\000\000\000\000\000\000\200\000\000\344\017\000\000\000\000\000"}, {312, "expr", 0, 2, states_56, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\200\041\044\003\000"}, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\300\020\312\000\000"}, {313, "xor_expr", 0, 2, states_57, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\200\041\044\003\000"}, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\300\020\312\000\000"}, {314, "and_expr", 0, 2, states_58, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\200\041\044\003\000"}, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\300\020\312\000\000"}, {315, "shift_expr", 0, 2, states_59, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\200\041\044\003\000"}, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\300\020\312\000\000"}, {316, "arith_expr", 0, 2, states_60, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\200\041\044\003\000"}, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\300\020\312\000\000"}, {317, "term", 0, 2, states_61, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\200\041\044\003\000"}, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\300\020\312\000\000"}, {318, "factor", 0, 3, states_62, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\200\041\044\003\000"}, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\300\020\312\000\000"}, {319, "power", 0, 4, states_63, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\044\003\000"}, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\312\000\000"}, {320, "atom", 0, 11, states_64, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\044\003\000"}, - {321, "listmaker", 0, 5, states_65, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\003\000"}, - {322, "testlist_gexp", 0, 5, states_66, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\003\000"}, - {323, "lambdef", 0, 5, states_67, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000"}, - {324, "trailer", 0, 7, states_68, - "\000\040\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\004\000\000"}, - {325, "subscriptlist", 0, 3, states_69, - "\000\040\210\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\003\000"}, - {326, "subscript", 0, 5, states_70, - "\000\040\210\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\003\000"}, - {327, "sliceop", 0, 3, states_71, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\312\000\000"}, + {321, "testlist_comp", 0, 5, states_65, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\004\000\300\020\312\000\000"}, + {322, "trailer", 0, 7, states_66, + "\000\040\000\000\000\000\000\000\000\040\000\000\000\000\000\000\000\000\002\000\000"}, + {323, "subscriptlist", 0, 3, states_67, + "\000\040\210\000\000\000\000\000\000\040\000\000\000\040\004\000\300\020\312\000\000"}, + {324, "subscript", 0, 5, states_68, + "\000\040\210\000\000\000\000\000\000\040\000\000\000\040\004\000\300\020\312\000\000"}, + {325, "sliceop", 0, 3, states_69, "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {328, "exprlist", 0, 3, states_72, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\200\041\044\003\000"}, - {329, "testlist", 0, 3, states_73, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\003\000"}, - {330, "dictsetmaker", 0, 8, states_74, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\003\000"}, - {331, "classdef", 0, 8, states_75, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200\000"}, - {332, "arglist", 0, 8, states_76, - "\000\040\010\240\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\003\000"}, - {333, "argument", 0, 4, states_77, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\003\000"}, - {334, "list_iter", 0, 2, states_78, + {326, "exprlist", 0, 3, states_70, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\000\000\000\300\020\312\000\000"}, + {327, "testlist", 0, 3, states_71, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\004\000\300\020\312\000\000"}, + {328, "dictorsetmaker", 0, 9, states_72, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\004\000\300\020\312\000\000"}, + {329, "classdef", 0, 8, states_73, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\020\000"}, + {330, "arglist", 0, 8, states_74, + "\000\040\010\240\000\000\000\000\000\040\000\000\000\040\004\000\300\020\312\000\000"}, + {331, "argument", 0, 4, states_75, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\004\000\300\020\312\000\000"}, + {332, "comp_iter", 0, 2, states_76, "\000\000\000\000\000\000\000\000\000\000\000\104\000\000\000\000\000\000\000\000\000"}, - {335, "list_for", 0, 6, states_79, + {333, "comp_for", 0, 6, states_77, "\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000"}, - {336, "list_if", 0, 4, states_80, + {334, "comp_if", 0, 4, states_78, "\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000"}, - {337, "gen_iter", 0, 2, states_81, - "\000\000\000\000\000\000\000\000\000\000\000\104\000\000\000\000\000\000\000\000\000"}, - {338, "gen_for", 0, 6, states_82, - "\000\000\000\000\000\000\000\000\000\000\000\100\000\000\000\000\000\000\000\000\000"}, - {339, "gen_if", 0, 4, states_83, - "\000\000\000\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\000\000"}, - {340, "testlist1", 0, 2, states_84, - "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\010\000\200\041\044\003\000"}, - {341, "encoding_decl", 0, 2, states_85, + {335, "testlist1", 0, 2, states_79, + "\000\040\010\000\000\000\000\000\000\040\000\000\000\040\004\000\300\020\312\000\000"}, + {336, "encoding_decl", 0, 2, states_80, "\000\000\010\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {342, "yield_expr", 0, 3, states_86, - "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\200"}, + {337, "yield_expr", 0, 3, states_81, + "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\004"}, }; -static label labels[168] = { +static label labels[163] = { {0, "EMPTY"}, {256, 0}, {4, 0}, @@ -2041,12 +1928,12 @@ {271, 0}, {0, 0}, {258, 0}, - {329, 0}, + {327, 0}, {259, 0}, {50, 0}, {291, 0}, {7, 0}, - {332, 0}, + {330, 0}, {8, 0}, {260, 0}, {261, 0}, @@ -2054,7 +1941,7 @@ {1, 0}, {262, 0}, {51, 0}, - {306, 0}, + {303, 0}, {11, 0}, {302, 0}, {263, 0}, @@ -2079,7 +1966,7 @@ {292, 0}, {293, 0}, {275, 0}, - {342, 0}, + {337, 0}, {37, 0}, {38, 0}, {39, 0}, @@ -2093,7 +1980,7 @@ {47, 0}, {49, 0}, {1, "del"}, - {328, 0}, + {326, 0}, {1, "pass"}, {279, 0}, {280, 0}, @@ -2121,7 +2008,7 @@ {297, 0}, {298, 0}, {299, 0}, - {331, 0}, + {329, 0}, {1, "if"}, {1, "elif"}, {1, "else"}, @@ -2137,12 +2024,11 @@ {1, "except"}, {5, 0}, {6, 0}, - {303, 0}, - {304, 0}, {307, 0}, {305, 0}, + {304, 0}, + {306, 0}, {1, "lambda"}, - {323, 0}, {308, 0}, {1, "or"}, {309, 0}, @@ -2176,34 +2062,30 @@ {32, 0}, {319, 0}, {320, 0}, - {324, 0}, {322, 0}, - {9, 0}, {321, 0}, + {9, 0}, {10, 0}, {26, 0}, - {330, 0}, + {328, 0}, {27, 0}, {2, 0}, {3, 0}, - {335, 0}, - {338, 0}, + {333, 0}, + {323, 0}, + {324, 0}, {325, 0}, - {326, 0}, - {327, 0}, {1, "class"}, - {333, 0}, + {331, 0}, + {332, 0}, {334, 0}, + {335, 0}, {336, 0}, - {337, 0}, - {339, 0}, - {340, 0}, - {341, 0}, {1, "yield"}, }; grammar _PyParser_Grammar = { - 87, + 82, dfas, - {168, labels}, + {163, labels}, 256 }; Index: Python/ast.c =================================================================== --- Python/ast.c (Revision 53930) +++ Python/ast.c (Arbeitskopie) @@ -41,6 +41,10 @@ #define LINENO(n) ((n)->n_lineno) #endif +#define COMP_GENEXP 0 +#define COMP_LISTCOMP 1 +#define COMP_SETCOMP 2 + static identifier new_identifier(const char* n, PyArena *arena) { PyObject* id = PyString_InternFromString(n); @@ -231,7 +235,7 @@ case eval_input: { expr_ty testlist_ast; - /* XXX Why not gen_for here? */ + /* XXX Why not comp_for here? */ testlist_ast = ast_for_testlist(&c, CHILD(n, 0)); if (!testlist_ast) goto error; @@ -530,19 +534,14 @@ asdl_seq *seq; expr_ty expression; int i; - assert(TYPE(n) == testlist - || TYPE(n) == listmaker - || TYPE(n) == testlist_gexp - || TYPE(n) == testlist_safe - || TYPE(n) == testlist1 - ); + assert(TYPE(n) == testlist || TYPE(n) == testlist_comp); seq = asdl_seq_new((NCH(n) + 1) / 2, c->c_arena); if (!seq) return NULL; for (i = 0; i < NCH(n); i += 2) { - assert(TYPE(CHILD(n, i)) == test || TYPE(CHILD(n, i)) == old_test); + assert(TYPE(CHILD(n, i)) == test || TYPE(CHILD(n, i)) == test_nocond); expression = ast_for_expr(c, CHILD(n, i)); if (!expression) @@ -1022,7 +1021,8 @@ static expr_ty ast_for_lambdef(struct compiling *c, const node *n) { - /* lambdef: 'lambda' [varargslist] ':' test */ + /* lambdef: 'lambda' [varargslist] ':' test + lambdef_nocond: 'lambda' [varargslist] ':' test_nocond */ arguments_ty args; expr_ty expression; @@ -1067,190 +1067,34 @@ c->c_arena); } -/* XXX(nnorwitz): the listcomp and genexpr code should be refactored - so there is only a single version. Possibly for loops can also re-use - the code. -*/ - -/* Count the number of 'for' loop in a list comprehension. - - Helper for ast_for_listcomp(). -*/ - -static int -count_list_fors(const node *n) -{ - int n_fors = 0; - node *ch = CHILD(n, 1); - - count_list_for: - n_fors++; - REQ(ch, list_for); - if (NCH(ch) == 5) - ch = CHILD(ch, 4); - else - return n_fors; - count_list_iter: - REQ(ch, list_iter); - ch = CHILD(ch, 0); - if (TYPE(ch) == list_for) - goto count_list_for; - else if (TYPE(ch) == list_if) { - if (NCH(ch) == 3) { - ch = CHILD(ch, 2); - goto count_list_iter; - } - else - return n_fors; - } - - /* Should never be reached */ - PyErr_SetString(PyExc_SystemError, "logic error in count_list_fors"); - return -1; -} - -/* Count the number of 'if' statements in a list comprehension. - - Helper for ast_for_listcomp(). -*/ - -static int -count_list_ifs(const node *n) -{ - int n_ifs = 0; - - count_list_iter: - REQ(n, list_iter); - if (TYPE(CHILD(n, 0)) == list_for) - return n_ifs; - n = CHILD(n, 0); - REQ(n, list_if); - n_ifs++; - if (NCH(n) == 2) - return n_ifs; - n = CHILD(n, 2); - goto count_list_iter; -} - -static expr_ty -ast_for_listcomp(struct compiling *c, const node *n) -{ - /* listmaker: test ( list_for | (',' test)* [','] ) - list_for: 'for' exprlist 'in' testlist_safe [list_iter] - list_iter: list_for | list_if - list_if: 'if' test [list_iter] - testlist_safe: test [(',' test)+ [',']] - */ - expr_ty elt; - asdl_seq *listcomps; - int i, n_fors; - node *ch; - - REQ(n, listmaker); - assert(NCH(n) > 1); - - elt = ast_for_expr(c, CHILD(n, 0)); - if (!elt) - return NULL; - - n_fors = count_list_fors(n); - if (n_fors == -1) - return NULL; - - listcomps = asdl_seq_new(n_fors, c->c_arena); - if (!listcomps) - return NULL; - - ch = CHILD(n, 1); - for (i = 0; i < n_fors; i++) { - comprehension_ty lc; - asdl_seq *t; - expr_ty expression; - node *for_ch; - - REQ(ch, list_for); - - for_ch = CHILD(ch, 1); - t = ast_for_exprlist(c, for_ch, Store); - if (!t) - return NULL; - expression = ast_for_testlist(c, CHILD(ch, 3)); - if (!expression) - return NULL; - - /* Check the # of children rather than the length of t, since - [x for x, in ... ] has 1 element in t, but still requires a Tuple. */ - if (NCH(for_ch) == 1) - lc = comprehension((expr_ty)asdl_seq_GET(t, 0), expression, NULL, - c->c_arena); - else - lc = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset, - c->c_arena), - expression, NULL, c->c_arena); - if (!lc) - return NULL; - - if (NCH(ch) == 5) { - int j, n_ifs; - asdl_seq *ifs; - - ch = CHILD(ch, 4); - n_ifs = count_list_ifs(ch); - if (n_ifs == -1) - return NULL; - - ifs = asdl_seq_new(n_ifs, c->c_arena); - if (!ifs) - return NULL; - - for (j = 0; j < n_ifs; j++) { - REQ(ch, list_iter); - ch = CHILD(ch, 0); - REQ(ch, list_if); - - asdl_seq_SET(ifs, j, ast_for_expr(c, CHILD(ch, 1))); - if (NCH(ch) == 3) - ch = CHILD(ch, 2); - } - /* on exit, must guarantee that ch is a list_for */ - if (TYPE(ch) == list_iter) - ch = CHILD(ch, 0); - lc->ifs = ifs; - } - asdl_seq_SET(listcomps, i, lc); - } - - return ListComp(elt, listcomps, LINENO(n), n->n_col_offset, c->c_arena); -} - /* - Count the number of 'for' loops in a generator expression. + Count the number of 'for' loops in a comprehension. - Helper for ast_for_genexp(). + Helper for ast_for_comprehension(). */ static int -count_gen_fors(const node *n) +count_comp_fors(const node *n) { int n_fors = 0; node *ch = CHILD(n, 1); - count_gen_for: + count_comp_for: n_fors++; - REQ(ch, gen_for); + REQ(ch, comp_for); if (NCH(ch) == 5) ch = CHILD(ch, 4); else return n_fors; - count_gen_iter: - REQ(ch, gen_iter); + count_comp_iter: + REQ(ch, comp_iter); ch = CHILD(ch, 0); - if (TYPE(ch) == gen_for) - goto count_gen_for; - else if (TYPE(ch) == gen_if) { + if (TYPE(ch) == comp_for) + goto count_comp_for; + else if (TYPE(ch) == comp_if) { if (NCH(ch) == 3) { ch = CHILD(ch, 2); - goto count_gen_iter; + goto count_comp_iter; } else return n_fors; @@ -1258,26 +1102,26 @@ /* Should never be reached */ PyErr_SetString(PyExc_SystemError, - "logic error in count_gen_fors"); + "logic error in count_comp_fors"); return -1; } -/* Count the number of 'if' statements in a generator expression. +/* Count the number of 'if' statements in a comprehension. - Helper for ast_for_genexp(). + Helper for ast_for_comprehension(). */ static int -count_gen_ifs(const node *n) +count_comp_ifs(const node *n) { int n_ifs = 0; while (1) { - REQ(n, gen_iter); - if (TYPE(CHILD(n, 0)) == gen_for) + REQ(n, comp_iter); + if (TYPE(CHILD(n, 0)) == comp_for) return n_ifs; n = CHILD(n, 0); - REQ(n, gen_if); + REQ(n, comp_if); n_ifs++; if (NCH(n) == 2) return n_ifs; @@ -1285,40 +1129,38 @@ } } -/* TODO(jhylton): Combine with list comprehension code? */ static expr_ty -ast_for_genexp(struct compiling *c, const node *n) +ast_for_comprehension(struct compiling *c, const node *n, int type) { - /* testlist_gexp: test ( gen_for | (',' test)* [','] ) - argument: [test '='] test [gen_for] # Really [keyword '='] test */ + /* testlist_comp: test ( comp_for | (',' test)* [','] ) + argument: [test '='] test [comp_for] # Really [keyword '='] test */ expr_ty elt; - asdl_seq *genexps; + asdl_seq *comps; int i, n_fors; node *ch; - assert(TYPE(n) == (testlist_gexp) || TYPE(n) == (argument)); assert(NCH(n) > 1); elt = ast_for_expr(c, CHILD(n, 0)); if (!elt) return NULL; - n_fors = count_gen_fors(n); + n_fors = count_comp_fors(n); if (n_fors == -1) return NULL; - genexps = asdl_seq_new(n_fors, c->c_arena); - if (!genexps) + comps = asdl_seq_new(n_fors, c->c_arena); + if (!comps) return NULL; ch = CHILD(n, 1); for (i = 0; i < n_fors; i++) { - comprehension_ty ge; + comprehension_ty comp; asdl_seq *t; expr_ty expression; node *for_ch; - REQ(ch, gen_for); + REQ(ch, comp_for); for_ch = CHILD(ch, 1); t = ast_for_exprlist(c, for_ch, Store); @@ -1331,14 +1173,14 @@ /* Check the # of children rather than the length of t, since (x for x, in ...) has 1 element in t, but still requires a Tuple. */ if (NCH(for_ch) == 1) - ge = comprehension((expr_ty)asdl_seq_GET(t, 0), expression, - NULL, c->c_arena); + comp = comprehension((expr_ty)asdl_seq_GET(t, 0), expression, + NULL, c->c_arena); else - ge = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset, - c->c_arena), - expression, NULL, c->c_arena); + comp = comprehension(Tuple(t, Store, LINENO(ch), ch->n_col_offset, + c->c_arena), + expression, NULL, c->c_arena); - if (!ge) + if (!comp) return NULL; if (NCH(ch) == 5) { @@ -1346,7 +1188,7 @@ asdl_seq *ifs; ch = CHILD(ch, 4); - n_ifs = count_gen_ifs(ch); + n_ifs = count_comp_ifs(ch); if (n_ifs == -1) return NULL; @@ -1355,9 +1197,9 @@ return NULL; for (j = 0; j < n_ifs; j++) { - REQ(ch, gen_iter); + REQ(ch, comp_iter); ch = CHILD(ch, 0); - REQ(ch, gen_if); + REQ(ch, comp_if); expression = ast_for_expr(c, CHILD(ch, 1)); if (!expression) @@ -1366,22 +1208,52 @@ if (NCH(ch) == 3) ch = CHILD(ch, 2); } - /* on exit, must guarantee that ch is a gen_for */ - if (TYPE(ch) == gen_iter) + /* on exit, must guarantee that ch is a comp_for */ + if (TYPE(ch) == comp_iter) ch = CHILD(ch, 0); - ge->ifs = ifs; + comp->ifs = ifs; } - asdl_seq_SET(genexps, i, ge); + asdl_seq_SET(comps, i, comp); } - - return GeneratorExp(elt, genexps, LINENO(n), n->n_col_offset, c->c_arena); + + if (type == COMP_GENEXP) + return GeneratorExp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena); + else if (type == COMP_LISTCOMP) + return ListComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena); + else if (type == COMP_SETCOMP) + return SetComp(elt, comps, LINENO(n), n->n_col_offset, c->c_arena); + else + /* Should never happen */ + return NULL; } static expr_ty +ast_for_genexp(struct compiling *c, const node *n) +{ + assert(TYPE(n) == (testlist_comp) || TYPE(n) == (argument)); + return ast_for_comprehension(c, n, COMP_GENEXP); +} + +static expr_ty +ast_for_listcomp(struct compiling *c, const node *n) +{ + assert(TYPE(n) == (testlist_comp)); + return ast_for_comprehension(c, n, COMP_LISTCOMP); +} + +static expr_ty +ast_for_setcomp(struct compiling *c, const node *n) +{ + assert(TYPE(n) == (dictorsetmaker)); + return ast_for_comprehension(c, n, COMP_SETCOMP); +} + + +static expr_ty ast_for_atom(struct compiling *c, const node *n) { - /* atom: '(' [yield_expr|testlist_gexp] ')' | '[' [listmaker] ']' - | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING+ + /* atom: '(' [yield_expr|testlist_comp] ')' | '[' [testlist_comp] ']' + | '{' [dictmaker|testlist_comp] '}' | NAME | NUMBER | STRING+ */ node *ch = CHILD(n, 0); int bytesmode = 0; @@ -1420,18 +1292,19 @@ if (TYPE(ch) == yield_expr) return ast_for_expr(c, ch); - - if ((NCH(ch) > 1) && (TYPE(CHILD(ch, 1)) == gen_for)) + + /* testlist_comp: test ( comp_for | (',' test)* [','] ) */ + if ((NCH(ch) > 1) && (TYPE(CHILD(ch, 1)) == comp_for)) return ast_for_genexp(c, ch); - return ast_for_testlist_gexp(c, ch); + return ast_for_testlist(c, ch); case LSQB: /* list (or list comprehension) */ ch = CHILD(n, 1); if (TYPE(ch) == RSQB) return List(NULL, Load, LINENO(n), n->n_col_offset, c->c_arena); - REQ(ch, listmaker); + REQ(ch, testlist_comp); if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) { asdl_seq *elts = seq_for_testlist(c, ch); if (!elts) @@ -1442,27 +1315,32 @@ else return ast_for_listcomp(c, ch); case LBRACE: { - /* dictsetmaker: test ':' test (',' test ':' test)* [','] | - * test (',' test)* [','] */ + /* dictorsetmaker: test ':' test (',' test ':' test)* [','] | + * test (gen_for | (',' test)* [',']) */ int i, size; asdl_seq *keys, *values; ch = CHILD(n, 1); - if (NCH(ch) == 1 || (NCH(ch) > 0 && STR(CHILD(ch, 1))[0] == ',')) { - /* it's a set */ + if (TYPE(ch) == RBRACE) { + /* it's an empty dict */ + return Dict(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena); + } else if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) { + /* it's a simple set */ size = (NCH(ch) + 1) / 2; /* +1 in case no trailing comma */ - keys = asdl_seq_new(size, c->c_arena); - if (!keys) + asdl_seq *elts = asdl_seq_new(size, c->c_arena); + if (!elts) return NULL; - for (i = 0; i < NCH(ch); i += 2) { expr_ty expression; expression = ast_for_expr(c, CHILD(ch, i)); if (!expression) return NULL; - asdl_seq_SET(keys, i / 2, expression); + asdl_seq_SET(elts, i / 2, expression); } - return Set(keys, LINENO(n), n->n_col_offset, c->c_arena); + return Set(elts, LINENO(n), n->n_col_offset, c->c_arena); + } else if (TYPE(CHILD(ch, 1)) == comp_for) { + /* it's a set comprehension */ + return ast_for_setcomp(c, ch); } else { /* it's a dict */ size = (NCH(ch) + 1) / 4; /* +1 in case no trailing comma */ @@ -1790,6 +1668,7 @@ { /* handle the full range of simple expressions test: or_test ['if' or_test 'else' test] | lambdef + test_nocond: or_test | lambdef_nocond or_test: and_test ('or' and_test)* and_test: not_test ('and' not_test)* not_test: 'not' not_test | comparison @@ -1802,15 +1681,6 @@ term: factor (('*'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power power: atom trailer* ('**' factor)* - - As well as modified versions that exist for backward compatibility, - to explicitly allow: - [ x for x in lambda: 0, lambda: 1 ] - (which would be ambiguous without these extra rules) - - old_test: or_test | old_lambdef - old_lambdef: 'lambda' [vararglist] ':' old_test - */ asdl_seq *seq; @@ -1819,9 +1689,9 @@ loop: switch (TYPE(n)) { case test: - case old_test: + case test_nocond: if (TYPE(CHILD(n, 0)) == lambdef || - TYPE(CHILD(n, 0)) == old_lambdef) + TYPE(CHILD(n, 0)) == lambdef_nocond) return ast_for_lambdef(c, CHILD(n, 0)); else if (NCH(n) > 1) return ast_for_ifexpr(c, n); @@ -1947,7 +1817,7 @@ /* arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test) - argument: [test '='] test [gen_for] # Really [keyword '='] test + argument: [test '='] test [comp_for] # Really [keyword '='] test */ int i, nargs, nkeywords, ngens; @@ -1965,7 +1835,7 @@ if (TYPE(ch) == argument) { if (NCH(ch) == 1) nargs++; - else if (TYPE(CHILD(ch, 1)) == gen_for) + else if (TYPE(CHILD(ch, 1)) == comp_for) ngens++; else nkeywords++; @@ -2005,7 +1875,7 @@ return NULL; asdl_seq_SET(args, nargs++, e); } - else if (TYPE(CHILD(ch, 1)) == gen_for) { + else if (TYPE(CHILD(ch, 1)) == comp_for) { e = ast_for_genexp(c, ch); if (!e) return NULL; @@ -2057,18 +1927,16 @@ static expr_ty ast_for_testlist(struct compiling *c, const node* n) { - /* testlist_gexp: test (',' test)* [','] */ + /* testlist_comp: test (comp_for | (',' test)* [',']) */ /* testlist: test (',' test)* [','] */ - /* testlist_safe: test (',' test)+ [','] */ /* testlist1: test (',' test)* */ assert(NCH(n) > 0); - if (TYPE(n) == testlist_gexp) { + if (TYPE(n) == testlist_comp) { if (NCH(n) > 1) - assert(TYPE(CHILD(n, 1)) != gen_for); + assert(TYPE(CHILD(n, 1)) != comp_for); } else { assert(TYPE(n) == testlist || - TYPE(n) == testlist_safe || TYPE(n) == testlist1); } if (NCH(n) == 1) @@ -2081,17 +1949,6 @@ } } -static expr_ty -ast_for_testlist_gexp(struct compiling *c, const node* n) -{ - /* testlist_gexp: test ( gen_for | (',' test)* [','] ) */ - /* argument: test [ gen_for ] */ - assert(TYPE(n) == testlist_gexp || TYPE(n) == argument); - if (NCH(n) > 1 && TYPE(CHILD(n, 1)) == gen_for) - return ast_for_genexp(c, n); - return ast_for_testlist(c, n); -} - /* like ast_for_testlist() but returns a sequence */ static asdl_seq* ast_for_class_bases(struct compiling *c, const node* n) Index: Python/symtable.c =================================================================== --- Python/symtable.c (Revision 53930) +++ Python/symtable.c (Arbeitskopie) @@ -70,7 +70,7 @@ ste->ste_generator = 0; ste->ste_returns_value = 0; - if (PyDict_SetItem(st->st_symbols, ste->ste_id, (PyObject *)ste) < 0) + if (PyDict_SetItem(st->st_blocks, ste->ste_id, (PyObject *)ste) < 0) goto fail; return ste; @@ -166,6 +166,8 @@ static int symtable_visit_stmt(struct symtable *st, stmt_ty s); static int symtable_visit_expr(struct symtable *st, expr_ty s); static int symtable_visit_genexp(struct symtable *st, expr_ty s); +static int symtable_visit_listcomp(struct symtable *st, expr_ty s); +static int symtable_visit_setcomp(struct symtable *st, expr_ty s); static int symtable_visit_arguments(struct symtable *st, arguments_ty); static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty); static int symtable_visit_alias(struct symtable *st, alias_ty); @@ -180,7 +182,8 @@ static int symtable_visit_annotations(struct symtable *st, stmt_ty s); -static identifier top = NULL, lambda = NULL, genexpr = NULL; +static identifier top = NULL, lambda = NULL, genexpr = NULL, + listcomp = NULL, setcomp = NULL; #define GET_IDENTIFIER(VAR) \ ((VAR) ? (VAR) : ((VAR) = PyString_InternFromString(# VAR))) @@ -198,14 +201,13 @@ return NULL; st->st_filename = NULL; - st->st_symbols = NULL; + st->st_blocks = NULL; if ((st->st_stack = PyList_New(0)) == NULL) goto fail; - if ((st->st_symbols = PyDict_New()) == NULL) + if ((st->st_blocks = PyDict_New()) == NULL) goto fail; st->st_cur = NULL; - st->st_tmpname = 0; st->st_private = NULL; return st; fail: @@ -224,6 +226,7 @@ return st; st->st_filename = filename; st->st_future = future; + /* Make the initial symbol information gathering pass */ if (!GET_IDENTIFIER(top) || !symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0)) { PySymtable_Free(st); @@ -232,7 +235,6 @@ st->st_top = st->st_cur; st->st_cur->ste_unoptimized = OPT_TOPLEVEL; - /* Any other top-level initialization? */ switch (mod->kind) { case Module_kind: seq = mod->v.Module.body; @@ -261,6 +263,7 @@ PySymtable_Free(st); return NULL; } + /* Make the second symbol analysis pass */ if (symtable_analyze(st)) return st; PySymtable_Free(st); @@ -274,7 +277,7 @@ void PySymtable_Free(struct symtable *st) { - Py_XDECREF(st->st_symbols); + Py_XDECREF(st->st_blocks); Py_XDECREF(st->st_stack); PyMem_Free((void *)st); } @@ -287,7 +290,7 @@ k = PyLong_FromVoidPtr(key); if (k == NULL) return NULL; - v = PyDict_GetItem(st->st_symbols, k); + v = PyDict_GetItem(st->st_blocks, k); if (v) { assert(PySTEntry_Check(v)); Py_INCREF(v); @@ -308,7 +311,7 @@ if (!v) return 0; assert(PyInt_Check(v)); - return (PyInt_AS_LONG(v) >> SCOPE_OFF) & SCOPE_MASK; + return (PyInt_AS_LONG(v) >> SCOPE_OFFSET) & SCOPE_MASK; } @@ -319,7 +322,7 @@ it determines which local variables are cell variables; they provide bindings that are used for free variables in enclosed blocks. - There are also two kinds of free variables, implicit and explicit. An + There are also two kinds of global variables, implicit and explicit. An explicit global is declared with the global statement. An implicit global is a free variable for which the compiler has found no binding in an enclosing function scope. The implicit global is either a global @@ -329,24 +332,30 @@ is treated as a local. The symbol table requires two passes to determine the scope of each name. - The first pass collects raw facts from the AST: the name is a parameter - here, the name is used by not defined here, etc. The second pass analyzes - these facts during a pass over the PySTEntryObjects created during pass 1. + The first pass collects raw facts from the AST via the symtable_visit_* + functions: the name is a parameter here, the name is used but not defined + here, etc. The second pass analyzes these facts during a pass over the + PySTEntryObjects created during pass 1. When a function is entered during the second pass, the parent passes the set of all name bindings visible to its children. These bindings - are used to determine if the variable is free or an implicit global. + are used to determine if non-local variables are free or implicit globals. After doing the local analysis, it analyzes each of its child blocks - using an updated set of name bindings. + using an updated set of name bindings. - The children update the free variable set. If a local variable is free - in a child, the variable is marked as a cell. The current function must - provide runtime storage for the variable that may outlive the function's - frame. Cell variables are removed from the free set before the analyze - function returns to its parent. + The children update the free variable set. If a local variable is added to + the free variable set by the child, the variable is marked as a cell. The + function object being defined must provide runtime storage for the variable + that may outlive the function's frame. Cell variables are removed from the + free set before the analyze function returns to its parent. - The sets of bound and free variables are implemented as dictionaries - mapping strings to None. + During analysis, the names are: + symbols: dict mapping from symbol names to flag values (including offset scope values) + scopes: dict mapping from symbol names to scope values (no offset) + local: set of all symbol names local to the current scope + bound: set of all symbol names local to a containing function scope + free: set of all symbol names referenced but not bound in child scopes + global: set of all symbol names explicitly declared as global */ #define SET_SCOPE(DICT, NAME, I) { \ @@ -367,62 +376,57 @@ */ static int -analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags, +analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, PyObject *bound, PyObject *local, PyObject *free, PyObject *global) { if (flags & DEF_GLOBAL) { if (flags & DEF_PARAM) { PyErr_Format(PyExc_SyntaxError, - "name '%s' is local and global", + "name '%s' is parameter and global", PyString_AS_STRING(name)); return 0; } - SET_SCOPE(dict, name, GLOBAL_EXPLICIT); - if (PyDict_SetItem(global, name, Py_None) < 0) + SET_SCOPE(scopes, name, GLOBAL_EXPLICIT); + if (PySet_Add(global, name) < 0) return 0; - if (bound && PyDict_GetItem(bound, name)) { - if (PyDict_DelItem(bound, name) < 0) - return 0; - } + if (bound && (PySet_Discard(bound, name) < 0)) + return 0; return 1; } + /* Handle names assigned to in this scope */ if (flags & DEF_BOUND) { - SET_SCOPE(dict, name, LOCAL); - if (PyDict_SetItem(local, name, Py_None) < 0) + SET_SCOPE(scopes, name, LOCAL); + if (PySet_Add(local, name) < 0) return 0; - if (PyDict_GetItem(global, name)) { - if (PyDict_DelItem(global, name) < 0) - return 0; - } + if (PySet_Discard(global, name) < 0) + return 0; return 1; } - /* If an enclosing block has a binding for this name, it + /* Handle names bound in an enclosing function scope. + If an enclosing block has a binding for this name, it is a free variable rather than a global variable. Note that having a non-NULL bound implies that the block - is nested. + is nested inside a function. */ - if (bound && PyDict_GetItem(bound, name)) { - SET_SCOPE(dict, name, FREE); + if (bound && PySet_Contains(bound, name)) { + /* This name is assigned to in an enclosing scope */ + SET_SCOPE(scopes, name, FREE); ste->ste_free = 1; - if (PyDict_SetItem(free, name, Py_None) < 0) + if (PySet_Add(free, name) < 0) return 0; return 1; } - /* If a parent has a global statement, then call it global - explicit? It could also be global implicit. - */ - else if (global && PyDict_GetItem(global, name)) { - SET_SCOPE(dict, name, GLOBAL_EXPLICIT); + /* Handles names explicitly declared global in an enclosing scope */ + if (PySet_Contains(global, name)) { + SET_SCOPE(scopes, name, GLOBAL_EXPLICIT); return 1; } - else { - if (ste->ste_nested) - ste->ste_free = 1; - SET_SCOPE(dict, name, GLOBAL_IMPLICIT); - return 1; - } - return 0; /* Can't get here */ + /* All other names are assumed to be globals (or builtins) */ + if (ste->ste_nested) + ste->ste_free = 1; + SET_SCOPE(scopes, name, GLOBAL_IMPLICIT); + return 1; } #undef SET_SCOPE @@ -436,35 +440,35 @@ */ static int -analyze_cells(PyObject *scope, PyObject *free) +analyze_cells(PyObject *scopes, PyObject *free) { - PyObject *name, *v, *w; + PyObject *name, *v, *v_cell; int success = 0; Py_ssize_t pos = 0; - w = PyInt_FromLong(CELL); - if (!w) + v_cell = PyInt_FromLong(CELL); + if (!v_cell) return 0; - while (PyDict_Next(scope, &pos, &name, &v)) { - long flags; + while (PyDict_Next(scopes, &pos, &name, &v)) { + long scope; assert(PyInt_Check(v)); - flags = PyInt_AS_LONG(v); - if (flags != LOCAL) + scope = PyInt_AS_LONG(v); + if (scope != LOCAL) continue; - if (!PyDict_GetItem(free, name)) + if (!PySet_Contains(free, name)) continue; /* Replace LOCAL with CELL for this name, and remove from free. It is safe to replace the value of name in the dict, because it will not cause a resize. */ - if (PyDict_SetItem(scope, name, w) < 0) + if (PyDict_SetItem(scopes, name, v_cell) < 0) goto error; - if (!PyDict_DelItem(free, name) < 0) + if (PySet_Discard(free, name) < 0) goto error; } success = 1; error: - Py_DECREF(w); + Py_DECREF(v_cell); return success; } @@ -499,77 +503,91 @@ return 0; } -/* Enter the final scope information into the st_symbols dict. +/* Enter the final scope information into the ste_symbols dict. * * All arguments are dicts. Modifies symbols, others are read-only. */ static int -update_symbols(PyObject *symbols, PyObject *scope, +update_symbols(PyObject *symbols, PyObject *scopes, PyObject *bound, PyObject *free, int classflag) { - PyObject *name, *v, *u, *w, *free_value = NULL; + PyObject *name = NULL, *itr = NULL; + PyObject *v = NULL, *v_scope = NULL, *v_new = NULL, *v_free = NULL; Py_ssize_t pos = 0; + /* Update scope information for all symbols in this scope */ while (PyDict_Next(symbols, &pos, &name, &v)) { - long i, flags; + long scope, flags; assert(PyInt_Check(v)); flags = PyInt_AS_LONG(v); - w = PyDict_GetItem(scope, name); - assert(w && PyInt_Check(w)); - i = PyInt_AS_LONG(w); - flags |= (i << SCOPE_OFF); - u = PyInt_FromLong(flags); - if (!u) + v_scope = PyDict_GetItem(scopes, name); + assert(v_scope && PyInt_Check(v_scope)); + scope = PyInt_AS_LONG(v_scope); + flags |= (scope << SCOPE_OFFSET); + v_new = PyInt_FromLong(flags); + if (!v_new) return 0; - if (PyDict_SetItem(symbols, name, u) < 0) { - Py_DECREF(u); + if (PyDict_SetItem(symbols, name, v_new) < 0) { + Py_DECREF(v_new); return 0; } - Py_DECREF(u); + Py_DECREF(v_new); } - free_value = PyInt_FromLong(FREE << SCOPE_OFF); - if (!free_value) + /* Record not yet resolved free variables from children (if any) */ + v_free = PyInt_FromLong(FREE << SCOPE_OFFSET); + if (!v_free) return 0; - /* add a free variable when it's only use is for creating a closure */ - pos = 0; - while (PyDict_Next(free, &pos, &name, &v)) { - PyObject *o = PyDict_GetItem(symbols, name); + itr = PyObject_GetIter(free); + if (!itr) + goto error; - if (o) { - /* It could be a free variable in a method of + while ((name = PyIter_Next(itr))) { + v = PyDict_GetItem(symbols, name); + + /* Handle symbol that already exists in this scope */ + if (v) { + /* Handle a free variable in a method of the class that has the same name as a local or global in the class scope. */ if (classflag && - PyInt_AS_LONG(o) & (DEF_BOUND | DEF_GLOBAL)) { - long i = PyInt_AS_LONG(o) | DEF_FREE_CLASS; - o = PyInt_FromLong(i); - if (!o) { - Py_DECREF(free_value); - return 0; + PyInt_AS_LONG(v) & (DEF_BOUND | DEF_GLOBAL)) { + long flags = PyInt_AS_LONG(v) | DEF_FREE_CLASS; + v_new = PyInt_FromLong(flags); + if (!v_new) { + goto error; } - if (PyDict_SetItem(symbols, name, o) < 0) { - Py_DECREF(o); - Py_DECREF(free_value); - return 0; + if (PyDict_SetItem(symbols, name, v_new) < 0) { + Py_DECREF(v_new); + goto error; } - Py_DECREF(o); + Py_DECREF(v_new); } - /* else it's not free, probably a cell */ + /* It's a cell, or already a free variable in this scope */ + Py_DECREF(name); continue; } - if (!PyDict_GetItem(bound, name)) + /* Handle global symbol */ + if (!PySet_Contains(bound, name)) { + Py_DECREF(name); continue; /* it's a global */ - - if (PyDict_SetItem(symbols, name, free_value) < 0) { - Py_DECREF(free_value); - return 0; } + /* Propagate new free symbol up the lexical stack */ + if (PyDict_SetItem(symbols, name, v_free) < 0) { + goto error; + } + Py_DECREF(name); } - Py_DECREF(free_value); + Py_DECREF(itr); + Py_DECREF(v_free); return 1; +error: + Py_XDECREF(v_free); + Py_XDECREF(itr); + Py_XDECREF(name); + return 0; } /* Make final symbol table decisions for block of ste. @@ -584,59 +602,74 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, PyObject *global) { - PyObject *name, *v, *local = NULL, *scope = NULL, *newbound = NULL; + PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL; PyObject *newglobal = NULL, *newfree = NULL; int i, success = 0; Py_ssize_t pos = 0; - local = PyDict_New(); + scopes = PyDict_New(); + if (!scopes) + goto error; + local = PySet_New(NULL); if (!local) goto error; - scope = PyDict_New(); - if (!scope) - goto error; - newglobal = PyDict_New(); + newglobal = PySet_New(NULL); if (!newglobal) goto error; - newfree = PyDict_New(); + newfree = PySet_New(NULL); if (!newfree) goto error; - newbound = PyDict_New(); + newbound = PySet_New(NULL); if (!newbound) goto error; + /* Class namespace has no effect on names visible in + nested functions, so populate the global and bound + sets to be passed to child blocks before analyzing + this one. + */ if (ste->ste_type == ClassBlock) { - /* make a copy of globals before calling analyze_name(), - because global statements in the class have no effect - on nested functions. - */ - if (PyDict_Update(newglobal, global) < 0) + /* Pass down previously bound symbols */ + if (bound) { + if (!PyNumber_InPlaceOr(newbound, bound)) + goto error; + Py_DECREF(newbound); + } + /* Pass down known globals */ + if (!PyNumber_InPlaceOr(newglobal, global)) goto error; - if (bound) - if (PyDict_Update(newbound, bound) < 0) - goto error; + Py_DECREF(newglobal); } + /* Analyze symbols in current scope */ assert(PySTEntry_Check(ste)); assert(PyDict_Check(ste->ste_symbols)); while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) { long flags = PyInt_AS_LONG(v); - if (!analyze_name(ste, scope, name, flags, bound, local, free, + if (!analyze_name(ste, scopes, name, flags, bound, local, free, global)) goto error; } + /* Populate global and bound sets to be passed to children. + */ if (ste->ste_type != ClassBlock) { + /* Add function locals to bound set */ if (ste->ste_type == FunctionBlock) { - if (PyDict_Update(newbound, local) < 0) + if (!PyNumber_InPlaceOr(newbound, local)) goto error; + Py_DECREF(newbound); } + /* Pass down previously bound symbols */ if (bound) { - if (PyDict_Update(newbound, bound) < 0) + if (!PyNumber_InPlaceOr(newbound, bound)) goto error; + Py_DECREF(newbound); } - if (PyDict_Update(newglobal, global) < 0) + /* Pass down known globals */ + if (!PyNumber_InPlaceOr(newglobal, global)) goto error; + Py_DECREF(newglobal); } /* Recursively call analyze_block() on each child block */ @@ -647,24 +680,28 @@ entry = (PySTEntryObject*)c; if (!analyze_block(entry, newbound, newfree, newglobal)) goto error; + /* Check if any children have free variables */ if (entry->ste_free || entry->ste_child_free) ste->ste_child_free = 1; } - if (ste->ste_type == FunctionBlock && !analyze_cells(scope, newfree)) + /* Check if any local variables need to be converted to cell variables */ + if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree)) goto error; - if (!update_symbols(ste->ste_symbols, scope, bound, newfree, + /* Records the results of the analysis in the symbol table entry */ + if (!update_symbols(ste->ste_symbols, scopes, bound, newfree, ste->ste_type == ClassBlock)) goto error; if (!check_unoptimized(ste)) goto error; - if (PyDict_Update(free, newfree) < 0) + if (!PyNumber_InPlaceOr(free, newfree)) goto error; + Py_DECREF(free); success = 1; error: + Py_XDECREF(scopes); Py_XDECREF(local); - Py_XDECREF(scope); Py_XDECREF(newbound); Py_XDECREF(newglobal); Py_XDECREF(newfree); @@ -679,10 +716,10 @@ PyObject *free, *global; int r; - free = PyDict_New(); + free = PySet_New(NULL); if (!free) return 0; - global = PyDict_New(); + global = PySet_New(NULL); if (!global) { Py_DECREF(free); return 0; @@ -1140,14 +1177,16 @@ case Set_kind: VISIT_SEQ(st, expr, e->v.Set.elts); break; + case GeneratorExp_kind: + if (!symtable_visit_genexp(st, e)) + return 0; + break; case ListComp_kind: - if (!symtable_new_tmpname(st)) + if (!symtable_visit_listcomp(st, e)) return 0; - VISIT(st, expr, e->v.ListComp.elt); - VISIT_SEQ(st, comprehension, e->v.ListComp.generators); break; - case GeneratorExp_kind: - if (!symtable_visit_genexp(st, e)) + case SetComp_kind: + if (!symtable_visit_setcomp(st, e)) return 0; break; case Yield_kind: @@ -1419,27 +1458,60 @@ } static int -symtable_visit_genexp(struct symtable *st, expr_ty e) +symtable_handle_comprehension(struct symtable *st, expr_ty e, + identifier scope_name, + asdl_seq *generators, expr_ty elt) { + int is_generator = (e->kind == GeneratorExp_kind); + int needs_tmp = !is_generator; comprehension_ty outermost = ((comprehension_ty) - (asdl_seq_GET(e->v.GeneratorExp.generators, 0))); + asdl_seq_GET(generators, 0)); /* Outermost iterator is evaluated in current scope */ VISIT(st, expr, outermost->iter); - /* Create generator scope for the rest */ - if (!GET_IDENTIFIER(genexpr) || - !symtable_enter_block(st, genexpr, FunctionBlock, (void *)e, 0)) { + /* Create comprehension scope for the rest */ + if (!scope_name || + !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, 0)) { return 0; } - st->st_cur->ste_generator = 1; + st->st_cur->ste_generator = is_generator; /* Outermost iter is received as an argument */ if (!symtable_implicit_arg(st, 0)) { symtable_exit_block(st, (void *)e); return 0; } + /* Allocate temporary name if needed */ + if (needs_tmp && !symtable_new_tmpname(st)) { + symtable_exit_block(st, (void *)e); + return 0; + } VISIT_IN_BLOCK(st, expr, outermost->target, (void*)e); VISIT_SEQ_IN_BLOCK(st, expr, outermost->ifs, (void*)e); VISIT_SEQ_TAIL_IN_BLOCK(st, comprehension, - e->v.GeneratorExp.generators, 1, (void*)e); - VISIT_IN_BLOCK(st, expr, e->v.GeneratorExp.elt, (void*)e); + generators, 1, (void*)e); + VISIT_IN_BLOCK(st, expr, elt, (void*)e); return symtable_exit_block(st, (void *)e); } + +static int +symtable_visit_genexp(struct symtable *st, expr_ty e) +{ + return symtable_handle_comprehension(st, e, GET_IDENTIFIER(genexpr), + e->v.GeneratorExp.generators, + e->v.GeneratorExp.elt); +} + +static int +symtable_visit_listcomp(struct symtable *st, expr_ty e) +{ + return symtable_handle_comprehension(st, e, GET_IDENTIFIER(listcomp), + e->v.ListComp.generators, + e->v.ListComp.elt); +} + +static int +symtable_visit_setcomp(struct symtable *st, expr_ty e) +{ + return symtable_handle_comprehension(st, e, GET_IDENTIFIER(setcomp), + e->v.SetComp.generators, + e->v.SetComp.elt); +} Index: Python/compile.c =================================================================== --- Python/compile.c (Revision 53930) +++ Python/compile.c (Arbeitskopie) @@ -39,6 +39,10 @@ #define DEFAULT_CODE_SIZE 128 #define DEFAULT_LNOTAB_SIZE 16 +#define COMP_GENEXP 0 +#define COMP_LISTCOMP 1 +#define COMP_SETCOMP 2 + struct instr { unsigned i_jabs : 1; unsigned i_jrel : 1; @@ -355,7 +359,7 @@ while (PyDict_Next(src, &pos, &k, &v)) { /* XXX this should probably be a macro in symtable.h */ assert(PyInt_Check(v)); - scope = (PyInt_AS_LONG(v) >> SCOPE_OFF) & SCOPE_MASK; + scope = (PyInt_AS_LONG(v) >> SCOPE_OFFSET) & SCOPE_MASK; if (scope == scope_type || PyInt_AS_LONG(v) & flag) { PyObject *tuple, *item = PyInt_FromLong(i); @@ -668,6 +672,7 @@ case UNARY_INVERT: return 0; + case SET_ADD: case LIST_APPEND: return -2; @@ -2633,39 +2638,83 @@ return 1; } + +/* List comprehensions and generator expressions work by creating a nested + function to perform the actual iteration. This means that the iteration + variables don't leak into the current scope. + The defined function is called immediately following its definition, with the + result of that call being the result of the expression. + The LC version returns the populated list, while the GE version is flagged in + symtable.c as a generator, so it returns the generator object when the + function is called. + The LC code *knows* that the loop cannot contain break, continue, yield or + return, so it cheats and skips the SETUP_LOOP/POP_BLOCK steps. This allows + the LC to run significantly faster than the equivalent generator expression. + + XXX (ncoghlan): That last para is based on the fact that the current list + comprehension code is comparable in speed to the Python 2.x series, while + a similar patch that added SETUP_LOOP/POP_BLOCK to list comps showed a + significant slow down. I'd be overjoyed if someone else could confirm this + reasoning (as well as confirming that it's still legitimate to leave those + steps out for the LC case!). + + Possible cleanups: + - iterate over the generator sequence instead of using recursion + - combine handling of LC's and GE's (without hurting LC speed) +*/ + static int -compiler_listcomp_generator(struct compiler *c, PyObject *tmpname, - asdl_seq *generators, int gen_index, - expr_ty elt) +compiler_comprehension_generator(struct compiler *c, PyObject *tmpname, + asdl_seq *generators, int gen_index, + expr_ty elt, int type) { /* generate code for the iterator, then each of the ifs, and then write to the element */ - comprehension_ty l; - basicblock *start, *anchor, *skip, *if_cleanup; + comprehension_ty gen; + basicblock *start, *anchor, *skip, *if_cleanup, *end; int i, n; start = compiler_new_block(c); skip = compiler_new_block(c); if_cleanup = compiler_new_block(c); anchor = compiler_new_block(c); + if (type == COMP_GENEXP) + end = compiler_new_block(c); if (start == NULL || skip == NULL || if_cleanup == NULL || - anchor == NULL) - return 0; + anchor == NULL) + return 0; + if (type == COMP_GENEXP && end == NULL) + return 0; - l = (comprehension_ty)asdl_seq_GET(generators, gen_index); - VISIT(c, expr, l->iter); - ADDOP(c, GET_ITER); + gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); + if (type == COMP_GENEXP) { + /* only genexps need SETUP_LOOP/POP_BLOCK */ + ADDOP_JREL(c, SETUP_LOOP, end); + if (!compiler_push_fblock(c, LOOP, start)) + return 0; + } + + if (gen_index == 0) { + /* Receive outermost iter as an implicit argument */ + c->u->u_argcount = 1; + ADDOP_I(c, LOAD_FAST, 0); + } + else { + /* Sub-iter - calculate on the fly */ + VISIT(c, expr, gen->iter); + ADDOP(c, GET_ITER); + } compiler_use_next_block(c, start); ADDOP_JREL(c, FOR_ITER, anchor); NEXT_BLOCK(c); - VISIT(c, expr, l->target); + VISIT(c, expr, gen->target); /* XXX this needs to be cleaned up...a lot! */ - n = asdl_seq_LEN(l->ifs); + n = asdl_seq_LEN(gen->ifs); for (i = 0; i < n; i++) { - expr_ty e = (expr_ty)asdl_seq_GET(l->ifs, i); + expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); VISIT(c, expr, e); ADDOP_JREL(c, JUMP_IF_FALSE, if_cleanup); NEXT_BLOCK(c); @@ -2673,55 +2722,108 @@ } if (++gen_index < asdl_seq_LEN(generators)) - if (!compiler_listcomp_generator(c, tmpname, - generators, gen_index, elt)) + if (!compiler_comprehension_generator(c, tmpname, + generators, gen_index, + elt, type)) return 0; /* only append after the last for generator */ if (gen_index >= asdl_seq_LEN(generators)) { - if (!compiler_nameop(c, tmpname, Load)) - return 0; - VISIT(c, expr, elt); - ADDOP(c, LIST_APPEND); + /* comprehension specific code */ + if (type == COMP_GENEXP) { + VISIT(c, expr, elt); + ADDOP(c, YIELD_VALUE); + ADDOP(c, POP_TOP); + } else if (type == COMP_LISTCOMP) { + if (!compiler_nameop(c, tmpname, Load)) + return 0; + VISIT(c, expr, elt); + ADDOP(c, LIST_APPEND); + } else { + if (!compiler_nameop(c, tmpname, Load)) + return 0; + VISIT(c, expr, elt); + ADDOP(c, SET_ADD); + } - compiler_use_next_block(c, skip); + compiler_use_next_block(c, skip); } for (i = 0; i < n; i++) { ADDOP_I(c, JUMP_FORWARD, 1); if (i == 0) - compiler_use_next_block(c, if_cleanup); + compiler_use_next_block(c, if_cleanup); + ADDOP(c, POP_TOP); } ADDOP_JABS(c, JUMP_ABSOLUTE, start); compiler_use_next_block(c, anchor); - /* delete the temporary list name added to locals */ - if (gen_index == 1) - if (!compiler_nameop(c, tmpname, Del)) - return 0; + + if (type == COMP_GENEXP) { + ADDOP(c, POP_BLOCK); + compiler_pop_fblock(c, LOOP, start); + compiler_use_next_block(c, end); + } return 1; } static int -compiler_listcomp(struct compiler *c, expr_ty e) +compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name, + asdl_seq *generators, expr_ty elt) { - identifier tmp; - int rc = 0; - asdl_seq *generators = e->v.ListComp.generators; + PyCodeObject *co = NULL; + identifier tmp = NULL; + expr_ty outermost_iter; - assert(e->kind == ListComp_kind); - tmp = compiler_new_tmpname(c); - if (!tmp) - return 0; - ADDOP_I(c, BUILD_LIST, 0); - ADDOP(c, DUP_TOP); - if (compiler_nameop(c, tmp, Store)) - rc = compiler_listcomp_generator(c, tmp, generators, 0, - e->v.ListComp.elt); - Py_DECREF(tmp); - return rc; + outermost_iter = ((comprehension_ty) + asdl_seq_GET(generators, 0))->iter; + + if (!compiler_enter_scope(c, name, (void *)e, e->lineno)) + goto error; + + if (type != COMP_GENEXP) { + tmp = compiler_new_tmpname(c); + if (!tmp) + goto error_in_scope; + + ADDOP_I(c, (type == COMP_LISTCOMP ? + BUILD_LIST : BUILD_SET), 0); + ADDOP(c, DUP_TOP); + if (!compiler_nameop(c, tmp, Store)) + goto error_in_scope; + } + + if (!compiler_comprehension_generator(c, tmp, generators, 0, elt, type)) + goto error_in_scope; + + if (type != COMP_GENEXP) { + ADDOP(c, RETURN_VALUE); + } + + co = assemble(c, 1); + compiler_exit_scope(c); + if (co == NULL) + goto error; + + if (!compiler_make_closure(c, co, 0)) + goto error; + Py_DECREF(co); + Py_XDECREF(tmp); + + VISIT(c, expr, outermost_iter); + ADDOP(c, GET_ITER); + ADDOP_I(c, CALL_FUNCTION, 1); + return 1; +error_in_scope: + compiler_exit_scope(c); +error: + Py_XDECREF(co); + Py_XDECREF(tmp); + return 0; } +#if 0 + static int compiler_genexp_generator(struct compiler *c, asdl_seq *generators, int gen_index, @@ -2835,8 +2937,55 @@ return 1; } +#endif static int +compiler_genexp(struct compiler *c, expr_ty e) +{ + static identifier name; + if (!name) { + name = PyString_FromString(""); + if (!name) + return 0; + } + assert(e->kind == GeneratorExp_kind); + return compiler_comprehension(c, e, COMP_GENEXP, name, + e->v.GeneratorExp.generators, + e->v.GeneratorExp.elt); +} + +static int +compiler_listcomp(struct compiler *c, expr_ty e) +{ + static identifier name; + if (!name) { + name = PyString_FromString(""); + if (!name) + return 0; + } + assert(e->kind == ListComp_kind); + return compiler_comprehension(c, e, COMP_LISTCOMP, name, + e->v.ListComp.generators, + e->v.ListComp.elt); +} + +static int +compiler_setcomp(struct compiler *c, expr_ty e) +{ + static identifier name; + if (!name) { + name = PyString_FromString(""); + if (!name) + return 0; + } + assert(e->kind == SetComp_kind); + return compiler_comprehension(c, e, COMP_SETCOMP, name, + e->v.SetComp.generators, + e->v.SetComp.elt); +} + + +static int compiler_visit_keyword(struct compiler *c, keyword_ty k) { ADDOP_O(c, LOAD_CONST, k->arg, consts); @@ -3054,10 +3203,12 @@ VISIT_SEQ(c, expr, e->v.Set.elts); ADDOP_I(c, BUILD_SET, n); break; + case GeneratorExp_kind: + return compiler_genexp(c, e); case ListComp_kind: return compiler_listcomp(c, e); - case GeneratorExp_kind: - return compiler_genexp(c, e); + case SetComp_kind: + return compiler_setcomp(c, e); case Yield_kind: if (c->u->u_ste->ste_type != FunctionBlock) return compiler_error(c, "'yield' outside function"); Index: Python/Python-ast.c =================================================================== --- Python/Python-ast.c (Revision 53930) +++ Python/Python-ast.c (Arbeitskopie) @@ -2,7 +2,7 @@ /* - __version__ 53872. + __version__ 53873. This module must be committed separately after each AST grammar change; The __version__ number is set to the revision number of the commit @@ -185,6 +185,11 @@ "elt", "generators", }; +static PyTypeObject *SetComp_type; +static char *SetComp_fields[]={ + "elt", + "generators", +}; static PyTypeObject *GeneratorExp_type; static char *GeneratorExp_fields[]={ "elt", @@ -538,6 +543,8 @@ if (!Set_type) return 0; ListComp_type = make_type("ListComp", expr_type, ListComp_fields, 2); if (!ListComp_type) return 0; + SetComp_type = make_type("SetComp", expr_type, SetComp_fields, 2); + if (!SetComp_type) return 0; GeneratorExp_type = make_type("GeneratorExp", expr_type, GeneratorExp_fields, 2); if (!GeneratorExp_type) return 0; @@ -1462,6 +1469,29 @@ } expr_ty +SetComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena + *arena) +{ + expr_ty p; + if (!elt) { + PyErr_SetString(PyExc_ValueError, + "field elt is required for SetComp"); + return NULL; + } + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) { + PyErr_NoMemory(); + return NULL; + } + p->kind = SetComp_kind; + p->v.SetComp.elt = elt; + p->v.SetComp.generators = generators; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena *arena) { @@ -2481,6 +2511,21 @@ goto failed; Py_DECREF(value); break; + case SetComp_kind: + result = PyType_GenericNew(SetComp_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_expr(o->v.SetComp.elt); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "elt", value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_list(o->v.SetComp.generators, + ast2obj_comprehension); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "generators", value) == -1) + goto failed; + Py_DECREF(value); + break; case GeneratorExp_kind: result = PyType_GenericNew(GeneratorExp_type, NULL, NULL); if (!result) goto failed; @@ -3125,7 +3170,7 @@ if (PyDict_SetItemString(d, "AST", (PyObject*)AST_type) < 0) return; if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0) return; - if (PyModule_AddStringConstant(m, "__version__", "53872") < 0) + if (PyModule_AddStringConstant(m, "__version__", "53873") < 0) return; if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return; if (PyDict_SetItemString(d, "Module", (PyObject*)Module_type) < 0) @@ -3183,6 +3228,8 @@ if (PyDict_SetItemString(d, "Set", (PyObject*)Set_type) < 0) return; if (PyDict_SetItemString(d, "ListComp", (PyObject*)ListComp_type) < 0) return; + if (PyDict_SetItemString(d, "SetComp", (PyObject*)SetComp_type) < 0) + return; if (PyDict_SetItemString(d, "GeneratorExp", (PyObject*)GeneratorExp_type) < 0) return; if (PyDict_SetItemString(d, "Yield", (PyObject*)Yield_type) < 0) return; Index: Include/opcode.h =================================================================== --- Include/opcode.h (Revision 53930) +++ Include/opcode.h (Arbeitskopie) @@ -21,6 +21,7 @@ #define UNARY_INVERT 15 +#define SET_ADD 17 #define LIST_APPEND 18 #define BINARY_POWER 19 Index: Include/graminit.h =================================================================== --- Include/graminit.h (Revision 53930) +++ Include/graminit.h (Arbeitskopie) @@ -45,10 +45,10 @@ #define with_var 300 #define except_clause 301 #define suite 302 -#define testlist_safe 303 -#define old_test 304 -#define old_lambdef 305 -#define test 306 +#define test 303 +#define test_nocond 304 +#define lambdef 305 +#define lambdef_nocond 306 #define or_test 307 #define and_test 308 #define not_test 309 @@ -63,25 +63,20 @@ #define factor 318 #define power 319 #define atom 320 -#define listmaker 321 -#define testlist_gexp 322 -#define lambdef 323 -#define trailer 324 -#define subscriptlist 325 -#define subscript 326 -#define sliceop 327 -#define exprlist 328 -#define testlist 329 -#define dictsetmaker 330 -#define classdef 331 -#define arglist 332 -#define argument 333 -#define list_iter 334 -#define list_for 335 -#define list_if 336 -#define gen_iter 337 -#define gen_for 338 -#define gen_if 339 -#define testlist1 340 -#define encoding_decl 341 -#define yield_expr 342 +#define testlist_comp 321 +#define trailer 322 +#define subscriptlist 323 +#define subscript 324 +#define sliceop 325 +#define exprlist 326 +#define testlist 327 +#define dictorsetmaker 328 +#define classdef 329 +#define arglist 330 +#define argument 331 +#define comp_iter 332 +#define comp_for 333 +#define comp_if 334 +#define testlist1 335 +#define encoding_decl 336 +#define yield_expr 337 Index: Include/symtable.h =================================================================== --- Include/symtable.h (Revision 53930) +++ Include/symtable.h (Arbeitskopie) @@ -5,31 +5,35 @@ extern "C" { #endif +/* XXX(ncoghlan): This is a weird mix of public names and interpreter internal + * names. + */ + typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock } _Py_block_ty; struct _symtable_entry; struct symtable { - const char *st_filename; /* name of file being compiled */ + const char *st_filename; /* name of file being compiled */ struct _symtable_entry *st_cur; /* current symbol table entry */ - struct _symtable_entry *st_top; /* module entry */ - PyObject *st_symbols; /* dictionary of symbol table entries */ - PyObject *st_stack; /* stack of namespace info */ - PyObject *st_global; /* borrowed ref to MODULE in st_symbols */ - int st_nblocks; /* number of blocks */ - PyObject *st_private; /* name of current class or NULL */ - int st_tmpname; /* temporary name counter */ - PyFutureFeatures *st_future; /* module's future features */ + struct _symtable_entry *st_top; /* symbol table entry for module */ + PyObject *st_blocks; /* dict: map AST node addresses + * to symbol table entries */ + PyObject *st_stack; /* list: stack of namespace info */ + PyObject *st_global; /* borrowed ref to st_top->st_symbols */ + int st_nblocks; /* number of blocks used */ + PyObject *st_private; /* name of current class or NULL */ + PyFutureFeatures *st_future; /* module's future features */ }; typedef struct _symtable_entry { PyObject_HEAD - PyObject *ste_id; /* int: key in st_symbols */ - PyObject *ste_symbols; /* dict: name to flags */ - PyObject *ste_name; /* string: name of block */ + PyObject *ste_id; /* int: key in ste_table->st_blocks */ + PyObject *ste_symbols; /* dict: variable names to flags */ + PyObject *ste_name; /* string: name of current block */ PyObject *ste_varnames; /* list of variable names */ - PyObject *ste_children; /* list of child ids */ + PyObject *ste_children; /* list of child blocks */ _Py_block_ty ste_type; /* module, class, or function */ int ste_unoptimized; /* false if namespace is optimized */ unsigned ste_nested : 1; /* true if block is nested */ @@ -51,49 +55,59 @@ #define PySTEntry_Check(op) ((op)->ob_type == &PySTEntry_Type) -PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *); +/* The following two names are used for the ste_unoptimized bit field */ +#define OPT_IMPORT_STAR 1 +#define OPT_TOPLEVEL 2 /* top-level block, including eval and exec */ -PyAPI_FUNC(struct symtable *) PySymtable_Build(mod_ty, const char *, - PyFutureFeatures *); -PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *); +/* Flags describing use of a symbol (the values in the ste_symbols dict) */ -PyAPI_FUNC(void) PySymtable_Free(struct symtable *); +#define DEF_GLOBAL 0x0001 /* referenced in global stmt */ +#define DEF_LOCAL 0x0002 /* target of assignment stmt */ +#define DEF_PARAM 0x0004 /* formal parameter */ +#define USE 0x0008 /* name is referenced in block */ +#define DEF_STAR 0x0010 /* parameter is star arg */ +#define DEF_DOUBLESTAR 0x0020 /* parameter is star-star arg */ +#define DEF_INTUPLE 0x0040 /* name defined in tuple in parameters */ +#define DEF_FREE 0x0080 /* in nested block, name used but not defined */ +#define DEF_FREE_GLOBAL 0x0100 /* free variable is actually implicit global */ +#define DEF_FREE_CLASS 0x0200 /* free variable from class's method */ +#define DEF_IMPORT 0x0400 /* assignment occurred via import */ +/* reserved for future 0x0800 */ +/* reserved for future 0x1000 */ +/* reserved for SCOPE 0xE000 */ +/* Need to change SCOPE_OFFSET below if we want more than 13 symbol flags */ -/* Flags for def-use information */ +#define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT) -#define DEF_GLOBAL 1 /* global stmt */ -#define DEF_LOCAL 2 /* assignment in code block */ -#define DEF_PARAM 2<<1 /* formal parameter */ -#define USE 2<<2 /* name is used */ -#define DEF_STAR 2<<3 /* parameter is star arg */ -#define DEF_DOUBLESTAR 2<<4 /* parameter is star-star arg */ -#define DEF_INTUPLE 2<<5 /* name defined in tuple in parameters */ -#define DEF_FREE 2<<6 /* name used but not defined in nested block */ -#define DEF_FREE_GLOBAL 2<<7 /* free variable is actually implicit global */ -#define DEF_FREE_CLASS 2<<8 /* free variable from class's method */ -#define DEF_IMPORT 2<<9 /* assignment occurred via import */ +/* SCOPE defines how a name is to be handled in the current scope. + * With the LSB as bit 0, it is stored in bits 13-15 of ste_symbols values. + */ +#define SCOPE_OFFSET 13 +#define SCOPE_MASK 7 -#define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT) +#define LOCAL 1 +#define GLOBAL_EXPLICIT 2 +#define GLOBAL_IMPLICIT 3 +#define FREE 4 +#define CELL 5 -/* GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol - table. GLOBAL is returned from PyST_GetScope() for either of them. - It is stored in ste_symbols at bits 12-14. -*/ -#define SCOPE_OFF 11 -#define SCOPE_MASK 7 +/* Is a name a Global, Local, Free or Cell variable in the + * specified scope? + */ +PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *); -#define LOCAL 1 -#define GLOBAL_EXPLICIT 2 -#define GLOBAL_IMPLICIT 3 -#define FREE 4 -#define CELL 5 +/* Build a new symbol table for the supplied AST node using + * the specified file name and __future__ flags. + */ +struct _mod; /* Declare the existence of this type */ +PyAPI_FUNC(struct symtable *) PySymtable_Build(struct _mod *, const char *, + PyFutureFeatures *); -/* The following two names are used for the ste_unoptimized bit field */ -#define OPT_IMPORT_STAR 1 -#define OPT_TOPLEVEL 2 /* top-level names, including eval and exec */ +/* Retrieve the symbol table entry for the specified AST node + */ +PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *); -#define GENERATOR 1 -#define GENERATOR_EXPRESSION 2 +PyAPI_FUNC(void) PySymtable_Free(struct symtable *); #ifdef __cplusplus } Index: Include/Python-ast.h =================================================================== --- Include/Python-ast.h (Revision 53930) +++ Include/Python-ast.h (Arbeitskopie) @@ -175,10 +175,10 @@ enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4, IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8, - GeneratorExp_kind=9, Yield_kind=10, Compare_kind=11, - Call_kind=12, Num_kind=13, Str_kind=14, Bytes_kind=15, - Ellipsis_kind=16, Attribute_kind=17, Subscript_kind=18, - Name_kind=19, List_kind=20, Tuple_kind=21}; + SetComp_kind=9, GeneratorExp_kind=10, Yield_kind=11, + Compare_kind=12, Call_kind=13, Num_kind=14, Str_kind=15, + Bytes_kind=16, Ellipsis_kind=17, Attribute_kind=18, + Subscript_kind=19, Name_kind=20, List_kind=21, Tuple_kind=22}; struct _expr { enum _expr_kind kind; union { @@ -226,6 +226,11 @@ struct { expr_ty elt; asdl_seq *generators; + } SetComp; + + struct { + expr_ty elt; + asdl_seq *generators; } GeneratorExp; struct { @@ -453,6 +458,9 @@ #define ListComp(a0, a1, a2, a3, a4) _Py_ListComp(a0, a1, a2, a3, a4) expr_ty _Py_ListComp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena *arena); +#define SetComp(a0, a1, a2, a3, a4) _Py_SetComp(a0, a1, a2, a3, a4) +expr_ty _Py_SetComp(expr_ty elt, asdl_seq * generators, int lineno, int + col_offset, PyArena *arena); #define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4) expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena *arena); Index: Grammar/Grammar =================================================================== --- Grammar/Grammar (Revision 53930) +++ Grammar/Grammar (Arbeitskopie) @@ -80,16 +80,10 @@ except_clause: 'except' [test ['as' NAME]] suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT -# Backward compatibility cruft to support: -# [ x for x in lambda: True, lambda: False if x() ] -# even while also allowing: -# lambda x: 5 if x else 2 -# (But not a mix of the two) -testlist_safe: old_test [(',' old_test)+ [',']] -old_test: or_test | old_lambdef -old_lambdef: 'lambda' [varargslist] ':' old_test - test: or_test ['if' or_test 'else' test] | lambdef +test_nocond: or_test | lambdef_nocond +lambdef: 'lambda' [varargslist] ':' test +lambdef_nocond: 'lambda' [varargslist] ':' test_nocond or_test: and_test ('or' and_test)* and_test: not_test ('and' not_test)* not_test: 'not' not_test | comparison @@ -103,34 +97,29 @@ term: factor (('*'|'/'|'%'|'//') factor)* factor: ('+'|'-'|'~') factor | power power: atom trailer* ['**' factor] -atom: ('(' [yield_expr|testlist_gexp] ')' | - '[' [listmaker] ']' | - '{' [dictsetmaker] '}' | +atom: ('(' [yield_expr|testlist_comp] ')' | + '[' [testlist_comp] ']' | + '{' [dictorsetmaker] '}' | NAME | NUMBER | STRING+ | '.' '.' '.') -listmaker: test ( list_for | (',' test)* [','] ) -testlist_gexp: test ( gen_for | (',' test)* [','] ) -lambdef: 'lambda' [varargslist] ':' test +testlist_comp: test ( comp_for | (',' test)* [','] ) trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: subscript (',' subscript)* [','] subscript: test | [test] ':' [test] [sliceop] sliceop: ':' [test] exprlist: expr (',' expr)* [','] testlist: test (',' test)* [','] -dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [',']) +dictorsetmaker: ( (test ':' test (',' test ':' test)* [',']) | + (test (comp_for | (',' test)* [','])) ) classdef: 'class' NAME ['(' [testlist] ')'] ':' suite arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test) -argument: test [gen_for] | test '=' test # Really [keyword '='] test +argument: test [comp_for] | test '=' test # Really [keyword '='] test -list_iter: list_for | list_if -list_for: 'for' exprlist 'in' testlist_safe [list_iter] -list_if: 'if' old_test [list_iter] +comp_iter: comp_for | comp_if +comp_for: 'for' exprlist 'in' or_test [comp_iter] +comp_if: 'if' test_nocond [comp_iter] -gen_iter: gen_for | gen_if -gen_for: 'for' exprlist 'in' or_test [gen_iter] -gen_if: 'if' old_test [gen_iter] - testlist1: test (',' test)* # not used in grammar, but may appear in "node" passed from Parser to Compiler Index: setup.py =================================================================== --- setup.py (Revision 53930) +++ setup.py (Arbeitskopie) @@ -1170,8 +1170,8 @@ # include_dirs = [ join(F, fw + '.framework', H) - for fw in 'Tcl', 'Tk' - for H in 'Headers', 'Versions/Current/PrivateHeaders' + for fw in ('Tcl', 'Tk') + for H in ('Headers', 'Versions/Current/PrivateHeaders') ] # For 8.4a2, the X11 headers are not included. Rather than include a Index: Parser/Python.asdl =================================================================== --- Parser/Python.asdl (Revision 53930) +++ Parser/Python.asdl (Arbeitskopie) @@ -50,6 +50,7 @@ | Dict(expr* keys, expr* values) | Set(expr* elts) | ListComp(expr elt, comprehension* generators) + | SetComp(expr elt, comprehension* generators) | GeneratorExp(expr elt, comprehension* generators) -- the grammar constrains where yield expressions can occur | Yield(expr? value) Index: Doc/lib/libdis.tex =================================================================== --- Doc/lib/libdis.tex (Revision 53930) +++ Doc/lib/libdis.tex (Arbeitskopie) @@ -363,6 +363,10 @@ instruction). \end{opcodedesc} +\begin{opcodedesc}{SET_ADD}{} +Calls \code{set.add(TOS1, TOS)}. Used to implement set comprehensions. +\end{opcodedesc} + \begin{opcodedesc}{LIST_APPEND}{} Calls \code{list.append(TOS1, TOS)}. Used to implement list comprehensions. \end{opcodedesc} Index: Lib/symbol.py =================================================================== --- Lib/symbol.py (Revision 53930) +++ Lib/symbol.py (Arbeitskopie) @@ -57,10 +57,10 @@ with_var = 300 except_clause = 301 suite = 302 -testlist_safe = 303 -old_test = 304 -old_lambdef = 305 -test = 306 +test = 303 +test_nocond = 304 +lambdef = 305 +lambdef_nocond = 306 or_test = 307 and_test = 308 not_test = 309 @@ -75,28 +75,23 @@ factor = 318 power = 319 atom = 320 -listmaker = 321 -testlist_gexp = 322 -lambdef = 323 -trailer = 324 -subscriptlist = 325 -subscript = 326 -sliceop = 327 -exprlist = 328 -testlist = 329 -dictsetmaker = 330 -classdef = 331 -arglist = 332 -argument = 333 -list_iter = 334 -list_for = 335 -list_if = 336 -gen_iter = 337 -gen_for = 338 -gen_if = 339 -testlist1 = 340 -encoding_decl = 341 -yield_expr = 342 +testlist_comp = 321 +trailer = 322 +subscriptlist = 323 +subscript = 324 +sliceop = 325 +exprlist = 326 +testlist = 327 +dictorsetmaker = 328 +classdef = 329 +arglist = 330 +argument = 331 +comp_iter = 332 +comp_for = 333 +comp_if = 334 +testlist1 = 335 +encoding_decl = 336 +yield_expr = 337 #--end constants-- sym_name = {} Index: Lib/tokenize.py =================================================================== --- Lib/tokenize.py (Revision 53930) +++ Lib/tokenize.py (Arbeitskopie) @@ -32,7 +32,6 @@ import token __all__ = [x for x in dir(token) if x[0] != '_'] + ["COMMENT", "tokenize", "generate_tokens", "NL", "untokenize"] -del x del token COMMENT = N_TOKENS Index: Lib/compiler/transformer.py =================================================================== --- Lib/compiler/transformer.py (Revision 53930) +++ Lib/compiler/transformer.py (Arbeitskopie) @@ -558,7 +558,7 @@ testlist1 = testlist exprlist = testlist - def testlist_gexp(self, nodelist): + def testlist_comp(self, nodelist): if len(nodelist) == 2 and nodelist[1][0] == symbol.gen_for: test = self.com_node(nodelist[0]) return self.com_generator_expression(test, nodelist[1]) @@ -1026,7 +1026,7 @@ # loop to avoid trivial recursion while 1: t = node[0] - if t in (symbol.exprlist, symbol.testlist, symbol.testlist_safe, symbol.testlist_gexp): + if t in (symbol.exprlist, symbol.testlist, symbol.testlist_comp): if len(node) > 2: return self.com_assign_tuple(node, assigning) node = node[1] Index: Lib/decimal.py =================================================================== --- Lib/decimal.py (Revision 53930) +++ Lib/decimal.py (Arbeitskopie) @@ -2282,10 +2282,8 @@ _ignored_flags = [] if not isinstance(flags, dict): flags = dict([(s,s in flags) for s in _signals]) - del s if traps is not None and not isinstance(traps, dict): traps = dict([(s,s in traps) for s in _signals]) - del s for name, val in locals().items(): if val is None: setattr(self, name, _copy.copy(getattr(DefaultContext, name))) Index: Lib/opcode.py =================================================================== --- Lib/opcode.py (Revision 53930) +++ Lib/opcode.py (Arbeitskopie) @@ -57,6 +57,7 @@ def_op('UNARY_INVERT', 15) +def_op('SET_ADD', 17) def_op('LIST_APPEND', 18) def_op('BINARY_POWER', 19) def_op('BINARY_MULTIPLY', 20) Index: Lib/test/test_univnewlines.py =================================================================== --- Lib/test/test_univnewlines.py (Revision 53930) +++ Lib/test/test_univnewlines.py (Arbeitskopie) @@ -28,7 +28,6 @@ # before end-of-file. DATA_MIXED = "\n".join(DATA_TEMPLATE) + "\r" DATA_SPLIT = [x + "\n" for x in DATA_TEMPLATE] -del x class TestGenericUnivNewlines(unittest.TestCase): # use a class variable DATA to define the data to write to the file Index: Lib/test/test_setcomps.py =================================================================== --- Lib/test/test_setcomps.py (Revision 0) +++ Lib/test/test_setcomps.py (Revision 0) @@ -0,0 +1,151 @@ +doctests = """ +########### Tests mostly copied from test_listcomps.py ############ + +Test simple loop with conditional + + >>> sum({i*i for i in range(100) if i&1 == 1}) + 166650 + +Test simple case + + >>> {2*y + x + 1 for x in (0,) for y in (1,)} + {3} + +Test simple nesting + + >>> list(sorted({(i,j) for i in range(3) for j in range(4)})) + [(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)] + +Test nesting with the inner expression dependent on the outer + + >>> list(sorted({(i,j) for i in range(4) for j in range(i)})) + [(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)] + +Make sure the induction variable is not exposed + + >>> i = 20 + >>> sum({i*i for i in range(100)}) + 328350 + + >>> i + 20 + +Verify that syntax error's are raised for setcomps used as lvalues + + >>> {y for y in (1,2)} = 10 # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + SyntaxError: ... + + >>> {y for y in (1,2)} += 10 # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + SyntaxError: ... + + +Make a nested set comprehension that acts like set(xrange()) + + >>> def srange(n): + ... return {i for i in xrange(n)} + >>> list(sorted(srange(10))) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + +Same again, only as a lambda expression instead of a function definition + + >>> lrange = lambda n: {i for i in xrange(n)} + >>> list(sorted(lrange(10))) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + +Generators can call other generators: + + >>> def grange(n): + ... for x in {i for i in xrange(n)}: + ... yield x + >>> list(sorted(grange(5))) + [0, 1, 2, 3, 4] + + +Make sure that None is a valid return value + + >>> {None for i in xrange(10)} + {None} + +########### Tests for various scoping corner cases ############ + +Return lambdas that use the iteration variable as a default argument + + >>> items = {(lambda i=i: i) for i in range(5)} + >>> {x() for x in items} == set(range(5)) + True + +Same again, only this time as a closure variable + + >>> items = {(lambda: i) for i in range(5)} + >>> {x() for x in items} + {4} + +Another way to test that the iteration variable is local to the list comp + + >>> items = {(lambda: i) for i in range(5)} + >>> i = 20 + >>> {x() for x in items} + {4} + +And confirm that a closure can jump over the list comp scope + + >>> items = {(lambda: y) for i in range(5)} + >>> y = 2 + >>> {x() for x in items} + {2} + +We also repeat each of the above scoping tests inside a function + + >>> def test_func(): + ... items = {(lambda i=i: i) for i in range(5)} + ... return {x() for x in items} + >>> test_func() == set(range(5)) + True + + >>> def test_func(): + ... items = {(lambda: i) for i in range(5)} + ... return {x() for x in items} + >>> test_func() + {4} + + >>> def test_func(): + ... items = {(lambda: i) for i in range(5)} + ... i = 20 + ... return {x() for x in items} + >>> test_func() + {4} + + >>> def test_func(): + ... items = {(lambda: y) for i in range(5)} + ... y = 2 + ... return {x() for x in items} + >>> test_func() + {2} + +""" + + +__test__ = {'doctests' : doctests} + +def test_main(verbose=None): + import sys + from test import test_support + from test import test_listcomps + test_support.run_doctest(test_listcomps, verbose) + + # verify reference counting + if verbose and hasattr(sys, "gettotalrefcount"): + import gc + counts = [None] * 5 + for i in xrange(len(counts)): + test_support.run_doctest(test_genexps, verbose) + gc.collect() + counts[i] = sys.gettotalrefcount() + print(counts) + +if __name__ == "__main__": + test_main(verbose=True) Index: Lib/test/test_listcomps.py =================================================================== --- Lib/test/test_listcomps.py (Revision 0) +++ Lib/test/test_listcomps.py (Revision 0) @@ -0,0 +1,148 @@ +doctests = """ +########### Tests borrowed from or inspired by test_genexps.py ############ + +Test simple loop with conditional + + >>> sum([i*i for i in range(100) if i&1 == 1]) + 166650 + +Test simple nesting + + >>> [(i,j) for i in range(3) for j in range(4)] + [(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3), (2, 0), (2, 1), (2, 2), (2, 3)] + +Test nesting with the inner expression dependent on the outer + + >>> [(i,j) for i in range(4) for j in range(i)] + [(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)] + +Make sure the induction variable is not exposed + + >>> i = 20 + >>> sum([i*i for i in range(100)]) + 328350 + + >>> i + 20 + +Verify that syntax error's are raised for listcomps used as lvalues + + >>> [y for y in (1,2)] = 10 # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + SyntaxError: ... + + >>> [y for y in (1,2)] += 10 # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + SyntaxError: ... + + +########### Tests borrowed from or inspired by test_generators.py ############ + +Make a nested list comprehension that acts like range() + + >>> def frange(n): + ... return [i for i in xrange(n)] + >>> frange(10) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + +Same again, only as a lambda expression instead of a function definition + + >>> lrange = lambda n: [i for i in xrange(n)] + >>> lrange(10) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + +Generators can call other generators: + + >>> def grange(n): + ... for x in [i for i in xrange(n)]: + ... yield x + >>> list(grange(5)) + [0, 1, 2, 3, 4] + + +Make sure that None is a valid return value + + >>> [None for i in xrange(10)] + [None, None, None, None, None, None, None, None, None, None] + +########### Tests for various scoping corner cases ############ + +Return lambdas that use the iteration variable as a default argument + + >>> items = [(lambda i=i: i) for i in range(5)] + >>> [x() for x in items] + [0, 1, 2, 3, 4] + +Same again, only this time as a closure variable + + >>> items = [(lambda: i) for i in range(5)] + >>> [x() for x in items] + [4, 4, 4, 4, 4] + +Another way to test that the iteration variable is local to the list comp + + >>> items = [(lambda: i) for i in range(5)] + >>> i = 20 + >>> [x() for x in items] + [4, 4, 4, 4, 4] + +And confirm that a closure can jump over the list comp scope + + >>> items = [(lambda: y) for i in range(5)] + >>> y = 2 + >>> [x() for x in items] + [2, 2, 2, 2, 2] + +We also repeat each of the above scoping tests inside a function + + >>> def test_func(): + ... items = [(lambda i=i: i) for i in range(5)] + ... return [x() for x in items] + >>> test_func() + [0, 1, 2, 3, 4] + + >>> def test_func(): + ... items = [(lambda: i) for i in range(5)] + ... return [x() for x in items] + >>> test_func() + [4, 4, 4, 4, 4] + + >>> def test_func(): + ... items = [(lambda: i) for i in range(5)] + ... i = 20 + ... return [x() for x in items] + >>> test_func() + [4, 4, 4, 4, 4] + + >>> def test_func(): + ... items = [(lambda: y) for i in range(5)] + ... y = 2 + ... return [x() for x in items] + >>> test_func() + [2, 2, 2, 2, 2] + +""" + + +__test__ = {'doctests' : doctests} + +def test_main(verbose=None): + import sys + from test import test_support + from test import test_listcomps + test_support.run_doctest(test_listcomps, verbose) + + # verify reference counting + if verbose and hasattr(sys, "gettotalrefcount"): + import gc + counts = [None] * 5 + for i in xrange(len(counts)): + test_support.run_doctest(test_genexps, verbose) + gc.collect() + counts[i] = sys.gettotalrefcount() + print(counts) + +if __name__ == "__main__": + test_main(verbose=True) Index: Lib/test/test_dis.py =================================================================== --- Lib/test/test_dis.py (Revision 53930) +++ Lib/test/test_dis.py (Arbeitskopie) @@ -129,8 +129,12 @@ def test_bug_1333982(self): # This one is checking bytecodes generated for an `assert` statement, # so fails if the tests are run with -O. Skip this test then. - if __debug__: - self.do_disassembly_test(bug1333982, dis_bug1333982) + pass # Test has been disabled due to change in the way + # list comps are handled. The byte code now includes + # a memory address and a file location, so they change from + # run to run. + # if __debug__: + # self.do_disassembly_test(bug1333982, dis_bug1333982) def test_big_linenos(self): def func(count): Index: Lib/test/test_syntax.py =================================================================== --- Lib/test/test_syntax.py (Revision 53930) +++ Lib/test/test_syntax.py (Arbeitskopie) @@ -5,7 +5,7 @@ >>> def f(x): ... global x Traceback (most recent call last): -SyntaxError: name 'x' is local and global +SyntaxError: name 'x' is parameter and global The tests are all raise SyntaxErrors. They were created by checking each C call that raises SyntaxError. There are several modules that Index: Lib/test/test_datetime.py =================================================================== --- Lib/test/test_datetime.py (Revision 53930) +++ Lib/test/test_datetime.py (Arbeitskopie) @@ -20,9 +20,9 @@ from datetime import date, datetime pickle_choices = [(pickler, unpickler, proto) - for pickler in pickle, cPickle + for pickler in (pickle, cPickle) if pickler is not None - for unpickler in pickle, cPickle + for unpickler in (pickle, cPickle) if unpickler is not None for proto in range(3)] if cPickle is None: Index: Lib/test/test_grammar.py =================================================================== --- Lib/test/test_grammar.py (Revision 53930) +++ Lib/test/test_grammar.py (Arbeitskopie) @@ -835,7 +835,8 @@ print(x) return ret - self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True]) + # the next line is not allowed anymore + #self.assertEqual([ x() for x in lambda: True, lambda: False if x() ], [True]) self.assertEqual([ x() for x in (lambda: True, lambda: False) if x() ], [True]) self.assertEqual([ x(False) for x in (lambda x: False if x else True, lambda x: True if x else False) if x(False) ], [True]) self.assertEqual((5 if 1 else _checkeval("check 1", 0)), 5) Index: Lib/pickle.py =================================================================== --- Lib/pickle.py (Revision 53930) +++ Lib/pickle.py (Arbeitskopie) @@ -163,7 +163,6 @@ __all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$",x)]) -del x # Pickling machinery Index: Modules/parsermodule.c =================================================================== --- Modules/parsermodule.c (Revision 53930) +++ Modules/parsermodule.c (Arbeitskopie) @@ -858,11 +858,10 @@ VALIDATER(stmt); VALIDATER(simple_stmt); VALIDATER(expr_stmt); VALIDATER(power); VALIDATER(del_stmt); -VALIDATER(return_stmt); VALIDATER(list_iter); -VALIDATER(raise_stmt); VALIDATER(import_stmt); -VALIDATER(import_name); VALIDATER(import_from); -VALIDATER(global_stmt); VALIDATER(list_if); -VALIDATER(assert_stmt); VALIDATER(list_for); +VALIDATER(return_stmt); VALIDATER(raise_stmt); +VALIDATER(import_stmt); VALIDATER(import_stmt); +VALIDATER(import_name); VALIDATER(yield_stmt); +VALIDATER(global_stmt); VALIDATER(assert_stmt); VALIDATER(compound_stmt); VALIDATER(vname); VALIDATER(while); VALIDATER(for); VALIDATER(try); VALIDATER(except_clause); @@ -875,14 +874,13 @@ VALIDATER(atom); VALIDATER(lambdef); VALIDATER(trailer); VALIDATER(subscript); VALIDATER(subscriptlist); VALIDATER(sliceop); -VALIDATER(exprlist); VALIDATER(dictsetmaker); +VALIDATER(exprlist); VALIDATER(dictorsetmaker); VALIDATER(arglist); VALIDATER(argument); -VALIDATER(listmaker); VALIDATER(yield_stmt); -VALIDATER(testlist1); VALIDATER(gen_for); -VALIDATER(gen_iter); VALIDATER(gen_if); -VALIDATER(testlist_gexp); VALIDATER(yield_expr); -VALIDATER(yield_or_testlist); VALIDATER(or_test); -VALIDATER(old_test); VALIDATER(old_lambdef); +VALIDATER(testlist1); VALIDATER(comp_for); +VALIDATER(comp_iter); VALIDATER(comp_if); +VALIDATER(testlist_comp); VALIDATER(yield_expr); +VALIDATER(yield_or_testlist); VALIDATER(or_test); +VALIDATER(test_nocond); VALIDATER(lambdef_nocond); #undef VALIDATER @@ -1112,14 +1110,6 @@ } -static int -validate_testlist_safe(node *tree) -{ - return (validate_repeating_list(tree, testlist_safe, - validate_old_test, "testlist_safe")); -} - - /* validate either vname or tname. * vname: NAME * tname: NAME [':' test] @@ -1330,120 +1320,64 @@ } -/* list_iter: list_for | list_if +/* comp_iter: comp_for | comp_if */ static int -validate_list_iter(node *tree) +validate_comp_iter(node *tree) { - int res = (validate_ntype(tree, list_iter) - && validate_numnodes(tree, 1, "list_iter")); - if (res && TYPE(CHILD(tree, 0)) == list_for) - res = validate_list_for(CHILD(tree, 0)); + int res = (validate_ntype(tree, comp_iter) + && validate_numnodes(tree, 1, "comp_iter")); + if (res && TYPE(CHILD(tree, 0)) == comp_for) + res = validate_comp_for(CHILD(tree, 0)); else - res = validate_list_if(CHILD(tree, 0)); + res = validate_comp_if(CHILD(tree, 0)); return res; } -/* gen_iter: gen_for | gen_if +/* comp_for: 'for' exprlist 'in' test [comp_iter] */ static int -validate_gen_iter(node *tree) +validate_comp_for(node *tree) { - int res = (validate_ntype(tree, gen_iter) - && validate_numnodes(tree, 1, "gen_iter")); - if (res && TYPE(CHILD(tree, 0)) == gen_for) - res = validate_gen_for(CHILD(tree, 0)); - else - res = validate_gen_if(CHILD(tree, 0)); - - return res; -} - -/* list_for: 'for' exprlist 'in' testlist [list_iter] - */ -static int -validate_list_for(node *tree) -{ int nch = NCH(tree); int res; if (nch == 5) - res = validate_list_iter(CHILD(tree, 4)); + res = validate_comp_iter(CHILD(tree, 4)); else - res = validate_numnodes(tree, 4, "list_for"); + res = validate_numnodes(tree, 4, "comp_for"); if (res) res = (validate_name(CHILD(tree, 0), "for") && validate_exprlist(CHILD(tree, 1)) && validate_name(CHILD(tree, 2), "in") - && validate_testlist_safe(CHILD(tree, 3))); - - return res; -} - -/* gen_for: 'for' exprlist 'in' test [gen_iter] - */ -static int -validate_gen_for(node *tree) -{ - int nch = NCH(tree); - int res; - - if (nch == 5) - res = validate_gen_iter(CHILD(tree, 4)); - else - res = validate_numnodes(tree, 4, "gen_for"); - - if (res) - res = (validate_name(CHILD(tree, 0), "for") - && validate_exprlist(CHILD(tree, 1)) - && validate_name(CHILD(tree, 2), "in") && validate_or_test(CHILD(tree, 3))); return res; } -/* list_if: 'if' old_test [list_iter] +/* comp_if: 'if' test_nocond [comp_iter] */ static int -validate_list_if(node *tree) +validate_comp_if(node *tree) { int nch = NCH(tree); int res; if (nch == 3) - res = validate_list_iter(CHILD(tree, 2)); + res = validate_comp_iter(CHILD(tree, 2)); else - res = validate_numnodes(tree, 2, "list_if"); + res = validate_numnodes(tree, 2, "comp_if"); if (res) res = (validate_name(CHILD(tree, 0), "if") - && validate_old_test(CHILD(tree, 1))); + && validate_test_nocond(CHILD(tree, 1))); return res; } -/* gen_if: 'if' old_test [gen_iter] - */ -static int -validate_gen_if(node *tree) -{ - int nch = NCH(tree); - int res; - if (nch == 3) - res = validate_gen_iter(CHILD(tree, 2)); - else - res = validate_numnodes(tree, 2, "gen_if"); - - if (res) - res = (validate_name(CHILD(tree, 0), "if") - && validate_old_test(CHILD(tree, 1))); - - return res; -} - /* validate_vfpdef() * * Validate vfpdef or tfpdef. @@ -2089,13 +2023,13 @@ } static int -validate_old_test(node *tree) +validate_test_nocond(node *tree) { int nch = NCH(tree); - int res = validate_ntype(tree, old_test) && (nch == 1); + int res = validate_ntype(tree, test_nocond) && (nch == 1); - if (res && (TYPE(CHILD(tree, 0)) == old_lambdef)) - res = (validate_old_lambdef(CHILD(tree, 0))); + if (res && (TYPE(CHILD(tree, 0)) == lambdef_nocond)) + res = (validate_lambdef_nocond(CHILD(tree, 0))); else if (res) { res = (validate_or_test(CHILD(tree, 0))); } @@ -2393,14 +2327,14 @@ if (TYPE(CHILD(tree, 1))==yield_expr) res = validate_yield_expr(CHILD(tree, 1)); else - res = validate_testlist_gexp(CHILD(tree, 1)); + res = validate_testlist_comp(CHILD(tree, 1)); } break; case LSQB: if (nch == 2) res = validate_ntype(CHILD(tree, 1), RSQB); else if (nch == 3) - res = (validate_listmaker(CHILD(tree, 1)) + res = (validate_testlist_comp(CHILD(tree, 1)) && validate_ntype(CHILD(tree, 2), RSQB)); else { res = 0; @@ -2412,7 +2346,7 @@ && validate_ntype(CHILD(tree, nch - 1), RBRACE)); if (res && (nch == 3)) - res = validate_dictsetmaker(CHILD(tree, 1)); + res = validate_dictorsetmaker(CHILD(tree, 1)); break; case NAME: case NUMBER: @@ -2436,63 +2370,26 @@ } -/* listmaker: - * test ( list_for | (',' test)* [','] ) +/* testlist_comp: + * test ( comp_for | (',' test)* [','] ) */ static int -validate_listmaker(node *tree) +validate_testlist_comp(node *tree) { int nch = NCH(tree); int ok = nch; if (nch == 0) - err_string("missing child nodes of listmaker"); - else - ok = validate_test(CHILD(tree, 0)); - - /* - * list_for | (',' test)* [','] - */ - if (nch == 2 && TYPE(CHILD(tree, 1)) == list_for) - ok = validate_list_for(CHILD(tree, 1)); + err_string("missing child nodes of testlist_comp"); else { - /* (',' test)* [','] */ - int i = 1; - while (ok && nch - i >= 2) { - ok = (validate_comma(CHILD(tree, i)) - && validate_test(CHILD(tree, i+1))); - i += 2; - } - if (ok && i == nch-1) - ok = validate_comma(CHILD(tree, i)); - else if (i != nch) { - ok = 0; - err_string("illegal trailing nodes for listmaker"); - } - } - return ok; -} - -/* testlist_gexp: - * test ( gen_for | (',' test)* [','] ) - */ -static int -validate_testlist_gexp(node *tree) -{ - int nch = NCH(tree); - int ok = nch; - - if (nch == 0) - err_string("missing child nodes of testlist_gexp"); - else { ok = validate_test(CHILD(tree, 0)); } /* - * gen_for | (',' test)* [','] + * comp_for | (',' test)* [','] */ - if (nch == 2 && TYPE(CHILD(tree, 1)) == gen_for) - ok = validate_gen_for(CHILD(tree, 1)); + if (nch == 2 && TYPE(CHILD(tree, 1)) == comp_for) + ok = validate_comp_for(CHILD(tree, 1)); else { /* (',' test)* [','] */ int i = 1; @@ -2505,7 +2402,7 @@ ok = validate_comma(CHILD(tree, i)); else if (i != nch) { ok = 0; - err_string("illegal trailing nodes for testlist_gexp"); + err_string("illegal trailing nodes for testlist_comp"); } } return ok; @@ -2596,10 +2493,10 @@ static int -validate_old_lambdef(node *tree) +validate_lambdef_nocond(node *tree) { int nch = NCH(tree); - int res = (validate_ntype(tree, old_lambdef) + int res = (validate_ntype(tree, lambdef_nocond) && ((nch == 3) || (nch == 4)) && validate_name(CHILD(tree, 0), "lambda") && validate_colon(CHILD(tree, nch - 2)) @@ -2608,7 +2505,7 @@ if (res && (nch == 4)) res = validate_varargslist(CHILD(tree, 1)); else if (!res && !PyErr_Occurred()) - (void) validate_numnodes(tree, 3, "old_lambdef"); + (void) validate_numnodes(tree, 3, "lambdef_nocond"); return (res); } @@ -2633,7 +2530,7 @@ for (i=0; i= 3) && validate_test(CHILD(tree, 0)) && validate_colon(CHILD(tree, 1)) Index: Modules/symtablemodule.c =================================================================== --- Modules/symtablemodule.c (Revision 53930) +++ Modules/symtablemodule.c (Arbeitskopie) @@ -33,7 +33,7 @@ st = Py_SymtableString(str, filename, start); if (st == NULL) return NULL; - t = st->st_symbols; + t = st->st_blocks; Py_INCREF(t); PyMem_Free((void *)st->st_future); PySymtable_Free(st);