OLD | NEW |
1 #include "Python.h" | 1 #include "Python.h" |
2 | 2 |
3 #include "JIT/ConstantMirror.h" | 3 #include "JIT/ConstantMirror.h" |
4 #include "JIT/global_llvm_data.h" | 4 #include "JIT/global_llvm_data.h" |
5 #include "JIT/llvm_fbuilder.h" | 5 #include "JIT/llvm_fbuilder.h" |
6 #include "JIT/opcodes/call.h" | 6 #include "JIT/opcodes/call.h" |
7 | 7 |
8 #include "llvm/BasicBlock.h" | 8 #include "llvm/BasicBlock.h" |
9 #include "llvm/Function.h" | 9 #include "llvm/Function.h" |
10 #include "llvm/Instructions.h" | 10 #include "llvm/Instructions.h" |
(...skipping 17 matching lines...) Expand all Loading... |
28 no_opt_no_data(0), no_opt_polymorphic(0) { | 28 no_opt_no_data(0), no_opt_polymorphic(0) { |
29 } | 29 } |
30 | 30 |
31 ~CallFunctionStats() { | 31 ~CallFunctionStats() { |
32 errs() << "\nCALL_FUNCTION optimization:\n"; | 32 errs() << "\nCALL_FUNCTION optimization:\n"; |
33 errs() << "Total opcodes: " << this->total << "\n"; | 33 errs() << "Total opcodes: " << this->total << "\n"; |
34 errs() << "Direct C calls: " << this->direct_calls << "\n"; | 34 errs() << "Direct C calls: " << this->direct_calls << "\n"; |
35 errs() << "Inlined: " << this->inlined << "\n"; | 35 errs() << "Inlined: " << this->inlined << "\n"; |
36 errs() << "No opt: callsite kwargs: " << this->no_opt_kwargs << "\n"; | 36 errs() << "No opt: callsite kwargs: " << this->no_opt_kwargs << "\n"; |
37 errs() << "No opt: function params: " << this->no_opt_params << "\n"; | 37 errs() << "No opt: function params: " << this->no_opt_params << "\n"; |
| 38 errs() << "No opt: not C function: " << this->no_opt_not_cfunc << "\n"; |
38 errs() << "No opt: no data: " << this->no_opt_no_data << "\n"; | 39 errs() << "No opt: no data: " << this->no_opt_no_data << "\n"; |
39 errs() << "No opt: polymorphic: " << this->no_opt_polymorphic << "\n"; | 40 errs() << "No opt: polymorphic: " << this->no_opt_polymorphic << "\n"; |
40 } | 41 } |
41 | 42 |
42 // How many CALL_FUNCTION opcodes were compiled. | 43 // How many CALL_FUNCTION opcodes were compiled. |
43 unsigned total; | 44 unsigned total; |
44 // How many CALL_FUNCTION opcodes were optimized to direct calls to C | 45 // How many CALL_FUNCTION opcodes were optimized to direct calls to C |
45 // functions. | 46 // functions. |
46 unsigned direct_calls; | 47 unsigned direct_calls; |
47 // How many calls were inlined into the caller. | 48 // How many calls were inlined into the caller. |
48 unsigned inlined; | 49 unsigned inlined; |
49 // We only optimize call sites without keyword, *args or **kwargs arguments. | 50 // We only optimize call sites without keyword, *args or **kwargs arguments. |
50 unsigned no_opt_kwargs; | 51 unsigned no_opt_kwargs; |
51 // We only optimize METH_ARG_RANGE functions so far. | 52 // We only optimize METH_ARG_RANGE functions so far. |
52 unsigned no_opt_params; | 53 unsigned no_opt_params; |
53 // We only optimize callsites where we've collected data. Note that since | 54 // We only optimize callsites where we've collected data. Note that since |
54 // we record only PyCFunctions, any call to a Python function will show up | 55 // we record only PyCFunctions, any call to a Python function will show up |
55 // as having no data. | 56 // as having no data. |
56 unsigned no_opt_no_data; | 57 unsigned no_opt_no_data; |
57 // We only optimize monomorphic callsites so far. | 58 // We only optimize monomorphic callsites so far. |
58 unsigned no_opt_polymorphic; | 59 unsigned no_opt_polymorphic; |
| 60 // We only optimize direct calls to C functions. |
| 61 unsigned no_opt_not_cfunc; |
59 }; | 62 }; |
60 | 63 |
61 static llvm::ManagedStatic<CallFunctionStats> call_function_stats; | 64 static llvm::ManagedStatic<CallFunctionStats> call_function_stats; |
62 | 65 |
63 #define CF_INC_STATS(field) call_function_stats->field++ | 66 #define CF_INC_STATS(field) call_function_stats->field++ |
64 #else | 67 #else |
65 #define CF_INC_STATS(field) | 68 #define CF_INC_STATS(field) |
66 #endif /* Py_WITH_INSTRUMENTATION */ | 69 #endif /* Py_WITH_INSTRUMENTATION */ |
67 | 70 |
68 namespace py { | 71 namespace py { |
69 | 72 |
70 OpcodeCall::OpcodeCall(LlvmFunctionBuilder *fbuilder) : | 73 OpcodeCall::OpcodeCall(LlvmFunctionBuilder *fbuilder) : |
71 fbuilder_(fbuilder), | 74 fbuilder_(fbuilder), |
72 state_(fbuilder->state()), | 75 state_(fbuilder->state()), |
73 builder_(fbuilder->builder()), | 76 builder_(fbuilder->builder()), |
74 llvm_data_(fbuilder->llvm_data()) | 77 llvm_data_(fbuilder->llvm_data()) |
75 { | 78 { |
76 } | 79 } |
77 | 80 |
78 void | 81 bool |
79 OpcodeCall::CALL_FUNCTION_fast(int oparg, | 82 OpcodeCall::CALL_FUNCTION_fast(int oparg) |
80 const PyRuntimeFeedback *feedback) | |
81 { | 83 { |
82 CF_INC_STATS(total); | |
83 | |
84 // Check for keyword arguments; we only optimize callsites with positional | 84 // Check for keyword arguments; we only optimize callsites with positional |
85 // arguments. | 85 // arguments. |
86 if ((oparg >> 8) & 0xff) { | 86 if ((oparg >> 8) & 0xff) { |
87 CF_INC_STATS(no_opt_kwargs); | 87 CF_INC_STATS(no_opt_kwargs); |
88 this->CALL_FUNCTION_safe(oparg); | 88 return false; |
89 return; | |
90 } | 89 } |
91 | 90 |
92 // Only optimize monomorphic callsites. | 91 // Only optimize monomorphic callsites. |
93 llvm::SmallVector<PyMethodDef*, 3> fdo_data; | 92 const PyRuntimeFeedback *feedback = this->fbuilder_->GetFeedback(); |
| 93 if (!feedback) { |
| 94 CF_INC_STATS(no_opt_no_data); |
| 95 return false; |
| 96 } |
| 97 if (feedback->FuncsOverflowed()) { |
| 98 CF_INC_STATS(no_opt_polymorphic); |
| 99 return false; |
| 100 } |
| 101 |
| 102 llvm::SmallVector<PyTypeMethodPair, 3> fdo_data; |
94 feedback->GetSeenFuncsInto(fdo_data); | 103 feedback->GetSeenFuncsInto(fdo_data); |
95 if (fdo_data.size() != 1) { | 104 if (fdo_data.size() != 1) { |
96 #ifdef Py_WITH_INSTRUMENTATION | 105 #ifdef Py_WITH_INSTRUMENTATION |
97 if (fdo_data.size() == 0) | 106 if (fdo_data.size() == 0) |
98 CF_INC_STATS(no_opt_no_data); | 107 CF_INC_STATS(no_opt_no_data); |
99 else | 108 else |
100 CF_INC_STATS(no_opt_polymorphic); | 109 CF_INC_STATS(no_opt_polymorphic); |
101 #endif | 110 #endif |
102 this->CALL_FUNCTION_safe(oparg); | 111 return false; |
103 return; | |
104 } | 112 } |
105 | 113 |
106 PyMethodDef *func_record = fdo_data[0]; | 114 PyMethodDef *func_record = fdo_data[0].second; |
| 115 PyTypeObject *type_record = (PyTypeObject *)fdo_data[0].first; |
| 116 // We embed a pointer to type_record but we don't incref it because it can |
| 117 // only be PyCFunction_Type or PyMethodDescr_Type, which are statically |
| 118 // allocated anyway. |
| 119 if (type_record != &PyCFunction_Type && |
| 120 type_record != &PyMethodDescr_Type) { |
| 121 CF_INC_STATS(no_opt_not_cfunc); |
| 122 return false; |
| 123 } |
| 124 bool func_is_cfunc = (type_record == &PyCFunction_Type); |
107 | 125 |
108 // Only optimize calls to C functions with a known number of parameters, | 126 // Only optimize calls to C functions with a known number of parameters, |
109 // where the number of arguments we have is in that range. | 127 // where the number of arguments we have is in that range. |
110 int flags = func_record->ml_flags; | 128 int flags = func_record->ml_flags; |
111 int min_arity = func_record->ml_min_arity; | 129 int min_arity = func_record->ml_min_arity; |
112 int max_arity = func_record->ml_max_arity; | 130 int max_arity = func_record->ml_max_arity; |
113 int num_args = oparg & 0xff; | 131 int num_args = oparg & 0xff; |
114 if (!(flags & METH_ARG_RANGE && | 132 if (!(flags & METH_ARG_RANGE && |
115 min_arity <= num_args && num_args <= max_arity)) { | 133 min_arity <= num_args && num_args <= max_arity)) { |
116 CF_INC_STATS(no_opt_params); | 134 CF_INC_STATS(no_opt_params); |
117 this->CALL_FUNCTION_safe(oparg); | 135 return false; |
118 return; | |
119 } | 136 } |
120 assert(num_args <= PY_MAX_ARITY); | 137 assert(num_args <= PY_MAX_ARITY); |
121 | 138 |
122 PyCFunction cfunc_ptr = func_record->ml_meth; | 139 PyCFunction cfunc_ptr = func_record->ml_meth; |
123 | 140 |
124 // Expose the C function pointer to LLVM. This is what will actually get | 141 // Expose the C function pointer to LLVM. This is what will actually get |
125 // called. | 142 // called. |
126 Constant *llvm_func = | 143 Constant *llvm_func = |
127 llvm_data_->constant_mirror().GetGlobalForCFunction( | 144 llvm_data_->constant_mirror().GetGlobalForCFunction( |
128 cfunc_ptr, | 145 cfunc_ptr, |
(...skipping 25 matching lines...) Expand all Loading... |
154 this->builder_.CreateLoad(this->fbuilder_->stack_pointer_addr()); | 171 this->builder_.CreateLoad(this->fbuilder_->stack_pointer_addr()); |
155 llvm_data_->tbaa_stack.MarkInstruction(stack_pointer); | 172 llvm_data_->tbaa_stack.MarkInstruction(stack_pointer); |
156 | 173 |
157 Value *actual_func = this->builder_.CreateLoad( | 174 Value *actual_func = this->builder_.CreateLoad( |
158 this->builder_.CreateGEP( | 175 this->builder_.CreateGEP( |
159 stack_pointer, | 176 stack_pointer, |
160 ConstantInt::getSigned( | 177 ConstantInt::getSigned( |
161 Type::getInt64Ty(this->fbuilder_->context()), | 178 Type::getInt64Ty(this->fbuilder_->context()), |
162 -num_args - 1))); | 179 -num_args - 1))); |
163 | 180 |
164 // Make sure it's a PyCFunction; if not, bail. | 181 // Make sure it's the right type; if not, bail. |
165 Value *is_cfunction = this->state_->CreateCall( | 182 Value *actual_type = this->builder_.CreateLoad( |
166 this->state_->GetGlobalFunction<int(PyObject *)>( | 183 ObjectTy::ob_type(this->builder_, actual_func)); |
167 "_PyLlvm_WrapCFunctionCheck"), | 184 Value *right_type = this->state_->EmbedPointer<PyTypeObject*>(type_record); |
168 actual_func, | 185 Value *is_right_type = this->builder_.CreateICmpEQ( |
169 "is_cfunction"); | 186 actual_type, right_type, "is_right_type"); |
170 Value *is_cfunction_guard = this->builder_.CreateICmpEQ( | 187 this->builder_.CreateCondBr(is_right_type, check_is_same_func, |
171 is_cfunction, ConstantInt::get(is_cfunction->getType(), 1), | |
172 "is_cfunction_guard"); | |
173 this->builder_.CreateCondBr(is_cfunction_guard, check_is_same_func, | |
174 invalid_assumptions); | 188 invalid_assumptions); |
175 | 189 |
176 // Make sure we got the same underlying function pointer; if not, bail. | 190 // Make sure we got the same underlying function pointer; if not, bail. |
177 this->builder_.SetInsertPoint(check_is_same_func); | 191 this->builder_.SetInsertPoint(check_is_same_func); |
178 Value *actual_as_pycfunc = this->builder_.CreateBitCast( | 192 Value *actual_as_righttype, *actual_method_def; |
179 actual_func, | 193 if (func_is_cfunc) { |
180 PyTypeBuilder<PyCFunctionObject *>::get(this->fbuilder_->context())); | 194 const Type *pycfunction_ty = |
181 Value *actual_method_def = this->builder_.CreateLoad( | 195 PyTypeBuilder<PyCFunctionObject *>::get(this->fbuilder_->context()); |
182 CFunctionTy::m_ml(this->builder_, actual_as_pycfunc), | 196 actual_as_righttype = this->builder_.CreateBitCast( |
183 "CALL_FUNCTION_actual_method_def"); | 197 actual_func, pycfunction_ty); |
| 198 actual_method_def = this->builder_.CreateLoad( |
| 199 CFunctionTy::m_ml(this->builder_, actual_as_righttype), |
| 200 "CALL_FUNCTION_actual_method_def"); |
| 201 } else { |
| 202 actual_as_righttype = this->builder_.CreateBitCast( |
| 203 actual_func, |
| 204 PyTypeBuilder<PyMethodDescrObject *>::get(this->fbuilder_->context()
)); |
| 205 actual_method_def = this->builder_.CreateLoad( |
| 206 MethodDescrTy::d_method(this->builder_, actual_as_righttype), |
| 207 "CALL_FUNCTION_actual_method_def"); |
| 208 } |
| 209 |
184 Value *actual_func_ptr = this->builder_.CreateLoad( | 210 Value *actual_func_ptr = this->builder_.CreateLoad( |
185 MethodDefTy::ml_meth(this->builder_, actual_method_def), | 211 MethodDefTy::ml_meth(this->builder_, actual_method_def), |
186 "CALL_FUNCTION_actual_func_ptr"); | 212 "CALL_FUNCTION_actual_func_ptr"); |
187 Value *is_same = this->builder_.CreateICmpEQ( | 213 Value *is_same = this->builder_.CreateICmpEQ( |
188 // TODO(jyasskin): change this to "llvm_func" when | 214 // TODO(jyasskin): change this to "llvm_func" when |
189 // http://llvm.org/PR5126 is fixed. | 215 // http://llvm.org/PR5126 is fixed. |
190 this->state_->EmbedPointer<PyCFunction>((void*)cfunc_ptr), | 216 this->state_->EmbedPointer<PyCFunction>((void*)cfunc_ptr), |
191 actual_func_ptr); | 217 actual_func_ptr); |
192 this->builder_.CreateCondBr(is_same, | 218 this->builder_.CreateCondBr(is_same, |
193 all_assumptions_valid, invalid_assumptions); | 219 all_assumptions_valid, invalid_assumptions); |
(...skipping 19 matching lines...) Expand all Loading... |
213 else if (arg1_type == &PyTuple_Type) | 239 else if (arg1_type == &PyTuple_Type) |
214 function_name = "_PyLlvm_BuiltinLen_Tuple"; | 240 function_name = "_PyLlvm_BuiltinLen_Tuple"; |
215 else if (arg1_type == &PyDict_Type) | 241 else if (arg1_type == &PyDict_Type) |
216 function_name = "_PyLlvm_BuiltinLen_Dict"; | 242 function_name = "_PyLlvm_BuiltinLen_Dict"; |
217 | 243 |
218 if (function_name != NULL) { | 244 if (function_name != NULL) { |
219 this->CALL_FUNCTION_fast_len(actual_func, stack_pointer, | 245 this->CALL_FUNCTION_fast_len(actual_func, stack_pointer, |
220 invalid_assumptions, | 246 invalid_assumptions, |
221 function_name); | 247 function_name); |
222 CF_INC_STATS(inlined); | 248 CF_INC_STATS(inlined); |
223 return; | 249 return true; |
224 } | 250 } |
225 } | 251 } |
226 | 252 |
227 // If we get here, we know we have a C function pointer | 253 // If we get here, we know we have a C function pointer |
228 // that takes some number of arguments: first the invocant, then some | 254 // that takes some number of arguments: first the invocant, then some |
229 // PyObject *s. If the underlying function is nullary, we use NULL for the | 255 // PyObject *s. If the underlying function is nullary, we use NULL for the |
230 // second argument. Because "the invocant" differs between built-in | 256 // second argument. Because "the invocant" differs between built-in |
231 // functions like len() and C-level methods like list.append(), we pull the | 257 // functions like len() and C-level methods like list.append(), we pull the |
232 // invocant (called m_self) from the PyCFunction object we popped | 258 // invocant (called m_self) from the PyCFunction object we popped |
233 // off the stack. Once the function returns, we patch up the stack pointer. | 259 // off the stack. Once the function returns, we patch up the stack pointer. |
234 Value *self = this->builder_.CreateLoad( | |
235 CFunctionTy::m_self(this->builder_, actual_as_pycfunc), | |
236 "CALL_FUNCTION_actual_self"); | |
237 llvm::SmallVector<Value*, PY_MAX_ARITY + 1> args; // +1 for self. | 260 llvm::SmallVector<Value*, PY_MAX_ARITY + 1> args; // +1 for self. |
238 args.push_back(self); | 261 // If the function is a PyCFunction, pass its self member. |
239 if (num_args == 0 && max_arity == 0) { | 262 if (func_is_cfunc) { |
240 args.push_back(this->state_->GetNull<PyObject *>()); | 263 Value *self = this->builder_.CreateLoad( |
| 264 CFunctionTy::m_self(this->builder_, actual_as_righttype), |
| 265 "CALL_FUNCTION_actual_self"); |
| 266 args.push_back(self); |
241 } | 267 } |
| 268 |
| 269 // Pass the arguments that are on the stack. |
242 for (int i = num_args; i >= 1; --i) { | 270 for (int i = num_args; i >= 1; --i) { |
243 args.push_back( | 271 args.push_back( |
244 this->builder_.CreateLoad( | 272 this->builder_.CreateLoad( |
245 this->builder_.CreateGEP( | 273 this->builder_.CreateGEP( |
246 stack_pointer, | 274 stack_pointer, |
247 ConstantInt::getSigned( | 275 ConstantInt::getSigned( |
248 Type::getInt64Ty(this->fbuilder_->context()), -i)))); | 276 Type::getInt64Ty(this->fbuilder_->context()), -i)))); |
249 } | 277 } |
250 for(int i = 0; i < (max_arity - num_args); ++i) { | 278 |
| 279 // Pad optional arguments with NULLs. |
| 280 for (int i = args.size(); i < max_arity + 1; ++i) { |
| 281 args.push_back(this->state_->GetNull<PyObject *>()); |
| 282 } |
| 283 |
| 284 // Pass a single NULL after self for METH_NOARGS functions. |
| 285 if (args.size() == 1 && max_arity == 0) { |
251 args.push_back(this->state_->GetNull<PyObject *>()); | 286 args.push_back(this->state_->GetNull<PyObject *>()); |
252 } | 287 } |
253 | 288 |
254 #ifdef WITH_TSC | 289 #ifdef WITH_TSC |
255 this->state_->LogTscEvent(CALL_ENTER_C); | 290 this->state_->LogTscEvent(CALL_ENTER_C); |
256 #endif | 291 #endif |
257 Value *result = | 292 Value *result = |
258 this->state_->CreateCall(llvm_func, args.begin(), args.end()); | 293 this->state_->CreateCall(llvm_func, args.begin(), args.end()); |
259 | 294 |
| 295 // Decref and the function and all of its arguments. |
260 this->state_->DecRef(actual_func); | 296 this->state_->DecRef(actual_func); |
261 // Decrefing args[0] will cause self to be double-decrefed, so avoid that. | 297 // If func is a cfunc, decrefing args[0] will cause self to be |
262 for (int i = 1; i <= num_args; ++i) { | 298 // double-decrefed, so avoid that. |
263 this->state_->DecRef(args[i]); | 299 for (unsigned i = (func_is_cfunc ? 1 : 0); i < args.size(); ++i) { |
| 300 // If LLVM knows that args[i] is NULL, it will delete this code. |
| 301 this->state_->XDecRef(args[i]); |
264 } | 302 } |
| 303 |
| 304 // Adjust the stack pointer to pop the function and its arguments. |
265 Value *new_stack_pointer = this->builder_.CreateGEP( | 305 Value *new_stack_pointer = this->builder_.CreateGEP( |
266 stack_pointer, | 306 stack_pointer, |
267 ConstantInt::getSigned( | 307 ConstantInt::getSigned( |
268 Type::getInt64Ty(this->fbuilder_->context()), | 308 Type::getInt64Ty(this->fbuilder_->context()), |
269 -num_args - 1)); | 309 -num_args - 1)); |
270 this->builder_.CreateStore(new_stack_pointer, | 310 this->builder_.CreateStore(new_stack_pointer, |
271 this->fbuilder_->stack_pointer_addr()); | 311 this->fbuilder_->stack_pointer_addr()); |
272 this->fbuilder_->PropagateExceptionOnNull(result); | 312 this->fbuilder_->PropagateExceptionOnNull(result); |
273 this->fbuilder_->Push(result); | 313 this->fbuilder_->Push(result); |
274 | 314 |
275 // Check signals and maybe switch threads after each function call. | 315 // Check signals and maybe switch threads after each function call. |
276 this->fbuilder_->CheckPyTicker(); | 316 this->fbuilder_->CheckPyTicker(); |
| 317 |
277 CF_INC_STATS(direct_calls); | 318 CF_INC_STATS(direct_calls); |
| 319 return true; |
278 } | 320 } |
279 | 321 |
280 void | 322 void |
281 OpcodeCall::CALL_FUNCTION_fast_len(Value *actual_func, | 323 OpcodeCall::CALL_FUNCTION_fast_len(Value *actual_func, |
282 Value *stack_pointer, | 324 Value *stack_pointer, |
283 BasicBlock *invalid_assumptions, | 325 BasicBlock *invalid_assumptions, |
284 const char *function_name) | 326 const char *function_name) |
285 { | 327 { |
286 BasicBlock *success = this->state_->CreateBasicBlock("BUILTIN_LEN_success"); | 328 BasicBlock *success = this->state_->CreateBasicBlock("BUILTIN_LEN_success"); |
287 | 329 |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
343 this->fbuilder_->PropagateExceptionOnNull(result); | 385 this->fbuilder_->PropagateExceptionOnNull(result); |
344 this->fbuilder_->Push(result); | 386 this->fbuilder_->Push(result); |
345 | 387 |
346 // Check signals and maybe switch threads after each function call. | 388 // Check signals and maybe switch threads after each function call. |
347 this->fbuilder_->CheckPyTicker(); | 389 this->fbuilder_->CheckPyTicker(); |
348 } | 390 } |
349 | 391 |
350 void | 392 void |
351 OpcodeCall::CALL_FUNCTION(int oparg) | 393 OpcodeCall::CALL_FUNCTION(int oparg) |
352 { | 394 { |
353 const PyRuntimeFeedback *feedback = this->fbuilder_->GetFeedback(); | 395 CF_INC_STATS(total); |
354 if (feedback == NULL || feedback->FuncsOverflowed()) | 396 if (!this->CALL_FUNCTION_fast(oparg)) { |
355 this->CALL_FUNCTION_safe(oparg); | 397 this->CALL_FUNCTION_safe(oparg); |
356 else | 398 } |
357 this->CALL_FUNCTION_fast(oparg, feedback); | 399 } |
| 400 |
| 401 void |
| 402 OpcodeCall::CALL_METHOD(int oparg) |
| 403 { |
| 404 // We only want to generate code to handle one case, but we need to be |
| 405 // robust in the face of malformed code objects, which might cause there to |
| 406 // be mismatched LOAD_METHOD/CALL_METHOD opcodes. Therefore the value we |
| 407 // get from the loads_optimized_ stack is only a guess, but it should be |
| 408 // accurate for all code objects with matching loads and calls. |
| 409 bool load_optimized = false; |
| 410 std::vector<bool> &loads_optimized = this->fbuilder_->loads_optimized(); |
| 411 if (!loads_optimized.empty()) { |
| 412 load_optimized = loads_optimized.back(); |
| 413 loads_optimized.pop_back(); |
| 414 } |
| 415 |
| 416 BasicBlock *call_block = state_->CreateBasicBlock("CALL_METHOD_call"); |
| 417 BasicBlock *bail_block = state_->CreateBasicBlock("CALL_METHOD_bail"); |
| 418 |
| 419 int num_args = (oparg & 0xff); |
| 420 int num_kwargs = (oparg>>8) & 0xff; |
| 421 // +1 for the actual function object, +1 for self. |
| 422 int num_stack_slots = num_args + 2 * num_kwargs + 1 + 1; |
| 423 |
| 424 // Look down the stack for the cell that is either padding or a method. |
| 425 Value *stack_pointer = |
| 426 this->builder_.CreateLoad(this->fbuilder_->stack_pointer_addr()); |
| 427 Value *stack_idx = |
| 428 ConstantInt::getSigned(Type::getInt32Ty(this->fbuilder_->context()), |
| 429 -num_stack_slots); |
| 430 Value *padding_ptr = this->builder_.CreateGEP(stack_pointer, |
| 431 stack_idx); |
| 432 Value *method_or_padding = this->builder_.CreateLoad(padding_ptr); |
| 433 |
| 434 // Depending on how we optimized the load, we either expect it to be NULL |
| 435 // or we expect it to be non-NULL. We bail if it's not what we expect. |
| 436 Value *bail_cond = state_->IsNull(method_or_padding); |
| 437 if (!load_optimized) { |
| 438 bail_cond = this->builder_.CreateNot(bail_cond); |
| 439 } |
| 440 this->builder_.CreateCondBr(bail_cond, bail_block, call_block); |
| 441 |
| 442 // Restore the stack in the bail bb. |
| 443 this->builder_.SetInsertPoint(bail_block); |
| 444 this->fbuilder_->CreateGuardBailPoint(_PYGUARD_CALL_METHOD); |
| 445 |
| 446 this->builder_.SetInsertPoint(call_block); |
| 447 if (load_optimized) { |
| 448 // Increment the argument count in oparg and do a regular CALL_FUNCTION. |
| 449 assert((num_args + 1) <= 0xff && |
| 450 "TODO(rnk): Deal with oparg overflow."); |
| 451 oparg = (oparg & ~0xff) | ((num_args + 1) & 0xff); |
| 452 } |
| 453 |
| 454 this->CALL_FUNCTION(oparg); |
| 455 |
| 456 if (!load_optimized) { |
| 457 // Pop off the padding cell. |
| 458 Value *result = fbuilder_->Pop(); |
| 459 Value *padding = fbuilder_->Pop(); |
| 460 state_->Assert(state_->IsNull(padding), "Padding was non-NULL!"); |
| 461 fbuilder_->Push(result); |
| 462 } |
358 } | 463 } |
359 | 464 |
360 // Keep this in sync with eval.cc | 465 // Keep this in sync with eval.cc |
361 #define CALL_FLAG_VAR 1 | 466 #define CALL_FLAG_VAR 1 |
362 #define CALL_FLAG_KW 2 | 467 #define CALL_FLAG_KW 2 |
363 | 468 |
364 void | 469 void |
365 OpcodeCall::CallVarKwFunction(int oparg, int call_flag) | 470 OpcodeCall::CallVarKwFunction(int oparg, int call_flag) |
366 { | 471 { |
367 #ifdef WITH_TSC | 472 #ifdef WITH_TSC |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
429 #ifdef WITH_TSC | 534 #ifdef WITH_TSC |
430 this->state_->LogTscEvent(CALL_START_LLVM); | 535 this->state_->LogTscEvent(CALL_START_LLVM); |
431 #endif | 536 #endif |
432 this->CallVarKwFunction(oparg, CALL_FLAG_KW | CALL_FLAG_VAR); | 537 this->CallVarKwFunction(oparg, CALL_FLAG_KW | CALL_FLAG_VAR); |
433 } | 538 } |
434 | 539 |
435 #undef CALL_FLAG_VAR | 540 #undef CALL_FLAG_VAR |
436 #undef CALL_FLAG_KW | 541 #undef CALL_FLAG_KW |
437 | 542 |
438 } | 543 } |
OLD | NEW |