Postgresql源码（115）LLVM JIT运行逻辑分析（上）

1 JIT入口开关

1. 总入口：jit_enabled打开 且 生成计划成本超过jit_above_cost启动JIT。
   • 计划成本超过jit_optimize_above_cost，执行PGJIT_OPT3使用O3对IR进行优化。
   • 计划成本超过jit_inline_above_cost，执行PGJIT_INLINE。
   • jit_expressions开关如果打开，执行PGJIT_EXPR表达式优化。
   • jit_tuple_deforming开关如果打开，执行PGJIT_DEFORM优化拆解元组流程。

standard_planner
	...
	...
	result->jitFlags = PGJIT_NONE;
	
	if (jit_enabled && jit_above_cost >= 0 &&
		top_plan->total_cost > jit_above_cost)
	{
		result->jitFlags |= PGJIT_PERFORM;

		/*
		 * Decide how much effort should be put into generating better code.
		 */
		if (jit_optimize_above_cost >= 0 &&
			top_plan->total_cost > jit_optimize_above_cost)
			result->jitFlags |= PGJIT_OPT3;
		if (jit_inline_above_cost >= 0 &&
			top_plan->total_cost > jit_inline_above_cost)
			result->jitFlags |= PGJIT_INLINE;

		/*
		 * Decide which operations should be JITed.
		 */
		if (jit_expressions)
			result->jitFlags |= PGJIT_EXPR;
		if (jit_tuple_deforming)
			result->jitFlags |= PGJIT_DEFORM;
	}

2 从表达式堆栈进入JIT逻辑jit_compile_expr

《Postgresql源码（113）表达式JIT计算简单分析》

#0  jit_compile_expr (state=0x1deae18) at jit.c:180
#1  0x000000000071fa6b in ExecReadyExpr (state=0x1deae18) at execExpr.c:874
#2  0x000000000071e60b in ExecInitExpr (node=0x1dfabb8, parent=0x0) at execExpr.c:152
#3  0x00000000008b3395 in evaluate_expr (expr=0x1dfabb8, result_type=23, result_typmod=-1, result_collation=0) at clauses.c:4892
#4  0x00000000008b26f8 in evaluate_function (funcid=1397, result_type=23, result_typmod=-1, result_collid=0, input_collid=0, args=0x1dfab68, funcvariadic=false, func_tuple=0x7fd9588871a8, context=0x7ffdd8867f20) at clauses.c:4409

...

3 jit_compile_expr初始化加载llvmjit.so

jit_compile_expr
	provider_init
		load_external_function(path, "_PG_jit_provider_init", true, NULL)

dlopen动态加载llvmjit.so，并调用so中的_PG_jit_provider_init初始化：

void
_PG_jit_provider_init(JitProviderCallbacks *cb)
{
	cb->reset_after_error = llvm_reset_after_error;
	cb->release_context = llvm_release_context;
	cb->compile_expr = llvm_compile_expr;
}

为provider配置入口函数：

typedef struct JitProviderCallbacks JitProviderCallbacks;

struct JitProviderCallbacks
{
	JitProviderResetAfterErrorCB reset_after_error;
	JitProviderReleaseContextCB release_context;
	JitProviderCompileExprCB compile_expr;
};

static JitProviderCallbacks provider;

jit_compile_expr继续调用hook：provider.compile_expr进入llvm逻辑：

jit_compile_expr
	provider_init
	provider.compile_expr(state)  -> llvm_compile_expr

4 llvm_compile_expr执行初始化llvm_create_context

llvm_create_context初始化生成LLVMJitContext结构：

typedef struct JitContext
{
	/* see PGJIT_* above */
	int			flags;

	ResourceOwner resowner;

	JitInstrumentation instr;
} JitContext;

typedef struct LLVMJitContext
{
	JitContext	base;               // 上面的JIT FLAG、ResourceOwner
	size_t		module_generation;  // 当前context存了几个Module？
	LLVMModuleRef module;           // 当前正在使用的module
	bool		compiled;           // 已经编译过了？
	List	   *handles;            // 所有挂在当前context下的module
} LLVMJitContext;

llvm_create_context初始化流程

llvm_create_context
	llvm_session_initialize
		【库函数】LLVMInitializeNativeTarget
		【库函数】LLVMInitializeNativeAsmPrinter
		【库函数】LLVMInitializeNativeAsmParser
		【库函数】LLVMContextSetOpaquePointers
		
		读取llvmjit_types.bc中需要的类型、函数签名：llvm_create_types
			LLVMCreateMemoryBufferWithContentsOfFile
			LLVMParseBitcode2
			LLVMDisposeMemoryBuffer
			
		【库函数】LLVMGetTargetFromTriple
		...
		【库函数】LLVMLoadLibraryPermanently
		
		llvm_ts_context = LLVMOrcCreateNewThreadSafeContext
		llvm_opt0_orc = llvm_create_jit_instance
			【库函数】若干
			传入机器信息，构造LLVMJIT环境
			【库函数】若干
			LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine
			【库函数】若干
			LLVMOrcCreateLLJIT
			
		llvm_opt3_orc = llvm_create_jit_instance
			【库函数】若干
			传入机器信息，构造LLVMJIT环境
			【库函数】若干
			LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine
			【库函数】若干
			LLVMOrcCreateLLJIT
			
	ResourceOwnerEnlargeJIT

从llvmjit_types.bc读取的类型、函数

	/*
	 * Load triple & layout from clang emitted file so we're guaranteed to be
	 * compatible.
	 */
	llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
	llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));

	TypeSizeT = llvm_pg_var_type("TypeSizeT");
	TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
	TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
	TypePGFunction = llvm_pg_var_type("TypePGFunction");
	StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
	StructExprContext = llvm_pg_var_type("StructExprContext");
	StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
	StructExprState = llvm_pg_var_type("StructExprState");
	StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
	StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
	StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
	StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
	StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
	StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
	StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
	StructAggState = llvm_pg_var_type("StructAggState");
	StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
	StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");

	AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");

读取到的所有类型、函数指针等记录在全局变量llvm_types_module中，用llvm_pg_var_type等函数调用LLVM库函数转换为LLVM能识别的类型、函数。

5 llvm_compile_expr创建module

创建Module需要的llvm_triple、llvm_layout都来自llvm_create_types函数，读取llvmjit_types.bc拿到的信息。

LLVMModuleRef
llvm_mutable_module(LLVMJitContext *context)
{
	llvm_assert_in_fatal_section();

	/*
	 * If there's no in-progress module, create a new one.
	 */
	if (!context->module)
	{
		context->compiled = false;
		context->module_generation = llvm_generation++;
		context->module = LLVMModuleCreateWithName("pg");
		LLVMSetTarget(context->module, llvm_triple);
		LLVMSetDataLayout(context->module, llvm_layout);
	}

	return context->module;
}

6 llvm_compile_expr新增函数到module中

llvm_compile_expr

新增函数到module

	eval_fn = LLVMAddFunction(mod, funcname,
							  llvm_pg_var_func_type("TypeExprStateEvalFunc"));

函数中加BB

	entry = LLVMAppendBasicBlock(eval_fn, "entry");

按表达式分支逻辑为BB添加代码

			case EEOP_FUNCEXPR_STRICT:
				{
					FunctionCallInfo fcinfo = op->d.func.fcinfo_data;
					LLVMValueRef v_fcinfo_isnull;
					LLVMValueRef v_retval;

					if (opcode == EEOP_FUNCEXPR_STRICT)
					{
						LLVMBasicBlockRef b_nonull;
						LLVMBasicBlockRef *b_checkargnulls;
						LLVMValueRef v_fcinfo;

						/*
						 * Block for the actual function call, if args are
						 * non-NULL.
						 */
						b_nonull = l_bb_before_v(opblocks[opno + 1],
												 "b.%d.no-null-args", opno);

						/* should make sure they're optimized beforehand */
						if (op->d.func.nargs == 0)
							elog(ERROR, "argumentless strict functions are pointless");

						v_fcinfo =
							l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData));

						/*
						 * set resnull to true, if the function is actually
						 * called, it'll be reset
						 */
						LLVMBuildStore(b, l_sbool_const(1), v_resnullp);

						/* create blocks for checking args, one for each */
						b_checkargnulls =
							palloc(sizeof(LLVMBasicBlockRef *) * op->d.func.nargs);
						for (int argno = 0; argno < op->d.func.nargs; argno++)
							b_checkargnulls[argno] =
								l_bb_before_v(b_nonull, "b.%d.isnull.%d", opno,
											  argno);

						/* jump to check of first argument */
						LLVMBuildBr(b, b_checkargnulls[0]);

						/* check each arg for NULLness */
						for (int argno = 0; argno < op->d.func.nargs; argno++)
						{
							LLVMValueRef v_argisnull;
							LLVMBasicBlockRef b_argnotnull;

							LLVMPositionBuilderAtEnd(b, b_checkargnulls[argno]);

							/*
							 * Compute block to jump to if argument is not
							 * null.
							 */
							if (argno + 1 == op->d.func.nargs)
								b_argnotnull = b_nonull;
							else
								b_argnotnull = b_checkargnulls[argno + 1];

							/* and finally load & check NULLness of arg */
							v_argisnull = l_funcnull(b, v_fcinfo, argno);
							LLVMBuildCondBr(b,
											LLVMBuildICmp(b, LLVMIntEQ,
														  v_argisnull,
														  l_sbool_const(1),
														  ""),
											opblocks[opno + 1],
											b_argnotnull);
						}

						LLVMPositionBuilderAtEnd(b, b_nonull);
					}

					v_retval = BuildV1Call(context, b, mod, fcinfo,
										   &v_fcinfo_isnull);
					LLVMBuildStore(b, v_retval, v_resvaluep);
					LLVMBuildStore(b, v_fcinfo_isnull, v_resnullp);

					LLVMBuildBr(b, opblocks[opno + 1]);
					break;
				}

7 （核心步骤）ExecRunCompiledExpr对module进行编译、优化、执行

ExecRunCompiledExpr找到jit函数并执行，惰性编译、优化。

ExecRunCompiledExpr
	llvm_get_function
		重要：llvm_compile_module
		LLVMOrcLLJITLookup

在找函数执行时，编译这一步是核心逻辑，编译会对上面逻辑进行优化处理：

llvm_compile_module
	llvm_inline
	llvm_optimize_module

优化一：llvm_inline

、llvm_build_inline_plan会查询module里面的function，到函数目录查找对应的bc文件，并加载bc文件中函数的逻辑（增加LLVM编译后，所有源码文件都会用clang额外生成一个bc文件，提供给inline使用）。function_inlinable函数会检查当前函数引用的其他函数时候能inline。

优化二：llvm_optimize_module

将IR过一遍PASS，下一篇继续分析后面的流程。