27 const auto window_func_context =
31 switch (window_func->getKind()) {
36 return code_generator.codegenWindowPosition(window_func_context,
37 code_generator.posArg(
nullptr));
40 return cgen_state_->emitCall(
"percent_window_func",
41 {cgen_state_->llInt(reinterpret_cast<const int64_t>(
42 window_func_context->output())),
43 code_generator.posArg(
nullptr)});
51 const auto&
args = window_func->getArgs();
53 const auto arg_lvs = code_generator.codegen(
args.front().get(),
true, co);
55 return arg_lvs.front();
65 return codegenWindowFunctionAggregate(&code_generator, co);
73 return codegenWindowNavigationFunctionOnFrame(co);
75 LOG(
FATAL) <<
"Invalid window function kind";
101 agg_name =
"agg_count";
105 agg_name =
"agg_count_if";
109 agg_name =
"agg_sum_if";
113 LOG(
FATAL) <<
"Invalid window function kind";
116 switch (window_func_ti.
get_type()) {
118 agg_name +=
"_float";
122 agg_name +=
"_double";
136 ?
args.front()->get_type_info()
149 }
else if (size == 2) {
151 }
else if (size == 4) {
158 if (col_ti.
is_fp()) {
166 return cgen_state->
llInt((int16_t)inline_int_null_value<int16_t>());
169 return cgen_state->
llInt((int32_t)inline_int_null_value<int32_t>());
173 return cgen_state->
llInt((int16_t)inline_int_null_value<int16_t>());
175 return cgen_state->
llInt((int32_t)inline_int_null_value<int32_t>());
177 return cgen_state->
llInt((int64_t)inline_int_null_value<int64_t>());
183 return cgen_state->
llInt((int8_t)inline_int_null_value<int8_t>());
188 return cgen_state->
llInt((int16_t)inline_int_null_value<int16_t>());
193 return cgen_state->
llInt((int32_t)inline_int_null_value<int32_t>());
203 return cgen_state->
llInt((int64_t)inline_int_null_value<int64_t>());
208 return cgen_state->
llInt((int64_t)inline_int_null_value<int64_t>());
218 if (col_ti.
is_fp()) {
225 llvm::Value* ret_val{
nullptr};
233 if (ret_val->getType()->getIntegerBitWidth() > ret_val_col_size_in_bytes) {
234 return cgen_state->
castToTypeIn(ret_val, ret_val_col_size_in_bytes);
245 const auto window_func_context =
247 const auto window_func = window_func_context->getWindowFunction();
250 arg_ti.get_type() ==
kFLOAT
251 ? llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0)
252 : llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
253 const auto aggregate_state_i64 = cgen_state_->llInt(
254 reinterpret_cast<const int64_t>(window_func_context->aggregateState()));
260 aggregate_state_type,
268 auto [reset_state_false_bb, aggregate_state] =
269 codegenWindowResetStateControlFlow(code_generator, co);
270 llvm::Value* aggregate_state_count =
nullptr;
271 const auto window_func_context =
273 const auto window_func = window_func_context->getWindowFunction();
275 const auto aggregate_state_count_i64 = cgen_state_->llInt(
276 reinterpret_cast<const int64_t>(window_func_context->aggregateStateCount()));
277 const auto pi64_type =
278 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
283 aggregate_state_count_i64,
288 codegenWindowFunctionStateInit(code_generator, co, aggregate_state);
290 const auto count_zero = cgen_state_->llInt(int64_t(0));
291 cgen_state_->emitCall(
"agg_id", {aggregate_state_count, count_zero});
293 cgen_state_->ir_builder_.CreateBr(reset_state_false_bb);
294 cgen_state_->ir_builder_.SetInsertPoint(reset_state_false_bb);
296 return codegenWindowFunctionAggregateCalls(aggregate_state, co);
303 const auto window_func_context =
305 auto aggregate_state = aggregateWindowStatePtr(code_generator, co);
306 const auto bitset = cgen_state_->llInt(
307 reinterpret_cast<const int64_t>(window_func_context->partitionStart()));
308 const auto bitset_lv =
314 llvm::PointerType::get(
get_int_type(8, cgen_state_->context_), 0),
317 const auto min_val = cgen_state_->llInt(int64_t(0));
318 const auto max_val = cgen_state_->llInt(window_func_context->elementCount() - 1);
319 const auto null_val = cgen_state_->llInt(inline_int_null_value<int64_t>());
320 const auto null_bool_val = cgen_state_->llInt<int8_t>(inline_int_null_value<int8_t>());
321 const auto reset_state =
322 code_generator->
toBool(cgen_state_->emitCall(
"bit_is_set",
324 code_generator->
posArg(
nullptr),
329 const auto reset_state_true_bb = llvm::BasicBlock::Create(
330 cgen_state_->context_,
"reset_state.true", cgen_state_->current_func_);
331 const auto reset_state_false_bb = llvm::BasicBlock::Create(
332 cgen_state_->context_,
"reset_state.false", cgen_state_->current_func_);
333 cgen_state_->ir_builder_.CreateCondBr(
334 reset_state, reset_state_true_bb, reset_state_false_bb);
335 cgen_state_->ir_builder_.SetInsertPoint(reset_state_true_bb);
336 return std::make_pair(reset_state_false_bb, aggregate_state);
341 llvm::Value* aggregate_state) {
343 const auto window_func_context =
345 const auto window_func = window_func_context->getWindowFunction();
347 const auto window_func_null_val =
348 window_func_ti.is_fp()
349 ? cgen_state_->inlineFpNull(window_func_ti)
350 : cgen_state_->castToTypeIn(cgen_state_->inlineIntNull(window_func_ti), 64);
351 llvm::Value* window_func_init_val;
352 const auto window_func_kind = window_func_context->getWindowFunction()->getKind();
355 switch (window_func_ti.get_type()) {
357 window_func_init_val = cgen_state_->llFp(
float(0));
361 window_func_init_val = cgen_state_->llFp(
double(0));
365 window_func_init_val = cgen_state_->llInt(int64_t(0));
370 window_func_init_val = window_func_null_val;
372 const auto pi32_type =
373 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
374 switch (window_func_ti.get_type()) {
376 cgen_state_->emitCall(
"agg_id_double", {aggregate_state, window_func_init_val});
381 cgen_state_->ir_builder_.CreateBitCast(aggregate_state, pi32_type);
382 cgen_state_->emitCall(
"agg_id_float", {aggregate_state, window_func_init_val});
386 cgen_state_->emitCall(
"agg_id", {aggregate_state, window_func_init_val});
395 const auto window_func_context =
397 const auto window_func = window_func_context->getWindowFunction();
398 const auto window_func_kind = window_func->getKind();
399 const auto&
args = window_func->getArgs();
403 const auto target_col_ti =
args.front()->get_type_info();
404 const auto target_col_size = target_col_ti.get_size();
405 const auto target_col_type_name =
408 window_func->get_type_info().get_size(), window_func->get_type_info().is_fp());
413 auto logical_null_val_lv =
415 auto target_col_null_val_lv =
417 if (window_func_context->elementCount() == 0) {
419 return target_col_null_val_lv;
422 auto current_row_pos_lv = code_generator.posArg(
nullptr);
423 auto partition_index_lv = codegenCurrentPartitionIndex(
424 window_func_context, &code_generator, co, current_row_pos_lv);
427 size_t target_col_size_in_byte = target_col_size * 8;
429 target_col_ti.is_fp()
430 ?
get_fp_type(target_col_size_in_byte, cgen_state_->context_)
431 :
get_int_type(target_col_size_in_byte, cgen_state_->context_);
432 auto col_buf_type = llvm::PointerType::get(col_buf_ptr_type, 0);
433 auto target_col_buf_ptr_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
434 window_func_context->getColumnBufferForWindowFunctionExpressions().front()));
439 target_col_buf_ptr_lv,
445 auto partition_buf_ptrs = codegenLoadPartitionBuffers(
446 window_func_context, &code_generator, co, partition_index_lv);
449 const auto order_key_buf_ti =
450 window_func_context->getOrderKeyColumnBufferTypes().front();
451 auto const ordering_spec = window_func->getCollation().front();
452 auto order_key_col_null_val_lv =
456 auto [order_col_type_name, order_key_buf_ptr_lv] =
457 codegenLoadOrderKeyBufPtr(window_func_context, &code_generator, co);
460 auto [null_start_pos_lv, null_end_pos_lv] =
461 codegenFrameNullRange(window_func_context, &code_generator, co, partition_index_lv);
464 std::string row_idx_on_frame_func =
"compute_";
465 row_idx_on_frame_func += order_col_type_name;
466 row_idx_on_frame_func += ordering_spec.is_desc ?
"_greater_equal" :
"_less_equal";
467 row_idx_on_frame_func +=
"_current_row_idx_in_frame";
468 auto int64_t_one_val_lv = cgen_state_->llInt((int64_t)1);
469 auto nulls_first_lv = cgen_state_->llBool(ordering_spec.nulls_first);
470 auto cur_row_idx_in_frame_lv =
471 cgen_state_->emitCall(row_idx_on_frame_func,
472 {partition_buf_ptrs.num_elem_current_partition_lv,
474 order_key_buf_ptr_lv,
475 partition_buf_ptrs.target_partition_rowid_ptr_lv,
476 partition_buf_ptrs.target_partition_sorted_rowid_ptr_lv,
477 order_key_col_null_val_lv,
482 if (window_func->isMissingValueFillingFunction()) {
490 std::string func_name =
"fill_" + target_col_type_name +
"_missing_value";
492 llvm::Value* forward_fill_lv =
494 return cgen_state_->emitCall(func_name,
495 {cur_row_idx_in_frame_lv,
496 target_col_null_val_lv,
498 partition_buf_ptrs.num_elem_current_partition_lv,
499 partition_buf_ptrs.target_partition_rowid_ptr_lv,
500 partition_buf_ptrs.target_partition_sorted_rowid_ptr_lv,
505 auto [frame_start_bound_expr_lv, frame_end_bound_expr_lv] =
506 codegenFrameBoundRange(window_func, code_generator, co);
509 auto const int64_t_zero_val_lv = cgen_state_->llInt((int64_t)0);
511 frame_start_bound_expr_lv,
512 frame_end_bound_expr_lv,
513 window_func->hasRangeModeFraming() ? current_row_pos_lv : cur_row_idx_in_frame_lv,
515 window_func->hasRangeModeFraming()
516 ? int64_t_zero_val_lv
520 partition_buf_ptrs.num_elem_current_partition_lv,
521 order_key_buf_ptr_lv,
523 partition_buf_ptrs.target_partition_rowid_ptr_lv,
524 partition_buf_ptrs.target_partition_sorted_rowid_ptr_lv,
528 auto [frame_start_bound_lv, frame_end_bound_lv] =
529 codegenWindowFrameBounds(window_func_context,
530 window_func->getFrameStartBound(),
531 window_func->getFrameEndBound(),
532 order_key_col_null_val_lv,
537 llvm::Value* modified_cur_row_idx_in_frame_lv{
nullptr};
538 llvm::Value* offset_lv{
nullptr};
539 switch (window_func_kind) {
541 offset_lv = cgen_state_->castToTypeIn(
542 code_generator.codegen(
args[1].get(),
true, co)[0], 64);
543 modified_cur_row_idx_in_frame_lv =
544 cgen_state_->ir_builder_.CreateSub(cur_row_idx_in_frame_lv, offset_lv);
547 offset_lv = cgen_state_->castToTypeIn(
548 code_generator.codegen(
args[1].get(),
true, co)[0], 64);
549 modified_cur_row_idx_in_frame_lv =
550 cgen_state_->ir_builder_.CreateAdd(cur_row_idx_in_frame_lv, offset_lv);
553 modified_cur_row_idx_in_frame_lv = frame_start_bound_lv;
556 modified_cur_row_idx_in_frame_lv = frame_end_bound_lv;
559 offset_lv = cgen_state_->castToTypeIn(
560 code_generator.codegen(
args[1].get(),
true, co)[0], 64);
561 auto candidate_offset_lv =
562 cgen_state_->ir_builder_.CreateAdd(frame_start_bound_lv, offset_lv);
563 auto out_of_frame_bound_lv =
564 cgen_state_->ir_builder_.CreateICmpSGT(candidate_offset_lv, frame_end_bound_lv);
565 modified_cur_row_idx_in_frame_lv = cgen_state_->ir_builder_.CreateSelect(
566 out_of_frame_bound_lv, cgen_state_->llInt((int64_t)-1), candidate_offset_lv);
570 UNREACHABLE() <<
"Unsupported window function to navigate a window frame.";
572 CHECK(modified_cur_row_idx_in_frame_lv);
575 std::string target_func_name =
"get_";
576 target_func_name += target_col_type_name +
"_value_";
577 target_func_name += target_col_logical_type_name +
"_type_";
578 target_func_name +=
"in_frame";
580 cgen_state_->emitCall(target_func_name,
581 {modified_cur_row_idx_in_frame_lv,
582 frame_start_bound_lv,
585 partition_buf_ptrs.target_partition_rowid_ptr_lv,
586 partition_buf_ptrs.target_partition_sorted_rowid_ptr_lv,
588 target_col_null_val_lv});
590 res_lv = cgen_state_->emitCall(
592 {res_lv, logical_null_val_lv, cgen_state_->llInt((int64_t)
kSecsPerDay)});
606 const auto order_col_ti = window_func->
getOrderKeys().front()->get_type_info();
607 auto encode_date_col_val = [&order_col_ti,
this](llvm::Value* bound_expr_lv) {
608 if (order_col_ti.get_comp_param() == 16) {
609 return cgen_state_->emitCall(
610 "fixed_width_date_encode_noinline",
616 return cgen_state_->emitCall(
"fixed_width_date_encode_noinline",
622 llvm::Value* bound_expr_lv{
nullptr};
623 if (needs_bound_expr_codegen(frame_bound)) {
625 if (
auto dateadd_expr = dynamic_cast<const Analyzer::DateaddExpr*>(bound_expr)) {
626 if (dateadd_expr->get_datetime_expr()->get_type_info().is_encoded_timestamp()) {
627 dateadd_expr->set_fixed_encoding_null_val();
630 auto bound_expr_lvs = code_generator.
codegen(bound_expr,
true, co);
631 bound_expr_lv = bound_expr_lvs.front();
634 throw std::runtime_error(
635 "Range mode with date type ordering column is not supported yet.");
637 bound_expr_lv = encode_date_col_val(bound_expr_lv);
640 bound_expr_lv = cgen_state_->castToTypeIn(bound_expr_lv, 64);
643 bound_expr_lv = cgen_state_->llInt((int64_t)-1);
645 CHECK(bound_expr_lv);
646 return bound_expr_lv;
651 bool for_window_frame_naviation,
653 bool is_timestamp_type_frame,
654 llvm::Value* order_key_null_val,
657 auto adjust_frame_end_bound = [&](llvm::Value* target_bound_lv) {
661 CHECK(for_start_bound) <<
"frame end cannot be UNBOUNDED PRECEDING";
664 CHECK(!for_start_bound) <<
"frame start cannot be UNBOUNDED FOLLOWING";
666 return for_window_frame_naviation
670 std::vector<llvm::Value*> func_args;
671 std::string op_name =
673 if (!for_range_mode) {
674 llvm::Value* current_row_bound_expr_lv{
nullptr};
675 if (for_window_frame_naviation) {
680 current_row_bound_expr_lv =
683 current_row_bound_expr_lv =
690 if (for_start_bound) {
691 return cgen_state_->ir_builder_.CreateSelect(
692 cgen_state_->ir_builder_.CreateICmpSLT(current_row_bound_expr_lv,
695 current_row_bound_expr_lv);
697 return cgen_state_->ir_builder_.CreateSelect(
698 cgen_state_->ir_builder_.CreateICmpSGE(current_row_bound_expr_lv,
701 current_row_bound_expr_lv);
704 std::string func_class = for_start_bound ?
"start" :
"end";
705 auto const func_name =
"compute_row_mode_" + func_class +
"_index_" + op_name;
706 func_args = prepareRowModeFuncArgs(for_start_bound, bound_type, args);
707 current_row_bound_expr_lv = cgen_state_->emitCall(func_name, func_args);
709 return current_row_bound_expr_lv;
711 std::string func_class = for_start_bound ?
"lower" :
"upper";
712 auto const func_name = getFramingFuncName(
717 func_args = prepareRangeModeFuncArgs(
718 for_start_bound, frame_bound, is_timestamp_type_frame, order_key_null_val, args);
719 auto frame_bound_lv = cgen_state_->emitCall(func_name, func_args);
720 if (!for_start_bound && for_window_frame_naviation) {
722 frame_bound_lv = cgen_state_->ir_builder_.CreateSelect(
723 cgen_state_->ir_builder_.CreateICmpSGE(frame_bound_lv,
728 return frame_bound_lv;
735 return window_func->
getOrderKeys().front()->get_type_info();
739 const auto order_key_size = getFirstOrderColTypeInfo(window_func_context).get_size();
740 return order_key_size;
745 auto const order_key_size = getOrderKeySize(window_func_context);
746 auto const order_key_ptr =
748 CHECK(order_key_ptr);
750 order_key_ptr->get_type_info().is_fp());
757 llvm::Value* current_col_value_ptr_lv{
nullptr};
758 const auto order_key_size_in_byte = getOrderKeySize(window_func_context) * 8;
759 auto const order_key_ptr =
761 CHECK(order_key_ptr);
762 auto const order_col_ti = order_key_ptr->get_type_info();
763 auto const order_col_llvm_type =
764 order_col_ti.is_fp() ?
get_fp_type(order_key_size_in_byte, cgen_state_->context_)
765 :
get_int_type(order_key_size_in_byte, cgen_state_->context_);
769 current_col_value_ptr_lv = cgen_state_->ir_builder_.CreateGEP(
772 current_col_value_ptr_lv = cgen_state_->ir_builder_.CreateGEP(
775 return cgen_state_->ir_builder_.CreateLoad(
776 current_col_value_ptr_lv->getType()->getPointerElementType(),
777 current_col_value_ptr_lv,
778 "current_col_value");
785 llvm::Value* current_row_pos_lv) {
786 const auto pi64_type =
787 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
788 const auto pi32_type =
789 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
790 auto row_pos_lv = current_row_pos_lv;
808 auto*
const hash_slot_idx_ptr =
810 auto hash_slot_idx_buf_lv =
811 cgen_state_->llInt(reinterpret_cast<int64_t>(hash_slot_idx_ptr));
816 hash_slot_idx_buf_lv,
820 auto hash_slot_idx_load_lv = cgen_state_->ir_builder_.CreateGEP(
821 hash_slot_idx_ptr_lv->getType()->getPointerElementType(),
822 hash_slot_idx_ptr_lv,
824 row_pos_lv = cgen_state_->castToTypeIn(
825 cgen_state_->ir_builder_.CreateLoad(
826 hash_slot_idx_load_lv->getType()->getPointerElementType(),
827 hash_slot_idx_load_lv,
828 "cur_row_hash_slot_idx"),
831 auto partition_count_lv = cgen_state_->llInt(window_func_context->
partitionCount());
832 auto partition_num_count_buf_lv = cgen_state_->llInt(
838 partition_num_count_buf_lv,
842 return cgen_state_->emitCall(
843 "compute_int64_t_lower_bound",
844 {partition_count_lv, row_pos_lv, partition_num_count_ptr_lv});
848 const std::string& order_col_type,
849 const std::string& op_type,
850 bool for_timestamp_type)
const {
851 auto target_val_type = for_timestamp_type ?
"int64_t" : order_col_type;
852 auto null_type = for_timestamp_type ?
"int64_t" : order_col_type;
853 return "range_mode_" + target_val_type +
"_" + order_col_type +
"_" + null_type +
"_" +
854 op_type +
"_frame_" + bound_type +
"_bound";
858 bool for_start_bound,
876 bool for_start_bound,
878 bool is_timestamp_type_frame,
879 llvm::Value* order_key_null_val,
881 llvm::Value* bound_expr_lv =
883 llvm::Value* target_val_lv =
887 llvm::Value* frame_bound_val_lv =
908 llvm::Value* partition_index_lv)
const {
909 const auto pi64_type =
910 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
911 const auto null_start_pos_buf = cgen_state_->llInt(
921 const auto null_start_pos_ptr =
922 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
923 null_start_pos_buf_ptr,
925 auto null_start_pos_lv = cgen_state_->ir_builder_.CreateLoad(
926 null_start_pos_ptr->getType()->getPointerElementType(),
929 const auto null_end_pos_buf = cgen_state_->llInt(
939 const auto null_end_pos_ptr = cgen_state_->ir_builder_.CreateGEP(
940 get_int_type(64, cgen_state_->context_), null_end_pos_buf_ptr, partition_index_lv);
941 auto null_end_pos_lv = cgen_state_->ir_builder_.CreateLoad(
942 null_end_pos_ptr->getType()->getPointerElementType(),
945 return std::make_pair(null_start_pos_lv, null_end_pos_lv);
952 auto const order_key_ti =
954 auto const order_key_size = order_key_ti.get_size();
958 size_t order_key_size_in_byte = order_key_size * 8;
959 auto const order_key_type =
960 order_key_ti.is_fp() ?
get_fp_type(order_key_size_in_byte, cgen_state_->context_)
961 :
get_int_type(order_key_size_in_byte, cgen_state_->context_);
962 auto const order_key_buf_type = llvm::PointerType::get(order_key_type, 0);
963 auto const order_key_buf = cgen_state_->llInt(
973 return std::make_pair(order_col_type_name, order_key_buf_ptr_lv);
980 llvm::Value* partition_index_lv)
const {
982 const auto pi64_type =
983 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
984 const auto pi32_type =
985 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
988 auto partition_start_offset_buf_lv = cgen_state_->llInt(
994 partition_start_offset_buf_lv,
1000 auto current_partition_start_offset_ptr_lv =
1001 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
1002 partition_start_offset_ptr_lv,
1003 partition_index_lv);
1005 current_partition_start_offset_ptr_lv->getType()->getPointerElementType(),
1006 current_partition_start_offset_ptr_lv);
1009 const auto partition_rowid_buf_lv =
1010 cgen_state_->llInt(reinterpret_cast<int64_t>(window_func_context->
payload()));
1015 partition_rowid_buf_lv,
1020 cgen_state_->ir_builder_.CreateGEP(
get_int_type(32, cgen_state_->context_),
1021 partition_rowid_ptr_lv,
1025 const auto sorted_rowid_lv = cgen_state_->llInt(
1036 cgen_state_->ir_builder_.CreateGEP(
get_int_type(64, cgen_state_->context_),
1037 sorted_rowid_ptr_lv,
1041 const auto partition_count_buf =
1042 cgen_state_->llInt(reinterpret_cast<int64_t>(window_func_context->
counts()));
1047 partition_count_buf,
1053 const auto num_elem_current_partition_ptr =
1054 cgen_state_->ir_builder_.CreateGEP(
get_int_type(32, cgen_state_->context_),
1055 partition_count_buf_ptr_lv,
1056 partition_index_lv);
1058 cgen_state_->ir_builder_.CreateLoad(
1059 num_elem_current_partition_ptr->getType()->getPointerElementType(),
1060 num_elem_current_partition_ptr),
1071 auto frame_start_bound_expr_lv =
1072 codegenFrameBoundExpr(window_func, frame_start_bound, code_generator, co);
1073 auto frame_end_bound_expr_lv =
1074 codegenFrameBoundExpr(window_func, frame_end_bound, code_generator, co);
1075 CHECK(frame_start_bound_expr_lv);
1076 CHECK(frame_end_bound_expr_lv);
1077 return std::make_pair(frame_start_bound_expr_lv, frame_end_bound_expr_lv);
1084 llvm::Value* order_key_col_null_val_lv,
1092 if (window_func->hasRangeModeFraming()) {
1094 CHECK(window_func->getOrderKeys().size() == 1UL);
1098 codegenLoadCurrentValueFromColBuf(window_func_context, code_generator, args);
1101 auto get_order_key_null_val = [is_timestamp_type_frame,
1102 &order_key_col_null_val_lv,
1104 return is_timestamp_type_frame && !frame_bound->isCurrentRowBound()
1105 ? cgen_state_->castToTypeIn(order_key_col_null_val_lv, 64)
1106 : order_key_col_null_val_lv;
1108 auto frame_start_bound_lv =
1109 codegenFrameBound(
true,
1110 window_func->hasRangeModeFraming(),
1111 window_func->isFrameNavigateWindowFunction(),
1113 is_timestamp_type_frame,
1114 get_order_key_null_val(frame_start_bound),
1116 auto frame_end_bound_lv =
1117 codegenFrameBound(
false,
1118 window_func->hasRangeModeFraming(),
1119 window_func->isFrameNavigateWindowFunction(),
1121 is_timestamp_type_frame,
1122 get_order_key_null_val(frame_end_bound),
1124 CHECK(frame_start_bound_lv);
1125 CHECK(frame_end_bound_lv);
1126 return std::make_pair(frame_start_bound_lv, frame_end_bound_lv);
1132 const auto window_func_context =
1134 const auto window_func = window_func_context->getWindowFunction();
1136 const auto window_func_null_val =
1137 window_func_ti.is_fp()
1138 ? cgen_state_->inlineFpNull(window_func_ti)
1139 : cgen_state_->castToTypeIn(cgen_state_->inlineIntNull(window_func_ti), 64);
1140 if (window_func_context->elementCount() == 0) {
1144 : window_func_null_val;
1146 const auto&
args = window_func->getArgs();
1148 if (window_func_context->needsToBuildAggregateTree()) {
1153 const auto pi64_type =
1154 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
1155 const auto ppi64_type = llvm::PointerType::get(
1156 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0), 0);
1158 auto [frame_start_bound_expr_lv, frame_end_bound_expr_lv] =
1159 codegenFrameBoundRange(window_func, code_generator, co);
1162 auto current_row_pos_lv = code_generator.
posArg(
nullptr);
1163 auto partition_index_lv = codegenCurrentPartitionIndex(
1164 window_func_context, &code_generator, co, current_row_pos_lv);
1167 const auto target_col_ti =
args.front()->get_type_info();
1168 const auto target_col_size = target_col_ti.get_size();
1169 const auto col_type_name =
1172 const auto partition_buf_ptrs = codegenLoadPartitionBuffers(
1173 window_func_context, &code_generator, co, partition_index_lv);
1175 auto [order_col_type_name, order_key_buf_ptr_lv] =
1176 codegenLoadOrderKeyBufPtr(window_func_context, &code_generator, co);
1179 const auto order_key_buf_ti =
1180 window_func_context->getOrderKeyColumnBufferTypes().front();
1181 auto const ordering_spec = window_func->getCollation().front();
1182 llvm::Value* order_key_col_null_val_lv{
nullptr};
1183 switch (order_key_buf_ti.get_type()) {
1190 order_key_col_null_val_lv = cgen_state_->llInt((int32_t)null_val);
1195 order_key_col_null_val_lv = cgen_state_->inlineNull(order_key_buf_ti);
1200 auto [null_start_pos_lv, null_end_pos_lv] = codegenFrameNullRange(
1201 window_func_context, &code_generator, co, partition_index_lv);
1202 auto nulls_first_lv = cgen_state_->llBool(ordering_spec.nulls_first);
1205 frame_start_bound_expr_lv,
1206 frame_end_bound_expr_lv,
1210 cgen_state_->llInt((int64_t)0),
1211 cgen_state_->llInt((int64_t)1),
1212 partition_buf_ptrs.num_elem_current_partition_lv,
1213 order_key_buf_ptr_lv,
1215 partition_buf_ptrs.target_partition_rowid_ptr_lv,
1216 partition_buf_ptrs.target_partition_sorted_rowid_ptr_lv,
1220 auto [frame_start_bound_lv, frame_end_bound_lv] =
1221 codegenWindowFrameBounds(window_func_context,
1222 window_func->getFrameStartBound(),
1223 window_func->getFrameEndBound(),
1224 order_key_col_null_val_lv,
1229 llvm::ConstantInt* aggregation_trees_lv{
nullptr};
1230 llvm::Value* invalid_val_lv{
nullptr};
1231 llvm::Value* null_val_lv{
nullptr};
1232 std::string aggregation_tree_search_func_name{
"search_"};
1233 std::string aggregation_tree_getter_func_name{
"get_"};
1237 auto agg_expr_ti =
args.front()->get_type_info();
1238 if (agg_expr_ti.is_fp()) {
1240 invalid_val_lv = cgen_state_->llFp(std::numeric_limits<double>::max());
1242 invalid_val_lv = cgen_state_->llFp(std::numeric_limits<double>::lowest());
1244 invalid_val_lv = cgen_state_->llFp((
double)0);
1247 aggregation_tree_search_func_name +=
"double";
1248 aggregation_tree_getter_func_name +=
"double";
1251 invalid_val_lv = cgen_state_->llInt(std::numeric_limits<int64_t>::max());
1253 invalid_val_lv = cgen_state_->llInt(std::numeric_limits<int64_t>::lowest());
1255 invalid_val_lv = cgen_state_->llInt((int64_t)0);
1257 null_val_lv = cgen_state_->llInt(inline_int_null_value<int64_t>());
1258 aggregation_tree_search_func_name +=
"int64_t";
1259 aggregation_tree_getter_func_name +=
"integer";
1264 aggregation_tree_search_func_name +=
"_derived";
1265 aggregation_tree_getter_func_name +=
"_derived";
1269 if (agg_expr_ti.is_fp()) {
1271 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1272 window_func_context->getDerivedAggregationTreesForDoubleTypeWindowExpr()));
1274 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1275 window_func_context->getAggregationTreesForDoubleTypeWindowExpr()));
1279 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1280 window_func_context->getDerivedAggregationTreesForIntegerTypeWindowExpr()));
1282 aggregation_trees_lv = cgen_state_->llInt(reinterpret_cast<int64_t>(
1283 window_func_context->getAggregationTreesForIntegerTypeWindowExpr()));
1287 CHECK(aggregation_trees_lv);
1288 CHECK(invalid_val_lv);
1289 aggregation_tree_search_func_name +=
"_aggregation_tree";
1290 aggregation_tree_getter_func_name +=
"_aggregation_tree";
1297 aggregation_trees_lv,
1301 auto target_aggregation_tree_lv = cgen_state_->emitCall(
1302 aggregation_tree_getter_func_name, {aggregation_trees_ptr, partition_index_lv});
1305 const auto tree_depth_buf = cgen_state_->llInt(
1306 reinterpret_cast<int64_t>(window_func_context->getAggregateTreeDepth()));
1315 const auto current_partition_tree_depth_buf_ptr = cgen_state_->ir_builder_.CreateGEP(
1316 get_int_type(64, cgen_state_->context_), tree_depth_buf_ptr, partition_index_lv);
1317 const auto current_partition_tree_depth_lv = cgen_state_->ir_builder_.CreateLoad(
1318 current_partition_tree_depth_buf_ptr->getType()->getPointerElementType(),
1319 current_partition_tree_depth_buf_ptr);
1322 const auto aggregation_tree_fanout_lv = cgen_state_->llInt(
1323 static_cast<int64_t>(window_func_context->getAggregateTreeFanout()));
1326 const auto agg_type_lv =
1327 cgen_state_->llInt(static_cast<int32_t>(window_func->getKind()));
1332 cgen_state_->emitCall(aggregation_tree_search_func_name,
1333 {target_aggregation_tree_lv,
1334 frame_start_bound_lv,
1336 current_partition_tree_depth_lv,
1337 aggregation_tree_fanout_lv,
1338 cgen_state_->llBool(agg_expr_ti.is_decimal()),
1339 cgen_state_->llInt((int64_t)agg_expr_ti.get_scale()),
1345 std::string null_handler_func_name{
"handle_null_val_"};
1346 std::vector<llvm::Value*> null_handler_args{res_lv, null_val_lv};
1352 if (agg_expr_ti.is_fp()) {
1354 null_handler_func_name +=
"double_double";
1357 null_handler_func_name +=
"double_int64_t";
1359 }
else if (agg_expr_ti.is_fp()) {
1361 null_handler_func_name +=
"double_double";
1364 null_handler_func_name +=
"int64_t_int64_t";
1366 null_handler_func_name +=
"_window_framing_agg";
1370 if (agg_expr_ti.is_fp()) {
1371 null_handler_args.push_back(cgen_state_->llFp((
double)0));
1373 null_handler_args.push_back(cgen_state_->llInt((int64_t)0));
1378 null_handler_args.push_back(cgen_state_->castToTypeIn(window_func_null_val, 64));
1380 res_lv = cgen_state_->emitCall(null_handler_func_name, null_handler_args);
1385 return cgen_state_->ir_builder_.CreateFPToSI(
1388 agg_expr_ti.is_date_in_days()) {
1391 if (date_null_val->getType()->getScalarSizeInBits() != 32) {
1392 date_null_val = cgen_state_->castToTypeIn(date_null_val, 32);
1394 return cgen_state_->emitCall(
"fixed_width_date_decode",
1395 {res_lv, date_null_val, null_val_lv});
1401 std::vector<llvm::Value*> agg_func_args{aggregate_state};
1402 auto modified_window_func_null_val = window_func_null_val;
1407 agg_func_args.push_back(cgen_state_->llInt(int64_t(1)));
1411 arg_target_expr =
args.front().get();
1412 const auto arg_lvs = code_generator.
codegen(arg_target_expr,
true, co);
1413 CHECK_EQ(arg_lvs.size(), size_t(1));
1415 auto crt_val = arg_lvs.front();
1418 !window_func_ti.is_fp()) {
1420 arg_lvs.front(),
args.front()->get_type_info(), window_func_ti,
false);
1422 agg_func_args.push_back(window_func_ti.get_type() ==
kFLOAT
1424 : cgen_state_->castToTypeIn(crt_val, 64));
1426 llvm::Value* cond_lv{
nullptr};
1428 switch (window_func->getKind()) {
1431 modified_window_func_null_val = cgen_state_->castToTypeIn(
1437 if (
args.front()->get_type_info().is_integer()) {
1438 agg_func_args[1] = cgen_state_->castToTypeIn(agg_func_args[1], 64);
1440 modified_window_func_null_val =
1441 cgen_state_->castToTypeIn(window_func_null_val, 64);
1443 auto cond_expr_lv = code_generator.
codegen(
args[1].
get(),
true, co).front();
1445 codegenConditionalAggregateCondValSelector(cond_expr_lv,
kSUM_IF, co);
1451 agg_name +=
"_skip_val";
1452 agg_func_args.push_back(modified_window_func_null_val);
1454 agg_func_args.push_back(cond_lv);
1457 cgen_state_->emitCall(agg_name, agg_func_args);
1459 codegenWindowAvgEpilogue(
1460 &code_generator, co, agg_func_args[1], window_func_null_val);
1462 return codegenAggregateWindowState(&code_generator, co, aggregate_state);
1468 llvm::Value* crt_val,
1469 llvm::Value* window_func_null_val) {
1471 const auto window_func_context =
1473 const auto window_func = window_func_context->getWindowFunction();
1475 const auto pi32_type =
1476 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
1477 const auto pi64_type =
1478 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
1479 const auto aggregate_state_type =
1480 window_func_ti.get_type() ==
kFLOAT ? pi32_type : pi64_type;
1481 const auto aggregate_state_count_i64 = cgen_state_->llInt(
1482 reinterpret_cast<const int64_t>(window_func_context->aggregateStateCount()));
1487 aggregate_state_count_i64,
1488 aggregate_state_type,
1491 std::string agg_count_func_name =
"agg_count";
1492 switch (window_func_ti.get_type()) {
1494 agg_count_func_name +=
"_float";
1498 agg_count_func_name +=
"_double";
1505 agg_count_func_name +=
"_skip_val";
1506 cgen_state_->emitCall(agg_count_func_name,
1507 {aggregate_state_count, crt_val, window_func_null_val});
1512 llvm::Value* aggregate_state) {
1514 const auto pi32_type =
1515 llvm::PointerType::get(
get_int_type(32, cgen_state_->context_), 0);
1516 const auto pi64_type =
1517 llvm::PointerType::get(
get_int_type(64, cgen_state_->context_), 0);
1518 const auto window_func_context =
1522 const auto aggregate_state_type =
1523 window_func_ti.get_type() ==
kFLOAT ? pi32_type : pi64_type;
1525 const auto aggregate_state_count_i64 = cgen_state_->llInt(
1526 reinterpret_cast<const int64_t>(window_func_context->aggregateStateCount()));
1531 aggregate_state_count_i64,
1532 aggregate_state_type,
1536 switch (window_func_ti.get_type()) {
1538 return cgen_state_->emitCall(
1539 "load_avg_float", {aggregate_state, aggregate_state_count, double_null_lv});
1542 return cgen_state_->emitCall(
1543 "load_avg_double", {aggregate_state, aggregate_state_count, double_null_lv});
1546 return cgen_state_->emitCall(
1549 aggregate_state_count,
1551 cgen_state_->llInt<int32_t>(window_func_ti.get_scale())});
1554 return cgen_state_->emitCall(
1555 "load_avg_int", {aggregate_state, aggregate_state_count, double_null_lv});
1560 return cgen_state_->ir_builder_.CreateLoad(
1561 aggregate_state->getType()->getPointerElementType(), aggregate_state);
1563 switch (window_func_ti.get_type()) {
1565 return cgen_state_->emitCall(
"load_float", {aggregate_state});
1568 return cgen_state_->emitCall(
"load_double", {aggregate_state});
1571 return cgen_state_->ir_builder_.CreateLoad(
1572 aggregate_state->getType()->getPointerElementType(), aggregate_state);
1578 llvm::Value* cond_lv,
1581 llvm::Value* res_cond_lv{
nullptr};
1584 if (cond_lv->getType()->isIntegerTy(1)) {
1587 res_cond_lv = cgen_state_->castToTypeIn(cond_lv, 8);
1589 CHECK(cond_lv->getType()->isIntegerTy(8));
1595 cgen_state_->ir_builder_.CreateICmpEQ(cond_lv, cgen_state_->llInt((int8_t)1));
1596 res_cond_lv = cgen_state_->ir_builder_.CreateSelect(
1597 true_cond_lv, cgen_state_->llInt((int8_t)1), cgen_state_->llInt((int8_t)0));
std::string get_col_type_name_by_size(const size_t size, const bool is_fp)
bool hasRangeModeFraming() const
bool isCurrentRowBound() const
llvm::Value * castToTypeIn(llvm::Value *val, const size_t bit_width)
SqlWindowFunctionKind getKind() const
llvm::Value * num_elem_current_partition_lv
HOST DEVICE int get_size() const
bool hasTimestampTypeFrameBound() const
llvm::Value * current_col_value_lv
bool isFrameNavigateWindowFunction() const
void codegenWindowFunctionStateInit(CodeGenerator *code_generator, const CompilationOptions &co, llvm::Value *aggregate_state)
int64_t * getNullValueEndPos() const
bool window_function_conditional_aggregate(const SqlWindowFunctionKind kind)
llvm::Value * current_partition_start_offset_lv
llvm::Value * current_row_pos_lv
llvm::Value * target_partition_rowid_ptr_lv
llvm::Value * codegenFrameBound(bool for_start_bound, bool for_range_mode, bool for_window_frame_naviation, const Analyzer::WindowFrame *frame_bound, bool is_timestamp_type_frame, llvm::Value *order_key_null_val, const WindowFrameBoundFuncArgs &args)
llvm::Value * frame_end_bound_expr_lv
llvm::Value * num_elem_current_partition_lv
llvm::Value * nulls_first_lv
const std::vector< SQLTypeInfo > & getOrderKeyColumnBufferTypes() const
llvm::Value * posArg(const Analyzer::Expr *) const
llvm::Value * aggregateWindowStatePtr(CodeGenerator *code_generator, const CompilationOptions &co)
size_t elementCount() const
llvm::Type * get_fp_type(const int width, llvm::LLVMContext &context)
std::string getFramingFuncName(const std::string &bound_type, const std::string &order_col_type, const std::string &op_type, bool for_timestamp_type) const
const int32_t * counts() const
const Analyzer::WindowFrame * getFrameStartBound() const
HOST DEVICE SQLTypes get_type() const
llvm::Value * target_partition_sorted_rowid_ptr_lv
const Analyzer::WindowFrame * getFrameEndBound() const
llvm::Value * target_partition_rowid_ptr_lv
llvm::Type * get_int_type(const int width, llvm::LLVMContext &context)
const std::vector< std::shared_ptr< Analyzer::Expr > > & getOrderKeys() const
static WindowFunctionContext * getActiveWindowFunctionContext(Executor *executor)
std::string get_window_agg_name(const SqlWindowFunctionKind kind, const SQLTypeInfo &window_func_ti)
int64_t * getNullValueStartPos() const
const int64_t * partitionStartOffset() const
std::pair< llvm::Value *, llvm::Value * > codegenFrameNullRange(WindowFunctionContext *window_func_context, CodeGenerator *code_generator, const CompilationOptions &co, llvm::Value *partition_index_lv) const
size_t partitionCount() const
SqlWindowFrameBoundType getBoundType() const
llvm::Value * codegenWindowPosition(const WindowFunctionContext *window_func_context, llvm::Value *pos_arg)
llvm::Value * null_start_pos_lv
static const WindowProjectNodeContext * get(Executor *executor)
const std::vector< std::shared_ptr< Analyzer::Expr > > & getArgs() const
int get_logical_size() const
std::vector< llvm::Value * > prepareRangeModeFuncArgs(bool for_start_bound, const Analyzer::WindowFrame *frame_bound, bool is_timestamp_type_frame, llvm::Value *order_key_null_val, const WindowFrameBoundFuncArgs &frame_args) const
const SQLTypeInfo getFirstOrderColTypeInfo(WindowFunctionContext *window_func_context) const
const WindowFunctionContext * activateWindowFunctionContext(Executor *executor, const size_t target_index) const
std::pair< llvm::Value *, llvm::Value * > codegenWindowFrameBounds(WindowFunctionContext *window_func_context, const Analyzer::WindowFrame *frame_start_bound, const Analyzer::WindowFrame *frame_end_bound, llvm::Value *order_key_col_null_val_lv, WindowFrameBoundFuncArgs &args, CodeGenerator &code_generator)
llvm::Value * codegenCastBetweenIntTypes(llvm::Value *operand_lv, const SQLTypeInfo &operand_ti, const SQLTypeInfo &ti, bool upscale=true)
const Analyzer::Expr * getBoundExpr() const
llvm::ConstantFP * llFp(const float v) const
std::pair< llvm::BasicBlock *, llvm::Value * > codegenWindowResetStateControlFlow(CodeGenerator *code_generator, const CompilationOptions &co)
const SQLTypeInfo & get_type_info() const
static const int NUM_EXECUTION_DEVICES
std::vector< llvm::Value * > codegen(const Analyzer::Expr *, const bool fetch_columns, const CompilationOptions &)
llvm::Value * current_partition_start_offset_lv
llvm::Value * codegenLoadCurrentValueFromColBuf(WindowFunctionContext *window_func_context, CodeGenerator &code_generator, WindowFrameBoundFuncArgs &args) const
HOST DEVICE EncodingType get_compression() const
llvm::Value * target_partition_sorted_rowid_ptr_lv
void codegenWindowAvgEpilogue(CodeGenerator *code_generator, const CompilationOptions &co, llvm::Value *crt_val, llvm::Value *window_func_null_val)
const std::string getOrderKeyTypeName(WindowFunctionContext *window_func_context) const
llvm::Value * codegenConditionalAggregateCondValSelector(llvm::Value *cond_lv, SQLAgg const aggKind, CompilationOptions const &co) const
constexpr float inline_fp_null_value< float >()
std::pair< llvm::Value *, llvm::Value * > codegenFrameBoundRange(const Analyzer::WindowFunction *window_func, CodeGenerator &code_generator, const CompilationOptions &co)
const std::vector< const int8_t * > & getOrderKeyColumnBuffers() const
constexpr double inline_fp_null_value< double >()
llvm::Value * null_end_pos_lv
const int64_t * partitionNumCountBuf() const
llvm::Value * codegenWindowFunctionAggregate(CodeGenerator *code_generator, const CompilationOptions &co)
llvm::Value * toBool(llvm::Value *)
llvm::Value * codegenCurrentPartitionIndex(const WindowFunctionContext *window_func_context, CodeGenerator *code_generator, const CompilationOptions &co, llvm::Value *current_row_pos_lv)
HOST DEVICE int get_comp_param() const
llvm::Value * codegenWindowNavigationFunctionOnFrame(const CompilationOptions &co)
llvm::Value * codegenFrameBoundExpr(const Analyzer::WindowFunction *window_func, const Analyzer::WindowFrame *frame_bound, CodeGenerator &code_generator, const CompilationOptions &co)
llvm::ConstantInt * llInt(const T v) const
llvm::Value * order_key_buf_ptr_lv
llvm::Value * int64_t_zero_val_lv
const int64_t * sortedPartition() const
WindowPartitionBufferPtrs codegenLoadPartitionBuffers(WindowFunctionContext *window_func_context, CodeGenerator *code_generator, const CompilationOptions &co, llvm::Value *partition_index_lv) const
int64_t inline_int_null_val(const SQL_TYPE_INFO &ti)
int64_t inline_fixed_encoding_null_val(const SQL_TYPE_INFO &ti)
std::pair< std::string, llvm::Value * > codegenLoadOrderKeyBufPtr(WindowFunctionContext *window_func_context, CodeGenerator *code_generator, const CompilationOptions &co) const
const Analyzer::WindowFunction * getWindowFunction() const
bool is_dict_encoded_string() const
const int32_t * payload() const
llvm::Value * get_null_value_by_size(CgenState *cgen_state, SQLTypeInfo col_ti)
std::vector< llvm::Value * > createPtrWithHoistedMemoryAddr(CgenState *cgen_state, CodeGenerator *code_generator, CompilationOptions const &co, llvm::ConstantInt *ptr_int_val, llvm::Type *type, size_t num_devices_to_hoist_literal)
size_t getOrderKeySize(WindowFunctionContext *window_func_context) const
llvm::Value * codegenAggregateWindowState(CodeGenerator *code_generator, const CompilationOptions &co, llvm::Value *aggregate_state)
llvm::Value * int64_t_one_val_lv
llvm::Value * codegenWindowFunctionAggregateCalls(llvm::Value *aggregate_state, const CompilationOptions &co)
llvm::Value * frame_start_bound_expr_lv
llvm::Value * codegenWindowFunction(const size_t target_index, const CompilationOptions &co)
std::string order_type_col_name
llvm::Value * get_null_value_by_size_with_encoding(CgenState *cgen_state, SQLTypeInfo col_ti)
std::vector< llvm::Value * > prepareRowModeFuncArgs(bool for_start_bound, SqlWindowFrameBoundType bound_type, const WindowFrameBoundFuncArgs &args) const
SQLTypeInfo get_adjusted_window_type_info(const Analyzer::WindowFunction *window_func)