//===----------------------------------------------------------------------===// // DuckDB // // duckdb/common/vector_operations/binary_executor.hpp // // //===----------------------------------------------------------------------===// #pragma once #include "duckdb/common/exception.hpp" #include "duckdb/common/types/vector.hpp" #include "duckdb/common/vector_operations/vector_operations.hpp" #include namespace duckdb { struct DefaultNullCheckOperator { template static inline bool Operation(LEFT_TYPE left, RIGHT_TYPE right) { return false; } }; struct BinaryStandardOperatorWrapper { template static inline RESULT_TYPE Operation(FUNC fun, LEFT_TYPE left, RIGHT_TYPE right, ValidityMask &mask, idx_t idx) { return OP::template Operation(left, right); } static bool AddsNulls() { return false; } }; struct BinarySingleArgumentOperatorWrapper { template static inline RESULT_TYPE Operation(FUNC fun, LEFT_TYPE left, RIGHT_TYPE right, ValidityMask &mask, idx_t idx) { return OP::template Operation(left, right); } static bool AddsNulls() { return false; } }; struct BinaryLambdaWrapper { template static inline RESULT_TYPE Operation(FUNC fun, LEFT_TYPE left, RIGHT_TYPE right, ValidityMask &mask, idx_t idx) { return fun(left, right); } static bool AddsNulls() { return false; } }; struct BinaryLambdaWrapperWithNulls { template static inline RESULT_TYPE Operation(FUNC fun, LEFT_TYPE left, RIGHT_TYPE right, ValidityMask &mask, idx_t idx) { return fun(left, right, mask, idx); } static bool AddsNulls() { return true; } }; struct BinaryExecutor { template static void ExecuteFlatLoop(const LEFT_TYPE *__restrict ldata, const RIGHT_TYPE *__restrict rdata, RESULT_TYPE *__restrict result_data, idx_t count, ValidityMask &mask, FUNC fun) { if (!LEFT_CONSTANT) { ASSERT_RESTRICT(ldata, ldata + count, result_data, result_data + count); } if (!RIGHT_CONSTANT) { ASSERT_RESTRICT(rdata, rdata + count, result_data, result_data + count); } if (!mask.AllValid()) { idx_t base_idx = 0; auto entry_count = ValidityMask::EntryCount(count); for (idx_t entry_idx = 0; entry_idx < entry_count; entry_idx++) { auto validity_entry = mask.GetValidityEntry(entry_idx); idx_t next = MinValue(base_idx + ValidityMask::BITS_PER_VALUE, count); if (ValidityMask::AllValid(validity_entry)) { // all valid: perform operation for (; base_idx < next; base_idx++) { auto lentry = ldata[LEFT_CONSTANT ? 0 : base_idx]; auto rentry = rdata[RIGHT_CONSTANT ? 0 : base_idx]; result_data[base_idx] = OPWRAPPER::template Operation( fun, lentry, rentry, mask, base_idx); } } else if (ValidityMask::NoneValid(validity_entry)) { // nothing valid: skip all base_idx = next; continue; } else { // partially valid: need to check individual elements for validity idx_t start = base_idx; for (; base_idx < next; base_idx++) { if (ValidityMask::RowIsValid(validity_entry, base_idx - start)) { auto lentry = ldata[LEFT_CONSTANT ? 0 : base_idx]; auto rentry = rdata[RIGHT_CONSTANT ? 0 : base_idx]; result_data[base_idx] = OPWRAPPER::template Operation( fun, lentry, rentry, mask, base_idx); } } } } } else { for (idx_t i = 0; i < count; i++) { auto lentry = ldata[LEFT_CONSTANT ? 0 : i]; auto rentry = rdata[RIGHT_CONSTANT ? 0 : i]; result_data[i] = OPWRAPPER::template Operation( fun, lentry, rentry, mask, i); } } } template static void ExecuteConstant(Vector &left, Vector &right, Vector &result, FUNC fun) { result.SetVectorType(VectorType::CONSTANT_VECTOR); auto ldata = ConstantVector::GetData(left); auto rdata = ConstantVector::GetData(right); auto result_data = ConstantVector::GetData(result); if (ConstantVector::IsNull(left) || ConstantVector::IsNull(right)) { ConstantVector::SetNull(result, true); return; } *result_data = OPWRAPPER::template Operation( fun, *ldata, *rdata, ConstantVector::Validity(result), 0); } template static void ExecuteFlat(Vector &left, Vector &right, Vector &result, idx_t count, FUNC fun) { auto ldata = FlatVector::GetData(left); auto rdata = FlatVector::GetData(right); if ((LEFT_CONSTANT && ConstantVector::IsNull(left)) || (RIGHT_CONSTANT && ConstantVector::IsNull(right))) { // either left or right is constant NULL: result is constant NULL result.SetVectorType(VectorType::CONSTANT_VECTOR); ConstantVector::SetNull(result, true); return; } result.SetVectorType(VectorType::FLAT_VECTOR); auto result_data = FlatVector::GetData(result); auto &result_validity = FlatVector::Validity(result); if (LEFT_CONSTANT) { if (OPWRAPPER::AddsNulls()) { result_validity.Copy(FlatVector::Validity(right), count); } else { FlatVector::SetValidity(result, FlatVector::Validity(right)); } } else if (RIGHT_CONSTANT) { if (OPWRAPPER::AddsNulls()) { result_validity.Copy(FlatVector::Validity(left), count); } else { FlatVector::SetValidity(result, FlatVector::Validity(left)); } } else { if (OPWRAPPER::AddsNulls()) { result_validity.Copy(FlatVector::Validity(left), count); if (result_validity.AllValid()) { result_validity.Copy(FlatVector::Validity(right), count); } else { result_validity.Combine(FlatVector::Validity(right), count); } } else { FlatVector::SetValidity(result, FlatVector::Validity(left)); result_validity.Combine(FlatVector::Validity(right), count); } } ExecuteFlatLoop( ldata, rdata, result_data, count, result_validity, fun); } template static void ExecuteGenericLoop(const LEFT_TYPE *__restrict ldata, const RIGHT_TYPE *__restrict rdata, RESULT_TYPE *__restrict result_data, const SelectionVector *__restrict lsel, const SelectionVector *__restrict rsel, idx_t count, ValidityMask &lvalidity, ValidityMask &rvalidity, ValidityMask &result_validity, FUNC fun) { if (!lvalidity.AllValid() || !rvalidity.AllValid()) { for (idx_t i = 0; i < count; i++) { auto lindex = lsel->get_index(i); auto rindex = rsel->get_index(i); if (lvalidity.RowIsValid(lindex) && rvalidity.RowIsValid(rindex)) { auto lentry = ldata[lindex]; auto rentry = rdata[rindex]; result_data[i] = OPWRAPPER::template Operation( fun, lentry, rentry, result_validity, i); } else { result_validity.SetInvalid(i); } } } else { for (idx_t i = 0; i < count; i++) { auto lentry = ldata[lsel->get_index(i)]; auto rentry = rdata[rsel->get_index(i)]; result_data[i] = OPWRAPPER::template Operation( fun, lentry, rentry, result_validity, i); } } } template static void ExecuteGeneric(Vector &left, Vector &right, Vector &result, idx_t count, FUNC fun) { UnifiedVectorFormat ldata, rdata; left.ToUnifiedFormat(count, ldata); right.ToUnifiedFormat(count, rdata); result.SetVectorType(VectorType::FLAT_VECTOR); auto result_data = FlatVector::GetData(result); ExecuteGenericLoop( UnifiedVectorFormat::GetData(ldata), UnifiedVectorFormat::GetData(rdata), result_data, ldata.sel, rdata.sel, count, ldata.validity, rdata.validity, FlatVector::Validity(result), fun); } template static void ExecuteSwitch(Vector &left, Vector &right, Vector &result, idx_t count, FUNC fun) { auto left_vector_type = left.GetVectorType(); auto right_vector_type = right.GetVectorType(); if (left_vector_type == VectorType::CONSTANT_VECTOR && right_vector_type == VectorType::CONSTANT_VECTOR) { ExecuteConstant(left, right, result, fun); } else if (left_vector_type == VectorType::FLAT_VECTOR && right_vector_type == VectorType::CONSTANT_VECTOR) { ExecuteFlat(left, right, result, count, fun); } else if (left_vector_type == VectorType::CONSTANT_VECTOR && right_vector_type == VectorType::FLAT_VECTOR) { ExecuteFlat(left, right, result, count, fun); } else if (left_vector_type == VectorType::FLAT_VECTOR && right_vector_type == VectorType::FLAT_VECTOR) { ExecuteFlat(left, right, result, count, fun); } else { ExecuteGeneric(left, right, result, count, fun); } } public: template > static void Execute(Vector &left, Vector &right, Vector &result, idx_t count, FUNC fun) { ExecuteSwitch(left, right, result, count, fun); } template static void Execute(Vector &left, Vector &right, Vector &result, idx_t count) { ExecuteSwitch(left, right, result, count, false); } template static void ExecuteStandard(Vector &left, Vector &right, Vector &result, idx_t count) { ExecuteSwitch(left, right, result, count, false); } template > static void ExecuteWithNulls(Vector &left, Vector &right, Vector &result, idx_t count, FUNC fun) { ExecuteSwitch(left, right, result, count, fun); } public: template static idx_t SelectConstant(Vector &left, Vector &right, const SelectionVector *sel, idx_t count, SelectionVector *true_sel, SelectionVector *false_sel) { auto ldata = ConstantVector::GetData(left); auto rdata = ConstantVector::GetData(right); // both sides are constant, return either 0 or the count // in this case we do not fill in the result selection vector at all if (ConstantVector::IsNull(left) || ConstantVector::IsNull(right) || !OP::Operation(*ldata, *rdata)) { if (false_sel) { for (idx_t i = 0; i < count; i++) { false_sel->set_index(i, sel->get_index(i)); } } return 0; } else { if (true_sel) { for (idx_t i = 0; i < count; i++) { true_sel->set_index(i, sel->get_index(i)); } } return count; } } template static inline idx_t SelectFlatLoop(const LEFT_TYPE *__restrict ldata, const RIGHT_TYPE *__restrict rdata, const SelectionVector *sel, idx_t count, ValidityMask &validity_mask, SelectionVector *true_sel, SelectionVector *false_sel) { idx_t true_count = 0, false_count = 0; idx_t base_idx = 0; auto entry_count = ValidityMask::EntryCount(count); for (idx_t entry_idx = 0; entry_idx < entry_count; entry_idx++) { auto validity_entry = validity_mask.GetValidityEntry(entry_idx); idx_t next = MinValue(base_idx + ValidityMask::BITS_PER_VALUE, count); if (ValidityMask::AllValid(validity_entry)) { // all valid: perform operation for (; base_idx < next; base_idx++) { idx_t result_idx = sel->get_index(base_idx); idx_t lidx = LEFT_CONSTANT ? 0 : base_idx; idx_t ridx = RIGHT_CONSTANT ? 0 : base_idx; bool comparison_result = OP::Operation(ldata[lidx], rdata[ridx]); if (HAS_TRUE_SEL) { true_sel->set_index(true_count, result_idx); true_count += comparison_result; } if (HAS_FALSE_SEL) { false_sel->set_index(false_count, result_idx); false_count += !comparison_result; } } } else if (ValidityMask::NoneValid(validity_entry)) { // nothing valid: skip all if (HAS_FALSE_SEL) { for (; base_idx < next; base_idx++) { idx_t result_idx = sel->get_index(base_idx); false_sel->set_index(false_count, result_idx); false_count++; } } base_idx = next; continue; } else { // partially valid: need to check individual elements for validity idx_t start = base_idx; for (; base_idx < next; base_idx++) { idx_t result_idx = sel->get_index(base_idx); idx_t lidx = LEFT_CONSTANT ? 0 : base_idx; idx_t ridx = RIGHT_CONSTANT ? 0 : base_idx; bool comparison_result = ValidityMask::RowIsValid(validity_entry, base_idx - start) && OP::Operation(ldata[lidx], rdata[ridx]); if (HAS_TRUE_SEL) { true_sel->set_index(true_count, result_idx); true_count += comparison_result; } if (HAS_FALSE_SEL) { false_sel->set_index(false_count, result_idx); false_count += !comparison_result; } } } } if (HAS_TRUE_SEL) { return true_count; } else { return count - false_count; } } template static inline idx_t SelectFlatLoopSwitch(const LEFT_TYPE *__restrict ldata, const RIGHT_TYPE *__restrict rdata, const SelectionVector *sel, idx_t count, ValidityMask &mask, SelectionVector *true_sel, SelectionVector *false_sel) { if (true_sel && false_sel) { return SelectFlatLoop( ldata, rdata, sel, count, mask, true_sel, false_sel); } else if (true_sel) { return SelectFlatLoop( ldata, rdata, sel, count, mask, true_sel, false_sel); } else { D_ASSERT(false_sel); return SelectFlatLoop( ldata, rdata, sel, count, mask, true_sel, false_sel); } } template static idx_t SelectFlat(Vector &left, Vector &right, const SelectionVector *sel, idx_t count, SelectionVector *true_sel, SelectionVector *false_sel) { auto ldata = FlatVector::GetData(left); auto rdata = FlatVector::GetData(right); if (LEFT_CONSTANT && ConstantVector::IsNull(left)) { if (false_sel) { for (idx_t i = 0; i < count; i++) { false_sel->set_index(i, sel->get_index(i)); } } return 0; } if (RIGHT_CONSTANT && ConstantVector::IsNull(right)) { if (false_sel) { for (idx_t i = 0; i < count; i++) { false_sel->set_index(i, sel->get_index(i)); } } return 0; } if (LEFT_CONSTANT) { return SelectFlatLoopSwitch( ldata, rdata, sel, count, FlatVector::Validity(right), true_sel, false_sel); } else if (RIGHT_CONSTANT) { return SelectFlatLoopSwitch( ldata, rdata, sel, count, FlatVector::Validity(left), true_sel, false_sel); } else { ValidityMask combined_mask = FlatVector::Validity(left); combined_mask.Combine(FlatVector::Validity(right), count); return SelectFlatLoopSwitch( ldata, rdata, sel, count, combined_mask, true_sel, false_sel); } } template static inline idx_t SelectGenericLoop(const LEFT_TYPE *__restrict ldata, const RIGHT_TYPE *__restrict rdata, const SelectionVector *__restrict lsel, const SelectionVector *__restrict rsel, const SelectionVector *__restrict result_sel, idx_t count, ValidityMask &lvalidity, ValidityMask &rvalidity, SelectionVector *true_sel, SelectionVector *false_sel) { idx_t true_count = 0, false_count = 0; for (idx_t i = 0; i < count; i++) { auto result_idx = result_sel->get_index(i); auto lindex = lsel->get_index(i); auto rindex = rsel->get_index(i); if ((NO_NULL || (lvalidity.RowIsValid(lindex) && rvalidity.RowIsValid(rindex))) && OP::Operation(ldata[lindex], rdata[rindex])) { if (HAS_TRUE_SEL) { true_sel->set_index(true_count++, result_idx); } } else { if (HAS_FALSE_SEL) { false_sel->set_index(false_count++, result_idx); } } } if (HAS_TRUE_SEL) { return true_count; } else { return count - false_count; } } template static inline idx_t SelectGenericLoopSelSwitch(const LEFT_TYPE *__restrict ldata, const RIGHT_TYPE *__restrict rdata, const SelectionVector *__restrict lsel, const SelectionVector *__restrict rsel, const SelectionVector *__restrict result_sel, idx_t count, ValidityMask &lvalidity, ValidityMask &rvalidity, SelectionVector *true_sel, SelectionVector *false_sel) { if (true_sel && false_sel) { return SelectGenericLoop( ldata, rdata, lsel, rsel, result_sel, count, lvalidity, rvalidity, true_sel, false_sel); } else if (true_sel) { return SelectGenericLoop( ldata, rdata, lsel, rsel, result_sel, count, lvalidity, rvalidity, true_sel, false_sel); } else { D_ASSERT(false_sel); return SelectGenericLoop( ldata, rdata, lsel, rsel, result_sel, count, lvalidity, rvalidity, true_sel, false_sel); } } template static inline idx_t SelectGenericLoopSwitch(const LEFT_TYPE *__restrict ldata, const RIGHT_TYPE *__restrict rdata, const SelectionVector *__restrict lsel, const SelectionVector *__restrict rsel, const SelectionVector *__restrict result_sel, idx_t count, ValidityMask &lvalidity, ValidityMask &rvalidity, SelectionVector *true_sel, SelectionVector *false_sel) { if (!lvalidity.AllValid() || !rvalidity.AllValid()) { return SelectGenericLoopSelSwitch( ldata, rdata, lsel, rsel, result_sel, count, lvalidity, rvalidity, true_sel, false_sel); } else { return SelectGenericLoopSelSwitch( ldata, rdata, lsel, rsel, result_sel, count, lvalidity, rvalidity, true_sel, false_sel); } } template static idx_t SelectGeneric(Vector &left, Vector &right, const SelectionVector *sel, idx_t count, SelectionVector *true_sel, SelectionVector *false_sel) { UnifiedVectorFormat ldata, rdata; left.ToUnifiedFormat(count, ldata); right.ToUnifiedFormat(count, rdata); return SelectGenericLoopSwitch( UnifiedVectorFormat::GetData(ldata), UnifiedVectorFormat::GetData(rdata), ldata.sel, rdata.sel, sel, count, ldata.validity, rdata.validity, true_sel, false_sel); } template static idx_t Select(Vector &left, Vector &right, const SelectionVector *sel, idx_t count, SelectionVector *true_sel, SelectionVector *false_sel) { if (!sel) { sel = FlatVector::IncrementalSelectionVector(); } if (left.GetVectorType() == VectorType::CONSTANT_VECTOR && right.GetVectorType() == VectorType::CONSTANT_VECTOR) { return SelectConstant(left, right, sel, count, true_sel, false_sel); } else if (left.GetVectorType() == VectorType::CONSTANT_VECTOR && right.GetVectorType() == VectorType::FLAT_VECTOR) { return SelectFlat(left, right, sel, count, true_sel, false_sel); } else if (left.GetVectorType() == VectorType::FLAT_VECTOR && right.GetVectorType() == VectorType::CONSTANT_VECTOR) { return SelectFlat(left, right, sel, count, true_sel, false_sel); } else if (left.GetVectorType() == VectorType::FLAT_VECTOR && right.GetVectorType() == VectorType::FLAT_VECTOR) { return SelectFlat(left, right, sel, count, true_sel, false_sel); } else { return SelectGeneric(left, right, sel, count, true_sel, false_sel); } } }; } // namespace duckdb