Skip to content

Commit 3cf1b67

Browse files
hmohrdaurathmohrdaurat
authored
Merge pull request #1 from hmohrdaurat/updated_velox
make velox-docker-image independent from dps-docker-image, update new version of velox, remove fmt and spdlog (use internal versions instead) as they make MultiKernelBOSS build to fail
2 parents 73430d4 + 3abfd37 commit 3cf1b67

2 files changed

Lines changed: 306 additions & 16 deletions

File tree

‎velox-docker-image/Dockerfile‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,15 @@
1-
FROM ghcr.io/little-big-h/dps-docker-image:nightly
1+
FROM ubuntu:22.04
2+
ENV DEBIAN_FRONTEND=noninteractive
3+
RUN apt-get update -y
4+
RUN apt-get install curl wget git build-essential sudo cmake -y
5+
RUN apt-get install lldb clang -y
6+
RUN cd /usr/local && git clone --recursive https://github.com/facebookincubator/velox.git \
7+
&& cd velox && git checkout 3020196b001130a9027b09d6b3b95385b90bc2cb && git submodule sync --recursive && git submodule update --init --recursive
8+
RUN cd /usr/local/velox && ./scripts/setup-ubuntu.sh
9+
RUN cd /usr/local/velox && make VELOX_BUILD_TESTING=OFF
10+
RUN apt-get install openssh-server -y
11+
RUN apt install nano emacs-nox -y
212
ADD ./velox.patch /usr/local/velox/
313
WORKDIR /usr/local/velox
414
RUN git apply velox.patch
5-
RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -yy uuid-dev libfmt-dev libspdlog-dev libopenblas-dev libfftw3-dev liblapacke-dev
15+
RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -yy uuid-dev libopenblas-dev libfftw3-dev liblapacke-dev

‎velox-docker-image/velox.patch‎

Lines changed: 294 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,18 @@
1+
diff --git a/scripts/setup-helper-functions.sh b/scripts/setup-helper-functions.sh
2+
index 4f0a11e15..52c9180e9 100644
3+
--- a/scripts/setup-helper-functions.sh
4+
+++ b/scripts/setup-helper-functions.sh
5+
@@ -163,6 +163,8 @@ function cmake_install {
6+
cmake -Wno-dev -B"${BINARY_DIR}" \
7+
-GNinja \
8+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
9+
+ -DCMAKE_CXX_COMPILER=clang++ \
10+
+ -DCMAKE_C_COMPILER=clang \
11+
-DCMAKE_CXX_STANDARD=17 \
12+
"${INSTALL_PREFIX+-DCMAKE_PREFIX_PATH=}${INSTALL_PREFIX-}" \
13+
"${INSTALL_PREFIX+-DCMAKE_INSTALL_PREFIX=}${INSTALL_PREFIX-}" \
114
diff --git a/velox/dwio/dwrf/proto/CMakeLists.txt b/velox/dwio/dwrf/proto/CMakeLists.txt
2-
index 02ff2c8d..0ae65562 100644
15+
index 02ff2c8d7..0ae655626 100644
316
--- a/velox/dwio/dwrf/proto/CMakeLists.txt
417
+++ b/velox/dwio/dwrf/proto/CMakeLists.txt
518
@@ -24,8 +24,6 @@ foreach(PROTO ${PROTO_FILES})
@@ -22,11 +35,215 @@ index 02ff2c8d..0ae65562 100644
2235
DEPENDS ${Protobuf_PROTOC_EXECUTABLE}
2336
COMMENT "Running PROTO compiler"
2437
VERBATIM)
38+
diff --git a/velox/exec/HashBuild.cpp b/velox/exec/HashBuild.cpp
39+
index bfeb1cd6f..360a54b08 100644
40+
--- a/velox/exec/HashBuild.cpp
41+
+++ b/velox/exec/HashBuild.cpp
42+
@@ -158,7 +158,8 @@ void HashBuild::setupTable() {
43+
operatorCtx_->driverCtx()
44+
->queryConfig()
45+
.minTableRowsForParallelJoinBuild(),
46+
- pool());
47+
+ pool(),
48+
+ operatorCtx_->driverCtx()->queryConfig().hashAdaptivityEnabled());
49+
} else {
50+
// (Left) semi and anti join with no extra filter only needs to know whether
51+
// there is a match. Hence, no need to store entries with duplicate keys.
52+
@@ -178,7 +179,8 @@ void HashBuild::setupTable() {
53+
operatorCtx_->driverCtx()
54+
->queryConfig()
55+
.minTableRowsForParallelJoinBuild(),
56+
- pool());
57+
+ pool(),
58+
+ operatorCtx_->driverCtx()->queryConfig().hashAdaptivityEnabled());
59+
} else {
60+
// Ignore null keys
61+
table_ = HashTable<true>::createForJoin(
62+
@@ -189,7 +191,8 @@ void HashBuild::setupTable() {
63+
operatorCtx_->driverCtx()
64+
->queryConfig()
65+
.minTableRowsForParallelJoinBuild(),
66+
- pool());
67+
+ pool(),
68+
+ operatorCtx_->driverCtx()->queryConfig().hashAdaptivityEnabled());
69+
}
70+
}
71+
analyzeKeys_ = table_->hashMode() != BaseHashTable::HashMode::kHash;
72+
diff --git a/velox/exec/HashProbe.cpp b/velox/exec/HashProbe.cpp
73+
index 228358c35..37c7c2510 100644
74+
--- a/velox/exec/HashProbe.cpp
75+
+++ b/velox/exec/HashProbe.cpp
76+
@@ -23,9 +23,6 @@ namespace facebook::velox::exec {
77+
78+
namespace {
79+
80+
-// Batch size used when iterating the row container.
81+
-constexpr int kBatchSize = 1024;
82+
-
83+
// Returns the type for the hash table row. Build side keys first,
84+
// then dependent build side columns.
85+
RowTypePtr makeTableType(
86+
@@ -996,10 +993,10 @@ void HashProbe::fillFilterInput(vector_size_t size) {
87+
void HashProbe::prepareFilterRowsForNullAwareJoin(
88+
vector_size_t numRows,
89+
bool filterPropagateNulls) {
90+
- VELOX_CHECK_LE(numRows, kBatchSize);
91+
+ VELOX_CHECK_LE(numRows, outputBatchSize_);
92+
if (filterTableInput_ == nullptr) {
93+
- filterTableInput_ =
94+
- BaseVector::create<RowVector>(filterInputType_, kBatchSize, pool());
95+
+ filterTableInput_ = BaseVector::create<RowVector>(
96+
+ filterInputType_, outputBatchSize_, pool());
97+
}
98+
99+
if (filterPropagateNulls) {
100+
@@ -1067,8 +1064,8 @@ void HashProbe::applyFilterOnTableRowsForNullAwareJoin(
101+
}
102+
auto* tableRows = table_->rows();
103+
VELOX_CHECK(tableRows, "Should not move rows in hash joins");
104+
- char* data[kBatchSize];
105+
- while (auto numRows = iterator(data, kBatchSize)) {
106+
+ char** data = new char*[outputBatchSize_];
107+
+ while (auto numRows = iterator(data, outputBatchSize_)) {
108+
filterTableInput_->resize(numRows);
109+
filterTableInputRows_.resizeFill(numRows, true);
110+
for (auto& projection : filterTableProjections_) {
111+
@@ -1112,6 +1109,7 @@ void HashProbe::applyFilterOnTableRowsForNullAwareJoin(
112+
}
113+
});
114+
}
115+
+ delete[] data;
116+
}
117+
118+
SelectivityVector HashProbe::evalFilterForNullAwareJoin(
119+
diff --git a/velox/exec/HashTable.cpp b/velox/exec/HashTable.cpp
120+
index 44cd1e82e..e45a95a90 100644
121+
--- a/velox/exec/HashTable.cpp
122+
+++ b/velox/exec/HashTable.cpp
123+
@@ -53,16 +53,23 @@ HashTable<ignoreNullKeys>::HashTable(
124+
bool hasProbedFlag,
125+
uint32_t minTableSizeForParallelJoinBuild,
126+
memory::MemoryPool* pool,
127+
- const std::shared_ptr<velox::HashStringAllocator>& stringArena)
128+
+ const std::shared_ptr<velox::HashStringAllocator>& stringArena, bool hashAdaptivityEnabled)
129+
: BaseHashTable(std::move(hashers)),
130+
minTableSizeForParallelJoinBuild_(minTableSizeForParallelJoinBuild),
131+
isJoinBuild_(isJoinBuild) {
132+
std::vector<TypePtr> keys;
133+
- for (auto& hasher : hashers_) {
134+
- keys.push_back(hasher->type());
135+
- if (!VectorHasher::typeKindSupportsValueIds(hasher->typeKind())) {
136+
- hashMode_ = HashMode::kHash;
137+
+ if (hashAdaptivityEnabled) {
138+
+ for (auto& hasher : hashers_) {
139+
+ keys.push_back(hasher->type());
140+
+ if (!VectorHasher::typeKindSupportsValueIds(hasher->typeKind())) {
141+
+ hashMode_ = HashMode::kHash;
142+
+ }
143+
}
144+
+ } else {
145+
+ for (auto& hasher : hashers_) {
146+
+ keys.push_back(hasher->type());
147+
+ }
148+
+ hashMode_ = HashMode::kHash;
149+
}
150+
151+
rows_ = std::make_unique<RowContainer>(
152+
diff --git a/velox/exec/HashTable.h b/velox/exec/HashTable.h
153+
index eec394caf..f5fb64b9c 100644
154+
--- a/velox/exec/HashTable.h
155+
+++ b/velox/exec/HashTable.h
156+
@@ -427,7 +427,8 @@ class HashTable : public BaseHashTable {
157+
bool hasProbedFlag,
158+
uint32_t minTableSizeForParallelJoinBuild,
159+
memory::MemoryPool* pool,
160+
- const std::shared_ptr<velox::HashStringAllocator>& stringArena = nullptr);
161+
+ const std::shared_ptr<velox::HashStringAllocator>& stringArena = nullptr,
162+
+ bool hashAdaptivityEnabled = true);
163+
164+
static std::unique_ptr<HashTable> createForAggregation(
165+
std::vector<std::unique_ptr<VectorHasher>>&& hashers,
166+
@@ -453,7 +454,8 @@ class HashTable : public BaseHashTable {
167+
bool allowDuplicates,
168+
bool hasProbedFlag,
169+
uint32_t minTableSizeForParallelJoinBuild,
170+
- memory::MemoryPool* pool) {
171+
+ memory::MemoryPool* pool,
172+
+ bool hashAdaptivityEnabled = true) {
173+
return std::make_unique<HashTable>(
174+
std::move(hashers),
175+
std::vector<Accumulator>{},
176+
@@ -462,7 +464,9 @@ class HashTable : public BaseHashTable {
177+
true, // isJoinBuild
178+
hasProbedFlag,
179+
minTableSizeForParallelJoinBuild,
180+
- pool);
181+
+ pool,
182+
+ nullptr,
183+
+ hashAdaptivityEnabled);
184+
}
185+
186+
void groupProbe(HashLookup& lookup) override;
187+
diff --git a/velox/exec/Task.h b/velox/exec/Task.h
188+
index cb4a8507f..f4012b6be 100644
189+
--- a/velox/exec/Task.h
190+
+++ b/velox/exec/Task.h
191+
@@ -618,6 +618,10 @@ class Task : public std::enable_shared_from_this<Task> {
192+
terminate(TaskState::kFinished).wait();
193+
}
194+
195+
+ auto const& childPools() const {
196+
+ return childPools_;
197+
+ }
198+
+
199+
private:
200+
Task(
201+
const std::string& taskId,
202+
diff --git a/velox/expression/Expr.cpp b/velox/expression/Expr.cpp
203+
index bd35ea322..8b90f1784 100644
204+
--- a/velox/expression/Expr.cpp
205+
+++ b/velox/expression/Expr.cpp
206+
@@ -1109,7 +1109,7 @@ bool Expr::removeSureNulls(
207+
continue;
208+
}
209+
210+
- if (values->mayHaveNulls()) {
211+
+ /*if (values->mayHaveNulls()) {
212+
LocalDecodedVector decoded(context, *values, rows);
213+
if (auto* rawNulls = decoded->nulls()) {
214+
if (!result) {
215+
@@ -1118,7 +1118,7 @@ bool Expr::removeSureNulls(
216+
auto bits = result->asMutableRange().bits();
217+
bits::andBits(bits, rawNulls, rows.begin(), rows.end());
218+
}
219+
- }
220+
+ }*/
221+
}
222+
if (result) {
223+
result->updateBounds();
224+
@@ -1144,7 +1144,7 @@ void Expr::evalWithNulls(
225+
return;
226+
}
227+
228+
- if (propagatesNulls_ && !skipFieldDependentOptimizations()) {
229+
+ if (false/*propagatesNulls_ && !skipFieldDependentOptimizations()*/) {
230+
bool mayHaveNulls = false;
231+
for (auto* field : distinctFields_) {
232+
const auto& vector = context.getField(field->index(context));
233+
@@ -1199,7 +1199,7 @@ void Expr::evalWithMemo(
234+
}
235+
++baseOfDictionaryRepeats_;
236+
237+
- if (baseOfDictionaryRepeats_ == 1) {
238+
+ if (true/*baseOfDictionaryRepeats_ == 1*/) {
239+
evalWithNulls(rows, context, result);
240+
baseOfDictionary_ = base;
241+
dictionaryCache_ = result;
25242
diff --git a/velox/functions/prestosql/Arithmetic.h b/velox/functions/prestosql/Arithmetic.h
26-
index 36d449ce..1948374d 100644
243+
index 05d427bf1..4539d65b6 100644
27244
--- a/velox/functions/prestosql/Arithmetic.h
28245
+++ b/velox/functions/prestosql/Arithmetic.h
29-
@@ -67,6 +67,17 @@ struct MultiplyFunction {
246+
@@ -107,6 +107,17 @@ struct IntervalMultiplyFunction {
30247
}
31248
};
32249

@@ -44,8 +261,42 @@ index 36d449ce..1948374d 100644
44261
template <typename T>
45262
struct DivideFunction {
46263
template <typename TInput>
264+
@@ -123,6 +134,33 @@ struct DivideFunction {
265+
}
266+
};
267+
268+
+template <typename T>
269+
+struct MixedDivideFunction {
270+
+ FOLLY_ALWAYS_INLINE void
271+
+ call(double& result, const double& a, const int64_t& b)
272+
+// depend on compiler have correct behaviour for divide by zero
273+
+#if defined(__has_feature)
274+
+#if __has_feature(__address_sanitizer__)
275+
+ __attribute__((__no_sanitize__("float-divide-by-zero")))
276+
+#endif
277+
+#endif
278+
+ {
279+
+ result = a / b;
280+
+ }
281+
+
282+
+ FOLLY_ALWAYS_INLINE void
283+
+ call(double& result, const int64_t& a, const double& b)
284+
+// depend on compiler have correct behaviour for divide by zero
285+
+#if defined(__has_feature)
286+
+#if __has_feature(__address_sanitizer__)
287+
+ __attribute__((__no_sanitize__("float-divide-by-zero")))
288+
+#endif
289+
+#endif
290+
+ {
291+
+ result = a / b;
292+
+ }
293+
+};
294+
+
295+
template <typename T>
296+
struct IntervalDivideFunction {
297+
FOLLY_ALWAYS_INLINE void call(int64_t& result, int64_t a, double b)
47298
diff --git a/velox/functions/prestosql/ArithmeticImpl.h b/velox/functions/prestosql/ArithmeticImpl.h
48-
index 9b4d1ae1..f87df490 100644
299+
index 11fe000d7..44bbe9fb3 100644
49300
--- a/velox/functions/prestosql/ArithmeticImpl.h
50301
+++ b/velox/functions/prestosql/ArithmeticImpl.h
51302
@@ -86,6 +86,17 @@ T multiply(const T a, const T b)
@@ -66,18 +317,47 @@ index 9b4d1ae1..f87df490 100644
66317
// This is used by Velox for floating points divide.
67318
template <typename T>
68319
T divide(const T& a, const T& b)
320+
@@ -99,6 +110,18 @@ T divide(const T& a, const T& b)
321+
return result;
322+
}
323+
324+
+template <typename T, typename U>
325+
+T divide(const T& a, const U& b)
326+
+#if defined(__has_feature)
327+
+#if __has_feature(__address_sanitizer__)
328+
+ __attribute__((__no_sanitize__("float-divide-by-zero")))
329+
+#endif
330+
+#endif
331+
+{
332+
+ T result = a / b;
333+
+ return result;
334+
+}
335+
+
336+
// This is used by Velox for floating points modulus.
337+
template <typename T>
338+
T modulus(const T a, const T b) {
69339
diff --git a/velox/functions/prestosql/registration/ArithmeticFunctionsRegistration.cpp b/velox/functions/prestosql/registration/ArithmeticFunctionsRegistration.cpp
70-
index 652367e5..6af1897a 100644
340+
index b04695d11..d629967f5 100644
71341
--- a/velox/functions/prestosql/registration/ArithmeticFunctionsRegistration.cpp
72342
+++ b/velox/functions/prestosql/registration/ArithmeticFunctionsRegistration.cpp
73-
@@ -26,6 +26,10 @@ void registerSimpleFunctions() {
74-
registerBinaryFloatingPoint<PlusFunction>({"plus"});
75-
registerBinaryFloatingPoint<MinusFunction>({"minus"});
76-
registerBinaryFloatingPoint<MultiplyFunction>({"multiply"});
343+
@@ -51,12 +51,20 @@ void registerSimpleFunctions(const std::string& prefix) {
344+
IntervalDayTime,
345+
double,
346+
IntervalDayTime>({prefix + "multiply"});
77347
+ registerFunction<MixedMultiplyFunction, double, double, int64_t>(
78-
+ {"multiply"});
348+
+ {prefix + "multiply"});
79349
+ registerFunction<MixedMultiplyFunction, double, int64_t, double>(
80-
+ {"multiply"});
81-
registerBinaryFloatingPoint<DivideFunction>({"divide"});
82-
registerBinaryFloatingPoint<ModulusFunction>({"mod"});
83-
registerUnaryNumeric<CeilFunction>({"ceil", "ceiling"});
350+
+ {prefix + "multiply"});
351+
registerBinaryFloatingPoint<DivideFunction>({prefix + "divide"});
352+
registerFunction<
353+
IntervalDivideFunction,
354+
IntervalDayTime,
355+
IntervalDayTime,
356+
double>({prefix + "divide"});
357+
+ registerFunction<MixedDivideFunction, double, double, int64_t>(
358+
+ {prefix + "divide"});
359+
+ registerFunction<MixedDivideFunction, double, int64_t, double>(
360+
+ {prefix + "divide"});
361+
registerBinaryFloatingPoint<ModulusFunction>({prefix + "mod"});
362+
registerUnaryNumeric<CeilFunction>({prefix + "ceil", prefix + "ceiling"});
363+
registerUnaryNumeric<FloorFunction>({prefix + "floor"});

0 commit comments

Comments
 (0)