Skip to content

[SPARK-56179][PYTHON] Consolidate error classes for type mismatch - part 1 #54978

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
zhengruifeng wants to merge 2 commits into from
Closed

[SPARK-56179][PYTHON] Consolidate error classes for type mismatch - part 1 #54978

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0f13ce5
[SPARK-XXXXX][PYTHON] Consolidate 11 NOT_* type-check error classes i…
zhengruifeng Mar 24, 2026
123a2dd
reformat
zhengruifeng Mar 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 6 additions & 66 deletions python/pyspark/errors/error-conditions.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -602,72 +602,6 @@
"Value for `<arg_name>` must be greater than or equal to 0, got '<arg_value>'."
]
},
"NOT_BOOL": {
"message": [
"Argument `<arg_name>` should be a bool, got <arg_type>."
],
"sqlState": "42K09"
},
"NOT_BOOL_OR_DICT_OR_FLOAT_OR_INT_OR_LIST_OR_STR_OR_TUPLE": {
"message": [
"Argument `<arg_name>` should be a bool, dict, float, int, str or tuple, got <arg_type>."
],
"sqlState": "42K09"
},
"NOT_BOOL_OR_DICT_OR_FLOAT_OR_INT_OR_STR": {
"message": [
"Argument `<arg_name>` should be a bool, dict, float, int or str, got <arg_type>."
],
"sqlState": "42K09"
},
"NOT_BOOL_OR_FLOAT_OR_INT": {
"message": [
"Argument `<arg_name>` should be a bool, float or int, got <arg_type>."
],
"sqlState": "42K09"
},
"NOT_BOOL_OR_FLOAT_OR_INT_OR_LIST_OR_NONE_OR_STR_OR_TUPLE": {
"message": [
"Argument `<arg_name>` should be a bool, float, int, list, None, str or tuple, got <arg_type>."
],
"sqlState": "42K09"
},
"NOT_BOOL_OR_FLOAT_OR_INT_OR_STR": {
"message": [
"Argument `<arg_name>` should be a bool, float, int or str, got <arg_type>."
],
"sqlState": "42K09"
},
"NOT_BOOL_OR_STR": {
"message": [
"Argument `<arg_name>` should be a bool or str, got <arg_type>."
],
"sqlState": "42K09"
},
"NOT_CALLABLE": {
"message": [
"Argument `<arg_name>` should be a callable, got <arg_type>."
],
"sqlState": "42K09"
},
"NOT_COLUMN": {
"message": [
"Argument `<arg_name>` should be a Column, got <arg_type>."
],
"sqlState": "42K09"
},
"NOT_COLUMN_OR_DATATYPE_OR_STR": {
"message": [
"Argument `<arg_name>` should be a Column, str or DataType, but got <arg_type>."
],
"sqlState": "42K09"
},
"NOT_COLUMN_OR_FLOAT_OR_INT_OR_LIST_OR_STR": {
"message": [
"Argument `<arg_name>` should be a Column, float, integer, list or string, got <arg_type>."
],
"sqlState": "42K09"
},
"NOT_COLUMN_OR_INT": {
"message": [
"Argument `<arg_name>` should be a Column or int, got <arg_type>."
Expand Down Expand Up @@ -716,6 +650,12 @@
],
"sqlState": "42K09"
},
"NOT_EXPECTED_TYPE": {
"message": [
"Argument `<arg_name>` should be <expected_type>, got <arg_type>."
],
"sqlState": "42K09"
},
"NOT_EXPRESSION": {
"message": [
"Argument `<arg_name>` should be an Expression, got <arg_type>."
Expand Down
32 changes: 24 additions & 8 deletions python/pyspark/pandas/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -955,27 +955,43 @@ def spark_column_equals(left: Column, right: Column) -> bool:

if not isinstance(left, ConnectColumn):
raise PySparkTypeError(
errorClass="NOT_COLUMN",
messageParameters={"arg_name": "left", "arg_type": type(left).__name__},
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "Column",
"arg_name": "left",
"arg_type": type(left).__name__,
},
)
if not isinstance(right, ConnectColumn):
raise PySparkTypeError(
errorClass="NOT_COLUMN",
messageParameters={"arg_name": "right", "arg_type": type(right).__name__},
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "Column",
"arg_name": "right",
"arg_type": type(right).__name__,
},
)
return repr(left).replace("`", "") == repr(right).replace("`", "")
else:
from pyspark.sql.classic.column import Column as ClassicColumn

if not isinstance(left, ClassicColumn):
raise PySparkTypeError(
errorClass="NOT_COLUMN",
messageParameters={"arg_name": "left", "arg_type": type(left).__name__},
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "Column",
"arg_name": "left",
"arg_type": type(left).__name__,
},
)
if not isinstance(right, ClassicColumn):
raise PySparkTypeError(
errorClass="NOT_COLUMN",
messageParameters={"arg_name": "right", "arg_type": type(right).__name__},
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "Column",
"arg_name": "right",
"arg_type": type(right).__name__,
},
)
return left._jc.equals(right._jc)

Expand Down
16 changes: 12 additions & 4 deletions python/pyspark/sql/classic/column.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,8 +381,12 @@ def withField(self, fieldName: str, col: ParentColumn) -> ParentColumn:

if not isinstance(col, Column):
raise PySparkTypeError(
errorClass="NOT_COLUMN",
messageParameters={"arg_name": "col", "arg_type": type(col).__name__},
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "Column",
"arg_name": "col",
"arg_type": type(col).__name__,
},
)

return Column(self._jc.withField(fieldName, col._jc))
Expand Down Expand Up @@ -587,8 +591,12 @@ def between(
def when(self, condition: ParentColumn, value: Any) -> ParentColumn:
if not isinstance(condition, Column):
raise PySparkTypeError(
errorClass="NOT_COLUMN",
messageParameters={"arg_name": "condition", "arg_type": type(condition).__name__},
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "Column",
"arg_name": "condition",
"arg_type": type(condition).__name__,
},
)
v = value._jc if isinstance(value, Column) else enum_to_value(value)
jc = self._jc.when(condition._jc, v)
Expand Down
44 changes: 32 additions & 12 deletions python/pyspark/sql/classic/dataframe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,8 +252,9 @@ def explain(
if not (is_no_argument or is_extended_case or is_extended_as_mode or is_mode_case):
if (extended is not None) and (not isinstance(extended, (bool, str))):
raise PySparkTypeError(
errorClass="NOT_BOOL_OR_STR",
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "bool or str",
"arg_name": "extended",
"arg_type": type(extended).__name__,
},
Expand Down Expand Up @@ -306,8 +307,12 @@ def _show_string(

if not isinstance(vertical, bool):
raise PySparkTypeError(
errorClass="NOT_BOOL",
messageParameters={"arg_name": "vertical", "arg_type": type(vertical).__name__},
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "bool",
"arg_name": "vertical",
"arg_type": type(vertical).__name__,
},
)

if isinstance(truncate, bool) and truncate:
Expand All @@ -317,8 +322,9 @@ def _show_string(
int_truncate = int(truncate)
except ValueError:
raise PySparkTypeError(
errorClass="NOT_BOOL",
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "bool",
"arg_name": "truncate",
"arg_type": type(truncate).__name__,
},
Expand Down Expand Up @@ -1013,8 +1019,12 @@ def __getitem__(
return Column(jc)
else:
raise PySparkTypeError(
errorClass="NOT_COLUMN_OR_FLOAT_OR_INT_OR_LIST_OR_STR",
messageParameters={"arg_name": "item", "arg_type": type(item).__name__},
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "Column, float, integer, list or string",
"arg_name": "item",
"arg_type": type(item).__name__,
},
)

def __getattr__(self, name: str) -> Column:
Expand Down Expand Up @@ -1285,8 +1295,12 @@ def fillna(
) -> ParentDataFrame:
if not isinstance(value, (float, int, str, bool, dict)):
raise PySparkTypeError(
errorClass="NOT_BOOL_OR_DICT_OR_FLOAT_OR_INT_OR_STR",
messageParameters={"arg_name": "value", "arg_type": type(value).__name__},
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "bool, dict, float, int or str",
"arg_name": "value",
"arg_type": type(value).__name__,
},
)

# Note that bool validates isinstance(int), but we don't want to
Expand Down Expand Up @@ -1384,8 +1398,9 @@ def all_of_(xs: Iterable) -> bool:
valid_types = (bool, float, int, str, list, tuple)
if not isinstance(to_replace, valid_types + (dict,)):
raise PySparkTypeError(
errorClass="NOT_BOOL_OR_DICT_OR_FLOAT_OR_INT_OR_LIST_OR_STR_OR_TUPLE",
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "bool, dict, float, int, str or tuple",
"arg_name": "to_replace",
"arg_type": type(to_replace).__name__,
},
Expand All @@ -1397,8 +1412,9 @@ def all_of_(xs: Iterable) -> bool:
and not isinstance(to_replace, dict)
):
raise PySparkTypeError(
errorClass="NOT_BOOL_OR_FLOAT_OR_INT_OR_LIST_OR_NONE_OR_STR_OR_TUPLE",
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "bool, float, int, list, None, str or tuple",
"arg_name": "value",
"arg_type": type(value).__name__,
},
Expand Down Expand Up @@ -1648,8 +1664,12 @@ def withColumns(self, *colsMap: Dict[str, Column]) -> ParentDataFrame:
def withColumn(self, colName: str, col: Column) -> ParentDataFrame:
if not isinstance(col, Column):
raise PySparkTypeError(
errorClass="NOT_COLUMN",
messageParameters={"arg_name": "col", "arg_type": type(col).__name__},
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "Column",
"arg_name": "col",
"arg_type": type(col).__name__,
},
)
return DataFrame(self._jdf.withColumn(colName, col._jc), self.sparkSession)

Expand Down
16 changes: 12 additions & 4 deletions python/pyspark/sql/connect/column.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -301,8 +301,12 @@ def endswith(
def when(self, condition: ParentColumn, value: Any) -> ParentColumn:
if not isinstance(condition, Column):
raise PySparkTypeError(
errorClass="NOT_COLUMN",
messageParameters={"arg_name": "condition", "arg_type": type(condition).__name__},
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "Column",
"arg_name": "condition",
"arg_type": type(condition).__name__,
},
)

if not isinstance(self._expr, CaseWhen):
Expand Down Expand Up @@ -527,8 +531,12 @@ def withField(self, fieldName: str, col: ParentColumn) -> ParentColumn:

if not isinstance(col, Column):
raise PySparkTypeError(
errorClass="NOT_COLUMN",
messageParameters={"arg_name": "col", "arg_type": type(col).__name__},
errorClass="NOT_EXPECTED_TYPE",
messageParameters={
"expected_type": "Column",
"arg_name": "col",
"arg_type": type(col).__name__,
},
)

return Column(WithField(self._expr, fieldName, col._expr))
Expand Down
Loading