Enable macOS M1 wheel builds

nicholasjng · nicholasjng · commit 35806b409841 · 2023-12-12T12:50:39.000+01:00
This commit contains fixes for `ml-metadata`'s build setup to enable wheel builds on macOS M1.

The core build files have not been touched except for version increments of Bazel dependencies, and for `mariadbconnector-c`, whose build file needed to be updated to support
the latest version.

Several dependencies were upgraded to newer versions, with the update policy being "as recent as possible without breaking changes". A few gRPC dependencies were eliminated from
`ml-metadata`'s workspace to avoid friction on subsequent gRPC updates (this includes e.g. `protobuf`, `zlib`).

This change allows for M1 wheel builds without more external system dependencies outside of Bazel - the used build definitions contain M1 capabilities for sufficiently new versions of Bazel.
diff --git a/.bazelrc b/.bazelrc
@@ -23,5 +23,7 @@ build --protocopt=--experimental_allow_proto3_optional
 # parameter 'user_link_flags' is deprecated and will be removed soon.
 # It may be temporarily re-enabled by setting --incompatible_require_linker_input_cc_api=false
 build --incompatible_require_linker_input_cc_api=false
+
+# Support for Apple M1.
 build:macos --apple_platform_type=macos
 build:macos_arm64 --cpu=darwin_arm64
diff --git a/WORKSPACE b/WORKSPACE
@@ -88,22 +88,22 @@ http_archive(
 
 http_archive(
     name = "bazel_skylib",
-    sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44",
+    sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
-        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
     ],
 )
 
 # Needed by abseil-py by zetasql.
 http_archive(
     name = "six_archive",
     urls = [
-        "http://mirror.bazel.build/pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
-        "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
+        "http://mirror.bazel.build/pypi.python.org/packages/source/s/six/six-1.16.0.tar.gz",
+        "https://pypi.python.org/packages/source/s/six/six-1.16.0.tar.gz",
     ],
-    sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
-    strip_prefix = "six-1.10.0",
+    sha256 = "1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+    strip_prefix = "six-1.16.0",
     build_file = "//ml_metadata/third_party:six.BUILD"
 )
 
@@ -166,11 +166,12 @@ http_archive(
     ],
 )
 
+# gRPC. Official release 1.50.0. Name is required by Google APIs.
 http_archive(
     name = "com_github_grpc_grpc",
-    urls = ["https://github.com/grpc/grpc/archive/v1.46.3.tar.gz"],
-    sha256 = "d6cbf22cb5007af71b61c6be316a79397469c58c82a942552a62e708bce60964",
-    strip_prefix = "grpc-1.46.3",
+    sha256 = "76900ab068da86378395a8e125b5cc43dfae671e09ff6462ddfef18676e2165a",
+    strip_prefix = "grpc-1.50.0",
+    urls = ["https://github.com/grpc/grpc/archive/v1.50.0.tar.gz"],
 )
 
 load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
@@ -191,34 +192,8 @@ bind(
     actual = "@com_github_grpc_grpc//:grpc++",
 )
 
-# Needed by gRPC.
-http_archive(
-    name = "build_bazel_rules_swift",
-    sha256 = "d0833bc6dad817a367936a5f902a0c11318160b5e80a20ece35fb85a5675c886",
-    strip_prefix = "rules_swift-3eeeb53cebda55b349d64c9fc144e18c5f7c0eb8",
-    urls = ["https://github.com/bazelbuild/rules_swift/archive/3eeeb53cebda55b349d64c9fc144e18c5f7c0eb8.tar.gz"],
-)
-
-http_archive(
-    name = "io_bazel_rules_go",
-    urls = [
-        "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/rules_go/releases/download/v0.20.3/rules_go-v0.20.3.tar.gz",
-        "https://github.com/bazelbuild/rules_go/releases/download/v0.20.3/rules_go-v0.20.3.tar.gz",
-    ],
-    sha256 = "e88471aea3a3a4f19ec1310a55ba94772d087e9ce46e41ae38ecebe17935de7b",
-)
-
+# pulled in transitively by gRPC.
 load("@io_bazel_rules_go//go:deps.bzl", "go_rules_dependencies", "go_register_toolchains")
-
-http_archive(
-    name = "bazel_gazelle",
-    urls = [
-        "https://storage.googleapis.com/bazel-mirror/github.com/bazelbuild/bazel-gazelle/releases/download/v0.19.1/bazel-gazelle-v0.19.1.tar.gz",
-        "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.19.1/bazel-gazelle-v0.19.1.tar.gz",
-    ],
-    sha256 = "86c6d481b3f7aedc1d60c1c211c6f76da282ae197c3b3160f54bd3a8f847896f",
-)
-
 load("@bazel_gazelle//:deps.bzl", "go_repository", "gazelle_dependencies")
 
 go_repository(
@@ -235,7 +210,7 @@ go_repository(
 
 go_rules_dependencies()
 
-go_register_toolchains()
+go_register_toolchains("host")
 
 gazelle_dependencies()
 
diff --git a/ml_metadata/libmysqlclient.BUILD b/ml_metadata/libmysqlclient.BUILD
@@ -34,8 +34,10 @@ genrule(
 )
 
 config_setting(
-    name = "darwin",
-    values = {"cpu": "darwin"},
+    name = "macos",
+    constraint_values = [
+        "@bazel_tools//platforms:osx",
+    ],
     visibility = ["//visibility:public"],
 )
 
@@ -45,13 +47,14 @@ cc_library(
         # plugins.
         "plugins/auth/my_auth.c",
         "plugins/auth/old_password.c",
+        "plugins/compress/c_zlib.c",
         "plugins/pvio/pvio_socket.c",
         # ssl.
         "libmariadb/secure/openssl.c",
         # core libmariadb
         "libmariadb/ma_array.c",
         "libmariadb/ma_charset.c",
-        "libmariadb/ma_hash.c",
+        "libmariadb/ma_hashtbl.c",
         "libmariadb/ma_net.c",
         "libmariadb/mariadb_charset.c",
         "libmariadb/ma_time.c",
@@ -91,7 +94,10 @@ cc_library(
         "-DLIBICONV_PLUG",
         "-DHAVE_OPENSSL",
         "-DHAVE_TLS",
-    ],
+    ] + select({
+        ":macos": ["-D_XOPEN_SOURCE"],
+        "//conditions:default": [],
+    }),
     includes = [
         "build/include/",
         "include/",
@@ -102,7 +108,7 @@ cc_library(
         "-ldl",
         "-lm",
     ] + select({
-        ":darwin": ["-liconv"],
+        ":macos": ["-liconv"],
         "//conditions:default": [],
     }),
     visibility = ["//visibility:public"],
diff --git a/ml_metadata/metadata_store/BUILD b/ml_metadata/metadata_store/BUILD
@@ -71,7 +71,7 @@ cc_library(
         ":metadata_access_object_base",
         ":query_executor",
         "@com_google_protobuf//:protobuf",
-        
+
         "@com_google_absl//absl/algorithm:container",
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/container:flat_hash_set",
diff --git a/ml_metadata/metadata_store/pywrap/BUILD b/ml_metadata/metadata_store/pywrap/BUILD
@@ -37,6 +37,7 @@ ml_metadata_pybind_extension(
         "//ml_metadata/proto:metadata_store_proto",
         "//ml_metadata/simple_types/proto:simple_types_proto",
         "@pybind11",
+        "@local_config_python//:python_headers",
     ],
 )
 
diff --git a/ml_metadata/ml_metadata.bzl b/ml_metadata/ml_metadata.bzl
@@ -262,6 +262,7 @@ def ml_metadata_pybind_extension(
                 "-Wl,-exported_symbols_list,$(location %s)" % exported_symbols_file,
             ],
             "//conditions:default": [
+                "-Wl,-Bsymbolic",
                 "-Wl,--version-script",
                 "$(location %s)" % version_script_file,
             ],
@@ -270,7 +271,7 @@ def ml_metadata_pybind_extension(
             exported_symbols_file,
             version_script_file,
         ],
-        features = ["-use_header_modules"],
+        features = ["-use_header_modules", "-parse_headers"],
         linkshared = 1,
         visibility = visibility,
     )
diff --git a/ml_metadata/mysql_configure.bzl b/ml_metadata/mysql_configure.bzl
@@ -25,7 +25,7 @@ def mysql_configure():
         name = "libmysqlclient",
         build_file = "//ml_metadata:libmysqlclient.BUILD",
         workspace_file = "//ml_metadata:libmysqlclient.WORKSPACE",
+        patches = ["//ml_metadata/third_party:mariadbconnector.patch"],
         remote = "https://github.com/MariaDB/mariadb-connector-c.git",
-        tag = "v3.0.8-release",
-        patches = ["//ml_metadata/third_party:libmysqlclient.patch"],
+        tag = "v3.3.2",
     )
diff --git a/ml_metadata/third_party/grpc.patch b/ml_metadata/third_party/grpc.patch
diff --git a/ml_metadata/third_party/mariadbconnector.patch b/ml_metadata/third_party/mariadbconnector.patch
@@ -0,0 +1,15 @@
+--- libmariadb/secure/openssl.c
++++ libmariadb/secure/openssl.c
+@@ -329,11 +329,7 @@ static int ma_tls_set_certs(MYSQL *mysql, SSL_CTX *ctx)
+   if ((mysql->options.ssl_cipher &&
+         mysql->options.ssl_cipher[0] != 0))
+   {
+-    if(
+-#ifdef TLS1_3_VERSION
+-       SSL_CTX_set_ciphersuites(ctx, mysql->options.ssl_cipher) == 0 &&
+-#endif
+-       SSL_CTX_set_cipher_list(ctx, mysql->options.ssl_cipher) == 0)
++    if(SSL_CTX_set_cipher_list(ctx, mysql->options.ssl_cipher) == 0)
+       goto error;
+   }
+ 
diff --git a/setup.py b/setup.py
@@ -115,6 +115,8 @@ def run(self):
         # file, which is the root directory.
         cwd=os.path.dirname(os.path.realpath(__file__)),)
 
+    # explicitly call shutdown to free up resources.
+    subprocess.check_call([self._bazel_cmd, "shutdown"])
 
 # Get version from version module.
 with open('ml_metadata/version.py') as fp:

Original file line number	Diff line number	Diff line change
`@@ -37,6 +37,7 @@ ml_metadata_pybind_extension(`
`37`	`37`	`"//ml_metadata/proto:metadata_store_proto",`
`38`	`38`	`"//ml_metadata/simple_types/proto:simple_types_proto",`
`39`	`39`	`"@pybind11",`
	`40`	`+ "@local_config_python//:python_headers",`
`40`	`41`	`],`
`41`	`42`	`)`
`42`	`43`