tiktoken.cpp/CMakeLists.txt at main · KolosalAI/tiktoken.cpp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
cmake_minimum_required(VERSION 3.20)
project(tiktoken_cpp LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

# Aggressive optimization flags for maximum performance
if(CMAKE_BUILD_TYPE MATCHES "Release")
  if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|AppleClang")
    add_compile_options(-O3 -march=native -mtune=native -ffast-math -funroll-loops)
    add_compile_options(-flto)
    add_link_options(-flto)
  endif()
endif()

add_library(tiktoken INTERFACE)
target_include_directories(tiktoken INTERFACE src)

set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ON)
# Help CMake locate Conan CMakeDeps files if the toolchain didn't wire the path
list(APPEND CMAKE_PREFIX_PATH
  "${CMAKE_BINARY_DIR}/generators"
  "${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/generators"
)

find_package(PCRE2 REQUIRED CONFIG)
find_package(utf8cpp REQUIRED CONFIG)

set(_PCRE2_TARGET "")
if (TARGET PCRE2::8BIT)
  set(_PCRE2_TARGET PCRE2::8BIT)
elseif (TARGET pcre2::pcre2-8)
  set(_PCRE2_TARGET pcre2::pcre2-8)
elseif (TARGET pcre2::pcre2)
  set(_PCRE2_TARGET pcre2::pcre2)
else()
  message(FATAL_ERROR "Could not detect a PCRE2 CMake target")
endif()

# utf8cpp can expose different target names depending on the package/provider
set(_UTF8CPP_TARGET "")
if (TARGET utf8::cpp)
  set(_UTF8CPP_TARGET utf8::cpp)
elseif (TARGET utf8cpp)
  set(_UTF8CPP_TARGET utf8cpp)
elseif (TARGET utf8cpp::utf8cpp)
  # Some distros/exporters may provide this namespace
  set(_UTF8CPP_TARGET utf8cpp::utf8cpp)
else()
  message(FATAL_ERROR "Could not detect a utf8cpp CMake target")
endif()

target_link_libraries(tiktoken INTERFACE ${_PCRE2_TARGET} ${_UTF8CPP_TARGET})

# Examples
option(TIKTOKEN_BUILD_EXAMPLES "Build tiktoken.cpp examples" ON)
option(TIKTOKEN_BUILD_BENCHMARKS "Build tiktoken.cpp benchmarks" ON)
if (TIKTOKEN_BUILD_EXAMPLES)
  add_executable(cpp_basic examples/cpp_basic.cpp)
  target_link_libraries(cpp_basic PRIVATE tiktoken)
  target_compile_features(cpp_basic PRIVATE cxx_std_17)
  target_include_directories(cpp_basic PRIVATE src)
endif()

if (TIKTOKEN_BUILD_BENCHMARKS)
  find_package(Threads REQUIRED)

  add_executable(tiktoken_benchmark benchmarks/encode_benchmark.cpp)
  target_link_libraries(tiktoken_benchmark PRIVATE tiktoken)
  target_compile_features(tiktoken_benchmark PRIVATE cxx_std_17)
  target_include_directories(tiktoken_benchmark PRIVATE src)

  add_executable(benchmark benchmarks/benchmark.cpp)
  target_link_libraries(benchmark PRIVATE tiktoken Threads::Threads)
  target_compile_features(benchmark PRIVATE cxx_std_17)
  target_include_directories(benchmark PRIVATE src)
endif()