feat: replace Algolia with local CJK search

README.md

@@ -26,7 +26,7 @@ MX Space Core is a headless CMS server built with **NestJS**, **MongoDB**, and *
 | **AI Workflow** | Summary generation, multi-language translation, comment moderation, writing assistance, streaming responses |
 | **LLM Providers** | OpenAI, OpenAI-compatible, Anthropic, OpenRouter |
 | **Real-time** | WebSocket via Socket.IO with Redis adapter for multi-instance broadcast |
-| **Distribution** | RSS/Atom feeds, sitemap, Algolia search, aggregate API |
+| **Distribution** | RSS/Atom feeds, sitemap, local search, aggregate API |
 | **Auth** | JWT sessions, passkeys, OAuth, API keys (via better-auth) |
 | **Deployment** | Docker (multi-arch), PM2, standalone binary |
 

apps/core/package.json

@@ -52,7 +52,6 @@
     "changelog": true
   },
   "dependencies": {
-    "@algolia/client-search": "^5.49.1",
     "@antfu/install-pkg": "1.1.0",
     "@anthropic-ai/sdk": "^0.78.0",
     "@babel/core": "7.29.0",
@@ -90,7 +89,6 @@
     "@typegoose/auto-increment": "^5.0.0",
     "@typegoose/typegoose": "^13.2.0",
     "@types/jsonwebtoken": "9.0.10",
-    "algoliasearch": "5.49.1",
     "axios": "^1.13.3",
     "axios-retry": "4.5.0",
     "bcryptjs": "^3.0.3",

apps/core/src/constants/db.constant.ts

@@ -24,6 +24,7 @@ export const PROJECT_COLLECTION_NAME = 'projects'
 export const READER_COLLECTION_NAME = 'readers'
 export const RECENTLY_COLLECTION_NAME = 'recentlies'
 export const SAY_COLLECTION_NAME = 'says'
+export const SEARCH_DOCUMENT_COLLECTION_NAME = 'search_documents'
 export const SERVERLESS_LOG_COLLECTION_NAME = 'serverless_logs'
 export const SERVERLESS_STORAGE_COLLECTION_NAME = 'serverless_storages'
 export const SLUG_TRACKER_COLLECTION_NAME = 'slug_trackers'

apps/core/src/constants/error-code.constant.ts

@@ -51,8 +51,6 @@ export enum ErrorCodeEnum {
   // biz - disabled/not enabled (400/403)
   LinkDisabled = 13000,
   SubpathLinkDisabled = 13001,
-  AlgoliaNotEnabled = 13002,
-  AlgoliaNotConfigured = 13003,
   BackupNotEnabled = 13004,
   SubscribeNotEnabled = 13005,
   PasswordLoginDisabled = 13006,
@@ -204,8 +202,6 @@ export const ErrorCode = Object.freeze<Record<ErrorCodeEnum, [string, number]>>(
       '管理员当前禁用了子路径友链申请',
       422,
     ],
-    [ErrorCodeEnum.AlgoliaNotEnabled]: ['Algolia 未开启', 400],
-    [ErrorCodeEnum.AlgoliaNotConfigured]: ['Algolia 未配置', 400],
     [ErrorCodeEnum.BackupNotEnabled]: ['请先在设置中开启备份功能', 400],
     [ErrorCodeEnum.SubscribeNotEnabled]: ['订阅功能未开启', 400],
     [ErrorCodeEnum.PasswordLoginDisabled]: ['密码登录已禁用', 400],

apps/core/src/constants/event-bus.constant.ts

@@ -1,6 +1,5 @@
 export enum EventBusEvents {
   EmailInit = 'email.init',
-  PushSearch = 'search.push',
   TokenExpired = 'token.expired',
   CleanAggregateCache = 'cache.aggregate',
   SystemException = 'system.exception',

apps/core/src/modules/configs/configs.default.ts

@@ -82,13 +82,6 @@ export const generateDefaultConfig: () => IConfig = () => ({
   },
   baiduSearchOptions: { enable: false, token: null! },
   bingSearchOptions: { enable: false, token: null! },
-  algoliaSearchOptions: {
-    enable: false,
-    apiKey: '',
-    appId: '',
-    indexName: '',
-    maxTruncateSize: 10000,
-  },
   adminExtra: {
     enableAdminProxy: true,
 

apps/core/src/modules/configs/configs.dsl.util.ts

@@ -1,4 +1,5 @@
 import { z } from 'zod'
+
 import { configSchemaMapping, FullConfigSchema } from './configs.schema'
 import { getMeta, type SchemaMetadata } from './configs.zod-schema.util'
 
@@ -108,11 +109,7 @@ const groupConfigs: GroupConfig[] = [
     title: '搜索推送',
     description: '搜索引擎、全文检索',
     icon: 'search',
-    sectionKeys: [
-      'baiduSearchOptions',
-      'bingSearchOptions',
-      'algoliaSearchOptions',
-    ],
+    sectionKeys: ['baiduSearchOptions', 'bingSearchOptions'],
   },
   {
     key: 'storage',
@@ -459,8 +456,7 @@ function formatProviderLabel(provider: AIProviderInfo): string {
   const id = provider.id || ''
 
   const nameLooksLikeUuid =
-    !!name &&
-    /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name)
+    !!name && /^[\da-f]{8}(?:-[\da-f]{4}){3}-[\da-f]{12}$/i.test(name)
 
   const displayName = !nameLooksLikeUuid && name ? name : ''
 

apps/core/src/modules/configs/configs.interface.ts

@@ -1,15 +1,15 @@
 import type { z } from 'zod'
+
 import {
-  configSchemaMapping,
   type AdminExtraSchema,
   type AISchema,
-  type AlgoliaSearchOptionsSchema,
   type AuthSecuritySchema,
   type BackupOptionsSchema,
   type BaiduSearchOptionsSchema,
   type BarkOptionsSchema,
   type BingSearchOptionsSchema,
   type CommentOptionsSchema,
+  configSchemaMapping,
   type FeatureListSchema,
   type FileUploadOptionsSchema,
   type FriendLinkOptionsSchema,
@@ -43,7 +43,6 @@ export abstract class IConfig {
   fileUploadOptions: Required<z.infer<typeof FileUploadOptionsSchema>>
   baiduSearchOptions: Required<z.infer<typeof BaiduSearchOptionsSchema>>
   bingSearchOptions: Required<z.infer<typeof BingSearchOptionsSchema>>
-  algoliaSearchOptions: Required<z.infer<typeof AlgoliaSearchOptionsSchema>>
   featureList: Required<z.infer<typeof FeatureListSchema>>
   thirdPartyServiceIntegration: Required<
     z.infer<typeof ThirdPartyServiceIntegrationSchema>

apps/core/src/modules/configs/configs.schema.ts

@@ -206,31 +206,6 @@ export class BingSearchOptionsDto extends createZodDto(
 ) {}
 export type BingSearchOptionsConfig = z.infer<typeof BingSearchOptionsSchema>
 
-// ==================== Algolia Search Options ====================
-export const AlgoliaSearchOptionsSchema = section('Algolia Search', {
-  enable: field.plain(z.boolean().optional(), '开启 Algolia Search'),
-  apiKey: field.password(z.string().optional(), 'ApiKey'),
-  appId: field.plain(z.string().optional(), 'AppId'),
-  indexName: field.plain(z.string().optional(), 'IndexName'),
-  maxTruncateSize: field.plain(
-    z.preprocess(
-      (val) => (val ? Number(val) : val),
-      z.number().int().min(100).optional(),
-    ),
-    '最大文档大小',
-    {
-      description:
-        'Algolia 文档大小限制，单位为字节，免费版本为 10K, 填写为 10000',
-    },
-  ),
-})
-export class AlgoliaSearchOptionsDto extends createZodDto(
-  AlgoliaSearchOptionsSchema,
-) {}
-export type AlgoliaSearchOptionsConfig = z.infer<
-  typeof AlgoliaSearchOptionsSchema
->
-
 // ==================== Admin Extra ====================
 export const AdminExtraSchema = section('后台附加设置', {
   enableAdminProxy: field.toggle(z.boolean().optional(), '开启后台管理反代', {
@@ -445,7 +420,6 @@ export const configSchemaMapping = {
   fileUploadOptions: FileUploadOptionsSchema,
   baiduSearchOptions: BaiduSearchOptionsSchema,
   bingSearchOptions: BingSearchOptionsSchema,
-  algoliaSearchOptions: AlgoliaSearchOptionsSchema,
   featureList: FeatureListSchema,
   thirdPartyServiceIntegration: ThirdPartyServiceIntegrationSchema,
   authSecurity: AuthSecuritySchema,
@@ -471,7 +445,6 @@ export const FullConfigSchema = withMeta(
     fileUploadOptions: FileUploadOptionsSchema,
     baiduSearchOptions: BaiduSearchOptionsSchema,
     bingSearchOptions: BingSearchOptionsSchema,
-    algoliaSearchOptions: AlgoliaSearchOptionsSchema,
     featureList: FeatureListSchema,
     thirdPartyServiceIntegration: ThirdPartyServiceIntegrationSchema,
     authSecurity: AuthSecuritySchema,

apps/core/src/modules/configs/configs.service.ts

@@ -1,6 +1,10 @@
 import cluster from 'node:cluster'
+
 import { Injectable, Logger } from '@nestjs/common'
 import type { ReturnModelType } from '@typegoose/typegoose'
+import { cloneDeep, merge, mergeWith } from 'es-toolkit/compat'
+import type { z, ZodError } from 'zod'
+
 import { BizException } from '~/common/exceptions/biz.exception'
 import { EventScope } from '~/constants/business-event.constant'
 import { RedisKeys } from '~/constants/cache.constant'
@@ -13,8 +17,7 @@ import { SubPubBridgeService } from '~/processors/redis/subpub.service'
 import { InjectModel } from '~/transformers/model.transformer'
 import { getRedisKey } from '~/utils/redis.util'
 import { camelcaseKeys, sleep } from '~/utils/tool.util'
-import { cloneDeep, merge, mergeWith } from 'es-toolkit/compat'
-import type { z, ZodError } from 'zod'
+
 import { generateDefaultConfig } from './configs.default'
 import {
   decryptObject,
@@ -122,7 +125,7 @@ export class ConfigsService {
       } catch (error) {
         await this.configInit()
         if (errorRetryCount > 0) {
-          return await this.getConfig(--errorRetryCount)
+          return await this.getConfig(errorRetryCount - 1)
         }
         this.logger.error('获取配置失败')
         throw error
@@ -251,19 +254,6 @@ export class ConfigsService {
 
         return option
       }
-      case 'algoliaSearchOptions': {
-        const option = await this.patch(
-          key as 'algoliaSearchOptions',
-          instanceValue as any,
-        )
-        if (option.enable) {
-          this.eventManager.emit(EventBusEvents.PushSearch, null, {
-            scope: EventScope.TO_SYSTEM,
-          })
-        }
-        return option
-      }
-
       case 'oauth': {
         const value = instanceValue as unknown as OAuthConfig
         const current = await this.get('oauth')

apps/core/src/modules/search/search-document.model.ts

@@ -0,0 +1,65 @@
+import { index, modelOptions, prop } from '@typegoose/typegoose'
+
+import { SEARCH_DOCUMENT_COLLECTION_NAME } from '~/constants/db.constant'
+
+export type SearchDocumentRefType = 'post' | 'note' | 'page'
+
+@index({ refType: 1, refId: 1 }, { unique: true })
+@index({ title: 'text', searchText: 'text' })
+@index({ terms: 1 })
+@index({ refType: 1, modified: -1, created: -1 })
+@index({ refType: 1, isPublished: 1, publicAt: 1, hasPassword: 1 })
+@modelOptions({
+  options: {
+    customName: SEARCH_DOCUMENT_COLLECTION_NAME,
+  },
+})
+export class SearchDocumentModel {
+  @prop({ required: true, enum: ['post', 'note', 'page'] })
+  refType!: SearchDocumentRefType
+
+  @prop({ required: true, index: true })
+  refId!: string
+
+  @prop({ required: true, trim: true })
+  title!: string
+
+  @prop({ required: true, trim: true })
+  searchText!: string
+
+  @prop({ type: () => [String], default: [] })
+  terms!: string[]
+
+  @prop({ type: () => [String], default: [] })
+  titleTerms!: string[]
+
+  @prop({ type: () => [String], default: [] })
+  bodyTerms!: string[]
+
+  @prop({ default: 0 })
+  titleLength!: number
+
+  @prop({ default: 0 })
+  bodyLength!: number
+
+  @prop({ trim: true })
+  slug?: string
+
+  @prop()
+  nid?: number
+
+  @prop({ default: true })
+  isPublished!: boolean
+
+  @prop()
+  publicAt?: Date | null
+
+  @prop({ default: false })
+  hasPassword!: boolean
+
+  @prop()
+  created?: Date | null
+
+  @prop()
+  modified?: Date | null
+}

apps/core/src/modules/search/search.constants.ts

@@ -1,4 +1,8 @@
-export const SEARCH_TITLE_WEIGHT = 3
-export const SEARCH_TEXT_WEIGHT = 20
-
-export const DEFAULT_ALGOLIA_MAX_SIZE_IN_BYTES = 10_000
+export const SEARCH_EXACT_TITLE_BONUS = 80
+export const SEARCH_PREFIX_TITLE_BONUS = 30
+export const SEARCH_CANDIDATE_MULTIPLIER = 12
+export const SEARCH_MAX_CANDIDATES = 300
+export const SEARCH_BM25_TITLE_WEIGHT = 3
+export const SEARCH_BM25_BODY_WEIGHT = 1.2
+export const SEARCH_BM25_K1 = 1.2
+export const SEARCH_BM25_B = 0.75

apps/core/src/modules/search/search.controller.ts

@@ -1,22 +1,36 @@
-import { Get, Param, Post, Query, Res } from '@nestjs/common'
+import { Get, Param, Post, Query } from '@nestjs/common'
+
 import { ApiController } from '~/common/decorators/api-controller.decorator'
 import { Auth } from '~/common/decorators/auth.decorator'
 import { HttpCache } from '~/common/decorators/cache.decorator'
-import { HTTPDecorators } from '~/common/decorators/http.decorator'
 import { IsAuthenticated } from '~/common/decorators/role.decorator'
 import { BizException } from '~/common/exceptions/biz.exception'
 import { ErrorCodeEnum } from '~/constants/error-code.constant'
 import { SearchDto } from '~/modules/search/search.schema'
-import type { FastifyReply } from 'fastify'
+
 import { SearchService } from './search.service'
 
 @ApiController('search')
 export class SearchController {
   constructor(private readonly searchService: SearchService) {}
 
+  @HttpCache.disable
+  @Get()
+  search(
+    @Query() query: SearchDto,
+    @IsAuthenticated() isAuthenticated: boolean,
+  ) {
+    return this.searchService.search(query, isAuthenticated)
+  }
+
+  @Post('/rebuild')
+  @Auth()
+  rebuild() {
+    return this.searchService.rebuildSearchDocuments()
+  }
+
   @Get('/:type')
   @HttpCache.disable
-  @HTTPDecorators.Paginator
   searchByType(
     @Query() query: SearchDto,
     @IsAuthenticated() isAuthenticated: boolean,
@@ -25,33 +39,18 @@ export class SearchController {
     type = type.toLowerCase()
     switch (type) {
       case 'post': {
-        return this.searchService.searchPost(query)
+        return this.searchService.searchPost(query, isAuthenticated)
       }
-      case 'note':
+      case 'note': {
         return this.searchService.searchNote(query, isAuthenticated)
+      }
+      case 'page': {
+        return this.searchService.searchPage(query)
+      }
 
-      default:
+      default: {
         throw new BizException(ErrorCodeEnum.InvalidSearchType, type)
+      }
     }
   }
-
-  @Get('/algolia')
-  async search(@Query() query: SearchDto) {
-    return this.searchService.searchAlgolia(query)
-  }
-
-  @Post('/algolia/push')
-  @Auth()
-  async pushAlgoliaAllManually() {
-    return this.searchService.pushAllToAlgoliaSearch()
-  }
-
-  @Get('/algolia/import-json')
-  @Auth()
-  async getAlgoliaIndexJsonFile(@Res() res: FastifyReply) {
-    const documents = await this.searchService.buildAlgoliaIndexData()
-    res.header('Content-Type', 'application/json')
-    res.header('Content-Disposition', 'attachment; filename=algolia-index.json')
-    res.send(JSON.stringify(documents))
-  }
 }