diff --git a/docs/reference/search.md b/docs/reference/search.md index b7af8138..daa9c0e7 100644 --- a/docs/reference/search.md +++ b/docs/reference/search.md @@ -289,6 +289,20 @@ directly. | `date` | `range` (inclusive) | yes | yes | ISO 8601 string (surface) | | `boolean` | `is` | yes | – | boolean (absent = false) | +A **`date`** is stored as Unix seconds – a sortable, range-filterable integer – +and is ISO 8601 at every edge: the projection converts a value on the way in, +the query compiler converts a filter bound on the way out, and the surface emits +a string. A year outside the plain four-digit form is accepted however it is written – +expanded (`-001100-01-01T00:00:00.000Z`) or not (`-1100`, `-250000`, `25000`): +the codec pads it to the signed six digits `Date` needs before parsing, so the +same value indexes and filters alike. Deep time is real data – SCHEMA-AP-NDE +blesses years beyond four digits for `schema:dateCreated` – and an unpadded year +would otherwise parse as something else entirely (a BCE year as a UTC offset, +landing in the wrong era; a five-digit CE year as a legacy local-time date, +landing a year early and differently per host timezone), without ever failing. +The window is the one `Date` can represent, ±271,821 years around 1970; a year +outside it leaves the field absent, as any unparseable value does. + **`array` decides a field’s shape**, whatever the graph carries: a declared `array` field stores a list, and a single-valued one stores the first value – for every kind alike, so the projection, the engine collection definition diff --git a/packages/search-typesense/test/query-compiler.test.ts b/packages/search-typesense/test/query-compiler.test.ts index 22f5512f..d8d7951c 100644 --- a/packages/search-typesense/test/query-compiler.test.ts +++ b/packages/search-typesense/test/query-compiler.test.ts @@ -413,6 +413,30 @@ describe('buildSearchParams', () => { ).toBe(`datePosted:<=${max}`); }); + it('reads an unpadded BCE bound as a year, so the range is not inverted', () => { + const min = Date.parse('-001100-01-01T00:00:00.000Z') / 1000; + const max = Date.parse('-000800-01-01T00:00:00.000Z') / 1000; + expect( + buildSearchParams( + { + ...base, + where: [ + { + or: [ + { + field: 'datePosted', + range: { min: '-1100', max: '-0800' }, + }, + ], + }, + ], + }, + schema, + ).filter_by, + ).toBe(`datePosted:[${min}..${max}]`); + expect(min).toBeLessThan(max); + }); + it('compiles orderBy: RELEVANCE → _text_match and a localized field → its sort key', () => { expect( buildSearchParams( diff --git a/packages/search/src/project.ts b/packages/search/src/project.ts index d17f1dd1..75a95716 100644 --- a/packages/search/src/project.ts +++ b/packages/search/src/project.ts @@ -230,11 +230,17 @@ function applyField( ? derivedIris(derived) : // A derived `date` goes through the same storage codec as a read one: // the field is stored as Unix seconds, so an ISO string returned here - // would otherwise land as a string in an int64 field. A derive that - // already computed seconds returns a number and passes through. + // would otherwise land as a string in an int64 field. field.kind === 'date' && typeof derived === 'string' ? isoToUnixSeconds(derived) - : derived; + : // A derive that computed seconds itself returns a number, which the + // codec never saw and so never range-checked. Seconds outside what + // `Date` can represent – reachable now that deep time is – would + // store fine and then throw on every read at the surface, where an + // unparseable string merely leaves the field absent. Drop it too. + field.kind === 'date' && typeof derived === 'number' + ? storableSeconds(derived) + : derived; // NaN is not a value any kind stores, and a derive is the only route that // can produce one (`setNumber` guards the read route). It serializes as // `null`, which an engine rejects for a numeric field – so drop it, leaving @@ -588,6 +594,13 @@ function derivedIris(value: unknown): unknown { return typeof value === 'string' && isAbsoluteIri(value) ? value : undefined; } +/** Stored Unix seconds a `Date` can represent, or `undefined` – the same + * answer {@link isoToUnixSeconds} gives a string it cannot parse, so the two + * routes into a `date` field agree on what is storable. */ +function storableSeconds(seconds: number): number | undefined { + return Number.isNaN(new Date(seconds * 1000).getTime()) ? undefined : seconds; +} + function toInteger(literal: string | undefined): number | undefined { return literal === undefined ? undefined : Math.trunc(Number(literal)); } diff --git a/packages/search/src/schema.ts b/packages/search/src/schema.ts index eb431a18..bdfa755b 100644 --- a/packages/search/src/schema.ts +++ b/packages/search/src/schema.ts @@ -1454,10 +1454,38 @@ const ABSOLUTE_IRI = /^[A-Za-z][A-Za-z0-9+.-]*:\S*$/; * unparseable value. */ export function isoToUnixSeconds(iso: string): number | undefined { - const millis = new Date(iso).getTime(); + // Trim first: XSD collapses whitespace around a date literal before parsing, + // so a padded lexical form is legal and reaches here verbatim. Left in place + // it defeats the year expansion below in both directions – a leading space + // stops the match, a trailing one survives it into the legacy parser – and + // each lands the value in a different year than it reads. + const millis = new Date(expandYear(iso.trim())).getTime(); return Number.isNaN(millis) ? undefined : Math.trunc(millis / 1000); } +/** + * Pad a year outside the plain four-digit form to the signed six digits ISO + * 8601 expects, so `Date` reads it as a year at all. Deep time is real data, + * not a corner case: SCHEMA-AP-NDE blesses years beyond four digits for + * `schema:dateCreated`. Left unexpanded, neither era parses as written and + * neither fails loudly – `-1100` is read as a bare UTC offset and lands + * around 1100 **CE** (shifted by the host’s offset), `25000` is read as a + * legacy local-time date and lands a year early, on a boundary that moves + * with the host’s timezone. Either way nothing throws, no field is left + * absent, and the value sorts and filters as if it were another date + * entirely. A plain four-digit year and an already-expanded one pass through + * unchanged, so the form the API itself emits round-trips. + */ +function expandYear(iso: string): string { + // The year, then the rest: a non-digit stops the run, so a longer digit + // string (epoch millis, say) is left alone rather than cut into a year. + const year = /^([+-]?)(\d{1,6})(\D.*|)$/.exec(iso); + if (year === null || (year[1] === '' && year[2].length === 4)) { + return iso; + } + return `${year[1] || '+'}${year[2].padStart(6, '0')}${year[3]}`; +} + /** The inverse of {@link isoToUnixSeconds}: stored Unix seconds → ISO 8601. */ export function unixSecondsToIso(seconds: number): string { return new Date(seconds * 1000).toISOString(); diff --git a/packages/search/test/project.test.ts b/packages/search/test/project.test.ts index 444e1173..ff347185 100644 --- a/packages/search/test/project.test.ts +++ b/packages/search/test/project.test.ts @@ -557,6 +557,14 @@ describe('projectDocument', () => { kind: 'date', derive: () => Date.parse('not-a-date') / 1000, }, + // Seconds past what `Date` can represent are dropped too: stored, + // they read back as an error at the surface rather than as a value, + // where an unparseable string simply leaves the field absent. + { + name: 'temporal', + kind: 'date', + derive: () => -8_640_000_000_001, + }, ], }, ); @@ -564,6 +572,7 @@ describe('projectDocument', () => { expect(document.modified).toBe(1_704_067_200); expect(document).not.toHaveProperty('created'); expect(document).not.toHaveProperty('available'); + expect(document).not.toHaveProperty('temporal'); }); it('holds a reference to absolute IRIs on every route a value can arrive by', () => { diff --git a/packages/search/test/schema.test.ts b/packages/search/test/schema.test.ts index b26f7165..67166e7e 100644 --- a/packages/search/test/schema.test.ts +++ b/packages/search/test/schema.test.ts @@ -1445,4 +1445,68 @@ describe('date storage codec', () => { it('returns undefined for an unparseable date', () => { expect(isoToUnixSeconds('not-a-date')).toBeUndefined(); }); + + it('reads an unpadded BCE year as a year, not as a UTC offset', () => { + const seconds = isoToUnixSeconds('-1100'); + expect(unixSecondsToIso(seconds ?? 0)).toBe('-001100-01-01T00:00:00.000Z'); + expect(seconds).toBe(isoToUnixSeconds('-001100-01-01T00:00:00.000Z')); + }); + + it('reads a CE year beyond four digits, independent of the host timezone', () => { + expect(unixSecondsToIso(isoToUnixSeconds('25000') ?? 0)).toBe( + '+025000-01-01T00:00:00.000Z', + ); + expect(unixSecondsToIso(isoToUnixSeconds('25000-06-01') ?? 0)).toBe( + '+025000-06-01T00:00:00.000Z', + ); + }); + + it('leaves a plain four-digit year and an expanded one alone', () => { + expect(unixSecondsToIso(isoToUnixSeconds('1999') ?? 0)).toBe( + '1999-01-01T00:00:00.000Z', + ); + expect( + unixSecondsToIso(isoToUnixSeconds('+025000-01-01T00:00:00.000Z') ?? 0), + ).toBe('+025000-01-01T00:00:00.000Z'); + }); + + it('reads a deep-time BCE year beyond four digits', () => { + expect(unixSecondsToIso(isoToUnixSeconds('-250000') ?? 0)).toBe( + '-250000-01-01T00:00:00.000Z', + ); + expect(unixSecondsToIso(isoToUnixSeconds('-38000') ?? 0)).toBe( + '-038000-01-01T00:00:00.000Z', + ); + }); + + it('expands the year of a fuller BCE date, leaving the rest alone', () => { + expect(unixSecondsToIso(isoToUnixSeconds('-1100-05-03') ?? 0)).toBe( + '-001100-05-03T00:00:00.000Z', + ); + }); + + it('reads a whitespace-padded year as the year it spells', () => { + // XSD collapses whitespace around a date literal, so a padded form is legal + // and reaches the codec verbatim. + for (const padded of [' -1100', '\t-1100', '-1100 ', ' -1100 ']) { + expect(isoToUnixSeconds(padded)).toBe(isoToUnixSeconds('-1100')); + } + expect(isoToUnixSeconds(' 25000 ')).toBe(isoToUnixSeconds('25000')); + expect(isoToUnixSeconds(' 1999 ')).toBe(isoToUnixSeconds('1999')); + }); + + it('returns undefined for a year beyond what Date can represent', () => { + // ±271,821 years is the edge of the Date range; past it there is no value + // to store, so the field is left absent as an unparseable string is. + expect(isoToUnixSeconds('-500000')).toBeUndefined(); + expect(isoToUnixSeconds('-271821')).toBeUndefined(); + }); + + it('orders BCE before CE, and earlier BCE before later', () => { + const deepTime = isoToUnixSeconds('-250000') ?? 0; + const bce = isoToUnixSeconds('-1100') ?? 0; + const ce = isoToUnixSeconds('1100') ?? 0; + expect(deepTime).toBeLessThan(bce); + expect(bce).toBeLessThan(ce); + }); }); diff --git a/packages/search/vite.config.ts b/packages/search/vite.config.ts index 661f9376..e3f60542 100644 --- a/packages/search/vite.config.ts +++ b/packages/search/vite.config.ts @@ -12,7 +12,7 @@ export default mergeConfig( thresholds: { functions: 100, lines: 100, - branches: 99.56, + branches: 99.57, statements: 100, }, },