Skip to content

Commit b9ec465

Browse files
caio-pizzolcaio-pizzol
committed
feat: disable chat until documents are fully indexed
Track vectorCount and chunkCount from the API. Chat only enables when vectorCount >= chunkCount. Documents show green pulsing "Indexing..." state between processing completion and vector availability. Error state with 2-minute timeout for stuck documents. Configurable base path for Vite builds.
1 parent b9260be commit b9ec465

10 files changed

Lines changed: 185 additions & 55 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
dist/

‎rag/apps/api/src/cf/d1.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,13 @@ export function createD1Client(db: D1Database) {
146146
};
147147
},
148148

149+
async chunkCount(): Promise<number> {
150+
const result = await db
151+
.prepare("SELECT COUNT(*) as count FROM chunks")
152+
.first();
153+
return (result as any)?.count ?? 0;
154+
},
155+
149156
async getChunkIdsByDocument(documentId: number): Promise<string[]> {
150157
const result = await db
151158
.prepare("SELECT id FROM chunks WHERE document_id = ?")

‎rag/apps/api/src/cf/vectorize.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,11 @@ export function createVectorizeClient(index: VectorizeIndex) {
2020
}
2121
},
2222

23+
async vectorCount(): Promise<number> {
24+
const info = await index.describe();
25+
return info.vectorCount;
26+
},
27+
2328
async deleteByIds(ids: string[]): Promise<void> {
2429
const batchSize = 1000;
2530
for (let i = 0; i < ids.length; i += batchSize) {

‎rag/apps/api/src/worker.ts‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,17 +59,35 @@ export default {
5959
return handleQuery(request, env);
6060
}
6161

62-
// GET /api/documents — list documents
62+
// GET /api/documents — list documents + indexing status
6363
if (url.pathname === "/api/documents" && request.method === "GET") {
6464
const d1 = createD1Client(env.DB);
65-
const docs = await d1.listDocuments();
66-
return json(
67-
docs.map((d) => ({
65+
const vectorize = createVectorizeClient(env.VECTORIZE);
66+
const [docs, vectorCount, chunkCount] = await Promise.all([
67+
d1.listDocuments(),
68+
vectorize.vectorCount(),
69+
d1.chunkCount(),
70+
]);
71+
const TIMEOUT_MS = 2 * 60 * 1000;
72+
const now = Date.now();
73+
for (const d of docs) {
74+
if (d.status === "processing") {
75+
const created = new Date(d.createdAt).getTime();
76+
if (now - created > TIMEOUT_MS) {
77+
await d1.updateDocumentStatus(d.id, "error");
78+
d.status = "error";
79+
}
80+
}
81+
}
82+
return json({
83+
documents: docs.map((d) => ({
6884
id: d.id,
6985
filename: d.filename,
7086
status: d.status,
7187
})),
72-
);
88+
vectorCount,
89+
chunkCount,
90+
});
7391
}
7492

7593
// DELETE /api/documents/:id — remove document, chunks, vectors, and file
@@ -327,6 +345,14 @@ async function handleQuery(request: Request, env: Env): Promise<Response> {
327345
`[query] D1 returned ${chunks.length} chunks for ${chunkIds.length} IDs`,
328346
);
329347

348+
if (chunks.length === 0) {
349+
return json({
350+
answer:
351+
"Documents are still being indexed. Please try again in a moment.",
352+
citations: [],
353+
});
354+
}
355+
330356
const context = chunks
331357
.map(
332358
(c) =>

‎rag/apps/web/App.tsx‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ function App() {
99
const [activeDocId, setActiveDocId] = useState<number | null>(null);
1010
const [activeFilename, setActiveFilename] = useState<string | null>(null);
1111
const [activeCitation, setActiveCitation] = useState<Citation | null>(null);
12+
const [hasReadyDocs, setHasReadyDocs] = useState(false);
1213

1314
const handleSelectDoc = useCallback((doc: DocumentInfo) => {
1415
setActiveDocId(doc.id);
@@ -19,8 +20,8 @@ function App() {
1920
async (citation: Citation) => {
2021
if (citation.documentId !== activeDocId) {
2122
setActiveDocId(citation.documentId);
22-
const docs = await listDocuments();
23-
const doc = docs.find((d) => d.id === citation.documentId);
23+
const resp = await listDocuments();
24+
const doc = resp.documents.find((d) => d.id === citation.documentId);
2425
if (doc) setActiveFilename(doc.filename);
2526
}
2627
setActiveCitation(null);
@@ -50,13 +51,20 @@ function App() {
5051
</div>
5152
</header>
5253
<main className="main">
53-
<FileSidebar activeDocId={activeDocId} onSelectDoc={handleSelectDoc} />
54+
<FileSidebar
55+
activeDocId={activeDocId}
56+
onSelectDoc={handleSelectDoc}
57+
onReadyChange={setHasReadyDocs}
58+
/>
5459
<DocumentViewer
5560
documentId={activeDocId}
5661
citation={activeCitation}
5762
filename={activeFilename}
5863
/>
59-
<ChatPanel onCitationClick={handleCitationClick} />
64+
<ChatPanel
65+
onCitationClick={handleCitationClick}
66+
disabled={!hasReadyDocs}
67+
/>
6068
</main>
6169
</div>
6270
);

‎rag/apps/web/components/ChatPanel.tsx‎

Lines changed: 39 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,12 @@ const SAMPLE_QUESTIONS = [
1818
"What was decided about pricing?",
1919
];
2020

21-
type Props = { onCitationClick: (citation: Citation) => void };
21+
type Props = {
22+
onCitationClick: (citation: Citation) => void;
23+
disabled?: boolean;
24+
};
2225

23-
export function ChatPanel({ onCitationClick }: Props) {
26+
export function ChatPanel({ onCitationClick, disabled }: Props) {
2427
const [messages, setMessages] = useState<Message[]>([]);
2528
const [input, setInput] = useState("");
2629
const [loading, setLoading] = useState(false);
@@ -34,7 +37,7 @@ export function ChatPanel({ onCitationClick }: Props) {
3437

3538
async function handleSend() {
3639
const q = input.trim();
37-
if (!q || loading) return;
40+
if (!q || loading || disabled) return;
3841

3942
const userMsg: Message = {
4043
id: nextIdRef.current++,
@@ -89,25 +92,37 @@ export function ChatPanel({ onCitationClick }: Props) {
8992
<div className="chat-body">
9093
{messages.length === 0 && (
9194
<div className="chat-empty">
92-
<span className="chat-empty-title">Ask your documents</span>
93-
<span className="chat-empty-hint">
94-
Get cited answers from paragraphs, comments, and tracked changes.
95-
Click a citation to see the source.
96-
</span>
97-
<div className="sample-questions">
98-
{SAMPLE_QUESTIONS.map((q) => (
99-
<button
100-
type="button"
101-
key={q}
102-
className="sample-question"
103-
onClick={() => {
104-
setInput(q);
105-
}}
106-
>
107-
{q}
108-
</button>
109-
))}
110-
</div>
95+
{disabled ? (
96+
<>
97+
<span className="chat-empty-title">Waiting for documents</span>
98+
<span className="chat-empty-hint">
99+
Documents need to be uploaded, processed, and indexed before
100+
you can ask questions.
101+
</span>
102+
</>
103+
) : (
104+
<>
105+
<span className="chat-empty-title">Ask your documents</span>
106+
<span className="chat-empty-hint">
107+
Get cited answers from paragraphs, comments, and tracked
108+
changes. Click a citation to see the source.
109+
</span>
110+
<div className="sample-questions">
111+
{SAMPLE_QUESTIONS.map((q) => (
112+
<button
113+
type="button"
114+
key={q}
115+
className="sample-question"
116+
onClick={() => {
117+
setInput(q);
118+
}}
119+
>
120+
{q}
121+
</button>
122+
))}
123+
</div>
124+
</>
125+
)}
111126
</div>
112127
)}
113128
{messages.map((msg) => (
@@ -128,13 +143,13 @@ export function ChatPanel({ onCitationClick }: Props) {
128143
value={input}
129144
onChange={(e) => setInput(e.target.value)}
130145
onKeyDown={(e) => e.key === "Enter" && handleSend()}
131-
disabled={loading}
146+
disabled={loading || disabled}
132147
/>
133148
<button
134149
type="button"
135150
className="btn-send"
136151
onClick={handleSend}
137-
disabled={loading || !input.trim()}
152+
disabled={loading || disabled || !input.trim()}
138153
>
139154
Send
140155
</button>

‎rag/apps/web/components/FileSidebar.tsx‎

Lines changed: 67 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ type FileEntry = {
1111
filename: string;
1212
hash?: string;
1313
docId?: number;
14-
status: "ready" | "queued" | "extracting" | "deleting" | "error";
14+
status: "ready" | "queued" | "extracting" | "indexing" | "deleting" | "error";
1515
detail?: string;
1616
};
1717

@@ -26,40 +26,83 @@ async function fileHash(file: File): Promise<string> {
2626
type Props = {
2727
activeDocId: number | null;
2828
onSelectDoc: (doc: DocumentInfo) => void;
29+
onReadyChange: (hasReady: boolean) => void;
2930
};
3031

3132
let entrySeq = 0;
3233

33-
export function FileSidebar({ activeDocId, onSelectDoc }: Props) {
34+
export function FileSidebar({
35+
activeDocId,
36+
onSelectDoc,
37+
onReadyChange,
38+
}: Props) {
3439
const [entries, setEntries] = useState<FileEntry[]>([]);
3540
const [documents, setDocuments] = useState<DocumentInfo[]>([]);
41+
const [vectorCount, setVectorCount] = useState(0);
42+
const [chunkCount, setChunkCount] = useState(0);
3643
const fileRef = useRef<HTMLInputElement>(null);
3744

3845
const refreshDocs = useCallback(async () => {
39-
const docs = await listDocuments();
40-
setDocuments(docs);
46+
const resp = await listDocuments();
47+
setDocuments(resp.documents);
48+
setVectorCount(resp.vectorCount);
49+
setChunkCount(resp.chunkCount);
4150
setEntries((prev) => {
4251
const uploading = prev.filter(
43-
(e) => e.status === "queued" && !docs.some((d) => d.id === e.docId),
52+
(e) =>
53+
e.status === "queued" &&
54+
!resp.documents.some((d) => d.id === e.docId),
4455
);
45-
const fromApi: FileEntry[] = docs.map((d) => ({
46-
key: `doc-${d.id}`,
47-
filename: d.filename,
48-
docId: d.id,
49-
status: d.status === "ready" ? "ready" : "extracting",
50-
}));
51-
return [...uploading, ...fromApi];
56+
const deleting = prev.filter((e) => e.status === "deleting");
57+
const deletingIds = new Set(deleting.map((e) => e.docId));
58+
const fromApi: FileEntry[] = resp.documents
59+
.filter((d) => !deletingIds.has(d.id))
60+
.map((d) => {
61+
let status: FileEntry["status"];
62+
let detail: string | undefined;
63+
if (d.status === "error") {
64+
status = "error";
65+
detail = "Processing failed";
66+
} else if (
67+
d.status === "ready" &&
68+
resp.chunkCount > 0 &&
69+
resp.vectorCount < resp.chunkCount
70+
) {
71+
status = "indexing";
72+
} else if (d.status === "ready") {
73+
status = "ready";
74+
} else {
75+
status = "extracting";
76+
}
77+
return {
78+
key: `doc-${d.id}`,
79+
filename: d.filename,
80+
docId: d.id,
81+
status,
82+
detail,
83+
};
84+
});
85+
return [...uploading, ...deleting, ...fromApi];
5286
});
5387
}, []);
5488

5589
useEffect(() => {
5690
refreshDocs();
5791
}, [refreshDocs]);
5892

59-
// Auto-poll while any documents are still processing
93+
// Notify parent when documents are ready AND vectors are fully indexed
94+
const indexed = chunkCount > 0 && vectorCount >= chunkCount;
6095
useEffect(() => {
61-
const hasProcessing = entries.some((e) => e.status === "extracting");
62-
if (!hasProcessing) return;
96+
const hasReady = entries.some((e) => e.status === "ready");
97+
onReadyChange(hasReady && indexed);
98+
}, [entries, indexed, onReadyChange]);
99+
100+
// Auto-poll while documents are processing or vectors are not yet indexed
101+
useEffect(() => {
102+
const needsPoll = entries.some(
103+
(e) => e.status === "extracting" || e.status === "indexing",
104+
);
105+
if (!needsPoll) return;
63106
const interval = setInterval(refreshDocs, 3000);
64107
return () => clearInterval(interval);
65108
}, [entries, refreshDocs]);
@@ -134,7 +177,11 @@ export function FileSidebar({ activeDocId, onSelectDoc }: Props) {
134177
}
135178

136179
function handleClick(entry: FileEntry) {
137-
if (entry.status !== "ready" || !entry.docId) return;
180+
if (
181+
(entry.status !== "ready" && entry.status !== "indexing") ||
182+
!entry.docId
183+
)
184+
return;
138185
const doc = documents.find((d) => d.id === entry.docId);
139186
if (doc) onSelectDoc(doc);
140187
}
@@ -156,6 +203,7 @@ export function FileSidebar({ activeDocId, onSelectDoc }: Props) {
156203
const STATUS_LABEL: Record<string, string> = {
157204
queued: "Queued",
158205
extracting: "Processing...",
206+
indexing: "Indexing...",
159207
deleting: "Removing...",
160208
error: "Error",
161209
};
@@ -187,20 +235,20 @@ export function FileSidebar({ activeDocId, onSelectDoc }: Props) {
187235
{entries.map((entry) => (
188236
<div
189237
key={entry.key}
190-
className={`file-item ${entry.docId === activeDocId && entry.status === "ready" ? "active" : ""} ${entry.status !== "ready" ? "file-item--processing" : ""}`}
238+
className={`file-item ${entry.docId === activeDocId && (entry.status === "ready" || entry.status === "indexing") ? "active" : ""} ${entry.status === "error" ? "file-item--error" : entry.status === "indexing" ? "file-item--indexing" : entry.status !== "ready" ? "file-item--processing" : ""}`}
191239
>
192240
<button
193241
type="button"
194242
className="file-item-btn"
195243
onClick={() => handleClick(entry)}
196-
disabled={entry.status !== "ready"}
244+
disabled={entry.status !== "ready" && entry.status !== "indexing"}
197245
>
198246
<span
199247
className={`file-item-dot file-item-dot--${entry.status}`}
200248
/>
201249
<span className="file-item-label">{entry.filename}</span>
202250
</button>
203-
{entry.status === "ready" && entry.docId && (
251+
{entry.status !== "deleting" && entry.docId && (
204252
<button
205253
type="button"
206254
className="file-item-delete"

‎rag/apps/web/lib/api.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,15 @@ export async function deleteDocument(id: number): Promise<void> {
6565
}
6666
}
6767

68-
export async function listDocuments(): Promise<DocumentInfo[]> {
68+
export type DocumentsResponse = {
69+
documents: DocumentInfo[];
70+
vectorCount: number;
71+
chunkCount: number;
72+
};
73+
74+
export async function listDocuments(): Promise<DocumentsResponse> {
6975
const res = await fetch(`${API_BASE}/api/documents`);
70-
if (!res.ok) return [];
76+
if (!res.ok) return { documents: [], vectorCount: 0, chunkCount: 0 };
7177
return res.json();
7278
}
7379

0 commit comments

Comments
 (0)