fix (chat): resolving buffering issues (WIP)

Author: itsskofficial

src/client/app/api/chat/message/route.js

@@ -72,7 +72,8 @@ export const POST = withAuth(async function POST(request, { authHeader }) {
 				"Content-Type": "application/x-ndjson",
 				"Cache-Control": "no-cache",
 				Connection: "keep-alive",
-				"X-Accel-Buffering": "no" // Disable buffering on Netlify/Vercel
+				"X-Accel-Buffering": "no", // Disable buffering on Netlify/Vercel
+				"Content-Encoding": "identity"
 			}
 		})
 	} catch (error) {

src/client/netlify.toml

@@ -1,3 +1,9 @@
 [build]
    command = "npm run build"
-   publish = ".next"
+   publish = ".next"
+[[headers]]
+  for = "/api/chat/message"
+  [headers.values]
+    Cache-Control = "no-cache, no-store, must-revalidate"
+    Content-Encoding = "identity"
+    X-Accel-Buffering = "no"

src/server/main/chat/routes.py

@@ -41,7 +41,6 @@ async def chat_endpoint(
 
     async def event_stream_generator():
         try:
-            # Pass the full message history directly to the LLM stream generator
             async for event in generate_chat_llm_stream(
                 user_id,
                 request_body.messages,
@@ -53,16 +52,28 @@ async def event_stream_generator():
                 enable_maps=request_body.enable_maps,
                 enable_shopping=request_body.enable_shopping
             ):
-                if not event: continue
-                yield json.dumps(event) + "\n"
+                if not event:
+                    continue
+                # yield as bytes and flush
+                chunk = (json.dumps(event) + "\n").encode("utf-8")
+                yield chunk
+                await asyncio.sleep(0)  # Force control back to event loop
         except asyncio.CancelledError:
-             logger.info(f"Client disconnected, stream cancelled for user {user_id}.")
+            logger.info(f"Client disconnected, stream cancelled for user {user_id}.")
         except Exception as e:
             logger.error(f"Error in chat stream for user {user_id}: {e}", exc_info=True)
             error_response = {
                 "type": "error",
                 "message": "Sorry, I encountered an error while processing your request."
             }
-            yield json.dumps(error_response) + "\n"
+            yield (json.dumps(error_response) + "\n").encode("utf-8")
 
-    return StreamingResponse(event_stream_generator(), media_type="application/x-ndjson")
+    return StreamingResponse(
+        event_stream_generator(),
+        media_type="application/x-ndjson",
+        headers={
+            "Cache-Control": "no-cache",
+            "X-Accel-Buffering": "no",  # Disable Nginx buffering
+            "Transfer-Encoding": "chunked",  # Hint chunked encoding
+        }
+    )

src/server/nginx.conf

@@ -15,35 +15,25 @@ server {
     location /chat/message {
         proxy_pass http://backend_app;
 
-        # Streaming-Specific Settings
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
-
-        # Critical: Disable buffering for this endpoint
+        # Prevent buffering
         proxy_buffering off;
+        proxy_cache off;
+        proxy_request_buffering off;
 
-        # Increase timeouts for long-lived stream
-        proxy_read_timeout 86400s;
-        proxy_send_timeout 86400s;
+        # Disable gzip (interferes with chunked encoding)
+        gzip off;
 
-        # Required for keep-alive connections
+        # Streaming-friendly headers
         proxy_http_version 1.1;
         proxy_set_header Connection '';
-        
-        # Add CORS headers if needed, though your app likely handles this.
-        add_header 'Access-Control-Allow-Origin' '*' always;
-        add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS' always;
-        add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization' always;
-        if ($request_method = 'OPTIONS') {
-            add_header 'Access-Control-Max-Age' 1728000;
-            add_header 'Content-Type' 'text/plain; charset=utf-8';
-            add_header 'Content-Length' 0;
-            return 204;
-        }
+        chunked_transfer_encoding on;
+
+        # CORS and other headers (keep yours)
+        add_header Cache-Control "no-cache";
+        add_header X-Accel-Buffering "no" always;  # Netlify/Vercel/CDNs respect this
     }
 
+
     # ADDED: Location block for WebSocket notifications
     location /api/ws/notifications {
         proxy_pass http://backend_app;