Skip to content

Commit ec2fcce

Browse files
Chetan11-devChetan11-dev
committed
fixes
1 parent 12a28f7 commit ec2fcce

5 files changed

Lines changed: 93 additions & 25 deletions

File tree

‎js/botasaurus-server-js/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,4 @@ todo.md
1818
temp.js
1919
todo copy.md
2020
tsconfig.tsbuildinfo
21+
.history/

‎js/botasaurus-server-js/src/routes-db-logic.ts‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import { Server } from './server';
1111
import { applyPagination, calculatePageEnd, calculatePageStart } from './apply-pagination';
1212
import { applyFiltersInPlace } from './filters';
1313
import { applySorts } from './sorts';
14-
import { _applyViewForUi, _applyViewForUiLargeTask, findView, transformRecord, getFields } from './views';
14+
import { _applyViewForUi, _applyViewForUiLargeTask, findView, transformRecordStream, getFields } from './views';
1515
import { getExecutor } from './executor'
1616
import { TaskPriority } from './task-executor'
1717
import { sleep } from 'botasaurus/utils'
@@ -955,10 +955,17 @@ async function performGetTaskResults(taskId: number): Promise<[string, boolean,
955955
let streamFn;
956956
if (viewObj) {
957957
const targetFields: any[] = isNotNullish(inputData) ? getFields(viewObj.fields, inputData, []) : viewObj.fields;
958-
streamFn = (item: any) => {
959-
item = transformRecord(targetFields, item);
960-
return convertToEnglish ? convertUnicodeDictToAsciiDict(item) : item;
961-
};
958+
if (convertToEnglish) {
959+
streamFn = function*(item: any) {
960+
for (const row of transformRecordStream(targetFields, item)) {
961+
yield convertUnicodeDictToAsciiDict(row);
962+
}
963+
};
964+
} else {
965+
streamFn = (item: any) => {
966+
return transformRecordStream(targetFields, item);
967+
};
968+
}
962969
} else {
963970
streamFn = (item: any) => {
964971
return convertToEnglish ? convertUnicodeDictToAsciiDict(item) : item;

‎js/botasaurus-server-js/src/views.ts‎

Lines changed: 72 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,67 @@ function transformRecord(targetFields: (Field | CustomField | ExpandDictField |
335335
return expandedRecords
336336
}
337337

338+
function* transformRecordStream(targetFields: (Field | CustomField | ExpandDictField | ExpandListField)[], record: any): Generator<any> {
339+
let expandedRecords: any[] = [{}]
340+
341+
for (let i = 0; i < targetFields.length; i++) {
342+
const field = targetFields[i]
343+
344+
if (field instanceof Field) {
345+
const value = record[field.key]
346+
const result = field.map ? field.map(value, record) : value
347+
for (const expandedRecord of expandedRecords) {
348+
expandedRecord[field.outputKey] = result
349+
}
350+
} else if (field instanceof CustomField) {
351+
const result = field.map(record)
352+
for (const expandedRecord of expandedRecords) {
353+
expandedRecord[field.outputKey] = result
354+
}
355+
} else if (field instanceof ExpandDictField) {
356+
const nestedFieldValues = createNestedFieldValues(record, field)
357+
for (const expandedRecord of expandedRecords) {
358+
Object.assign(expandedRecord, nestedFieldValues)
359+
}
360+
} else if (field instanceof ExpandListField) {
361+
const nestedList = record[field.key] || []
362+
const remainingFields = targetFields.slice(i + 1)
363+
364+
for (const item of nestedList) {
365+
for (const baseExpandedRecord of expandedRecords) {
366+
const newRecord = { ...baseExpandedRecord }
367+
for (const nestedField of field.fields) {
368+
if (nestedField instanceof Field) {
369+
const value = item[nestedField.key]
370+
newRecord[nestedField.outputKey] = nestedField.map ? nestedField.map(value, item, record) : value
371+
} else if (nestedField instanceof CustomField) {
372+
newRecord[nestedField.outputKey] = nestedField.map(item, record)
373+
} else if (nestedField instanceof ExpandDictField) {
374+
Object.assign(newRecord, createNestedFieldValuesListed(item, nestedField, record))
375+
}
376+
}
377+
for (const rf of remainingFields) {
378+
if (rf instanceof Field) {
379+
const value = record[rf.key]
380+
newRecord[rf.outputKey] = rf.map ? rf.map(value, record) : value
381+
} else if (rf instanceof CustomField) {
382+
newRecord[rf.outputKey] = rf.map(record)
383+
} else if (rf instanceof ExpandDictField) {
384+
Object.assign(newRecord, createNestedFieldValues(record, rf))
385+
}
386+
}
387+
yield newRecord
388+
}
389+
}
390+
return
391+
}
392+
}
393+
394+
for (const r of expandedRecords) {
395+
yield r
396+
}
397+
}
398+
338399
function performApplyView(results: any[], viewObj: View, inputData?: any): [any[], string[]] {
339400
const hidden_fields: string[] = [];
340401
const targetFields: (Field | CustomField | ExpandDictField | ExpandListField)[] = isNotNullish(inputData)?getFields(viewObj.fields, inputData, hidden_fields):viewObj.fields
@@ -366,21 +427,16 @@ function _applyViewForUi(
366427
if (pagination) {
367428
const { start, end, containsListField } = pagination;
368429
if (containsListField) {
369-
// Need to count all expanded items, but only keep items in range
370430
const result: any[] = [];
371431
let items_count = 0;
372432
for (let i = 0; i < results.length; i++) {
373433
const record = results[i];
374-
const expandedRecords = transformRecord(targetFields, record);
375-
const prevCount = items_count;
376-
items_count += expandedRecords.length;
377434
results[i] = null; // free memory
378-
379-
// Add only the items that fall within [start, end)
380-
if (prevCount < end && items_count > start) {
381-
const sliceStart = Math.max(0, start - prevCount);
382-
const sliceEnd = Math.min(expandedRecords.length, end - prevCount);
383-
result.push(...expandedRecords.slice(sliceStart, sliceEnd));
435+
for (const expandedRecord of transformRecordStream(targetFields, record)) {
436+
if (items_count >= start && items_count < end) {
437+
result.push(expandedRecord);
438+
}
439+
items_count++;
384440
}
385441
}
386442
return [result, hidden_fields, items_count];
@@ -432,15 +488,11 @@ async function _applyViewForUiLargeTask(taskId: number, view: string, views: Vie
432488
// @ts-ignore
433489
await TaskResults.streamTask(taskId, (record, _index) => {
434490
if (items_count >= end) return false;
435-
const expandedRecords: any[] = transformRecord(targetFields, record);
436-
const prevCount = items_count;
437-
items_count += expandedRecords.length;
438-
439-
// Add only the items that fall within [start, end)
440-
if (prevCount < end && items_count > start) {
441-
const sliceStart = Math.max(0, start - prevCount);
442-
const sliceEnd = Math.min(expandedRecords.length, end - prevCount);
443-
result.push(...expandedRecords.slice(sliceStart, sliceEnd));
491+
for (const expandedRecord of transformRecordStream(targetFields, record)) {
492+
if (items_count >= start && items_count < end) {
493+
result.push(expandedRecord);
494+
}
495+
items_count++;
444496
}
445497
});
446498

@@ -490,5 +542,5 @@ export {
490542
findView,
491543
_applyViewForUi,
492544
applyView,
493-
transformRecord,getFields,
545+
transformRecord,transformRecordStream,getFields,
494546
};

‎js/botasaurus-server-js/src/writer-http.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ function applyFunctionToResultSync(result: any, fn: (item: any) => any) {
1111
for (const item of result) {
1212
fn(item);
1313
}
14+
} else if (result != null && typeof result === 'object' && Symbol.iterator in result) {
15+
for (const item of result) {
16+
fn(item);
17+
}
1418
} else {
1519
return fn(result);
1620
}

‎js/botasaurus-server-js/src/writer.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ async function applyFunctionToResult(result: any, fn: (item:any)=> any) {
3030
for (const item of result) {
3131
await fn(item);
3232
}
33+
} else if (result != null && typeof result === 'object' && Symbol.iterator in result) {
34+
for (const item of result) {
35+
await fn(item);
36+
}
3337
} else {
3438
return fn(result);
3539
}

0 commit comments

Comments
 (0)