Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | 9x 9x 9x 9x 9x 9x 9x 13x 13x 13x 24x 24x 24x 24x 24x 2x 24x 24x 24x 3x 13x 9x 9x 9x 13x 9x | /**
* Batch processor — concurrency-controlled batch embedding.
*
* Implements backpressure, progress callbacks, and timeout per item.
*/
import type { BatchOptions } from './embedder-interface';
export async function processBatch<T>(
items: T[],
processItem: (item: T, index: number) => Promise<void>,
options: BatchOptions = {},
): Promise<{ errors: Array<{ index: number; error: Error }> }> {
const concurrency =
options.concurrency ??
Math.max(1, Math.floor((typeof navigator !== 'undefined' ? 2 : await importNodeCores()) / 2));
const timeout = options.timeout ?? 30000;
let completed = 0;
const total = items.length;
let index = 0;
const workers: Promise<{ errors: Array<{ index: number; error: Error }> }>[] = [];
for (let i = 0; i < Math.min(concurrency, total); i++) {
workers.push(worker());
}
async function worker(): Promise<{ errors: Array<{ index: number; error: Error }> }> {
const errors: Array<{ index: number; error: Error }> = [];
while (index < items.length) {
const i = index++;
Iif (i >= items.length) break;
let timer: ReturnType<typeof setTimeout>;
try {
await Promise.race([
processItem(items[i]!, i),
new Promise<void>((_, reject) => {
timer = setTimeout(() => reject(new Error(`Timeout after ${timeout}ms`)), timeout);
}),
]);
} catch (err) {
errors.push({ index: i, error: err instanceof Error ? err : new Error(String(err)) });
} finally {
clearTimeout(timer!);
}
completed++;
if (options.onProgress) {
options.onProgress(completed, total);
}
}
return { errors };
}
const results = await Promise.all(workers);
const allErrors: Array<{ index: number; error: Error }> = [];
for (const r of results) {
allErrors.push(...r.errors);
}
return { errors: allErrors };
}
async function importNodeCores(): Promise<number> {
try {
const os = await import('node:os');
return os.cpus().length;
} catch {
return 4; // browser default
}
}
|