Skip to content

Commit 9308e90

Browse files
jenningsandersonjenningsanderson
committed
sort of working better
1 parent 619b553 commit 9308e90

3 files changed

Lines changed: 385 additions & 587 deletions

File tree

‎js/explorer.js‎

Lines changed: 164 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,27 @@
5050
let initialized = false;
5151
let drawHandler = null;
5252

53+
// ============================================================
54+
// Coordinate clamping helpers
55+
// ============================================================
56+
function clampLat(v) { return Math.max(-90, Math.min(90, v)); }
57+
function clampLon(v) { return Math.max(-180, Math.min(180, v)); }
58+
function clampBbox(bbox) {
59+
return {
60+
west: clampLon(bbox.west),
61+
south: clampLat(bbox.south),
62+
east: clampLon(bbox.east),
63+
north: clampLat(bbox.north)
64+
};
65+
}
66+
5367
// ============================================================
5468
// Initialize Explorer (called after setup completes)
5569
// ============================================================
5670
window.initExplorer = function () {
5771
if (initialized) {
58-
const type = document.getElementById('type-select').value;
59-
loadTypeBboxes(type);
72+
var types = getSelectedTypes();
73+
if (types.length > 0) loadTypeBboxes(types);
6074
return;
6175
}
6276
initialized = true;
@@ -67,19 +81,28 @@
6781
maxZoom: 19
6882
}).addTo(map);
6983

70-
document.getElementById('type-select').addEventListener('change', function () {
71-
loadTypeBboxes(this.value);
84+
// Re-render map when explorer section is toggled open
85+
var explorerDetails = document.querySelector('#explorer-section details');
86+
if (explorerDetails) {
87+
explorerDetails.addEventListener('toggle', function () {
88+
if (this.open) {
89+
setTimeout(function () { map.invalidateSize(); }, 150);
90+
}
91+
});
92+
}
93+
94+
document.getElementById('type-checkboxes').addEventListener('change', function () {
95+
var types = getSelectedTypes();
96+
if (types.length > 0) loadTypeBboxes(types);
7297
});
7398

7499
document.getElementById('bbox-select').addEventListener('change', function () {
75100
const val = this.value;
76-
const customInputs = document.getElementById('custom-bbox-inputs');
77101
if (drawHandler) {
78102
drawHandler.disable();
79103
drawHandler = null;
80104
}
81105
if (val === 'custom') {
82-
customInputs.style.display = 'flex';
83106
drawHandler = new L.Draw.Rectangle(map, {
84107
shapeOptions: {
85108
color: '#e65100',
@@ -93,23 +116,30 @@
93116
drawHandler.enable();
94117
return;
95118
}
96-
customInputs.style.display = 'none';
97119
if (val === '') {
120+
document.getElementById('bbox-west').value = '';
121+
document.getElementById('bbox-south').value = '';
122+
document.getElementById('bbox-east').value = '';
123+
document.getElementById('bbox-north').value = '';
98124
clearQueryBbox();
99125
return;
100126
}
101127
const preset = PRESETS[val];
102128
if (preset) {
129+
document.getElementById('bbox-west').value = preset.west;
130+
document.getElementById('bbox-south').value = preset.south;
131+
document.getElementById('bbox-east').value = preset.east;
132+
document.getElementById('bbox-north').value = preset.north;
103133
applyQueryBbox(preset);
104134
}
105135
});
106136

107137
map.on(L.Draw.Event.CREATED, function (e) {
108138
const bounds = e.layer.getBounds();
109-
const west = parseFloat(bounds.getWest().toFixed(3));
110-
const south = parseFloat(bounds.getSouth().toFixed(3));
111-
const east = parseFloat(bounds.getEast().toFixed(3));
112-
const north = parseFloat(bounds.getNorth().toFixed(3));
139+
const west = clampLon(parseFloat(bounds.getWest().toFixed(3)));
140+
const south = clampLat(parseFloat(bounds.getSouth().toFixed(3)));
141+
const east = clampLon(parseFloat(bounds.getEast().toFixed(3)));
142+
const north = clampLat(parseFloat(bounds.getNorth().toFixed(3)));
113143

114144
document.getElementById('bbox-west').value = west;
115145
document.getElementById('bbox-south').value = south;
@@ -121,14 +151,22 @@
121151
});
122152

123153
document.getElementById('btn-apply-custom-bbox').addEventListener('click', function () {
124-
const west = parseFloat(document.getElementById('bbox-west').value);
125-
const south = parseFloat(document.getElementById('bbox-south').value);
126-
const east = parseFloat(document.getElementById('bbox-east').value);
127-
const north = parseFloat(document.getElementById('bbox-north').value);
154+
var west = parseFloat(document.getElementById('bbox-west').value);
155+
var south = parseFloat(document.getElementById('bbox-south').value);
156+
var east = parseFloat(document.getElementById('bbox-east').value);
157+
var north = parseFloat(document.getElementById('bbox-north').value);
128158
if (isNaN(west) || isNaN(south) || isNaN(east) || isNaN(north)) {
129159
alert('Please enter valid numeric coordinates for all four bbox fields.');
130160
return;
131161
}
162+
west = clampLon(west);
163+
east = clampLon(east);
164+
south = clampLat(south);
165+
north = clampLat(north);
166+
document.getElementById('bbox-west').value = west;
167+
document.getElementById('bbox-south').value = south;
168+
document.getElementById('bbox-east').value = east;
169+
document.getElementById('bbox-north').value = north;
132170
applyQueryBbox({ west, south, east, north });
133171
});
134172

@@ -153,53 +191,53 @@
153191
drawHandler.enable();
154192
});
155193

156-
// Insert into Query button
157-
document.getElementById('btn-copy-to-query').addEventListener('click', function () {
158-
const sqlOutput = document.getElementById('bbox-sql-output');
159-
const sql = sqlOutput.textContent;
160-
if (!sql || sql.startsWith('--')) return;
161-
const queryEditor = document.getElementById('query-editor');
162-
queryEditor.value = sql;
163-
});
164-
165-
loadTypeBboxes('place');
194+
loadTypeBboxes(getSelectedTypes());
166195
};
167196

168197
// ============================================================
169-
// Load file bboxes for a collection type
198+
// Get selected types from checkboxes
199+
// ============================================================
200+
function getSelectedTypes() {
201+
return Array.from(document.querySelectorAll('#type-checkboxes input:checked')).map(function (cb) { return cb.value; });
202+
}
203+
204+
// ============================================================
205+
// Load file bboxes for one or more collection types
170206
// ============================================================
171-
async function loadTypeBboxes(type) {
207+
async function loadTypeBboxes(types) {
172208
const conn = window.duckdbConn;
173209
if (!conn || !window.setupComplete) return;
210+
if (!Array.isArray(types) || types.length === 0) return;
174211

175212
const fileListEl = document.getElementById('file-list');
176-
const sqlOutput = document.getElementById('bbox-sql-output');
177213

178214
clearFileRectangles();
179215
clearQueryBbox();
180216
fileListEl.innerHTML = '';
181-
sqlOutput.textContent = '-- Select a bounding box to see the generated SQL';
182217

183218
try {
184219
const t0 = performance.now();
220+
const inList = types.map(function (t) { return "'" + t + "'"; }).join(', ');
185221
const result = await conn.query(`
186222
SELECT
187223
id,
224+
collection,
188225
assets.aws.href AS href,
189226
bbox.xmin AS xmin,
190227
bbox.ymin AS ymin,
191228
bbox.xmax AS xmax,
192229
bbox.ymax AS ymax
193230
FROM overture_collections
194-
WHERE collection = '${type}'
195-
ORDER BY id
231+
WHERE collection IN (${inList})
232+
ORDER BY collection, id
196233
`);
197234

198235
allFiles = result.toArray().map(r => {
199236
const obj = r.toJSON();
200237
const href = String(obj.href);
201238
return {
202239
id: String(obj.id),
240+
collection: String(obj.collection),
203241
filename: href.split('/').pop() || String(obj.id),
204242
href: href,
205243
xmin: Number(obj.xmin),
@@ -210,7 +248,7 @@
210248
});
211249

212250
const elapsed = Math.round(performance.now() - t0);
213-
console.log(`[explorer] Loaded ${allFiles.length} files for type '${type}' (${elapsed} ms)`);
251+
console.log(`[explorer] Loaded ${allFiles.length} files for types [${types.join(', ')}] (${elapsed} ms)`);
214252

215253
renderFileBboxes(allFiles);
216254
} catch (err) {
@@ -310,77 +348,88 @@
310348
map.fitBounds([[south, west], [north, east]], { padding: [50, 50] });
311349

312350
const fileListEl = document.getElementById('file-list');
313-
const sqlOutput = document.getElementById('bbox-sql-output');
314351
const intersectingFiles = intersecting.map(r => r.fileData);
315352

316353
if (intersectingFiles.length === 0) {
317354
fileListEl.innerHTML = '<p class="placeholder" style="padding:1rem;">No files intersect this bounding box.</p>';
318-
sqlOutput.textContent = '-- No files intersect this bounding box';
355+
document.getElementById('query-from-where').textContent = '-- No files intersect this bounding box';
319356
return;
320357
}
321358

322-
// Query parquet_file_metadata for features and row groups
323-
let metaMap = {};
324-
try {
325-
const urls = intersectingFiles.map(f => `'${f.href}'`).join(', ');
326-
const metaResult = await window.duckdbConn.query(`
327-
SELECT file_name, num_rows, num_row_groups
328-
FROM parquet_file_metadata([${urls}])
329-
`);
330-
const metaRows = metaResult.toArray().map(r => r.toJSON());
331-
for (const m of metaRows) {
332-
metaMap[m.file_name] = { num_rows: Number(m.num_rows), num_row_groups: Number(m.num_row_groups) };
333-
}
334-
} catch (err) {
335-
console.warn('[explorer] Could not fetch parquet metadata:', err);
336-
}
337-
338-
// Build STAC link info
339-
const type = document.getElementById('type-select').value;
340-
const theme = TYPE_TO_THEME[type] || type;
359+
// Build STAC link info — group by collection for multi-type support
360+
const types = getSelectedTypes();
341361
const release = window.latestVersion;
342362

343-
let totalFeatures = 0;
344-
let totalRowGroups = 0;
363+
// Render table with cache buttons
364+
renderFileTable(fileListEl, intersectingFiles, release);
345365

366+
// Show FROM/WHERE SQL fragment in the locked query zone
367+
const urlList = intersectingFiles.map(f => ` '${f.href}'`).join(',\n');
368+
const fromWhereFragment =
369+
`FROM read_parquet([\n${urlList}\n])\nWHERE bbox.xmin <= ${east}\n AND bbox.xmax >= ${west}\n AND bbox.ymin <= ${north}\n AND bbox.ymax >= ${south}`;
370+
document.getElementById('query-from-where').textContent = fromWhereFragment;
371+
}
372+
373+
// ============================================================
374+
// Render file table
375+
// ============================================================
376+
function renderFileTable(fileListEl, files, release) {
346377
let html = `<p style="padding:0.5rem 0;font-size:0.85rem;color:var(--om-color-text-muted);">
347-
<strong>${intersectingFiles.length}</strong> of ${fileRectangles.length} files intersect this bounding box
378+
<strong>${files.length}</strong> of ${fileRectangles.length} files intersect this bounding box
348379
</p>`;
349380
html += '<div class="table-wrapper" style="max-height:none;"><table><thead><tr>';
350-
html += '<th>Partition</th><th>Features</th><th>Row Groups</th><th>Bbox</th>';
381+
html += '<th>Type</th><th>Partition</th><th>Bbox</th><th>Cache</th>';
351382
html += '</tr></thead><tbody>';
352383

353-
for (const f of intersectingFiles) {
384+
for (const f of files) {
354385
const shortName = f.filename.match(/^(part-\d+)/)?.[1] || f.filename;
355-
const stacUrl = `https://stac.overturemaps.org/${release}/${theme}/${type}/${f.id}/${f.id}.json`;
356-
const meta = metaMap[f.href] || {};
357-
const numRows = meta.num_rows ?? '\u2014';
358-
const numRG = meta.num_row_groups ?? '\u2014';
359-
if (typeof numRows === 'number') totalFeatures += numRows;
360-
if (typeof numRG === 'number') totalRowGroups += numRG;
386+
const theme = TYPE_TO_THEME[f.collection] || f.collection;
387+
const stacUrl = `https://stac.overturemaps.org/${release}/${theme}/${f.collection}/${f.id}/${f.id}.json`;
361388
html += '<tr>';
389+
html += `<td>${escapeHtml(f.collection)}</td>`;
362390
html += `<td title="${escapeHtml(f.href)}"><a href="${escapeHtml(stacUrl)}" target="_blank" style="color:var(--om-link-color);">${escapeHtml(shortName)}</a></td>`;
363-
html += `<td style="text-align:right;">${typeof numRows === 'number' ? numRows.toLocaleString() : numRows}</td>`;
364-
html += `<td style="text-align:right;">${typeof numRG === 'number' ? numRG.toLocaleString() : numRG}</td>`;
365391
html += `<td style="font-size:0.7rem;">${f.xmin.toFixed(1)}, ${f.ymin.toFixed(1)}, ${f.xmax.toFixed(1)}, ${f.ymax.toFixed(1)}</td>`;
392+
html += `<td><button class="btn btn-primary cache-run-btn" data-href="${escapeHtml(f.href)}" style="font-size:0.7rem;padding:0.15rem 0.5rem;">Cache</button></td>`;
366393
html += '</tr>';
367394
}
368395

369-
// Total row
370-
html += '<tr style="font-weight:600;border-top:2px solid var(--om-color-border);">';
371-
html += `<td>Total (${intersectingFiles.length} files)</td>`;
372-
html += `<td style="text-align:right;">${totalFeatures.toLocaleString()}</td>`;
373-
html += `<td style="text-align:right;">${totalRowGroups.toLocaleString()}</td>`;
374-
html += '<td></td>';
375-
html += '</tr>';
376-
377-
html += '</tbody></table></div>';
396+
html += '</tbody><tfoot><tr>';
397+
html += '<td colspan="3"></td>';
398+
html += '<td><button id="btn-cache-all" class="btn btn-primary" style="font-size:0.7rem;padding:0.15rem 0.5rem;">Cache All</button></td>';
399+
html += '</tr></tfoot></table></div>';
378400
fileListEl.innerHTML = html;
379401

380-
// Show full resolved SQL with proper bbox intersection filter
381-
const urlList = intersectingFiles.map(f => ` '${f.href}'`).join(',\n');
382-
sqlOutput.textContent =
383-
`SELECT *\nFROM read_parquet([\n${urlList}\n])\nWHERE bbox.xmin <= ${east}\n AND bbox.xmax >= ${west}\n AND bbox.ymin <= ${north}\n AND bbox.ymax >= ${south}`;
402+
// Wire up individual Cache buttons
403+
fileListEl.querySelectorAll('.cache-run-btn').forEach(function (btn) {
404+
btn.addEventListener('click', function () {
405+
runCacheQuery(this);
406+
});
407+
});
408+
409+
// Wire up Cache All button
410+
var btnAll = document.getElementById('btn-cache-all');
411+
if (btnAll) {
412+
btnAll.addEventListener('click', async function () {
413+
this.disabled = true;
414+
this.innerHTML = '<span class="btn-spinner"></span>';
415+
var buttons = Array.from(fileListEl.querySelectorAll('.cache-run-btn:not(:disabled):not(#btn-cache-all)'));
416+
var batchSize = 6;
417+
var tAll = performance.now();
418+
console.log(`[cache] Starting Cache All: ${buttons.length} files in batches of ${batchSize}`);
419+
for (var i = 0; i < buttons.length; i += batchSize) {
420+
var batch = buttons.slice(i, i + batchSize);
421+
var batchNum = Math.floor(i / batchSize) + 1;
422+
var totalBatches = Math.ceil(buttons.length / batchSize);
423+
console.log(`[cache] Batch ${batchNum}/${totalBatches} (${batch.length} files)...`);
424+
var tBatch = performance.now();
425+
await Promise.all(batch.map(function (b) { return runCacheQuery(b); }));
426+
console.log(`[cache] Batch ${batchNum} done (${Math.round(performance.now() - tBatch)} ms)`);
427+
}
428+
var totalElapsed = Math.round(performance.now() - tAll);
429+
console.log(`[cache] Cache All complete: ${buttons.length} files cached in ${totalElapsed} ms`);
430+
this.textContent = 'Done';
431+
});
432+
}
384433
}
385434

386435
// ============================================================
@@ -402,10 +451,45 @@
402451
});
403452
}
404453

405-
document.getElementById('bbox-sql-output').textContent = '-- Select a bounding box to see the generated SQL';
454+
document.getElementById('query-from-where').textContent = '-- Select a type and bounding box in the Explorer above';
406455
document.getElementById('file-list').innerHTML = '';
407456
}
408457

458+
async function runCacheQuery(btn) {
459+
var href = btn.dataset.href;
460+
var localName = href.split('/').pop();
461+
462+
btn.disabled = true;
463+
btn.innerHTML = '<span class="btn-spinner"></span>';
464+
465+
try {
466+
// Register file with HTTP protocol for efficient range requests
467+
// DuckDBDataProtocol.HTTP = 4, directIO = false (don't cache whole file)
468+
if (window.duckdbDB) {
469+
var tReg = performance.now();
470+
await window.duckdbDB.registerFileURL(localName, href, 4, false);
471+
console.log(`[cache] Registered ${localName} with HTTP protocol (${Math.round(performance.now() - tReg)} ms)`);
472+
}
473+
474+
var tQuery = performance.now();
475+
console.log(`[cache] Fetching parquet footer for ${localName}...`);
476+
var queryTarget = window.duckdbDB ? localName : href;
477+
await window.duckdbConn.query("SELECT 1 FROM read_parquet('" + queryTarget + "') LIMIT 0");
478+
var elapsed = Math.round(performance.now() - tQuery);
479+
console.log(`[cache] \u2713 Cached ${localName} (${elapsed} ms)`);
480+
481+
btn.textContent = '\u2713';
482+
btn.style.opacity = '0.5';
483+
btn.style.cursor = 'default';
484+
} catch (err) {
485+
var elapsed = Math.round(performance.now() - (tQuery || performance.now()));
486+
console.error(`[cache] \u2717 Failed ${localName} after ${elapsed} ms:`, err.message);
487+
btn.textContent = '\u2717';
488+
btn.style.background = 'var(--om-color-error)';
489+
btn.title = err.message;
490+
}
491+
}
492+
409493
// ============================================================
410494
// Clear file rectangles
411495
// ============================================================

0 commit comments

Comments
 (0)