Skip to content

Commit 1c21473

Browse files
committed
Added cesium view, adjusted tutorial
1 parent 5ca1ea4 commit 1c21473

File tree

7 files changed

+296
-44
lines changed

7 files changed

+296
-44
lines changed

_quarto.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ website:
4444
href: tutorials/index.qmd
4545
- text: "iSamples Parquet Tutorial"
4646
href: tutorials/parquet.qmd
47+
- text: "Cesium View"
48+
href: tutorials/parquet_cesium.qmd
4749

4850

4951

@@ -70,7 +72,7 @@ format:
7072
number-sections: true
7173
anchor-sections: false
7274
number-depth: 8
73-
theme: sandstone
75+
theme: cosmo
7476
css: styles.css
7577

7678
#filters:

models/assets/geome_mapping.drawio.svg

Lines changed: 0 additions & 1 deletion
This file was deleted.

models/assets/geome_mapping_1.drawio.svg

Lines changed: 0 additions & 1 deletion
This file was deleted.

models/assets/geome_mapping_2.drawio.svg

Lines changed: 0 additions & 1 deletion
This file was deleted.

pyproject.toml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
[project]
2+
name = "isamplesorg-github-io"
3+
version = "0.5.0"
4+
description = "Documentation for the iSamples project."
5+
authors = [{ name = "datadavev", email = "605409+datadavev@users.noreply.github.com" }]
6+
requires-python = ">=3.9,<3.13"
7+
readme = "README.md"
8+
license = "MIT"
9+
dependencies = [
10+
"rdflib>=6.2.0,<7",
11+
"click>=8.1.3,<9",
12+
"pandoc-plantuml-filter>=0.1.2,<0.2",
13+
"graphviz>=0.20.1,<0.21",
14+
"itables>=1.3.0,<2",
15+
"requests>=2.28.1,<3",
16+
"nbformat>=5.9.2,<6",
17+
"nbclient>=0.8.0,<0.9",
18+
"jupyter>=1.0.0,<2",
19+
"anywidget[dev]>=0.7.0,<0.8",
20+
"matplotlib>=3.8.0,<4",
21+
"ipyleaflet>=0.17.4,<0.18",
22+
"mapwidget>=0.1.2,<0.2",
23+
"pygithub>=2.4.0,<3",
24+
]
25+
26+
[tool.hatch.build.targets.sdist]
27+
include = ["isamplesorg"]
28+
29+
[tool.hatch.build.targets.wheel]
30+
include = ["isamplesorg"]
31+
32+
[build-system]
33+
requires = ["hatchling"]
34+
build-backend = "hatchling.build"

tutorials/parquet.qmd

Lines changed: 28 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -4,61 +4,49 @@ title: "Parquet"
44

55
Let's query Eric's parquet file using duckdb+parquet
66

7-
8-
simpler query:
9-
107
```{ojs}
11-
//| echo: true
12-
13-
// Import Observable's libraries
14-
import {DuckDBClient} from "@observablehq/duckdb"
8+
//| code-fold: true
9+
//| output: false
10+
//
1511
1612
// Create a DuckDB instance
1713
db = DuckDBClient.of()
1814
1915
// Set the Parquet file path
2016
parquet_path = 'https://storage.googleapis.com/opencontext-parquet/oc_isamples_pqg.parquet'
2117
22-
// For testing, use a smaller dataset or limit rows
23-
// Option 1: Use LIMIT to reduce data transferred
24-
viewof testResults = {
25-
// Show loading indicator
26-
const loadingElement = html`<div>Running query...</div>`;
27-
document.body.appendChild(loadingElement);
28-
29-
try {
30-
// Test with a small LIMIT to verify connection works
31-
const data = await db.query(`
32-
SELECT otype, pid
33-
FROM read_parquet('${parquet_path}')
34-
LIMIT 10
35-
`);
36-
return Inputs.table(data);
37-
} finally {
38-
// Remove loading indicator when done (whether success or error)
39-
loadingElement.remove();
40-
}
18+
row_count = {
19+
const result = await db.queryRow(`select count(*) as n from read_parquet('${parquet_path}');`);
20+
return result.n;
4121
}
4222
43-
```
44-
45-
now the full query
23+
results = {
24+
const data = await db.query(`SELECT COUNT(*) as count, otype FROM read_parquet('${parquet_path}') GROUP BY otype ORDER BY count DESC`);
25+
document.getElementById("loading_1").hidden = true;
26+
return Inputs.table(data);
27+
}
4628
29+
rows1k = {
30+
const data = await db.query(`SELECT row_id, pid, otype, label FROM read_parquet('${parquet_path}') limit 1000`);
31+
document.getElementById("loading_2").hidden = true;
32+
return Inputs.table(data);
33+
}
34+
```
4735

4836
```{ojs}
49-
//| echo: true
50-
37+
//| code-fold: true
5138
52-
// Query the Parquet file
53-
viewof results = Inputs.table(
54-
await db.query(`
55-
SELECT COUNT(pid) as count, otype
56-
FROM read_parquet('${parquet_path}')
57-
GROUP BY otype
58-
ORDER BY count DESC
59-
`)
60-
)
39+
md`There are ${row_count} rows in the source <code>${parquet_path}</code>.`
6140
```
6241

42+
<div>
43+
<div id="loading_1">Loading type counts...</div>
44+
${results}
45+
</div>
6346

47+
The first 1000 rows:
6448

49+
<div>
50+
<div id="loading_2">Loading...</div>
51+
${rows1k}
52+
</div>

tutorials/parquet_cesium.qmd

Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
---
2+
title: Using Cesium for display of remote parquet.
3+
categories: [parquet, spatial, recipe]
4+
---
5+
6+
This page renders points from an iSamples parquet file on cesium using point primitives.
7+
8+
<script src="https://cesium.com/downloads/cesiumjs/releases/1.127/Build/Cesium/Cesium.js"></script>
9+
<link href="https://cesium.com/downloads/cesiumjs/releases/1.127/Build/Cesium/Widgets/widgets.css" rel="stylesheet"></link>
10+
<style>
11+
div.cesium-topleft {
12+
display: block;
13+
position: absolute;
14+
background: #00000099;
15+
color: white;
16+
height: auto;
17+
z-index: 999;
18+
}
19+
#cesiumContainer {
20+
aspect-ratio: 1/1;
21+
}
22+
</style>
23+
24+
```{ojs}
25+
//| output: false
26+
Cesium.Ion.defaultAccessToken = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiIwNzk3NjkyMy1iNGI1LTRkN2UtODRiMy04OTYwYWE0N2M3ZTkiLCJpZCI6Njk1MTcsImlhdCI6MTYzMzU0MTQ3N30.e70dpNzOCDRLDGxRguQCC-tRzGzA-23Xgno5lNgCeB4';
27+
28+
viewof parquet_path = Inputs.text({label:"Source", value:"https://storage.googleapis.com/opencontext-parquet/oc_isamples_pqg.parquet", width:"100%", submit:true});
29+
```
30+
31+
```{ojs}
32+
//| code-fold: true
33+
// Import Observable's libraries
34+
import {DuckDBClient} from "@observablehq/duckdb"
35+
36+
// Create a DuckDB instance
37+
db = DuckDBClient.of();
38+
39+
40+
async function loadData(query, params=[], waiting_id=null) {
41+
// Get loading indicator
42+
const waiter = document.getElementById(waiting_id);
43+
if (waiter) {
44+
waiter.hidden = false;
45+
}
46+
try {
47+
// Run the (slow) query
48+
const _results = await db.query(query, ...params);
49+
return _results;
50+
} catch (error) {
51+
if (waiter) {
52+
waiter.innerHtml = `<pre>${error}</pre>`;
53+
}
54+
return null;
55+
} finally {
56+
// Hide the waiter (if there is one)
57+
if (waiter) {
58+
waiter.hidden = true;
59+
}
60+
}
61+
}
62+
63+
locations = {
64+
// get the content form the parquet file
65+
const query = `SELECT row_id, pid, latitude, longitude FROM read_parquet('${parquet_path}') WHERE otype='GeospatialCoordLocation'`;
66+
const data = await loadData(query, [], "loading_1");
67+
68+
// create point primitives for cesium display
69+
const scalar = new Cesium.NearFarScalar(1.5e2, 2, 8.0e6, 0.2);
70+
const color = Cesium.Color.PINK;
71+
const point_size = 4;
72+
for (const row of data) {
73+
content.points.add({
74+
id: row.pid,
75+
// https://cesium.com/learn/cesiumjs/ref-doc/Cartesian3.html#.fromDegrees
76+
position: Cesium.Cartesian3.fromDegrees(
77+
row.longitude, //longitude
78+
row.latitude, //latitude
79+
0,//randomCoordinateJitter(10.0, 10.0), //elevation, m
80+
),
81+
row_id: row.row_id,
82+
pixelSize: point_size,
83+
color: color,
84+
scaleByDistance: scalar,
85+
});
86+
}
87+
content.enableTracking();
88+
return data;
89+
}
90+
91+
92+
function createShowPrimitive(viewer) {
93+
return function(movement) {
94+
// Get the point at the mouse end position
95+
const selectPoint = viewer.viewer.scene.pick(movement.endPosition);
96+
97+
// Clear the current selection, if there is one and it is different to the selectPoint
98+
if (viewer.currentSelection !== null) {
99+
//console.log(`selected.p ${viewer.currentSelection}`)
100+
if (Cesium.defined(selectPoint) && selectPoint !== viewer.currentSelection) {
101+
console.log(`selected.p 2 ${viewer.currentSelection}`)
102+
viewer.currentSelection.primitive.pixelSize = 4;
103+
viewer.currentSelection.primitive.outlineColor = Cesium.Color.TRANSPARENT;
104+
viewer.currentSelection.outlineWidth = 0;
105+
viewer.currentSelection = null;
106+
}
107+
}
108+
109+
// If selectPoint is valid and no currently selected point
110+
if (Cesium.defined(selectPoint) && selectPoint.hasOwnProperty("primitive")) {
111+
//console.log(`showPrimitiveId ${selectPoint.id}`);
112+
const carto = Cesium.Cartographic.fromCartesian(selectPoint.primitive.position)
113+
viewer.pointLabel.position = selectPoint.primitive.position;
114+
viewer.pointLabel.label.show = true;
115+
//viewer.pointLabel.label.text = `id:${selectPoint.id}, ${carto}`;
116+
viewer.pointLabel.label.text = `${selectPoint.id}`;
117+
selectPoint.primitive.pixelSize = 20;
118+
selectPoint.primitive.outlineColor = Cesium.Color.YELLOW;
119+
selectPoint.primitive.outlineWidth = 3;
120+
viewer.currentSelection = selectPoint;
121+
} else {
122+
viewer.pointLabel.label.show = false;
123+
}
124+
}
125+
}
126+
127+
class CView {
128+
constructor(target) {
129+
this.viewer = new Cesium.Viewer(
130+
target, {
131+
timeline: false,
132+
animation: false,
133+
baseLayerPicker: false,
134+
fullscreenElement: target,
135+
terrain: Cesium.Terrain.fromWorldTerrain()
136+
});
137+
this.currentSelection = null;
138+
this.point_size = 1;
139+
this.n_points = 0;
140+
// https://cesium.com/learn/cesiumjs/ref-doc/PointPrimitiveCollection.html
141+
this.points = new Cesium.PointPrimitiveCollection();
142+
this.viewer.scene.primitives.add(this.points);
143+
144+
this.pointLabel = this.viewer.entities.add({
145+
label: {
146+
show: false,
147+
showBackground: true,
148+
font: "14px monospace",
149+
horizontalOrigin: Cesium.HorizontalOrigin.LEFT,
150+
verticalOrigin: Cesium.VerticalOrigin.BOTTOM,
151+
pixelOffset: new Cesium.Cartesian2(15, 0),
152+
// this attribute will prevent this entity clipped by the terrain
153+
disableDepthTestDistance: Number.POSITIVE_INFINITY,
154+
text:"",
155+
},
156+
});
157+
158+
this.pickHandler = new Cesium.ScreenSpaceEventHandler(this.viewer.scene.canvas);
159+
// Can also do this rather than wait for the points to be generated
160+
//this.pickHandler.setInputAction(createShowPrimitive(this), Cesium.ScreenSpaceEventType.MOUSE_MOVE);
161+
162+
this.selectHandler = new Cesium.ScreenSpaceEventHandler(this.viewer.scene.canvas);
163+
this.selectHandler.setInputAction((e) => {
164+
const selectPoint = this.viewer.scene.pick(e.position);
165+
if (Cesium.defined(selectPoint) && selectPoint.hasOwnProperty("primitive")) {
166+
mutable clickedPointId = selectPoint.id;
167+
}
168+
},Cesium.ScreenSpaceEventType.LEFT_CLICK);
169+
170+
}
171+
172+
enableTracking() {
173+
this.pickHandler.setInputAction(createShowPrimitive(this), Cesium.ScreenSpaceEventType.MOUSE_MOVE);
174+
}
175+
}
176+
177+
content = new CView("cesiumContainer");
178+
179+
async function getGeoRecord(pid) {
180+
if (pid === null || pid ==="" || pid == "unset") {
181+
return "unset";
182+
}
183+
const q = `SELECT row_id, pid, otype, latitude, longitude FROM read_parquet('${parquet_path}') WHERE otype='GeospatialCoordLocation' AND pid=?`;
184+
const result = await db.queryRow(q, [pid]);
185+
return result;
186+
}
187+
188+
mutable clickedPointId = "unset";
189+
selectedGeoRecord = await getGeoRecord(clickedPointId);
190+
191+
md`Retrieved ${pointdata.length} locations from ${parquet_path}.`;
192+
```
193+
194+
::: {.panel-tabset}
195+
196+
## Map
197+
198+
<div id="cesiumContainer"></div>
199+
200+
## Data
201+
202+
<div id="loading_1">Loading...</div>
203+
204+
```{ojs}
205+
//| code-fold: true
206+
207+
viewof pointdata = {
208+
const data_table = Inputs.table(locations, {
209+
header: {
210+
row_id:"Row ID",
211+
pid: "PID",
212+
latitude: "Latitude",
213+
longitude: "Longitude"
214+
},
215+
});
216+
return data_table;
217+
}
218+
```
219+
220+
:::
221+
222+
The click point ID is "${clickedPointId}".
223+
224+
```{ojs}
225+
//| echo: false
226+
md`\`\`\`
227+
${JSON.stringify(selectedGeoRecord, null, 2)}
228+
\`\`\`
229+
`
230+
```
231+

0 commit comments

Comments
 (0)