Skip to content

Commit 3aaf914

Browse files
jrentlezjrentlez
committed
(TODO:SPLIT THIS DOWN) big commit that fixes almost everything
1 parent 5268475 commit 3aaf914

51 files changed

Lines changed: 1261 additions & 490 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/interpreter/test/assets/broken-model.jv‎

Lines changed: 38 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -28,27 +28,50 @@ pipeline CarsPipeline {
2828
write: ["name"];
2929
}
3030

31+
valuetype Car {
32+
property name oftype text;
33+
property mpg oftype decimal;
34+
property cyl oftype integer;
35+
property disp oftype decimal;
36+
property hp oftype integer;
37+
property drat oftype decimal;
38+
property wt oftype decimal;
39+
property qsec oftype decimal;
40+
property vs oftype integer;
41+
property am oftype integer;
42+
property gear oftype integer;
43+
property carb oftype integer;
44+
}
45+
46+
transform CarParser {
47+
from r oftype Collection<text>;
48+
to car oftype Car;
49+
50+
car: {
51+
name: asText (r cellInColumn "name"),
52+
mpg: asDecimal (r cellInColumn "mpg"),
53+
cyl: asInteger (r cellInColumn 2),
54+
disp: asDecimal (r cellInColumn 3),
55+
hp: asInteger (r cellInColumn "hp"),
56+
drat: asDecimal (r cellInColumn "drat"),
57+
wt: asDecimal (r cellInColumn "wt"),
58+
qsec: asDecimal (r cellInColumn "qsec"),
59+
vs: asInteger (r cellInColumn "vs"),
60+
am: asInteger (r cellInColumn "am"),
61+
gear: asInteger (r cellInColumn "gear"),
62+
carb: asInteger (r cellInColumn "carb")
63+
};
64+
}
65+
3166
block CarsTableInterpreter oftype TableInterpreter {
3267
header: true;
33-
columns: [
34-
"name" oftype text,
35-
"mpg" oftype decimal,
36-
"cyl" oftype integer,
37-
"disp" oftype decimal,
38-
"hp" oftype integer,
39-
"drat" oftype decimal,
40-
"wt" oftype decimal,
41-
"qsec" oftype decimal,
42-
"vs" oftype integer,
43-
"am" oftype integer,
44-
"gear" oftype integer,
45-
"carb" oftype integer
46-
];
68+
columns: Car;
69+
parseWith: CarParser;
4770
}
4871

4972
block CarsLoader oftype SQLiteLoader {
5073
table: "Cars";
5174
file: "./cars.sqlite";
5275
}
5376

54-
}
77+
}

‎example/cars.jv‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,12 +88,33 @@ pipeline CarsPipeline {
8888
property carb oftype integer;
8989
}
9090

91+
transform CarParser {
92+
from r oftype Collection<text>;
93+
to car oftype Car;
94+
95+
car: {
96+
name: asText (r cellInColumn "name"),
97+
mpg: asDecimal (r cellInColumn "mpg"),
98+
cyl: asInteger (r cellInColumn 2),
99+
disp: asDecimal (r cellInColumn 3),
100+
hp: asInteger (r cellInColumn "hp"),
101+
drat: asDecimal (r cellInColumn "drat"),
102+
wt: asDecimal (r cellInColumn "wt"),
103+
qsec: asDecimal (r cellInColumn "qsec"),
104+
vs: asInteger (r cellInColumn "vs"),
105+
am: asInteger (r cellInColumn "am"),
106+
gear: asInteger (r cellInColumn "gear"),
107+
carb: asInteger (r cellInColumn "carb")
108+
};
109+
}
110+
91111
// 15. As a next step, we interpret the sheet as a table, using the valuetype
92112
// defined above. Rows that include values that are not valid according to the
93113
// their value types are dropped automatically.
94114
block CarsTableInterpreter oftype TableInterpreter {
95115
header: true;
96116
columns: Car;
117+
parseWith: CarParser;
97118
}
98119

99120
// 16. As a last step, we load the table into a sink, here into a sqlite file.

‎example/electric-vehicles.jv‎

Lines changed: 50 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -49,30 +49,56 @@ pipeline ElectricVehiclesPipeline {
4949

5050
block ElectricVehiclesCSVInterpreter oftype CSVInterpreter { }
5151

52+
valuetype ElectricVehicle {
53+
property vin oftype VehicleIdentificationNumber10;
54+
property county oftype text;
55+
property city oftype text;
56+
property state oftype UsStateCode;
57+
property postal oftype text;
58+
property modelYear oftype integer;
59+
property make oftype text;
60+
property model oftype text;
61+
property evType oftype text;
62+
property cafvEligibility oftype text;
63+
property electricRange oftype integer;
64+
property baseMSRP oftype integer;
65+
property legislativeDistrict oftype text;
66+
property dolID oftype integer;
67+
property location oftype text;
68+
property utility oftype text;
69+
property censusTract oftype text;
70+
}
71+
72+
transform ElectricVehicleParser {
73+
from r oftype Collection<text>;
74+
to ev oftype ElectricVehicle;
75+
76+
ev: {
77+
vin: r cellInColumn "VIN (1-10)",
78+
county: asText (r cellInColumn "County"),
79+
city: asText (r cellInColumn "City"),
80+
state: asText (r cellInColumn "State"),
81+
postal: asText (r cellInColumn "Postal Code"),
82+
modelYear: asInteger (r cellInColumn "Model Year"),
83+
make: asText (r cellInColumn "Make"),
84+
model: asText (r cellInColumn "Model"),
85+
evType: asText (r cellInColumn "Electric Vehicle Type"),
86+
cafvEligibility: asText (r cellInColumn "Clean Alternative Fuel Vehicle (CAFV) Eligibility"),
87+
electricRange: asInteger (r cellInColumn "Electric Range"),
88+
baseMSRP: asInteger (r cellInColumn "Base MSRP"),
89+
legislativeDistrict: asText (r cellInColumn "LegislativeDistrict"),
90+
dolID: asInteger (r cellInColumn "DOL Vehicle ID"),
91+
location: asText (r cellInColumn "Vehicle Location"),
92+
utility: asText (r cellInColumn "Electric Utility"),
93+
censusTract: asText (r cellInColumn "2020 Census Tract"),
94+
};
95+
}
96+
97+
5298
block ElectricVehiclesTableInterpreter oftype TableInterpreter {
5399
header: true;
54-
columns: [
55-
// 4. Here, a user-deifned value type is used to describe this column.
56-
// The capital letter indicates that the value type is not built-in
57-
// by convention. The value type itself is defined further below.
58-
"VIN (1-10)" oftype VehicleIdentificationNumber10,
59-
"County" oftype text,
60-
"City" oftype text,
61-
"State" oftype UsStateCode, // We can just use the element as if it was defined in this file.
62-
"Postal Code" oftype text,
63-
"Model Year" oftype integer,
64-
"Make" oftype text,
65-
"Model" oftype text,
66-
"Electric Vehicle Type" oftype text,
67-
"Clean Alternative Fuel Vehicle (CAFV) Eligibility" oftype text,
68-
"Electric Range" oftype integer,
69-
"Base MSRP" oftype integer,
70-
"Legislative District" oftype text,
71-
"DOL Vehicle ID" oftype integer,
72-
"Vehicle Location" oftype text,
73-
"Electric Utility" oftype text,
74-
"2020 Census Tract" oftype text,
75-
];
100+
columns: ElectricVehicle;
101+
parseWith: ElectricVehicleParser;
76102
}
77103

78104
// 5. This block describes the application of a transform function
@@ -81,9 +107,9 @@ pipeline ElectricVehiclesPipeline {
81107
// by the "use" property.
82108
block ElectricRangeTransformer oftype TableTransformer {
83109
inputColumns: [
84-
"Electric Range"
110+
"electricRange"
85111
];
86-
outputColumn: "Electric Range (km)";
112+
outputColumn: "electricRange (km)";
87113
uses: MilesToKilometers;
88114
}
89115

‎example/gtfs-rt.jv‎

Lines changed: 90 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -58,49 +58,105 @@ pipeline GtfsRTSimplePipeline {
5858
}
5959

6060
// 5. Next, we interpret the sheets as tables
61+
valuetype TripUpdate {
62+
property headerGtfsRealtimeVersion oftype text;
63+
property headerTimestamp oftype text;
64+
property headerIncrementality oftype text;
65+
property entityId oftype text;
66+
property entityTripUpdateTripTripId oftype text;
67+
property entityTripUpdateTripRouteId oftype text;
68+
property entityTripUpdateStopTimeUpdateStopSequence oftype text;
69+
property entityTripUpdateStopTimeUpdateStopId oftype text;
70+
property entityTripUpdateStopTimeUpdateArrivalTime oftype text;
71+
property entityTripUpdateStopTimeUpdateDepartureTime oftype text;
72+
}
73+
transform TripUpdateParser {
74+
from r oftype Collection<text>;
75+
to tripUpdate oftype TripUpdate;
76+
77+
tripUpdate: {
78+
headerGtfsRealtimeVersion: r cellInColumn "header.gtfs_realtime_version",
79+
headerTimestamp: r cellInColumn "header.timestamp",
80+
headerIncrementality: r cellInColumn "header.incrementality",
81+
entityId: r cellInColumn "entity.id",
82+
entityTripUpdateTripTripId: r cellInColumn "entity.trip_update.trip.trip_id",
83+
entityTripUpdateTripRouteId: r cellInColumn "entity.trip_update.trip.route_id",
84+
entityTripUpdateStopTimeUpdateStopSequence: r cellInColumn "entity.trip_update.stop_time_update.stop_sequence",
85+
entityTripUpdateStopTimeUpdateStopId: r cellInColumn "entity.trip_update.stop_time_update.stop_id",
86+
entityTripUpdateStopTimeUpdateArrivalTime: r cellInColumn "entity.trip_update.stop_time_update.arrival.time",
87+
entityTripUpdateStopTimeUpdateDepartureTime: r cellInColumn "entity.trip_update.stop_time_update.departure.time",
88+
};
89+
}
6190
block TripUpdateTableInterpreter oftype TableInterpreter {
6291
header: true;
63-
columns: [
64-
"header.gtfs_realtime_version" oftype text,
65-
"header.timestamp" oftype text,
66-
"header.incrementality" oftype text,
67-
"entity.id" oftype text,
68-
"entity.trip_update.trip.trip_id" oftype text,
69-
"entity.trip_update.trip.route_id" oftype text,
70-
"entity.trip_update.stop_time_update.stop_sequence" oftype text,
71-
"entity.trip_update.stop_time_update.stop_id" oftype text,
72-
"entity.trip_update.stop_time_update.arrival.time" oftype text,
73-
"entity.trip_update.stop_time_update.departure.time" oftype text,
74-
];
92+
columns: TripUpdate;
93+
parseWith: TripUpdateParser;
7594
}
7695

96+
97+
valuetype VehiclePosition {
98+
property headerGtfsRealtimeVersion oftype text;
99+
property headerTimestamp oftype text;
100+
property headerIncrementality oftype text;
101+
property entityId oftype text;
102+
property entityVehiclePositionVehicleDescriptorId oftype text;
103+
property entityVehiclePositionTripTripId oftype text;
104+
property entityVehiclePositionTripRouteId oftype text;
105+
property entityVehiclePositionPositionLatitude oftype text;
106+
property entityVehiclePositionPositionLongitude oftype text;
107+
property entityVehiclePositionTimestamp oftype text;
108+
}
109+
transform VehiclePositionParser {
110+
from r oftype Collection<text>;
111+
to vehiclePosition oftype VehiclePosition;
112+
113+
vehiclePosition: {
114+
headerGtfsRealtimeVersion: r cellInColumn "header.gtfs_realtime_version",
115+
headerTimestamp: r cellInColumn "header.timestamp",
116+
headerIncrementality: r cellInColumn "header.incrementality",
117+
entityId: r cellInColumn "entity.id",
118+
entityVehiclePositionVehicleDescriptorId: r cellInColumn "entity.vehicle_position.vehicle_descriptor.id",
119+
entityVehiclePositionTripTripId: r cellInColumn "entity.vehicle_position.trip.trip_id",
120+
entityVehiclePositionTripRouteId: r cellInColumn "entity.vehicle_position.trip.route_id",
121+
entityVehiclePositionPositionLatitude: r cellInColumn "entity.vehicle_position.position.latitude",
122+
entityVehiclePositionPositionLongitude: r cellInColumn "entity.vehicle_position.position.longitude",
123+
entityVehiclePositionTimestamp: r cellInColumn "entity.vehicle_position.timestamp",
124+
};
125+
}
77126
block VehiclePositionTableInterpreter oftype TableInterpreter {
78127
header: true;
79-
columns: [
80-
"header.gtfs_realtime_version" oftype text,
81-
"header.timestamp" oftype text,
82-
"header.incrementality" oftype text,
83-
"entity.id" oftype text,
84-
"entity.vehicle_position.vehicle_descriptor.id" oftype text,
85-
"entity.vehicle_position.trip.trip_id" oftype text,
86-
"entity.vehicle_position.trip.route_id" oftype text,
87-
"entity.vehicle_position.position.latitude" oftype text,
88-
"entity.vehicle_position.position.longitude" oftype text,
89-
"entity.vehicle_position.timestamp" oftype text
90-
];
128+
columns: VehiclePosition;
129+
parseWith: VehiclePositionParser;
91130
}
92131

132+
133+
valuetype Alert {
134+
property headerGtfsRealtimeVersion oftype text;
135+
property headerTimestamp oftype text;
136+
property headerIncrementality oftype text;
137+
property entityId oftype text;
138+
property entityAlertInformedEntityRouteId oftype text;
139+
property entityAlertHeaderText oftype text;
140+
property entityAlertDescriptionText oftype text;
141+
}
142+
transform AlertParser {
143+
from r oftype Collection<text>;
144+
to alert oftype Alert;
145+
146+
alert: {
147+
headerGtfsRealtimeVersion: r cellInColumn "header.gtfs_realtime_version",
148+
headerTimestamp: r cellInColumn "header.timestamp",
149+
headerIncrementality: r cellInColumn "header.incrementality",
150+
entityId: r cellInColumn "entity.id",
151+
entityAlertInformedEntityRouteId: r cellInColumn "entity.alert.informed_entity.route_id",
152+
entityAlertHeaderText: r cellInColumn "entity.alert.header_text",
153+
entityAlertDescriptionText: r cellInColumn "entity.alert.description_text",
154+
};
155+
}
93156
block AlertTableInterpreter oftype TableInterpreter {
94157
header: true;
95-
columns: [
96-
'header.gtfs_realtime_version' oftype text,
97-
'header.timestamp' oftype text,
98-
'header.incrementality' oftype text,
99-
'entity.id' oftype text,
100-
'entity.alert.informed_entity.route_id' oftype text,
101-
'entity.alert.header_text' oftype text,
102-
'entity.alert.description_text' oftype text,
103-
];
158+
columns: Alert;
159+
parseWith: AlertParser;
104160
}
105161

106162
// 6. Last, we load the tables into the same SQLite file.
@@ -124,4 +180,4 @@ pipeline GtfsRTSimplePipeline {
124180
file: "./gtfs.sqlite";
125181
dropTable: false;
126182
}
127-
}
183+
}

‎libs/execution/src/lib/types/io-types/sheet.ts‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,8 @@ export class Sheet implements IOTypeImplementation<IOType.SHEET> {
4545
return this.numberOfColumns;
4646
}
4747

48-
getHeaderRow(): string[] {
49-
assert(
50-
this.getNumberOfRows() > 0,
51-
'The sheet is expected to be non-empty and have a header row',
52-
);
53-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
54-
return this.data[0]!;
48+
popHeaderRow(): string[] | undefined {
49+
return this.data.shift();
5550
}
5651

5752
iterateRows(callbackFn: (row: string[], rowIndex: number) => void) {

‎libs/execution/src/lib/types/io-types/table.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ export class Table implements IOTypeImplementation<IOType.TABLE> {
118118
return this.columns.has(name);
119119
}
120120

121-
getColumns(): ReadonlyMap<string, TableColumn> {
121+
getColumns(): Map<string, TableColumn> {
122122
return this.columns;
123123
}
124124

‎libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,6 @@ describe('Validation of CellRangeSelectorExecutor', () => {
9696
expect(result.right.ioType).toEqual(IOType.SHEET);
9797
expect(result.right.getNumberOfColumns()).toEqual(3);
9898
expect(result.right.getNumberOfRows()).toEqual(16);
99-
expect(result.right.getHeaderRow()).toEqual(['0', 'Test', 'true']);
10099
expect(result.right.getData()).toEqual(
101100
expect.arrayContaining([
102101
['0', 'Test', 'true'],
@@ -121,7 +120,6 @@ describe('Validation of CellRangeSelectorExecutor', () => {
121120
expect(result.right.ioType).toEqual(IOType.SHEET);
122121
expect(result.right.getNumberOfColumns()).toEqual(3);
123122
expect(result.right.getNumberOfRows()).toEqual(2);
124-
expect(result.right.getHeaderRow()).toEqual(['', 'Test', 'true']);
125123
expect(result.right.getData()).toEqual([
126124
['', 'Test', 'true'],
127125
['', 'Test', 'false'],

‎libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,6 @@ describe('Validation of CellWriterExecutor', () => {
9696
expect(result.right.ioType).toEqual(IOType.SHEET);
9797
expect(result.right.getNumberOfColumns()).toEqual(3);
9898
expect(result.right.getNumberOfRows()).toEqual(16);
99-
expect(result.right.getHeaderRow()).toEqual(['16', 'Test', 'true']);
10099
expect(result.right.getData()).toEqual(
101100
expect.arrayContaining([
102101
['16', 'Test', 'true'],
@@ -121,7 +120,6 @@ describe('Validation of CellWriterExecutor', () => {
121120
expect(result.right.ioType).toEqual(IOType.SHEET);
122121
expect(result.right.getNumberOfColumns()).toEqual(3);
123122
expect(result.right.getNumberOfRows()).toEqual(16);
124-
expect(result.right.getHeaderRow()).toEqual(['16', 'Test2', '']);
125123
expect(result.right.getData()).toEqual(
126124
expect.arrayContaining([
127125
['16', 'Test2', ''],

0 commit comments

Comments
 (0)