Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@ import org.apache.texera.amber.engine.architecture.worker.statistics.{
import scala.collection.mutable

class StatisticsManager {
// DataProcessor
// Plain maps (no withDefaultValue) so they survive Kryo round-trip.
private val inputStatistics: mutable.Map[PortIdentity, (Long, Long)] =
mutable.Map.empty.withDefaultValue((0L, 0L))
mutable.Map.empty
private val outputStatistics: mutable.Map[PortIdentity, (Long, Long)] =
mutable.Map.empty.withDefaultValue((0L, 0L))
mutable.Map.empty
private var dataProcessingTime: Long = 0L
private var totalExecutionTime: Long = 0L
private var workerStartTime: Long = 0L
Expand Down Expand Up @@ -82,8 +82,10 @@ class StatisticsManager {
*/
def increaseInputStatistics(portId: PortIdentity, size: Long): Unit = {
require(size >= 0, "Tuple size must be non-negative")
val (count, totalSize) = inputStatistics(portId)
inputStatistics.update(portId, (count + 1, totalSize + size))
inputStatistics.updateWith(portId) {
case Some((count, totalSize)) => Some((count + 1, totalSize + size))
case None => Some((1L, size))
}
}

/**
Expand All @@ -93,8 +95,10 @@ class StatisticsManager {
*/
def increaseOutputStatistics(portId: PortIdentity, size: Long): Unit = {
require(size >= 0, "Tuple size must be non-negative")
val (count, totalSize) = outputStatistics(portId)
outputStatistics.update(portId, (count + 1, totalSize + size))
outputStatistics.updateWith(portId) {
case Some((count, totalSize)) => Some((count + 1, totalSize + size))
case None => Some((1L, size))
}
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ class WorkflowEmailNotifier(
private def createDashboardUrl(): String = {
val host = sessionUri.getHost
val port = sessionUri.getPort
val path = s"/dashboard/user/workspace/$workflowId"
val path = s"/user/workspace/$workflowId"
if (port == -1 || port == 80 || port == 443) {
s"http://$host$path"
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,15 @@ class WorkerManagersSpec extends AnyFlatSpec {
val sm = new StatisticsManager()
sm.increaseOutputStatistics(PortIdentity(0), 30)
sm.increaseOutputStatistics(PortIdentity(0), 70)
assert(sm.getOutputTupleCount == 2L)
val out = sm.getStatistics(nullExec).outputTupleMetrics
assert(out.size == 1)
assert(out.head.tupleMetrics.count == 2L)
assert(out.head.tupleMetrics.size == 100L)
sm.increaseOutputStatistics(PortIdentity(1), 25)
assert(sm.getOutputTupleCount == 3L)
val byPort = sm
.getStatistics(nullExec)
.outputTupleMetrics
.map(m => m.portId -> (m.tupleMetrics.count, m.tupleMetrics.size))
.toMap
assert(byPort(PortIdentity(0)) == (2L, 100L))
assert(byPort(PortIdentity(1)) == (1L, 25L))
}

it should "reject negative tuple sizes" in {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ class CheckpointSpec extends AnyFlatSpecLike with BeforeAndAfterAll {
system.actorOf(Props[SingleNodeListener](), "cluster-info")
}

"Default controller state" should "be serializable" in {
"Default controller state" should "round-trip through CheckpointState" in {
val cp =
new ControllerProcessor(
workflow.context,
Expand All @@ -73,16 +73,20 @@ class CheckpointSpec extends AnyFlatSpecLike with BeforeAndAfterAll {
)
val chkpt = new CheckpointState()
chkpt.save(CP_STATE_KEY, cp)
val restored: ControllerProcessor = chkpt.load(CP_STATE_KEY)
assert(restored.actorId == cp.actorId)
}

"Default worker state" should "be serializable" in {
"Default worker state" should "round-trip through CheckpointState" in {
val dp = new DataProcessor(
SELF,
msg => {},
inputMessageQueue = new LinkedBlockingQueue[DPInputQueueElement]()
)
val chkpt = new CheckpointState()
chkpt.save(DP_STATE_KEY, dp)
val restored: DataProcessor = chkpt.load(DP_STATE_KEY)
assert(restored.actorId == dp.actorId)
}

"CheckpointState" should "fail loudly on an unknown key" in {
Expand Down
16 changes: 7 additions & 9 deletions frontend/git-version.js → frontend/build-version.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,12 @@
* under the License.
*/

const { gitDescribeSync } = require("git-describe");
const { generate } = require("build-number-generator");
const { version } = require("./package.json");
const { resolve, relative } = require("path");
const { writeFileSync, existsSync, mkdirSync } = require("fs-extra");

const gitInfo = gitDescribeSync({
dirtyMark: false,
dirtySemver: false,
});

gitInfo.version = version;
const buildNumber = generate(version);

if (!existsSync(__dirname + "/src/environments")) {
mkdirSync(__dirname + "/src/environments");
Expand All @@ -37,10 +32,13 @@ writeFileSync(
file,
`// IMPORTANT: THIS FILE IS AUTO GENERATED! DO NOT MANUALLY EDIT OR CHECKIN!
/* tslint:disable */
export const Version = ${JSON.stringify(gitInfo, null, 4)};
export const Version = {
"buildNumber": ${JSON.stringify(buildNumber)},
"version": ${JSON.stringify(version)}
};
/* tslint:enable */
`,
{ encoding: "utf-8" }
);

console.log(`Wrote version info ${gitInfo.raw} to ${relative(resolve(__dirname, ".."), file)}`);
console.log(`Wrote build number ${buildNumber} to ${relative(resolve(__dirname, ".."), file)}`);
4 changes: 2 additions & 2 deletions frontend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"eslint:fix": "yarn eslint --fix ./src",
"format:fix": "yarn prettier-eslint --write \"src/**/*.{ts,js,html,scss,less,json}\"",
"format:ci": "yarn prettier-eslint --list-different \"src/**/*.{ts,js,html,scss,less,json}\" && yarn eslint ./src",
"postinstall": "node git-version.js"
"postinstall": "node build-version.js"
},
"private": true,
"dependencies": {
Expand Down Expand Up @@ -122,12 +122,12 @@
"@vitest/browser": "4.1.5",
"@vitest/browser-playwright": "4.1.5",
"@vitest/coverage-v8": "4.1.5",
"build-number-generator": "3.1.0",
"concurrently": "7.4.0",
"eslint": "8.57.0",
"eslint-plugin-rxjs": "5.0.3",
"eslint-plugin-rxjs-angular": "2.0.1",
"fs-extra": "10.0.1",
"git-describe": "4.1.0",
"jsdom": "25.0.1",
"nodecat": "2.0.0",
"nx": "22.7.0",
Expand Down
2 changes: 1 addition & 1 deletion frontend/src/app/app-routing.constant.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

export const DASHBOARD = "/dashboard";
export const DASHBOARD = "";
export const DASHBOARD_HOME = `${DASHBOARD}/home`;
export const DASHBOARD_ABOUT = `${DASHBOARD}/about`;

Expand Down
33 changes: 9 additions & 24 deletions frontend/src/app/app-routing.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
* under the License.
*/

import { inject, NgModule } from "@angular/core";
import { CanActivateFn, Router, RouterModule, Routes } from "@angular/router";
import { NgModule } from "@angular/core";
import { RouterModule, Routes } from "@angular/router";
import { DashboardComponent } from "./dashboard/component/dashboard.component";
import { UserWorkflowComponent } from "./dashboard/component/user/user-workflow/user-workflow.component";
import { UserQuotaComponent } from "./dashboard/component/user/user-quota/user-quota.component";
Expand All @@ -38,28 +38,21 @@ import { DatasetDetailComponent } from "./dashboard/component/user/user-dataset/
import { UserDatasetComponent } from "./dashboard/component/user/user-dataset/user-dataset.component";
import { HubWorkflowDetailComponent } from "./hub/component/workflow/detail/hub-workflow-detail.component";
import { LandingPageComponent } from "./hub/component/landing-page/landing-page.component";
import { DASHBOARD_ABOUT, DASHBOARD_USER_WORKFLOW } from "./app-routing.constant";
import { DASHBOARD_USER_WORKFLOW } from "./app-routing.constant";
import { HubSearchResultComponent } from "./hub/component/hub-search-result/hub-search-result.component";
import { AdminSettingsComponent } from "./dashboard/component/admin/settings/admin-settings.component";
import { GuiConfigService } from "./common/service/gui-config.service";

const rootRedirectGuard: CanActivateFn = () => {
const config = inject(GuiConfigService);
const router = inject(Router);
try {
return router.parseUrl(DASHBOARD_ABOUT);
} catch {
// config not loaded yet, swallow the error and let the app handle it
}
return true;
};

const routes: Routes = [];

routes.push({
path: "dashboard",
path: "",
component: DashboardComponent,
children: [
{
path: "",
redirectTo: "about",
pathMatch: "full",
},
{
path: "home",
component: LandingPageComponent,
Expand Down Expand Up @@ -174,14 +167,6 @@ routes.push({
],
});

// default route renders the workspace editor directly; if userSystem is enabled at runtime,
// AppComponent will navigate to DASHBOARD_ABOUT instead.
routes.push({
path: "",
component: WorkspaceComponent,
canActivate: [rootRedirectGuard],
});

// redirect all other paths to index.
routes.push({
path: "**",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@
<tr *ngFor="let execution of basicTable.data">
<td>
<div *ngIf="execution.access; else normalWorkflowName">
<a href="/dashboard/user/workflow/{{execution.workflowId}}">
<a href="/user/workflow/{{execution.workflowId}}">
{{ maxStringLength(execution.workflowName, 16) }} ({{ execution.workflowId }})
</a>
</div>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@
<span>About</span>
</li>
</ul>
<span id="git-commit-id">Git hash: {{ gitCommitHash }}</span>
<span id="build-number">Build: {{ buildNumber }}</span>
<span
*ngIf="!isCollapsed && config.env.attributionEnabled"
id="powered-by">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ nz-content {
padding: 5px 0;
}

#git-commit-id {
#build-number {
position: absolute;
left: 5px;
bottom: 5px;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,8 @@ describe("DashboardComponent", () => {
};

routerMock = {
events: of(new NavigationEnd(1, "/dashboard", "/dashboard")),
url: "/dashboard",
events: of(new NavigationEnd(1, "/", "/")),
url: "/",
navigateByUrl: vi.fn(),
};

Expand Down
4 changes: 2 additions & 2 deletions frontend/src/app/dashboard/component/dashboard.component.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ export class DashboardComponent implements OnInit {

isAdmin: boolean = this.userService.isAdmin();
isLogin = this.userService.isLogin();
public gitCommitHash: string = Version.raw;
public buildNumber: string = Version.buildNumber;
displayForum: boolean = true;
displayNavbar: boolean = true;
isCollapsed: boolean = false;
Expand Down Expand Up @@ -232,7 +232,7 @@ export class DashboardComponent implements OnInit {

isNavbarEnabled(currentRoute: string) {
// Hide navbar for workflow workspace pages (with numeric ID)
if (currentRoute.match(/\/dashboard\/user\/workflow\/\d+/)) {
if (currentRoute.match(/\/user\/workflow\/\d+/)) {
return false;
}
return true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ export class ShareAccessComponent implements OnInit, OnDestroy {
this.emailTags.forEach(email => {
let message = `${this.userService.getCurrentUser()?.email} shared a ${this.type} with you`;
if (this.type !== "computing-unit")
message += `, access the ${this.type} at ${location.origin}/dashboard/user/workflow/${this.id}`;
message += `, access the ${this.type} at ${location.origin}/user/workflow/${this.id}`;
this.accessService
.grantAccess(this.type, this.id, email, this.validateForm.value.accessLevel)
.pipe(untilDestroyed(this))
Expand Down
27 changes: 11 additions & 16 deletions frontend/yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -6397,7 +6397,7 @@ __metadata:
languageName: node
linkType: hard

"@types/semver@npm:^7.3.12, @types/semver@npm:^7.3.8":
"@types/semver@npm:^7.3.12":
version: 7.7.1
resolution: "@types/semver@npm:7.7.1"
checksum: 10c0/c938aef3bf79a73f0f3f6037c16e2e759ff40c54122ddf0b2583703393d8d3127130823facb880e694caa324eb6845628186aac1997ee8b31dc2d18fafe26268
Expand Down Expand Up @@ -8135,6 +8135,15 @@ __metadata:
languageName: node
linkType: hard

"build-number-generator@npm:3.1.0":
version: 3.1.0
resolution: "build-number-generator@npm:3.1.0"
bin:
buildnumgen: bin/buildnumgen.js
checksum: 10c0/10d2366b74493a12aa7390e08dc5cd27f77ba600234c757e59ae9436eb7b7dde837322d34f0e177de5e5da3f4e5cabb089a03f9fcb32195ceaecd3ebab0b3c5b
languageName: node
linkType: hard

"bundle-name@npm:^4.1.0":
version: 4.1.0
resolution: "bundle-name@npm:4.1.0"
Expand Down Expand Up @@ -10869,20 +10878,6 @@ __metadata:
languageName: node
linkType: hard

"git-describe@npm:4.1.0":
version: 4.1.0
resolution: "git-describe@npm:4.1.0"
dependencies:
"@types/semver": "npm:^7.3.8"
lodash: "npm:^4.17.21"
semver: "npm:^5.6.0"
dependenciesMeta:
semver:
optional: true
checksum: 10c0/2e5cbb0f5aa4a6f4dc9135276a85a29ec627a88dd0d73b63d1a3cd2ec2477a8d1d5fc83ae480a00c587c0ea8a193c6a9696a5b549dc6af02db0be74cdfac0eb5
languageName: node
linkType: hard

"glob-parent@npm:^5.1.2, glob-parent@npm:~5.1.2":
version: 5.1.2
resolution: "glob-parent@npm:5.1.2"
Expand Down Expand Up @@ -11094,6 +11089,7 @@ __metadata:
"@vitest/coverage-v8": "npm:4.1.5"
ai: "npm:5.0.93"
ajv: "npm:8.10.0"
build-number-generator: "npm:3.1.0"
concaveman: "npm:2.0.0"
concurrently: "npm:7.4.0"
d3-shape: "npm:2.1.0"
Expand All @@ -11104,7 +11100,6 @@ __metadata:
file-saver: "npm:2.0.5"
fs-extra: "npm:10.0.1"
fuse.js: "npm:6.5.3"
git-describe: "npm:4.1.0"
html2canvas: "npm:1.4.1"
jointjs: "npm:3.5.4"
jsdom: "npm:25.0.1"
Expand Down
Loading