Skip to content

Commit a8b8975

Browse files
GoldinGuyGoldinGuy
committed
bg removal functional
1 parent eca086e commit a8b8975

6 files changed

Lines changed: 76 additions & 90 deletions

File tree

‎example/index.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ import Draggable from 'react-draggable';
77
const App = () => {
88
return (
99
<>
10-
{/* <GeneralTest /> */}
11-
<HoursTest />
10+
<GeneralTest />
11+
{/* <HoursTest /> */}
1212
</>
1313
);
1414
};

‎src/CatalystChat.tsx‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ const CatalystChat = ({
7070
const [cookies, setCookie] = useCookies(['PERSISTENT_CLIENT_ID']);
7171
const [audioOn, setAudioOn] = useState(audioOffDefault ? false : true);
7272
const [videoOn, setVideoOn] = useState(videoOffDefault ? false : true);
73+
const [bgRemovalKey, setBgRemovalKey] = useState('');
7374
const [token, setToken] = useState('');
7475
const isMounted = useIsMounted();
7576
if (!handleComponentRefresh) handleComponentRefresh = () => {};
@@ -115,6 +116,9 @@ const CatalystChat = ({
115116
.then((user: CatalystUserData) => {
116117
if (!isMounted()) return;
117118
setToken(user.token);
119+
if (user.vectorlyToken.length > 0) {
120+
setBgRemovalKey(user.vectorlyToken);
121+
}
118122
if (handleUserData) handleUserData(user);
119123
// console.log(user);
120124
return user.token;
@@ -170,6 +174,7 @@ const CatalystChat = ({
170174
disableChat={disableChat}
171175
disableSelfieMode={disableSelfieMode}
172176
arbData={arbData}
177+
bgRemovalKey={bgRemovalKey}
173178
bgRemoval={bgRemoval}
174179
handleReceiveArbData={handleReceiveArbData}
175180
cstmWelcomeMsg={cstmWelcomeMsg}

‎src/components/toolbar/Toolbar.tsx‎

Lines changed: 18 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -65,10 +65,9 @@ const Toolbar = ({
6565
outputDevice,
6666
chatOpen,
6767
bgRemoval,
68+
bgRemovalKey,
6869
bgFilter,
6970
setBgFilter,
70-
// bgRemovalEffect,
71-
// setBgRemovalEffect,
7271
setChatOpen,
7372
disableChat,
7473
cstmSupportUrl,
@@ -82,10 +81,9 @@ const Toolbar = ({
8281
outputDevice?: MediaDeviceInfo;
8382
chatOpen: boolean;
8483
bgRemoval?: 'blur' | string;
84+
bgRemovalKey: string;
8585
bgFilter?: BackgroundFilter;
8686
setBgFilter: Function;
87-
// bgRemovalEffect?: 'blur' | string;
88-
// setBgRemovalEffect: Function;
8987
setChatOpen: Function;
9088
disableChat?: boolean;
9189
cstmSupportUrl?: string;
@@ -238,15 +236,16 @@ const Toolbar = ({
238236
}, [videoDevice]);
239237

240238
const handleBgEffect = async () => {
241-
var track: LocalVideoTrack | MediaStreamTrack | undefined;
242239
var filter: BackgroundFilter | undefined | null = bgFilter;
243-
if (!filter && video) {
240+
if (!filter && video && bgRemovalKey.length > 0) {
241+
var track: LocalVideoTrack | MediaStreamTrack | undefined;
244242
createLocalVideoTrack({ deviceId: videoDevice?.deviceId })
245243
.then(async (track: LocalVideoTrack | MediaStreamTrack) => {
246244
if (video) {
247245
unpublishTrack(video.track as LocalVideoTrack);
248246
}
249247
filter = await initBgFilter(
248+
bgRemovalKey,
250249
video.track as LocalVideoTrack,
251250
bgRemovalEffect
252251
);
@@ -277,35 +276,33 @@ const Toolbar = ({
277276
if (!filterEnabled) {
278277
const filterEnabledStream = await filter.enable();
279278
// console.log('filterEnabledStream', filterEnabledStream);
280-
if (filterEnabledStream) {
279+
if (filterEnabledStream) {
281280
// track = filterEnabledStream.getVideoTracks()[0];
282-
track = await filter.getOutputTrack();
283-
if (track) {
281+
track = await filter.getOutputTrack();
282+
if (track) {
284283
// console.log(track);
285-
if (video) {
286-
unpublishTrack(video.track as LocalVideoTrack);
287-
}
288-
room.localParticipant.publishTrack(track);
289-
setFilterEnabled(true);
290-
}
291-
}
284+
if (video) {
285+
unpublishTrack(video.track as LocalVideoTrack);
286+
}
287+
room.localParticipant.publishTrack(track);
288+
setFilterEnabled(true);
289+
}
290+
}
292291
}
293292
if (bgRemovalEffect === 'blur' && filter.background !== 'blur') {
294293
await filter.changeBackground('blur');
295294
} else if (filter.background !== bgRemovalEffect) {
296-
await filter.changeBackground(bgRemovalEffect);
295+
await filter.changeBackground(bgRemovalEffect);
297296
}
298297
}
299298
}
300-
301299
};
302300

303301
useEffect(() => {
304302
// console.log(bgRemovalEffect);
305303
handleBgEffect();
306304
}, [bgRemovalEffect]);
307305

308-
309306
const toggleVideo = () => {
310307
if (video?.track) {
311308
if (video) unpublishTrack(video.track as LocalVideoTrack);
@@ -316,8 +313,8 @@ const Toolbar = ({
316313
.then(async (track: LocalVideoTrack) => {
317314
let newTrack: LocalVideoTrack | MediaStreamTrack = track;
318315
if (bgRemovalEffect && bgFilter) {
319-
await applyBgFilterToTrack(track.mediaStreamTrack, bgFilter);
320-
newTrack = await bgFilter.getOutputTrack();
316+
await applyBgFilterToTrack(track.mediaStreamTrack, bgFilter);
317+
newTrack = await bgFilter.getOutputTrack();
321318
}
322319
room.localParticipant.publishTrack(newTrack);
323320
})
@@ -492,28 +489,6 @@ const Toolbar = ({
492489
}}
493490
/>
494491
)}
495-
{/* TODO: fixed right chat button {!disableChat && !isMobile && (
496-
<div
497-
// className={`absolute ${
498-
// chatOpen ? 'right-48 animate-fade-in-right' : 'right-3 ' // animate-fade-in-left
499-
// } z-40 bottom-1 animate-fade-in-up`}
500-
className={`absolute ${
501-
chatOpen ? 'hidden' : ''
502-
} right-3 z-0 bottom-1 animate-fade-in-up`}
503-
>
504-
<ToolbarButton
505-
type="chat"
506-
tooltip="Toggle Chat"
507-
icon={faCommentAlt}
508-
bgColor={
509-
chatOpen ? `bg-primary` : 'bg-tertiary hover:bg-quaternary '
510-
}
511-
onClick={() => {
512-
setChatOpen(chatOpen => !chatOpen);
513-
}}
514-
/>
515-
</div>
516-
)} */}
517492
</div>
518493
);
519494
};

‎src/typings/interfaces.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ export interface CatalystUserData {
104104
__v: number;
105105
};
106106
isNewUser: boolean;
107+
vectorlyToken: string;
107108
}
108109

109110
export interface RoomData {

‎src/utils/bg_removal.ts‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,25 @@ import { BackgroundFilter } from "@vectorly-io/ai-filters";
22
import { LocalTrack, LocalVideoTrack } from "livekit-client";
33

44
export const initBgFilter = async (
5+
bgRemovalKey: string,
56
vidTrack: LocalTrack ,
67
effectType?: string
78
): Promise<BackgroundFilter | null> => {
8-
if (effectType && effectType.length > 0 && effectType !== 'none') {
9+
if (
10+
effectType &&
11+
effectType.length > 0 &&
12+
effectType !== 'none' &&
13+
bgRemovalKey.length > 0
14+
) {
915
if (vidTrack?.mediaStreamTrack) {
1016
return new BackgroundFilter(vidTrack.mediaStreamTrack, {
11-
token: 'BG_REMOVAL_TOKEN',
17+
token: bgRemovalKey ?? 'BG_REMOVAL_TOKEN',
1218
background: effectType,
13-
blurRadius: 6
19+
blurRadius: 6,
1420
});
1521
// console.log('bg', bgRemovedTrack);
1622
}
17-
}
23+
}
1824
return null;
1925
};
2026

‎src/views/CatalystChatView.tsx‎

Lines changed: 40 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ const CatalystChatView = ({
7575
cstmSupportUrl,
7676
arbData,
7777
bgRemoval,
78+
bgRemovalKey,
7879
handleReceiveArbData,
7980
onJoinCall,
8081
onMemberJoin,
@@ -92,6 +93,7 @@ const CatalystChatView = ({
9293
cstmSupportUrl?: string;
9394
arbData?: Uint8Array;
9495
bgRemoval?: 'blur' | string;
96+
bgRemovalKey: string;
9597
handleReceiveArbData?: (arbData: Uint8Array) => void;
9698
onJoinCall?: () => void;
9799
onMemberJoin?: () => void;
@@ -107,7 +109,7 @@ const CatalystChatView = ({
107109
const [chatOpen, setChatOpen] = useState(false);
108110
const [outputDevice, setOutputDevice] = useState<MediaDeviceInfo>();
109111
// const [bgRemovalEffect, setBgRemovalEffect] = useState(bgRemoval ?? 'none');
110-
const [bgFilter, setBgFilter] = useState <BackgroundFilter>();
112+
const [bgFilter, setBgFilter] = useState<BackgroundFilter>();
111113
const roomState = useRoom();
112114
const isMounted = useIsMounted();
113115
const audDId = useReadLocalStorage('PREFERRED_AUDIO_DEVICE_ID');
@@ -172,17 +174,17 @@ const CatalystChatView = ({
172174
: false,
173175
});
174176
// apply bg removal filters
175-
if (bgRemoval && bgRemoval !== 'none') {
177+
if (bgRemoval && bgRemoval !== 'none' && bgRemovalKey.length >0) {
176178
const vidTrack = localTracks.find(track => track.kind === 'video');
177179
if (vidTrack) {
178-
const filter = await initBgFilter(vidTrack, bgRemoval);
180+
const filter = await initBgFilter(bgRemovalKey, vidTrack, bgRemoval);
179181
if (filter) {
180182
setBgFilter(filter);
181183
localTracks = await applyBgFilterToTracks(localTracks, filter);
182184
}
183185
}
184186
}
185-
187+
186188
localTracks.forEach(track => {
187189
if (!isMounted()) return;
188190
room.localParticipant.publishTrack(
@@ -221,25 +223,25 @@ const CatalystChatView = ({
221223
}, [token]);
222224

223225
useEffect(() => {
224-
if (!outputDevice) {
225-
navigator.mediaDevices.enumerateDevices().then(devices => {
226-
if (!isMounted()) return;
227-
const outputDevices = devices.filter(
228-
id => id.kind === 'audiooutput' && id?.deviceId
229-
);
230-
let outDevice: MediaDeviceInfo | undefined;
231-
if (outDId) {
232-
outDevice = outputDevices.find(d => d?.deviceId === outDId);
233-
}
234-
if (!outDevice) {
235-
outDevice = outputDevices[0];
236-
}
237-
setOutputDevice(outDevice);
238-
if (outDId !== outDevice?.deviceId && outDevice?.deviceId) {
239-
setOutDId(outDevice?.deviceId);
240-
}
241-
});
242-
}
226+
if (!outputDevice) {
227+
navigator.mediaDevices.enumerateDevices().then(devices => {
228+
if (!isMounted()) return;
229+
const outputDevices = devices.filter(
230+
id => id.kind === 'audiooutput' && id?.deviceId
231+
);
232+
let outDevice: MediaDeviceInfo | undefined;
233+
if (outDId) {
234+
outDevice = outputDevices.find(d => d?.deviceId === outDId);
235+
}
236+
if (!outDevice) {
237+
outDevice = outputDevices[0];
238+
}
239+
setOutputDevice(outDevice);
240+
if (outDId !== outDevice?.deviceId && outDevice?.deviceId) {
241+
setOutDId(outDevice?.deviceId);
242+
}
243+
});
244+
}
243245
// disconnect on unmount
244246
return () => {
245247
roomState.disconnectAll();
@@ -261,10 +263,9 @@ const CatalystChatView = ({
261263
};
262264

263265
// roomState.audioTracks.map(track => console.log(track));
264-
if (fade > 0) {
265-
266-
// animate toolbar & header fade in/out
267-
useEffect(() => {
266+
if (fade > 0) {
267+
// animate toolbar & header fade in/out
268+
useEffect(() => {
268269
const delayCheck = () => {
269270
if (!isMounted()) return;
270271
const hClasses = headerRef.current?.classList;
@@ -315,16 +316,15 @@ const CatalystChatView = ({
315316
videoChatRef.current?.addEventListener('mousemove', debounceHandleMouse);
316317
var _delay = setInterval(delayCheck, fade);
317318

318-
return () => {
319-
clearInterval(_delay);
320-
videoChatRef.current?.removeEventListener(
321-
'mousemove',
322-
debounceHandleMouse
323-
);
324-
};
325-
}, []);
326-
}
327-
319+
return () => {
320+
clearInterval(_delay);
321+
videoChatRef.current?.removeEventListener(
322+
'mousemove',
323+
debounceHandleMouse
324+
);
325+
};
326+
}, []);
327+
}
328328

329329
return (
330330
<div id="video-chat" className="relative w-full h-full" ref={videoChatRef}>
@@ -485,10 +485,9 @@ const CatalystChatView = ({
485485
updateOutputDevice={updateOutputDevice}
486486
outputDevice={outputDevice}
487487
bgFilter={bgFilter}
488-
setBgFilter={setBgFilter}
489-
bgRemoval={bgRemoval}
490-
// bgRemovalEffect={bgRemovalEffect}
491-
// setBgRemovalEffect={setBgRemovalEffect}
488+
setBgFilter={setBgFilter}
489+
bgRemoval={bgRemoval}
490+
bgRemovalKey={bgRemovalKey}
492491
chatOpen={chatOpen}
493492
setChatOpen={setChatOpen}
494493
disableChat={disableChat}

0 commit comments

Comments
 (0)