Media Element Audio Source Node

apps/common-app/src/examples/AudioTag/AudioTag.tsx

@@ -1,58 +1,140 @@
-import React, { useRef } from 'react';
-import { Button, View } from 'react-native';
+import React, { useCallback, useMemo, useRef, useState, useEffect } from 'react';
+import { Text, useWindowDimensions, View } from 'react-native';
 import {
   Audio,
   AudioTagHandle,
 } from 'react-native-audio-api/development/react';
+import { AudioContext, GainNode } from 'react-native-audio-api';
 
-import { Container } from '../../components';
+import { Button, Container, Slider, Spacer } from '../../components';
 
 // const DEMO_AUDIO_URL = 'https://filesamples.com/samples/audio/m4a/sample4.m4a';
 const DEMO_AUDIO_URL = 'https://filesamples.com/samples/audio/mp3/sample4.mp3';
 
 const AudioTag: React.FC = () => {
+  const { width: screenWidth } = useWindowDimensions();
   const audioRef = useRef<AudioTagHandle>(null);
+  const [sliderVolume, setSliderVolume] = useState(1);
+  const volumeRef = useRef(1);
+  const audioContextRef = useRef<AudioContext>(new AudioContext());
+  const gainNodeRef = useRef<GainNode | null>(null);
 
-  // const handlePlay = () => {
-  //   audioRef.current?.play();
-  // };
+  const ensureMediaElementRoute = useCallback(() => {
+    if (!audioRef.current) {
+      throw new Error('Audio tag handle is not ready yet.');
+    }
 
-  // const handlePause = () => {
-  //   audioRef.current?.pause();
-  // };
+    const ctx = audioContextRef.current;
+    const mediaElementSource = ctx.createMediaElementSource(audioRef.current);
+    gainNodeRef.current = ctx.createGain();
+    gainNodeRef.current.gain.value = volumeRef.current;
 
-  // const handleSeekToTime = (time: number) => {
-  //   console.log('handleSeekToTime', time);
-  //   audioRef.current?.seekToTime(time);
-  // };
+    const biquad = ctx.createBiquadFilter();
+    biquad.type = 'highpass';
+    biquad.frequency.value = 5000;
+    mediaElementSource.connect(biquad);
+    biquad.connect(gainNodeRef.current);
+    gainNodeRef.current.connect(ctx.destination);
+    audioRef.current?.play();
+  }, []);
 
-  // const handleSetVolume = (volume: number) => {
-  //   audioRef.current?.setVolume(volume);
-  // };
+  const handleVolumeChange = useCallback((nextVolume: number) => {
+    setSliderVolume(nextVolume);
+    volumeRef.current = nextVolume;
+    audioRef.current?.setVolume(nextVolume);
+    if (gainNodeRef.current) {
+      gainNodeRef.current.gain.value = nextVolume;
+    }
+  }, []);
 
-  // const handleSetMuted = (muted: boolean) => {
-  //   audioRef.current?.setMuted(muted);
-  // };
+  const handleLoadStart = useCallback(() => {
+    // console.log('onLoadStart');
+  }, []);
+  const handleLoad = useCallback(() => {
+    // console.log('onLoad');
+  }, []);
+  const handleError = useCallback((error: Error) => {
+    // console.log('onError', error);
+  }, []);
+  const handlePositionChange = useCallback(
+    (seconds: number) => {
+      // console.log('onPositionChange', seconds);
+    },
+    []
+  );
+  const handleEnded = useCallback(() => {
+    // console.log('onEnded');
+  }, []);
+  const handlePlay = useCallback(() => {
+    // console.log('onPlay');
+  }, []);
+  const handlePause = useCallback(() => {
+    // console.log('onPause');
+  }, []);
+  const handleVolumeEvent = useCallback(
+    (volume: number) => {
+      // console.log('onVolumeChange', volume);
+    },
+    []
+  );
+
+  const audioTagElement = useMemo(
+    () => (
+      <Audio
+        source={DEMO_AUDIO_URL}
+        ref={audioRef}
+        context={audioContextRef.current}
+        controls
+        onLoadStart={handleLoadStart}
+        onLoad={handleLoad}
+        onError={handleError}
+        onPositionChange={handlePositionChange}
+        onEnded={handleEnded}
+        onPlay={handlePlay}
+        onPause={handlePause}
+        onVolumeChange={handleVolumeEvent}
+      />
+    ),
+    [
+      handleEnded,
+      handleError,
+      handleLoad,
+      handleLoadStart,
+      handlePause,
+      handlePlay,
+      handlePositionChange,
+      handleVolumeEvent,
+    ]
+  );
 
   return (
     <Container disablePadding>
       <View style={{ flex: 1, justifyContent: 'center', alignItems: 'center' }}>
         <View style={{ width: '90%' }}>
-          <Audio
-            source={DEMO_AUDIO_URL}
-            ref={audioRef}
-            controls
-            onLoadStart={() => console.log('onLoadStart')}
-            onLoad={() => console.log('onLoad')}
-            onError={(error) => console.log('onError', error)}
-            onPositionChange={(seconds) =>
-              console.log('onPositionChange', seconds)
-            }
-            onEnded={() => console.log('onEnded')}
-            onPlay={() => console.log('onPlay')}
-            onPause={() => console.log('onPause')}
-            onVolumeChange={(volume) => console.log('onVolumeChange', volume)}
+          {audioTagElement}
+          <Spacer.Vertical size={20} />
+          <Slider
+            label="Volume"
+            value={sliderVolume}
+            onValueChange={handleVolumeChange}
+            min={0}
+            max={1}
+            step={0.01}
+            minLabelWidth={70}
           />
+          <Spacer.Vertical size={20} />
+        </View>
+        <Button
+          title="Route via MediaElement node"
+          onPress={ensureMediaElementRoute}
+          width={screenWidth * 0.8}
+        />
+        <Spacer.Vertical size={12} />
+        <View style={{ width: '90%' }}>
+          <Text style={{ color: 'white', textAlign: 'center' }}>
+            The button initializes MediaElementAudioSourceNode from the Audio tag
+            handle and routes it through GainNode to destination.
+          </Text>
         </View>
       </View>
     </Container>

apps/fabric-example/ios/Podfile.lock

@@ -2550,7 +2550,7 @@ SPEC CHECKSUMS:
   ReactCodegen: d07ee3c8db75b43d1cbe479ae6affebf9925c733
   ReactCommon: fe2a3af8975e63efa60f95fca8c34dc85deee360
   ReactNativeDependencies: 4d5ce2683b6d74f7c686bf90a88c7d381295cf3c
-  RNAudioAPI: 764858df27270ed9a55803bb4c9c0ccb5bb14e9a
+  RNAudioAPI: 6668f71bdd9850005984acf39a3daef4935cec02
   RNGestureHandler: 187c5c7936abf427bc4d22d6c3b1ac80ad1f63c0
   RNReanimated: 64f4b3b33b48b19e0ba76a352571b52b1e931981
   RNScreens: 01b065ded2dfe7987bcce770ff3a196be417ff41

packages/audiodocs/docs/core/audio-context.mdx

@@ -33,6 +33,16 @@ It inherits all properties from [`BaseAudioContext`](/docs/core/base-audio-conte
 
 It inherits all methods from [`BaseAudioContext`](/docs/core/base-audio-context#methods).
 
+### `createMediaElementSource`
+
+Creates [`MediaElementAudioSourceNode`](/docs/sources/media-element-audio-source-node), routing media playback into the audio graph.
+
+| Parameter | Type | Description |
+| :---: | :---: | :---- |
+| `mediaElement` | `HTMLMediaElement` (web) \| [`AudioTagHandle`](/docs/experimental/audio-tag#ref-handle-audiotaghandle-methods) (native) | Media element to route into the graph. On native, wait for [`onLoad`](/docs/experimental/audio-tag#props-audioprops) on `<Audio>` before calling. |
+
+#### Returns [`MediaElementAudioSourceNode`](/docs/sources/media-element-audio-source-node).
+
 ### `close`
 
 Closes the audio context, releasing any system audio resources that it uses.

packages/audiodocs/docs/other/web-audio-api-coverage.mdx

@@ -23,6 +23,7 @@ sidebar_position: 2
 | DelayNode | ✅ |
 | GainNode | ✅ |
 | IIRFilterNode | ✅  |
+| MediaElementAudioSourceNode | ✅ | Full <Audio /> spec is still in a development phase, however most important functionalities are implemented. |
 | OfflineAudioContext | ✅ |
 | OscillatorNode | ✅ |
 | PeriodicWave | ✅ |
@@ -39,7 +40,6 @@ sidebar_position: 2
 | ChannelMergerNode | ❌ |
 | ChannelSplitterNode | ❌ |
 | DynamicsCompressorNode | ❌ |
-| MediaElementAudioSourceNode | ❌ |
 | MediaStreamAudioDestinationNode | ❌ |
 | MediaStreamAudioSourceNode | ❌ |
 | PannerNode | ❌ |

packages/audiodocs/docs/sources/media-element-audio-source-node.mdx

@@ -0,0 +1,107 @@
+---
+sidebar_position: 6
+---
+
+import AudioNodePropsTable from "@site/src/components/AudioNodePropsTable"
+import { ReadOnly } from '@site/src/components/Badges';
+
+# MediaElementAudioSourceNode
+
+The `MediaElementAudioSourceNode` is an [`AudioNode`](/docs/core/audio-node) that routes media playback into the audio graph.
+On web it mirrors [`AudioContext.createMediaElementSource()`](https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/createMediaElementSource) and accepts an `HTMLMediaElement`.
+On native it wraps a file-backed source created by the experimental [`<Audio>`](/docs/experimental/audio-tag) tag.
+
+## Constructor
+
+```tsx
+constructor(context: BaseAudioContext, options: MediaElementAudioSourceOptions)
+```
+
+### `MediaElementAudioSourceOptions`
+
+| Parameter | Type | Default | |
+| :---: | :---: | :----: | :---- |
+| `mediaElement` | `HTMLMediaElement` (web) \| [`AudioTagHandle`](/docs/experimental/audio-tag#ref-handle-audiotaghandle-methods) (native) | — | Media element whose audio is routed into the graph. On native, wait for [`onLoad`](/docs/experimental/audio-tag#props-audioprops) before constructing this node. |
+
+Or by using [`AudioContext`](/docs/core/audio-context) factory method:
+[`AudioContext.createMediaElementSource()`](/docs/core/audio-context#createmediaelementsource)
+
+## Example
+
+```tsx
+import React, { useRef } from 'react';
+import {
+  AudioContext,
+  GainNode,
+} from 'react-native-audio-api';
+import { Audio, AudioTagHandle } from 'react-native-audio-api/development/react';
+
+function WebExample() {
+  const audioContextRef = useRef<AudioContext | null>(null);
+  if (!audioContextRef.current) {
+    audioContextRef.current = new AudioContext();
+  }
+
+  const audioElement = document.querySelector('audio')!;
+  const sourceNode = audioContextRef.current.createMediaElementSource(audioElement);
+  sourceNode.connect(audioContextRef.current.destination);
+}
+
+function NativeExample() {
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const audioRef = useRef<AudioTagHandle>(null);
+
+  if (!audioContextRef.current) {
+    audioContextRef.current = new AudioContext();
+  }
+
+  const routeThroughGraph = () => {
+    if (!audioRef.current || !audioContextRef.current) {
+      return;
+    }
+
+    const mediaElementSource = audioContextRef.current.createMediaElementSource(
+      audioRef.current
+    );
+    const gain = audioContextRef.current.createGain();
+    mediaElementSource.connect(gain);
+    gain.connect(audioContextRef.current.destination);
+    audioRef.current.play();
+  };
+
+  return (
+    <Audio
+      ref={audioRef}
+      source="https://example.com/audio.mp3"
+      context={audioContextRef.current}
+      onLoad={routeThroughGraph}
+    />
+  );
+}
+```
+
+## Properties
+
+It inherits all properties from [`AudioNode`](/docs/core/audio-node#properties).
+
+<AudioNodePropsTable numberOfInputs={0} numberOfOutputs={1} channelCount="defined by associated media element" channelCountMode="max" channelInterpretation="speakers" />
+
+| Name | Type | Default value | Description |
+| :----: | :----: | :-------- | :------- |
+| `mediaElement` <ReadOnly /> | `HTMLMediaElement` \| [`AudioTagHandle`](/docs/experimental/audio-tag#ref-handle-audiotaghandle-methods) | — | The media element routed into the audio graph. |
+
+## Methods
+
+It inherits all methods from [`AudioNode`](/docs/core/audio-node#methods).
+
+## Remarks
+
+#### Binding
+
+- A single media element can be bound only once per [`AudioContext`](/docs/core/audio-context).
+- Calling `createMediaElementSource` again for the same element throws an error.
+
+#### Native routing
+
+- On native, the underlying file source is disconnected from the destination when this node is created so audio is not processed twice.
+- Use the same [`AudioTagHandle`](/docs/experimental/audio-tag#ref-handle-audiotaghandle-methods) for transport controls (`play`, `pause`, `seekToTime`, volume).

packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.cpp

@@ -1,5 +1,7 @@
 #include <audioapi/HostObjects/AudioContextHostObject.h>
 
+#include <audioapi/HostObjects/sources/AudioFileSourceNodeHostObject.h>
+#include <audioapi/HostObjects/sources/MediaElementAudioSourceNodeHostObject.h>
 #include <audioapi/core/AudioContext.h>
 #include <memory>
 #include <utility>
@@ -19,7 +21,8 @@ AudioContextHostObject::AudioContextHostObject(
   addFunctions(
       JSI_EXPORT_FUNCTION(AudioContextHostObject, close),
       JSI_EXPORT_FUNCTION(AudioContextHostObject, resume),
-      JSI_EXPORT_FUNCTION(AudioContextHostObject, suspend));
+      JSI_EXPORT_FUNCTION(AudioContextHostObject, suspend),
+      JSI_EXPORT_FUNCTION(AudioContextHostObject, createMediaElementSource));
 }
 
 JSI_HOST_FUNCTION_IMPL(AudioContextHostObject, close) {
@@ -57,4 +60,19 @@ JSI_HOST_FUNCTION_IMPL(AudioContextHostObject, suspend) {
   return promise;
 }
 
+JSI_HOST_FUNCTION_IMPL(AudioContextHostObject, createMediaElementSource) {
+  auto sourceObject = args[0].asObject(runtime);
+  auto fileSourceHostObject = sourceObject.getHostObject<AudioFileSourceNodeHostObject>(runtime);
+  auto mediaElementSourceNode =
+      context_->createMediaElementSource(fileSourceHostObject->getAudioFileSourceNode());
+
+  if (mediaElementSourceNode == nullptr) {
+    throw jsi::JSError(runtime, "This media element is already connected to this AudioContext.");
+  }
+
+  auto mediaElementSourceHostObject =
+      std::make_shared<MediaElementAudioSourceNodeHostObject>(mediaElementSourceNode);
+  return jsi::Object::createFromHostObject(runtime, mediaElementSourceHostObject);
+}
+
 } // namespace audioapi

packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.h

@@ -24,5 +24,6 @@ class AudioContextHostObject : public BaseAudioContextHostObject {
   JSI_HOST_FUNCTION_DECL(close);
   JSI_HOST_FUNCTION_DECL(resume);
   JSI_HOST_FUNCTION_DECL(suspend);
+  JSI_HOST_FUNCTION_DECL(createMediaElementSource);
 };
 } // namespace audioapi

packages/react-native-audio-api/common/cpp/audioapi/HostObjects/BaseAudioContextHostObject.cpp

@@ -23,6 +23,7 @@
 #include <audioapi/HostObjects/utils/JsEnumParser.h>
 #include <audioapi/HostObjects/utils/NodeOptionsParser.h>
 #include <audioapi/core/BaseAudioContext.h>
+#include <audioapi/core/OfflineAudioContext.h>
 #include <audioapi/core/utils/AudioDecoder.h>
 
 #include <memory>
@@ -254,7 +255,7 @@ JSI_HOST_FUNCTION_IMPL(BaseAudioContextHostObject, createBufferSource) {
 }
 
 JSI_HOST_FUNCTION_IMPL(BaseAudioContextHostObject, createFileSource) {
-  auto makeFileSourceHostObject = [&](AudioFileSourceOptions opts) -> jsi::Value {
+  auto makeFileSourceHostObject = [&](const AudioFileSourceOptions &opts) -> jsi::Value {
 #if RN_AUDIO_API_FFMPEG_DISABLED
     if (opts.requiresFFmpeg) {
       return jsi::Value::undefined();