Search code examples
javajavafxgluon-mobilegraalvm-native-image

Loading and playing audio on desktop, and mobile with GraalVM


I'm trying to load and play an AudioClip in 2 scenarios:

  1. On desktop during development (from an IDE) so that I can test that things work correctly during development.
  2. On mobile, using Gluon's GraalVM native solution.

The media module is not supported on Android and Attach's audio service is used instead.

What I tried: check if AudioService is present; if yes, load the sound and play it with the service; if not, load the sound and play it with Media.

  • Is this the correct approach? I'm getting an exception as shown below.
  • Do I (can I) exclude the media module dependency when I deploy to mobile because it is unusable?

Example classes:

package com.audioTest;

import static com.gluonhq.charm.glisten.application.AppManager.*;

import javafx.application.Application;
import javafx.scene.control.Button;
import javafx.stage.Stage;

import com.gluonhq.charm.glisten.application.AppManager;
import com.gluonhq.charm.glisten.mvc.View;

public class MyApplication extends Application {

    AppManager appManager = AppManager.initialize();

    @Override
    public void init() {
        class HomeView extends View {
            
            HomeView() {
                var button = new Button("Play Sound");
                button.setOnAction(e -> Notifier.create());
                setCenter(button);
            }
        };

        appManager.addViewFactory(HOME_VIEW, HomeView::new);
    }

    @Override
    public void start(Stage stage) throws Exception {
        appManager.start(stage);
        if (com.gluonhq.attach.util.Platform.isDesktop()) {
            stage.setHeight(600);
            stage.setWidth(360);
            stage.centerOnScreen();
        }
    }

    public static void main(String args[]) {
        launch(args);
    }
}

package com.audioTest;

import javafx.scene.media.AudioClip;

import com.gluonhq.attach.audio.Audio;
import com.gluonhq.attach.audio.AudioService;

public abstract sealed class Notifier {

    public static Notifier create() {
        return AudioService.create().<Notifier>map(MobileNotifier::new).orElseGet(DesktopNotifier::new);
    }

    private Notifier() {}

    private final static class MobileNotifier extends Notifier {

        private final Audio BEEP;

        private MobileNotifier(AudioService service) {
            BEEP = service.loadSound(getClass().getResource("/sounds/Beep.wav")).orElseThrow(); // throws here
            BEEP.play();
        }
    }

    private final static class DesktopNotifier extends Notifier {

        private static final AudioClip BEEP = new AudioClip(Notifier.class.getResource("/sounds/Beep.wav").toExternalForm());

        private DesktopNotifier() {
            BEEP.play();
        }
    }
}

The pom is the standard one given by the Gluon plugin with the added audio service and the resource list:

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.audioTest</groupId>
    <artifactId>AudioTest</artifactId>
    <version>1.0.0-SNAPSHOT</version>

    <name>AudioTest</name>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

        <maven-compiler-plugin-version>3.10.1</maven-compiler-plugin-version>
        <javafx-maven-plugin-version>0.0.8</javafx-maven-plugin-version>
        <gluonfx-maven-plugin-version>1.0.14</gluonfx-maven-plugin-version>

        <java-version>17</java-version>
        <javafx-version>18.0.1</javafx-version>
        <charm-version>6.1.0</charm-version>
        <attach-version>4.0.14</attach-version>
    
        <main.class>com.audioTest.MyApplication</main.class>
        <app.identifier>${main.class}</app.identifier>
        <app.description>The AudioTest app</app.description>
        <package.type />
        <mac.app.store />
        <mac.signing.user.name />
        <bundle.version />
        <bundle.short.version />
        <version.code />
        <provided.keystore.path />
        <provided.keystore.password />
        <provided.key.alias />
        <provided.key.alias.password />
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.openjfx</groupId>
            <artifactId>javafx-controls</artifactId>
            <version>${javafx-version}</version>
        </dependency>
        <dependency>
            <groupId>org.openjfx</groupId>
            <artifactId>javafx-media</artifactId>
            <version>${javafx-version}</version>
        </dependency>

        <dependency>
            <groupId>com.gluonhq</groupId>
            <artifactId>charm-glisten</artifactId>
            <version>${charm-version}</version>
        </dependency>
        <dependency>
            <groupId>com.gluonhq.attach</groupId>
            <artifactId>display</artifactId>
            <version>${attach-version}</version>
        </dependency>
        <dependency>
            <groupId>com.gluonhq.attach</groupId>
            <artifactId>lifecycle</artifactId>
            <version>${attach-version}</version>
        </dependency>
        <dependency>
            <groupId>com.gluonhq.attach</groupId>
            <artifactId>statusbar</artifactId>
            <version>${attach-version}</version>
        </dependency>
        <dependency>
            <groupId>com.gluonhq.attach</groupId>
            <artifactId>storage</artifactId>
            <version>${attach-version}</version>
        </dependency>
        <dependency>
            <groupId>com.gluonhq.attach</groupId>
            <artifactId>audio</artifactId>
            <version>${attach-version}</version>
        </dependency>
        <dependency>
            <groupId>com.gluonhq.attach</groupId>
            <artifactId>util</artifactId>
            <version>${attach-version}</version>
        </dependency>
    </dependencies>

    <repositories>
        <repository>
            <id>Gluon</id>
            <url>https://nexus.gluonhq.com/nexus/content/repositories/releases</url>
        </repository>
        <repository>
            <id>snapshot</id>
            <url>https://oss.sonatype.org/content/repositories/snapshots/</url>
        </repository>
    </repositories>

    <pluginRepositories>
        <pluginRepository>
            <id>snapshot</id>
            <url>https://oss.sonatype.org/content/repositories/snapshots/</url>
        </pluginRepository>
    </pluginRepositories>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>${maven-compiler-plugin-version}</version>
                <configuration>
                    <release>${java-version}</release>
                </configuration>
            </plugin>

            <plugin>
                <groupId>org.openjfx</groupId>
                <artifactId>javafx-maven-plugin</artifactId>
                <version>${javafx-maven-plugin-version}</version>
                <configuration>
                    <mainClass>${main.class}</mainClass>
                </configuration>
            </plugin>

            <plugin>
                <groupId>com.gluonhq</groupId>
                <artifactId>gluonfx-maven-plugin</artifactId>
                <version>${gluonfx-maven-plugin-version}</version>
                <configuration>
                    <verbose>true</verbose>
                    <target>${gluonfx.target}</target>
                    <attachList>
                        <list>display</list>
                        <list>lifecycle</list>
                        <list>statusbar</list>
                        <list>storage</list>
                        <list>audio</list>
                    </attachList>
                    <resourcesList>
                        <item>Beep.wav</item>
                    </resourcesList>
                    <mainClass>${main.class}</mainClass>
                    <appIdentifier>${app.identifier}</appIdentifier>
                    <releaseConfiguration>
                        <vendor>Gluon</vendor>
                        <description>${app.description}</description>
                        <packageType>${package.type}</packageType>
                        <!-- for macOS/iOS -->
                        <macAppStore>${mac.app.store}</macAppStore>
                        <bundleShortVersion>${bundle.short.version}</bundleShortVersion>
                        <bundleVersion>${bundle.version}</bundleVersion>
                        <!-- for Android -->
                        <versionCode>${version.code}</versionCode>
                        <providedKeyStorePath>${provided.keystore.path}</providedKeyStorePath>
                        <providedKeyStorePassword>${provided.keystore.password}</providedKeyStorePassword>
                        <providedKeyAlias>${provided.key.alias}</providedKeyAlias>
                        <providedKeyAliasPassword>${provided.key.alias.password}</providedKeyAliasPassword>
                    </releaseConfiguration>
                </configuration>
            </plugin>
        </plugins>
    </build>

    <profiles>
        <profile>
            <id>desktop</id>
            <activation>
                <activeByDefault>true</activeByDefault>
            </activation>
            <properties>
                <gluonfx.target>host</gluonfx.target>
            </properties>
        </profile>
        <profile>
            <id>mac</id>
            <properties>
                <package.type>pkg</package.type>
                <mac.app.store>false</mac.app.store>
                <bundle.version>${env.GITHUB_RUN_NUMBER}</bundle.version>
                <bundle.short.version>1.0</bundle.short.version>
            </properties>
        </profile>
        <profile>
            <id>macstore</id>
            <properties>
                <package.type>pkg</package.type>
                <mac.app.store>true</mac.app.store>
                <bundle.version>1.${env.GITHUB_RUN_NUMBER}</bundle.version>
                <bundle.short.version>1.6</bundle.short.version>
            </properties>
        </profile>
        <profile>
            <id>ios</id>
            <properties>
                <gluonfx.target>ios</gluonfx.target>
                <bundle.version>${env.GITHUB_RUN_NUMBER}</bundle.version>
                <bundle.short.version>1.0</bundle.short.version>
            </properties>
        </profile>
        <profile>
            <id>android</id>
            <properties>
                <gluonfx.target>android</gluonfx.target>
                <app.identifier>com.audioTest</app.identifier>
                <version.code>${env.GITHUB_RUN_NUMBER}</version.code>
                <provided.keystore.path>${env.GLUON_ANDROID_KEYSTOREPATH}</provided.keystore.path>
                <provided.keystore.password>${env.GLUON_ANDROID_KEYSTORE_PASSWORD}</provided.keystore.password>
                <provided.key.alias>${env.GLUON_ANDROID_KEYALIAS}</provided.key.alias>
                <provided.key.alias.password>${env.GLUON_ANDROID_KEYALIAS_PASSWORD}</provided.key.alias.password>
            </properties>
        </profile>
        <profile>
            <id>pi</id>
            <properties>
                <gluonfx.target>linux-aarch64</gluonfx.target>
            </properties>
        </profile>
    </profiles>
</project>

The file is located under src/main/resources/sounds/Beep.wav.

The exception:

D/GraalCompiled( 7319): Exception in thread "JavaFX Application Thread" java.util.NoSuchElementException: No value present
D/GraalCompiled( 7319):         at java.util.Optional.orElseThrow(Optional.java:377)
D/GraalCompiled( 7319):         at com.audioTest.Notifier$MobileNotifier.<init>(Notifier.java:21)
D/GraalCompiled( 7319):         at java.util.Optional.map(Optional.java:260)
D/GraalCompiled( 7319):         at com.audioTest.Notifier.create(Notifier.java:11)
D/GraalCompiled( 7319):         at com.audioTest.MyApplication$1HomeView.lambda$new$0(MyApplication.java:22)
D/GraalCompiled( 7319):         at com.sun.javafx.event.CompositeEventHandler.dispatchBubblingEvent(CompositeEventHandler.java:86)
D/GraalCompiled( 7319):         at com.sun.javafx.event.EventHandlerManager.dispatchBubblingEvent(EventHandlerManager.java:234)
D/GraalCompiled( 7319):         at com.sun.javafx.event.EventHandlerManager.dispatchBubblingEvent(EventHandlerManager.java:191)
D/GraalCompiled( 7319):         at com.sun.javafx.event.CompositeEventDispatcher.dispatchBubblingEvent(CompositeEventDispatcher.java:59)
D/GraalCompiled( 7319):         at com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:58)D/GraalCompiled( 7319):         at com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114)
D/GraalCompiled( 7319):         at com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56)D/GraalCompiled( 7319):         at com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114)
D/GraalCompiled( 7319):         at com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56)D/GraalCompiled( 7319):         at com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114)
D/GraalCompiled( 7319):         at com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56)D/GraalCompiled( 7319):         at com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114)
D/GraalCompiled( 7319):         at com.sun.javafx.event.EventUtil.fireEventImpl(EventUtil.java:74)
D/GraalCompiled( 7319):         at com.sun.javafx.event.EventUtil.fireEvent(EventUtil.java:49)
D/GraalCompiled( 7319):         at javafx.event.Event.fireEvent(Event.java:198)
D/GraalCompiled( 7319):         at javafx.scene.Node.fireEvent(Node.java:8797)
D/GraalCompiled( 7319):         at javafx.scene.control.Button.fire(Button.java:203)
D/GraalCompiled( 7319):         at com.sun.javafx.scene.control.behavior.ButtonBehavior.mouseReleased(ButtonBehavior.java:208)
D/GraalCompiled( 7319):         at com.sun.javafx.scene.control.inputmap.InputMap.handle(InputMap.java:274)
D/GraalCompiled( 7319):         at com.sun.javafx.event.CompositeEventHandler$NormalEventHandlerRecord.handleBubblingEvent(CompositeEventHandler.java:247)
D/GraalCompiled( 7319):         at com.sun.javafx.event.CompositeEventHandler.dispatchBubblingEvent(CompositeEventHandler.java:80)
D/GraalCompiled( 7319):         at com.sun.javafx.event.EventHandlerManager.dispatchBubblingEvent(EventHandlerManager.java:234)
D/GraalCompiled( 7319):         at com.sun.javafx.event.EventHandlerManager.dispatchBubblingEvent(EventHandlerManager.java:191)
D/GraalCompiled( 7319):         at com.sun.javafx.event.CompositeEventDispatcher.dispatchBubblingEvent(CompositeEventDispatcher.java:59)
D/GraalCompiled( 7319):         at com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:58)D/GraalCompiled( 7319):         at com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114)
D/GraalCompiled( 7319):         at com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56)D/GraalCompiled( 7319):         at com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114)
D/GraalCompiled( 7319):         at com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56)D/GraalCompiled( 7319):         at com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114)
D/GraalCompiled( 7319):         at com.sun.javafx.event.BasicEventDispatcher.dispatchEvent(BasicEventDispatcher.java:56)D/GraalCompiled( 7319):         at com.sun.javafx.event.EventDispatchChainImpl.dispatchEvent(EventDispatchChainImpl.java:114)
D/GraalCompiled( 7319):         at com.sun.javafx.event.EventUtil.fireEventImpl(EventUtil.java:74)
D/GraalCompiled( 7319):         at com.sun.javafx.event.EventUtil.fireEvent(EventUtil.java:54)
D/GraalCompiled( 7319):         at javafx.event.Event.fireEvent(Event.java:198)
D/GraalCompiled( 7319):         at javafx.scene.Scene$MouseHandler.process(Scene.java:3881)
D/GraalCompiled( 7319):         at javafx.scene.Scene.processMouseEvent(Scene.java:1874)
D/GraalCompiled( 7319):         at javafx.scene.Scene$ScenePeerListener.mouseEvent(Scene.java:2607)
D/GraalCompiled( 7319):         at com.sun.javafx.tk.quantum.GlassViewEventHandler$MouseEventNotification.run(GlassViewEventHandler.java:411)
D/GraalCompiled( 7319):         at com.sun.javafx.tk.quantum.GlassViewEventHandler$MouseEventNotification.run(GlassViewEventHandler.java:301)
D/GraalCompiled( 7319):         at java.security.AccessController.executePrivileged(AccessController.java:169)
D/GraalCompiled( 7319):         at java.security.AccessController.doPrivileged(AccessController.java:399)
D/GraalCompiled( 7319):         at com.sun.javafx.tk.quantum.GlassViewEventHandler.lambda$handleMouseEvent$2(GlassViewEventHandler.java:450)
D/GraalCompiled( 7319):         at com.sun.javafx.tk.quantum.QuantumToolkit.runWithoutRenderLock(QuantumToolkit.java:424)
D/GraalCompiled( 7319):         at com.sun.javafx.tk.quantum.GlassViewEventHandler.handleMouseEvent(GlassViewEventHandler.java:449)
D/GraalCompiled( 7319):         at com.sun.glass.ui.View.handleMouseEvent(View.java:551)
D/GraalCompiled( 7319):         at com.sun.glass.ui.View.notifyMouse(View.java:937)
D/GraalCompiled( 7319):         at com.sun.glass.ui.monocle.MonocleView.notifyMouse(MonocleView.java:116)
D/GraalCompiled( 7319):         at com.sun.glass.ui.monocle.MouseInput.notifyMouse(MouseInput.java:328)
D/GraalCompiled( 7319):         at com.sun.glass.ui.monocle.MouseInput.lambda$postMouseEvent$3(MouseInput.java:241)
D/GraalCompiled( 7319):         at com.sun.glass.ui.monocle.RunnableProcessor.runLoop(RunnableProcessor.java:92)
D/GraalCompiled( 7319):         at com.sun.glass.ui.monocle.RunnableProcessor.run(RunnableProcessor.java:51)
D/GraalCompiled( 7319):         at java.lang.Thread.run(Thread.java:833)
D/GraalCompiled( 7319):         at com.oracle.svm.core.thread.PlatformThreads.threadStartRoutine(PlatformThreads.java:704)
D/GraalCompiled( 7319):         at com.oracle.svm.core.posix.thread.PosixPlatformThreads.pthreadStartRoutine(PosixPlatformThreads.java:202)

Solution

  • This is a fine approach. There is no need to change anything in the Media module dependency as it will not be used on mobile where the Attach service is available.

    For every resource that is not automatically supported, it needs to be declared in the <resourceList>, and this is also where the error comes from. it requires a path to be specified, and just specifying <item>Beep.wav</item> will work only if the file is directly under the resources folder. To find the file, this pattern needs to be used: <item>.*/Beep.wav$</item>.

    Sources: https://github.com/oracle/graal/blob/master/docs/reference-manual/native-image/Resources.md