feat: added initial impl of speech recognition API

AlmasB · AlmasB · commit 1ad4d4322d6c · 2024-01-27T11:40:09.000Z
diff --git a/fxgl-intelligence/pom.xml b/fxgl-intelligence/pom.xml
@@ -29,6 +29,12 @@
             <artifactId>selenium-java</artifactId>
             <version>${selenium.version}</version>
         </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-nop</artifactId>
+            <version>${slf4j.version}</version>
+        </dependency>
     </dependencies>
 
     <build>
diff --git a/fxgl-intelligence/src/main/java/com/almasb/fxgl/intelligence/SpeechRecognitionService.java b/fxgl-intelligence/src/main/java/com/almasb/fxgl/intelligence/SpeechRecognitionService.java
diff --git a/fxgl-intelligence/src/main/java/com/almasb/fxgl/intelligence/WebAPI.java b/fxgl-intelligence/src/main/java/com/almasb/fxgl/intelligence/WebAPI.java
@@ -0,0 +1,22 @@
+/*
+ * FXGL - JavaFX Game Library. The MIT License (MIT).
+ * Copyright (c) AlmasB (almaslvl@gmail.com).
+ * See LICENSE for details.
+ */
+
+package com.almasb.fxgl.intelligence;
+
+/**
+ * Stores constants related to web-api projects.
+ * Changes to these values must be synchronized with the web-api project (https://github.com/AlmasB/web-api).
+ *
+ * @author Almas Baim (https://github.com/AlmasB)
+ */
+public final class WebAPI {
+
+    public static final String SPEECH_RECOGNITION_API = "https://almasb.github.io/web-api/speech-recog/";
+    public static final String GESTURE_RECOGNITION_API = "https://almasb.github.io/web-api/gesture-recog-v1/";
+
+    public static final int SPEECH_RECOGNITION_PORT = 55555;
+    public static final int GESTURE_RECOGNITION_PORT = 55560;
+}
diff --git a/fxgl-intelligence/src/main/java/com/almasb/fxgl/speechrecog/SpeechRecognitionService.java b/fxgl-intelligence/src/main/java/com/almasb/fxgl/speechrecog/SpeechRecognitionService.java
@@ -0,0 +1,96 @@
+/*
+ * FXGL - JavaFX Game Library. The MIT License (MIT).
+ * Copyright (c) AlmasB (almaslvl@gmail.com).
+ * See LICENSE for details.
+ */
+
+package com.almasb.fxgl.speechrecog;
+
+import com.almasb.fxgl.core.EngineService;
+import com.almasb.fxgl.core.concurrent.Async;
+import com.almasb.fxgl.logging.Logger;
+import com.almasb.fxgl.ws.LocalWebSocketServer;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.chrome.ChromeDriver;
+import org.openqa.selenium.chrome.ChromeOptions;
+
+import java.util.function.Consumer;
+
+import static com.almasb.fxgl.intelligence.WebAPI.SPEECH_RECOGNITION_API;
+import static com.almasb.fxgl.intelligence.WebAPI.SPEECH_RECOGNITION_PORT;
+
+public final class SpeechRecognitionService extends EngineService {
+
+    private static final Logger log = Logger.get(SpeechRecognitionService.class);
+    private LocalWebSocketServer server = new LocalWebSocketServer("SpeechRecogServer", SPEECH_RECOGNITION_PORT);
+
+    private WebDriver webDriver = null;
+
+    @Override
+    public void onInit() {
+        server.start();
+    }
+
+    /**
+     * Starts speech recognition in a background thread.
+     * Can be called after stop() to restart speech recognition.
+     * If the service has already started, then calls stop() and restarts it.
+     */
+    public void start() {
+        Async.INSTANCE.startAsync(() -> {
+            try {
+                if (webDriver != null) {
+                    stop();
+                }
+
+                ChromeOptions options = new ChromeOptions();
+                options.addArguments("--headless=new");
+                options.addArguments("--use-fake-ui-for-media-stream");
+
+                webDriver = new ChromeDriver(options);
+                webDriver.get(SPEECH_RECOGNITION_API);
+
+                // we are ready to use the web api service
+
+            } catch (Exception e) {
+                log.warning("Failed to start Chrome web driver. Ensure Chrome is installed in default location");
+                log.warning("Error data", e);
+            }
+        });
+    }
+
+    /**
+     * Stops speech recognition.
+     * No-op if it has not started via start() before.
+     */
+    public void stop() {
+        try {
+            if (webDriver != null) {
+                webDriver.quit();
+                webDriver = null;
+            }
+        } catch (Exception e) {
+            log.warning("Failed to quit web driver", e);
+        }
+    }
+
+    /**
+     * Add a [handler] for incoming speech input.
+     */
+    public void addInputHandler(Consumer<String> handler) {
+        server.addMessageHandler(handler);
+    }
+
+    /**
+     * Remove an existing input handler.
+     */
+    public void removeInputHandler(Consumer<String> handler) {
+        server.removeMessageHandler(handler);
+    }
+
+    @Override
+    public void onExit() {
+        stop();
+        server.stop();
+    }
+}
diff --git a/fxgl-intelligence/src/main/java/com/almasb/fxgl/ws/LocalWebSocketServer.java b/fxgl-intelligence/src/main/java/com/almasb/fxgl/ws/LocalWebSocketServer.java
@@ -32,7 +32,7 @@ public final class LocalWebSocketServer extends WebSocketServer {
 
     private final Logger log;
 
-    private boolean isLoggingEnabled = true;
+    private boolean hasStarted = false;
 
     private String serverName;
     private List<Consumer<String>> messageHandlers = new ArrayList<>();
@@ -58,12 +58,8 @@ public LocalWebSocketServer(String serverName, int port) {
         thread = new SendMessageThread(serverName);
     }
 
-    public boolean isLoggingEnabled() {
-        return isLoggingEnabled;
-    }
-
-    public void setLoggingEnabled(boolean isLoggingEnabled) {
-        this.isLoggingEnabled = isLoggingEnabled;
+    public boolean hasStarted() {
+        return hasStarted;
     }
 
     /**
@@ -109,14 +105,14 @@ public boolean isConnected() {
 
     @Override
     public void onOpen(WebSocket conn, ClientHandshake handshake) {
-        log.info("Opened connection: " + conn.getRemoteSocketAddress());
+        log.debug("Opened connection: " + conn.getRemoteSocketAddress());
 
         socket = conn;
     }
 
     @Override
     public void onClose(WebSocket conn, int code, String reason, boolean remote) {
-        log.info("Closed connection, code: " + code);
+        log.debug("Closed connection, code: " + code);
     }
 
     @Override
@@ -131,14 +127,16 @@ public void onError(WebSocket conn, Exception ex) {
 
     @Override
     public void onStart() {
-        log.info("Server started successfully");
+        log.debug("Server started successfully");
     }
 
     @Override
     public void start() {
         try {
             thread.start();
             super.start();
+
+            hasStarted = true;
         } catch (Exception e) {
             log.warning("Failed to start WS server", e);
         }
diff --git a/fxgl-samples/pom.xml b/fxgl-samples/pom.xml
@@ -27,6 +27,12 @@
             <artifactId>fxgl-controllerinput</artifactId>
             <version>${fxgl.version}</version>
         </dependency>
+
+        <dependency>
+            <groupId>com.github.almasb</groupId>
+            <artifactId>fxgl-intelligence</artifactId>
+            <version>${fxgl.version}</version>
+        </dependency>
     </dependencies>
 
     <build>
diff --git a/fxgl-samples/src/main/java/module-info.java b/fxgl-samples/src/main/java/module-info.java
diff --git a/fxgl-samples/src/main/java/sandbox/net/SpeechRecogSample.java b/fxgl-samples/src/main/java/sandbox/net/SpeechRecogSample.java
@@ -0,0 +1,55 @@
+/*
+ * FXGL - JavaFX Game Library. The MIT License (MIT).
+ * Copyright (c) AlmasB (almaslvl@gmail.com).
+ * See LICENSE for details.
+ */
+
+package sandbox.net;
+
+import com.almasb.fxgl.app.GameApplication;
+import com.almasb.fxgl.app.GameSettings;
+import com.almasb.fxgl.speechrecog.SpeechRecognitionService;
+import com.almasb.fxgl.ui.FontType;
+import javafx.scene.control.TextArea;
+
+import static com.almasb.fxgl.dsl.FXGL.*;
+
+/**
+ * Shows how to use the speech recognition service.
+ *
+ * @author Almas Baim (https://github.com/AlmasB)
+ */
+public class SpeechRecogSample extends GameApplication {
+
+    private TextArea output;
+
+    @Override
+    protected void initSettings(GameSettings settings) {
+        settings.addEngineService(SpeechRecognitionService.class);
+    }
+
+    @Override
+    protected void initGame() {
+        getService(SpeechRecognitionService.class).addInputHandler(input -> {
+            getExecutor().startAsyncFX(() -> {
+                output.appendText(input + "\n");
+            });
+        });
+
+        getService(SpeechRecognitionService.class).start();
+    }
+
+    @Override
+    protected void initUI() {
+        output = new TextArea();
+        output.setWrapText(true);
+        output.setPrefSize(getAppWidth(), getAppHeight());
+        output.setFont(getUIFactoryService().newFont(FontType.MONO, 18));
+
+        addUINode(output);
+    }
+
+    public static void main(String[] args) {
+        launch(args);
+    }
+}
diff --git a/pom.xml b/pom.xml
@@ -96,6 +96,7 @@
         <jackson.version>2.14.2</jackson.version>
         <websocket.version>1.5.5</websocket.version>
         <selenium.version>4.17.0</selenium.version>
+        <slf4j.version>2.0.6</slf4j.version>
 
         <!-- test dependencies -->
         <junit.jupiter.version>5.10.0</junit.jupiter.version>
