pikvm/pikvm#1440: Websocket-based transport and decoding for H.264

8 files changed, 305 insertions, 33 deletions

diff --git a/web/kvm/index.html b/web/kvm/index.html
index fbf9c5df..1db915ee 100644
--- a/web/kvm/index.html
+++ b/web/kvm/index.html
@@ -170,6 +170,17 @@
                 </div>
             <hr>
           </div>
+          <div class="hidden" id="stream-message-no-vd">
+                <div class="text">
+                  <table>
+                    <tr>
+                      <td rowspan="2"><img class="sign " src="/share/svg/warning.svg"></td>
+                      <td style="line-height:1.5"><b>Direct HTTP H.264 streaming is not supported</b></td>
+                    </tr>
+                  </table>
+                </div>
+            <hr>
+          </div>
           <div class="hidden" id="stream-message-no-h264">
                 <div class="text">
                   <table>
@@ -220,10 +231,12 @@
               <td>Video <a target="_blank" href="https://docs.pikvm.org/webrtc">mode</a>:</td>
               <td>
                 <div class="radio-box">
-                  <input checked type="radio" id="stream-mode-radio-mjpeg" name="stream-mode-radio" value="mjpeg">
-                  <label for="stream-mode-radio-mjpeg">MJPEG / HTTP</label>
                   <input type="radio" id="stream-mode-radio-janus" name="stream-mode-radio" value="janus">
-                  <label for="stream-mode-radio-janus">H.264 / WebRTC</label>
+                  <label for="stream-mode-radio-janus">WebRTC</label>
+                  <input type="radio" id="stream-mode-radio-media" name="stream-mode-radio" value="media">
+                  <label for="stream-mode-radio-media">H.264</label>
+                  <input checked type="radio" id="stream-mode-radio-mjpeg" name="stream-mode-radio" value="mjpeg">
+                  <label for="stream-mode-radio-mjpeg">MJPEG</label>
                 </div>
               </td>
             </tr>
@@ -914,6 +927,7 @@
       <button class="window-button-exit-full-tab">&#9660;</button>
       <div class="stream-box-offline" id="stream-box"><img id="stream-image" src="/share/png/blank-stream.png">
         <video class="hidden" id="stream-video" disablePictureInPicture="true" autoplay playsinline muted></video>
+        <canvas class="hidden" id="stream-canvas"></canvas>
         <div id="stream-fullscreen-active"></div>
       </div>
       <div class="keypad" id="stream-mouse-buttons" align="center">
diff --git a/web/kvm/navbar-system.pug b/web/kvm/navbar-system.pug
index 62cbda25..a3438486 100644
--- a/web/kvm/navbar-system.pug
+++ b/web/kvm/navbar-system.pug
@@ -19,6 +19,9 @@ li(id="system-dropdown" class="right")
 		div(id="stream-message-no-webrtc" class="hidden")
 			+menu_message("warning", "WebRTC is not supported by this browser")
 			hr
+		div(id="stream-message-no-vd" class="hidden")
+			+menu_message("warning", "Direct HTTP H.264 streaming is not supported")
+			hr
 		div(id="stream-message-no-h264" class="hidden")
 			+menu_message("warning", "H.264 is not supported by this browser")
 			hr
@@ -46,10 +49,12 @@ li(id="system-dropdown" class="right")
 				td Video #[a(target="_blank" href="https://docs.pikvm.org/webrtc") mode]:
 				td
 					div(class="radio-box")
-						input(checked type="radio" id="stream-mode-radio-mjpeg" name="stream-mode-radio" value="mjpeg")
-						label(for="stream-mode-radio-mjpeg") MJPEG / HTTP
 						input(type="radio" id="stream-mode-radio-janus" name="stream-mode-radio" value="janus")
-						label(for="stream-mode-radio-janus") H.264 / WebRTC
+						label(for="stream-mode-radio-janus") WebRTC
+						input(type="radio" id="stream-mode-radio-media" name="stream-mode-radio" value="media")
+						label(for="stream-mode-radio-media") H.264
+						input(checked type="radio" id="stream-mode-radio-mjpeg" name="stream-mode-radio" value="mjpeg")
+						label(for="stream-mode-radio-mjpeg") MJPEG
 			tr(id="stream-orient" class="feature-disabled")
 				td Orientation:
 				td
diff --git a/web/kvm/window-stream.pug b/web/kvm/window-stream.pug
index 5ac6e15c..cbe998e3 100644
--- a/web/kvm/window-stream.pug
+++ b/web/kvm/window-stream.pug
@@ -16,6 +16,7 @@ div(id="stream-window" class="window window-resizable")
 	div(id="stream-box" class="stream-box-offline")
 		img(id="stream-image" src=`${png_dir}/blank-stream.png`)
 		video(id="stream-video" class="hidden" disablePictureInPicture="true" autoplay playsinline muted)
+		canvas(id="stream-canvas" class="hidden")
 		div(id="stream-fullscreen-active")
 
 	div(id="stream-mouse-buttons" class="keypad" align="center")
diff --git a/web/share/css/kvm/stream.css b/web/share/css/kvm/stream.css
index 16132c21..f6fdb62c 100644
--- a/web/share/css/kvm/stream.css
+++ b/web/share/css/kvm/stream.css
@@ -85,7 +85,8 @@ div.stream-box-mouse-none {
 }
 
 img#stream-image,
-video#stream-video {
+video#stream-video,
+canvas#stream-canvas {
 	width: 100%;
 	height: 100%;
 	object-fit: contain;
diff --git a/web/share/js/kvm/stream.js b/web/share/js/kvm/stream.js
index b436c093..7d947736 100644
--- a/web/share/js/kvm/stream.js
+++ b/web/share/js/kvm/stream.js
@@ -27,6 +27,7 @@ import {tools, $} from "../tools.js";
 import {wm} from "../wm.js";
 
 import {JanusStreamer} from "./stream_janus.js";
+import {MediaStreamer} from "./stream_media.js";
 import {MjpegStreamer} from "./stream_mjpeg.js";
 
 
@@ -168,17 +169,20 @@ export function Streamer() {
 		if (state.features) {
 			let f = state.features;
 			let l = state.limits;
-			let has_webrtc = JanusStreamer.is_webrtc_available();
-			let has_h264 = JanusStreamer.is_h264_available();
-			let has_janus = (__janus_imported && f.h264 && has_webrtc); // Don't check has_h264 for sure
+			let sup_h264 = $("stream-video").canPlayType("video/mp4; codecs=\"avc1.42E01F\"");
+			let sup_vd = MediaStreamer.is_videodecoder_available();
+			let sup_webrtc = JanusStreamer.is_webrtc_available();
+			let has_media = (f.h264 && sup_vd); // Don't check sup_h264 for sure
+			let has_janus = (__janus_imported && f.h264 && sup_webrtc); // Same
 
 			tools.info(
 				`Stream: Janus WebRTC state: features.h264=${f.h264},`
-				+ ` webrtc=${has_webrtc}, h264=${has_h264}, janus_imported=${__janus_imported}`
+				+ ` webrtc=${sup_webrtc}, h264=${sup_h264}, janus_imported=${__janus_imported}`
 			);
 
-			tools.hidden.setVisible($("stream-message-no-webrtc"), __janus_imported && f.h264 && !has_webrtc);
-			tools.hidden.setVisible($("stream-message-no-h264"), __janus_imported && f.h264 && !has_h264);
+			tools.hidden.setVisible($("stream-message-no-webrtc"), __janus_imported && f.h264 && !sup_webrtc);
+			tools.hidden.setVisible($("stream-message-no-vd"), f.h264 && !sup_vd);
+			tools.hidden.setVisible($("stream-message-no-h264"), __janus_imported && f.h264 && !sup_h264);
 
 			tools.slider.setRange($("stream-desired-fps-slider"), l.desired_fps.min, l.desired_fps.max);
 			if (f.resolution) {
@@ -190,21 +194,27 @@ export function Streamer() {
 			} else {
 				$("stream-resolution-selector").options.length = 0;
 			}
-			if (has_janus) {
+			if (f.h264) {
 				tools.slider.setRange($("stream-h264-bitrate-slider"), l.h264_bitrate.min, l.h264_bitrate.max);
 				tools.slider.setRange($("stream-h264-gop-slider"), l.h264_gop.min, l.h264_gop.max);
 			}
 
 			// tools.feature.setEnabled($("stream-quality"), f.quality); // Only on s.encoder.quality
 			tools.feature.setEnabled($("stream-resolution"), f.resolution);
-			tools.feature.setEnabled($("stream-h264-bitrate"), has_janus);
-			tools.feature.setEnabled($("stream-h264-gop"), has_janus);
-			tools.feature.setEnabled($("stream-mode"), has_janus);
-			if (!has_janus) {
+			tools.feature.setEnabled($("stream-h264-bitrate"), f.h264);
+			tools.feature.setEnabled($("stream-h264-gop"), f.h264);
+			tools.feature.setEnabled($("stream-mode"), f.h264);
+			if (!f.h264) {
 				tools.feature.setEnabled($("stream-audio"), false);
 			}
 
-			let mode = (has_janus ? tools.storage.get("stream.mode", "janus") : "mjpeg");
+			let mode = tools.storage.get("stream.mode", "janus");
+			if (mode === "janus" && !has_janus) {
+				mode = "media";
+			}
+			if (mode === "media" && !has_media) {
+				mode = "mjpeg";
+			}
 			tools.radio.clickValue("stream-mode-radio", mode);
 		}
 
@@ -285,8 +295,12 @@ export function Streamer() {
 			// Firefox doesn't support RTP orientation:
 			//  - https://bugzilla.mozilla.org/show_bug.cgi?id=1316448
 			tools.feature.setEnabled($("stream-orient"), !tools.browser.is_firefox);
-		} else { // mjpeg
-			__streamer = new MjpegStreamer(__setActive, __setInactive, __setInfo);
+		} else {
+			if (mode === "media") {
+				__streamer = new MediaStreamer(__setActive, __setInactive, __setInfo);
+			} else { // mjpeg
+				__streamer = new MjpegStreamer(__setActive, __setInactive, __setInfo);
+			}
 			tools.feature.setEnabled($("stream-orient"), false);
 			tools.feature.setEnabled($("stream-audio"), false); // Enabling in stream_janus.js
 		}
@@ -299,7 +313,8 @@ export function Streamer() {
 		let mode = tools.radio.getValue("stream-mode-radio");
 		tools.storage.set("stream.mode", mode);
 		if (mode !== __streamer.getMode()) {
-			tools.hidden.setVisible($("stream-image"), (mode !== "janus"));
+			tools.hidden.setVisible($("stream-canvas"), (mode === "media"));
+			tools.hidden.setVisible($("stream-image"), (mode === "mjpeg"));
 			tools.hidden.setVisible($("stream-video"), (mode === "janus"));
 			__resetStream(mode);
 		}
diff --git a/web/share/js/kvm/stream_janus.js b/web/share/js/kvm/stream_janus.js
index 27d3f55d..df7c48f1 100644
--- a/web/share/js/kvm/stream_janus.js
+++ b/web/share/js/kvm/stream_janus.js
@@ -32,6 +32,8 @@ var _Janus = null;
 export function JanusStreamer(__setActive, __setInactive, __setInfo, __orient, __allow_audio) {
 	var self = this;
 
+	/************************************************************************/
+
 	var __stop = false;
 	var __ensuring = false;
 
@@ -45,10 +47,12 @@ export function JanusStreamer(__setActive, __setInactive, __setInfo, __orient, _
 	var __state = null;
 	var __frames = 0;
 
+	/************************************************************************/
+
 	self.getOrientation = () => __orient;
 	self.isAudioAllowed = () => __allow_audio;
 
-	self.getName = () => (__allow_audio ? "H.264 + Audio" : "H.264");
+	self.getName = () => (__allow_audio ? "WebRTC H.264 + Audio" : "WebRTC H.264");
 	self.getMode = () => "janus";
 
 	self.getResolution = function() {
@@ -75,9 +79,9 @@ export function JanusStreamer(__setActive, __setInactive, __setInfo, __orient, _
 
 	var __ensureJanus = function(internal) {
 		if (__janus === null && !__stop && (!__ensuring || internal)) {
+			__ensuring = true;
 			__setInactive();
 			__setInfo(false, false, "");
-			__ensuring = true;
 			__logInfo("Starting Janus ...");
 			__janus = new _Janus({
 				"server": `${tools.is_https ? "wss" : "ws"}://${location.host}/janus/ws`,
@@ -447,11 +451,3 @@ JanusStreamer.ensure_janus = function(callback) {
 JanusStreamer.is_webrtc_available = function() {
 	return !!window.RTCPeerConnection;
 };
-
-JanusStreamer.is_h264_available = function() {
-	let ok = true;
-	if ($("stream-video").canPlayType) {
-		ok = $("stream-video").canPlayType("video/mp4; codecs=\"avc1.42E01F\"");
-	}
-	return ok;
-};
diff --git a/web/share/js/kvm/stream_media.js b/web/share/js/kvm/stream_media.js
new file mode 100644
index 00000000..8c84e8de
--- /dev/null
+++ b/web/share/js/kvm/stream_media.js
@@ -0,0 +1,240 @@
+/*****************************************************************************
+#                                                                            #
+#    KVMD - The main PiKVM daemon.                                           #
+#                                                                            #
+#    Copyright (C) 2018-2024  Maxim Devaev <[email protected]>               #
+#                                                                            #
+#    This program is free software: you can redistribute it and/or modify    #
+#    it under the terms of the GNU General Public License as published by    #
+#    the Free Software Foundation, either version 3 of the License, or       #
+#    (at your option) any later version.                                     #
+#                                                                            #
+#    This program is distributed in the hope that it will be useful,         #
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of          #
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           #
+#    GNU General Public License for more details.                            #
+#                                                                            #
+#    You should have received a copy of the GNU General Public License       #
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.  #
+#                                                                            #
+*****************************************************************************/
+
+
+"use strict";
+
+
+import {tools, $} from "../tools.js";
+
+
+export function MediaStreamer(__setActive, __setInactive, __setInfo) {
+	var self = this;
+
+	/************************************************************************/
+
+	var __stop = false;
+	var __ensuring = false;
+
+	var __ws = null;
+	var __ping_timer = null;
+	var __missed_heartbeats = 0;
+	var __decoder = null;
+	var __codec = "";
+
+	var __state = null;
+	var __frames = 0;
+
+	/************************************************************************/
+
+	self.getName = () => "HTTP H.264";
+	self.getMode = () => "media";
+
+	self.getResolution = function() {
+		let el = $("stream-canvas");
+		return {
+			// Разрешение видео или элемента
+			"real_width": (el.width || el.offsetWidth),
+			"real_height": (el.height || el.offsetHeight),
+			"view_width": el.offsetWidth,
+			"view_height": el.offsetHeight,
+		};
+	};
+
+	self.ensureStream = function(state) {
+		__state = state;
+		__stop = false;
+		__ensureMedia(false);
+	};
+
+	self.stopStream = function() {
+		__stop = true;
+		__ensuring = false;
+		__wsForceClose();
+		__setInfo(false, false, "");
+	};
+
+	var __ensureMedia = function(internal) {
+		if (__ws === null && !__stop && (!__ensuring || internal)) {
+			__ensuring = true;
+			__setInactive();
+			__setInfo(false, false, "");
+			__logInfo("Starting Media ...");
+			__ws = new WebSocket(`${tools.is_https ? "wss" : "ws"}://${location.host}/api/media/ws`);
+			__ws.binaryType = "arraybuffer";
+			__ws.onopen = __wsOpenHandler;
+			__ws.onerror = __wsErrorHandler;
+			__ws.onclose = __wsCloseHandler;
+			__ws.onmessage = async (event) => {
+				if (typeof event.data === "string") {
+					__wsJsonHandler(JSON.parse(event.data));
+				} else { // Binary
+					await __wsBinHandler(event.data);
+				}
+			};
+		}
+	};
+
+	var __wsOpenHandler = function(event) {
+		__logInfo("Socket opened:", event);
+		__missed_heartbeats = 0;
+		__ping_timer = setInterval(__ping, 1000);
+	};
+
+	var __ping = function() {
+		try {
+			__missed_heartbeats += 1;
+			if (__missed_heartbeats >= 5) {
+				throw new Error("Too many missed heartbeats");
+			}
+			__ws.send(new Uint8Array([0]));
+
+			if (__decoder && __decoder.state === "configured") {
+				let online = !!(__state && __state.source.online);
+				let info = `${__frames} fps dynamic`;
+				__frames = 0;
+				__setInfo(true, online, info);
+			}
+		} catch (ex) {
+			__wsErrorHandler(ex.message);
+		}
+	};
+
+	var __wsForceClose = function() {
+		if (__ws) {
+			__ws.onclose = null;
+			__ws.close();
+		}
+		__wsCloseHandler(null);
+		__setInactive();
+	};
+
+	var __wsErrorHandler = function(event) {
+		__logInfo("Socket error:", event);
+		__setInfo(false, false, event);
+		__wsForceClose();
+	};
+
+	var __wsCloseHandler = function(event) {
+		__logInfo("Socket closed:", event);
+		if (__ping_timer) {
+			clearInterval(__ping_timer);
+			__ping_timer = null;
+		}
+		if (__decoder) {
+			__decoder.close();
+			__decoder = null;
+		}
+		__missed_heartbeats = 0;
+		__frames = 0;
+		__ws = null;
+		if (!__stop) {
+			setTimeout(() => __ensureMedia(true), 1000);
+		}
+	};
+
+	var __wsJsonHandler = function(event) {
+		if (event.event_type === "media") {
+			__decoderCreate(event.event.video);
+		}
+	};
+
+	var __wsBinHandler = async (data) => {
+		let header = new Uint8Array(data.slice(0, 2));
+
+		if (header[0] === 255) { // Pong
+			__missed_heartbeats = 0;
+
+		} else if (header[0] === 1 && __decoder !== null) { // Video frame
+			let key = !!header[1];
+			if (__decoder.state !== "configured") {
+				if (!key) {
+					return;
+				}
+				await __decoder.configure({"codec": __codec, "optimizeForLatency": true});
+			}
+
+			let chunk = new EncodedVideoChunk({ // eslint-disable-line no-undef
+				"timestamp": (performance.now() + performance.timeOrigin) * 1000,
+				"type": (key ? "key" : "delta"),
+				"data": data.slice(2),
+			});
+			await __decoder.decode(chunk);
+		}
+	};
+
+	var __decoderCreate = function(formats) {
+		__decoderDestroy();
+
+		if (formats.h264 === undefined) {
+			let msg = "No H.264 stream available on PiKVM";
+			__setInfo(false, false, msg);
+			__logInfo(msg);
+			return;
+		}
+		if (!window.VideoDecoder) {
+			let msg = "This browser can't handle direct H.264 stream";
+			if (!tools.is_https) {
+				msg = "Direct H.264 requires HTTPS";
+			}
+			__setInfo(false, false, msg);
+			__logInfo(msg);
+			return;
+		}
+
+		__decoder = new VideoDecoder({ // eslint-disable-line no-undef
+			"output": (frame) => {
+				try {
+					let canvas = $("stream-canvas");
+					if (canvas.width !== frame.displayWidth || canvas.height !== frame.displayHeight) {
+						canvas.width = frame.displayWidth;
+						canvas.height = frame.displayHeight;
+					}
+					canvas.getContext("2d").drawImage(frame, 0, 0);
+					__frames += 1;
+				} finally {
+					frame.close();
+				}
+			},
+			"error": (err) => __logInfo(err.message),
+		});
+		__codec = `avc1.${formats.h264.profile_level_id}`;
+
+		__ws.send(JSON.stringify({
+			"event_type": "start",
+			"event": {"kind": "video", "format": "h264"},
+		}));
+	};
+
+	var __decoderDestroy = function() {
+		if (__decoder !== null) {
+			__decoder.close();
+			__decoder = null;
+			__codec = "";
+		}
+	};
+
+	var __logInfo = (...args) => tools.info("Stream [Media]:", ...args);
+}
+
+MediaStreamer.is_videodecoder_available = function() {
+	return !!window.VideoDecoder;
+};
diff --git a/web/share/js/kvm/stream_mjpeg.js b/web/share/js/kvm/stream_mjpeg.js
index 6efbff7d..1fd1548e 100644
--- a/web/share/js/kvm/stream_mjpeg.js
+++ b/web/share/js/kvm/stream_mjpeg.js
@@ -41,7 +41,7 @@ export function MjpegStreamer(__setActive, __setInactive, __setInfo) {
 
 	/************************************************************************/
 
-	self.getName = () => "MJPEG";
+	self.getName = () => "HTTP MJPEG";
 	self.getMode = () => "mjpeg";
 
 	self.getResolution = function() {