ocr

9 files changed, 255 insertions, 45 deletions

diff --git a/web/share/css/kvm/hid.css b/web/share/css/kvm/hid.css
index 02d2f8a3..e27812ce 100644
--- a/web/share/css/kvm/hid.css
+++ b/web/share/css/kvm/hid.css
@@ -23,8 +23,8 @@
 textarea#hid-pak-text {
 	display: block;
 	resize: none;
-	height: 150px;
-	width: 300px;
+	height: 120px;
+	width: 320px;
 	border: var(--border-default-thin);
 	border-radius: 4px;
 	color: var(--cs-code-default-fg);
diff --git a/web/share/css/kvm/stream.css b/web/share/css/kvm/stream.css
index a641e296..0ead41cb 100644
--- a/web/share/css/kvm/stream.css
+++ b/web/share/css/kvm/stream.css
@@ -29,6 +29,26 @@ div#stream-info {
 	display: none;
 }
 
+div#stream-ocr-window {
+	cursor: crosshair;
+	position: absolute;
+	width: 100%;
+	height: 100%;
+	top: 0;
+	left: 0;
+	z-index: 1;
+	background-color: unset !important;
+	border-radius: unset !important;
+	border: unset !important;
+	padding: 0px !important;
+	background: radial-gradient(transparent 15%, black);
+}
+div#stream-ocr-selection {
+	position: relative;
+	background-color: #5b90bb50;
+	box-shadow: inset 0 0 0px 1px #e8e8e8cd;
+}
+
 div#stream-box {
 	width: 100%;
 	height: 100%;
diff --git a/web/share/js/kvm/hid.js b/web/share/js/kvm/hid.js
index 67d919cf..ecf8d39f 100644
--- a/web/share/js/kvm/hid.js
+++ b/web/share/js/kvm/hid.js
@@ -30,7 +30,7 @@ import {Keyboard} from "./keyboard.js";
 import {Mouse} from "./mouse.js";
 
 
-export function Hid(__getResolution, __recorder) {
+export function Hid(__getGeometry, __recorder) {
 	var self = this;
 
 	/************************************************************************/
@@ -40,7 +40,7 @@ export function Hid(__getResolution, __recorder) {
 
 	var __init__ = function() {
 		__keyboard = new Keyboard(__recorder.recordWsEvent);
-		__mouse = new Mouse(__getResolution, __recorder.recordWsEvent);
+		__mouse = new Mouse(__getGeometry, __recorder.recordWsEvent);
 
 		let hidden_attr = null;
 		let visibility_change_attr = null;
diff --git a/web/share/js/kvm/mouse.js b/web/share/js/kvm/mouse.js
index d00fae8d..cbcd99da 100644
--- a/web/share/js/kvm/mouse.js
+++ b/web/share/js/kvm/mouse.js
@@ -27,7 +27,7 @@ import {tools, $} from "../tools.js";
 import {Keypad} from "../keypad.js";
 
 
-export function Mouse(__getResolution, __recordWsEvent) {
+export function Mouse(__getGeometry, __recordWsEvent) {
 	var self = this;
 
 	/************************************************************************/
@@ -227,10 +227,10 @@ export function Mouse(__getResolution, __recordWsEvent) {
 		if (__absolute) {
 			let pos = __current_pos;
 			if (pos.x !== __sent_pos.x || pos.y !== __sent_pos.y) {
-				let geo = __getVideoGeometry();
+				let geo = __getGeometry();
 				let to = {
-					"x": __translatePosition(pos.x, geo.x, geo.width, -32768, 32767),
-					"y": __translatePosition(pos.y, geo.y, geo.height, -32768, 32767),
+					"x": tools.remap(pos.x, geo.x, geo.width, -32768, 32767),
+					"y": tools.remap(pos.y, geo.y, geo.height, -32768, 32767),
 				};
 				tools.debug("Mouse: moved:", to);
 				__sendEvent("mouse_move", {"to": to});
@@ -243,36 +243,6 @@ export function Mouse(__getResolution, __recordWsEvent) {
 		}
 	};
 
-	var __getVideoGeometry = function() {
-		// Первоначально обновление геометрии считалось через ResizeObserver.
-		// Но оно не ловило некоторые события, например в последовательности:
-		//   - Находять в HD переходим в фулскрин
-		//   - Меняем разрешение на маленькое
-		//   - Убираем фулскрин
-		//   - Переходим в HD
-		//   - Видим нарушение пропорций
-		// Так что теперь используются быстре рассчеты через offset*
-		// вместо getBoundingClientRect().
-		let res = __getResolution();
-		let ratio = Math.min(res.view_width / res.real_width, res.view_height / res.real_height);
-		return {
-			"x": Math.round((res.view_width - ratio * res.real_width) / 2),
-			"y": Math.round((res.view_height - ratio * res.real_height) / 2),
-			"width": Math.round(ratio * res.real_width),
-			"height": Math.round(ratio * res.real_height),
-		};
-	};
-
-	var __translatePosition = function(x, a, b, c, d) {
-		let translated = Math.round((x - a) / b * (d - c) + c);
-		if (translated < c) {
-			return c;
-		} else if (translated > d) {
-			return d;
-		}
-		return translated;
-	};
-
 	var __streamWheelHandler = function(event) {
 		// https://learn.javascript.ru/mousewheel
 		// https://stackoverflow.com/a/24595588
diff --git a/web/share/js/kvm/ocr.js b/web/share/js/kvm/ocr.js
new file mode 100644
index 00000000..c7fcf643
--- /dev/null
+++ b/web/share/js/kvm/ocr.js
@@ -0,0 +1,181 @@
+/*****************************************************************************
+#                                                                            #
+#    KVMD - The main PiKVM daemon.                                           #
+#                                                                            #
+#    Copyright (C) 2018-2022  Maxim Devaev <[email protected]>               #
+#                                                                            #
+#    This program is free software: you can redistribute it and/or modify    #
+#    it under the terms of the GNU General Public License as published by    #
+#    the Free Software Foundation, either version 3 of the License, or       #
+#    (at your option) any later version.                                     #
+#                                                                            #
+#    This program is distributed in the hope that it will be useful,         #
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of          #
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the           #
+#    GNU General Public License for more details.                            #
+#                                                                            #
+#    You should have received a copy of the GNU General Public License       #
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.  #
+#                                                                            #
+*****************************************************************************/
+
+
+"use strict";
+
+
+import {tools, $} from "../tools.js";
+import {wm} from "../wm.js";
+
+
+export function Ocr(__getGeometry) {
+	var self = this;
+
+	/************************************************************************/
+
+	var __start_pos = null;
+	var __end_pos = null;
+	var __selection = null;
+
+	var __init__ = function() {
+		tools.el.setOnClick($("stream-ocr-button"), function() {
+			__resetSelection();
+			wm.showWindow($("stream-window"));
+			wm.showWindow($("stream-ocr-window"));
+		});
+
+		$("stream-ocr-lang-selector").addEventListener("change", function() {
+			tools.storage.set("stream.ocr.lang", $("stream-ocr-lang-selector").value);
+		});
+
+		$("stream-ocr-window").addEventListener("blur", __resetSelection);
+		$("stream-ocr-window").addEventListener("resize", __resetSelection);
+		$("stream-ocr-window").close_hook = __resetSelection;
+
+		$("stream-ocr-window").onkeyup = function(event) {
+			event.preventDefault();
+			if (event.code === "Enter") {
+				__recognizeSelection();
+				wm.closeWindow($("stream-ocr-window"));
+			} else if (event.code === "Escape") {
+				wm.closeWindow($("stream-ocr-window"));
+			}
+		};
+
+		$("stream-ocr-window").onmousedown = __startSelection;
+		$("stream-ocr-window").onmousemove = __changeSelection;
+		$("stream-ocr-window").onmouseup = __endSelection;
+	};
+
+	/************************************************************************/
+
+	self.setState = function(state) {
+		let enabled = (state && state.ocr.enabled && navigator.clipboard && !tools.browser.is_ios);
+		if (enabled) {
+			let selected = tools.storage.get("stream.ocr.lang", state.ocr.langs["default"]);
+			let html = "";
+			for (let variant of state.ocr.langs.available) {
+				html += `<option value=${variant} ${variant === selected ? "selected" : ""}>${variant}</option>`;
+			}
+			$("stream-ocr-lang-selector").innerHTML = html;
+		}
+		tools.feature.setEnabled($("stream-ocr"), enabled);
+		$("stream-ocr-led").className = (enabled ? "led-gray" : "hidden");
+	};
+
+	var __startSelection = function(event) {
+		if (__start_pos === null) {
+			tools.hidden.setVisible($("stream-ocr-selection"), false);
+			__start_pos = __getGlobalPosition(event);
+			__end_pos = null;
+		}
+	};
+
+	var __changeSelection = function(event) {
+		if (__start_pos !== null) {
+			__end_pos = __getGlobalPosition(event);
+			let width = Math.abs(__start_pos.x - __end_pos.x);
+			let height = Math.abs(__start_pos.y - __end_pos.y);
+			let el_selection = $("stream-ocr-selection");
+			el_selection.style.left = Math.min(__start_pos.x, __end_pos.x) + "px";
+			el_selection.style.top = Math.min(__start_pos.y, __end_pos.y) + "px";
+			el_selection.style.width = width + "px";
+			el_selection.style.height = height + "px";
+			tools.hidden.setVisible(el_selection, (width > 1 || height > 1));
+		}
+	};
+
+	var __endSelection = function(event) {
+		__changeSelection(event);
+		let el_selection = $("stream-ocr-selection");
+		let ok = (
+			el_selection.offsetWidth > 1 && el_selection.offsetHeight > 1
+			&& __start_pos !== null && __end_pos !== null
+		);
+		tools.hidden.setVisible(el_selection, ok);
+		if (ok) {
+			let rect = $("stream-box").getBoundingClientRect();
+			let rel_left = Math.min(__start_pos.x, __end_pos.x) - rect.left;
+			let rel_right = Math.max(__start_pos.x, __end_pos.x) - rect.left;
+			let rel_top = Math.min(__start_pos.y, __end_pos.y) - rect.top;
+			let rel_bottom = Math.max(__start_pos.y, __end_pos.y) - rect.top;
+			let geo = __getGeometry();
+			__selection = {
+				left: tools.remap(rel_left, geo.x, geo.width, 0, geo.real_width),
+				right: tools.remap(rel_right, geo.x, geo.width, 0, geo.real_width),
+				top: tools.remap(rel_top, geo.y, geo.height, 0, geo.real_height),
+				bottom: tools.remap(rel_bottom, geo.y, geo.height, 0, geo.real_height),
+			};
+		} else {
+			__selection = null;
+		}
+		__start_pos = null;
+		__end_pos = null;
+	};
+
+	var __getGlobalPosition = function(event) {
+		let rect = $("stream-box").getBoundingClientRect();
+		let geo = __getGeometry();
+		return {
+			x: Math.min(Math.max(event.clientX, rect.left + geo.x), rect.right - geo.x),
+			y: Math.min(Math.max(event.clientY, rect.top + geo.y), rect.bottom - geo.y),
+		};
+	};
+
+	var __resetSelection = function() {
+		tools.hidden.setVisible($("stream-ocr-selection"), false);
+		__start_pos = null;
+		__end_pos = null;
+		__selection = null;
+	};
+
+	var __recognizeSelection = function() {
+		tools.el.setEnabled($("stream-ocr-button"), false);
+		tools.el.setEnabled($("stream-ocr-lang-selector"), false);
+		$("stream-ocr-led").className = "led-yellow-rotating-fast";
+
+		let lang = $("stream-ocr-lang-selector").value;
+		let url = `/api/streamer/snapshot?ocr=1&ocr_lang=${lang}`;
+		url += `&ocr_left=${__selection.left}&ocr_top=${__selection.top}`;
+		url += `&ocr_right=${__selection.right}&ocr_bottom=${__selection.bottom}`;
+
+		let http = tools.makeRequest("GET", url, function() {
+			if (http.readyState === 4) {
+				if (http.status === 200) {
+					navigator.clipboard.writeText(http.responseText).then(function() {
+						wm.info("The text is copied to the clipboard");
+					}, function() {
+						wm.error("Can't copy text to the clipboard");
+					});
+				} else {
+					wm.error("OCR error:<br>", http.responseText);
+				}
+
+				tools.el.setEnabled($("stream-ocr-button"), true);
+				tools.el.setEnabled($("stream-ocr-lang-selector"), true);
+				$("stream-ocr-led").className = "led-gray";
+			}
+		});
+	};
+
+	__init__();
+}
diff --git a/web/share/js/kvm/session.js b/web/share/js/kvm/session.js
index 81c7c12e..68320655 100644
--- a/web/share/js/kvm/session.js
+++ b/web/share/js/kvm/session.js
@@ -32,6 +32,7 @@ import {Atx} from "./atx.js";
 import {Msd} from "./msd.js";
 import {Streamer} from "./stream.js";
 import {Gpio} from "./gpio.js";
+import {Ocr} from "./ocr.js";
 
 
 export function Session() {
@@ -46,10 +47,11 @@ export function Session() {
 
 	var __streamer = new Streamer();
 	var __recorder = new Recorder();
-	var __hid = new Hid(__streamer.getResolution, __recorder);
+	var __hid = new Hid(__streamer.getGeometry, __recorder);
 	var __atx = new Atx(__recorder);
 	var __msd = new Msd();
 	var __gpio = new Gpio(__recorder);
+	var __ocr = new Ocr(__streamer.getGeometry);
 
 	var __init__ = function() {
 		__startSession();
@@ -251,6 +253,7 @@ export function Session() {
 			case "atx_state": __atx.setState(data.event); break;
 			case "msd_state": __msd.setState(data.event); break;
 			case "streamer_state": __streamer.setState(data.event); break;
+			case "streamer_ocr_state": __ocr.setState(data.event); break;
 		}
 	};
 
@@ -273,6 +276,7 @@ export function Session() {
 			__ping_timer = null;
 		}
 
+		__ocr.setState(null);
 		__gpio.setState(null);
 		__hid.setSocket(null);
 		__recorder.setSocket(null);
diff --git a/web/share/js/kvm/stream.js b/web/share/js/kvm/stream.js
index fad90123..8e68e4a6 100644
--- a/web/share/js/kvm/stream.js
+++ b/web/share/js/kvm/stream.js
@@ -455,8 +455,26 @@ export function Streamer() {
 
 	/************************************************************************/
 
-	self.getResolution = function() {
-		return __streamer.getResolution();
+	self.getGeometry = function() {
+		// Первоначально обновление геометрии считалось через ResizeObserver.
+		// Но оно не ловило некоторые события, например в последовательности:
+		//   - Находять в HD переходим в фулскрин
+		//   - Меняем разрешение на маленькое
+		//   - Убираем фулскрин
+		//   - Переходим в HD
+		//   - Видим нарушение пропорций
+		// Так что теперь используются быстре рассчеты через offset*
+		// вместо getBoundingClientRect().
+		let res = __streamer.getResolution();
+		let ratio = Math.min(res.view_width / res.real_width, res.view_height / res.real_height);
+		return {
+			"x": Math.round((res.view_width - ratio * res.real_width) / 2),
+			"y": Math.round((res.view_height - ratio * res.real_height) / 2),
+			"width": Math.round(ratio * res.real_width),
+			"height": Math.round(ratio * res.real_height),
+			"real_width": res.real_width,
+			"real_height": res.real_height,
+		};
 	};
 
 	self.setJanusEnabled = function(enabled) {
diff --git a/web/share/js/tools.js b/web/share/js/tools.js
index a412cc0d..006453bd 100644
--- a/web/share/js/tools.js
+++ b/web/share/js/tools.js
@@ -83,6 +83,16 @@ export var tools = new function() {
 		return `${hours}:${mins}:${secs}.${millis}`;
 	};
 
+	self.remap = function(x, a1, b1, a2, b2) {
+		let remapped = Math.round((x - a1) / b1 * (b2 - a2) + a2);
+		if (remapped < a2) {
+			return a2;
+		} else if (remapped > b2) {
+			return b2;
+		}
+		return remapped;
+	};
+
 	/************************************************************************/
 
 	self.el = new function() {
diff --git a/web/share/js/wm.js b/web/share/js/wm.js
index 78a22140..823272d8 100644
--- a/web/share/js/wm.js
+++ b/web/share/js/wm.js
@@ -84,10 +84,7 @@ function __WindowManager() {
 			let el_close_button = el_window.querySelector(".window-header .window-button-close");
 			if (el_close_button) {
 				el_close_button.title = "Close window";
-				tools.el.setOnClick(el_close_button, function() {
-					__closeWindow(el_window);
-					__activateLastWindow(el_window);
-				});
+				tools.el.setOnClick(el_close_button, () => self.closeWindow(el_window));
 			}
 
 			let el_maximize_button = el_window.querySelector(".window-header .window-button-maximize");
@@ -139,6 +136,7 @@ function __WindowManager() {
 
 	/************************************************************************/
 
+	self.info = (...args) => __modalDialog("Info", args.join(" "), true, false, null);
 	self.error = (...args) => __modalDialog("Error", args.join(" "), true, false, null);
 	self.confirm = (...args) => __modalDialog("Question", args.join(" "), true, true, null);
 
@@ -253,6 +251,11 @@ function __WindowManager() {
 		};
 	};
 
+	self.closeWindow = function(el_window) {
+		__closeWindow(el_window);
+		__activateLastWindow(el_window);
+	};
+
 	var __closeWindow = function(el_window) {
 		el_window.focus();
 		el_window.blur();
@@ -460,6 +463,10 @@ function __WindowManager() {
 	var __makeWindowMovable = function(el_window) {
 		let el_header = el_window.querySelector(".window-header");
 		let el_grab = el_window.querySelector(".window-header .window-grab");
+		if (el_header === null || el_grab === null) {
+			// Для псевдоокна OCR
+			return;
+		}
 
 		let prev_pos = {x: 0, y: 0};