From bb23ab3ec22c8a30081e4ef4fdc90e73fc338c9c Mon Sep 17 00:00:00 2001 From: dtookey Date: Fri, 11 Aug 2023 20:47:59 -0400 Subject: [PATCH] i'm done for tonight --- .../kotlin/controllers/VisionController.kt | 7 +- src/main/kotlin/controllers/windows/User32.kt | 35 +++++ .../controllers/windows/WindowsOSProxy.kt | 121 +++++++++++++++++- .../controllers/VisionControllerTest.kt | 15 ++- 4 files changed, 170 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/controllers/VisionController.kt b/src/main/kotlin/controllers/VisionController.kt index 6587f87..20ce7a5 100644 --- a/src/main/kotlin/controllers/VisionController.kt +++ b/src/main/kotlin/controllers/VisionController.kt @@ -2,6 +2,7 @@ package controllers import controllers.windows.WindowsOSProxy import java.awt.image.BufferedImage +import java.awt.image.MultiResolutionImage /** * Interface for a vision controller that can take screenshots. @@ -22,7 +23,7 @@ interface VisionController : Automaton, OSProxy { * @return A BufferedImage containing the screenshot of the foreground * window. */ - fun takeScreenshotOfForeground(): BufferedImage + fun takeScreenshotOfForeground(): MultiResolutionImage } /** @@ -44,9 +45,9 @@ class ConcreteVisionController : VisionController, WindowsOSProxy, RobotAutomato * @return A [BufferedImage] containing a screenshot of the foreground window * scaled to account for the screen DPI. */ - override fun takeScreenshotOfForeground(): BufferedImage { + override fun takeScreenshotOfForeground(): MultiResolutionImage { val rect = getScaledForegroundWindowBounds() - return robot.createScreenCapture(rect) + return robot.createMultiResolutionScreenCapture(rect) } } \ No newline at end of file diff --git a/src/main/kotlin/controllers/windows/User32.kt b/src/main/kotlin/controllers/windows/User32.kt index 52bfd0b..51aaa19 100644 --- a/src/main/kotlin/controllers/windows/User32.kt +++ b/src/main/kotlin/controllers/windows/User32.kt @@ -1,6 +1,7 @@ package controllers.windows import com.sun.jna.Native +import com.sun.jna.NativeLong import com.sun.jna.Pointer import com.sun.jna.win32.StdCallLibrary import controllers.windows.WindowsOSProxy.WinRect @@ -129,6 +130,40 @@ interface User32 : StdCallLibrary { */ fun SetForegroundWindow(hWnd: Pointer?): Boolean + + /** + * Converts a logical point to a physical point for a window based on the DPI awareness per monitor. + * + * This calls the Win32 API LogicalToPhysicalPointForPerMonitorDPI function to convert a logical point + * (x, y coordinates) relative to a window into a physical point based on the DPI awareness context of + * the given window. This accounts for different DPI scaling settings on each monitor. + * + * The logical point is specified in the [lpPoint] struct as: + * + * ``` + * struct POINT { + * LONG x; + * LONG y; + * } + * ``` + * + * On return, [lpPoint] will contain the converted physical coordinates. + * + * @param hWnd The window handle (HWND) to use the DPI context of for conversion. + * @param lpPoint Pointer to a POINT struct containing the logical coordinates. + * This will be updated with the converted physical coordinates. + */ + fun LogicalToPhysicalPointForPerMonitorDPI(hWnd: Pointer?, lpPoint: Pointer?): Boolean + + fun PhysicalToLogicalPointForPerMonitorDPI(hWnd: Pointer?, lpPoint: Pointer?): Boolean + + fun DwmGetWindowAttribute( + hwnd: Pointer?, + dwAttribute: Int, + pvAttribute: Pointer?, + cbAttribute: Int + ): NativeLong + /** * Retrieves the coordinates of a window on the screen. * diff --git a/src/main/kotlin/controllers/windows/WindowsOSProxy.kt b/src/main/kotlin/controllers/windows/WindowsOSProxy.kt index 49c4dc1..8d7f3b2 100644 --- a/src/main/kotlin/controllers/windows/WindowsOSProxy.kt +++ b/src/main/kotlin/controllers/windows/WindowsOSProxy.kt @@ -74,12 +74,28 @@ interface WindowsOSProxy : MousePointerObserver, OSProxy { } + /** + * Mirror of the Windows POINT struct used for window/physical coordinates + */ + @FieldOrder("x", "y") + class WinPoint() : Structure() { + constructor(x: Int, y: Int) : this() { + this.x = x + this.y = y + } + + @JvmField + var x: Int = 0 + + @JvmField + var y: Int = 0 + } /** * Mirror of the Windows RECT struct used for window coordinates. */ @FieldOrder("left", "top", "right", "bottom") - public class WinRect : Structure() { + class WinRect : Structure() { /** The x coordinate of the left edge */ @JvmField @@ -98,6 +114,64 @@ interface WindowsOSProxy : MousePointerObserver, OSProxy { var bottom: Int = 0 } + enum class DwmWindowAttribute { + + DWMWA_NCRENDERING_ENABLED, + DWMWA_NCRENDERING_POLICY, + DWMWA_TRANSITIONS_FORCEDISABLED, + DWMWA_ALLOW_NCPAINT, + DWMWA_CAPTION_BUTTON_BOUNDS, + DWMWA_NONCLIENT_RTL_LAYOUT, + DWMWA_FORCE_ICONIC_REPRESENTATION, + DWMWA_FLIP3D_POLICY, + DWMWA_EXTENDED_FRAME_BOUNDS, + DWMWA_HAS_ICONIC_BITMAP, + DWMWA_DISALLOW_PEEK, + DWMWA_EXCLUDED_FROM_PEEK, + DWMWA_CLOAK, + DWMWA_CLOAKED, + DWMWA_FREEZE_REPRESENTATION, + DWMWA_PASSIVE_UPDATE_MODE, + DWMWA_USE_HOSTBACKDROPBRUSH, + DWMWA_USE_IMMERSIVE_DARK_MODE, + DWMWA_WINDOW_CORNER_PREFERENCE, + DWMWA_BORDER_COLOR, + DWMWA_CAPTION_COLOR, + DWMWA_TEXT_COLOR, + DWMWA_VISIBLE_FRAME_BORDER_THICKNESS, + DWMWA_SYSTEMBACKDROP_TYPE, + DWMWA_LAST; + + companion object { + val DWMWA_NCRENDERING_ENABLED = 0 + val DWMWA_NCRENDERING_POLICY = 1 + val DWMWA_TRANSITIONS_FORCEDISABLED = 2 + val DWMWA_ALLOW_NCPAINT = 3 + val DWMWA_CAPTION_BUTTON_BOUNDS = 4 + val DWMWA_NONCLIENT_RTL_LAYOUT = 5 + val DWMWA_FORCE_ICONIC_REPRESENTATION = 6 + val DWMWA_FLIP3D_POLICY = 7 + val DWMWA_EXTENDED_FRAME_BOUNDS = 8 + val DWMWA_HAS_ICONIC_BITMAP = 9 + val DWMWA_DISALLOW_PEEK = 10 + val DWMWA_EXCLUDED_FROM_PEEK = 11 + val DWMWA_CLOAK = 12 + val DWMWA_CLOAKED = 13 + val DWMWA_FREEZE_REPRESENTATION = 14 + val DWMWA_PASSIVE_UPDATE_MODE = 15 + val DWMWA_USE_HOSTBACKDROPBRUSH = 16 + val DWMWA_USE_IMMERSIVE_DARK_MODE = 17 + val DWMWA_WINDOW_CORNER_PREFERENCE = 18 + val DWMWA_BORDER_COLOR = 19 + val DWMWA_CAPTION_COLOR = 20 + val DWMWA_TEXT_COLOR = 21 + val DWMWA_VISIBLE_FRAME_BORDER_THICKNESS = 22 + val DWMWA_SYSTEMBACKDROP_TYPE = 23 + val DWMWA_LAST = 24 + } + } + + /** * Gets the title of the active/foreground window. * @@ -244,6 +318,25 @@ interface WindowsOSProxy : MousePointerObserver, OSProxy { override fun getScaledForegroundWindowBounds(): Rectangle? { val user32 = User32.INSTANCE val hWnd = user32.GetForegroundWindow() + + val rect = WinRect() + val success = user32.DwmGetWindowAttribute( + hWnd, + 8, + rect.pointer, + rect.size() + ) + return if (true){ + rect.read() + Rectangle(rect.top, rect.left, (rect.right - rect.left), rect.bottom - rect.top) + }else{ + null + } + } + + fun barelyFunctionalWindowQuery(): Rectangle? { + val user32 = User32.INSTANCE + val hWnd = user32.GetForegroundWindow() val rect = getRectFromWindowHandle(user32, hWnd) val defaultDPI = 96 @@ -297,10 +390,32 @@ interface WindowsOSProxy : MousePointerObserver, OSProxy { return if (success) { //the values are stuck down in memory, so we have to read these back out in order to proceed rect.read() - rect + + // Convert top-left point to physical coordinates + val topLeft = WinPoint(rect.left, rect.top) //rect.top, rect.left + val tlSuccess = user32.PhysicalToLogicalPointForPerMonitorDPI(hWnd, topLeft.pointer) + + // Convert bottom-right point to physical coordinates + val bottomRight = WinPoint(rect.right, rect.bottom) + val brSuccess = user32.PhysicalToLogicalPointForPerMonitorDPI(hWnd, bottomRight.pointer) + + + return if (tlSuccess && brSuccess) { + // Create new rect with physical coordinates + val returnValue = WinRect() + returnValue.top = topLeft.y + returnValue.left = topLeft.x + returnValue.bottom = bottomRight.y + returnValue.right = bottomRight.x + + returnValue + } else { + // Failed to convert logical to physical points + null + } } else { + // Failed to get window rect null } } - } \ No newline at end of file diff --git a/src/test/kotlin/controllers/VisionControllerTest.kt b/src/test/kotlin/controllers/VisionControllerTest.kt index b1253aa..77acfa2 100644 --- a/src/test/kotlin/controllers/VisionControllerTest.kt +++ b/src/test/kotlin/controllers/VisionControllerTest.kt @@ -1,12 +1,23 @@ package controllers +import java.awt.image.BufferedImage +import java.nio.file.Paths +import javax.imageio.ImageIO import kotlin.test.Test +import kotlin.test.assertNotNull class VisionControllerTest { @Test - fun testImageCapture(){ + fun testImageCapture() { val vc = ConcreteVisionController() - vc.takeScreenshotOfForeground() + val bi = vc.takeScreenshotOfForeground() + assertNotNull(bi) + + for (i in 0..