diff --git a/src/main/kotlin/controllers/Automaton.kt b/src/main/kotlin/controllers/Automaton.kt index 0508a02..e2af7f1 100644 --- a/src/main/kotlin/controllers/Automaton.kt +++ b/src/main/kotlin/controllers/Automaton.kt @@ -6,7 +6,7 @@ package controllers * * Automaton combines capabilities from other interfaces to create a controller that can: * - * - Get desktop and mouse state information like pointer location via [DesktopController] + * - Get desktop and mouse state information like pointer location via [MousePointerObserver] * * - Perform mouse and keyboard input like clicks, key presses, and scrolling via [InputController] * @@ -23,4 +23,4 @@ package controllers * * This interface allows the underlying OS/desktop implementation details to be abstracted and swapped as needed. */ -interface Automaton : DesktopController, InputController, TemporalController \ No newline at end of file +interface Automaton : MousePointerObserver, InputController, TemporalController \ No newline at end of file diff --git a/src/main/kotlin/controllers/DesktopController.kt b/src/main/kotlin/controllers/DesktopController.kt deleted file mode 100644 index 8987836..0000000 --- a/src/main/kotlin/controllers/DesktopController.kt +++ /dev/null @@ -1,438 +0,0 @@ -package controllers - -import com.sun.jna.Native -import com.sun.jna.Pointer -import com.sun.jna.win32.StdCallLibrary -import params.MouseWiggleParams -import java.awt.MouseInfo -import java.awt.Point -import kotlin.random.Random - - -/** - * Interface for controllers that interact with the desktop. - * - * This defines methods for getting desktop state like the mouse pointer - * location. - * - * Classes that implement this can serve as desktop automation controllers. - */ -interface DesktopController { - - /** - * Gets the current pointer/mouse location on the desktop. - * - * This returns a [Point] representing the x, y coordinates of the mouse pointer on the screen. - * - * For example: - * - * ``` - * val mouseLocation = getPointerLocation() - * - * println(mouseLocation) // Might print "Point[x=1920,y=1080]" - * ``` - * - * @return The current [Point] location of the mouse pointer on the screen. - */ - fun getPointerLocation(): Point { - return MouseInfo.getPointerInfo().location - } - - - /** - * Gets a point near the given [point], with some random wiggle. - * - * This generates a new point that is randomly offset from the given [point] - * by an amount controlled by the given [MouseWiggleParams]. - * - * Usage example: - * - * ``` - * val base = Point(10, 20) - * val params = WiggleParams(xWiggle = 5, yWiggle = 5) - * val randomPoint = getAlmostPoint(base, params) - * - * // randomPoint might be (8, 22) - * ``` - * - * Alternatively, params may be omitted to use the default WiggleParams values - * ``` - * val base = Point(10, 20) - * val randomPoint = getAlmostPoint(base) - * - * // randomPoint might be (8, 22) - * ``` - * - * @param point The base point to start from - * @param params The wiggle parameters that control the random offset amount - * @return A new [Point] near the given point with some random wiggle applied - */ - fun getAlmostPoint(point: Point, params: MouseWiggleParams = MouseWiggleParams()): Point { - val xDel = Random.nextInt(0, params.xWiggle) - val yDel = Random.nextInt(0, params.yWiggle) - val xDir = if (Random.nextDouble() > 0.5) { - 1 - } else { - -1 - } - val yDir = if (Random.nextDouble() > 0.5) { - 1 - } else { - -1 - } - return Point(point.x + (xDel * xDir), point.y + (yDel * yDir)) - } -} - -interface OSProxy { - fun getActiveWindowName(): String - - fun enumWindowNames(): ArrayList - - fun setForegroundWindowByName(name: String) -} - -/** - * Windows implementation of [DesktopController]. - * - * This class provides methods to interact with the desktop on Windows - * by calling Win32 APIs. - * - * It implements the [DesktopController] interface to provide desktop - * functionality like getting the mouse pointer location on Windows. - */ -class WindowsDesktopController : DesktopController, OSProxy { - - companion object { - /** - * Converts a native byte buffer to a String. - * - * This takes a byte array [byteBuffer] containing text from a native Win32 call, - * converts it to a String using JNA, and trims whitespace characters. - * - * Usage example: - * - * ``` - * val buffer = ByteArray(256) - * GetWindowTextA(hwnd, buffer, buffer.size) // Win32 call - * - * val windowTitle = nativeByteBufferToString(buffer) - * println(windowTitle) // Print title string - * ``` - * - * @param byteBuffer Byte array containing text from a native call - * @return The native text as a String - */ - private fun nativeByteBufferToString(byteBuffer: ByteArray): String { - // I guess this prunes anything that isn't on the ascii table? - val wText = Native.toString(byteBuffer).trim { it <= ' ' } - return wText - } - } - - /** - * Interface for calling Windows User32 API functions. - * - * This defines an interface extending StdCallLibrary to call native - * Windows User32 library functions like EnumWindows, GetWindowTextA etc. - * - * Classes can implement this interface to make direct calls to the - * User32 DLL on Windows. - */ - internal interface User32 : StdCallLibrary { - - /** - * Interface for a Windows callback function to enumerate windows. - * - * This extends the StdCallLibrary.StdCallCallback to define a callback - * method that will be invoked by the Windows API EnumWindows function. - * - * The callback method accepts a window handle (HWND) and a user-defined - * pointer, and returns a Boolean indicating whether to continue enumeration. - * - * Usage example: - * ``` - * val callback = object : WNDENUMPROC { - * override fun callback(hWnd: Pointer?, arg: Pointer?): Boolean { - * // Check if hWnd matches target window - * if (matchesTarget(hWnd)) { - * // Found target window, stop enumeration - * return false - * } - * // Keep enumerating - * return true - * } - * } - * ``` - * - * @param hWnd Window handle (HWND) for the current enumerated window. - * @param arg User-defined data pointer passed to EnumWindows. - * @return True to continue enumerating windows, false to stop. - */ - interface WNDENUMPROC : StdCallLibrary.StdCallCallback { - fun callback(hWnd: Pointer?, arg: Pointer?): Boolean - } - - /** - * Enumerates windows on the system. - * - * This calls the Windows API EnumWindows function to enumerate all top-level windows. - * - * For each window, it calls the provided [WNDENUMPROC] callback function, - * passing the window handle [hWnd] and user-defined [userData] pointer. - * - * Enumeration can be stopped by returning false from the callback. - * - * Usage example: - * - * ``` - * val windows = mutableListOf() - * - * val callback = object : WNDENUMPROC { - * override fun callback(hWnd: Pointer?, arg: Pointer?): Boolean { - * windows.add(hWnd) // Add hWnd to list - * return true // Continue enumerating - * } - * } - * - * EnumWindows(callback, null) // Get all top-level windows - * - * println(windows) // Print list of HWNDs - * ``` - * - * @param lpEnumFunc The [WNDENUMPROC] callback to call for each window. - * @param userData Optional user-defined data to pass to the callback. - * @return True if successful, false otherwise. - */ - fun EnumWindows(lpEnumFunc: WNDENUMPROC?, userData: Pointer?): Boolean - - /** - * Gets the title text of the specified window. - * - * This calls the Win32 API GetWindowTextA function to get the title - * text for the window handle [hWnd]. - * - * The window text is copied into the [lpString] buffer up to [nMaxCount] characters. - * - * Usage example: - * - * ``` - * val buffer = ByteArray(256) - * val hwnd = getWindowHandle() // get some HWND - * - * GetWindowTextA(hwnd, buffer, buffer.size) - * - * val windowTitle = String(buffer) - * println(windowTitle) - * ``` - * - * @param hWnd The window handle (HWND). - * @param lpString The buffer to receive the window text. - * @param nMaxCount The maximum number of characters to copy to the buffer. - * @return The length of the window text (excluding null-terminator). - */ - fun GetWindowTextA(hWnd: Pointer?, lpString: ByteArray?, nMaxCount: Int): Int - - /** - * Gets the handle of the foreground window. - * - * Calls the Win32 API [User32.GetForegroundWindow] to retrieve the window handle - * of the current foreground window - the one the user is currently interacting with. - * - * @return The window handle (HWND) of the current foreground window, or null if none. - */ - fun GetForegroundWindow(): Pointer? - - /** - * Sets the foreground window. - * - * Calls the Win32 API [User32.SetForegroundWindow] to bring the window with the given - * handle [hWnd] to the foreground. - * - * @param hWnd The native window handle (HWND) of the window to activate. - * @return True if successful, false otherwise. - */ - fun SetForegroundWindow(hWnd: Pointer?): Boolean - - /** - * Loads the User32.dll library and gets an instance of the [User32] interface. - * - * This uses JNA to dynamically load the user32.dll library at runtime. It casts - * the result to the [User32] interface to provide access to the Windows API - * functions defined there. - * - * The [User32] instance is stored in [INSTANCE] to be used throughout the class - * for calling Windows APIs. - * - * For example: - * - * ``` - * val user32 = User32.INSTANCE - * - * user32.EnumWindows(...) // Call Windows API - * ``` - * - * @see User32 - */ - companion object { - /** - * Loads the User32.dll library and gets an instance of the [User32] interface. - * - * This uses JNA to dynamically load the user32.dll library at runtime. It casts - * the result to the [User32] interface to provide access to the Windows API - * functions defined there. - * - * The [User32] instance is stored in [INSTANCE] to be used throughout the class - * for calling Windows APIs. - * - * For example: - * - * ``` - * val user32 = User32.INSTANCE - * - * user32.EnumWindows(...) // Call Windows API - * ``` - * - * @see User32 - */ - val INSTANCE = Native.load("user32", User32::class.java) as User32 - } - } - - - /** - * Callback class used for enumerating windows. - * - * This implements the [User32.WNDENUMPROC] interface required by the - * [User32.EnumWindows] API. An instance of this class is passed to - * [User32.EnumWindows] to receive callbacks for each window. - * - * @param cb The callback function to invoke for each window. It should return - * true to continue enumeration or false to stop. - */ - internal class WindowEnumCallback(val cb: (Pointer?, Pointer?) -> Boolean) : User32.WNDENUMPROC { - - /** - * Called by [User32.EnumWindows] for each window. - * - * Implements the callback method required by [User32.WNDENUMPROC]. - * This simply invokes the provided [cb] callback and returns its result. - * - * @param hWnd The window handle being enumerated. - * @param userDataPtr A pointer to user data passed to [User32.EnumWindows]. - * @return The return value from the [cb] callback. - */ - override fun callback(hWnd: Pointer?, userDataPtr: Pointer?): Boolean { - return cb(hWnd, userDataPtr) - } - - } - - /** - * Gets the title of the active/foreground window. - * - * This calls Win32 APIs to get the handle of the foreground window, - * then gets its title text. - * - * Usage example: - * - * ``` - * val activeWindowName = getActiveWindowName() - * - * println(activeWindowName) // Prints foreground window title - * ``` - * - * @return The title text of the current foreground window. - */ - override fun getActiveWindowName(): String { - val user32 = User32.INSTANCE - val foregroundWindowHwnd = user32.GetForegroundWindow() - return getWindowName(foregroundWindowHwnd) - } - - /** - * Gets the title/name of the window for the given handle. - * - * This calls the Win32 API [User32.GetWindowTextA] to retrieve the title - * text for the window referenced by [hWnd]. - * - * It allocates a [windowTitleBuffer] byte array to hold the result. This is - * passed to [User32.GetWindowTextA] to be populated. - * - * The buffer is then converted to a [String] via [nativeByteBufferToString]. - * - * @param hWnd The native window handle to get the title for. - * @return The window title text as a [String]. - */ - private fun getWindowName(hWnd: Pointer?): String { - val maxTitleLength = 512 - val user32 = User32.INSTANCE - val windowTitleBuffer = ByteArray(maxTitleLength) - - user32.GetWindowTextA(hWnd, windowTitleBuffer, windowTitleBuffer.size) - - return nativeByteBufferToString(windowTitleBuffer) - } - - /** - * Enumerates all open window names on the desktop. - * - * Calls the Win32 API [User32.EnumWindows] to iterate through all current open windows. - * For each window handle, it retrieves the window name using [getWindowName] - * and adds it to a list if the name is not blank. We filter out blank window names because that particular information - * is useless for any reason other than counting how many open windows there are. If we actually need that information, - * we can simply acquire a list of all HWND references. - * - * @return An [ArrayList] containing the name of each open window. - */ - override fun enumWindowNames(): ArrayList { - val user32 = User32.INSTANCE - val windowNames = ArrayList() - - val iterationCallback = WindowEnumCallback { hWnd, _ -> - val windowName = getWindowName(hWnd) - if (windowName.isNotBlank()) { - windowNames.add(windowName) - } - true - } - - user32.EnumWindows(iterationCallback, null) - return windowNames - } - - - /** - * Sets the foreground window by name on Windows. - * - * This calls the Win32 API [User32.EnumWindows] to iterate through all - * top-level windows, compares their name to the given [name], and calls - * [User32.SetForegroundWindow] on the matching window to bring it to the foreground. - * - * @param name The window name to search for and activate. - */ - override fun setForegroundWindowByName(name: String) { - val user32 = User32.INSTANCE - - val iterationCallback = WindowEnumCallback { hWnd, _ -> - val windowName = getWindowName(hWnd) - - if (windowName.isBlank()) { - //if the window is blank, tell the system that we should continue - true - } else { - //otherwise, we need to check the window name - if (windowName == name) { - user32.SetForegroundWindow(hWnd) - //we found a match, so tell the system that we don't need to proceed any further - false - } else { - true - } - } - } - - user32.EnumWindows(iterationCallback, null) - } - -} \ No newline at end of file diff --git a/src/main/kotlin/controllers/MousePointerObserver.kt b/src/main/kotlin/controllers/MousePointerObserver.kt new file mode 100644 index 0000000..2111cc3 --- /dev/null +++ b/src/main/kotlin/controllers/MousePointerObserver.kt @@ -0,0 +1,82 @@ +package controllers + +import params.MouseWiggleParams +import java.awt.MouseInfo +import java.awt.Point +import kotlin.random.Random + + +/** + * Interface for controllers that interact with the desktop. + * + * This defines methods for getting desktop state like the mouse pointer + * location. + * + * Classes that implement this can serve as desktop automation controllers. + */ +interface MousePointerObserver { + + /** + * Gets the current pointer/mouse location on the desktop. + * + * This returns a [Point] representing the x, y coordinates of the mouse pointer on the screen. + * + * For example: + * + * ``` + * val mouseLocation = getPointerLocation() + * + * println(mouseLocation) // Might print "Point[x=1920,y=1080]" + * ``` + * + * @return The current [Point] location of the mouse pointer on the screen. + */ + fun getPointerLocation(): Point { + return MouseInfo.getPointerInfo().location + } + + + /** + * Gets a point near the given [point], with some random wiggle. + * + * This generates a new point that is randomly offset from the given [point] + * by an amount controlled by the given [MouseWiggleParams]. + * + * Usage example: + * + * ``` + * val base = Point(10, 20) + * val params = WiggleParams(xWiggle = 5, yWiggle = 5) + * val randomPoint = getAlmostPoint(base, params) + * + * // randomPoint might be (8, 22) + * ``` + * + * Alternatively, params may be omitted to use the default WiggleParams values + * ``` + * val base = Point(10, 20) + * val randomPoint = getAlmostPoint(base) + * + * // randomPoint might be (8, 22) + * ``` + * + * @param point The base point to start from + * @param params The wiggle parameters that control the random offset amount + * @return A new [Point] near the given point with some random wiggle applied + */ + fun getAlmostPoint(point: Point, params: MouseWiggleParams = MouseWiggleParams()): Point { + val xDel = Random.nextInt(0, params.xWiggle) + val yDel = Random.nextInt(0, params.yWiggle) + val xDir = if (Random.nextDouble() > 0.5) { + 1 + } else { + -1 + } + val yDir = if (Random.nextDouble() > 0.5) { + 1 + } else { + -1 + } + return Point(point.x + (xDel * xDir), point.y + (yDel * yDir)) + } +} diff --git a/src/main/kotlin/controllers/OSProxy.kt b/src/main/kotlin/controllers/OSProxy.kt new file mode 100644 index 0000000..a4af171 --- /dev/null +++ b/src/main/kotlin/controllers/OSProxy.kt @@ -0,0 +1,14 @@ +package controllers + +import java.awt.Rectangle + + +interface OSProxy { + fun getActiveWindowName(): String + + fun enumWindowNames(): ArrayList + + fun setForegroundWindowByName(name: String) + + fun getForegroundWindowBounds(): Rectangle? +} diff --git a/src/main/kotlin/controllers/RobotAutomaton.kt b/src/main/kotlin/controllers/RobotAutomaton.kt index 2147ba0..09df4f5 100644 --- a/src/main/kotlin/controllers/RobotAutomaton.kt +++ b/src/main/kotlin/controllers/RobotAutomaton.kt @@ -38,7 +38,7 @@ import java.awt.event.InputEvent * * @param robot The Robot instance to use. A default is created if not provided. */ -open class RobotAutomaton(private val robot: Robot = Robot()) : Automaton { +open class RobotAutomaton(internal val robot: Robot = Robot()) : Automaton { /** * Moves the mouse pointer to the given [Point] coordinates. diff --git a/src/main/kotlin/controllers/VisionController.kt b/src/main/kotlin/controllers/VisionController.kt new file mode 100644 index 0000000..6ef005b --- /dev/null +++ b/src/main/kotlin/controllers/VisionController.kt @@ -0,0 +1,18 @@ +package controllers + +import controllers.windows.WindowsOSProxy +import java.nio.file.Paths +import javax.imageio.ImageIO + +interface VisionController: Automaton, OSProxy { + fun takeScreenshot() +} + +class ConcreteVisionController(): VisionController, WindowsOSProxy, RobotAutomaton(){ + override fun takeScreenshot() { + val rect = getForegroundWindowBounds() + val img = robot.createScreenCapture(rect) + val testPath = Paths.get(".", "test2.png") + ImageIO.write(img, "png", testPath.toFile()) + } +} \ No newline at end of file diff --git a/src/main/kotlin/controllers/windows/User32.kt b/src/main/kotlin/controllers/windows/User32.kt new file mode 100644 index 0000000..6551410 --- /dev/null +++ b/src/main/kotlin/controllers/windows/User32.kt @@ -0,0 +1,218 @@ +package controllers.windows + +import com.sun.jna.Native +import com.sun.jna.NativeLong +import com.sun.jna.Pointer +import com.sun.jna.win32.StdCallLibrary +import controllers.windows.WindowsOSProxy.WinRect + +/** + * Interface for calling Windows User32 API functions. + * + * This defines an interface extending StdCallLibrary to call native + * Windows User32 library functions like EnumWindows, GetWindowTextA etc. + * + * Classes can implement this interface to make direct calls to the + * User32 DLL on Windows. + */ +interface User32 : StdCallLibrary { + + /** + * Interface for a Windows callback function to enumerate windows. + * + * This extends the StdCallLibrary.StdCallCallback to define a callback + * method that will be invoked by the Windows API EnumWindows function. + * + * The callback method accepts a window handle (HWND) and a user-defined + * pointer, and returns a Boolean indicating whether to continue enumeration. + * + * Usage example: + * ``` + * val callback = object : WNDENUMPROC { + * override fun callback(hWnd: Pointer?, arg: Pointer?): Boolean { + * // Check if hWnd matches target window + * if (matchesTarget(hWnd)) { + * // Found target window, stop enumeration + * return false + * } + * // Keep enumerating + * return true + * } + * } + * ``` + * + * @param hWnd Window handle (HWND) for the current enumerated window. + * @param arg User-defined data pointer passed to EnumWindows. + * @return True to continue enumerating windows, false to stop. + */ + interface WNDENUMPROC : StdCallLibrary.StdCallCallback { + fun callback(hWnd: Pointer?, userDataPtr: Pointer?): Boolean + } + + /** + * Enumerates windows on the system. + * + * This calls the Windows API EnumWindows function to enumerate all top-level windows. + * + * For each window, it calls the provided [WNDENUMPROC] callback function, + * passing the window handle [hWnd] and user-defined [userData] pointer. + * + * Enumeration can be stopped by returning false from the callback. + * + * Usage example: + * + * ``` + * val windows = mutableListOf() + * + * val callback = object : WNDENUMPROC { + * override fun callback(hWnd: Pointer?, arg: Pointer?): Boolean { + * windows.add(hWnd) // Add hWnd to list + * return true // Continue enumerating + * } + * } + * + * EnumWindows(callback, null) // Get all top-level windows + * + * println(windows) // Print list of HWNDs + * ``` + * + * @param lpEnumFunc The [WNDENUMPROC] callback to call for each window. + * @param userData Optional user-defined data to pass to the callback. + * @return True if successful, false otherwise. + */ + fun EnumWindows(lpEnumFunc: WNDENUMPROC?, userData: Pointer?): Boolean + + /** + * Gets the title text of the specified window. + * + * This calls the Win32 API GetWindowTextA function to get the title + * text for the window handle [hWnd]. + * + * The window text is copied into the [lpString] buffer up to [nMaxCount] characters. + * + * Usage example: + * + * ``` + * val buffer = ByteArray(256) + * val hwnd = getWindowHandle() // get some HWND + * + * GetWindowTextA(hwnd, buffer, buffer.size) + * + * val windowTitle = String(buffer) + * println(windowTitle) + * ``` + * + * @param hWnd The window handle (HWND). + * @param lpString The buffer to receive the window text. + * @param nMaxCount The maximum number of characters to copy to the buffer. + * @return The length of the window text (excluding null-terminator). + */ + fun GetWindowTextA(hWnd: Pointer?, lpString: ByteArray?, nMaxCount: Int): Int + + /** + * Gets the handle of the foreground window. + * + * Calls the Win32 API [User32.GetForegroundWindow] to retrieve the window handle + * of the current foreground window - the one the user is currently interacting with. + * + * @return The window handle (HWND) of the current foreground window, or null if none. + */ + fun GetForegroundWindow(): Pointer? + + /** + * Sets the foreground window. + * + * Calls the Win32 API [User32.SetForegroundWindow] to bring the window with the given + * handle [hWnd] to the foreground. + * + * @param hWnd The native window handle (HWND) of the window to activate. + * @return True if successful, false otherwise. + */ + fun SetForegroundWindow(hWnd: Pointer?): Boolean + + /** + * Retrieves the coordinates of a window on the screen. + * + * This calls the Win32 API [User32.GetWindowRect] to populate the provided [lpRect] + * struct with the outer bounding rectangle of the window specified by [hWnd]. + * + * The rectangle coordinates represent the upper-left and lower-right corners + * relative to the screen. + * + * @param hWnd The native window handle (HWND) to get the bounds for. + * @param lpRect Pointer to a [WinRect] struct to populate with the coordinates. + * @return True if successful, false otherwise. + */ + fun GetWindowRect(hWnd: Pointer?, lpRect: Pointer?): Boolean + + + fun GetWindowLongA(hWnd: Pointer?, nIndex: Int): Int + + + /** + * Gets the DPI value for the system. + * + * This calls the Win32 API GetDpiForSystem() function to get the current DPI or scaling + * percentage value that is set for the system. + * + * For example, on a normal 100% scaled display this will return 96. On a 150% scaled 4K display, + * this may return 144. + * + * @return The DPI scaling value for the system. + */ + fun GetDpiForSystem(): Int + + + fun GetDpiForWindow(hWnd: Pointer?): Int + + /** + * Adjusts the specified window rectangle taking into account DPI scaling. + * + * This calls the Win32 API AdjustWindowRectExForDpi function to adjust the provided + * window rectangle [lpRect] for the given window styles and DPI scaling value [dpi]. + * + * The rectangle will be expanded or contracted based on the window styles, menu, + * and DPI scaling. This allows properly sizing windows for the current display. + * + * @param lpRect Pointer to the RECT structure to adjust. + * @param dwStyle The window style flags for the window. + * @param bMenu Whether the window has a menu. + * @param dwExStyle The extended window style flags. + * @param dpi The DPI scaling value of the display. + * @return True if successful, false otherwise. + */ + fun AdjustWindowRectExForDpi( + lpRect: Pointer?, + dwStyle: Int, + bMenu: Boolean, + dwExStyle: Int, + dpi: Int + ): Boolean + + + companion object { + /** + * Loads the User32.dll library and gets an instance of the [User32] interface. + * + * This uses JNA to dynamically load the user32.dll library at runtime, which provides access to the Windows + * API functions defined there. + * + * The [User32] instance is stored in the [INSTANCE] field to be used throughout the class + * for calling Windows APIs. + * + * For example: + * + * ``` + * val user32 = User32.INSTANCE + * + * user32.EnumWindows(...) // Call Windows API + * ``` + * + * @see User32 + */ + val INSTANCE = Native.load("user32", User32::class.java) as User32 + + public const val GWL_EXSTYLE = -20 + public const val GWL_STYLE = -16 + } +} \ No newline at end of file diff --git a/src/main/kotlin/controllers/windows/WindowsOSProxy.kt b/src/main/kotlin/controllers/windows/WindowsOSProxy.kt new file mode 100644 index 0000000..33530ab --- /dev/null +++ b/src/main/kotlin/controllers/windows/WindowsOSProxy.kt @@ -0,0 +1,295 @@ +package controllers.windows + +import com.sun.jna.Native +import com.sun.jna.Pointer +import com.sun.jna.Structure +import com.sun.jna.Structure.FieldOrder +import controllers.MousePointerObserver +import controllers.OSProxy +import java.awt.Rectangle + + +/** + * Windows implementation of [MousePointerObserver]. + * + * This class provides methods to interact with the desktop on Windows + * by calling Win32 APIs. + * + * It implements the [MousePointerObserver] interface to provide desktop + * functionality like getting the mouse pointer location on Windows. + */ +interface WindowsOSProxy : MousePointerObserver, OSProxy { + + companion object { + /** + * Converts a native byte buffer to a String. + * + * This takes a byte array [byteBuffer] containing text from a native Win32 call, + * converts it to a String using JNA, and trims whitespace characters. + * + * Usage example: + * + * ``` + * val buffer = ByteArray(256) + * GetWindowTextA(hwnd, buffer, buffer.size) // Win32 call + * + * val windowTitle = nativeByteBufferToString(buffer) + * println(windowTitle) // Print title string + * ``` + * + * @param byteBuffer Byte array containing text from a native call + * @return The native text as a String + */ + private fun nativeByteBufferToString(byteBuffer: ByteArray): String { + return Native.toString(byteBuffer).trim { it <= ' ' } + } + } + + + /** + * Callback class used for enumerating windows. + * + * This implements the [User32.WNDENUMPROC] interface required by the + * [User32.EnumWindows] API. An instance of this class is passed to + * [User32.EnumWindows] to receive callbacks for each window. + * + * @param cb The callback function to invoke for each window. It should return + * true to continue enumeration or false to stop. + */ + class WindowEnumCallback(val cb: (Pointer?, Pointer?) -> Boolean) : User32.WNDENUMPROC { + + /** + * Called by [User32.EnumWindows] for each window. + * + * Implements the callback method required by [User32.WNDENUMPROC]. + * This simply invokes the provided [cb] callback and returns its result. + * + * @param hWnd The window handle being enumerated. + * @param userDataPtr A pointer to user data passed to [User32.EnumWindows]. + * @return The return value from the [cb] callback. + */ + override fun callback(hWnd: Pointer?, userDataPtr: Pointer?): Boolean { + return cb(hWnd, userDataPtr) + } + + } + + + /** + * Mirror of the Windows RECT struct used for window coordinates. + */ + @FieldOrder("left", "top", "right", "bottom") + public class WinRect : Structure() { + + /** The x coordinate of the left edge */ + @JvmField + var left: Int = 0 + + /** The y coordinate of the top edge */ + @JvmField + var top: Int = 0 + + /** The x coordinate of the right edge */ + @JvmField + var right: Int = 0 + + /** The y coordinate of the bottom edge */ + @JvmField + var bottom: Int = 0 + } + + /** + * Gets the title of the active/foreground window. + * + * This calls Win32 APIs to get the handle of the foreground window, + * then gets its title text. + * + * Usage example: + * + * ``` + * val activeWindowName = getActiveWindowName() + * + * println(activeWindowName) // Prints foreground window title + * ``` + * + * @return The title text of the current foreground window. + */ + override fun getActiveWindowName(): String { + val user32 = User32.INSTANCE + val foregroundWindowHwnd = user32.GetForegroundWindow() + return getWindowName(foregroundWindowHwnd) + } + + fun getWindowHandleByName(name: String): Pointer? { + val user32 = User32.INSTANCE + var ptr: Pointer? = null + val enumFunction = WindowEnumCallback { hwnd, _ -> + val wName = getWindowName(hwnd) + if (name == wName) { + ptr = hwnd + false + } else { + true + } + } + + user32.EnumWindows(enumFunction, null) + return ptr + } + + /** + * Gets the title/name of the window for the given handle. + * + * This calls the Win32 API [User32.GetWindowTextA] to retrieve the title + * text for the window referenced by [hWnd]. + * + * It allocates a [windowTitleBuffer] byte array to hold the result. This is + * passed to [User32.GetWindowTextA] to be populated. + * + * The buffer is then converted to a [String] via [nativeByteBufferToString]. + * + * @param hWnd The native window handle to get the title for. + * @return The window title text as a [String]. + */ + private fun getWindowName(hWnd: Pointer?): String { + val maxTitleLength = 512 + val user32 = User32.INSTANCE + val windowTitleBuffer = ByteArray(maxTitleLength) + + user32.GetWindowTextA(hWnd, windowTitleBuffer, windowTitleBuffer.size) + + return nativeByteBufferToString(windowTitleBuffer) + } + + /** + * Enumerates all open window names on the desktop. + * + * Calls the Win32 API [User32.EnumWindows] to iterate through all current open windows. + * For each window handle, it retrieves the window name using [getWindowName] + * and adds it to a list if the name is not blank. We filter out blank window names because that particular information + * is useless for any reason other than counting how many open windows there are. If we actually need that information, + * we can simply acquire a list of all HWND references. + * + * @return An [ArrayList] containing the name of each open window. + */ + override fun enumWindowNames(): ArrayList { + val user32 = User32.INSTANCE + val windowNames = ArrayList() + + val iterationCallback = WindowEnumCallback { hWnd, _ -> + val windowName = getWindowName(hWnd) + if (windowName.isNotBlank()) { + windowNames.add(windowName) + } + true + } + + user32.EnumWindows(iterationCallback, null) + return windowNames + } + + + /** + * Sets the foreground window by name on Windows. + * + * This calls the Win32 API [User32.EnumWindows] to iterate through all + * top-level windows, compares their name to the given [name], and calls + * [User32.SetForegroundWindow] on the matching window to bring it to the foreground. + * + * @param name The window name to search for and activate. + */ + override fun setForegroundWindowByName(name: String) { + val user32 = User32.INSTANCE + + val iterationCallback = WindowEnumCallback { hWnd, _ -> + val windowName = getWindowName(hWnd) + + if (windowName.isBlank()) { + //if the window is blank, tell the system that we should continue + true + } else { + //otherwise, we need to check the window name + if (windowName == name) { + user32.SetForegroundWindow(hWnd) + //we found a match, so tell the system that we don't need to proceed any further + false + } else { + true + } + } + } + + user32.EnumWindows(iterationCallback, null) + } + + /** + * Gets the screen bounds of the current foreground window. + * + * Calls the Win32 API [User32.GetForegroundWindow] to get the HWND of the + * foreground window. + * + * Then calls [User32.GetWindowRect] to populate a [WinRect] struct with the + * window bounds. + * + * The rectangle coordinates are converted to a [Rectangle] and returned. This allows us to keep the implementation + * generic and constrained within the java std library. + * In the future we may want to add LinuxOSProxy or DarwinOSProxy. + * + * @return The outer bounding rectangle of the foreground window in screen coordinates, + * or an empty [Rectangle] if it failed. + */ + override fun getForegroundWindowBounds(): Rectangle? { + val user32 = User32.INSTANCE + val hWnd = user32.GetForegroundWindow() + + val rect = getRectFromWindowHandle(user32, hWnd) + + return if (rect != null) { + + //we need to make the calls to get the info to correct for the dpi scaling + val secondarySuccess = correctWinRectForDpi(user32, hWnd!!, rect.pointer) + + if (secondarySuccess) { + //the correctWinRectForDpi function doesn't have access to the underlying stuct in order to read it, so + // we have to call this + rect.read() + Rectangle(rect.top, rect.left, (rect.right - rect.left), (rect.bottom - rect.top)) + } else { + return null + } + + } else { + null + } + } + + private fun getRectFromWindowHandle(user32: User32, hWnd: Pointer?): WinRect? { + //we have to provide the system a native struct in order to hold the results of our request + val rect = WinRect() + + val success = user32.GetWindowRect(hWnd, rect.pointer) + + return if (success) { + //the values are stuck down in memory, so we have to read these back out in order to proceed + rect.read() + rect + } else { + null + } + } + + private fun correctWinRectForDpi(user32: User32, hWnd: Pointer, lpRect: Pointer): Boolean { + val user32 = User32.INSTANCE + val dwStyle = user32.GetWindowLongA(hWnd, User32.GWL_STYLE) + + + val bMenu = true + + val dwExStyle = user32.GetWindowLongA(hWnd, User32.GWL_EXSTYLE) + + val dpi = user32.GetDpiForWindow(hWnd) + + return user32.AdjustWindowRectExForDpi(lpRect, dwStyle, bMenu, dwExStyle, dpi) + } + +} \ No newline at end of file diff --git a/src/test/kotlin/controllers/DesktopControllerTest.kt b/src/test/kotlin/controllers/DesktopControllerTest.kt index f9ce130..dc50387 100644 --- a/src/test/kotlin/controllers/DesktopControllerTest.kt +++ b/src/test/kotlin/controllers/DesktopControllerTest.kt @@ -20,7 +20,7 @@ class DesktopControllerTest { */ @Test fun `getPointerLocation returns mouse position`() { - val controller = mock(DesktopController::class.java) + val controller = mock(MousePointerObserver::class.java) // Mock mouse position `when`(controller.getPointerLocation()).thenReturn(Point(100, 200)) @@ -46,7 +46,7 @@ class DesktopControllerTest { */ @Test fun `getAlmostPoint returns wiggly point`() { - val controller = mock(DesktopController::class.java) + val controller = mock(MousePointerObserver::class.java) val params = MouseWiggleParams(xWiggle = 10, yWiggle = 10) // Mock random wiggle @@ -60,10 +60,4 @@ class DesktopControllerTest { assertNotEquals(100, wiggly.x) assertNotEquals(200, wiggly.y) } - - @Test - fun devTest(){ - val c = WindowsDesktopController() - c.setForegroundWindowByName("RuneScape") - } } \ No newline at end of file diff --git a/src/test/kotlin/controllers/OSProxyTest.kt b/src/test/kotlin/controllers/OSProxyTest.kt new file mode 100644 index 0000000..c869cd0 --- /dev/null +++ b/src/test/kotlin/controllers/OSProxyTest.kt @@ -0,0 +1,13 @@ +package controllers + +import org.junit.jupiter.api.Test + +class OSProxyTest { + + @Test + fun test(){ + val vc = ConcreteVisionController() + val rect = vc.getForegroundWindowBounds() + println(rect) + } +} \ No newline at end of file diff --git a/src/test/kotlin/controllers/VisionControllerTest.kt b/src/test/kotlin/controllers/VisionControllerTest.kt new file mode 100644 index 0000000..e006097 --- /dev/null +++ b/src/test/kotlin/controllers/VisionControllerTest.kt @@ -0,0 +1,12 @@ +package controllers + +import kotlin.test.Test + +class VisionControllerTest { + + @Test + fun testImageCapture(){ + val vc = ConcreteVisionController() + vc.takeScreenshot() + } +} \ No newline at end of file