tookeyman/RuneFactory
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

checkin before work

Browse Source
This commit is contained in:
dtookey 2023-08-09 09:00:11 -04:00
parent 15a0321a37
commit d831da11b0
11 changed files with 657 additions and 449 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/kotlin/controllers/Automaton.kt
View File

@ -6,7 +6,7 @@ package controllers
* *
* Automaton combines capabilities from other interfaces to create a controller that can: * Automaton combines capabilities from other interfaces to create a controller that can:
* *
* - Get desktop and mouse state information like pointer location via [DesktopController] * - Get desktop and mouse state information like pointer location via [MousePointerObserver]
* *
* - Perform mouse and keyboard input like clicks, key presses, and scrolling via [InputController] * - Perform mouse and keyboard input like clicks, key presses, and scrolling via [InputController]
* *
@ -23,4 +23,4 @@ package controllers
* *
* This interface allows the underlying OS/desktop implementation details to be abstracted and swapped as needed. * This interface allows the underlying OS/desktop implementation details to be abstracted and swapped as needed.
*/ */
interface Automaton : DesktopController, InputController, TemporalController interface Automaton : MousePointerObserver, InputController, TemporalController

438
src/main/kotlin/controllers/DesktopController.kt
View File

@ -1,438 +0,0 @@
package controllers
import com.sun.jna.Native
import com.sun.jna.Pointer
import com.sun.jna.win32.StdCallLibrary
import params.MouseWiggleParams
import java.awt.MouseInfo
import java.awt.Point
import kotlin.random.Random
/**
* Interface for controllers that interact with the desktop.
*
* This defines methods for getting desktop state like the mouse pointer
* location.
*
* Classes that implement this can serve as desktop automation controllers.
*/
interface DesktopController {
/**
* Gets the current pointer/mouse location on the desktop.
*
* This returns a [Point] representing the x, y coordinates of the mouse pointer on the screen.
*
* For example:
*
* ```
* val mouseLocation = getPointerLocation()
*
* println(mouseLocation) // Might print "Point[x=1920,y=1080]"
* ```
*
* @return The current [Point] location of the mouse pointer on the screen.
*/
fun getPointerLocation(): Point {
return MouseInfo.getPointerInfo().location
}
/**
* Gets a point near the given [point], with some random wiggle.
*
* This generates a new point that is randomly offset from the given [point]
* by an amount controlled by the given [MouseWiggleParams].
*
* Usage example:
*
* ```
* val base = Point(10, 20)
* val params = WiggleParams(xWiggle = 5, yWiggle = 5)
* val randomPoint = getAlmostPoint(base, params)
*
* // randomPoint might be (8, 22)
* ```
*
* Alternatively, params may be omitted to use the default WiggleParams values
* ```
* val base = Point(10, 20)
* val randomPoint = getAlmostPoint(base)
*
* // randomPoint might be (8, 22)
* ```
*
* @param point The base point to start from
* @param params The wiggle parameters that control the random offset amount
* @return A new [Point] near the given point with some random wiggle applied
*/
fun getAlmostPoint(point: Point, params: MouseWiggleParams = MouseWiggleParams()): Point {
val xDel = Random.nextInt(0, params.xWiggle)
val yDel = Random.nextInt(0, params.yWiggle)
val xDir = if (Random.nextDouble() > 0.5) {
1
} else {
-1
}
val yDir = if (Random.nextDouble() > 0.5) {
1
} else {
-1
}
return Point(point.x + (xDel * xDir), point.y + (yDel * yDir))
}
}
interface OSProxy {
fun getActiveWindowName(): String
fun enumWindowNames(): ArrayList<String>
fun setForegroundWindowByName(name: String)
}
/**
* Windows implementation of [DesktopController].
*
* This class provides methods to interact with the desktop on Windows
* by calling Win32 APIs.
*
* It implements the [DesktopController] interface to provide desktop
* functionality like getting the mouse pointer location on Windows.
*/
class WindowsDesktopController : DesktopController, OSProxy {
companion object {
/**
* Converts a native byte buffer to a String.
*
* This takes a byte array [byteBuffer] containing text from a native Win32 call,
* converts it to a String using JNA, and trims whitespace characters.
*
* Usage example:
*
* ```
* val buffer = ByteArray(256)
* GetWindowTextA(hwnd, buffer, buffer.size) // Win32 call
*
* val windowTitle = nativeByteBufferToString(buffer)
* println(windowTitle) // Print title string
* ```
*
* @param byteBuffer Byte array containing text from a native call
* @return The native text as a String
*/
private fun nativeByteBufferToString(byteBuffer: ByteArray): String {
// I guess this prunes anything that isn't on the ascii table?
val wText = Native.toString(byteBuffer).trim { it <= ' ' }
return wText
}
}
/**
* Interface for calling Windows User32 API functions.
*
* This defines an interface extending StdCallLibrary to call native
* Windows User32 library functions like EnumWindows, GetWindowTextA etc.
*
* Classes can implement this interface to make direct calls to the
* User32 DLL on Windows.
*/
internal interface User32 : StdCallLibrary {
/**
* Interface for a Windows callback function to enumerate windows.
*
* This extends the StdCallLibrary.StdCallCallback to define a callback
* method that will be invoked by the Windows API EnumWindows function.
*
* The callback method accepts a window handle (HWND) and a user-defined
* pointer, and returns a Boolean indicating whether to continue enumeration.
*
* Usage example:
* ```
* val callback = object : WNDENUMPROC {
* override fun callback(hWnd: Pointer?, arg: Pointer?): Boolean {
* // Check if hWnd matches target window
* if (matchesTarget(hWnd)) {
* // Found target window, stop enumeration
* return false
* }
* // Keep enumerating
* return true
* }
* }
* ```
*
* @param hWnd Window handle (HWND) for the current enumerated window.
* @param arg User-defined data pointer passed to EnumWindows.
* @return True to continue enumerating windows, false to stop.
*/
interface WNDENUMPROC : StdCallLibrary.StdCallCallback {
fun callback(hWnd: Pointer?, arg: Pointer?): Boolean
}
/**
* Enumerates windows on the system.
*
* This calls the Windows API EnumWindows function to enumerate all top-level windows.
*
* For each window, it calls the provided [WNDENUMPROC] callback function,
* passing the window handle [hWnd] and user-defined [userData] pointer.
*
* Enumeration can be stopped by returning false from the callback.
*
* Usage example:
*
* ```
* val windows = mutableListOf<HWND>()
*
* val callback = object : WNDENUMPROC {
* override fun callback(hWnd: Pointer?, arg: Pointer?): Boolean {
* windows.add(hWnd) // Add hWnd to list
* return true // Continue enumerating
* }
* }
*
* EnumWindows(callback, null) // Get all top-level windows
*
* println(windows) // Print list of HWNDs
* ```
*
* @param lpEnumFunc The [WNDENUMPROC] callback to call for each window.
* @param userData Optional user-defined data to pass to the callback.
* @return True if successful, false otherwise.
*/
fun EnumWindows(lpEnumFunc: WNDENUMPROC?, userData: Pointer?): Boolean
/**
* Gets the title text of the specified window.
*
* This calls the Win32 API GetWindowTextA function to get the title
* text for the window handle [hWnd].
*
* The window text is copied into the [lpString] buffer up to [nMaxCount] characters.
*
* Usage example:
*
* ```
* val buffer = ByteArray(256)
* val hwnd = getWindowHandle() // get some HWND
*
* GetWindowTextA(hwnd, buffer, buffer.size)
*
* val windowTitle = String(buffer)
* println(windowTitle)
* ```
*
* @param hWnd The window handle (HWND).
* @param lpString The buffer to receive the window text.
* @param nMaxCount The maximum number of characters to copy to the buffer.
* @return The length of the window text (excluding null-terminator).
*/
fun GetWindowTextA(hWnd: Pointer?, lpString: ByteArray?, nMaxCount: Int): Int
/**
* Gets the handle of the foreground window.
*
* Calls the Win32 API [User32.GetForegroundWindow] to retrieve the window handle
* of the current foreground window - the one the user is currently interacting with.
*
* @return The window handle (HWND) of the current foreground window, or null if none.
*/
fun GetForegroundWindow(): Pointer?
/**
* Sets the foreground window.
*
* Calls the Win32 API [User32.SetForegroundWindow] to bring the window with the given
* handle [hWnd] to the foreground.
*
* @param hWnd The native window handle (HWND) of the window to activate.
* @return True if successful, false otherwise.
*/
fun SetForegroundWindow(hWnd: Pointer?): Boolean
/**
* Loads the User32.dll library and gets an instance of the [User32] interface.
*
* This uses JNA to dynamically load the user32.dll library at runtime. It casts
* the result to the [User32] interface to provide access to the Windows API
* functions defined there.
*
* The [User32] instance is stored in [INSTANCE] to be used throughout the class
* for calling Windows APIs.
*
* For example:
*
* ```
* val user32 = User32.INSTANCE
*
* user32.EnumWindows(...) // Call Windows API
* ```
*
* @see User32
*/
companion object {
/**
* Loads the User32.dll library and gets an instance of the [User32] interface.
*
* This uses JNA to dynamically load the user32.dll library at runtime. It casts
* the result to the [User32] interface to provide access to the Windows API
* functions defined there.
*
* The [User32] instance is stored in [INSTANCE] to be used throughout the class
* for calling Windows APIs.
*
* For example:
*
* ```
* val user32 = User32.INSTANCE
*
* user32.EnumWindows(...) // Call Windows API
* ```
*
* @see User32
*/
val INSTANCE = Native.load("user32", User32::class.java) as User32
}
}
/**
* Callback class used for enumerating windows.
*
* This implements the [User32.WNDENUMPROC] interface required by the
* [User32.EnumWindows] API. An instance of this class is passed to
* [User32.EnumWindows] to receive callbacks for each window.
*
* @param cb The callback function to invoke for each window. It should return
* true to continue enumeration or false to stop.
*/
internal class WindowEnumCallback(val cb: (Pointer?, Pointer?) -> Boolean) : User32.WNDENUMPROC {
/**
* Called by [User32.EnumWindows] for each window.
*
* Implements the callback method required by [User32.WNDENUMPROC].
* This simply invokes the provided [cb] callback and returns its result.
*
* @param hWnd The window handle being enumerated.
* @param userDataPtr A pointer to user data passed to [User32.EnumWindows].
* @return The return value from the [cb] callback.
*/
override fun callback(hWnd: Pointer?, userDataPtr: Pointer?): Boolean {
return cb(hWnd, userDataPtr)
}
}
/**
* Gets the title of the active/foreground window.
*
* This calls Win32 APIs to get the handle of the foreground window,
* then gets its title text.
*
* Usage example:
*
* ```
* val activeWindowName = getActiveWindowName()
*
* println(activeWindowName) // Prints foreground window title
* ```
*
* @return The title text of the current foreground window.
*/
override fun getActiveWindowName(): String {
val user32 = User32.INSTANCE
val foregroundWindowHwnd = user32.GetForegroundWindow()
return getWindowName(foregroundWindowHwnd)
}
/**
* Gets the title/name of the window for the given handle.
*
* This calls the Win32 API [User32.GetWindowTextA] to retrieve the title
* text for the window referenced by [hWnd].
*
* It allocates a [windowTitleBuffer] byte array to hold the result. This is
* passed to [User32.GetWindowTextA] to be populated.
*
* The buffer is then converted to a [String] via [nativeByteBufferToString].
*
* @param hWnd The native window handle to get the title for.
* @return The window title text as a [String].
*/
private fun getWindowName(hWnd: Pointer?): String {
val maxTitleLength = 512
val user32 = User32.INSTANCE
val windowTitleBuffer = ByteArray(maxTitleLength)
user32.GetWindowTextA(hWnd, windowTitleBuffer, windowTitleBuffer.size)
return nativeByteBufferToString(windowTitleBuffer)
}
/**
* Enumerates all open window names on the desktop.
*
* Calls the Win32 API [User32.EnumWindows] to iterate through all current open windows.
* For each window handle, it retrieves the window name using [getWindowName]
* and adds it to a list if the name is not blank. We filter out blank window names because that particular information
* is useless for any reason other than counting how many open windows there are. If we actually need that information,
* we can simply acquire a list of all HWND references.
*
* @return An [ArrayList] containing the name of each open window.
*/
override fun enumWindowNames(): ArrayList<String> {
val user32 = User32.INSTANCE
val windowNames = ArrayList<String>()
val iterationCallback = WindowEnumCallback { hWnd, _ ->
val windowName = getWindowName(hWnd)
if (windowName.isNotBlank()) {
windowNames.add(windowName)
}
true
}
user32.EnumWindows(iterationCallback, null)
return windowNames
}
/**
* Sets the foreground window by name on Windows.
*
* This calls the Win32 API [User32.EnumWindows] to iterate through all
* top-level windows, compares their name to the given [name], and calls
* [User32.SetForegroundWindow] on the matching window to bring it to the foreground.
*
* @param name The window name to search for and activate.
*/
override fun setForegroundWindowByName(name: String) {
val user32 = User32.INSTANCE
val iterationCallback = WindowEnumCallback { hWnd, _ ->
val windowName = getWindowName(hWnd)
if (windowName.isBlank()) {
//if the window is blank, tell the system that we should continue
true
} else {
//otherwise, we need to check the window name
if (windowName == name) {
user32.SetForegroundWindow(hWnd)
//we found a match, so tell the system that we don't need to proceed any further
false
} else {
true
}
}
}
user32.EnumWindows(iterationCallback, null)
}
}

82
src/main/kotlin/controllers/MousePointerObserver.kt Normal file
View File

@ -0,0 +1,82 @@
package controllers
import params.MouseWiggleParams
import java.awt.MouseInfo
import java.awt.Point
import kotlin.random.Random
/**
* Interface for controllers that interact with the desktop.
*
* This defines methods for getting desktop state like the mouse pointer
* location.
*
* Classes that implement this can serve as desktop automation controllers.
*/
interface MousePointerObserver {
/**
* Gets the current pointer/mouse location on the desktop.
*
* This returns a [Point] representing the x, y coordinates of the mouse pointer on the screen.
*
* For example:
*
* ```
* val mouseLocation = getPointerLocation()
*
* println(mouseLocation) // Might print "Point[x=1920,y=1080]"
* ```
*
* @return The current [Point] location of the mouse pointer on the screen.
*/
fun getPointerLocation(): Point {
return MouseInfo.getPointerInfo().location
}
/**
* Gets a point near the given [point], with some random wiggle.
*
* This generates a new point that is randomly offset from the given [point]
* by an amount controlled by the given [MouseWiggleParams].
*
* Usage example:
*
* ```
* val base = Point(10, 20)
* val params = WiggleParams(xWiggle = 5, yWiggle = 5)
* val randomPoint = getAlmostPoint(base, params)
*
* // randomPoint might be (8, 22)
* ```
*
* Alternatively, params may be omitted to use the default WiggleParams values
* ```
* val base = Point(10, 20)
* val randomPoint = getAlmostPoint(base)
*
* // randomPoint might be (8, 22)
* ```
*
* @param point The base point to start from
* @param params The wiggle parameters that control the random offset amount
* @return A new [Point] near the given point with some random wiggle applied
*/
fun getAlmostPoint(point: Point, params: MouseWiggleParams = MouseWiggleParams()): Point {
val xDel = Random.nextInt(0, params.xWiggle)
val yDel = Random.nextInt(0, params.yWiggle)
val xDir = if (Random.nextDouble() > 0.5) {
1
} else {
-1
}
val yDir = if (Random.nextDouble() > 0.5) {
1
} else {
-1
}
return Point(point.x + (xDel * xDir), point.y + (yDel * yDir))
}
}

14
src/main/kotlin/controllers/OSProxy.kt Normal file
View File

@ -0,0 +1,14 @@
package controllers
import java.awt.Rectangle
interface OSProxy {
fun getActiveWindowName(): String
fun enumWindowNames(): ArrayList<String>
fun setForegroundWindowByName(name: String)
fun getForegroundWindowBounds(): Rectangle?
}

2
src/main/kotlin/controllers/RobotAutomaton.kt
View File

@ -38,7 +38,7 @@ import java.awt.event.InputEvent
* *
* @param robot The Robot instance to use. A default is created if not provided. * @param robot The Robot instance to use. A default is created if not provided.
*/ */
open class RobotAutomaton(private val robot: Robot = Robot()) : Automaton { open class RobotAutomaton(internal val robot: Robot = Robot()) : Automaton {
/** /**
* Moves the mouse pointer to the given [Point] coordinates. * Moves the mouse pointer to the given [Point] coordinates.

18
src/main/kotlin/controllers/VisionController.kt Normal file
View File

@ -0,0 +1,18 @@
package controllers
import controllers.windows.WindowsOSProxy
import java.nio.file.Paths
import javax.imageio.ImageIO
interface VisionController: Automaton, OSProxy {
fun takeScreenshot()
}
class ConcreteVisionController(): VisionController, WindowsOSProxy, RobotAutomaton(){
override fun takeScreenshot() {
val rect = getForegroundWindowBounds()
val img = robot.createScreenCapture(rect)
val testPath = Paths.get(".", "test2.png")
ImageIO.write(img, "png", testPath.toFile())
}
}

218
src/main/kotlin/controllers/windows/User32.kt Normal file
View File

@ -0,0 +1,218 @@
package controllers.windows
import com.sun.jna.Native
import com.sun.jna.NativeLong
import com.sun.jna.Pointer
import com.sun.jna.win32.StdCallLibrary
import controllers.windows.WindowsOSProxy.WinRect
/**
* Interface for calling Windows User32 API functions.
*
* This defines an interface extending StdCallLibrary to call native
* Windows User32 library functions like EnumWindows, GetWindowTextA etc.
*
* Classes can implement this interface to make direct calls to the
* User32 DLL on Windows.
*/
interface User32 : StdCallLibrary {
/**
* Interface for a Windows callback function to enumerate windows.
*
* This extends the StdCallLibrary.StdCallCallback to define a callback
* method that will be invoked by the Windows API EnumWindows function.
*
* The callback method accepts a window handle (HWND) and a user-defined
* pointer, and returns a Boolean indicating whether to continue enumeration.
*
* Usage example:
* ```
* val callback = object : WNDENUMPROC {
* override fun callback(hWnd: Pointer?, arg: Pointer?): Boolean {
* // Check if hWnd matches target window
* if (matchesTarget(hWnd)) {
* // Found target window, stop enumeration
* return false
* }
* // Keep enumerating
* return true
* }
* }
* ```
*
* @param hWnd Window handle (HWND) for the current enumerated window.
* @param arg User-defined data pointer passed to EnumWindows.
* @return True to continue enumerating windows, false to stop.
*/
interface WNDENUMPROC : StdCallLibrary.StdCallCallback {
fun callback(hWnd: Pointer?, userDataPtr: Pointer?): Boolean
}
/**
* Enumerates windows on the system.
*
* This calls the Windows API EnumWindows function to enumerate all top-level windows.
*
* For each window, it calls the provided [WNDENUMPROC] callback function,
* passing the window handle [hWnd] and user-defined [userData] pointer.
*
* Enumeration can be stopped by returning false from the callback.
*
* Usage example:
*
* ```
* val windows = mutableListOf<HWND>()
*
* val callback = object : WNDENUMPROC {
* override fun callback(hWnd: Pointer?, arg: Pointer?): Boolean {
* windows.add(hWnd) // Add hWnd to list
* return true // Continue enumerating
* }
* }
*
* EnumWindows(callback, null) // Get all top-level windows
*
* println(windows) // Print list of HWNDs
* ```
*
* @param lpEnumFunc The [WNDENUMPROC] callback to call for each window.
* @param userData Optional user-defined data to pass to the callback.
* @return True if successful, false otherwise.
*/
fun EnumWindows(lpEnumFunc: WNDENUMPROC?, userData: Pointer?): Boolean
/**
* Gets the title text of the specified window.
*
* This calls the Win32 API GetWindowTextA function to get the title
* text for the window handle [hWnd].
*
* The window text is copied into the [lpString] buffer up to [nMaxCount] characters.
*
* Usage example:
*
* ```
* val buffer = ByteArray(256)
* val hwnd = getWindowHandle() // get some HWND
*
* GetWindowTextA(hwnd, buffer, buffer.size)
*
* val windowTitle = String(buffer)
* println(windowTitle)
* ```
*
* @param hWnd The window handle (HWND).
* @param lpString The buffer to receive the window text.
* @param nMaxCount The maximum number of characters to copy to the buffer.
* @return The length of the window text (excluding null-terminator).
*/
fun GetWindowTextA(hWnd: Pointer?, lpString: ByteArray?, nMaxCount: Int): Int
/**
* Gets the handle of the foreground window.
*
* Calls the Win32 API [User32.GetForegroundWindow] to retrieve the window handle
* of the current foreground window - the one the user is currently interacting with.
*
* @return The window handle (HWND) of the current foreground window, or null if none.
*/
fun GetForegroundWindow(): Pointer?
/**
* Sets the foreground window.
*
* Calls the Win32 API [User32.SetForegroundWindow] to bring the window with the given
* handle [hWnd] to the foreground.
*
* @param hWnd The native window handle (HWND) of the window to activate.
* @return True if successful, false otherwise.
*/
fun SetForegroundWindow(hWnd: Pointer?): Boolean
/**
* Retrieves the coordinates of a window on the screen.
*
* This calls the Win32 API [User32.GetWindowRect] to populate the provided [lpRect]
* struct with the outer bounding rectangle of the window specified by [hWnd].
*
* The rectangle coordinates represent the upper-left and lower-right corners
* relative to the screen.
*
* @param hWnd The native window handle (HWND) to get the bounds for.
* @param lpRect Pointer to a [WinRect] struct to populate with the coordinates.
* @return True if successful, false otherwise.
*/
fun GetWindowRect(hWnd: Pointer?, lpRect: Pointer?): Boolean
fun GetWindowLongA(hWnd: Pointer?, nIndex: Int): Int
/**
* Gets the DPI value for the system.
*
* This calls the Win32 API GetDpiForSystem() function to get the current DPI or scaling
* percentage value that is set for the system.
*
* For example, on a normal 100% scaled display this will return 96. On a 150% scaled 4K display,
* this may return 144.
*
* @return The DPI scaling value for the system.
*/
fun GetDpiForSystem(): Int
fun GetDpiForWindow(hWnd: Pointer?): Int
/**
* Adjusts the specified window rectangle taking into account DPI scaling.
*
* This calls the Win32 API AdjustWindowRectExForDpi function to adjust the provided
* window rectangle [lpRect] for the given window styles and DPI scaling value [dpi].
*
* The rectangle will be expanded or contracted based on the window styles, menu,
* and DPI scaling. This allows properly sizing windows for the current display.
*
* @param lpRect Pointer to the RECT structure to adjust.
* @param dwStyle The window style flags for the window.
* @param bMenu Whether the window has a menu.
* @param dwExStyle The extended window style flags.
* @param dpi The DPI scaling value of the display.
* @return True if successful, false otherwise.
*/
fun AdjustWindowRectExForDpi(
lpRect: Pointer?,
dwStyle: Int,
bMenu: Boolean,
dwExStyle: Int,
dpi: Int
): Boolean
companion object {
/**
* Loads the User32.dll library and gets an instance of the [User32] interface.
*
* This uses JNA to dynamically load the user32.dll library at runtime, which provides access to the Windows
* API functions defined there.
*
* The [User32] instance is stored in the [INSTANCE] field to be used throughout the class
* for calling Windows APIs.
*
* For example:
*
* ```
* val user32 = User32.INSTANCE
*
* user32.EnumWindows(...) // Call Windows API
* ```
*
* @see User32
*/
val INSTANCE = Native.load("user32", User32::class.java) as User32
public const val GWL_EXSTYLE = -20
public const val GWL_STYLE = -16
}
}

295
src/main/kotlin/controllers/windows/WindowsOSProxy.kt Normal file
View File

@ -0,0 +1,295 @@
package controllers.windows
import com.sun.jna.Native
import com.sun.jna.Pointer
import com.sun.jna.Structure
import com.sun.jna.Structure.FieldOrder
import controllers.MousePointerObserver
import controllers.OSProxy
import java.awt.Rectangle
/**
* Windows implementation of [MousePointerObserver].
*
* This class provides methods to interact with the desktop on Windows
* by calling Win32 APIs.
*
* It implements the [MousePointerObserver] interface to provide desktop
* functionality like getting the mouse pointer location on Windows.
*/
interface WindowsOSProxy : MousePointerObserver, OSProxy {
companion object {
/**
* Converts a native byte buffer to a String.
*
* This takes a byte array [byteBuffer] containing text from a native Win32 call,
* converts it to a String using JNA, and trims whitespace characters.
*
* Usage example:
*
* ```
* val buffer = ByteArray(256)
* GetWindowTextA(hwnd, buffer, buffer.size) // Win32 call
*
* val windowTitle = nativeByteBufferToString(buffer)
* println(windowTitle) // Print title string
* ```
*
* @param byteBuffer Byte array containing text from a native call
* @return The native text as a String
*/
private fun nativeByteBufferToString(byteBuffer: ByteArray): String {
return Native.toString(byteBuffer).trim { it <= ' ' }
}
}
/**
* Callback class used for enumerating windows.
*
* This implements the [User32.WNDENUMPROC] interface required by the
* [User32.EnumWindows] API. An instance of this class is passed to
* [User32.EnumWindows] to receive callbacks for each window.
*
* @param cb The callback function to invoke for each window. It should return
* true to continue enumeration or false to stop.
*/
class WindowEnumCallback(val cb: (Pointer?, Pointer?) -> Boolean) : User32.WNDENUMPROC {
/**
* Called by [User32.EnumWindows] for each window.
*
* Implements the callback method required by [User32.WNDENUMPROC].
* This simply invokes the provided [cb] callback and returns its result.
*
* @param hWnd The window handle being enumerated.
* @param userDataPtr A pointer to user data passed to [User32.EnumWindows].
* @return The return value from the [cb] callback.
*/
override fun callback(hWnd: Pointer?, userDataPtr: Pointer?): Boolean {
return cb(hWnd, userDataPtr)
}
}
/**
* Mirror of the Windows RECT struct used for window coordinates.
*/
@FieldOrder("left", "top", "right", "bottom")
public class WinRect : Structure() {
/** The x coordinate of the left edge */
@JvmField
var left: Int = 0
/** The y coordinate of the top edge */
@JvmField
var top: Int = 0
/** The x coordinate of the right edge */
@JvmField
var right: Int = 0
/** The y coordinate of the bottom edge */
@JvmField
var bottom: Int = 0
}
/**
* Gets the title of the active/foreground window.
*
* This calls Win32 APIs to get the handle of the foreground window,
* then gets its title text.
*
* Usage example:
*
* ```
* val activeWindowName = getActiveWindowName()
*
* println(activeWindowName) // Prints foreground window title
* ```
*
* @return The title text of the current foreground window.
*/
override fun getActiveWindowName(): String {
val user32 = User32.INSTANCE
val foregroundWindowHwnd = user32.GetForegroundWindow()
return getWindowName(foregroundWindowHwnd)
}
fun getWindowHandleByName(name: String): Pointer? {
val user32 = User32.INSTANCE
var ptr: Pointer? = null
val enumFunction = WindowEnumCallback { hwnd, _ ->
val wName = getWindowName(hwnd)
if (name == wName) {
ptr = hwnd
false
} else {
true
}
}
user32.EnumWindows(enumFunction, null)
return ptr
}
/**
* Gets the title/name of the window for the given handle.
*
* This calls the Win32 API [User32.GetWindowTextA] to retrieve the title
* text for the window referenced by [hWnd].
*
* It allocates a [windowTitleBuffer] byte array to hold the result. This is
* passed to [User32.GetWindowTextA] to be populated.
*
* The buffer is then converted to a [String] via [nativeByteBufferToString].
*
* @param hWnd The native window handle to get the title for.
* @return The window title text as a [String].
*/
private fun getWindowName(hWnd: Pointer?): String {
val maxTitleLength = 512
val user32 = User32.INSTANCE
val windowTitleBuffer = ByteArray(maxTitleLength)
user32.GetWindowTextA(hWnd, windowTitleBuffer, windowTitleBuffer.size)
return nativeByteBufferToString(windowTitleBuffer)
}
/**
* Enumerates all open window names on the desktop.
*
* Calls the Win32 API [User32.EnumWindows] to iterate through all current open windows.
* For each window handle, it retrieves the window name using [getWindowName]
* and adds it to a list if the name is not blank. We filter out blank window names because that particular information
* is useless for any reason other than counting how many open windows there are. If we actually need that information,
* we can simply acquire a list of all HWND references.
*
* @return An [ArrayList] containing the name of each open window.
*/
override fun enumWindowNames(): ArrayList<String> {
val user32 = User32.INSTANCE
val windowNames = ArrayList<String>()
val iterationCallback = WindowEnumCallback { hWnd, _ ->
val windowName = getWindowName(hWnd)
if (windowName.isNotBlank()) {
windowNames.add(windowName)
}
true
}
user32.EnumWindows(iterationCallback, null)
return windowNames
}
/**
* Sets the foreground window by name on Windows.
*
* This calls the Win32 API [User32.EnumWindows] to iterate through all
* top-level windows, compares their name to the given [name], and calls
* [User32.SetForegroundWindow] on the matching window to bring it to the foreground.
*
* @param name The window name to search for and activate.
*/
override fun setForegroundWindowByName(name: String) {
val user32 = User32.INSTANCE
val iterationCallback = WindowEnumCallback { hWnd, _ ->
val windowName = getWindowName(hWnd)
if (windowName.isBlank()) {
//if the window is blank, tell the system that we should continue
true
} else {
//otherwise, we need to check the window name
if (windowName == name) {
user32.SetForegroundWindow(hWnd)
//we found a match, so tell the system that we don't need to proceed any further
false
} else {
true
}
}
}
user32.EnumWindows(iterationCallback, null)
}
/**
* Gets the screen bounds of the current foreground window.
*
* Calls the Win32 API [User32.GetForegroundWindow] to get the HWND of the
* foreground window.
*
* Then calls [User32.GetWindowRect] to populate a [WinRect] struct with the
* window bounds.
*
* The rectangle coordinates are converted to a [Rectangle] and returned. This allows us to keep the implementation
* generic and constrained within the java std library.
* In the future we may want to add LinuxOSProxy or DarwinOSProxy.
*
* @return The outer bounding rectangle of the foreground window in screen coordinates,
* or an empty [Rectangle] if it failed.
*/
override fun getForegroundWindowBounds(): Rectangle? {
val user32 = User32.INSTANCE
val hWnd = user32.GetForegroundWindow()
val rect = getRectFromWindowHandle(user32, hWnd)
return if (rect != null) {
//we need to make the calls to get the info to correct for the dpi scaling
val secondarySuccess = correctWinRectForDpi(user32, hWnd!!, rect.pointer)
if (secondarySuccess) {
//the correctWinRectForDpi function doesn't have access to the underlying stuct in order to read it, so
// we have to call this
rect.read()
Rectangle(rect.top, rect.left, (rect.right - rect.left), (rect.bottom - rect.top))
} else {
return null
}
} else {
null
}
}
private fun getRectFromWindowHandle(user32: User32, hWnd: Pointer?): WinRect? {
//we have to provide the system a native struct in order to hold the results of our request
val rect = WinRect()
val success = user32.GetWindowRect(hWnd, rect.pointer)
return if (success) {
//the values are stuck down in memory, so we have to read these back out in order to proceed
rect.read()
rect
} else {
null
}
}
private fun correctWinRectForDpi(user32: User32, hWnd: Pointer, lpRect: Pointer): Boolean {
val user32 = User32.INSTANCE
val dwStyle = user32.GetWindowLongA(hWnd, User32.GWL_STYLE)
val bMenu = true
val dwExStyle = user32.GetWindowLongA(hWnd, User32.GWL_EXSTYLE)
val dpi = user32.GetDpiForWindow(hWnd)
return user32.AdjustWindowRectExForDpi(lpRect, dwStyle, bMenu, dwExStyle, dpi)
}
}

10
src/test/kotlin/controllers/DesktopControllerTest.kt
View File

@ -20,7 +20,7 @@ class DesktopControllerTest {
*/ */
@Test @Test
fun `getPointerLocation returns mouse position`() { fun `getPointerLocation returns mouse position`() {
val controller = mock(DesktopController::class.java) val controller = mock(MousePointerObserver::class.java)
// Mock mouse position // Mock mouse position
`when`(controller.getPointerLocation()).thenReturn(Point(100, 200)) `when`(controller.getPointerLocation()).thenReturn(Point(100, 200))
@ -46,7 +46,7 @@ class DesktopControllerTest {
*/ */
@Test @Test
fun `getAlmostPoint returns wiggly point`() { fun `getAlmostPoint returns wiggly point`() {
val controller = mock(DesktopController::class.java) val controller = mock(MousePointerObserver::class.java)
val params = MouseWiggleParams(xWiggle = 10, yWiggle = 10) val params = MouseWiggleParams(xWiggle = 10, yWiggle = 10)
// Mock random wiggle // Mock random wiggle
@ -60,10 +60,4 @@ class DesktopControllerTest {
assertNotEquals(100, wiggly.x) assertNotEquals(100, wiggly.x)
assertNotEquals(200, wiggly.y) assertNotEquals(200, wiggly.y)
} }
@Test
fun devTest(){
val c = WindowsDesktopController()
c.setForegroundWindowByName("RuneScape")
}
} }

13
src/test/kotlin/controllers/OSProxyTest.kt Normal file
View File

@ -0,0 +1,13 @@
package controllers
import org.junit.jupiter.api.Test
class OSProxyTest {
@Test
fun test(){
val vc = ConcreteVisionController()
val rect = vc.getForegroundWindowBounds()
println(rect)
}
}

12
src/test/kotlin/controllers/VisionControllerTest.kt Normal file
View File

@ -0,0 +1,12 @@
package controllers
import kotlin.test.Test
class VisionControllerTest {
@Test
fun testImageCapture(){
val vc = ConcreteVisionController()
vc.takeScreenshot()
}
}