cleaning up the AI docs. verbose fucker, ain't he?
This commit is contained in:
parent
bb23ab3ec2
commit
244ca8d134
@ -1,6 +1,5 @@
|
|||||||
package controllers
|
package controllers
|
||||||
|
|
||||||
import params.MouseWiggleParams
|
|
||||||
import java.awt.MouseInfo
|
import java.awt.MouseInfo
|
||||||
import java.awt.Point
|
import java.awt.Point
|
||||||
import kotlin.random.Random
|
import kotlin.random.Random
|
||||||
@ -67,6 +66,8 @@ interface MousePointerObserver {
|
|||||||
fun getAlmostPoint(point: Point, params: MouseWiggleParams = MouseWiggleParams()): Point {
|
fun getAlmostPoint(point: Point, params: MouseWiggleParams = MouseWiggleParams()): Point {
|
||||||
val xDel = Random.nextInt(0, params.xWiggle)
|
val xDel = Random.nextInt(0, params.xWiggle)
|
||||||
val yDel = Random.nextInt(0, params.yWiggle)
|
val yDel = Random.nextInt(0, params.yWiggle)
|
||||||
|
|
||||||
|
//flip two coins, determine direction based on the parity of the results
|
||||||
val xDir = if (Random.nextDouble() > 0.5) {
|
val xDir = if (Random.nextDouble() > 0.5) {
|
||||||
1
|
1
|
||||||
} else {
|
} else {
|
||||||
@ -77,6 +78,37 @@ interface MousePointerObserver {
|
|||||||
} else {
|
} else {
|
||||||
-1
|
-1
|
||||||
}
|
}
|
||||||
|
|
||||||
return Point(point.x + (xDel * xDir), point.y + (yDel * yDir))
|
return Point(point.x + (xDel * xDir), point.y + (yDel * yDir))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Data class to hold wiggle parameters for mouse movement.
|
||||||
|
*
|
||||||
|
* This simple data class holds two integer properties for x and y wiggle amounts.
|
||||||
|
* These are used when generating simulated mouse movements to add some variance
|
||||||
|
* and randomness to the coordinates.
|
||||||
|
*
|
||||||
|
* For example, if a target destination point is (100, 200), the wiggle params
|
||||||
|
* might generate an actual movement point like (102, 198) to add some randomness.
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
*
|
||||||
|
* ```
|
||||||
|
* val controller = DesktopController()
|
||||||
|
* val wiggle = WiggleParams(xWiggle = 10, yWiggle = 15)
|
||||||
|
*
|
||||||
|
* val target = Point(100, 200)
|
||||||
|
* val actual = controller.getAlmostPoint(target, wiggle) // (104, 197)
|
||||||
|
* ```
|
||||||
|
*
|
||||||
|
* @param xWiggle The max amount of variance in x direction. Default 25.
|
||||||
|
* @param yWiggle The max amount of variance in y direction. Default 25.
|
||||||
|
*/
|
||||||
|
data class MouseWiggleParams(
|
||||||
|
val xWiggle: Int = 25,
|
||||||
|
val yWiggle: Int = 25
|
||||||
|
)
|
||||||
|
|
||||||
|
|||||||
@ -4,78 +4,44 @@ import java.awt.Point
|
|||||||
import java.awt.Robot
|
import java.awt.Robot
|
||||||
import java.awt.event.InputEvent
|
import java.awt.event.InputEvent
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Desktop automation controller using java.awt.Robot.
|
* RobotAutomaton is an open class that controls a java.awt.Robot.
|
||||||
*
|
*
|
||||||
* This provides mouse, keyboard, and timing control capabilities by
|
* This provides a Kotlin API for automating mouse and keyboard actions.
|
||||||
* wrapping the java.awt.Robot class.
|
|
||||||
*
|
*
|
||||||
* Key features:
|
* @param robot The java.awt.Robot instance to control. Defaults to a new Robot().
|
||||||
*
|
|
||||||
* - Get current mouse/pointer location
|
|
||||||
* - Move mouse and perform clicks
|
|
||||||
* - Keyboard presses and hotkeys
|
|
||||||
* - Scroll wheel motions
|
|
||||||
* - Sleep/delay methods with variance
|
|
||||||
*
|
|
||||||
* RobotController aims to provide a simple and easy to use API for
|
|
||||||
* automating desktop interactions and workflows.
|
|
||||||
*
|
|
||||||
* Usage example:
|
|
||||||
*
|
|
||||||
* ```
|
|
||||||
* val robot = RobotController()
|
|
||||||
*
|
|
||||||
* // Move mouse to 100, 200
|
|
||||||
* robot.mouseMove(Point(100, 200))
|
|
||||||
*
|
|
||||||
* // Left click at current position
|
|
||||||
* robot.click(InputEvent.BUTTON1_MASK)
|
|
||||||
*
|
|
||||||
* // Press A key
|
|
||||||
* robot.keyPress(KeyEvent.VK_A)
|
|
||||||
* ```
|
|
||||||
*
|
|
||||||
* @param robot The Robot instance to use. A default is created if not provided.
|
|
||||||
*/
|
*/
|
||||||
open class RobotAutomaton(internal val robot: Robot = Robot()) : Automaton {
|
open class RobotAutomaton(internal val robot: Robot = Robot()) : Automaton {
|
||||||
|
|
||||||
/**
|
|
||||||
* Moves the mouse pointer to the given [Point] coordinates.
|
|
||||||
*
|
|
||||||
* This will use the [Robot] API to move the mouse to the x,y position.
|
|
||||||
* It also does validation to retry the move if ended up at incorrect location.
|
|
||||||
*
|
|
||||||
* Usage examples:
|
|
||||||
*
|
|
||||||
* ```
|
|
||||||
* val controller = DesktopController()
|
|
||||||
* controller.moveMouse(Point(100, 200))
|
|
||||||
* ```
|
|
||||||
*
|
|
||||||
*
|
|
||||||
* ```
|
|
||||||
* val target = Point(500, 300)
|
|
||||||
* controller.moveMouse(target)
|
|
||||||
* ```
|
|
||||||
*
|
|
||||||
* @param point The destination [Point] x,y coordinates to move the mouse to.
|
|
||||||
*
|
|
||||||
*
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Moves the mouse cursor to the given point.
|
||||||
*
|
*
|
||||||
|
* Uses the Robot's mouseMove() method to move to the x, y coordinates.
|
||||||
|
*
|
||||||
|
* Due to a bug in OpenJDK, the mouse may not move all the way to the given
|
||||||
|
* point if using a non-default Windows display scale. So this method checks
|
||||||
|
* the current mouse position after moving and retries up to 10 times until
|
||||||
|
* the mouse reaches the exact intended point.
|
||||||
|
*
|
||||||
|
* @param point The Point representing the x, y coordinates to move the mouse to.
|
||||||
*/
|
*/
|
||||||
override fun moveMouse(point: Point) {
|
override fun moveMouse(point: Point) {
|
||||||
robot.mouseMove(point.x, point.y)
|
robot.mouseMove(point.x, point.y)
|
||||||
|
|
||||||
//There's a bug in OpenJDK that results in incorrect cursor position in the [Robot.mouseMove] function if using
|
// There is a bug in OpenJDK's Robot.mouseMove() on Windows machines using non-default display scaling. The
|
||||||
//a Windows Resolution scale other than 100%. As a result, we have to check that the cursor made it all the way.
|
// cursor may not move fully to the intended coordinates.
|
||||||
//From some anecdotal observation, it has an overshoot/decay pattern sort of like a binary search. The mouse will
|
|
||||||
//usually be in the correct place within 2-3 loop itterations
|
// To work around this, we check the current cursor position after moving and retry mouseMove() up to 10 times
|
||||||
|
// until the cursor reaches the exact intended destination.
|
||||||
|
|
||||||
|
// Based on testing, the cursor typically lands at the correct coordinates within 2-3 retries due to the
|
||||||
|
// decaying overshoot behavior.
|
||||||
|
|
||||||
repeat(10) {
|
repeat(10) {
|
||||||
val rPoint = getPointerLocation()
|
val rPoint = getPointerLocation()
|
||||||
//here, we check the points and if we're good, we
|
|
||||||
if (rPoint.x == point.x && rPoint.y == point.y) {
|
if (rPoint.x == point.x && rPoint.y == point.y) {
|
||||||
return
|
return
|
||||||
} else {
|
} else {
|
||||||
@ -85,34 +51,28 @@ open class RobotAutomaton(internal val robot: Robot = Robot()) : Automaton {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Performs a mouse click of the specified button.
|
* Performs a mouse click with the given button.
|
||||||
*
|
*
|
||||||
* This uses the Robot to press and release the given mouse button.
|
* Uses the Robot to simulate pressing and releasing the mouse button.
|
||||||
*
|
*
|
||||||
* A random sleep is added in between pressing and releasing the button
|
* A random sleep is added after pressing the button before releasing,
|
||||||
* to add variance and avoid robotic timing.
|
* to add variance in timing and avoid robotic behavior.
|
||||||
*
|
*
|
||||||
* Example usage:
|
* Valid button values are:
|
||||||
|
* - InputEvent.BUTTON1 (left click)
|
||||||
|
* - InputEvent.BUTTON2 (right click)
|
||||||
|
* - InputEvent.BUTTON3 (middle click)
|
||||||
*
|
*
|
||||||
* ```
|
* @param button the mouse button to click
|
||||||
* val robot = RobotController()
|
|
||||||
*
|
|
||||||
* // Perform left click at current mouse position
|
|
||||||
* robot.click(InputEvent.BUTTON1_MASK)
|
|
||||||
* ```
|
|
||||||
*
|
|
||||||
* @param button The button to click. Must be a valid constant like [InputEvent.BUTTON1_MASK].
|
|
||||||
*
|
|
||||||
* Returns immediately If button is negative. Button must be a positive integer.
|
|
||||||
*/
|
*/
|
||||||
override fun mouseClick(button: Int) {
|
override fun mouseClick(button: Int) {
|
||||||
//guardian logic
|
// Check for valid button value
|
||||||
if (button < 0) {
|
if (button < 0) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
robot.mousePress(button)
|
robot.mousePress(button)
|
||||||
|
|
||||||
//we add in some random time variance here to appear less robotic
|
// Add random sleep to vary timing
|
||||||
sleepWithVariance(8, 8)
|
sleepWithVariance(8, 8)
|
||||||
|
|
||||||
robot.mouseRelease(button)
|
robot.mouseRelease(button)
|
||||||
@ -121,33 +81,22 @@ open class RobotAutomaton(internal val robot: Robot = Robot()) : Automaton {
|
|||||||
/**
|
/**
|
||||||
* Presses and releases the given key.
|
* Presses and releases the given key.
|
||||||
*
|
*
|
||||||
* This uses the Robot to simulate pressing and releasing the key with the given key code.
|
* Uses the Robot's keyPress() and keyRelease() methods to simulate pressing and releasing the key.
|
||||||
*
|
*
|
||||||
* A random sleep is added after pressing the key before releasing it to add variance
|
* A random sleep is added after pressing the key before releasing, to add variance in timing and avoid robotic
|
||||||
* and avoid robotic timing.
|
* behavior. The worst-case delay is ~1 frame at 60fps.
|
||||||
*
|
*
|
||||||
* Example usage:
|
* @param keyCode The key code of the key to press, such as KeyEvent.VK_A
|
||||||
*
|
|
||||||
* ```
|
|
||||||
* val robot = RobotController()
|
|
||||||
*
|
|
||||||
* // Press the 'A' key
|
|
||||||
* robot.keyPress(KeyEvent.VK_A)
|
|
||||||
* ```
|
|
||||||
*
|
|
||||||
* @param keyCode The key code of the key to press, such as [KeyEvent.VK_A].
|
|
||||||
*
|
|
||||||
* Returns immediately if keyCode < 0. This can be useful for skipping actions by passing -1
|
|
||||||
*/
|
*/
|
||||||
override fun keyPress(keyCode: Int) {
|
override fun keyPress(keyCode: Int) {
|
||||||
//guardian logic
|
// Check for valid keyCode
|
||||||
if (keyCode < 0) {
|
if (keyCode < 0) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
robot.keyPress(keyCode)
|
robot.keyPress(keyCode)
|
||||||
|
|
||||||
//we add in some random time variance here to appear less robotic
|
// Add random sleep to vary timing
|
||||||
sleepWithVariance(8, 8)
|
sleepWithVariance(8, 8)
|
||||||
|
|
||||||
robot.keyRelease(keyCode)
|
robot.keyRelease(keyCode)
|
||||||
@ -156,19 +105,11 @@ open class RobotAutomaton(internal val robot: Robot = Robot()) : Automaton {
|
|||||||
/**
|
/**
|
||||||
* Scrolls the mouse wheel down by one unit.
|
* Scrolls the mouse wheel down by one unit.
|
||||||
*
|
*
|
||||||
* Uses the [Robot.mouseWheel] method to scroll down and then sleeps
|
* Uses the [Robot.mouseWheel] method to scroll down and then sleeps for a random duration between the given
|
||||||
* for a random duration between 10-20ms to pace the scrolling.
|
* sleepDur + rand(0, sleepDurVariance) ms. This is to add variance in scrolling speed and avoid robotic behavior.
|
||||||
*
|
*
|
||||||
* Example usage:
|
* @param sleepDur The average duration to sleep after scrolling.
|
||||||
*
|
* @param sleepDurVariance The variance in sleep duration.
|
||||||
* ```
|
|
||||||
* val robot = RobotController()
|
|
||||||
*
|
|
||||||
* // Scroll down 5 units
|
|
||||||
* repeat(5) {
|
|
||||||
* robot.scrollDown()
|
|
||||||
* }
|
|
||||||
* ```
|
|
||||||
*/
|
*/
|
||||||
override fun scrollOut(sleepDur: Long, sleepDurVariance: Long) {
|
override fun scrollOut(sleepDur: Long, sleepDurVariance: Long) {
|
||||||
robot.mouseWheel(1)
|
robot.mouseWheel(1)
|
||||||
@ -178,19 +119,11 @@ open class RobotAutomaton(internal val robot: Robot = Robot()) : Automaton {
|
|||||||
/**
|
/**
|
||||||
* Scrolls the mouse wheel up by one unit.
|
* Scrolls the mouse wheel up by one unit.
|
||||||
*
|
*
|
||||||
* Uses the [Robot.mouseWheel] method to scroll up and then sleeps for a
|
* Uses the [Robot.mouseWheel] method to scroll up and then sleeps for a random duration between the given
|
||||||
* random duration between 10-20ms to pace the scrolling.
|
* sleepDur + rand(0, sleepDurVariance) ms. This adds variance in scrolling speed to avoid robotic behavior.
|
||||||
*
|
*
|
||||||
* Example usage:
|
* @param sleepDur The average duration to sleep after scrolling
|
||||||
*
|
* @param sleepDurVariance The variance in sleep duration
|
||||||
* ```
|
|
||||||
* val robot = RobotController()
|
|
||||||
*
|
|
||||||
* // Scroll up 10 units
|
|
||||||
* repeat(10) {
|
|
||||||
* robot.scrollUp()
|
|
||||||
* }
|
|
||||||
* ```
|
|
||||||
*/
|
*/
|
||||||
override fun scrollIn(sleepDur: Long, sleepDurVariance: Long) {
|
override fun scrollIn(sleepDur: Long, sleepDurVariance: Long) {
|
||||||
robot.mouseWheel(-1)
|
robot.mouseWheel(-1)
|
||||||
|
|||||||
@ -9,22 +9,21 @@ import controllers.windows.WindowsOSProxy.WinRect
|
|||||||
/**
|
/**
|
||||||
* Interface for calling Windows User32 API functions.
|
* Interface for calling Windows User32 API functions.
|
||||||
*
|
*
|
||||||
* This defines an interface extending StdCallLibrary to call native
|
* This defines an interface extending StdCallLibrary to call native Windows User32 library functions like EnumWindows,
|
||||||
* Windows User32 library functions like EnumWindows, GetWindowTextA etc.
|
* GetWindowTextA etc.
|
||||||
*
|
*
|
||||||
* Classes can implement this interface to make direct calls to the
|
* Classes can implement this interface to make direct calls to the User32 DLL on Windows.
|
||||||
* User32 DLL on Windows.
|
|
||||||
*/
|
*/
|
||||||
interface User32 : StdCallLibrary {
|
interface User32 : StdCallLibrary {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Interface for a Windows callback function to enumerate windows.
|
* Interface for a Windows callback function to enumerate windows.
|
||||||
*
|
*
|
||||||
* This extends the StdCallLibrary.StdCallCallback to define a callback
|
* This extends the StdCallLibrary.StdCallCallback to define a callback method that will be invoked by the Windows
|
||||||
* method that will be invoked by the Windows API EnumWindows function.
|
* API EnumWindows function.
|
||||||
*
|
*
|
||||||
* The callback method accepts a window handle (HWND) and a user-defined
|
* The callback method accepts a window handle (HWND) and a user-defined pointer, and returns a Boolean indicating
|
||||||
* pointer, and returns a Boolean indicating whether to continue enumeration.
|
* whether to continue enumeration.
|
||||||
*
|
*
|
||||||
* Usage example:
|
* Usage example:
|
||||||
* ```
|
* ```
|
||||||
|
|||||||
@ -1,36 +0,0 @@
|
|||||||
package params
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Data class to hold wiggle parameters for mouse movement.
|
|
||||||
*
|
|
||||||
* This simple data class holds two integer properties for x and y wiggle amounts.
|
|
||||||
* These are used when generating simulated mouse movements to add some variance
|
|
||||||
* and randomness to the coordinates.
|
|
||||||
*
|
|
||||||
* For example, if a target destination point is (100, 200), the wiggle params
|
|
||||||
* might generate an actual movement point like (102, 198) to add some randomness.
|
|
||||||
*
|
|
||||||
* Usage:
|
|
||||||
*
|
|
||||||
* ```
|
|
||||||
* val controller = DesktopController()
|
|
||||||
* val wiggle = WiggleParams(xWiggle = 10, yWiggle = 15)
|
|
||||||
*
|
|
||||||
* val target = Point(100, 200)
|
|
||||||
* val actual = controller.getAlmostPoint(target, wiggle) // (104, 197)
|
|
||||||
* ```
|
|
||||||
*
|
|
||||||
* @param xWiggle The max amount of variance in x direction. Default 25.
|
|
||||||
* @param yWiggle The max amount of variance in y direction. Default 25.
|
|
||||||
*/
|
|
||||||
data class MouseWiggleParams(
|
|
||||||
val xWiggle: Int = 25,
|
|
||||||
val yWiggle: Int = 25
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -1,7 +1,5 @@
|
|||||||
package params
|
package params
|
||||||
|
|
||||||
import java.awt.Point
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Interface for common task parameters used across automation routines.
|
* Interface for common task parameters used across automation routines.
|
||||||
*
|
*
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user