tookeyman/RuneFactory
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

VisionController can now pipe out visual stuff

Browse Source
This commit is contained in:
dtookey 2023-08-11 19:15:18 -04:00
parent 437a3b9fee
commit c7596a7b85
11 changed files with 181 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/kotlin/controllers/OSProxy.kt
View File

@ -10,5 +10,5 @@ interface OSProxy {
fun setForegroundWindowByName(name: String)
fun getForegroundWindowBounds(): Rectangle?
fun getScaledForegroundWindowBounds(): Rectangle?
}

54
src/main/kotlin/controllers/VisionController.kt
View File

@ -1,18 +1,52 @@
package controllers
import controllers.windows.WindowsOSProxy
import java.nio.file.Paths
import javax.imageio.ImageIO
import java.awt.image.BufferedImage
interface VisionController: Automaton, OSProxy {
fun takeScreenshot()
/**
* Interface for a vision controller that can take screenshots.
*
* A vision controller provides capabilities for automated visual
* interactions like taking screenshots.
*
* Implementations will provide OS-specific implementations for taking
* screenshots of the foreground window.
*/
interface VisionController : Automaton, OSProxy {
/**
* Takes a screenshot of the current foreground window.
*
* The implementation should use the appropriate OS APIs to get the
* window bounds and take a screenshot of the window.
*
* @return A BufferedImage containing the screenshot of the foreground
* window.
*/
fun takeScreenshotOfForeground(): BufferedImage
}
class ConcreteVisionController(): VisionController, WindowsOSProxy, RobotAutomaton(){
override fun takeScreenshot() {
val rect = getForegroundWindowBounds()
val img = robot.createScreenCapture(rect)
val testPath = Paths.get(".", "test2.png")
ImageIO.write(img, "png", testPath.toFile())
/**
* Concrete implementation of the [VisionController] interface.
* Implements the interface methods by extending [WindowsOSProxy] to get
* OS-specific functionality on Windows and [RobotAutomaton] to get access
* to the Robot class for taking automated screenshots.
*/
class ConcreteVisionController : VisionController, WindowsOSProxy, RobotAutomaton() {
/**
* Takes a screenshot of the foreground window scaled for high DPI.
*
* Gets the bounding rectangle of the current foreground window using
* [getScaledForegroundWindowBounds].
*
* Then uses the [RobotAutomaton.robot] to create a screenshot cropped to that
* rectangle area.
*
* @return A [BufferedImage] containing a screenshot of the foreground window
* scaled to account for the screen DPI.
*/
override fun takeScreenshotOfForeground(): BufferedImage {
val rect = getScaledForegroundWindowBounds()
return robot.createScreenCapture(rect)
}
}

56
src/main/kotlin/controllers/windows/User32.kt
View File

@ -1,7 +1,6 @@
package controllers.windows
import com.sun.jna.Native
import com.sun.jna.NativeLong
import com.sun.jna.Pointer
import com.sun.jna.win32.StdCallLibrary
import controllers.windows.WindowsOSProxy.WinRect
@ -146,6 +145,34 @@ interface User32 : StdCallLibrary {
fun GetWindowRect(hWnd: Pointer?, lpRect: Pointer?): Boolean
/**
* Gets information about the specified window.
*
* This calls the Win32 API GetWindowLongA function to get configuration
* information about the window handle [hWnd].
*
* The [nIndex] parameter specifies what window information to retrieve:
*
* - GWL_STYLE: Retrieves the window style.
* - GWL_EXSTYLE: Retrieves the extended window style.
* - GWL_ID: Retrieves the control ID.
*
* And more (see WinUser.h documentation).
*
* Usage example:
*
* ```
* val hwnd = findWindow() // Get some window handle
*
* val style = GetWindowLongA(hwnd, GWL_STYLE)
*
* println("Window style: $style")
* ```
*
* @param hWnd The window handle (HWND) to get info about.
* @param nIndex The info index to retrieve.
* @return The requested window information.
*/
fun GetWindowLongA(hWnd: Pointer?, nIndex: Int): Int
@ -162,7 +189,32 @@ interface User32 : StdCallLibrary {
*/
fun GetDpiForSystem(): Int
/**
* Gets the DPI for the specified window.
*
* This calls the Win32 API GetDpiForWindow function to get the DPI (dots per inch)
* value for the given window handle [hWnd].
*
* The DPI indicates the display resolution and is used for scaling elements
* appropriately on high resolution screens.
*
* For example, a 96 DPI screen means 96 pixels per inch. A 144 DPI screen has
* higher pixel density, so elements need to be scaled up in size accordingly.
*
* Usage:
*
* ```
* val hwnd = findWindow() // Get some window handle
* val dpi = GetDpiForWindow(hwnd)
*
* // Scale based on DPI
* val scale = dpi / 96f
* drawScaledElements(scale)
* ```
*
* @param hWnd The window handle (HWND) to get the DPI for.
* @return The DPI value for the given window.
*/
fun GetDpiForWindow(hWnd: Pointer?): Int
/**

83
src/main/kotlin/controllers/windows/WindowsOSProxy.kt
View File

@ -223,46 +223,71 @@ interface WindowsOSProxy : MousePointerObserver, OSProxy {
}
/**
* Gets the screen bounds of the current foreground window.
* Gets the screen bounds of the foreground window scaled for high DPI screens.
*
* Calls the Win32 API [User32.GetForegroundWindow] to get the HWND of the
* foreground window.
* Calls Win32 APIs to get the window handle (HWND) of the foreground window
* using [User32.GetForegroundWindow].
*
* Then calls [User32.GetWindowRect] to populate a [WinRect] struct with the
* window bounds.
* Then calls [User32.GetWindowRect] to get the outer bounding rectangle
* coordinates of the window and stores them in a [WinRect] struct.
*
* The rectangle coordinates are converted to a [Rectangle] and returned. This allows us to keep the implementation
* generic and constrained within the java std library.
* In the future we may want to add LinuxOSProxy or DarwinOSProxy.
* To support high DPI screens, the [User32.GetDpiForWindow] API is called to
* get the DPI scaling for the window. The rectangle coordinates are scaled by
* multiplying by the default DPI (96) and dividing by the actual DPI.
*
* @return The outer bounding rectangle of the foreground window in screen coordinates,
* or an empty [Rectangle] if it failed.
* The scaled rectangle coordinates are returned encapsulated in a [Rectangle]
* to provide a coordinate system agnostic result.
*
* @return The outer bounding rectangle of the foreground window scaled for the
* screen DPI, or null if it failed.
*/
override fun getForegroundWindowBounds(): Rectangle? {
override fun getScaledForegroundWindowBounds(): Rectangle? {
val user32 = User32.INSTANCE
val hWnd = user32.GetForegroundWindow()
val rect = getRectFromWindowHandle(user32, hWnd)
val defaultDPI = 96
return if (rect != null) {
// Get the DPI for the given window
val dpi = user32.GetDpiForWindow(hWnd)
//we need to make the calls to get the info to correct for the dpi scaling
val secondarySuccess = correctWinRectForDpi(user32, hWnd!!, rect.pointer)
require(dpi != 0)
if (secondarySuccess) {
//the correctWinRectForDpi function doesn't have access to the underlying stuct in order to read it, so
// we have to call this
rect.read()
Rectangle(rect.top, rect.left, (rect.right - rect.left), (rect.bottom - rect.top))
} else {
return null
}
// Default DPI is 96. Scale the coordinates based on the actual DPI. This handles cases where screen has
// high DPI/scaling.
// Calculate width by getting difference between right and left edges Then multiply by default DPI and
// divide by actual DPI to scale
val width = ((rect.right - rect.left) * defaultDPI) / dpi
// Same for height - get difference between bottom and top edges Multiply by default DPI and divide by
// actual to scale
val height = ((rect.bottom - rect.top) * defaultDPI) / dpi
Rectangle(rect.top, rect.left, width, height)
} else {
null
}
}
/**
* Gets the window rectangle coordinates for the given window handle.
*
* This calls the Win32 API [User32.GetWindowRect] function to populate a
* [WinRect] struct with the window coordinates.
*
* It first creates an instance of [WinRect] to hold the results.
* [User32.GetWindowRect] is called, passing the window handle [hWnd] and
* pointer to the [WinRect].
*
* If it succeeds, the [WinRect] values are read back out since they are
* populated in native memory. The [WinRect] is returned.
*
* If it fails, null is returned.
*
* @param user32 An instance of User32, used to call the Win32 API.
* @param hWnd The window handle to get the rect for.
* @return The WindowRect with populated coordinates, or null if failed.
*/
private fun getRectFromWindowHandle(user32: User32, hWnd: Pointer?): WinRect? {
//we have to provide the system a native struct in order to hold the results of our request
val rect = WinRect()
@ -278,18 +303,4 @@ interface WindowsOSProxy : MousePointerObserver, OSProxy {
}
}
private fun correctWinRectForDpi(user32: User32, hWnd: Pointer, lpRect: Pointer): Boolean {
val user32 = User32.INSTANCE
val dwStyle = user32.GetWindowLongA(hWnd, User32.GWL_STYLE)
val bMenu = true
val dwExStyle = user32.GetWindowLongA(hWnd, User32.GWL_EXSTYLE)
val dpi = user32.GetDpiForWindow(hWnd)
return user32.AdjustWindowRectExForDpi(lpRect, dwStyle, bMenu, dwExStyle, dpi)
}
}

2
src/main/kotlin/game_logic/runescape/RSAgent.kt
View File

@ -198,7 +198,7 @@ class RSAgent(override val automaton: Automaton = RobotAutomaton()) : RSOrchestr
*
* @param n The number of game ticks to sleep for.
*/
fun sleepForNTicks(n: Long) {
override fun sleepForNTicks(n: Long) {
val latencyPadding = LATENCY_PADDING_MS
val baseWaitTime = n * TICK_DURATION_MS
automaton.sleepWithVariance(latencyPadding + baseWaitTime, 150)

12
src/main/kotlin/game_logic/runescape/RSOrchestrator.kt
View File

@ -161,4 +161,16 @@ interface RSOrchestrator : Orchestrator {
* @return The Point representing the x,y screen coordinates of the bank location.
*/
fun getBankPoint(): Point
/**
* Pauses execution for the specified number of game ticks.
*
* This will sleep/wait for the given number of game ticks before continuing.
*
* Game ticks typically occur around once every 0.6 seconds. So this can be used
* to pause for a duration measured in game ticks rather than absolute time.
*
* @param n The number of game ticks to wait/sleep for.
*/
fun sleepForNTicks(n: Long)
}

11
src/main/kotlin/game_logic/runescape/RunescapeRoutines.kt
View File

@ -417,6 +417,17 @@ object RunescapeRoutines {
RSOrchestrator.doTravelTask(agent, params)
}
/**
* Crafts a specified volume of necromancy ink at the bank.
*
* This handles the entire workflow of crafting necromancy ink from scratch:
* - Withdrawing bank preset on F6
* - Crafting the ink using the bank crafting preset hotkey on Minus
*
* @param volume The number of necromancy ink to craft
* @param agent The orchestrator instance to use, defaults to global instance
* @param bankPoint The tile position of the bank booth to use
*/
fun createNecromancyInk(volume: Int, agent: RSOrchestrator = RSOrchestrator.getInstance(), bankPoint: Point = agent.getBankPoint()) {
val params = StandingTaskParams(
volume,

1
src/main/kotlin/params/StandingTaskParams.kt
View File

@ -11,7 +11,6 @@ import java.awt.Point
*
* @param totalVolume Total number of items to process.
* @param volumePerStep The volume of items to process per iteration.
* @param agent The Agent instance.
* @param bankPoint Location of the bank.
* @param bankPresetHotkey Bank preset hotkey to use.
* @param craftingDialogHotkey Hotkey to open crafting dialog.

11
src/main/kotlin/util/HelperFunctions.kt
View File

@ -56,8 +56,16 @@ object HelperFunctions {
*/
fun report(step: Int, of: Int, dur: Long) {
val remaining = (dur / step) * (of - step)
if (step == 0) {
print("\rGathering timing data...\t|\tNo current ETA...) ")
} else if (step < 8) { // The time estimation is terrible, so it converges on reality. it takes roughly 8-10 steps to get a decent picture
print("\rStep $step of $of (${prettyTimeString(dur)} complete\t|\t~${prettyTimeString(remaining * 2)} remaining [LOW CONFIDENCE]) ")
} else if (step == of) {
print("\rFinal step (${prettyTimeString(dur)} complete\t|\t~${prettyTimeString(dur / (of - 1))} remaining) ")
} else {
print("\rStep $step of $of (${prettyTimeString(dur)} complete\t|\t~${prettyTimeString(remaining)} remaining) ")
}
}
/**
* Converts a duration in milliseconds to a human-readable string.
@ -76,9 +84,6 @@ object HelperFunctions {
* @return A string representation of the duration, in the format XhYmZs
*/
fun prettyTimeString(durationMillis: Long): String {
if (durationMillis == 0L) {
return "No time data yet"
}
val millisPerSecond = 1000L
val millisPerMinute = 60L * millisPerSecond
val millisPerHour = 60L * millisPerMinute

2
src/test/kotlin/controllers/OSProxyTest.kt
View File

@ -7,7 +7,7 @@ class OSProxyTest {
@Test
fun test(){
val vc = ConcreteVisionController()
val rect = vc.getForegroundWindowBounds()
val rect = vc.getScaledForegroundWindowBounds()
println(rect)
}
}

2
src/test/kotlin/controllers/VisionControllerTest.kt
View File

@ -7,6 +7,6 @@ class VisionControllerTest {
@Test
fun testImageCapture(){
val vc = ConcreteVisionController()
vc.takeScreenshot()
vc.takeScreenshotOfForeground()
}
}