tookeyman/RuneFactory
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

i'm done for tonight

Browse Source
This commit is contained in:
dtookey 2023-08-11 20:47:59 -04:00
parent c7596a7b85
commit bb23ab3ec2
4 changed files with 170 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/kotlin/controllers/VisionController.kt
View File

@ -2,6 +2,7 @@ package controllers
import controllers.windows.WindowsOSProxy
import java.awt.image.BufferedImage
import java.awt.image.MultiResolutionImage
/**
* Interface for a vision controller that can take screenshots.
@ -22,7 +23,7 @@ interface VisionController : Automaton, OSProxy {
* @return A BufferedImage containing the screenshot of the foreground
* window.
*/
fun takeScreenshotOfForeground(): BufferedImage
fun takeScreenshotOfForeground(): MultiResolutionImage
}
/**
@ -44,9 +45,9 @@ class ConcreteVisionController : VisionController, WindowsOSProxy, RobotAutomato
* @return A [BufferedImage] containing a screenshot of the foreground window
* scaled to account for the screen DPI.
*/
override fun takeScreenshotOfForeground(): BufferedImage {
override fun takeScreenshotOfForeground(): MultiResolutionImage {
val rect = getScaledForegroundWindowBounds()
return robot.createScreenCapture(rect)
return robot.createMultiResolutionScreenCapture(rect)
}
}

35
src/main/kotlin/controllers/windows/User32.kt
View File

@ -1,6 +1,7 @@
package controllers.windows
import com.sun.jna.Native
import com.sun.jna.NativeLong
import com.sun.jna.Pointer
import com.sun.jna.win32.StdCallLibrary
import controllers.windows.WindowsOSProxy.WinRect
@ -129,6 +130,40 @@ interface User32 : StdCallLibrary {
*/
fun SetForegroundWindow(hWnd: Pointer?): Boolean
/**
* Converts a logical point to a physical point for a window based on the DPI awareness per monitor.
*
* This calls the Win32 API LogicalToPhysicalPointForPerMonitorDPI function to convert a logical point
* (x, y coordinates) relative to a window into a physical point based on the DPI awareness context of
* the given window. This accounts for different DPI scaling settings on each monitor.
*
* The logical point is specified in the [lpPoint] struct as:
*
* ```
* struct POINT {
* LONG x;
* LONG y;
* }
* ```
*
* On return, [lpPoint] will contain the converted physical coordinates.
*
* @param hWnd The window handle (HWND) to use the DPI context of for conversion.
* @param lpPoint Pointer to a POINT struct containing the logical coordinates.
* This will be updated with the converted physical coordinates.
*/
fun LogicalToPhysicalPointForPerMonitorDPI(hWnd: Pointer?, lpPoint: Pointer?): Boolean
fun PhysicalToLogicalPointForPerMonitorDPI(hWnd: Pointer?, lpPoint: Pointer?): Boolean
fun DwmGetWindowAttribute(
hwnd: Pointer?,
dwAttribute: Int,
pvAttribute: Pointer?,
cbAttribute: Int
): NativeLong
/**
* Retrieves the coordinates of a window on the screen.
*

121
src/main/kotlin/controllers/windows/WindowsOSProxy.kt
View File

@ -74,12 +74,28 @@ interface WindowsOSProxy : MousePointerObserver, OSProxy {
}
/**
* Mirror of the Windows POINT struct used for window/physical coordinates
*/
@FieldOrder("x", "y")
class WinPoint() : Structure() {
constructor(x: Int, y: Int) : this() {
this.x = x
this.y = y
}
@JvmField
var x: Int = 0
@JvmField
var y: Int = 0
}
/**
* Mirror of the Windows RECT struct used for window coordinates.
*/
@FieldOrder("left", "top", "right", "bottom")
public class WinRect : Structure() {
class WinRect : Structure() {
/** The x coordinate of the left edge */
@JvmField
@ -98,6 +114,64 @@ interface WindowsOSProxy : MousePointerObserver, OSProxy {
var bottom: Int = 0
}
enum class DwmWindowAttribute {
DWMWA_NCRENDERING_ENABLED,
DWMWA_NCRENDERING_POLICY,
DWMWA_TRANSITIONS_FORCEDISABLED,
DWMWA_ALLOW_NCPAINT,
DWMWA_CAPTION_BUTTON_BOUNDS,
DWMWA_NONCLIENT_RTL_LAYOUT,
DWMWA_FORCE_ICONIC_REPRESENTATION,
DWMWA_FLIP3D_POLICY,
DWMWA_EXTENDED_FRAME_BOUNDS,
DWMWA_HAS_ICONIC_BITMAP,
DWMWA_DISALLOW_PEEK,
DWMWA_EXCLUDED_FROM_PEEK,
DWMWA_CLOAK,
DWMWA_CLOAKED,
DWMWA_FREEZE_REPRESENTATION,
DWMWA_PASSIVE_UPDATE_MODE,
DWMWA_USE_HOSTBACKDROPBRUSH,
DWMWA_USE_IMMERSIVE_DARK_MODE,
DWMWA_WINDOW_CORNER_PREFERENCE,
DWMWA_BORDER_COLOR,
DWMWA_CAPTION_COLOR,
DWMWA_TEXT_COLOR,
DWMWA_VISIBLE_FRAME_BORDER_THICKNESS,
DWMWA_SYSTEMBACKDROP_TYPE,
DWMWA_LAST;
companion object {
val DWMWA_NCRENDERING_ENABLED = 0
val DWMWA_NCRENDERING_POLICY = 1
val DWMWA_TRANSITIONS_FORCEDISABLED = 2
val DWMWA_ALLOW_NCPAINT = 3
val DWMWA_CAPTION_BUTTON_BOUNDS = 4
val DWMWA_NONCLIENT_RTL_LAYOUT = 5
val DWMWA_FORCE_ICONIC_REPRESENTATION = 6
val DWMWA_FLIP3D_POLICY = 7
val DWMWA_EXTENDED_FRAME_BOUNDS = 8
val DWMWA_HAS_ICONIC_BITMAP = 9
val DWMWA_DISALLOW_PEEK = 10
val DWMWA_EXCLUDED_FROM_PEEK = 11
val DWMWA_CLOAK = 12
val DWMWA_CLOAKED = 13
val DWMWA_FREEZE_REPRESENTATION = 14
val DWMWA_PASSIVE_UPDATE_MODE = 15
val DWMWA_USE_HOSTBACKDROPBRUSH = 16
val DWMWA_USE_IMMERSIVE_DARK_MODE = 17
val DWMWA_WINDOW_CORNER_PREFERENCE = 18
val DWMWA_BORDER_COLOR = 19
val DWMWA_CAPTION_COLOR = 20
val DWMWA_TEXT_COLOR = 21
val DWMWA_VISIBLE_FRAME_BORDER_THICKNESS = 22
val DWMWA_SYSTEMBACKDROP_TYPE = 23
val DWMWA_LAST = 24
}
}
/**
* Gets the title of the active/foreground window.
*
@ -244,6 +318,25 @@ interface WindowsOSProxy : MousePointerObserver, OSProxy {
override fun getScaledForegroundWindowBounds(): Rectangle? {
val user32 = User32.INSTANCE
val hWnd = user32.GetForegroundWindow()
val rect = WinRect()
val success = user32.DwmGetWindowAttribute(
hWnd,
8,
rect.pointer,
rect.size()
)
return if (true){
rect.read()
Rectangle(rect.top, rect.left, (rect.right - rect.left), rect.bottom - rect.top)
}else{
null
}
}
fun barelyFunctionalWindowQuery(): Rectangle? {
val user32 = User32.INSTANCE
val hWnd = user32.GetForegroundWindow()
val rect = getRectFromWindowHandle(user32, hWnd)
val defaultDPI = 96
@ -297,10 +390,32 @@ interface WindowsOSProxy : MousePointerObserver, OSProxy {
return if (success) {
//the values are stuck down in memory, so we have to read these back out in order to proceed
rect.read()
rect
// Convert top-left point to physical coordinates
val topLeft = WinPoint(rect.left, rect.top) //rect.top, rect.left
val tlSuccess = user32.PhysicalToLogicalPointForPerMonitorDPI(hWnd, topLeft.pointer)
// Convert bottom-right point to physical coordinates
val bottomRight = WinPoint(rect.right, rect.bottom)
val brSuccess = user32.PhysicalToLogicalPointForPerMonitorDPI(hWnd, bottomRight.pointer)
return if (tlSuccess && brSuccess) {
// Create new rect with physical coordinates
val returnValue = WinRect()
returnValue.top = topLeft.y
returnValue.left = topLeft.x
returnValue.bottom = bottomRight.y
returnValue.right = bottomRight.x
returnValue
} else {
// Failed to convert logical to physical points
null
}
} else {
// Failed to get window rect
null
}
}
}

13
src/test/kotlin/controllers/VisionControllerTest.kt
View File

@ -1,12 +1,23 @@
package controllers
import java.awt.image.BufferedImage
import java.nio.file.Paths
import javax.imageio.ImageIO
import kotlin.test.Test
import kotlin.test.assertNotNull
class VisionControllerTest {
@Test
fun testImageCapture() {
val vc = ConcreteVisionController()
vc.takeScreenshotOfForeground()
val bi = vc.takeScreenshotOfForeground()
assertNotNull(bi)
for (i in 0..<bi.resolutionVariants.size) {
val path = Paths.get("image$i.png")
val image = bi.resolutionVariants[i] as BufferedImage
ImageIO.write(image , "png", path.toFile())
}
}
}