[request] Windows: act on a non-forground app-window (not or not fully visible)

to find an image in another image:

setBundlePath("/..../somewhere/");

base = "base.png"
test = "test.png";
finder = Finder(base);
finder.find(test);

if finder.hasNext():
    match = finder.next()
    print "found:", match
}

See also in the docs "Finder"

The match's (x.y) is relative to the base images's top left corner (0,0).

Currently as you might know, SikuliX can only handle, what is visible on the screen.
Hence a minimised app-window is not accessible.

In SikuliX we already use some win32 API features via JNA.
So if there are any features available, we can further discuss it.

So any further infos are welcome.

Thanks @Raiman I will send you a function later so that you can add it in the App class

/*
 Creator:Jobeth Pogi Billien
 Date:April 1 2022 GMT+8
 Website: elitegamingbot.com
*/

#include <iostream>
#include<Windows.h>
int main()
{
    /*
    * Forward declaration of functions
     */
    void SendMessageJB(HWND window, WORD key, char letter);
    int CaptureAnImage(HWND hWnd,LPCWSTR);

    //Create a window handle for the target application
    HWND hwnd=FindWindow(NULL, L"Your Application Window");

    /*
    *send W key to Target Application that is in the background
    *may not work to some application -- need to test your target app
    */
    SendMessageJB(hwnd, 'W', 'W');

    //a function to unminimize the target window
    //ShowWindow(hwnd, SW_RESTORE);

    /*
    *need the software not minimize but in the background
    * Arguments : window handle,name of image(not path)
    * Edit the save path in the CaptureAnImage function and maybe the parameter.
    */
    CaptureAnImage(hwnd,L"jb.png");
    system("pause");
    return 0;
}
void SendMessageJB(HWND window, WORD key, char letter)
{
    SendMessage(window, WM_KEYDOWN, key, 0);
    if (letter != 0)
        SendMessage(window, WM_CHAR, letter, 1);
    SendMessage(window, WM_KEYUP, key, 1);
    std::cout << "Last Error:" << GetLastError();
}
int CaptureAnImage(HWND hWnd,LPCWSTR fileName)
{
    HDC hdcScreen;
    HDC hdcWindow;
    HDC hdcMemDC = NULL;
    HBITMAP hbmScreen = NULL;
    BITMAP bmpScreen;
    DWORD dwBytesWritten = 0;
    DWORD dwSizeofDIB = 0;
    HANDLE hFile = NULL;
    char* lpbitmap = NULL;
    HANDLE hDIB = NULL;
    DWORD dwBmpSize = 0;
    int cx = 0;
    int cy = 0;
    // Retrieve the handle to a display device context for the client
    // area of the window.

    hdcWindow = GetDC(hWnd);
    hdcScreen = hdcWindow;
    // Create a compatible DC, which is used in a BitBlt from the window DC.
    hdcMemDC = CreateCompatibleDC(hdcWindow);

    if (!hdcMemDC)
    {
        MessageBox(hWnd, L"CreateCompatibleDC has failed", L"Failed", MB_OK);
        goto done;
    }

    // Get the client area for size calculation.
    RECT rcClient;
    GetClientRect(hWnd, &rcClient);
    cx = rcClient.right - rcClient.left;
    cy = rcClient.bottom - rcClient.top;
    // This is the best stretch mode.
    SetStretchBltMode(hdcWindow, HALFTONE);

    // The source DC is the entire screen, and the destination DC is the current window (HWND).
    if (!StretchBlt(hdcWindow,
        0, 0,
        rcClient.right, rcClient.bottom,
        hdcScreen,
        0, 0,
        cx,
        cy,
        SRCCOPY))
    {
        MessageBox(hWnd, L"StretchBlt has failed", L"Failed", MB_OK);
        goto done;
    }

    // Create a compatible bitmap from the Window DC.
    hbmScreen = CreateCompatibleBitmap(hdcWindow, rcClient.right - rcClient.left, rcClient.bottom - rcClient.top);

    if (!hbmScreen)
    {
        MessageBox(hWnd, L"CreateCompatibleBitmap Failed", L"Failed", MB_OK);
        goto done;
    }

    // Select the compatible bitmap into the compatible memory DC.
    SelectObject(hdcMemDC, hbmScreen);

    // Bit block transfer into our compatible memory DC.
    if (!BitBlt(hdcMemDC,
        0, 0,
        rcClient.right - rcClient.left, rcClient.bottom - rcClient.top,
        hdcWindow,
        0, 0,
        SRCCOPY))
    {
        MessageBox(hWnd, L"BitBlt has failed", L"Failed", MB_OK);
        goto done;
    }

    // Get the BITMAP from the HBITMAP.
    GetObject(hbmScreen, sizeof(BITMAP), &bmpScreen);

    BITMAPFILEHEADER bmfHeader;
    BITMAPINFOHEADER bi;

    bi.biSize = sizeof(BITMAPINFOHEADER);
    bi.biWidth = bmpScreen.bmWidth;
    bi.biHeight = bmpScreen.bmHeight;
    bi.biPlanes = 1;
    bi.biBitCount = 32;
    bi.biCompression = BI_RGB;
    bi.biSizeImage = 0;
    bi.biXPelsPerMeter = 0;
    bi.biYPelsPerMeter = 0;
    bi.biClrUsed = 0;
    bi.biClrImportant = 0;

    dwBmpSize = ((bmpScreen.bmWidth * bi.biBitCount + 31) / 32) * 4 * bmpScreen.bmHeight;

    // Starting with 32-bit Windows, GlobalAlloc and LocalAlloc are implemented as wrapper functions that
    // call HeapAlloc using a handle to the process's default heap. Therefore, GlobalAlloc and LocalAlloc
    // have greater overhead than HeapAlloc.
    hDIB = GlobalAlloc(GHND, dwBmpSize);
    lpbitmap = (char*)GlobalLock(hDIB);

    // Gets the "bits" from the bitmap, and copies them into a buffer
    // that's pointed to by lpbitmap.
    GetDIBits(hdcWindow, hbmScreen, 0,
        (UINT)bmpScreen.bmHeight,
        lpbitmap,
        (BITMAPINFO*)&bi, DIB_RGB_COLORS);

    // A file is created, this is where we will save the screen capture.
    hFile = CreateFile(fileName,
        GENERIC_WRITE,
        0,
        NULL,
        CREATE_ALWAYS,
        FILE_ATTRIBUTE_NORMAL, NULL);

    // Add the size of the headers to the size of the bitmap to get the total file size.
    dwSizeofDIB = dwBmpSize + sizeof(BITMAPFILEHEADER) + sizeof(BITMAPINFOHEADER);

    // Offset to where the actual bitmap bits start.
    bmfHeader.bfOffBits = (DWORD)sizeof(BITMAPFILEHEADER) + (DWORD)sizeof(BITMAPINFOHEADER);

    // Size of the file.
    bmfHeader.bfSize = dwSizeofDIB;

    // bfType must always be BM for Bitmaps.
    bmfHeader.bfType = 0x4D42; // BM.

    WriteFile(hFile, (LPSTR)&bmfHeader, sizeof(BITMAPFILEHEADER), &dwBytesWritten, NULL);
    WriteFile(hFile, (LPSTR)&bi, sizeof(BITMAPINFOHEADER), &dwBytesWritten, NULL);
    WriteFile(hFile, (LPSTR)lpbitmap, dwBmpSize, &dwBytesWritten, NULL);

    // Unlock and Free the DIB from the heap.
    GlobalUnlock(hDIB);
    GlobalFree(hDIB);

    // Close the handle for the file that was created.
    CloseHandle(hFile);

    // Clean up.
done:
    DeleteObject(hbmScreen);
    DeleteObject(hdcMemDC);
    ReleaseDC(NULL, hdcScreen);
    ReleaseDC(hWnd, hdcWindow);

    return 0;
}

I am very impressed.

I suppose, that there is a minimized app window, whose content is saved to a png-file, without making this window visible on the screen.

I have some problems:

- I cannot use the c++-code as is, since I do not have the environment to make it runnable on my Windows 11. So it would help me, if I could get an exe, that accepts the 2 parameters (window-title, file-name).

- I do not understand, what "SendMessageJB(hwnd, 'W', 'W');" does with the minimized window.

Testing with JNA I always get Rect(x = -32000,y = -32000, w = 168, h = 28) for any minimized window, when using user32.GetWindowRect(hWnd, rect).

You can mail me directly at sikulix---at---outlook---dot---com

You should not minimize it, it does not work when minimize, only put it in the background.The user can do other things that way also.It is because of the state of the window. I cannot find a capture image function that is in the SW_MINIMIZE state,some says it is not possible. Also you should edit the title of the program in the FindWindow function so that it points to your target .

The SendMessage function on win32 is for sending input to the target window(mouse or keyboard input) even if it is in the background.

@Raiman on the file that I sent you I remove this block of code

if (!BitBlt(hdcMemDC,
        0, 0,
        rcClient.right - rcClient.left, rcClient.bottom - rcClient.top,
        hdcWindow,
        0, 0,
        SRCCOPY))
    {
        MessageBox(hWnd, L"BitBlt has failed", L"Failed", MB_OK);
        goto done;
    }

I replace it with PrintWindow(hWnd, hdcMemDC, 3);

The only problem now is how to send keyboard and mouse input to all background application. Sometimes SendMessage is failing on some application. If you know any other function besides PostMessage and SendMessage I appreciate it.

Great thanks for now.
Will look into it tomorrow (for me it is late in the evening now).

ok tested your exe and it works.
To use it programmatically, it should accept commandline parameters.

- opened a Notepad++ window and put it behind other windows (background)
- used the exe, that produced an image of the app window (test.png)

then I used the following script, to test that test.png can be found, when the app window is visible:

app = App("Notepad++")
app.focus()
regWin = App.focusedWindow()
img = "test.png"
imgExe = Image.create(img)
print "fromExe:", imgExe
print "notepad:", regWin

print regWin.find(imgExe)

--- output:
fromExe: I[test.png(1899x625)]
notepad: R[6,4 1914x684 (neu 1 - Notepad++)]@S(0)
M[14,55 1899x625]IN(0) %99,85 C(963,367) [147/147 msec]

So your approach works, to get the window content of not fully visible windows.

I still do not understand the intention of your SendMessage usage in the beginning. What is it good for?

--- The only problem now is how to send keyboard and mouse input to a background application.
I do not have any experience with the Windows API, so if the net does not know a solution, I cannot help.

Send message is use for sending keytroke and mouse input to the target window without actually using the mouse and keyboard. The user can still use the mouse and keyboard.
//send left click at target 250,250 without using the mouse
SendMouseJB(hwnd, 'L', 250, 250);
 //send right click at target 350,350 without using the mouse
SendMouseJB(hwnd, 'R', 350, 350);
//open inventory of the game
SendKeyboardJB(hwnd, 'I', 'I');
//press F1(skill slot of the game). We 0 the 3rd parameter because it is not a char
SendKeyboardJb(hwnd,VK_F1,0);
void SendMouseJB(HWND hwnd, const char button, int x, int y)
{
    //left mouse at x,y
    if (button == 'L')
    {
        SendMessage(hwnd, WM_LBUTTONDOWN, MK_LBUTTON, MAKELPARAM(x, y));
        SendMessage(hwnd, WM_LBUTTONUP, MK_LBUTTON, MAKELPARAM(x, y));
    }
    //right mouse at x,y
    else if (button == 'R')
    {
        SendMessage(hwnd, WM_RBUTTONDOWN, MK_RBUTTON, MAKELPARAM(x, y));
        SendMessage(hwnd, WM_RBUTTONUP, MK_RBUTTON, MAKELPARAM(x, y));
    }
}
void SendKeyboardJB(HWND window, WORD key, char letter)
{
    SendMessage(window, WM_KEYDOWN, key, 0);
    if (letter != 0)
        SendMessage(window, WM_CHAR, letter, 1);
    SendMessage(window, WM_KEYUP, key, 1);
}

Note:
If you are using notepad to test the window to send keyboard input, the window is "EDIT" because on Win32 api "EDIT" is actually a textbox.
HWND hwnd=FindWindow(NULL,L"Untitled - Notepad");
HWND hwndChild = FindWindowEx(hwnd, NULL, L"EDIT", NULL);
//type I to the textbox of notepad
SendKeyboardJB(hwndChild, 'I', 'I');

ok, thanks for the additional information.

From now on I am able to compile and test such snippets myself: https://github.com/Embarcadero/Dev-Cpp
The next days I will play a bit with your input.

I will come back with any insights.