屏幕截图中的模板匹配

Question

我是 Python 新手，但想学一点，所以我决定创建一个与桌面输入进行模板匹配的程序。

有人能帮忙吗 ？如何编写与桌面流匹配的模板？

import time

import cv2
import mss
import numpy

template = cv2.imread('template.jpg', 0)
w, h = template.shape[::-1]

with mss.mss() as sct:
    # Part of the screen to capture
    monitor = {"top": 40, "left": 0, "width": 800, "height": 640}

    while "Screen capturing":
        last_time = time.time()

        # Get raw pixels from the screen, save it to a Numpy array
        img = numpy.array(sct.grab(monitor))

        # Display the picture
        # cv2.imshow("OpenCV/Numpy normal", img)

        # Display the picture in grayscale
        cv2.imshow('OpenCV/Numpy grayscale', cv2.cvtColor(img, cv2.COLOR_BGRA2GRAY))

        # Print fps
        print("fps: {}".format(1 / (time.time() - last_time)))

        # Search template in stream

        # Press "q" to quit
        if cv2.waitKey(25) & 0xFF == ord("q"):
            cv2.destroyAllWindows()
            break

Answer 1

我注意到的第一件事是您没有对模板图像应用任何边缘检测。边缘检测不是必需的，但对于查找模板图像的特征很有用。

假设我有以下图像：

为了精确检测上述模板图像，我应该应用边缘检测算法。

template = cv2.imread("three.png")
template = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)
template = cv2.Canny(template, 50, 200)

我还应该从桌面对流应用边缘检测。

img = sct.grab(mon)
gray = cv2.cvtColor(np.array(img), cv2.COLOR_BGR2GRAY)
edged = cv2.Canny(gray, 50, 200)

检查模板与捕获的图像是否匹配

result = cv2.matchTemplate(edged, template, cv2.TM_CCOEFF)
(_, maxVal, _, maxLoc) = cv2.minMaxLoc(result)

如果模板图像在桌面流中匹配，则获取坐标。

(_, maxLoc, r) = found
(startX, startY) = (int(maxLoc[0] * r), int(maxLoc[1] * r))
(endX, endY) = (int((maxLoc[0] + w) * r), int((maxLoc[1] + h) * r))

最后绘制用于显示位置的矩形：

cv2.rectangle(img, (startX, startY), (endX, endY), (180, 105, 255), 2)

结果：

从上面我们可以看到我们的模板 3 值与来自桌面的流相匹配。

代码：

import time
import cv2
import numpy as np
import imutils
from mss import mss

template = cv2.imread("three.png")
template = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)
template = cv2.Canny(template, 50, 200)
(h, w) = template.shape[:2]

start_time = time.time()
mon = {'top': 200, 'left': 200, 'width': 200, 'height': 200}
with mss() as sct:
    while True:
        last_time = time.time()
        img = sct.grab(mon)
        img = np.array(img)
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        edged = cv2.Canny(gray, 50, 200)
        
        found = None

        for scale in np.linspace(0.2, 1.0, 20)[::-1]:
            resized = imutils.resize(gray, width=int(gray.shape[1] * scale))
            r = gray.shape[1] / float(resized.shape[1])

            if resized.shape[0] < h or resized.shape[1] < w:
                break

            edged = cv2.Canny(resized, 50, 200)
            cv2.imwrite("canny_image.png", edged)
            result = cv2.matchTemplate(edged, template, cv2.TM_CCOEFF)
            (_, maxVal, _, maxLoc) = cv2.minMaxLoc(result)

            if found is None or maxVal > found[0]:
                found = (maxVal, maxLoc, r)

        (_, maxLoc, r) = found
        (startX, startY) = (int(maxLoc[0] * r), int(maxLoc[1] * r))
        (endX, endY) = (int((maxLoc[0] + w) * r), int((maxLoc[1] + h) * r))

        cv2.rectangle(img, (startX, startY), (endX, endY), (180, 105, 255), 2)

        print('The loop took: {0}'.format(time.time()-last_time))
        cv2.imshow('test', np.array(img))

        if cv2.waitKey(25) & 0xFF == ord('q'):
            cv2.destroyAllWindows()
            break