I've done mobile terminal ui Automation partners , You will find that the elements of many controls are the same or cannot be found , To solve this pain point , Then the coordinates are returned through the image grayscale processing x,y Find the location of the control . combining pytest+ Interface +UI Assert the overall project thinking .
1. Next, we will mainly talk about based on opencv Image recognition find control coordinates
2. We use two graphs , One is a screenshot of the mobile terminal , One is the diagram of the control ,
Java The code is as follows
public static void main(String[] args) {
run_opencv("D:/Search.png", "D:/Setting.png",50,50);
}
public static HashMap<String, Integer> run_opencv(String picturePath,String PagePicturePath,int xPercent,int yPercent) {
HashMap<String, Integer> location = new HashMap<>();
try {
//x,y = get_center_location('D:/Battery.png', 'D:/Setting.png',0,0)
String cmds = String.format("python D:\\Project\\Program\\PythonWorkspace\\myProject\\python_project\\apptest\\myopencv\\other_case\\get_location_by_opencv.py %s %s %d %d", picturePath,PagePicturePath,xPercent,yPercent);
System.out.println("Executing python script for picture location.");
Process pcs = Runtime.getRuntime().exec(cmds);
pcs.waitFor();
Thread.sleep(1000);
// Definition Python The return value of the script
String result = null;
// obtain CMD Return flow of
BufferedInputStream in = new BufferedInputStream(pcs.getInputStream());// Character stream to byte stream
BufferedReader br = new BufferedReader(new InputStreamReader(in));// You can also output text logs here
String lineStr = null;
while ((lineStr = br.readLine()) != null) {
result = lineStr;//Python In the code print The data of is the return value
//xLocation: 147
//yLocation: 212
if(lineStr.contains("xLocation")) {
int x = Integer.parseInt(lineStr.split(":")[1].trim());
location.put("x", x);
}
if(lineStr.contains("yLocation")) {
int x = Integer.parseInt(lineStr.split(":")[1].trim());
location.put("y", x);
}
}
// Close input stream
br.close();
in.close();
System.out.println(location.toString());
} catch (Exception e) {
e.printStackTrace();
}
return location;
}
Python Code ：
# -*- encoding=utf-8 -*-
__author__ = 'Jeff.xie'
import cv2
import os
import sys
import time
# Get the picture of the mobile terminal
def screencap():
cmd = "adb root"
cmd1 = "adb shell /system/bin/screencap -p /sdcard/da.png"
cmd2 = "adb pull /sdcard/da.png "
os.system(cmd)
time.sleep(1)
os.system(cmd1)
time.sleep(2)
os.system(cmd2)
def _tran_canny(image):
""" Eliminate noise """
image = cv2.GaussianBlur(image, (3, 3), 0)
return cv2.Canny(image, 50, 150)
def get_center_location(img_slider_path,image_background_path,x_percent,y_percent):
"""get_center_location"""
# print("img_slider_path: "+img_slider_path)
# print("image_background_path: "+image_background_path)
# print("x_percent: "+str(x_percent))
# print("y_percent: "+str(y_percent))
# java The parameters passed in are str type , So we need to turn it into int type
xper = int(x_percent)
yper = int(y_percent)
# # Parameters 0 It's grayscale mode
image = cv2.imread(img_slider_path, 0)
template = cv2.imread(image_background_path, 0)
# Looking for the best match
res = cv2.matchTemplate(_tran_canny(image), _tran_canny(template), cv2.TM_CCOEFF_NORMED)
# minimum value , Maximum , And get the minimum , Index of maximum value
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
# Get the height and width of the background image
src_img = cv2.imread(image_background_path,cv2.IMREAD_GRAYSCALE)
h,w = src_img.shape
# print("src_img_h:",h)
# print("src_img_w:",w)
# Get the image height and width you need to find
des_img = cv2.imread(img_slider_path,cv2.IMREAD_GRAYSCALE)
des_img_h,des_img_w = des_img.shape
# print("des_img_h:",des_img_h)
# print("des_img_w:",des_img_w)
trows,tcols = image.shape[:2] # Get the width of the picture , Either way
# print(trows)
# print(tcols)
top_left = max_loc[0] # Abscissa
# Show the circled area
x, y = max_loc
# max_loc This is the maximum , So what we get is x,y coordinates , The position of the lower right corner of the small picture , The one in the upper left corner should be min_loc
# print("x:",x)
# print("y:",y)
xLocation = x + int(des_img_w*xper/100)
yLocation = y + int(des_img_h*yper/100)
print("xLocation: "+str(xLocation))
print("yLocation: "+str(yLocation))
# print(max_loc)
# print(min_loc)
# print(min_val)
# print(max_val)
return xLocation,yLocation
# w, h = image.shape[::-1] # Wide and high
# cv2.rectangle(template, (x, y), (x + w, y + h), (7, 249, 151), 2)
# return top_left
if __name__ == '__main__':
# x,y = get_center_location('D:/Battery.png', 'D:/Setting.png',40,39)
img_slider_path = sys.argv[1] image_background_path = sys.argv[2] x_percent = sys.argv[3] y_percent = sys.argv[4]
get_center_location(img_slider_path, image_background_path,x_percent,y_percent)
# 0%
# getx: 29
# gety: 1390
# 50%
# getx: 49
# gety: 1415
# 100%
# getx: 69
# gety: 1441
