做過移動端ui自動化的小伙伴，就會發現很多控件的元素是一樣的或者是找不到的，為了解決這個痛點，於是通過圖片灰度處理返回坐標x，y找到控件的位置。再結合pytest+接口+UI斷言整體項目思路。
1.接下來我們主要說一下基於opencv圖片識別尋找控件坐標
2. 我們使用兩個圖，一個是移動端截圖，一個是控件的圖，
Java代碼如下
public static void main(String[] args) {
run_opencv("D:/Search.png", "D:/Setting.png",50,50);
}
public static HashMap<String, Integer> run_opencv(String picturePath,String PagePicturePath,int xPercent,int yPercent) {
HashMap<String, Integer> location = new HashMap<>();
try {
//x,y = get_center_location('D:/Battery.png', 'D:/Setting.png',0,0)
String cmds = String.format("python D:\\Project\\Program\\PythonWorkspace\\myProject\\python_project\\apptest\\myopencv\\other_case\\get_location_by_opencv.py %s %s %d %d", picturePath,PagePicturePath,xPercent,yPercent);
System.out.println("Executing python script for picture location.");
Process pcs = Runtime.getRuntime().exec(cmds);
pcs.waitFor();
Thread.sleep(1000);
// 定義Python腳本的返回值
String result = null;
// 獲取CMD的返回流
BufferedInputStream in = new BufferedInputStream(pcs.getInputStream());// 字符流轉換字節流
BufferedReader br = new BufferedReader(new InputStreamReader(in));// 這裡也可以輸出文本日志
String lineStr = null;
while ((lineStr = br.readLine()) != null) {
result = lineStr;//Python 代碼中print的數據就是返回值
//xLocation: 147
//yLocation: 212
if(lineStr.contains("xLocation")) {
int x = Integer.parseInt(lineStr.split(":")[1].trim());
location.put("x", x);
}
if(lineStr.contains("yLocation")) {
int x = Integer.parseInt(lineStr.split(":")[1].trim());
location.put("y", x);
}
}
// 關閉輸入流
br.close();
in.close();
System.out.println(location.toString());
} catch (Exception e) {
e.printStackTrace();
}
return location;
}
Python代碼：
# -*- encoding=utf-8 -*-
__author__ = 'Jeff.xie'
import cv2
import os
import sys
import time
#獲取移動端圖片
def screencap():
cmd = "adb root"
cmd1 = "adb shell /system/bin/screencap -p /sdcard/da.png"
cmd2 = "adb pull /sdcard/da.png "
os.system(cmd)
time.sleep(1)
os.system(cmd1)
time.sleep(2)
os.system(cmd2)
def _tran_canny(image):
"""消除噪聲"""
image = cv2.GaussianBlur(image, (3, 3), 0)
return cv2.Canny(image, 50, 150)
def get_center_location(img_slider_path,image_background_path,x_percent,y_percent):
"""get_center_location"""
# print("img_slider_path: "+img_slider_path)
# print("image_background_path: "+image_background_path)
# print("x_percent: "+str(x_percent))
# print("y_percent: "+str(y_percent))
# java傳遞過來的參數都是str類型，所以需要強轉成int類型
xper = int(x_percent)
yper = int(y_percent)
# # 參數0是灰度模式
image = cv2.imread(img_slider_path, 0)
template = cv2.imread(image_background_path, 0)
# 尋找最佳匹配
res = cv2.matchTemplate(_tran_canny(image), _tran_canny(template), cv2.TM_CCOEFF_NORMED)
# 最小值，最大值，並得到最小值, 最大值的索引
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
#獲得背景圖像高和寬
src_img = cv2.imread(image_background_path,cv2.IMREAD_GRAYSCALE)
h,w = src_img.shape
# print("src_img_h:",h)
# print("src_img_w:",w)
#獲得需要尋找圖像高和寬
des_img = cv2.imread(img_slider_path,cv2.IMREAD_GRAYSCALE)
des_img_h,des_img_w = des_img.shape
# print("des_img_h:",des_img_h)
# print("des_img_w:",des_img_w)
trows,tcols = image.shape[:2] #獲得圖片的寬度,兩種方式都可以
# print(trows)
# print(tcols)
top_left = max_loc[0] # 橫坐標
# 展示圈出來的區域
x, y = max_loc
# max_loc這個是最大值，所以獲取的是x,y位置坐標，小圖片右下角的位置，左上角的要用min_loc
# print("x:",x)
# print("y:",y)
xLocation = x + int(des_img_w*xper/100)
yLocation = y + int(des_img_h*yper/100)
print("xLocation: "+str(xLocation))
print("yLocation: "+str(yLocation))
# print(max_loc)
# print(min_loc)
# print(min_val)
# print(max_val)
return xLocation,yLocation
# w, h = image.shape[::-1] # 寬高
# cv2.rectangle(template, (x, y), (x + w, y + h), (7, 249, 151), 2)
# return top_left
if __name__ == '__main__':
# x,y = get_center_location('D:/Battery.png', 'D:/Setting.png',40,39)
img_slider_path = sys.argv[1] image_background_path = sys.argv[2] x_percent = sys.argv[3] y_percent = sys.argv[4]
get_center_location(img_slider_path, image_background_path,x_percent,y_percent)
# 0%
# getx: 29
# gety: 1390
# 50%
# getx: 49
# gety: 1415
# 100%
# getx: 69
# gety: 1441
