外部应用程序调用和ocr的使用

最新推荐文章于 2023-10-14 11:09:40 发布

萌兽

最新推荐文章于 2023-10-14 11:09:40 发布

阅读量1.2k

点赞数 1

WindowsAPI在每一台Windows系统上开放标准API供开发人员调用.功能齐全.在这里只介绍三个部分.

1.利用API控制鼠标键盘.

2.对外部程序的读写和监听

3.对外部程序窗口的改动.

外部程序的意思就是.假设我的程序是360.辣么我能控制腾讯QQ客户端啥的.

API的宏.以下任何常量和函数都可以在

const int MOUSEEVENTF_MOVE = 0x0001; // 移动鼠标
const int MOUSEEVENTF_LEFTDOWN = 0x0002; //模仿鼠标左键按下
const int MOUSEEVENTF_LEFTUP = 0x0004; //模仿鼠标左键抬起
const int MOUSEEVENTF_RIGHTDOWN = 0x0008; //模仿鼠标右键按下
const int MOUSEEVENTF_RIGHTUP = 0x0010; //模仿鼠标右键抬起
const int MOUSEEVENTF_MIDDLEDOWN = 0x0020;// 模仿鼠标中键按下
const int MOUSEEVENTF_MIDDLEUP = 0x0040;// 模仿鼠标中键抬起
const int MOUSEEVENTF_ABSOLUTE = 0x8000; //标示是否采取绝对坐标
private const int WM_SETTEXT = 0x000C;
const int BM_CLICK = 0xF5;//鼠标点击事件
const int WM_GETTEXT = 0xd;//获取文本
const int WM_CLOSE = 0x0010;//关闭窗口

调用时程序会在Bin下寻找同名DLL.如果没有会在C:\Windows\System32中寻找同名DLL.

[return: MarshalAs(UnmanagedType.Bool)]
[DllImport("user32.dll", CharSet = CharSet.Auto, ExactSpelling = true)]
public static extern bool BlockInput([In, MarshalAs(UnmanagedType.Bool)] bool fBlockIt);

//BlockInput(true)锁定鼠标键盘.BlockInput(false)激活鼠标键盘.锁定时需要Thread.Sleep(500)才能生效

//如果在锁定鼠标键盘后死机..可以用CTRL + ALT +DELETE 激活鼠标键盘.听说用IO读出任务管理器可以使CTRL + ALT +DELETE 无效

[DllImport("user32")]
public extern static void SetCursorPos(int x, int y);

//移动鼠标到指定坐标

[DllImport("user32")]
public static extern void mouse_event(int dwFlags, int dx, int dy, int dwData, IntPtr dwExtraInfo);

//鼠标的点击事件

SetCursorPos(X, Y);//移动鼠标
mouse_event((int)(MOUSEEVENTF_LEFTDOWN | MOUSEEVENTF_ABSOLUTE), 0, 0, 0, IntPtr.Zero);//摁下
SetCursorPos(X, Y);//移动鼠标
mouse_event((int)(MOUSEEVENTF_LEFTUP | MOUSEEVENTF_ABSOLUTE), 0, 0, 0, IntPtr.Zero);//放开

//移动鼠标到指定位置然后拖拽到指定位置

SetCursorPos(X, Y);//移动鼠标

mouse_event((int)(MOUSEEVENTF_LEFTDOWN | MOUSEEVENTF_LEFTUP), 0, 0, 0, IntPtr.Zero);//摁下

//移动鼠标到指定位置左键双击

c#提供封装的对象可以控制键盘

SendKeys.Send("1111高");//发送字符串
SendKeys.SendWait("{^c }");//发送键盘按键.组合键

API中有keybd_event函数.Win IO等也可以控制键盘.但是本人没有找到使用组合键的方法..

对剪切板的操作

IDataObject iData = Clipboard.GetDataObject();

var a = (String)iData.GetData(DataFormats.Text);

//读

Clipboard.SetDataObject("1高G");

//写

var g = Graphics.GetImage();

//读取剪切板里的图片

对外部程序的读写：

[DllImport("user32.dll")]
public static extern int SendMessage(IntPtr Hwnd, int Msg, int wpala, string lpala);

private const int WM_SETTEXT = 0x000C;//定义写的宏

private const int WM_GETTEXT = 0xd;//定义读的宏

public static int WM_CLICK = 0x00F5;//定义点击的宏

public const int BM_CLICK = 0xF5;//鼠标点击事件

//可以对外部程序的控件做读写.按钮点击.窗口关闭等

SendMessage(new IntPtr(“句柄”), WM_SETTEXT, 0,“数据”);

//写

StringBuilder code = new StringBuilder(1024);

SendMessage(new IntPtr(“句柄”), WM_GETTEXT, 1024, code);

//读

SendMessage(“句柄”,WM_CLOSE,0,null);

//关闭窗口

SendMessage(new IntPtr(“句柄”), BM_CLICK, 0, 0);

//按钮的点击

在Windows系统下。每生成一个控件或者窗口都会出现一个句柄.是本对象的唯一标识符.可以通过坐标抓取.也可以用SPY++和INSPECT捕获句柄.稍后讲

下面是监听外部程序按钮点击的源码.是我哭着闹着求着微软的大牛给我写的一个案例.不要问我是怎么写的.我只会抄.如果有兴趣可以自行搜索全局钩子.HOOK.以下案例是微软的大臂用MSAA技术做的钩子

const uint WINEVENT_INCONTEXT = 0x0004;
const uint EVENT_MIN = 0x00000001;
const uint EVENT_MAX = 0x7FFFFFFF;
const uint EVENT_OBJECT_INVOKED = 0x8013;
const uint EVENT_OBJECT_STATECHANGE = 0x800A;
const uint ROLE_SYSTEM_PUSHBUTTON = 43;
const uint ROLE_SYSTEM_WINDOW = 10;
const int STATE_SYSTE_PRESSED = 0X00000008;
const int STATE_SYSTE_FOCUSED = 0X00000004;

[DllImport("user32.dll")]
static extern IntPtr SetWinEventHook(
uint eventMin,
uint eventMax,
IntPtr hmodWinEventProc,
WinEventDelegate lpfnWinEventProc,
uint idProcess,
uint idThread,
uint dwFlags);

delegate void WinEventDelegate(
IntPtr hWinEventHook,
uint eventType,
IntPtr hwnd,
int idObject,
int idChild,
uint dwEventThread,
uint dwmsEventTime);

[DllImport("Oleacc.dll")]
static extern uint AccessibleObjectFromEvent(IntPtr hwnd, int dwObjectID, int dwChildID, out IAccessible ppacc, [MarshalAs(UnmanagedType.Struct)] out object pvarChild);

private void WinEventCallback(
IntPtr hWinEventHook,
uint eventType,
IntPtr hwnd,
int idObject,
int idChild,
uint dwEventThread,
uint dwmsEventTime)
{
if (eventType == EVENT_OBJECT_STATECHANGE)
{
IAccessible accObj = null;
object o = null;
AccessibleObjectFromEvent(hwnd, idObject, idChild, out accObj, out o);

int state;
if (accObj != null &&
accObj.accRole.ToString().Equals(ROLE_SYSTEM_PUSHBUTTON.ToString()) &&
accObj.accName == txtButtonName.Text.Trim() &&
int.TryParse(accObj.accState.ToString(), out state))
{
if ((state & STATE_SYSTE_PRESSED) == STATE_SYSTE_PRESSED &&
FindParentWindow(accObj, txtFormName.Text.Trim()))
{

txtOutput.AppendText(string.Format("{0}: {1} clicked.\r\n", DateTime.Now.ToShortTimeString(), accObj.accName));
}
}
}

}

bool FindParent(IAccessible acc, string parentName)
{
if (acc == null)
{
return false;
}

int maxRetry = 5;
int count = 0;
IAccessible cur = acc.accParent as IAccessible;
while (cur != null && count < maxRetry)
{
if (parentName.Equals(cur.accName.ToString())&&
cur.accRole.ToString().Equals(ROLE_SYSTEM_WINDOW.ToString()))
{
return true;
}

cur = cur.accParent as IAccessible;

count++;
}

return false;
}

bool FindParentWindow(IAccessible acc, string parentName)
{
if (acc == null)
{
return false;
}

int count = 0;
IAccessible cur = acc.accParent as IAccessible;
while (cur != null)
{
if (cur.accRole.ToString().Equals(ROLE_SYSTEM_WINDOW.ToString()))
{
if (parentName.Equals(cur.accName.ToString()))
{
return true;
}
else
{
return false;
}
}

cur = cur.accParent as IAccessible;

count++;
}

return false;
}

以上都是C底层的设计.啥都不能改.有性趣可以自行搜索IAccessible

private void button1_Click(object sender, EventArgs e)
{
//先获取Process
string targetProcessName = txtProcessName.Text.Trim();
if (!string.IsNullOrEmpty(targetProcessName))
{
Process targetProcess = Process.GetProcessesByName(targetProcessName).First();

if (targetProcess != null)
{
//IAccessible acc =
IntPtr result = SetWinEventHook(EVENT_MIN, EVENT_MAX, IntPtr.Zero,
new WinEventDelegate(WinEventCallback), (uint)targetProcess.Id, 0, 0);
Console.WriteLine(result);
}
}
}

这些是我唯一看懂的源码.targetProcessName是挂钩的进程名称.targetProcessName是挂钩的程序按钮.

如果调用这个方法.就可以监听外部程序的按钮.如果目标点击.就会触发我们的事件.txtOutput文本会记录事件.

以下讲窗口的改动和句柄的捕获.

[DllImport("user32.dll", EntryPoint = "FindWindow", SetLastError = true)]
private static extern IntPtr FindWindow(string lpClassName, string lpWindowName);

//第一个参数填NULL,第二个参数填窗口标题可以捕获该窗口的句柄

[DllImport("user32.dll")]
private static extern int GetWindowRect(IntPtr hwnd, out Rect lpRect);

public struct Rect
{
public int Left;
public int Top;
public int Right;
public int Bottom;
}

//第一个参数窗口句柄.声明Rect传进去就会返回窗口的Rect

[DllImport("user32.dll", EntryPoint = "WindowFromPoint")]
public static extern int WindowFromPoint(
int xPoint,
int yPoint
);

//传递XY就会返回坐标处的句柄

对于窗口的改动用API很蛋疼.步骤是先正常化窗口.然后设置活动窗口.最后置顶.由于是用代码置顶.所以最后还要手动取消置顶

[DllImport("user32.dll", EntryPoint = "ShowWindow", CharSet = CharSet.Auto)]
public static extern int ShowWindow(IntPtr hwnd, int nCmdShow);

//第一个填窗口句柄.后面填函数识别的整数

// //最大化3 最小化2 正常化1

[DllImport("user32.dll", EntryPoint = "SetForegroundWindow", SetLastError = true)]
private static extern void SetForegroundWindow(IntPtr hwnd);

//第一个填窗口句柄.

//设置活动窗口是必须的.不要问我为什么.

[DllImport("user32.dll", CharSet = CharSet.Auto)]
private static extern int SetWindowPos(IntPtr hWnd, int hWndInsertAfter, int x, int y, int Width, int Height, int flags);

SetWindowPos(“句柄”, -1, 0, 0, 0, 0, 1 | 2);

SetWindowPos(“句柄”, -2, 0, 0, 0, 0, 1 | 2);

//-1置顶.-2取消置顶

以上就是API.多了也懒得讲

public Bitmap GetScreenSnapshot(int x, int y, int wid, int hei)//截图
{
Rectangle rectangle = new Rectangle(x, y, wid, hei);
Bitmap bitmap = new Bitmap(rectangle.Width, rectangle.Height, PixelFormat.Format32bppArgb);
using (Graphics graphics = Graphics.FromImage(bitmap))
{
graphics.CopyFromScreen(rectangle.X, rectangle.Y, 0, 0, rectangle.Size, CopyPixelOperation.SourceCopy);
}
return bitmap;
}

做图片识别先做截图.传递TOP.LEFT.WIDTH.HEIGHT利用以上方法完成区域截图

当时吧.认为自己对图片识别底层有一定了解.然后自己重写了一套OCR.万万没想到在WIN 10上识别率过低.最后老老实实的用了网上的框架.

AspriseOCR.dll.

DevIL.dll

ILU.dll

private string OCRPrise(string imgfile,string width, string height)
{
String gcstr = Marshal.PtrToStringAnsi(OCRpart(@"" + imgfile, -1, 0, 0,Convert.ToInt32( width),Convert.ToInt32( height)));
gcstr = gcstr.Replace('O', '0');
gcstr = gcstr.Replace('o', '0');
gcstr = gcstr.Replace('Z', '2');
gcstr = gcstr.Replace('z', '2');
gcstr = gcstr.Replace('L', '1');
gcstr = gcstr.Replace('l', '1');
gcstr = gcstr.Replace('I', '1');
gcstr = gcstr.Replace('i', '1');

gcstr = gcstr.Replace('T', '7');
gcstr = gcstr.Replace('t', '7');

gcstr = gcstr.Replace('G', '9');
gcstr = gcstr.Replace('g', '9');

gcstr = gcstr.Replace('S', '5');
gcstr = gcstr.Replace('s', '5');

gcstr = gcstr.Replace('J', '2');
gcstr = gcstr.Replace('j', '2');

gcstr = gcstr.Replace(',', '.');

gcstr = gcstr.Replace("(P1C)", "");

gcstr = gcstr.Replace("-", "");

gcstr = gcstr.Replace("'", "");
string result = null;

for (int i = 0; i < gcstr.Length; i++)
{
if (!String.IsNullOrWhiteSpace(gcstr[i].ToString()))
{
result += gcstr[i];
}
}
return result;
}

以上就可以识别数字.在XP Win 7 Win10下识别率和兼容性还不错

以下讲本屌认知的OCR.在WIN7下还是很耐用的.因为WIN 10视网膜屏.所以..

比如目前要识别一张黑纸白字的图片中的数字

每一张图片都可以被解析成数据

数据中会存放图片的每一个像素和它对应的XY以及RGB

比如要识别白字.黑色像素的RGB是000.那么我们去除黑色像素的数据.保留白色像素的特征.相对坐标和像素值作为标准特征.

下一次在识别白字黑纸的图片就在轮回一边.

要注意数字的切割和与标准特征的对比.

一张图片有上千个像素.在底层进行处理时算法的工作量非常大.如果能做到快速和高效.重写的OCR就算成功.我在WIN 7上写的OCR明显很成功.

#region

int bytes = bmpData.Stride * bmp.Height;
byte[] rgbValues = new byte[bytes];

Marshal.Copy(ptr, rgbValues, 0, bytes);

byte red = 0;
byte green = 0;
byte blue = 0;

for (int x = 0; x < bmp.Width; x++)
{
for (int y = 0; y < bmp.Height; y++)
{
//See the link above for an explanation
//of this calculation
int position = (y * bmpData.Stride) + (x * Image.GetPixelFormatSize(bmpData.PixelFormat) / 8);
blue = rgbValues[position];
green = rgbValues[position + 1];
red = rgbValues[position + 2];
//Console.WriteLine("Fast: " + red + " "
// + green + " " + blue);
if (red == 128 && green == 0 && blue == 0)
{
divisionx.Add(x);
divisiony.Add(y);
break;
}
}
}
bmp.UnlockBits(bmpData);

#endregion7

//过滤

#region
//var ocrdata = frist[3].Split(new char[] { '|'});
//hdc = GetDC(IntPtr.Zero);
//for (int i = lpRect.Left + Convert.ToInt32( ocrdata[0]); i <= lpRect.Left + Convert.ToInt32(ocrdata[0]) + Convert.ToInt32(ocrdata[2]); i++)
//{
// for (int a1 = lpRect.Top + Convert.ToInt32(ocrdata[1]); a1 < lpRect.Top + Convert.ToInt32(ocrdata[1]) + Convert.ToInt32(ocrdata[3]); a1++)
// {
// Color color = GetColor(i, a1);
// if (color.R == 128 && color.G == 0 && color.B == 0)
// {
// divisionx.Add(i);
// divisiony.Add(a1);
// break;
// }

// }

//}
//ReleaseDC(IntPtr.Zero, hdc);

#endregion

//获取指定像素做截图范围

if (divisionx.Count > 0)
{
#region
for (int ic = 0; ic < divisionx.Count; ic++)
{
if (ic <= divisionx.Count - 2)
{
if (ic == 0)
{
if (divisionx[ic] + 1 != divisionx[ic + 1])
{
divisionx.Remove(divisionx[ic]);
break;
}
}
else
{
if (divisionx[ic] + 1 != divisionx[ic + 1] && divisionx[ic] - 1 != divisionx[ic - 1])
{
divisionx.Remove(divisionx[ic]);
break;
}
}
}
}
#endregion

//删除小数点

#region

for (int i = 0; i < divisionx.Count; i++)
{
if (i <= divisionx.Count - 2)
{
if (xystate == 0 && divisionx[i] + 1 == divisionx[i + 1])
{
xystate = 1;
startx.Add(divisionx[i]);
//MessageBox.Show(startx.ToString());
}
if (divisionx[i] + 1 != divisionx[i + 1] && xystate == 1)
{
xystate = 0;
endx.Add(divisionx[i]);
// MessageBox.Show(endx.ToString());//末尾数字无法确定像素范围.单独计算
}
}
}
#endregion

//分割

#region

int max = -1;
int test;
for (int i = 0; i < divisionx.Count; i++)
{
if (i <= divisionx.Count - 2)
{
max = (divisionx[i] > divisionx[i + 1] ? test = divisionx[i] : test = divisionx[i + 1]) > max ?
(divisionx[i] > divisionx[i + 1] ? max = divisionx[i] : max = divisionx[i + 1]) : max;

}

#endregion

endx.Add(max);

//添加末尾指定像素

#region

int maxy = -1;
int testy;
for (int i = 0; i < divisiony.Count; i++)
{
if (i <= divisiony.Count - 2)
{
if (maxy == -1)
{
maxy = (divisiony[i] > divisiony[i + 1] ? testy = divisiony[i + 1] : testy = divisiony[i]);
}
else
{
maxy = (divisiony[i] > divisiony[i + 1] ? testy = divisiony[i + 1] : testy = divisiony[i]) < maxy ?
(divisiony[i] > divisiony[i + 1] ? testy = divisiony[i + 1] : testy = divisiony[i]) : maxy;
}

}

#endregion

//获取TOP像素

#region

int buttommax = -1;
int buttomtest;
for (int i = 0; i < divisiony.Count; i++)
{
if (i <= divisiony.Count - 2)
{
buttommax = (divisiony[i] > divisiony[i + 1] ? buttomtest = divisiony[i] : buttomtest = divisiony[i + 1]) > buttommax ?
(divisiony[i] > divisiony[i + 1] ? buttomtest = divisiony[i] : buttomtest = divisiony[i + 1]) : buttommax;

}

#endregion

//获取Buttom像素

#region 截图
int screenx;
int screeny;
int screenWidth;
int screenHeight;
if (startx.Count == endx.Count)
{
for (int i = 0; i < startx.Count; i++)
{
screenx = startx[i];
screeny = maxy;
screenWidth = endx[i] - startx[i] + 2;
screenHeight = buttommax - maxy + 2;
//OCR.GetScreenSnapshot(screenx, screeny, screenWidth, screenHeight).Save(@"" + AppDomain.CurrentDomain.SetupInformation.ApplicationBase + i + ".bmp", ImageFormat.Bmp);
Image img = Image.FromHbitmap(bmp.GetHbitmap());
Bitmap newbmp = new Bitmap(screenWidth, screenHeight, PixelFormat.Format32bppArgb);

using (Graphics g = Graphics.FromImage(newbmp))
{
//Rectangle origReg = new Rectangle(0, 0, bmp.Width, bmp.Height);
//Rectangle destReg = new Rectangle(screenx, screeny, screenWidth, screenHeight);
g.DrawImage(img, 0, 0, new Rectangle(screenx, screeny, screenWidth, screenHeight), GraphicsUnit.Pixel);
}

newbmp.Save(@"" + AppDomain.CurrentDomain.SetupInformation.ApplicationBase + i + ".bmp", ImageFormat.Bmp);

}

}
else
{
MessageBox.Show("图片识别失败");
}
#endregion

#region 读取图片
string finallydata = null;
List<string> ocr = FileIO.BitmapSean();
List<string> value = new List<string>();
KeyValuePair<string, string> dicdata = new KeyValuePair<string, string>();

if (ocr.Count > 0)//图片
{
if (StaffData.OCRData.Count > 0)//模板
{
for (int g = 0; g < ocr.Count; g++)
{

for (int i = 0; i <= 9; i++)
{
value.Clear();
dicdata = StaffData.OCRData.Where(a => a.Key== i.ToString()).FirstOrDefault();//模板的数据

string[] ocrtest = dicdata.Value.Split(new char[] { '!' });

if (ocrtest.Count() > 0)
{

for (int a = 0; a < ocrtest.Count(); a++)
{
value.Add(ocrtest[a]);//dic转list
}

if (value.Count > 0)//匹配
{
string resultocr = OCR.TestOCRProperty(gaulxy: value, gaulFile: ocr[g], gaulKey: StaffData.OCRData.Where(a => a.Key == i.ToString()).FirstOrDefault().Key.ToString());//图片识别
if (resultocr != "-1")
{
finallydata += resultocr;
break;

}

}
}
}
}
if (!String.IsNullOrWhiteSpace(finallydata))
{

string last1 = finallydata[finallydata.Count() - 1].ToString();
string last2 = finallydata[finallydata.Count() - 2].ToString();
finallydata = finallydata.Substring(0, finallydata.Count() - 2);
finallydata = finallydata + "." + last2 + last1;

}
// MessageBox.Show(finallydata);
}

}
#endregion

#region 删除图片
//List<string> imgdata = FileIO.BitmapSean();
//if (imgdata.Count > 0)
//{
// for (int img = 0; img < imgdata.Count; img++)
// {
// if (File.Exists(imgdata[img]))
// {
// File.Delete(imgdata[img]);
// }
// }
//}
#endregion

以上就是本屌手写的OCR.仅仅提供一个思路和经验.

以上是我保存的数字特征.每一个像素的间距.仅供参考

萌兽

关注

1
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
外部应用程序调用和ocr的使用

WindowsAPI在每一台Windows系统上开放标准API供开发人员调用.功能齐全.在这里只介绍三个部分.1.利用API控制鼠标键盘.2.对外部程序的读写和监听3.对外部程序窗口的改动.外部程序的意思就是.假设我的程序是360.辣么我能控制腾讯QQ客户端啥的.API的宏.以下任何常量和函数都可以在const int MOUSEEVENTF_MOVE = 0x0001;
复制链接

扫一扫