AR Cut & Paste的服务器组网结构分析

本项目组网如下图：

本网络使用三个端口：基于expo和react-native的移动客户端，基于flask的远程服务器，PhotoShop的远程连接服务。

App移动端负责拍取限制了大小的cut-image和paste-image传回给Sever。使用expo这个软件可以节省app上机调试的时间。

expo的网页端，它可以通过远程服务器直接把代码发送到expo app的登陆账号进行调试。

import React, { useState, useEffect } from "react";
import {
  Text,
  View,
  Image,
  TouchableWithoutFeedback,
  StyleSheet,
} from "react-native";
import * as ImageManipulator from "expo-image-manipulator";
import { Camera } from "expo-camera";

import ProgressIndicator from "./components/ProgressIndicator";
import server from "./components/Server";

调用expo的相机集成模块，使用ping服务器。下面展示cut函数的使用。

async function cut(): Promise<string> {
    const start = Date.now();
    console.log("");
    console.log("Cut");

    console.log(camera.pictureSize);
    // const ratios = await camera.getSupportedRatiosAsync()
    // console.log(ratios)
    // const sizes = await camera.getAvailablePictureSizeAsync("2:1")
    // console.log(sizes)

    console.log("> taking image...");
    const opts = { skipProcessing: true, exif: false, quality: 0 };
    // const opts = {};
    let photo = await camera.takePictureAsync(opts);

    console.log("> resizing...");
    const { uri } = await ImageManipulator.manipulateAsync(
      photo.uri,
      [
        { resize: { width: 256, height: 512 } },
        { crop: { originX: 0, originY: 128, width: 256, height: 256 } },
        // { resize: { width: 256, height: 457 } },
        // { crop: { originX: 0, originY: 99, width: 256, height: 256 } },
        // { resize: { width: 256, height: 341 } },
        // { crop: { originX: 0, originY: 42, width: 256, height: 256 } },
      ]
      // { compress: 0, format: ImageManipulator.SaveFormat.JPEG, base64: false }
    );

    console.log("> sending to /cut...");
    const resp = await server.cut(uri);

    console.log(`Done in ${((Date.now() - start) / 1000).toFixed(3)}s`);
    return resp;
  }

App回传的cut-received.jpg

Sever远程服务器提供图片接入发送接口，Photoshop连接服务

1.cut函数实现端点执行显著性检测/背景去除。

2.同时存储结果的一个副本，以便稍后粘贴。传输方法都是使用request库里面的post方法。

# The cut endpoints performs the salience detection / background removal.cut
# And store a copy of the result to be pasted later.
@app.route('/cut', methods=['POST'])
def save():
    start = time.time()
    logging.info(' CUT')

    # Convert string of image data to uint8.
    if 'data' not in request.files:
        return jsonify({
            'status': 'error',
            'error': 'missing file param `data`'
        }), 400
    data = request.files['data'].read()
    if len(data) == 0:
        return jsonify({'status:': 'error', 'error': 'empty image'}), 400

    # Save debug locally.
    with open('cut_received.jpg', 'wb') as f:
        f.write(data)

    # Send to BASNet service.
    logging.info(' > sending to BASNet...')
    headers = {}
    if args.basnet_service_host is not None:
        headers['Host'] = args.basnet_service_host
    files= {'data': open('cut_received.jpg', 'rb')}
    res = requests.post(args.basnet_service_ip, headers=headers, files=files )
    # logging.info(res.status_code)

post请求常用于交互，可以用来修改数据。而常见的get请求常常用来获取（查询）数据，显然，本函数使用的是带header的post，用来传输cut-received.jpg文件到BASNet-http服务器。

def post(url, data=None, json=None, **kwargs):
    r"""Sends a POST request.

    :param url: URL for the new :class:`Request` object.
    :param data: (optional) Dictionary, list of tuples, bytes, or file-like
        object to send in the body of the :class:`Request`.
    :param json: (optional) json data to send in the body of the :class:`Request`.
    :param \*\*kwargs: Optional arguments that ``request`` takes.
    :return: :class:`Response <Response>` object
    :rtype: requests.Response
    """

    return request('post', url, data=data, json=json, **kwargs)

图片在BASNet服务器进行显著性检测和背景去除后，得到对应的cut_mask.png

cut_mask.png

    # Save mask locally.
    logging.info(' > saving results...')
    with open('cut_mask.png', 'wb') as f:
        f.write(res.content)
        # shutil.copyfileobj(res.raw, f)

    logging.info(' > opening mask...')
    mask = Image.open('cut_mask.png').convert("L")

同时，本函数使用Image.composite()函数来通过使用透明mask混合图像来创建复合图像。

    # Convert string data to PIL Image.
    logging.info(' > compositing final image...')
    ref = Image.open(io.BytesIO(data))
    empty = Image.new("RGBA", ref.size, 0)
    img = Image.composite(ref, empty, mask)

保存合成好的图片到本地服务器，同时抄送到客户端。响应耗时一般在2s

    # TODO: currently hack to manually scale up the images. Ideally this would
    # be done respective to the view distance from the screen.
    img_scaled = img.resize((img.size[0] * 3, img.size[1] * 3))

    # Save locally.
    logging.info(' > saving final image...')
    img_scaled.save('cut_current.png')

    # Save to buffer
    buff = io.BytesIO()
    img.save(buff, 'PNG')
    buff.seek(0)

    # Print stats
    logging.info(f'Completed in {time.time() - start:.2f}s')

    # Return data
    return send_file(buff, mimetype='image/png')

cut_current.png

3.paste处理新的粘贴请求。解码App传输过来的数据，还原成图片。

# The paste endpoints handles new paste requests.
@app.route('/paste', methods=['POST'])
def paste():
    start = time.time()
    logging.info(' PASTE')

    # Convert string of image data to uint8.
    if 'data' not in request.files:
        return jsonify({
            'status': 'error',
            'error': 'missing file param `data`'
        }), 400
    data = request.files['data'].read()
    if len(data) == 0:
        return jsonify({'status:': 'error', 'error': 'empty image'}), 400

    # Save debug locally.
    with open('paste_received.jpg', 'wb') as f:
        f.write(data)

    # Convert string data to PIL Image.
    logging.info(' > loading image...')
    view = Image.open(io.BytesIO(data))

使用 view.thumbnail()压缩需要paste的图片,确保图片不会超出画幅。

    # Ensure the view image size is under max_view_size.
    if view.size[0] > max_view_size or view.size[1] > max_view_size:
        view.thumbnail((max_view_size, max_view_size))
       
    # Take screenshot with pyscreenshot.
    logging.info(' > grabbing screenshot...')
    screen = pyscreenshot.grab()
    screen_width, screen_height = screen.size

    # Ensure screenshot is under max size.
    if screen.size[0] > max_screenshot_size or screen.size[1] > max_screenshot_size:
        screen.thumbnail((max_screenshot_size, max_screenshot_size))

在相机拍到的图片上定位粘贴位置。

paste_received.jpg

    # Finds view centroid coordinates in screen space.
    logging.info(' > finding projected point...')
    view_arr = np.array(view.convert('L'))
    screen_arr = np.array(screen.convert('L'))
    # logging.info(f'{view_arr.shape}, {screen_arr.shape}')
    x, y = screenpoint.project(view_arr, screen_arr, False)

    found = x != -1 and y != -1

    if found:
        # Bring back to screen space
        x = int(x / screen.size[0] * screen_width)
        y = int(y / screen.size[1] * screen_height)
        logging.info(f'{x}, {y}')

然后按照PIL识别出来的坐标传输图像到Photoshop上。

        # Paste the current image in photoshop at these coordinates.
        logging.info(' > sending to photoshop...')
        name = datetime.today().strftime('%Y-%m-%d-%H:%M:%S')
        img_path = os.path.join(os.getcwd(), 'cut_current.png')
        err = ps.paste(img_path, name, x, y, password=args.photoshop_password)
        if err is not None:
            logging.error('error sending to photoshop')
            logging.error(err)
            jsonify({'status': 'error sending to photoshop'})
    else:
        logging.info('screen not found')

    # Print stats.
    logging.info(f'Completed in {time.time() - start:.2f}s')

    # Return status.
    if found:
        return jsonify({'status': 'ok'})
    else:
        return jsonify({'status': 'screen not found'})

最终Sever存储的数据如图。