图像转铅笔素描

Introduce

       最近看了一篇论文《Combining Sketch and Tone for Pencil Drawing Production》，感觉很好玩，所以就想实现了一下，但是里面有些公式是在太难理解了，我就在github上找到了一份代码，对其进行了重构。

方法介绍

       这篇论文中前面形成stroke structure还是比较好理解的，流程如下：

1）计算图像的梯度图像；

2）用8个核对梯度图像进行卷积，每个核为指向各个方向的直线，共形成8张卷积图像（卷积）；

3）比较8张卷积图像，对应像素位置的值最大则保留，于是又形成8张图像（筛选）；

4）对3）中形成的8张图像，进行卷积，卷积核为每张图像对应的卷积图像的生成核（强化）；

5）将4）生成的8张图像相加，并规范化。

        后面形成tone map，以及进行texture rendering包含一些固定参数，不容易推到出合适的，作者给出了一些推荐值，还有一些解公式之类的工作，这里无法叙述，可参详http://www.cnblogs.com/Imageshop/p/4285566.html 。

效果图

       生成效果图如下：

  

        两张图像对比还是前一张香蕉的图像更好看，观察了多张图片，我个人认为画面内容简单生成的图像更好看。

代码    

       除了前面部分是理解后实现的，其余代码大部分是重构，原始代码是https://github.com/HVisionSensing/PicWall，我的代码如下：

#include <stdio.h>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>


//#define DEBUG_INFO

void generate_conv_kernel(int ksize, float angle, int hardness, cv::Mat &kernel)
{
    int halfSize = ksize / 2;
    int size = halfSize * 2 - 1;
    int y0 = halfSize - hardness + 1;
    int y1 = halfSize + hardness - 1;

    cv::Mat rect, rotateMat;
    cv::Point2i central;

    kernel = cv::Mat(size, size, CV_32FC1, cv::Scalar(0));

    rect = cv::Mat(kernel, cv::Rect(0, y0, size, y1 - y0 + 1));
    rect = cv::Scalar(255);

    central = cv::Point2i(halfSize, halfSize);
    rotateMat = cv::getRotationMatrix2D(central, angle, 1.0);

    cv::warpAffine(kernel, kernel, rotateMat, kernel.size());
}


void generate_stroke_structure(cv::Mat &gray, int hardness, int directions, float strength, cv::Mat &structure)
{
    cv::Mat gradient;
    cv::Mat *kernels = new cv::Mat[directions];
    cv::Mat *Gs = new cv::Mat[directions];
    cv::Mat *Cs = new cv::Mat[directions];

    int rows, cols;
    int ksize;

    assert(gray.depth() == CV_32F);

    rows = gray.rows;
    cols = gray.cols;

    {
        cv::Mat gx, gy;
        cv::Sobel(gray, gx, CV_32FC1, 1, 0);
        cv::Sobel(gray, gy, CV_32FC1, 0, 1);
        cv::magnitude(gx, gy, gradient);

#ifdef DEBUG_INFO
        cv::imshow("gradient", gradient/255);
        cv::waitKey();
#endif
    }

    ksize = (rows < cols ? rows : cols) / 30;

    for(int i = 0; i < directions; i++)
    {
        float angle = i * 180.0 / directions;

        generate_conv_kernel(ksize, angle, hardness, kernels[i]);
        cv::filter2D(gradient, Gs[i], -1, kernels[i]);

        Cs[i] = cv::Mat(rows, cols, CV_32FC1, cv::Scalar(0));
    }

    for(int y = 0; y < rows; y++)
    {
        for(int x = 0; x < cols; x++)
        {
            int idx = 0;
            float mmax = 0, t;

            for(int i = 0; i < directions; i++)
            {
                if( (t = Gs[i].at<float>(y, x)) > mmax)
                {
                    mmax = t;
                    idx = i;
                }
            }

            Cs[idx].at<float>(y, x) = gradient.at<float>(y, x);
        }
    }


    structure = cv::Mat(rows, cols, CV_32FC1, cv::Scalar(0));

    for(int i = 0; i < directions; i++)
    {
        cv::filter2D(Cs[i], Cs[i], -1, kernels[i]);
        structure += Cs[i];
    }

    cv::normalize(structure, structure, 0, 1, cv::NORM_MINMAX);
    structure = 1 - structure * strength;

    delete[] kernels;
    delete[] Gs;
    delete[] Cs;

}


void calc_hist(uchar *data, int width, int height, int stride, float hist[])
{
    float sq = width * height;

    memset(hist, 0, sizeof(float) * 256);

    for(int y = 0; y < height; y++)
    {
        for(int x = 0; x < width; x++)
            hist[data[x]] ++;
        data += stride;
    }

    for(int i = 0; i < 256; i++)
        hist[i] /= sq;
}


void specific_hist(float *hist, float *target, float histMap[])
{
    float cumHist[256], dist[256 * 256], *ptr;
    int start = 0, end = 0, lastStart = 0, lastEnd = 0;

    cumHist[0] = hist[0];

    for(int i = 1; i < 256; i++)
        cumHist[i] = cumHist[i-1] + hist[i];


    ptr = dist;
    for(int y = 0; y < 256; y++)
    {
        for(int x = 0; x < 256; x++)
            ptr[x] = fabs(cumHist[x] - target[y]);

        ptr += 256;
    }

    ptr = dist;
    for(int y = 0; y < 256; y++)
    {
        float mmin = ptr[0];

        for(int x = 1; x < 256; x++)
        {
            if(mmin >= ptr[x])
            {
                mmin = ptr[x];
                end = x;
            }
        }

        if(start != lastStart || end != lastEnd)
        {
            for(int x = start; x <= end; x++)
                histMap[x] = y;

            lastStart = start;
            lastEnd = end;
            start = end + 1;
        }

        ptr += 256;
    }

}


void generate_tone(cv::Mat &gray, float hist[], cv::Mat &tone)
{
    float target[256], bright[256], dark[256], mid[256];
    float sumB = 0, sumD = 0, sumM = 0, sum = 0;

    int rows = gray.rows;
    int cols = gray.cols;

    tone = cv::Mat(rows, cols, CV_32FC1, cv::Scalar(0));

    for(int i = 0; i < 256; i++)
    {
        bright[i] = expf((i - 255.0) / 9);
        dark[i] = expf( (i - 90) * (i - 90) / -242.0);

        sumB += bright[i];
        sumD += dark[i];

        if( 105 <= i && i <= 225)
            mid[i] = 1;
        else
            mid[i] = 0;

        sumM += mid[i];
    }

    sum = 0;
    for(int i = 0; i < 256; i++)
    {
        float b = bright[i] / sumB;
        float d = dark[i] / sumD;
        float m = mid[i] / sumM;

        sum +=  (52 * b + 37 * m + 11 * d) / 100;
        target[i] = sum;
    }

    {
        float histMap[256];

        specific_hist(hist, target, histMap);

        for(int y = 0; y < rows; y++)
        {
            float *ptr1 = (float*)(gray.data + gray.step * y);
            float *ptr2 = (float*)(tone.data + tone.step * y);

            for(int x = 0; x < cols; x++)
                ptr2[x] = histMap[(int)ptr1[x]] / 255.0;
        }

    }
}


void texture_rendering(cv::Mat &tone, cv::Mat &image,  cv::Mat &texture)
{
    int rows = tone.rows;
    int cols = tone.cols;

    int ny = ceil(float(rows) / image.rows);
    int nx = ceil(float(cols) / image.cols);

    cv::Mat beta(rows, cols, CV_32FC1, cv::Scalar(1));

    assert(tone.depth() == CV_32F && image.depth() == CV_32F);

    texture = cv::Mat(rows, cols, CV_32FC1, cv::Scalar(0));

    if(ny > 1 || nx > 1)
        cv::repeat(image, ny, nx, image);

    cv::resize(image, image, cv::Size(cols, rows));
    image /= 255;

    for(int x = 0; x < cols; x++)
        beta.at<float>(0, x) = logf(tone.at<float>(0, x) + FLT_EPSILON) /
            logf(image.at<float>(0, x) + FLT_EPSILON);


    for(int y = 0; y < rows; y++)
        beta.at<float>(y, 0) = logf(tone.at<float>(y, 0) + FLT_EPSILON) /
            logf(image.at<float>(y, 0) + FLT_EPSILON);


    for(int y = 1; y < rows; y++)
    {
        for(int x = 1; x < cols; x++)
        {
            float t = logf(image.at<float>(y, x) + FLT_EPSILON);
            float v = (t * logf(tone.at<float>(y, x) + FLT_EPSILON) +
                    0.2 * ( beta.at<float>(y-1, x) + beta.at<float>(y, x - 1) )) / (t * t + 0.4);

            beta.at<float>(y, x) = v;
        }
    }

#ifdef DEBUG_INFO
    cv::imshow("beta", beta);
    cv::waitKey();
#endif


    for(int y = 0; y < rows; y++)
        for(int x = 0; x < cols; x++)
            texture.at<float>(y, x) = powf(image.at<float>(y, x), beta.at<float>(y, x));
}


void generate_sketch_images(cv::Mat &img, cv::Mat &pencilSketch, cv::Mat &colorSketch)
{
    assert(img.channels() == 3);

    cv::Mat gray, structure, tone, texture;

    cv::cvtColor(img, gray, cv::COLOR_BGR2GRAY);

    float hist[256];

    calc_hist(gray.data, gray.cols, gray.rows, gray.step, hist);

    gray.convertTo(gray, CV_32FC1);

    generate_stroke_structure(gray, 1, 8, 0.9, structure);

#ifdef DEBUG_INFO
    cv::imshow("structure", structure);
    cv::waitKey();
#endif

    generate_tone(gray, hist, tone);

#ifdef DEBUG_INFO
    cv::imshow("tone", tone);
    cv::waitKey();
#endif

    texture_rendering(tone, gray, texture);

#ifdef DEBUG_INFO
    cv::imshow("texture", texture);
    cv::waitKey();
#endif

    pencilSketch = structure.mul(texture) * 255;


    pencilSketch.convertTo(pencilSketch, CV_8UC1);
#ifdef DEBUG_INFO
    cv::imshow("pencil", pencilSketch);
    cv::waitKey();
#endif

    {
        cv::Mat labImg;
        std::vector<cv::Mat> channels;
        cv::cvtColor(img, labImg, cv::COLOR_BGR2Lab);
        cv::split(labImg, channels);
        channels[0] = pencilSketch;
        cv::merge(channels, labImg);
        cv::cvtColor(labImg, colorSketch, cv::COLOR_Lab2BGR);
    }
#ifdef DEBUG_INFO
    cv::imshow("color", colorSketch);
    cv::waitKey();
#endif

}


int main(int argc, char **argv)
{
    if(argc < 4)
    {
        printf("Usage: %s [input image] [output pencil image] [output color pencil image]\n", argv[0]);
        return 1;
    }

    cv::Mat img = cv::imread(argv[1], CV_LOAD_IMAGE_COLOR);
    cv::Mat pencil, colorPencil;

    generate_sketch_images(img, pencil, colorPencil);


    cv::imshow("pencil", pencil);
    cv::imshow("color pencil", colorPencil);
    cv::waitKey();

    if(!cv::imwrite(argv[2], pencil))
    {
        printf("Can't write image %s\n", argv[2]);
        return 2;
    }

    if(! cv::imwrite(argv[3], colorPencil))
    {
        printf("Can't write image %s\n", argv[3]);
        return 3;
    }

    return 0;
}