Commit e67653ea authored by si40wiga's avatar si40wiga Committed by Alexander Alekhin
Browse files

Merge pull request #2296 from si40wiga:fsr-inpaint 

* new algorithm Rapid Frequency Selective Reconstruction (FSR) added

* fix compiler warning

* applied changes suggested in alalek's review

* fix trailing whitespace

* xphoto: update inpaint() test

* fix pre-processing of error mask

* xphoto: move inpainting FSR algorithm into a separate file

* xphoto: cleanup inpaining documentation

* xphoto: inpainting fsr - avoid uninitialized values
parent 7d7312a9

modules/xphoto/doc/xphoto.bib

+46 −0
Original line number Diff line number Diff line
@@ -29,3 +29,49 @@ 
  publisher = {ACM},

  url = {https://www.researchgate.net/profile/Julie_Dorsey/publication/220184746_Fast_Bilateral_Filtering_for_the_Display_of_High_-_dynamic_-_range_Images/links/54566b000cf26d5090a95f96/Fast-Bilateral-Filtering-for-the-Display-of-High-dynamic-range-Images.pdf}

}



@INPROCEEDINGS{GenserPCS2018,

   author={N. {Genser} and J. {Seiler} and F. {Schilling} and A. {Kaup}},

   booktitle={Proc. Picture Coding Symposium (PCS)},

   title={Signal and Loss Geometry Aware Frequency Selective Extrapolation for Error Concealment},

   year={2018},

   pages={159-163},

   keywords={extrapolation;image reconstruction;video coding;loss geometry aware frequency selective extrapolation;error concealment;complex models;moderate computational complexity;Full HD image;error pattern;adjacent samples;undistorted samples;reconstruction parameters;processing order;High Efficiency Video Coding;content based partitioning;signal characteristics;block based frequency selective extrapolation;Image reconstruction;Extrapolation;Geometry;Partitioning algorithms;Task analysis;Computational modeling;Standards},

   doi={10.1109/PCS.2018.8456259},

   month={June},

}



@ARTICLE{SeilerTIP2015,

   author={J. {Seiler} and M. {Jonscher} and M. {Schöberl} and A. {Kaup}},

   journal={IEEE Transactions on Image Processing},

   title={Resampling Images to a Regular Grid From a Non-Regular Subset of Pixel Positions Using Frequency Selective Reconstruction},

   year={2015},

   volume={24},

   number={11},

   pages={4540-4555},

   keywords={Fourier transforms;image reconstruction;resampling images;regular grid;nonregular subset;pixel positions;frequency selective reconstruction;displaying image signals;image signal reconstruction algorithm;Fourier domain;optical transfer function;visual quality;peak signal-to-noise ratio;Image reconstruction;Signal processing algorithms;Reconstruction algorithms;Signal processing;Spatial resolution;;Image reconstruction;non-regular sampling;interpolation},

   doi={10.1109/TIP.2015.2463084},

   month={Nov},

}



@INPROCEEDINGS{GroscheICIP2018,

   author={S. {Grosche} and J. {Seiler} and A. {Kaup}},

   booktitle={Proc. 25th IEEE International Conference on Image Processing (ICIP)},

   title={Iterative Optimization of Quarter Sampling Masks for Non-Regular Sampling Sensors},

   year={2018},

   pages={26-30},

   keywords={extrapolation;image enhancement;image reconstruction;image resolution;image sampling;image sensors;interpolation;iterative methods;optimisation;regression analysis;iterative optimization;nonregular sampling sensors;iterative algorithm;arbitrary quarter sampling mask;reconstruction algorithms;random quarter sampling mask;optimized mask;frequency selective extrapolation;steering kernel regression;nearest neighbor interpolation;linear interpolation;regular imaging sensor;reconstruction quality;noise figure 0.31 dB to 0.68 dB;Image resolution;Image reconstruction;Sensors;Optimization;Energy resolution;Reconstruction algorithms;Image sensors;Non-Regular Sampling;Image reconstruction},

   doi={10.1109/ICIP.2018.8451658},

   month={Oct},

}



@INPROCEEDINGS{GroscheIST2018,

   author={S. {Grosche} and J. {Seiler} and A. {Kaup}},

   booktitle={Proc. IEEE International Conference on Imaging Systems and Techniques (IST)},

   title={Design Techniques for Incremental Non-Regular Image Sampling Patterns},

   year={2018},

   pages={1-6},

   keywords={image reconstruction;image resolution;image sampling;design techniques;incremental nonregular image sampling patterns;image signals;regular two dimensional grid;nonregular sampling patterns;sampling positions;random patterns;regular patterns;arbitrary sampling densities;incremental sampling patterns;sampling density;Image reconstruction;Scanning electron microscopy;Probability distribution;Atomic force microscopy;Reconstruction algorithms;Measurement by laser beam;Image Reconstruction;non-Regular Sampling},

   doi={10.1109/IST.2018.8577090},

   month={Oct},

}

modules/xphoto/include/opencv2/xphoto/inpainting.hpp

+38 −8
Original line number Diff line number Diff line
@@ -9,9 +9,14 @@ 
//

//                           License Agreement

//                For Open Source Computer Vision Library

//                       (3-clause BSD License)

//

// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.

// Copyright (C) 2000-2019, Intel Corporation, all rights reserved.

// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.

// Copyright (C) 2009-2016, NVIDIA Corporation, all rights reserved.

// Copyright (C) 2010-2013, Advanced Micro Devices, Inc., all rights reserved.

// Copyright (C) 2015-2016, OpenCV Foundation, all rights reserved.

// Copyright (C) 2015-2016, Itseez Inc., all rights reserved.

// Third party copyrights are property of their respective owners.

//

// Redistribution and use in source and binary forms, with or without modification,
@@ -24,8 +29,9 @@ 
//     this list of conditions and the following disclaimer in the documentation

//     and/or other materials provided with the distribution.

//

//   * The name of the copyright holders may not be used to endorse or promote products

//     derived from this software without specific prior written permission.

//   * Neither the names of the copyright holders nor the names of the contributors

//    may be used to endorse or promote products derived from this software

//    without specific prior written permission.

//

// This software is provided by the copyright holders and contributors "as is" and

// any express or implied warranties, including, but not limited to, the implied
@@ -58,24 +64,48 @@ namespace xphoto 
//! @addtogroup xphoto

//! @{



    //! various inpainting algorithms

    //! @brief Various inpainting algorithms

    //! @sa inpaint

    enum InpaintTypes

    {

        /** This algorithm searches for dominant correspondences (transformations) of

        image patches and tries to seamlessly fill-in the area to be inpainted using this

        transformations */

        INPAINT_SHIFTMAP = 0

        INPAINT_SHIFTMAP = 0,

        /** Performs Frequency Selective Reconstruction (FSR).

        One of the two quality profiles BEST and FAST can be chosen, depending on the time available for reconstruction.

        See @cite GenserPCS2018 and @cite SeilerTIP2015 for details.



        The algorithm may be utilized for the following areas of application:

        1. %Error Concealment (Inpainting).

           The sampling mask indicates the missing pixels of the distorted input

           image to be reconstructed.

        2. Non-Regular Sampling.

           For more information on how to choose a good sampling mask, please review

           @cite GroscheICIP2018 and @cite GroscheIST2018.



        1-channel grayscale or 3-channel BGR image are accepted.



        Conventional accepted ranges:

        - 0-255 for CV_8U

        - 0-65535 for CV_16U

        - 0-1 for CV_32F/CV_64F.

        */

        INPAINT_FSR_BEST = 1,

        INPAINT_FSR_FAST = 2                     //!< See #INPAINT_FSR_BEST

    };



    /** @brief The function implements different single-image inpainting algorithms.



    See the original paper @cite He2012 for details.

    See the original papers @cite He2012 (Shiftmap) or @cite GenserPCS2018 and @cite SeilerTIP2015 (FSR) for details.



    @param src source image, it could be of any type and any number of channels from 1 to 4. In case of

    @param src source image

    - #INPAINT_SHIFTMAP: it could be of any type and any number of channels from 1 to 4. In case of

    3- and 4-channels images the function expect them in CIELab colorspace or similar one, where first

    color component shows intensity, while second and third shows colors. Nonetheless you can try any

    colorspaces.

    @param mask mask (CV_8UC1), where non-zero pixels indicate valid image area, while zero pixels

    - #INPAINT_FSR_BEST or #INPAINT_FSR_FAST: 1-channel grayscale or 3-channel BGR image.

    @param mask mask (#CV_8UC1), where non-zero pixels indicate valid image area, while zero pixels

    indicate area to be inpainted

    @param dst destination image

    @param algorithmType see xphoto::InpaintTypes

modules/xphoto/src/inpainting.cpp

+37 −15
Original line number Diff line number Diff line
@@ -9,11 +9,19 @@ 
//

//                           License Agreement

//                For Open Source Computer Vision Library

//                       (3-clause BSD License)

//

// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.

// Copyright (C) 2000-2019, Intel Corporation, all rights reserved.

// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.

// Copyright (C) 2009-2016, NVIDIA Corporation, all rights reserved.

// Copyright (C) 2010-2013, Advanced Micro Devices, Inc., all rights reserved.

// Copyright (C) 2015-2016, OpenCV Foundation, all rights reserved.

// Copyright (C) 2015-2016, Itseez Inc., all rights reserved.

// Third party copyrights are property of their respective owners.

//

// Redistribution and use in source and binary forms, with or without modification,

// are permitted provided that the following conditions are met:

//

//   * Redistribution's of source code must retain the above copyright notice,

//     this list of conditions and the following disclaimer.

//
@@ -21,8 +29,9 @@ 
//     this list of conditions and the following disclaimer in the documentation

//     and/or other materials provided with the distribution.

//

//   * The name of Intel Corporation may not be used to endorse or promote products

//     derived from this software without specific prior written permission.

//   * Neither the names of the copyright holders nor the names of the contributors

//    may be used to endorse or promote products derived from this software

//    without specific prior written permission.

//

// This software is provided by the copyright holders and contributors "as is" and

// any express or implied warranties, including, but not limited to, the implied
@@ -46,6 +55,8 @@ 
#include <fstream>

#include <time.h>

#include <functional>

#include <string>

#include <tuple>



#include "opencv2/xphoto.hpp"
@@ -62,6 +73,8 @@ 
#include "annf.hpp"

#include "advanced_types.hpp"



#include "inpainting_fsr.impl.hpp"



namespace cv

{

namespace xphoto
@@ -298,17 +311,9 @@ namespace xphoto 
        }

    }



    /*! The function reconstructs the selected image area from known area.

    *  \param src : source image.

    *  \param mask : inpainting mask, 8-bit 1-channel image. Zero pixels indicate the area that needs to be inpainted.

    *  \param dst : destination image.

    *  \param algorithmType : inpainting method.

    */

    void inpaint(const Mat &src, const Mat &mask, Mat &dst, const int algorithmType)

    static

    void inpaint_shiftmap(const Mat &src, const Mat &mask, Mat &dst, const int algorithmType)

    {

        CV_Assert( mask.channels() == 1 && mask.depth() == CV_8U );

        CV_Assert( src.rows == mask.rows && src.cols == mask.cols );



        switch ( src.type() )

        {

            case CV_8SC1:
@@ -399,8 +404,25 @@ namespace xphoto 
                CV_Error_( CV_StsNotImplemented,

                    ("Unsupported source image format (=%d)",

                    src.type()) );

                break;

        }

    }



void inpaint(const Mat &src, const Mat &mask, Mat &dst, const int algorithmType)

{

    CV_Assert(!src.empty());

    CV_Assert(!mask.empty());

    CV_CheckTypeEQ(mask.type(), CV_8UC1, "");

    CV_Assert(src.rows == mask.rows && src.cols == mask.cols);



    switch (algorithmType)

    {

        case xphoto::INPAINT_SHIFTMAP:

            return inpaint_shiftmap(src, mask, dst, algorithmType);

        case xphoto::INPAINT_FSR_BEST:

        case xphoto::INPAINT_FSR_FAST:

            return inpaint_fsr(src, mask, dst, algorithmType);

    }

    CV_Error_(Error::StsNotImplemented, ("Unsupported inpainting algorithm type (=%d)", algorithmType));

}



}}  // namespace

modules/xphoto/src/inpainting_fsr.impl.hpp

0 → 100644
+826 −0

File added.

Preview size limit exceeded, changes collapsed.

modules/xphoto/test/test_inpainting.cpp

0 → 100644
+75 −0
Original line number Diff line number Diff line 
// This file is part of OpenCV project.

// It is subject to the license terms in the LICENSE file found in the top-level directory

// of this distribution and at http://opencv.org/license.html.

#include "test_precomp.hpp"



namespace opencv_test { namespace {

using namespace xphoto;





static void test_inpainting(const Size inputSize, InpaintTypes mode, double expected_psnr, ImreadModes inputMode = IMREAD_COLOR)

{

    string original_path = cvtest::findDataFile("cv/shared/lena.png");

    string mask_path = cvtest::findDataFile("cv/inpaint/mask.png");



    Mat original_ = imread(original_path, inputMode);

    ASSERT_FALSE(original_.empty()) << "Could not load input image " << original_path;



    Mat mask_ = imread(mask_path, IMREAD_GRAYSCALE);

    ASSERT_FALSE(mask_.empty()) << "Could not load error mask " << mask_path;



    Mat original, mask;

    resize(original_, original, inputSize, 0.0, 0.0, INTER_AREA);

    resize(mask_, mask, inputSize, 0.0, 0.0, INTER_NEAREST);



    Mat mask_valid = (mask == 0);

    Mat im_distorted(inputSize, original.type(), Scalar::all(0));

    original.copyTo(im_distorted, mask_valid);



    Mat reconstructed;

    xphoto::inpaint(im_distorted, mask_valid, reconstructed, mode);



    double adiff_psnr = cvtest::PSNR(original, reconstructed);

    EXPECT_LE(expected_psnr, adiff_psnr);



#if 0

    imshow("original", original);

    imshow("im_distorted", im_distorted);

    imshow("reconstructed", reconstructed);

    std::cout << "adiff_psnr=" << adiff_psnr << std::endl;

    waitKey();

#endif

}



TEST(xphoto_inpaint, smoke_FSR_FAST)  // fast smoke test, input doesn't fit well for tested algorithm

{

    test_inpainting(Size(128, 128), INPAINT_FSR_FAST, 30);

}

TEST(xphoto_inpaint, smoke_FSR_BEST)  // fast smoke test, input doesn't fit well for tested algorithm

{

    applyTestTag(CV_TEST_TAG_LONG);

    test_inpainting(Size(128, 128), INPAINT_FSR_BEST, 30);

}



TEST(xphoto_inpaint, smoke_grayscale_FSR_FAST)  // fast smoke test, input doesn't fit well for tested algorithm

{

    test_inpainting(Size(128, 128), INPAINT_FSR_FAST, 30, IMREAD_GRAYSCALE);

}

TEST(xphoto_inpaint, smoke_grayscale_FSR_BEST)  // fast smoke test, input doesn't fit well for tested algorithm

{

    test_inpainting(Size(128, 128), INPAINT_FSR_BEST, 30, IMREAD_GRAYSCALE);

}





TEST(xphoto_inpaint, regression_FSR_FAST)

{

    test_inpainting(Size(512, 512), INPAINT_FSR_FAST, 39.5);

}

TEST(xphoto_inpaint, regression_FSR_BEST)

{

    applyTestTag(CV_TEST_TAG_VERYLONG);  // add --test_tag_enable=verylong to run this test

    test_inpainting(Size(512, 512), INPAINT_FSR_BEST, 39.6);

}





}} // namespace