Commit 40b238b0 authored by Tobias Senst's avatar Tobias Senst Committed by Alexander Alekhin
Browse files

Merge pull request #2367 from tsenst:add_robust_interpolation_of_correspondence 

Add RIC method for sparse match interpolation

* * add RIC

* sparse_match_interpolators EdgeAwareInterpolation - enhance limitation of the maximal number of matches to be interpolated from SHORT_MAX to INT_MAX by substituting short by int types

* * add RIC citation

* * update EdgeAwareInterpolatorImpl

* * add intermediate RIC implementation

* * implementation of new paralelization classes. First test where sucessfull.

* * update documentation RICInterpolatorImpl and RLOF

* * update CMakeLists.txt remove highgui
* add test for RIC
* add ASSERTION for curr image

* * add cost map interface

* * fix internal cost map allocation bug

* * remove white spaces
* fix warnings

* *fix compiler warnings

* * remove double whitespaces
* underscore from parameters
* substitute parallel_for_() classes with lambda functions
* remove complex assertion statements
* remove dead code
* substitute swap function with std::swap()
* ocv_define_module now contains video instead of tracking module

* * remove whitespace endings

* * documentation update

* * remove double declarations that lead to warnings

* * unrole tracker.py
*  remove double space
* use static for inner functions
* update create function
* modify mem init. to avoid manual memory management

* * uncomment parallel_for_ for parallelization

* * unrole unwanted changes
* uncomment paralellization

* * remove empty comment
* change CHECK to CHK_GD
* remove not necessary ;

* *documentation remove double space
parent cacec38e

modules/optflow/README.md

+1 −1
Original line number Diff line number Diff line 
Optical Flow Algorithms

=======================



Algorithms for running and evaluating deepflow, simpleflow, sparsetodenseflow and motion templates (silhouette flow).
 
Algorithms for running and evaluating deepflow, simpleflow, sparsetodenseflow, robust local optical flow and motion templates (silhouette flow).

modules/optflow/doc/optflow.bib

+17 −0
Original line number Diff line number Diff line
@@ -104,9 +104,26 @@ 
  number = {9},

}



@phdthesis{Senst2019,

  title={Estimation and analysis of motion in video data},

  author={Senst, Tobias},

  year={2019},

  school={Technical Univerity Berlin},

  doi = {10.14279/depositonce-9085},

  url = {https://doi.org/10.14279/depositonce-9085}

}



@article{Tibshirani2008,

  title={Fast computation of the median by successive binning},

  author={Tibshirani, Ryan J},

  journal={arXiv preprint arXiv:0806.3301},

  year={2008}

}



@inproceedings{Hu2017,

  title={Robust interpolation of correspondences for large displacement optical flow},

  author={Hu, Yinlin and Li, Yunsong and Song, Rui},

  booktitle={IEEE Conference on Computer Vision and Pattern Recognition},

  pages={481--489},

  year={2017}

}
 
 No newline at end of file

modules/optflow/include/opencv2/optflow/rlofflow.hpp

+57 −11
Original line number Diff line number Diff line
@@ -30,14 +30,16 @@ enum SolverType { 
enum InterpolationType

{

    INTERP_GEO = 0,    /**<  Fast geodesic interpolation, see @cite Geistert2016 */

    INTERP_EPIC = 1,    /**<  Edge-preserving interpolation, see @cite Revaud2015,Geistert2016. */

    INTERP_EPIC = 1,   /**<  Edge-preserving interpolation using ximgproc::EdgeAwareInterpolator, see @cite Revaud2015,Geistert2016. */

    INTERP_RIC = 2,    /**<  SLIC based robust interpolation using ximgproc::RICInterpolator, see @cite Hu2017. */

};



/** @brief This is used store and set up the parameters of the robust local optical flow (RLOF) algoritm.

 *

 * The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014

 * and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as

 * proposed by @cite Bouguet00. The implementation is derived from optflow::calcOpticalFlowPyrLK().

 * proposed by @cite Bouguet00. More details and experiments can be found in the following thesis @cite Senst2019.

 * The implementation is derived from optflow::calcOpticalFlowPyrLK().

 * This RLOF implementation can be seen as an improved pyramidal iterative Lucas-Kanade and includes

 * a set of improving modules. The main improvements in respect to the pyramidal iterative Lucas-Kanade

 * are:
@@ -200,7 +202,8 @@ public: 
 *

 * The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014

 * and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as

 * proposed by @cite Bouguet00. The implementation is derived from optflow::calcOpticalFlowPyrLK().

 * proposed by @cite Bouguet00. More details and experiments can be found in the following thesis @cite Senst2019.

 * The implementation is derived from optflow::calcOpticalFlowPyrLK().

 *

 * The sparse-to-dense interpolation scheme allows for fast computation of dense optical flow using RLOF (see @cite Geistert2016).

 * For this scheme the following steps are applied:
@@ -324,6 +327,35 @@ public: 
     *    @see ximgproc::fastGlobalSmootherFilter, setUsePostProc

     */

    CV_WRAP virtual bool getUsePostProc() const = 0;

    //! @brief enables VariationalRefinement

    /**

     * @see getUseVariationalRefinement

     */

    CV_WRAP virtual void setUseVariationalRefinement(bool val) = 0;

    /** @copybrief setUseVariationalRefinement

     *    @see ximgproc::fastGlobalSmootherFilter, setUsePostProc

     */

    CV_WRAP virtual bool getUseVariationalRefinement() const = 0;

    //! @brief Parameter to tune the approximate size of the superpixel used for oversegmentation.

    /**

     * @see cv::ximgproc::createSuperpixelSLIC, cv::ximgproc::RICInterpolator

     */

    CV_WRAP virtual void setRICSPSize(int val) = 0;

    /** @copybrief setRICSPSize

    *    @see setRICSPSize

    */

    CV_WRAP virtual int  getRICSPSize() const = 0;

    /** @brief Parameter to choose superpixel algorithm variant to use:

     * - cv::ximgproc::SLICType SLIC segments image using a desired region_size (value: 100)

     * - cv::ximgproc::SLICType SLICO will optimize using adaptive compactness factor (value: 101)

     * - cv::ximgproc::SLICType MSLIC will optimize using manifold methods resulting in more content-sensitive superpixels (value: 102).

     *  @see cv::ximgproc::createSuperpixelSLIC, cv::ximgproc::RICInterpolator

    */

    CV_WRAP virtual void setRICSLICType(int val) = 0;

    /** @copybrief setRICSLICType

     *    @see setRICSLICType

     */

    CV_WRAP virtual int  getRICSLICType() const = 0;

    //! @brief Creates instance of optflow::DenseRLOFOpticalFlow

    /**

     *    @param rlofParam see optflow::RLOFOpticalFlowParameter
@@ -333,9 +365,12 @@ public: 
     *    @param epicK see setEPICK

     *    @param epicSigma see setEPICSigma

     *    @param epicLambda see setEPICLambda

     *    @param ricSPSize see setRICSPSize

     *    @param ricSLICType see setRICSLICType

     *    @param use_post_proc see setUsePostProc

     *    @param fgsLambda see setFgsLambda

     *    @param fgsSigma see setFgsSigma

     *    @param use_variational_refinement see setUseVariationalRefinement

    */

    CV_WRAP static Ptr<DenseRLOFOpticalFlow> create(

        Ptr<RLOFOpticalFlowParameter> rlofParam = Ptr<RLOFOpticalFlowParameter>(),
@@ -345,16 +380,20 @@ public: 
        int epicK = 128,

        float epicSigma = 0.05f,

        float epicLambda = 999.0f,

        int ricSPSize = 15,

        int ricSLICType = 100,

        bool use_post_proc = true,

        float fgsLambda = 500.0f,

        float fgsSigma = 1.5f);

        float fgsSigma = 1.5f,

        bool use_variational_refinement = false);

};



/** @brief Class used for calculation sparse optical flow and feature tracking with robust local optical flow (RLOF) algorithms.

*

* The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014

 * and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as

* proposed by @cite Bouguet00. The implementation is derived from optflow::calcOpticalFlowPyrLK().

* proposed by @cite Bouguet00. More details and experiments can be found in the following thesis @cite Senst2019.

* The implementation is derived from optflow::calcOpticalFlowPyrLK().

*

* For the RLOF configuration see optflow::RLOFOpticalFlowParameter for further details.

* Parameters have been described in @cite Senst2012, @cite Senst2013, @cite Senst2014 and @cite Senst2016.
@@ -401,7 +440,8 @@ public: 
 *

 * The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014

 * and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as

 * proposed by @cite Bouguet00. The implementation is derived from optflow::calcOpticalFlowPyrLK().

 * proposed by @cite Bouguet00. More details and experiments can be found in the following thesis @cite Senst2019.

 * The implementation is derived from optflow::calcOpticalFlowPyrLK().

 *

 * The sparse-to-dense interpolation scheme allows for fast computation of dense optical flow using RLOF (see @cite Geistert2016).

 * For this scheme the following steps are applied:
@@ -430,12 +470,15 @@ public: 
 * supported:

 * - **INTERP_GEO** applies the fast geodesic interpolation, see @cite Geistert2016.

 * - **INTERP_EPIC_RESIDUAL** applies the edge-preserving interpolation, see @cite Revaud2015,Geistert2016.

 * @param epicK see ximgproc::EdgeAwareInterpolator() sets the respective parameter.

 * @param epicSigma see ximgproc::EdgeAwareInterpolator() sets the respective parameter.

 * @param epicLambda see ximgproc::EdgeAwareInterpolator() sets the respective parameter.

 * @param epicK see ximgproc::EdgeAwareInterpolator sets the respective parameter.

 * @param epicSigma see ximgproc::EdgeAwareInterpolator sets the respective parameter.

 * @param epicLambda see ximgproc::EdgeAwareInterpolator sets the respective parameter.

 * @param ricSPSize  see ximgproc::RICInterpolator sets the respective parameter.

 * @param ricSLICType see ximgproc::RICInterpolator sets the respective parameter.

 * @param use_post_proc enables ximgproc::fastGlobalSmootherFilter() parameter.

 * @param fgsLambda sets the respective ximgproc::fastGlobalSmootherFilter() parameter.

 * @param fgsSigma sets the respective ximgproc::fastGlobalSmootherFilter() parameter.

 * @param use_variational_refinement enables VariationalRefinement

 *

 * Parameters have been described in @cite Senst2012, @cite Senst2013, @cite Senst2014, @cite Senst2016.

 * For the RLOF configuration see optflow::RLOFOpticalFlowParameter for further details.
@@ -451,14 +494,17 @@ CV_EXPORTS_W void calcOpticalFlowDenseRLOF(InputArray I0, InputArray I1, InputOu 
    float forwardBackwardThreshold = 0, Size gridStep = Size(6, 6),

    InterpolationType interp_type = InterpolationType::INTERP_EPIC,

    int epicK = 128, float epicSigma = 0.05f, float epicLambda = 100.f,

    bool use_post_proc = true, float fgsLambda = 500.0f, float fgsSigma = 1.5f);

    int ricSPSize = 15, int ricSLICType = 100,

    bool use_post_proc = true, float fgsLambda = 500.0f, float fgsSigma = 1.5f,

    bool use_variational_refinement = false);



/** @brief Calculates fast optical flow for a sparse feature set using the robust local optical flow (RLOF) similar

* to optflow::calcOpticalFlowPyrLK().

*

* The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014

 * and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as

* proposed by @cite Bouguet00. The implementation is derived from optflow::calcOpticalFlowPyrLK().

* proposed by @cite Bouguet00. More details and experiments can be found in the following thesis @cite Senst2019.

* The implementation is derived from optflow::calcOpticalFlowPyrLK().

*

* @param prevImg first 8-bit input image. If The cross-based RLOF is used (by selecting optflow::RLOFOpticalFlowParameter::supportRegionType

* = SupportRegionType::SR_CROSS) image has to be a 8-bit 3 channel image.

modules/optflow/samples/optical_flow_evaluation.cpp

+15 −1
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ using namespace optflow; 
const String keys = "{help h usage ? |      | print this message   }"

        "{@image1        |      | image1               }"

        "{@image2        |      | image2               }"

        "{@algorithm     |      | [farneback, simpleflow, tvl1, deepflow, sparsetodenseflow, pcaflow, DISflow_ultrafast, DISflow_fast, DISflow_medium] }"

        "{@algorithm     |      | [farneback, simpleflow, tvl1, deepflow, sparsetodenseflow, RLOF_EPIC, RLOF_RIC, pcaflow, DISflow_ultrafast, DISflow_fast, DISflow_medium] }"

        "{@groundtruth   |      | path to the .flo file  (optional), Middlebury format }"

        "{m measure      |endpoint| error measure - [endpoint or angular] }"

        "{r region       |all   | region to compute stats about [all, discontinuities, untextured] }"
@@ -259,6 +259,20 @@ int main( int argc, char** argv ) 
        algorithm = createOptFlow_DeepFlow();

    else if ( method == "sparsetodenseflow" )

        algorithm = createOptFlow_SparseToDense();

    else if (method == "RLOF_EPIC")

    {

        algorithm = createOptFlow_DenseRLOF();

        Ptr<DenseRLOFOpticalFlow> rlof = algorithm.dynamicCast< DenseRLOFOpticalFlow>();

        rlof->setInterpolation(INTERP_EPIC);

        rlof->setForwardBackward(1.f);

    }

    else if (method == "RLOF_RIC")

    {

        algorithm = createOptFlow_DenseRLOF();

        Ptr<DenseRLOFOpticalFlow> rlof = algorithm.dynamicCast< DenseRLOFOpticalFlow>();;

        rlof->setInterpolation(INTERP_RIC);

        rlof->setForwardBackward(1.f);

    }

    else if ( method == "pcaflow" ) {

        if ( parser.has("prior") ) {

            String prior = parser.get<String>("prior");

modules/optflow/src/rlofflow.cpp

+50 −4
Original line number Diff line number Diff line
@@ -70,6 +70,9 @@ public: 
        , fgs_lambda(500.0f)

        , fgs_sigma(1.5f)

        , use_post_proc(true)

        , use_variational_refinement(false)

        , sp_size(15)

        , slic_type(ximgproc::SLIC)



    {

        prevPyramid[0] = cv::Ptr<CImageBuffer>(new CImageBuffer);
@@ -107,6 +110,15 @@ public: 
    virtual bool getUsePostProc() const CV_OVERRIDE { return use_post_proc; }

    virtual void setUsePostProc(bool val) CV_OVERRIDE { use_post_proc = val; }



    virtual void setUseVariationalRefinement(bool val) CV_OVERRIDE { use_variational_refinement = val; }

    virtual bool getUseVariationalRefinement() const CV_OVERRIDE { return use_variational_refinement; }



    virtual void setRICSPSize(int val) CV_OVERRIDE { sp_size = val; }

    virtual int  getRICSPSize() const CV_OVERRIDE { return sp_size; }



    virtual void setRICSLICType(int val) CV_OVERRIDE { slic_type = static_cast<ximgproc::SLICType>(val); }

    virtual int  getRICSLICType() const CV_OVERRIDE { return slic_type; }



    virtual void calc(InputArray I0, InputArray I1, InputOutputArray flow) CV_OVERRIDE

    {

        CV_Assert(!I0.empty() && I0.depth() == CV_8U && (I0.channels() == 3 || I0.channels() == 1));
@@ -116,7 +128,7 @@ public: 
            param = Ptr<RLOFOpticalFlowParameter>(new RLOFOpticalFlowParameter());

        if (param->supportRegionType == SR_CROSS)

            CV_Assert( I0.channels() == 3 && I1.channels() == 3);

        CV_Assert(interp_type == InterpolationType::INTERP_EPIC || interp_type == InterpolationType::INTERP_GEO);

        CV_Assert(interp_type == InterpolationType::INTERP_EPIC || interp_type == InterpolationType::INTERP_GEO || interp_type == InterpolationType::INTERP_RIC);

        // if no parameter is used use the default parameter



        Mat prevImage = I0.getMat();
@@ -184,9 +196,23 @@ public: 
            gd->setK(k);

            gd->setSigma(sigma);

            gd->setLambda(lambda);

            gd->setFGSLambda(fgs_lambda);

            gd->setFGSSigma(fgs_sigma);

            gd->setUsePostProcessing(false);

            gd->interpolate(prevImage, filtered_prevPoints, currImage, filtered_currPoints, dense_flow);

        }

        else if (interp_type == InterpolationType::INTERP_RIC)

        {

            Ptr<ximgproc::RICInterpolator> gd = ximgproc::createRICInterpolator();

            gd->setK(k);

            gd->setFGSLambda(fgs_lambda);

            gd->setFGSSigma(fgs_sigma);

            gd->setSuperpixelSize(sp_size);

            gd->setSuperpixelMode(slic_type);

            gd->setUseGlobalSmootherFilter(false);

            gd->setUseVariationalRefinement(false);

            gd->interpolate(prevImage, filtered_prevPoints, currImage, filtered_currPoints, dense_flow);

        }

        else

        {

            Mat blurredPrevImage, blurredCurrImage;
@@ -200,6 +226,15 @@ public: 
            cv::bilateralFilter(vecMats[1], vecMats2[1], 5, 2, 20);

            cv::merge(vecMats2, dense_flow);

        }

        if (use_variational_refinement)

        {

            Mat prevGrey, currGrey;

            Ptr<VariationalRefinement > variationalrefine = VariationalRefinement::create();

            cvtColor(prevImage, prevGrey, COLOR_BGR2GRAY);

            cvtColor(currImage, currGrey, COLOR_BGR2GRAY);

            variationalrefine->setOmega(1.9f);

            variationalrefine->calc(prevGrey, currGrey, flow);

        }

        if (use_post_proc)

        {

            ximgproc::fastGlobalSmootherFilter(prevImage, flow, flow, fgs_lambda, fgs_sigma);
@@ -227,6 +262,9 @@ protected: 
    float                         fgs_lambda;

    float                         fgs_sigma;

    bool                          use_post_proc;

    bool                          use_variational_refinement;

    int                           sp_size;

    ximgproc::SLICType            slic_type;

};



Ptr<DenseRLOFOpticalFlow> DenseRLOFOpticalFlow::create(
@@ -237,9 +275,12 @@ Ptr<DenseRLOFOpticalFlow> DenseRLOFOpticalFlow::create( 
    int epicK,

    float epicSigma,

    float epicLambda,

    int ricSPSize,

    int ricSLICType,

    bool use_post_proc,

    float fgs_lambda,

    float fgs_sigma)

    float fgs_sigma,

    bool use_variational_refinement)

{

    Ptr<DenseRLOFOpticalFlow> algo = makePtr<DenseOpticalFlowRLOFImpl>();

    algo->setRLOFOpticalFlowParameter(rlofParam);
@@ -252,6 +293,9 @@ Ptr<DenseRLOFOpticalFlow> DenseRLOFOpticalFlow::create( 
    algo->setUsePostProc(use_post_proc);

    algo->setFgsLambda(fgs_lambda);

    algo->setFgsSigma(fgs_sigma);

    algo->setRICSLICType(ricSLICType);

    algo->setRICSPSize(ricSPSize);

    algo->setUseVariationalRefinement(use_variational_refinement);

    return algo;

}
@@ -381,11 +425,13 @@ void calcOpticalFlowDenseRLOF(InputArray I0, InputArray I1, InputOutputArray flo 
    float forewardBackwardThreshold, Size gridStep,

    InterpolationType interp_type,

    int epicK, float epicSigma, float epicLambda,

    bool use_post_proc, float fgsLambda, float fgsSigma)

    int superpixelSize, int superpixelType,

    bool use_post_proc, float fgsLambda, float fgsSigma, bool use_variational_refinement)

{

    Ptr<DenseRLOFOpticalFlow> algo = DenseRLOFOpticalFlow::create(

        rlofParam, forewardBackwardThreshold, gridStep, interp_type,

        epicK, epicSigma, epicLambda, use_post_proc, fgsLambda, fgsSigma);

        epicK, epicSigma, epicLambda, superpixelSize, superpixelType,

        use_post_proc, fgsLambda, fgsSigma, use_variational_refinement);

    algo->calc(I0, I1, flow);

    algo->collectGarbage();

}