Clean-Up Multiscale Branch

.gitignore

@@ -0,0 +1,2 @@
+
+build/

.travis.yml

@@ -1,10 +1,18 @@
-language: c
+language: c++
+sudo: required
+
 
 matrix:
   include:
     - os: linux
     - os: osx
 
+before_install:
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get -qq update; fi
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libfftw3-dev; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install --force fftw; fi
+#  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then  (brew install gcc || brew link --overwrite gcc) && brew install fftw; fi
+
 script:
-  - make
-  - make test
+  - mkdir build && cd build && cmake .. && make 
+#  - make test

CMakeLists.txt

@@ -0,0 +1,95 @@
+#Copyright 2014, Gabriele Facciolo <[email protected]> 
+cmake_minimum_required(VERSION 2.8)
+project(mgm)
+
+
+## SETUP ##
+
+# SETUP A MAKE CLEAN TO CLEAN EVERYTHING
+set_directory_properties (PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES 
+      "CMakeCache.txt;CMakeFiles;Makefile;cmake_install.cmake")
+
+# DEFAULT BUILD: RELEASE
+if (NOT CMAKE_BUILD_TYPE)
+   message(STATUS "No build type selected, default to Release")
+   set(CMAKE_BUILD_TYPE "Release")
+endif()
+
+if (NOT TARGET IIOLIB)
+   include_directories(extern/iio)
+   add_subdirectory(extern/iio)
+endif() 
+SET( LIBS ${LIBS} IIOLIB)
+
+FIND_PACKAGE( OpenMP )
+if(OPENMP_FOUND)
+   message("[OPENMP FOUND]")
+   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
+   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
+endif()
+
+
+SET( LIBS ${LIBS} fftw3)
+#SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+# Enable C99
+if (CMAKE_VERSION VERSION_LESS "3.1")
+  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99")
+else ()
+  set (CMAKE_C_STANDARD 99)
+endif ()
+
+
+# Build as ROS package also, if possible
+find_package(catkin QUIET)
+if (catkin_FOUND)
+  message("[CATKIN FOUND] Configuring for ROS library compatible compilation!")
+  include_directories(
+    ${catkin_INCLUDE_DIRS}
+  )
+
+  # Specify export settings
+  catkin_package(
+    INCLUDE_DIRS include src/impl extern/iio
+    LIBRARIES mgm iio
+  )
+else()
+  message("[CATKIN NOT FOUND] Skipping ROS relevant compile steps...")
+  message("(This is ok if you're not using MGM for ROS!)")
+endif()
+
+# INCLUDES AND COMPILATION ##
+
+# Specify build include directories
+include_directories(
+  include/mgm
+  include
+  src/impl
+)
+
+add_library(mgm
+  src/impl/img.cc
+  src/impl/point.cc
+  src/impl/mgm_costvolume.cc
+  src/impl/mgm_core.cc
+  src/impl/census_tools.cc
+  src/impl/shear.c
+  src/impl/stereo_utils.cc
+  src/impl/mgm_multiscale.cc
+  src/impl/smartparameter.cc
+)
+target_link_libraries(mgm ${LIBS})
+
+add_library(iio extern/iio/iio.c)
+target_link_libraries(iio IIOLIB)
+
+add_executable(run_mgm_multi src/main_mgm_multi.cc)
+target_link_libraries(run_mgm_multi ${LIBS} mgm)
+
+add_executable(run_mgm src/main_mgm.cc)
+target_link_libraries(run_mgm ${LIBS} mgm)
+
+if (catkin_FOUND)
+  install(DIRECTORY include/${PROJECT_NAME} src/impl extern/iio
+    DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION})
+endif()

README.md

@@ -0,0 +1,135 @@
+# MGM: More Global Matching
+
+## Introduction:
+
+This is the MGM code written by Gabriele Facciolo, Carlo de Franchis, and Enric Meinhardt.
+
+**There are two variants, MGM and MGM_multiscale**
+
+If you want to use this software or results obtained with it, the following paper should be cited within your publication:
+
+```
+G. Facciolo and C. de Franchis and E. Meinhardt, "MGM: A Significantly More Global Matching for Stereovision", in British Machine Vision Conference, BMVC 2015.
+
+Site : http://dev.ipol.im/~facciolo/mgm/
+Email: [email protected]
+```
+
+
+
+## Overview of the algorithm:
+
+This C++ code can be used for approximately optimizing MRF energies, defined on the 4- or 8-connected image grids, of the form:
+
+![E(D) = \sum_p C_p(D_p) + \sum_{pq} w_{pq} V (D_p, D_q)](https://latex.codecogs.com/gif.latex?%5Cbg_white%20%5CLARGE%20E%28D%29%20%3D%20%5Csum_p%20C_p%28D_p%29%20+%20%5Csum_%7Bpq%7D%20w_%7Bpq%7D%20V%20%28D_p%2C%20D_q%29)
+
+Where ![C_p()](https://latex.codecogs.com/png.latex?%5Cinline%20%5Cdpi%7B150%7D%20%5Cbg_white%20%5Clarge%20C_p) denotes the unary term for the p-th node, the variables ![D_p](https://latex.codecogs.com/png.latex?%5Cinline%20%5Cdpi%7B150%7D%20%5Cbg_white%20%5Clarge%20D_p) take values in the range [0,L-1], ![C_p()](https://latex.codecogs.com/png.latex?%5Cinline%20%5Cdpi%7B150%7D%20%5Cbg_white%20%5Clarge%20C_p) is represented as a costvolume of size W x H x L.
+
+![V()](https://latex.codecogs.com/gif.latex?%5Cbg_white%20%5CLARGE%20V%28%29) denotes the distance function used for specifying the pairwise potentials, it can take one of these two forms (with params P1,P2):
+     1) SMG's potential (Hirschmuller'08)
+
+                    |  0  if |a - b|==0
+          Vh(a,b) = | P1  if |a - b|==1
+                    | P2  otherwise
+  or
+     2) any potential described in (Felzenszwalb-Huttenlocher'06),
+        this code implements truncated linear and linear (P2=inf)
+
+          Vl(a,b) =  min(P1*|a - b|, P2).
+
+The edge weights are given by ![w_{pq}](https://latex.codecogs.com/gif.latex?%5Cbg_white%20%5CLARGE%20w_%7Bpq%7D), w can actually be used to adapt the parameters P1 and P2 on the pixel basis as:
+                ![V (D_p, D_q, P1(w(p)), P2(w(p)) )), P2(w(p)) )](https://latex.codecogs.com/gif.latex?%5Cbg_white%20%5CLARGE%20V%20%28D_p%2C%20D_q%2C%20P1%28w%28p%29%29%2C%20P2%28w%28p%29%29%20%29%29%2C%20P2%28w%28p%29%29%20%29)
+
+But the current implementation just multiplies the potential. The weights are represented as a stack of 8 images. For a pixel p each image of the stack contain the weight to the corresponding neighboring pixel: West, Est, S, N, (NW, NE, SE, SW).
+
+That is the first image just contains the weights for pixels to the left. For 4-connectivity, only the first 4 images are read, while for 8-connectivity the whole stack of 8 images is used.
+
+
+
+## Stereo code
+
+The code in this directory uses MGM for stereo.
+The option of the mgm program are shown when called without parameters.
+
+
+
+### Simplified code and Matlab interface
+
+The `matlab` subdirectory contains a simplified version of MGM, and a matlab wrapper for solving optimization problems as described above.
+
+
+
+### Compilation
+
+Tested in Linux, OSX and OpenBSD using gcc and clang
+
+```shell
+# In the root directory,
+
+# Make and go into a build folder
+$ mkdir -p build
+$ cd build
+
+# Create the makefile and build the project
+# (Uses OpenMP)
+$ cmake ..
+$ make
+```
+
+You might need to install `fftw3` as it is a dependency
+
+```shell
+$ sudo apt install fftw-dev 
+```
+
+
+
+### Running The Examples
+
+Run these inside the build directory!
+
+#### **MGM**
+
+The following line runs MGM with 8 traversals (-O 8) with 3-neighbor recursion (TSGM=3, mgm is usually 2, but this is NEW!).
+
+     * The cost is measured in absolute differences of the horizontal
+       sobel derivatives (sobel_x),
+     * for the regularity it uses the FELZENSZWALB potential V
+       (USE_TRUNCATED_LINEAR_POTENTIALS=1),
+     * then refines disparities with V_fitting and postprocesses with
+       MEDIAN filter.
+
+    MEDIAN=1 USE_TRUNCATED_LINEAR_POTENTIALS=1  TSGM=3 ./run_mgm -P2 20000 -P1 4 -r -120 -R 30 -p sobel_x -truncDist 63 -s vfit -O 8 ../data/fountain23-im?.png /tmp/{disp,cost}.tif
+
+
+The following line runs MGM with 8 traversals (-O 8) with 3-neighbor recursion (TSGM=3, mgm is usually 2, but this is NEW!)
+
+      * the cost is CENSUS on 3x3 neighors,
+      * for the regularity uses the FELZENSZWALB potential V
+        (USE_TRUNCATED_LINEAR_POTENTIALS=1),
+      * then refines disparities with V_fitting and postprocesses
+        with MEDIAN filter.
+
+    MEDIAN=1 CENSUS_NCC_WIN=3 USE_TRUNCATED_LINEAR_POTENTIALS=1  TSGM=3 ./run_mgm -P2 20000 -P1 2 -r -120 -R 30 -t census -s vfit -O 8 ../data/fountain23-im?.png /tmp/{disp,cost}.tif
+
+The following line runs a similar experiment with a satellite image
+
+    OMP_NUM_THREADS=4 MEDIAN=1 CENSUS_NCC_WIN=5 TSGM=3 ./run_mgm -r -22 -R 19 -s vfit -t census -O 8 ../data/rectified_{ref,sec}.tif /tmp/{disp,cost}.tif
+
+
+
+**You may similarly invoke `./run_mgm_multi` to run the multiscale variant.** More information about the multiscale variant for MGM can be found in the docs.
+
+
+
+### Visualising the Results
+
+As the results of the disparity matching are 32bit float images, you might want to use [pvflip](<https://github.com/gfacciol/pvflip>) to visualise them!
+
+
+
+### Parameters
+
+Some of the parameters that are exposed by the library and their explanations can be found [here](https://docs.google.com/spreadsheets/d/1RWZpUHwzIbBTrZQG7sjQVR7uFMCn35e54tAg5_dAWRI/edit?usp=sharing).
+
+Cheers!

data

@@ -1 +1 @@
-matlab/data
+src/matlab/data/

docs/README_multiscale.md

@@ -0,0 +1,65 @@
+# Multiscale MGM summary
+
+Multiscale may be marginally slower than single scale, but the results are more 
+robust and for slanted surfaces it can have a big impact on the computation time.
+Census window of 5x5 give smoother disparity maps with less detailed than 3x3.
+Using subpixel duplicates the computing time but improves the computed edges.
+
+Recommended setups for S2P from below: #4 or #3 (if results of #4 are too noisy), 
+eventually set SUBPIX=1 if time is a concern.
+
+## Tested configurations
+
+    # 1. census 5x5 
+    # time: ~8s
+    REMOVESMALLCC=25 MINDIFF=1 CENSUS_NCC_WIN=5 SUBPIX=1 time ./mgm_multi -S 0 -s vfit -t census data/rectified_???.tif /tmp/w5_S0_sp1_{d,c,b}.tif 
+
+    # 2. census 5x5 + multiscale 
+    # time: ~10s
+    # multiscale yields less outliers: a more robust result
+    REMOVESMALLCC=25 MINDIFF=1 CENSUS_NCC_WIN=5 SUBPIX=1 time ./mgm_multi -S 3 -s vfit -t census data/rectified_???.tif /tmp/w5_S3_sp1_{d,c,b}.tif
+
+    # 3. census 5x5 + multiscale + subpixel
+    # time: ~20s
+    # subpixel improves the edges 
+    REMOVESMALLCC=25 MINDIFF=1 CENSUS_NCC_WIN=5 SUBPIX=2 time ./mgm_multi -S 3 -s vfit -t census data/rectified_???.tif /tmp/w5_S3_sp2_{d,c,b}.tif
+
+    # 4. census 3x3 + multiscale + subpixel 
+    # time: ~20s
+    # 3x3 window improves the edges, and recovers small features
+    REMOVESMALLCC=25 MINDIFF=1 CENSUS_NCC_WIN=3 SUBPIX=2 time ./mgm_multi -S 3 -s vfit -t census data/rectified_???.tif /tmp/w3_S3_sp2_{d,c,b}.tif
+
+
+## Important parameters
+
+    -S 4             : 5 coarse-to-fine scales 
+    -s vfit          : VFIT subpixel refinement algorithm
+    CENSUS_NCC_WIN=3 : controls the size of the CENSUS window: 3x3 or 5x5 are recommended  
+    REMOVESMALLCC=25 : remove regions with uniform disparity smaller than 25 pixels
+    MINDIFF=1        : remove some adherence pixels on a window CENSUS_NCC_WIN
+    SUBPIX=2         : performs a 1/2-pixel disparity refinement from the computed the integer disparities
+
+
+## Multiscale refinement
+
+The disparity is refined with coarse to fine multiscale algorithm. 
+The solution computed at the previous resolution is used to 
+determine the search range of the next scale. The default 
+parameters for computing these bounds are
+    MULTISCALE_MINMAX_UPSAMPLE_RADIUS=4
+    MULTISCALE_MINMAX_UPSAMPLE_SLACK=8
+they control the radius over which the min/max disparity of each
+pixel are estimated, and some extra range.
+
+Multiscale may miss a feature if these ranges are not properly adjusted.
+
+
+## Remarks about census cost
+
+It is important to note that raw census costs are very quantized. 
+For instance, a 3x3 window can take only one of 9 values: [0,..8]. 
+Although the costs computed by MGM are aggregated somehow they keep
+this quantized character. 
+Because of this the interpolation of census costs is a bit tricky. 
+VFIT interpolation does a nice job when SUBPIX=1, but with SUBPIX>1
+it tends to produce flattened surfaces. With SUBPIX=2 this is barely noticeable.