Here's the code. I'm using Halide on VS2013, Win64 trunk of Aug 5, 2015. When I execute diag.compile_to_lowered_stmt("diag.html", {}, HTML), I get a stack overflow inside halide.dll.
Image<uint8_t> orig_uint = Tools::load_image("../foo.ppm");
Var x, y, c;
Func orig("orig"), orig_lum("orig_lum"), m45("m45"), m135("m135"), f45("f45"), f135("f135"), f4x4_horiz("f4x4_horiz"), f4x4("f4x4"), diag("diag");
Func orig_clamped = BoundaryConditions::repeat_edge(orig_uint);
const float wta = 1.0f, wtb = 3.0f, wt0 = wta * wta, wt1 = wta * wtb, wt2 = wtb * wtb;
orig(x, y, c) = cast<float_t>(orig_clamped(x, y, c));
orig_lum(x, y) = 0.299f * orig(x, y, 0) + 0.587f * orig(x, y, 1) + 0.114f * orig(x, y, 2);
m45(x, y) = abs(orig_lum(x - 1, y - 1) - orig_lum(x, y)) + abs(orig_lum(x, y) - orig_lum(x + 1, y + 1)) + abs(orig_lum(x + 1, y + 1) - orig_lum(x + 2, y + 2));
m135(x, y) = abs(orig_lum(x + 2, y - 1) - orig_lum(x + 1, y)) + abs(orig_lum(x + 1, y) - orig_lum(x, y + 1)) + abs(orig_lum(x, y + 1) - orig_lum(x - 1, y + 2));
f45(x, y, c) = wta * (orig(x - 1, y - 1, c) + orig(x + 2, y + 2, c)) + wtb * (orig(x, y, c) + orig(x + 1, y + 1, c));
f135(x, y, c) = wta * (orig(x - 1, y + 2, c) + orig(x + 2, y - 1, c)) + wtb * (orig(x, y + 1, c) + orig(x + 1, y, c));
f4x4_horiz(x, y, c) = wta * (orig(x - 1, y, c) + orig(x + 2, y, c)) + wtb * (orig(x, y, c) + orig(x + 1, y, c));
f4x4(x, y, c) = wta * (f4x4_horiz(x, y - 1, c) + f4x4_horiz(x, y + 2, c)) + wtb * (f4x4_horiz(x, y, c) + f4x4_horiz(x, y + 1, c));
diag(x, y, c) = select(m135(x, y) > m45(x, y), f45(x, y, c), select(m45(x, y) > m135(x, y), f135(x, y, c), f4x4(x, y, c)));
// schedule
orig_lum.compute_root();
m45.compute_root().bound(x, 0, orig_uint.width()).bound(y, 0, orig_uint.height());
m135.compute_root().bound(x, 0, orig_uint.width()).bound(y, 0, orig_uint.height());
f45.compute_at(diag, x);
f135.compute_at(diag, x);
f4x4.compute_at(diag, x);
diag.compute_root();
// compile so we can take a look at the code
diag.compile_to_lowered_stmt("diag.html", {}, HTML); // stack oflo here
Any ideas? I'm also happy to accept an improved schedule if you want to offer one -- I just wanted to get something basic running first.
(I added a bound pair after diag.compute_root(), but that didn't seem to help. I do want to clamp the diag coefficients eventually.)
msvc uses a fairly small stack by default (1 MB), and many Halide compiler passes do deeply recursive stuff. I'd just increase the stack size to something like 8 megabytes (e.g. add the compile flag /STACK:8388608,1048576). That's what we do for the Halide tests on windows.