1 / 51

Towards Beautiful Test Cases for Compiler Bugs

Towards Beautiful Test Cases for Compiler Bugs. John Regehr University of Utah.

raisie
Download Presentation

Towards Beautiful Test Cases for Compiler Bugs

An Image/Link below is provided (as is) to download presentation Download Policy: Content on the Website is provided to you AS IS for your information and personal use and may not be sold / licensed / shared on other websites without getting consent from its author. Content is provided to you AS IS for your information and personal use only. Download presentation by click this link. While downloading, if for some reason you are not able to download a presentation, the publisher may have deleted the file from their server. During download, if you can't get a presentation, the file might be deleted by the publisher.

E N D

Presentation Transcript

  1. Towards Beautiful Test Cases for Compiler Bugs John Regehr University of Utah

  2. template< class A0 , class A1> __attribute__((always_inline)) typename boost::dispatch::meta::call<tag::shift_right_( A0 const& , A1 const& )>::type shift_right ( A0 const& a0 , A1 const& a1 ) { typename boost::dispatch::make_functor<tag::shift_right_, A0>::type callee; return callee( a0 , a1);; } template< class A0 , class A1> __attribute__((always_inline)) typename boost::dispatch::meta::call<tag::shift_right_( A0 const& , A1 const& )>::type shr ( A0 const& a0 , A1 const& a1 ) { typename boost::dispatch::make_functor<tag::shift_right_, A0>::type callee; return callee( a0 , a1);; } } } # 5 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" 1 # 14 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" namespace boost { namespace simd { namespace ext { } } } namespace boost { namespace dispatch { namespace meta { template< class A0 , class A1> __attribute__((always_inline)) boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< floating_<A0> > , scalar_< integer_<A1> >) , tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_ , scalar_< floating_<A0> > const , scalar_< integer_<A1> > const , adl_helper = adl_helper() ) { boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< floating_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template< class A0 , class A1 , class Dummy > struct implement < boost::simd::tag::shift_right_( scalar_< floating_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0 , A1 const& a1 ) const { typedeftypename dispatch::meta::as_integer<A0, unsigned>::type itype; return bitwise_cast<result_type>(shift_right(bitwise_cast<itype>(a0),a1)); } }; } } } namespace boost { namespace simd { namespace ext{ } } } namespace boost { namespace dispatch { namespace meta { template< class A0 , class A1> __attribute__((always_inline)) boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< integer_<A0> > , scalar_< integer_<A1> >) , tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_ , scalar_< integer_<A0> > const , scalar_< integer_<A1> > const , adl_helper = adl_helper() ) { boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< integer_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template< class A0 , class A1 , class Dummy > struct implement < boost::simd::tag::shift_right_( scalar_< integer_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0 , A1 const& a1 ) const { return a0 >> a1; } }; } } } # 6 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_right.hpp" 1 # 20 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_right.hpp" namespace boost { namespace simd { namespace ext {

  3. From GCC PR 50800: “Testcase is [here] (couldn't attach it due to bugzilla size restrictions)” template< class A0 , class A1> __attribute__((always_inline)) typename boost::dispatch::meta::call<tag::shift_right_( A0 const& , A1 const& )>::type shift_right ( A0 const& a0 , A1 const& a1 ) { typename boost::dispatch::make_functor<tag::shift_right_, A0>::type callee; return callee( a0 , a1);; } template< class A0 , class A1> __attribute__((always_inline)) typename boost::dispatch::meta::call<tag::shift_right_( A0 const& , A1 const& )>::type shr ( A0 const& a0 , A1 const& a1 ) { typename boost::dispatch::make_functor<tag::shift_right_, A0>::type callee; return callee( a0 , a1);; } } } # 5 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" 1 # 14 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" namespace boost { namespace simd { namespace ext { } } } namespace boost { namespace dispatch { namespace meta { template< class A0 , class A1> __attribute__((always_inline)) boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< floating_<A0> > , scalar_< integer_<A1> >) , tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_ , scalar_< floating_<A0> > const , scalar_< integer_<A1> > const , adl_helper = adl_helper() ) { boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< floating_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template< class A0 , class A1 , class Dummy > struct implement < boost::simd::tag::shift_right_( scalar_< floating_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0 , A1 const& a1 ) const { typedeftypename dispatch::meta::as_integer<A0, unsigned>::type itype; return bitwise_cast<result_type>(shift_right(bitwise_cast<itype>(a0),a1)); } }; } } } namespace boost { namespace simd { namespace ext{ } } } namespace boost { namespace dispatch { namespace meta { template< class A0 , class A1> __attribute__((always_inline)) boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< integer_<A0> > , scalar_< integer_<A1> >) , tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_ , scalar_< integer_<A0> > const , scalar_< integer_<A1> > const , adl_helper = adl_helper() ) { boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< integer_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template< class A0 , class A1 , class Dummy > struct implement < boost::simd::tag::shift_right_( scalar_< integer_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0 , A1 const& a1 ) const { return a0 >> a1; } }; } } } # 6 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_right.hpp" 1 # 20 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_right.hpp" namespace boost { namespace simd { namespace ext {

  4. From GCC PR 50800: “Testcase is [here] (couldn't attach it due to bugzilla size restrictions)” template< class A0 , class A1> __attribute__((always_inline)) typename boost::dispatch::meta::call<tag::shift_right_( A0 const& , A1 const& )>::type shift_right ( A0 const& a0 , A1 const& a1 ) { typename boost::dispatch::make_functor<tag::shift_right_, A0>::type callee; return callee( a0 , a1);; } template< class A0 , class A1> __attribute__((always_inline)) typename boost::dispatch::meta::call<tag::shift_right_( A0 const& , A1 const& )>::type shr ( A0 const& a0 , A1 const& a1 ) { typename boost::dispatch::make_functor<tag::shift_right_, A0>::type callee; return callee( a0 , a1);; } } } # 5 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" 1 # 14 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" namespace boost { namespace simd { namespace ext { } } } namespace boost { namespace dispatch { namespace meta { template< class A0 , class A1> __attribute__((always_inline)) boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< floating_<A0> > , scalar_< integer_<A1> >) , tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_ , scalar_< floating_<A0> > const , scalar_< integer_<A1> > const , adl_helper = adl_helper() ) { boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< floating_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template< class A0 , class A1 , class Dummy > struct implement < boost::simd::tag::shift_right_( scalar_< floating_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0 , A1 const& a1 ) const { typedeftypename dispatch::meta::as_integer<A0, unsigned>::type itype; return bitwise_cast<result_type>(shift_right(bitwise_cast<itype>(a0),a1)); } }; } } } namespace boost { namespace simd { namespace ext{ } } } namespace boost { namespace dispatch { namespace meta { template< class A0 , class A1> __attribute__((always_inline)) boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< integer_<A0> > , scalar_< integer_<A1> >) , tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_ , scalar_< integer_<A0> > const , scalar_< integer_<A1> > const , adl_helper = adl_helper() ) { boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< integer_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template< class A0 , class A1 , class Dummy > struct implement < boost::simd::tag::shift_right_( scalar_< integer_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0 , A1 const& a1 ) const { return a0 >> a1; } }; } } } # 6 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_right.hpp" 1 # 20 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_right.hpp" namespace boost { namespace simd { namespace ext { Next comment: “That you couldn't attach it should tell you something…”

  5. From GCC PR 50800: “Testcase is [here] (couldn't attach it due to bugzilla size restrictions)” template< class A0 , class A1> __attribute__((always_inline)) typename boost::dispatch::meta::call<tag::shift_right_( A0 const& , A1 const& )>::type shift_right ( A0 const& a0 , A1 const& a1 ) { typename boost::dispatch::make_functor<tag::shift_right_, A0>::type callee; return callee( a0 , a1);; } template< class A0 , class A1> __attribute__((always_inline)) typename boost::dispatch::meta::call<tag::shift_right_( A0 const& , A1 const& )>::type shr ( A0 const& a0 , A1 const& a1 ) { typename boost::dispatch::make_functor<tag::shift_right_, A0>::type callee; return callee( a0 , a1);; } } } # 5 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" 1 # 14 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" namespace boost { namespace simd { namespace ext { } } } namespace boost { namespace dispatch { namespace meta { template< class A0 , class A1> __attribute__((always_inline)) boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< floating_<A0> > , scalar_< integer_<A1> >) , tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_ , scalar_< floating_<A0> > const , scalar_< integer_<A1> > const , adl_helper = adl_helper() ) { boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< floating_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template< class A0 , class A1 , class Dummy > struct implement < boost::simd::tag::shift_right_( scalar_< floating_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0 , A1 const& a1 ) const { typedeftypename dispatch::meta::as_integer<A0, unsigned>::type itype; return bitwise_cast<result_type>(shift_right(bitwise_cast<itype>(a0),a1)); } }; } } } namespace boost { namespace simd { namespace ext{ } } } namespace boost { namespace dispatch { namespace meta { template< class A0 , class A1> __attribute__((always_inline)) boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< integer_<A0> > , scalar_< integer_<A1> >) , tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_ , scalar_< integer_<A0> > const , scalar_< integer_<A1> > const , adl_helper = adl_helper() ) { boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< integer_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template< class A0 , class A1 , class Dummy > struct implement < boost::simd::tag::shift_right_( scalar_< integer_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0 , A1 const& a1 ) const { return a0 >> a1; } }; } } } # 6 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2 # 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_right.hpp" 1 # 20 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_right.hpp" namespace boost { namespace simd { namespace ext { Next comment: “That you couldn't attach it should tell you something…” Next comment: 203 KB reduced test case attached

  6. (safe_47 (((g_1161 ^ 0x2893L), (((safe_9 ((!l_1747), ((l_1749 = func_37 (func_58 ((l_1748, 4294967293UL), g_110, l_1168, p_7, l_1747), g_381, p_9)), g_894))) != l_1747) <= p_10)), p_8))), g_11) <= p_7)), g_1315, p_8), 10))) > 4UL), 0x93L)) ^ g_1161))) <= l_1750), p_8), g_458); g_1122 = (l_1749 && func_37 (l_1750, (g_1161 = ((safe_17 (((safe_60 (l_1755, 4)) | (g_703, (safe_55 ((p_10, 3UL), p_7)))), ((safe_51 ((safe_42 (0UL, ((l_1747 != 0x3AL) < g_28))))) | 0xF862D3BEL))) != 0xC7L)), g_1487)); g_146 = 0xBC806FC8L; l_1788 = func_24 (((safe_13 ((~ (((g_458 != (safe_54 (((~ (l_1749 = (safe_43 ((safe_14 ((((safe_62 ((safe_4 ((safe_63 (((l_1750 = 1L), (safe_59 (l_1168, 1))), (((p_10 = func_46 (func_46 (l_1749, g_894, (safe_23 (l_1749, (l_1167 = g_1161))), ((l_1469 =

  7. (safe_47 (((g_1161 ^ 0x2893L), (((safe_9 ((!l_1747), ((l_1749 = func_37 (func_58 ((l_1748, 4294967293UL), g_110, l_1168, p_7, l_1747), g_381, p_9)), g_894))) != l_1747) <= p_10)), p_8))), g_11) <= p_7)), g_1315, p_8), 10))) > 4UL), 0x93L)) ^ g_1161))) <= l_1750), p_8), g_458); g_1122 = (l_1749 && func_37 (l_1750, (g_1161 = ((safe_17 (((safe_60 (l_1755, 4)) | (g_703, (safe_55 ((p_10, 3UL), p_7)))), ((safe_51 ((safe_42 (0UL, ((l_1747 != 0x3AL) < g_28))))) | 0xF862D3BEL))) != 0xC7L)), g_1487)); g_146 = 0xBC806FC8L; l_1788 = func_24 (((safe_13 ((~ (((g_458 != (safe_54 (((~ (l_1749 = (safe_43 ((safe_14 ((((safe_62 ((safe_4 ((safe_63 (((l_1750 = 1L), (safe_59 (l_1168, 1))), (((p_10 = func_46 (func_46 (l_1749, g_894, (safe_23 (l_1749, (l_1167 = g_1161))), ((l_1469 = Exposes an LLVM miscompilation bug Part of an 89 KB program randomly generated by Csmith

  8. Csmith is our C compiler fuzzer ~425 compiler bugs discovered and reported so far http://embed.cs.utah.edu/csmith/ (safe_47 (((g_1161 ^ 0x2893L), (((safe_9 ((!l_1747), ((l_1749 = func_37 (func_58 ((l_1748, 4294967293UL), g_110, l_1168, p_7, l_1747), g_381, p_9)), g_894))) != l_1747) <= p_10)), p_8))), g_11) <= p_7)), g_1315, p_8), 10))) > 4UL), 0x93L)) ^ g_1161))) <= l_1750), p_8), g_458); g_1122 = (l_1749 && func_37 (l_1750, (g_1161 = ((safe_17 (((safe_60 (l_1755, 4)) | (g_703, (safe_55 ((p_10, 3UL), p_7)))), ((safe_51 ((safe_42 (0UL, ((l_1747 != 0x3AL) < g_28))))) | 0xF862D3BEL))) != 0xC7L)), g_1487)); g_146 = 0xBC806FC8L; l_1788 = func_24 (((safe_13 ((~ (((g_458 != (safe_54 (((~ (l_1749 = (safe_43 ((safe_14 ((((safe_62 ((safe_4 ((safe_63 (((l_1750 = 1L), (safe_59 (l_1168, 1))), (((p_10 = func_46 (func_46 (l_1749, g_894, (safe_23 (l_1749, (l_1167 = g_1161))), ((l_1469 = Part of an 89 KB program randomly generated by Csmith Exposes an LLVM miscompilation bug

  9. short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } $ clang -O1 foo.c ; ./a.out 0 $ clang -O2 foo.c ; ./a.out 1

  10. Reported as LLVM PR 12189 on 3/5/2012, fixed same day short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } $ clang -O1 foo.c ; ./a.out 0 $ clang -O2 foo.c ; ./a.out 1

  11. intprintf(constchar *, …); char f[] = { -9L }; int main (void) { printf ("%d\n", 255 | f[0]); } Intel CC 12.0.5 for x86-64 is wrong at “-Ofast -ipo”

  12. intprintf (const char *, ...); const union { short f1; int f2 : 13; } a = { 30155 }; int main (void) { printf ("%d\n", a.f1); printf ("%d\n", a.f2); return 0; } GCC 4.4.3 from Ubuntu 10.04 LTS for x86-64 is wrong at -O1

  13. intprintf (const char *, ...); int b; static struct S0 { intf2 : 1; } c; intmain (void) { b = -1 ^ c.f2; printf ("%d\n", b); } GCC from mid-Feb 2012 for x86-64 is wrong at -O0 (PR 52209)

  14. Beautiful test case == • Small (but not obfuscated) • Well-defined • Unambiguously elicits bad behavior • Makes the compiler bug obvious Not all bugs have beautiful test cases

  15. Delta Debugging algorithm “ddmin” == • Greedy search • Delete chunks • Test results for “interestingness” • Keep reducing granularity short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; }

  16. ddmin == • Greedy search • Delete chunks • Test for “interestingness” • Keep reducing granularity short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } Is this still interesting?

  17. #!/bin/bash clang -O1 test.c &&\ ./a.out > out1.txt &&\ clang -O2 test.c &&\ ./a.out > out2.txt &&\ ! diff out1.txt out2.txt

  18. ddmin == • Greedy search • Delete chunks • Test for “interestingness” • Keep reducing granularity short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; } Assume not interesting, so we backtrack

  19. ddmin == • Greedy search • Delete chunks • Test for “interestingness” • Keep reducing granularity short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; }

  20. ddmin == • Greedy search • Delete chunks • Test for “interestingness” • Keep reducing granularity short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; }

  21. ddmin == • Greedy search • Delete chunks • Test for “interestingness” • Keep reducing granularity short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; }

  22. ddmin == • Greedy search • Delete chunks • Test for “interestingness” • Keep reducing granularity short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; }

  23. ddmin == • Greedy search • Delete chunks • Test for “interestingness” • Keep reducing granularity short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; }

  24. ddmin == • Greedy search • Delete chunks • Test for “interestingness” • Keep reducing granularity short a; int b, d, k, l, m; int fn1 (short p1, int p2) { return p1 - p2; } int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1; } int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2; } void fn4 (unsigned char p1) { if (p1) b = 1; } void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; }

  25. Problem: ddmin gets stuck • Output is often not beautiful • Our premise: • The Delta Debugging concept is sound • But ddmin is too hard-coded, inflexible

  26. Generalized Delta Debugging • One or more transformations • E.g. “remove chunks of text” • Interestingness test • Fitness function • E.g. “prefer smaller test cases” • Search algorithm • E.g. “greedy”

  27. Generalized Delta Debugging • One or more transformations • E.g. “remove chunks of text” • Interestingness test • Fitness function • E.g. “prefer smaller test cases” • Search algorithm • E.g. “greedy”

  28. C-Reduce: A Generalized Delta Debugger • Fixpoint computation • Parameterized by • Interestingness test • Plugins that perform transformations Plugins are iteratorsimplementing: • reset() • advance() • transform(file)

  29. void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; }

  30. void fn5 (p1) { short 0 = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; }

  31. void fn5 (p1) { short n = 0; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; }

  32. void fn5 (p1) { short n = 60018; 0 = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; }

  33. void fn5 (p1) { short n = 60018; k = 0 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; }

  34. void fn5 (p1) { short n = 60018; k = fn2 (0); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m); } int main () { fn5 (2); printf ("%d\n", b); return 0; }

  35. Example transformations: • Remove chunks of text (like ddmin) • C-specific peephole transformations • 0xfeedbeefULL 1 • x ^= yx = y • (x + 1)  x + 1 • while (…) if (…) • x ? y : z  y C-Reduce has 49 plugins so far

  36. Some transformations make coordinated changes across different parts of the program • Inline a function call • Scalar replacement of aggregates • Un-nest nested function calls • Remove dead arguments • Make function return void • Reduce array dimension or pointer level • Shorten identifier name • These use Clang as a rewriter

  37. current test case yes transformation transformation transformation transformation interesting? transformation no

  38. Beautiful test case == • Small (but not obfuscated) • Well-defined • Unambiguously elicits bad behavior • Makes the compiler bug obvious

  39. GCC PR 51962 #include <iostream> using namespace std; int r[3], x[3], y[3]; int main() { int xa=2,ya=5,xb=4,yb=2,n=3; x[0] = 3; x[1] = 5; x[2] = 1; y[0] = 1; y[1] = 3; y[2] = 3; r[0] = 2; r[1] = 1; r[2] = 2; inttcount = 0; for (int k=min(xa,xb); k<=max(xa,xb); k++) { boolfound1,found2 = false; for (int j=0; j<n; j++) { if (((k-x[j])*(k-x[j])+(y[j]-ya)*(y[j]-ya))<=r[j]*r[j]) { found1 = true; } if (((k-x[j])*(k-x[j])+(y[j]-yb)*(y[j]-yb))<=r[j]*r[j]) { found2 = true; } if (found1 && found2) break; } if (!found1) tcount++; if (!found2) tcount++; } cout<< tcount << endl; return 0; }

  40. GCC PR 51962 Bug report says: “Compile the following simple code without -O3, and run. Now compile it with -O3 option (for optimization), run again. Surprisingly 2 different outputs appear.” #include <iostream> using namespace std; int r[3], x[3], y[3]; int main() { int xa=2,ya=5,xb=4,yb=2,n=3; x[0] = 3; x[1] = 5; x[2] = 1; y[0] = 1; y[1] = 3; y[2] = 3; r[0] = 2; r[1] = 1; r[2] = 2; inttcount = 0; for (int k=min(xa,xb); k<=max(xa,xb); k++) { boolfound1,found2 = false; for (int j=0; j<n; j++) { if (((k-x[j])*(k-x[j])+(y[j]-ya)*(y[j]-ya))<=r[j]*r[j]) { found1 = true; } if (((k-x[j])*(k-x[j])+(y[j]-yb)*(y[j]-yb))<=r[j]*r[j]) { found2 = true; } if (found1 && found2) break; } if (!found1) tcount++; if (!found2) tcount++; } cout<< tcount << endl; return 0; }

  41. GCC PR 51962 Bug report says: “Compile the following simple code without -O3, and run. Now compile it with -O3 option (for optimization), run again. Surprisingly 2 different outputs appear.” #include <iostream> using namespace std; int r[3], x[3], y[3]; int main() { int xa=2,ya=5,xb=4,yb=2,n=3; x[0] = 3; x[1] = 5; x[2] = 1; y[0] = 1; y[1] = 3; y[2] = 3; r[0] = 2; r[1] = 1; r[2] = 2; inttcount = 0; for (int k=min(xa,xb); k<=max(xa,xb); k++) { boolfound1,found2 = false; for (int j=0; j<n; j++) { if (((k-x[j])*(k-x[j])+(y[j]-ya)*(y[j]-ya))<=r[j]*r[j]) { found1 = true; } if (((k-x[j])*(k-x[j])+(y[j]-yb)*(y[j]-yb))<=r[j]*r[j]) { found2 = true; } if (found1 && found2) break; } if (!found1) tcount++; if (!found2) tcount++; } cout<< tcount << endl; return 0; } GCC developer responds: “You do not initialise found1.” PR 51962 is RESOLVED INVALID And this person may have a hard time getting someone to read his next bug report

  42. GCC PR 51962 #include <iostream> using namespace std; int r[3], x[3], y[3]; int main() { int xa=2,ya=5,xb=4,yb=2,n=3; x[0] = 3; x[1] = 5; x[2] = 1; y[0] = 1; y[1] = 3; y[2] = 3; r[0] = 2; r[1] = 1; r[2] = 2; inttcount = 0; for (int k=min(xa,xb); k<=max(xa,xb); k++) { boolfound1,found2 = false; for (int j=0; j<n; j++) { if (((k-x[j])*(k-x[j])+(y[j]-ya)*(y[j]-ya))<=r[j]*r[j]) { found1 = true; } if (((k-x[j])*(k-x[j])+(y[j]-yb)*(y[j]-yb))<=r[j]*r[j]) { found2 = true; } if (found1 && found2) break; } if (!found1) tcount++; if (!found2) tcount++; } cout<< tcount << endl; return 0; }

  43. C99 has • 191 kinds of undefined behavior • 52 kinds of unspecified behavior • Code in a bug report must not execute these behaviors • ddmin and C-Reduce tend to introduce these behaviors

  44. #!/bin/bash is_valid_c99 test.c &&\ clang -O1 test.c &&\ ./a.out > out1.txt &&\ clang -O2 test.c &&\ ./a.out > out2.txt &&\ ! diff out1.txt out2.txt

  45. Validity checkers: KCC: executable semantics for C99 http://code.google.com/p/c-semantics/ Frama-C: static analyzer that supports an interpreter mode http://frama-c.com/

  46. Median size output from reducers • For 57 compiler-crash bugs • ddmin*: 8.6 KB • C-Reduce: 0.33 KB • For 43 wrong-code bugs • ddmin*: 6.5 KB • C-Reduce: 0.51 KB *http://delta.tigris.org/

  47. Reduction fails when compiler is non-deterministic • Usually due to memory unsafety + ASLR • Reduction works poorly when compiler bug stems from a resource overflow • Infinite loop, memory leak, register spill

  48. Can C-Reduce create reportable test cases from proprietary codes? • We hope so • Clearly “declassification” has to be done on a case-by-case basis • We would love to get feedback about successes and failures

  49. Does C-Reduce work for C++ code? • Yes, but not (yet) very well • C++-specific transformations are needed • Template instantiation • Namespace and class hierarchy flattening • Problem: No validity checker for C++

  50. Factoring the transformations out of a delta debugger isa good idea • C-Reduce can (sometimes) produce beautiful test cases • Ensuring validity of test cases is hard

More Related