Percona Server moved to https://jira.percona.com/projects/PS

percona crash while restarting after an innodb recovery

Asked by anandjeyahar on 2011-02-10

Hi,
Here are the steps i followed. Am running percona 5.1.54 on a CentOS 5.5 machine. I had an os crash and was recovering from it. Percona asked me to run recovery first. So i set the force_innodb =4 and restarted. it ran fine. now to get back to normal mode i restart and percona crashes with the following error..
Starting MySQL (Percona Server).*** glibc detected *** /bin/sh: corrupted double-linked list: 0x00000000197fe970 ***
======= Backtrace: =========
/lib64/libc.so.6[0x3749673280]
/lib64/libc.so.6(__libc_malloc+0x6e)[0x3749674cde]
/lib64/libc.so.6[0x374962d4e1]
/lib64/libc.so.6[0x374962d772]
/lib64/libc.so.6[0x374962d772]
/lib64/libc.so.6[0x374962bbfd]
/lib64/libc.so.6[0x374962b47f]
/bin/sh[0x429c0e]
/bin/sh(execute_command_internal+0x654)[0x42a784]
/bin/sh(execute_command_internal+0x79f)[0x42a8cf]
/bin/sh[0x428e31]
/bin/sh[0x429dc0]
/bin/sh(execute_command_internal+0x654)[0x42a784]
/bin/sh(execute_command+0x4f)[0x42ba5f]
/bin/sh(reader_loop+0x126)[0x41b7c6]
/bin/sh(main+0x120a)[0x41b2aa]
/lib64/libc.so.6(__libc_start_main+0xf4)[0x374961d994]
/bin/sh[0x419119]
======= Memory map: ========
00400000-004b2000 r-xp 00000000 09:02 61562939 /bin/bash
006b2000-006bc000 rw-p 000b2000 09:02 61562939 /bin/bash
006bc000-006c1000 rw-p 006bc000 00:00 0
008bb000-008c3000 rw-p 000bb000 09:02 61562939 /bin/bash
197ec000-1980d000 rw-p 197ec000 00:00 0 [heap]
3749200000-374921c000 r-xp 00000000 09:02 7151650 /lib64/ld-2.5.so
374941b000-374941c000 r--p 0001b000 09:02 7151650 /lib64/ld-2.5.so
374941c000-374941d000 rw-p 0001c000 09:02 7151650 /lib64/ld-2.5.so
3749600000-374974e000 r-xp 00000000 09:02 7151801 /lib64/libc-2.5.so
374974e000-374994d000 ---p 0014e000 09:02 7151801 /lib64/libc-2.5.so
374994d000-3749951000 r--p 0014d000 09:02 7151801 /lib64/libc-2.5.so
3749951000-3749952000 rw-p 00151000 09:02 7151801 /lib64/libc-2.5.so
3749952000-3749957000 rw-p 3749952000 00:00 0
3749a00000-3749a02000 r-xp 00000000 09:02 7151829 /lib64/libdl-2.5.so
3749a02000-3749c02000 ---p 00002000 09:02 7151829 /lib64/libdl-2.5.so
3749c02000-3749c03000 r--p 00002000 09:02 7151829 /lib64/libdl-2.5.so
3749c03000-3749c04000 rw-p 00003000 09:02 7151829 /lib64/libdl-2.5.so
374a600000-374a603000 r-xp 00000000 09:02 7151663 /lib64/libtermcap.so.2.0.8
374a603000-374a802000 ---p 00003000 09:02 7151663 /lib64/libtermcap.so.2.0.8
374a802000-374a803000 rw-p 00002000 09:02 7151663 /lib64/libtermcap.so.2.0.8
374b600000-374b60d000 r-xp 00000000 09:02 7151933 /lib64/libgcc_s-4.1.2-20080825.so.1
374b60d000-374b80d000 ---p 0000d000 09:02 7151933 /lib64/libgcc_s-4.1.2-20080825.so.1
374b80d000-374b80e000 rw-p 0000d000 09:02 7151933 /lib64/libgcc_s-4.1.2-20080825.so.1
2b6d788d6000-2b6d788d7000 rw-p 2b6d788d6000 00:00 0
2b6d788df000-2b6d788e2000 rw-p 2b6d788df000 00:00 0
2b6d788e2000-2b6d7beb2000 r--p 00000000 09:02 8774953 /usr/lib/locale/locale-archive
2b6d7beb2000-2b6d7beb9000 r--s 00000000 09:02 8691832 /usr/lib64/gconv/gconv-modules.cache
2b6d7beb9000-2b6d7beba000 rw-p 2b6d7beb9000 00:00 0
2b6d7bec1000-2b6d7becb000 r-xp 00000000 09:02 7151771 /lib64/libnss_files-2.5.so
2b6d7becb000-2b6d7c0ca000 ---p 0000a000 09:02 7151771 /lib64/libnss_files-2.5.so
2b6d7c0ca000-2b6d7c0cb000 r--p 00009000 09:02 7151771 /lib64/libnss_files-2.5.so
2b6d7c0cb000-2b6d7c0cc000 rw-p 0000a000 09:02 7151771 /lib64/libnss_files-2.5.so
2b6d80000000-2b6d80021000 rw-p 2b6d80000000 00:00 0
2b6d80021000-2b6d84000000 ---p 2b6d80021000 00:00 0
7fffbf8d8000-7fffbf8ed000 rw-p 7ffffffe9000 00:00 0 [stack]
ffffffffff600000-ffffffffffe00000 ---p 00000000 00:00 0 [vdso]
*** glibc detected *** /bin/sh: corrupted double-linked list: 0x00000000197fe970 ***
======= Backtrace: =========
/lib64/libc.so.6[0x3749673280]
/lib64/libc.so.6(__libc_malloc+0x6e)[0x3749674cde]
/lib64/libc.so.6[0x374962d4e1]
/lib64/libc.so.6[0x374962d772]
/lib64/libc.so.6[0x374962bbfd]
/lib64/libc.so.6[0x374962b47f]
/bin/sh[0x434ea8]
/bin/sh[0x4352f5]
/bin/sh[0x4357df]
/bin/sh(notify_and_cleanup+0x35)[0x437725]
/bin/sh(wait_for+0x20a)[0x43794a]
/bin/sh(execute_command_internal+0xf21)[0x42b051]
/bin/sh(execute_command_internal+0x79f)[0x42a8cf]
/bin/sh[0x428e31]
/bin/sh[0x429dc0]
/bin/sh(execute_command_internal+0x654)[0x42a784]
/bin/sh(execute_command+0x4f)[0x42ba5f]
/bin/sh(reader_loop+0x126)[0x41b7c6]
/bin/sh(main+0x120a)[0x41b2aa]
/lib64/libc.so.6(__libc_start_main+0xf4)[0x374961d994]
/bin/sh[0x419119]
======= Memory map: ========
00400000-004b2000 r-xp 00000000 09:02 61562939 /bin/bash
006b2000-006bc000 rw-p 000b2000 09:02 61562939 /bin/bash
006bc000-006c1000 rw-p 006bc000 00:00 0
008bb000-008c3000 rw-p 000bb000 09:02 61562939 /bin/bash
197ec000-1980d000 rw-p 197ec000 00:00 0 [heap]
3749200000-374921c000 r-xp 00000000 09:02 7151650 /lib64/ld-2.5.so
374941b000-374941c000 r--p 0001b000 09:02 7151650 /lib64/ld-2.5.so
374941c000-374941d000 rw-p 0001c000 09:02 7151650 /lib64/ld-2.5.so
3749600000-374974e000 r-xp 00000000 09:02 7151801 /lib64/libc-2.5.so
374974e000-374994d000 ---p 0014e000 09:02 7151801 /lib64/libc-2.5.so
374994d000-3749951000 r--p 0014d000 09:02 7151801 /lib64/libc-2.5.so
3749951000-3749952000 rw-p 00151000 09:02 7151801 /lib64/libc-2.5.so
3749952000-3749957000 rw-p 3749952000 00:00 0
3749a00000-3749a02000 r-xp 00000000 09:02 7151829 /lib64/libdl-2.5.so
3749a02000-3749c02000 ---p 00002000 09:02 7151829 /lib64/libdl-2.5.so
3749c02000-3749c03000 r--p 00002000 09:02 7151829 /lib64/libdl-2.5.so
3749c03000-3749c04000 rw-p 00003000 09:02 7151829 /lib64/libdl-2.5.so
374a600000-374a603000 r-xp 00000000 09:02 7151663 /lib64/libtermcap.so.2.0.8
374a603000-374a802000 ---p 00003000 09:02 7151663 /lib64/libtermcap.so.2.0.8
374a802000-374a803000 rw-p 00002000 09:02 7151663 /lib64/libtermcap.so.2.0.8
374b600000-374b60d000 r-xp 00000000 09:02 7151933 /lib64/libgcc_s-4.1.2-20080825.so.1
374b60d000-374b80d000 ---p 0000d000 09:02 7151933 /lib64/libgcc_s-4.1.2-20080825.so.1
374b80d000-374b80e000 rw-p 0000d000 09:02 7151933 /lib64/libgcc_s-4.1.2-20080825.so.1
2b6d788d6000-2b6d788d7000 rw-p 2b6d788d6000 00:00 0
2b6d788df000-2b6d788e2000 rw-p 2b6d788df000 00:00 0
2b6d788e2000-2b6d7beb2000 r--p 00000000 09:02 8774953 /usr/lib/locale/locale-archive
2b6d7beb2000-2b6d7beb9000 r--s 00000000 09:02 8691832 /usr/lib64/gconv/gconv-modules.cache
2b6d7beb9000-2b6d7bebb000 rw-p 2b6d7beb9000 00:00 0
2b6d7bec1000-2b6d7becb000 r-xp 00000000 09:02 7151771 /lib64/libnss_files-2.5.so
2b6d7becb000-2b6d7c0ca000 ---p 0000a000 09:02 7151771 /lib64/libnss_files-2.5.so
2b6d7c0ca000-2b6d7c0cb000 r--p 00009000 09:02 7151771 /lib64/libnss_files-2.5.so
2b6d7c0cb000-2b6d7c0cc000 rw-p 0000a000 09:02 7151771 /lib64/libnss_files-2.5.so
2b6d80000000-2b6d80021000 rw-p 2b6d80000000 00:00 0
2b6d80021000-2b6d84000000 ---p 2b6d80021000 00:00 0
7fffbf8d8000-7fffbf8ed000 rw-p 7ffffffe9000 00:00 0 [stack]
ffffffffff600000-ffffffffffe00000 ---p 00000000 00:00 0 [vdso]
/etc/init.d/mysql: line 152: 4440 Aborted $bindir/mysqld_safe --datadir=$datadir --pid-file=$server_pid_file $other_args > /dev/null 2>&1
ERROR! Manager of pid-file quit without updating file.