mem alloc Issue with ipvsadm on i386

Asked by Stephen Murcott

This looks like the bug listed here http://lists.openwall.net/netdev/2011/03/22/4 but I may be wrong. I have included as much information as possible to help make it clear.

I was able to replicate on 3 seperate clean installs of the following:

ubuntu-12.04-server-i386.iso with md5sum 32184a83c8b5e6031e1264e5c499bc03
(have reproduced on different kernels)

Linux lvs 3.2.0-27-generic-pae #43-Ubuntu SMP Fri Jul 6 15:06:05 UTC 2012 i686 i686 i386 GNU/Linux

Steps to reproduce:-

Setup - install ubuntu 12.04 i386 server with sshd
apt-get upgrade
reboot

enabled ipv4 forwarding
net.ipv4.ip_forward = 1
iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE

aptitude install ipvsadm keepalived

ipvsadm -A -t $VIP:$PORT -s rr
Memory allocation problem

lsmod | grep vs
ip_vs_wrr 12615 0
ip_vs_wlc 12471 0
ip_vs_sh 12572 0
ip_vs_sed 12471 0
ip_vs_rr 12538 0
ip_vs_nq 12468 0
ip_vs_lc 12468 0
ip_vs_lblcr 12802 0
ip_vs_lblc 12747 0
ip_vs_ftp 13014 0
ip_vs_dh 12572 0
nf_nat 24959 3 ip_vs_ftp,ipt_MASQUERADE,iptable_nat
ip_vs 121543 24 ip_vs_wrr,ip_vs_wlc,ip_vs_sh,ip_vs_sed,ip_vs_rr,ip_vs_nq,ip_vs_lc,ip_vs_lblcr,ip_vs_lblc,ip_vs_ftp,ip_vs_dh
nf_conntrack 73847 5 ipt_MASQUERADE,iptable_nat,nf_nat,nf_conntrack_ipv4,ip_vs
libcrc32c 12543 1 ip_vs

strace ipvsadm -A -t 192.168.122.21:80 -s -rr
execve("/sbin/ipvsadm", ["ipvsadm", "-A", "-t", "192.168.122.21:80", "-s", "-rr"], [/* 20 vars */]) = 0
brk(0) = 0x8cd1000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77ec000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=15720, ...}) = 0
mmap2(NULL, 15720, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb77e8000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/i386-linux-gnu/libpopt.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\30\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=47012, ...}) = 0
mmap2(NULL, 49804, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x768000
mmap2(0x773000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa) = 0x773000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/libnl-genl-3.so.200", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\21\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=18460, ...}) = 0
mmap2(NULL, 21116, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xc5d000
mmap2(0xc61000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0xc61000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/libnl-3.so.200", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200E\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=91856, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77e7000
mmap2(NULL, 94756, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xe5e000
mmap2(0xe74000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15) = 0xe74000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/i386-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0000\226\1\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1713640, ...}) = 0
mmap2(NULL, 1723100, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x374000
mmap2(0x513000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19f) = 0x513000
mmap2(0x516000, 10972, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x516000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/i386-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p[\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=124663, ...}) = 0
mmap2(NULL, 107008, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x173000
mmap2(0x18a000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16) = 0x18a000
mmap2(0x18c000, 4608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x18c000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/i386-linux-gnu/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0000D\0\0004\0\0\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=173576, ...}) = 0
mmap2(NULL, 176256, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xea6000
mmap2(0xed0000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x29) = 0xed0000
close(3) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77e6000
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77e5000
set_thread_area({entry_number:-1 -> 6, base_addr:0xb77e56c0, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
mprotect(0x513000, 8192, PROT_READ) = 0
mprotect(0xed0000, 4096, PROT_READ) = 0
mprotect(0x18a000, 4096, PROT_READ) = 0
mprotect(0xe74000, 4096, PROT_READ) = 0
mprotect(0xc61000, 4096, PROT_READ) = 0
mprotect(0x773000, 4096, PROT_READ) = 0
mprotect(0x8054000, 4096, PROT_READ) = 0
mprotect(0x171000, 4096, PROT_READ) = 0
munmap(0xb77e8000, 15720) = 0
set_tid_address(0xb77e5728) = 2297
set_robust_list(0xb77e5730, 0xc) = 0
futex(0xbf9f4ec4, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 1, NULL, b77e56c0) = -1 EAGAIN (Resource temporarily unavailable)
rt_sigaction(SIGRTMIN, {0x178570, [], SA_SIGINFO}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x1785f0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
uname({sys="Linux", node="lvs", ...}) = 0
brk(0) = 0x8cd1000
brk(0x8cf2000) = 0x8cf2000
open("/proc/net/psched", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77eb000
read(3, "000003e8 00000040 000f4240 3b9ac"..., 1024) = 36
close(3) = 0
munmap(0xb77eb000, 4096) = 0
time(NULL) = 1343125167
socket(PF_NETLINK, SOCK_RAW|SOCK_CLOEXEC, 16) = 3
setsockopt(3, SOL_SOCKET, SO_SNDBUF, [32768], 4) = 0
setsockopt(3, SOL_SOCKET, SO_RCVBUF, [32768], 4) = 0
bind(3, {sa_family=AF_NETLINK, pid=2297, groups=00000000}, 12) = 0
getsockname(3, {sa_family=AF_NETLINK, pid=2297, groups=00000000}, [12]) = 0
sendmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"\24\0\0\0\20\0\5\3\257v\16P\371\10\0\0\3\1\0\0", 20}], msg_controllen=0, msg_flags=0}, 0) = 20
recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"t\0\0\0\20\0\2\0\257v\16P\371\10\0\0\1\2\0\0\v\0\2\0nlctrl\0\0"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 1520
recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"\24\0\0\0\3\0\2\0\257v\16P\371\10\0\0\0\0\0\0\v\0\2\0nlctrl\0\0"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 20
close(3) = 0
time(NULL) = 1343125167
socket(PF_NETLINK, SOCK_RAW|SOCK_CLOEXEC, 16) = 3
setsockopt(3, SOL_SOCKET, SO_SNDBUF, [32768], 4) = 0
setsockopt(3, SOL_SOCKET, SO_RCVBUF, [32768], 4) = 0
bind(3, {sa_family=AF_NETLINK, pid=2297, groups=00000000}, 12) = 0
getsockname(3, {sa_family=AF_NETLINK, pid=2297, groups=00000000}, [12]) = 0
sendmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"\24\0\0\0\20\0\5\3\257v\16P\371\10\0\0\3\1\0\0", 20}], msg_controllen=0, msg_flags=0}, 0) = 20
recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"t\0\0\0\20\0\2\0\257v\16P\371\10\0\0\1\2\0\0\v\0\2\0nlctrl\0\0"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 1520
recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"\24\0\0\0\3\0\2\0\257v\16P\371\10\0\0\0\0\0\0\v\0\2\0nlctrl\0\0"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 20
sendmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"\24\0\0\0\30\0\5\0\260v\16P\371\10\0\0\17\1\0\0", 20}], msg_controllen=0, msg_flags=0}, 0) = 20
recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"$\0\0\0\30\0\0\0\260v\16P\371\10\0\0\16\1\0\0\10\0\1\0\1\2\1\0\10\0\2\0"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 36
close(3) = 0
time(NULL) = 1343125167
socket(PF_NETLINK, SOCK_RAW|SOCK_CLOEXEC, 16) = 3
setsockopt(3, SOL_SOCKET, SO_SNDBUF, [32768], 4) = 0
setsockopt(3, SOL_SOCKET, SO_RCVBUF, [32768], 4) = 0
bind(3, {sa_family=AF_NETLINK, pid=2297, groups=00000000}, 12) = 0
getsockname(3, {sa_family=AF_NETLINK, pid=2297, groups=00000000}, [12]) = 0
sendmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"\24\0\0\0\20\0\5\3\257v\16P\371\10\0\0\3\1\0\0", 20}], msg_controllen=0, msg_flags=0}, 0) = 20
recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"t\0\0\0\20\0\2\0\257v\16P\371\10\0\0\1\2\0\0\v\0\2\0nlctrl\0\0"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 1520
recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"\24\0\0\0\3\0\2\0\257v\16P\371\10\0\0\0\0\0\0\v\0\2\0nlctrl\0\0"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 20
sendmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"h\0\0\0\30\0\5\0\260v\16P\371\10\0\0\1\1\0\0T\0\1\0\6\0\1\0\2\0\0\0"..., 104}], msg_controllen=0, msg_flags=0}, 0) = 104
recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"|\0\0\0\2\0\0\0\260v\16P\371\10\0\0\376\377\377\377h\0\0\0\30\0\5\0\260v\16P"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 124
close(3) = 0
write(2, "Memory allocation problem\n", 26Memory allocation problem
) = 26
exit_group(-1) = ?

Question information

Language:
English Edit question
Status:
Open
For:
Ubuntu ipvsadm Edit question
Assignee:
No assignee Edit question
Last query:
Last reply:
Revision history for this message
Stephen Murcott (scmurcott) said :
#1

For anyone else that happens to be making a load balancer with 12.04 and has needs a more version without the Memory Allocation error it is very quick to work around.

I basically worked around this issue by doing the following on the same server:-

aptitude purge ipvsadm

aptitude install git-core build-essential autoconf automake autotools-dev dh-make debhelper devscripts fakeroot libpopt-dev libnl-dev dpatch

mkdir tmp && cd tmp

git clone git://github.com/formorer/pkg-ipvsadm.git

cd pkg-ipvsadm

dpkg-buildpackage -rfakeroot
cd ..
dpkg -i ipvsadm_1.25_i386.deb
dpkg-reconfigure ipvsadm
ipvsadm -A -t $VIP:$PORT -s rr

ipvsadm --list

Can you help with this problem?

Provide an answer of your own, or ask Stephen Murcott for more information if necessary.

To post a message you must log in.