Garbage uint64 values from ctypes

Question

I'm trying to read the fields ifi_ibytes/ifi_obytes from the struct if_data64 as described in this header file. I've got ctypes working and I can make the sysctl call and I'm getting what seems to be usable information out of it. However, I'm getting what appears to be total garbage out of the non-zero integers. For example, I have one interface with 0 bytes received and ~8.5 million bytes transmitted. However, my python code is saying 0 received and 3,590,592,659,456 bytes transmitted.

I'm copying the approach from this known to work program and the byte count I'm getting from Python doesn't resemble anything close to the MenuMeter numbers as seen above. I am using a slightly different sysctl (NET_RT_IFLIST2) which gives 64 bit integers which I've taken into account in my code.

Here's my code:

from ctypes import *
from ctypes.util import find_library
from ctypes import sizeof as c_sizeof
import errno

libc = CDLL(find_library('c'), use_errno=True)

# hardcoded macosx constants!
CTL_NET = 4
PF_ROUTE = 17
NET_RT_IFLIST2 = 6

RTM_IFINFO2 = 0x12
IFF_LOOPBACK = 0x8
AF_LINK = 18

class if_data64(Structure):
   _pack_ = 4
   _fields_ = [('ifi_type', c_ubyte),
               ('ifi_typelen', c_ubyte),
               ('ifi_physical', c_ubyte),
               ('ifi_addrlen', c_ubyte),
               ('ifi_hdrlen', c_ubyte),
               ('ifi_recvquota', c_ubyte),
               ('ifi_xmitquota', c_ubyte),
               ('ifi_unused1', c_ubyte),
               ('ifi_mtu', c_uint32),
               ('ifi_metric', c_uint32),
               ('ifi_baudrate', c_uint32),
               ('ifi_ipackets', c_uint64),
               ('ifi_ierrors', c_uint64),
               ('ifi_opackets', c_uint64),
               ('ifi_oerrors', c_uint64),
               ('ifi_collisions', c_uint64),
               ('ifi_ibytes', c_uint64),
               ('ifi_obytes', c_uint64),
               ('ifi_imcasts', c_uint64),
               ('ifi_omcasts', c_uint64),
               ('ifi_iqdrops', c_uint64),
               ('ifi_noproto', c_uint64),
               ('ifi_recvtiming', c_uint32),
               ('ifi_xmittiming', c_uint32)]

class if_msghdr2(Structure):
   _fields_ = [('ifm_msglen', c_ushort),
               ('ifm_version', c_ubyte),
               ('ifm_type', c_ubyte),
               ('ifm_addrs', c_int),
               ('ifm_flags', c_int),
               ('ifm_index', c_ushort),
               ('ifm_snd_len', c_int),
               ('ifm_snd_maxlen', c_int),
               ('ifm_snd_drops', c_int),
               ('ifm_timer', c_int),
               ('ifm_data', if_data64)]

class sockaddr_dl(Structure):
   _fields_ = [('sdl_len', c_ubyte),
               ('sdl_family', c_ubyte),
               ('sdl_index', c_ushort),
               ('sdl_type', c_ubyte),
               ('sdl_nlen', c_ubyte),
               ('sdl_alen', c_ubyte),
               ('sdl_slen', c_ubyte),
               ('sdl_data', c_char * 12)] # for now

MIB_TYPE = c_int * 6
mib = MIB_TYPE(CTL_NET, PF_ROUTE, 0, 0, NET_RT_IFLIST2, 0)

sysctl_buf_len = c_uint(0)

rval = libc.sysctl(mib, 6, None, byref(sysctl_buf_len), None, 0)
if rval != 0:
   raise Exception(errno.errorcode[get_errno()])

sysctl_buf = create_string_buffer(sysctl_buf_len.value)
rval = libc.sysctl(mib, 6, sysctl_buf, byref(sysctl_buf_len), None, 0)
if rval != 0:
   raise Exception(errno.errorcode[get_errno()])

# walk the structure.  you need to know the length from ifm_msglen
idx = addressof(sysctl_buf)
end = idx + sysctl_buf_len.value
while idx < end:
   batch_off = idx - addressof(sysctl_buf)
   ifmsg = cast(c_void_p(idx), POINTER(if_msghdr2))
   if ifmsg.contents.ifm_type != RTM_IFINFO2:
      idx += ifmsg.contents.ifm_msglen
      continue
   if ifmsg.contents.ifm_flags & IFF_LOOPBACK:
      idx += ifmsg.contents.ifm_msglen
      continue
   # 12 bytes to compensate for 32 bit alignment
   sdl = cast(c_void_p(idx + c_sizeof(if_msghdr2) + 12), POINTER(sockaddr_dl))
   if sdl.contents.sdl_family != AF_LINK:
      idx += ifmsg.contents.ifm_msglen
      continue

   print sdl.contents.sdl_data[0:sdl.contents.sdl_nlen]
   print ifmsg.contents.ifm_data.ifi_ibytes, ifmsg.contents.ifm_data.ifi_obytes
   idx += ifmsg.contents.ifm_msglen

Answer 1

In if_data64, the ifi_baudrate field should be c_uint64 instead of c_uint32. Fixing this should align ifi_bytes and ifo_bytes to the correct offset. It's also missing the last field, ifi_lastchange. This is an 8 byte timeval32 struct. All together, this explains why sdl needed an extra 12 byte offset.

class timeval32(Structure):
    _fields_ = [('tv_sec', c_int32), 
                ('tv_usec', c_int32)]

class if_data64(Structure):
    _pack_ = 4
    _fields_ = [('ifi_type', c_ubyte),
                ('ifi_typelen', c_ubyte),
                ('ifi_physical', c_ubyte),
                ('ifi_addrlen', c_ubyte),
                ('ifi_hdrlen', c_ubyte),
                ('ifi_recvquota', c_ubyte),
                ('ifi_xmitquota', c_ubyte),
                ('ifi_unused1', c_ubyte),
                ('ifi_mtu', c_uint32),
                ('ifi_metric', c_uint32),
                ('ifi_baudrate', c_uint64),    # was c_uint32
                ('ifi_ipackets', c_uint64),
                ('ifi_ierrors', c_uint64),
                ('ifi_opackets', c_uint64),
                ('ifi_oerrors', c_uint64),
                ('ifi_collisions', c_uint64),
                ('ifi_ibytes', c_uint64),
                ('ifi_obytes', c_uint64),
                ('ifi_imcasts', c_uint64),
                ('ifi_omcasts', c_uint64),
                ('ifi_iqdrops', c_uint64),
                ('ifi_noproto', c_uint64),
                ('ifi_recvtiming', c_uint32),
                ('ifi_xmittiming', c_uint32),
                ('ifi_lastchange', timeval32)] # was missing

Instead of using cast with pointers, I'd write the loop using the from_buffer method (Python 2.6+) and offsets.

buf_offset = 0
while buf_offset < sysctl_buf_len.value:
   msg_offset = buf_offset
   ifmsg = if_msghdr2.from_buffer(sysctl_buf, msg_offset)
   buf_offset += ifmsg.ifm_msglen
   if ifmsg.ifm_type != RTM_IFINFO2 or ifmsg.ifm_flags & IFF_LOOPBACK:
      continue
   sdl_offset = msg_offset + c_sizeof(if_msghdr2)
   sdl = sockaddr_dl.from_buffer(sysctl_buf, sdl_offset)
   if sdl.sdl_family != AF_LINK:
      continue
   print sdl.sdl_data[:sdl.sdl_nlen]
   print ifmsg.ifm_data.ifi_ibytes, ifmsg.ifm_data.ifi_obytes