python script gets killed by oom killer

Question

Here is a code sample of a 'blackjack-simulator' program,giving some percentages,also plotting them. I included the three main parts everything builds upon,you can see the data structure. I believe the rest of the code is irrelevant to my question. The script just runs fine. Unless I put the ranges too high (how many Shoes to be dealt, how many times) After about one hour of running (the RAM gets eaten by 3MB/sec),the process gets killed. I tried to test with pympler, and the built-in sys module,but I cannot discover any reason why it actually eats up the memory. Any help/suggestion would be appreciated.

import datetime
from random import choice
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.backends.backend_pdf import PdfPages

class Rules(object):

    """a dictionary of dictionaries describing the drawing rules in certain situations""" 

    def __init__(self):
        self.rules = {  2:{"5":"H", "6":"H", "7":"H", "8":"H", "9":"H", "10":"D", "11":"D", "12":"H", "13":"S", "14":"S", "15":"S", "16":"S", "17":"S", "18":"S", "19":"S", "20":"S", "21":"S", "A2":"H", "A3":"H", "A4":"H", "A5":"H", "A6":"H", "A7":"S", "A8":"S", "A9":"S", "A10":"S","D2":"SP", "D3":"SP", "D4":"H", "D5":"D","D6":"SP", "D7":"SP", "D8":"SP", "D9":"SP", "D10":"S", "D11":"SP" },
                        3:{"5":"H", "6":"H", "7":"H", "8":"H", "9":"D", "10":"D", "11":"D", "12":"H", "13":"S", "14":"S", "15":"S", "16":"S", "17":"S", "18":"S", "19":"S", "20":"S", "21":"S", "A2":"H", "A3":"H", "A4":"H", "A5":"H", "A6":"D", "A7":"D", "A8":"S", "A9":"S", "A10":"S","D2":"SP", "D3":"SP", "D4":"H", "D5":"D","D6":"SP", "D7":"SP", "D8":"SP", "D9":"SP", "D10":"S", "D11":"SP" },
                        4:{"5":"H", "6":"H", "7":"H", "8":"H", "9":"D", "10":"D", "11":"D", "12":"S", "13":"S", "14":"S", "15":"S", "16":"S", "17":"S", "18":"S", "19":"S", "20":"S", "21":"S", "A2":"H", "A3":"H", "A4":"D", "A5":"D", "A6":"D", "A7":"D", "A8":"S", "A9":"S", "A10":"S","D2":"SP", "D3":"SP", "D4":"H", "D5":"D","D6":"SP", "D7":"SP", "D8":"SP", "D9":"SP", "D10":"S", "D11":"SP" },
                        5:{"5":"H", "6":"H", "7":"H", "8":"H", "9":"D", "10":"D", "11":"D", "12":"S", "13":"S", "14":"S", "15":"S", "16":"S", "17":"S", "18":"S", "19":"S", "20":"S", "21":"S", "A2":"D", "A3":"D", "A4":"D", "A5":"D", "A6":"D", "A7":"D", "A8":"S", "A9":"S", "A10":"S","D2":"SP", "D3":"SP", "D4":"SP", "D5":"D","D6":"SP", "D7":"SP", "D8":"SP", "D9":"SP", "D10":"S", "D11":"SP" },
                        6:{"5":"H", "6":"H", "7":"H", "8":"H", "9":"D", "10":"D", "11":"D", "12":"S", "13":"S", "14":"S", "15":"S", "16":"S", "17":"S", "18":"S", "19":"S", "20":"S", "21":"S", "A2":"D", "A3":"D", "A4":"D", "A5":"D", "A6":"D", "A7":"D", "A8":"S", "A9":"S", "A10":"S","D2":"SP", "D3":"SP", "D4":"SP", "D5":"D","D6":"SP", "D7":"SP", "D8":"SP", "D9":"SP", "D10":"S", "D11":"SP" },
                        7:{"5":"H", "6":"H", "7":"H", "8":"H", "9":"H", "10":"D", "11":"D", "12":"H", "13":"H", "14":"H", "15":"H", "16":"H", "17":"S", "18":"S", "19":"S", "20":"S", "21":"S", "A2":"H", "A3":"H", "A4":"H", "A5":"H", "A6":"H", "A7":"S", "A8":"S", "A9":"S", "A10":"S","D2":"SP", "D3":"SP", "D4":"H", "D5":"D","D6":"H", "D7":"SP", "D8":"SP", "D9":"S", "D10":"S", "D11":"SP" },
                        8:{"5":"H", "6":"H", "7":"H", "8":"H", "9":"H", "10":"D", "11":"D", "12":"H", "13":"H", "14":"H", "15":"H", "16":"H", "17":"S", "18":"S", "19":"S", "20":"S", "21":"S", "A2":"H", "A3":"H", "A4":"H", "A5":"H", "A6":"H", "A7":"S", "A8":"S", "A9":"S", "A10":"S","D2":"H", "D3":"H", "D4":"H", "D5":"D","D6":"H", "D7":"H", "D8":"SP", "D9":"SP", "D10":"S", "D11":"SP" },
                        9:{"5":"H", "6":"H", "7":"H", "8":"H", "9":"H", "10":"D", "11":"D", "12":"H", "13":"H", "14":"H", "15":"H", "16":"H", "17":"S", "18":"S", "19":"S", "20":"S", "21":"S", "A2":"H", "A3":"H", "A4":"H", "A5":"H", "A6":"H", "A7":"H", "A8":"S", "A9":"S", "A10":"S","D2":"H", "D3":"H", "D4":"H", "D5":"D","D6":"H", "D7":"H", "D8":"SP", "D9":"SP", "D10":"S", "D11":"SP" },
                        10:{"5":"H", "6":"H", "7":"H", "8":"H", "9":"H", "10":"H", "11":"H", "12":"H", "13":"H", "14":"H", "15":"H", "16":"H", "17":"S", "18":"S", "19":"S", "20":"S", "21":"S", "A2":"H", "A3":"H", "A4":"H", "A5":"H", "A6":"H", "A7":"H", "A8":"S", "A9":"S", "A10":"S","D2":"H", "D3":"H", "D4":"H", "D5":"H","D6":"H", "D7":"H", "D8":"H", "D9":"S", "D10":"S", "D11":"SP" },
                        11:{"5":"H", "6":"H", "7":"H", "8":"H", "9":"H", "10":"H", "11":"H", "12":"H", "13":"H", "14":"H", "15":"H", "16":"H", "17":"S", "18":"S", "19":"S", "20":"S", "21":"S", "A2":"H", "A3":"H", "A4":"H", "A5":"H", "A6":"H", "A7":"H", "A8":"S", "A9":"S", "A10":"S","D2":"H", "D3":"H", "D4":"H", "D5":"H","D6":"H", "D7":"H", "D8":"H", "D9":"S", "D10":"S", "D11":"H" }

                        }


    def lookup(self, bank_val, player_str): #!!!! bank_val > int , player_str > str 
        return self.rules[bank_val][player_str]


class Deck(object):

    """4*13 cards"""

    def __init__(self):
        self.cards = [2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10, 11]
        self.onedeck = []
        for decks in range(0, 4):
            self.onedeck[len(self.onedeck):]=self.cards


class Shoe(Deck):

    """this shoe contains 6 decks (312 cards)
        --todo: set number of decks as a parameter"""

    def __init__(self):

        deck = Deck()
        self.handcounter = 0
        self.decks=[]
        for i in range (0, 6):
            self.decks[len(self.decks):]=deck.onedeck


class Money(object):

    """acts as our wallet,doesnt change after we start a new shoe. works with a single parameter,the start amount of money"""

    def __init__(self, money):
        self.money = money
        self.moneylist = []
        self.handlist = []  

    def display(self):
        return self.money

    def lose(self, m, shoe):
        self.money = self.money - m
        self.moneylist.append(self.money)
        self.handlist.append(shoe.handcounter)

    def win(self, m, shoe):
        self.money = self.money + m
        self.moneylist.append(self.money)
        self.handlist.append(shoe.handcounter)

    def zero(self):
        self.money = 0


def money_eval(dic_player, dic_bank, shoe):

    """decides who wins,and how much in every situation, once the hand finished"""

    bank_val=sum(dic_bank["bank"])
    #current = shoe.money
    for key, value in dic_player.iteritems():
        for i in range(len(value)):
            shoe.handcounter += 1
            if value[i].count("D")<1:
                multi =1
            else:
                value[i].pop()
                multi = 2
            if len(value[i])==2 and sum(value[i])==21:
                multi=1.5
            if multi==1 and sum(value[i])>21:
                money.lose(multi, shoe)
                #print "lost ", multi," ", sum(value[i]),  " vs ",  bank_val
            elif multi==1 and sum(value[i])<=21 and bank_val > 21:
                money.win(multi, shoe)
                #print "won ", multi," ", sum(value[i]),  " vs ",  bank_val
            elif multi==1 and sum(value[i])<=21 and bank_val <= 21 and sum(value[i])>bank_val:
                money.win(multi, shoe)
                #print "won ", multi," ", sum(value[i]),  " vs ",  bank_val
            elif multi==1 and len(dic_bank["bank"])>2 and sum(value[i])<=21 and bank_val <= 21 and sum(value[i])<bank_val:
                money.lose(multi, shoe)
                #print "lost ", multi," ", sum(value[i]),  " vs ",  bank_val
            elif multi==1.5 and len(dic_bank["bank"])>2:
                money.win(multi, shoe)
                #print "BJ won ", multi," ", sum(value[i]),  " vs ",  bank_val
            elif multi==1 and len(dic_bank["bank"])==2 and bank_val==21:
                money.lose(multi, shoe)
                #print "lost against BJ", multi," ", sum(value[i]),  " vs ",  bank_val
            else:
                pass
                #print "draw!"
    return

def lngth(shoe):

    """how many cards are still in the current shoe"""

    return len(shoe.decks)

def deal_next_card(shoe, list):

    """the random process where a card is selected from the remaining ones,and put in a certain list"""

    card=choice(shoe.decks)
    list.append(card)
    shoe.decks.remove(card)

def construct_player_str(list):

    """translates the players hand into readable categories for Rules.lookup"""

    if list[0]==list[1]:
        return "D"+str(list[0])
    elif list[0]==11 or list[1]==11:
        return "A"+str(list[0]+list[1]-11)
    else:
        return str(list[0]+list[1])

def deal_a_hand(shoe,  playerboxes=1):

    """the process of dealing a single hand"""

    dic_player = { "hand" + str(i ) : [] for i in range(int(playerboxes)) }
    dic_bank = { "bank" : []}
    for key in dic_player.keys():                                                       #deal first cards
        tmp_list1=[]
        deal_next_card(shoe,tmp_list1)
        dic_player[key] =[tmp_list1]
    deal_next_card(shoe, dic_bank["bank"])                                    #deal to bank
    for key in dic_player.keys():                                                       #deal second cards
        deal_next_card(shoe,dic_player[key][0])

    for key in dic_player.keys():
        review_player_options(dic_player, key, shoe,dic_bank)                                   #all splitting and doubling first
    draw_bank_cards(dic_bank, shoe)
    money_eval(dic_player, dic_bank, shoe)
    shoe.handcounter=shoe.handcounter+1
    return

def split(dic_player,key, shoe, dic_bank):
    for tosplit in dic_player[key]:
        rule = rules.lookup(dic_bank["bank"][0], construct_player_str(tosplit))
        if tosplit[0]==tosplit[1] and rule == "SP":
            tmp0=[tosplit[0]]
            tmp1=[tosplit[1]]
            deal_next_card(shoe, tmp0)
            deal_next_card(shoe, tmp1)
            #print "splitting:", tmp0, tmp1
            dic_player[key].remove(tosplit)
            dic_player[key].append(tmp0)
            dic_player[key].append(tmp1)
    return 1

def has_splittable(dic_player, key, dic_bank):
    for hands in dic_player[key]:
        if hands[0]==hands[1] and rules.lookup(dic_bank["bank"][0], construct_player_str(hands)) == "SP":
            return 1
    return 0

def has_doubleable(dic_player, key, dic_bank):
    for hands in dic_player[key]:
        if  hands.count("D")<1 and rules.lookup(dic_bank["bank"][0], construct_player_str(hands)) == "D":
            return 1
    return 0

def double(dic_player, key, shoe, dic_bank):
    for todbl in dic_player[key]:
        if len(todbl)<3 and rules.lookup(dic_bank["bank"][0], construct_player_str(todbl)) == "D":
            deal_next_card(shoe, todbl)
            todbl.append("D")
    return

def draw_player_cards(dic_player, key, shoe):

    """once all the splitting and doubling done,draws all the remaining cards to the unfinished hands of the player boxes,not forgetting the optional techniqe to count an Ace as 1. 0 at the end of the list is signalling the Bust."""

    for hands in dic_player[key]:
        if hands.count("D")<1:
            while sum(hands)<17 and hands.count(0)<1:
                deal_next_card(shoe, hands)
                if sum(hands)>21:
                    if hands.count(11)>0:
                        hands.remove(11)
                        hands.append(1)
                    else:
                        hands.append(0)

def draw_bank_cards(dic_bank, shoe):

    """similar to draw_player_cards"""

    while sum(dic_bank["bank"])<17 and dic_bank["bank"].count(0)<1:
        deal_next_card(shoe, dic_bank["bank"])
        if sum(dic_bank["bank"])>21:
            if dic_bank["bank"].count(11)>0:
                dic_bank["bank"].remove(11)
                dic_bank["bank"].append(1)
            else:
                dic_bank["bank"].append(0)

def review_player_options(dic_player, key, shoe, dic_bank):

    """first all the splits,next the doubles,and the rest"""

    while has_splittable(dic_player, key, dic_bank):
        split(dic_player, key, shoe, dic_bank)
    while has_doubleable(dic_player, key, dic_bank):
        #print "D D D", dic_player[key]
        double(dic_player, key, shoe, dic_bank)
    draw_player_cards(dic_player, key, shoe)


def compute():
    sum_money = []
    sum_handcounter = []
    Shoes= [ Shoe() for i in range(1000) ]                       # how many shoes
    for i in Shoes:
        if (Shoes.index(i)%100==0):
           print Shoes.index(i)
        while lngth(i)>52:
            deal_a_hand(i,  2)
            sum_money.append(money.display())
            sum_handcounter.append(len(sum_money))
    percents.append(-1*sum_money[-1]/sum_handcounter[-1]*100)
    money.zero()


percents = []
money = Money(0)
rules = Rules()
times=range(1000)
for i in times:
    print '<<starting the '+str(i+1)+'th set of shoes of '+str(len(times)-1)+'>>'
    compute()
print percents
dt = str(datetime.datetime.now())
datedfilename = 'bjresults_'+dt+'.pdf'
pdf = PdfPages(datedfilename)
plot = plt.hist(percents, 3000)
plt.savefig(pdf, format='pdf')
plt.close()
pdf.close()
#plot.savefig('lol.png')
#plt.show()

Answer 1

The reason your program is crashing is that you're not considering memory management at all. If you want to try a quick solution, you should not create a list of Shoe objects, maybe

for i in range(1000):
    Shoe = Shoe()
    #Do stuff

OR

for i in (Shoe() for s in range(1000)):
    #Do stuff

..which would create a generator of Shoe objects as they're called, and not pre allocate all the objects.

Your Shoe and Deck object interaction is horrible and unnecessary. You need to use the super() method. When you're done with an object (after you've appended its stats to a list) use the del() method to delete it manually; instead of relying on garbage collection.

At a glance your code is consuming so much memory because you're not using objects properly. Just because Python can syntactically handle your program doesn't mean it's running "correctly." When you're dealing with scale (in this case iterations of different decks of cards) you need to be more wary with how objects are create and memory is allocated. Instead of using so many class(object) structures, learn to use generators (yield) or functions as classes with inner methods that act as an array of functions on the reduce() method.

while ratio_generator():
    generators = [
        file_id_generator(),
        types_generator(),
        date_generator(),
        names_generator(),
        folder_generator(),
        content_generator(),
        id_chain_generator(),
    ]
    dynamics = [
        get_datetime,
        get_code_status,
    ]
    statics = [
        self.today,
        self.test_id,
        self.author
    ]

    yield flatten([x for x in statics], [x() for x in dynamics], [next(x) for x in generators])

Filter, Map, Reduce may be your new best friend instead of using so many static look ups.

Good luck.