Memory allocated to Python is not released back in Linux even after gc.collect()

Question

I have written code within Python that doesn't release memory the way it should. The memory is taken by Python but never gets released even after not being used anymore. Even if you break the running program with ctrl+c. Delete the variable and run gc.collect() it doesn't seem to collect. Or the same as in Ipython and running %reset. The memory won't be freed and running gc.collect() has no effect. I tested this in Windows because I wanted to see if it could possibly be with the garbage collector library. It appears that is the case. Run the code below in Linux and then also in windows. Then compare the memory usage. You will need numpy and scipy installed. Any help or insight on this issue would be much appreciated.

Import the Model, create an instance, and then run createSpecific().

Here is a code that exhibits this behavior in Ubuntu 10.04:

from numpy import array, maximum,intersect1d, meshgrid, std, log, log10, zeros, ones, argwhere, abs, arange, size, copy, sqrt, sin, cos, pi, vstack, hstack, zeros, exp, max, mean, savetxt, loadtxt,  minimum,  linspace,  where
from numpy.fft import fft
from scipy.stats import f_oneway, kruskal, sem, scoreatpercentile
#import matplotlib
#matplotlib.use('cairo.pdf')
from matplotlib.pyplot import plot, clf, show, cla, xlim, xscale, imshow, ylabel, xlabel, figure, savefig, close,  bar,  title,  xticks, yticks, axes, axis
from matplotlib.axes import Axes
from mpl_toolkits.mplot3d import Axes3D
#from enthought.mayavi import mlab
from matplotlib import cm
import matplotlib.pyplot as plt
import os
from time import clock
from timeit import Timer
class Model:

#Constructors and default includes
    def __init__(self, prevAud = None,  debug=False):

        if (prevAud == None):
            self.fs=16000. #sample rate
            self.lowFreq=60. 
            self.hiFreq=5000.     
            self.numFilt=300 #number of channel
            self.EarQ = 9.26449   #9.26449
            self.minBW = 24.7     #24.7
            self.integrationWindow=.01
            self.sliceAt=.035
            self.maxOverallInhibit = 0.1
            self.winLen = int(self.fs*self.integrationWindow+.01) #default integration window 10 ms
            self.fullWind = 0.300
            self.outShortWindow = None
            self.siderArray = None
            self.maxNormalizeValue = .284     # Optimized at .284
            self.outputSemiModel = None
            self.semitones = 11
            self.activationTrace = None
        return




    def setErbScale(self, erbScale = None):
        if (erbScale ==None):
            self.erbScale = arange(100,500,5)
        else:
            self.erbScale = erbScale        

    def trainModel(self,soundVec=None, fs=None, lowfreq=None, highfreq=None, numfilt=None, figto=0, savefig = 'N', prompts=False, plotter=False):
        self.setErbScale()
        templateArray = self.intWindow(self.halfWaveRec(self.creGammatone(soundVec))) 
        for i in xrange(templateArray[0].size):        
            self.outerTest(self.innerTest(templateArray[:,i]))

        return templateArray   


    def createSpecific(self, freqArray = None, semitones = 11, timeforHarm = .3, soundVec=None, fs=None, lowfreq=None, highfreq=None, numfilt=None, figto=0, saveData='N', fileDir='TempRunT/', prompts=False, plotter=False):
        if (freqArray == None):
            self.setErbScale()
            freqArray = self.erbScale
        if (type(semitones) == int):
            semitones = arange(semitones+1)
        totalRuns = int(timeforHarm/self.integrationWindow+.001)
        inhibitWindowArray = zeros((freqArray.size,(semitones.size),self.numFilt,totalRuns))
        for x in xrange(freqArray.size):
            tempHarm = self.makeHarmonicAmpMod(freqArray[x],timeforHarm, numHarm=7,modulation=10)
            for y in semitones:
                tempChord = self.makeSemiChordAmpMod(tempHarm, freqArray[x],timeforHarm,modulation=10,numHarm=7,semi=y)
                inhibitWindowArray[x,y] = self.trainModel( tempChord, savefig = 'N', plotter=plotter)


        self.inhibitWindowArray = inhibitWindowArray

    def creGammatone(self, soundVec):

        temp = zeros((300,soundVec.size))
        for i in xrange(temp[:,0].size):
            temp[i] = -1**i*soundVec
        return temp

    def halfWaveRec(self, halfWaveFilts):

        filtShape = halfWaveFilts.shape
        if (filtShape[1] != int(self.fs*self.fullWind)):
            halfWaveFilts = hstack((halfWaveFilts,zeros((self.numFilt,int(self.fs*self.fullWind)-filtShape[1]))))
        temp = zeros((halfWaveFilts[:,0].size,halfWaveFilts[0].size))
        halfWaveFilts = maximum(halfWaveFilts,temp)

        del temp                
        return halfWaveFilts

    def intWindow(self, integratedFilts):
        winlen = self.winLen

        length = integratedFilts[0].size/winlen
        mod = integratedFilts[0].size%winlen
        outShortWindow = zeros((integratedFilts[:,0].size,length))
        meanval = 0

        if (mod != 0):
            for i in xrange(integratedFilts[:,0].size):
                mean(integratedFilts[i,0:-mod].reshape(length,winlen),1,out=outShortWindow[i])
        else:
            for i in xrange(integratedFilts[:,0].size):
                mean(integratedFilts[i].reshape(length,winlen),1,out=outShortWindow[i])
        del integratedFilts
        return outShortWindow    

    def innerTest(self, window):
        temper = copy(window)
        sider = 7
        st = .04
        sizer = temper.size
        inhibVal = 0
        for j in xrange(sider):
            inhibVal = (temper[0:j+sider+1].sum())*(sider*2+1)/(sider+1+j)
            window[j] += - st*(inhibVal)
        for j in xrange(sider,sizer - sider):
            inhibVal = temper[j-sider:j+sider+1].sum()
            window[j] += - st*(inhibVal)
        for j in xrange(sizer-sider, sizer):
            inhibVal = (temper[j-sider:sizer].sum())*(sider*2+1)/(sider+sizer-j)
            window[j] += - st*(inhibVal)

        maxsub = max(window) * self.maxOverallInhibit
        window += - maxsub    
        del temper
        return window

    def outerTest(self, window):
        newSatValue = scoreatpercentile(window, (76))
        numones = where(window > newSatValue)
        window[numones]=1
        self.maxSatValue = newSatValue
        del numones
        return window

    def makeHarmonicAmpMod(self, freq = 100, time = 1.,modulation=10, fsamp=None, numHarm=7):
        if fsamp == None: fsamp = self.fs
        samples = arange(time*fsamp)
        signal = 0
        for x in xrange(1,(numHarm+1),1):
            signal = signal + sin(samples/float(fsamp)*x*freq*2*pi)
        signal = (signal)*maximum(zeros(time*fsamp),sin((samples/float(fsamp)*modulation*2*pi)))
        return signal

    def makeSemiChordAmpMod(self, harmVec = None, freq=100, time = 1.,  modulation=10, fsamp=None, numHarm=7, semi = 2):
        if (harmVec == None): harmVec = self.makeHarmonicAmpMod(freq,time,modulation,fsamp,numHarm)
        if (semi == 0): return harmVec
        return harmVec + self.makeHarmonicAmpMod(freq*(2**(semi/12.)),time,modulation,fsamp,numHarm)

Answer 1

Virtual memory is not a scarce resource. There is no need to return it to the system since each process has its own address space. What is your actual problem? What issue is this behavior causing you?