How to find the largest objects in memory?

Question

I want to have a table of 10 largest objects in memory, with size.

Equivalent function in R: Tricks to manage the available memory in an R session

lsos()
#                          Type Size(MB)   Rows Columns
#d2                  data.table 157.8364 281444      74
#d                   data.table  62.2658 816078      11

Edit: @ 9.0 Here is my attempt.

I have to use globals(), using gc.get_objects() makes my computer very slow. I am not sure globals() gives me what I want. How to get the list of all initialized objects and function definitions alive in python?

def lsos(n=10):

    import pandas as pd
    import sys

    all_obj =globals()

    object_name = list(all_obj).copy()
    object_size = [sys.getsizeof(all_obj[x]) for x in object_name]

    d = pd.DataFrame(dict(name = object_name, size = object_size))
    d.sort_values(['size'], ascending=[0],inplace=True)

    return(d.head(n))


v  = list(range(1000))
v2 = list(range(10000))
v3 = list(range(100000))
v4 = v3
lsos()

    # name    size
# 0     v4  900112
# 22    v3  900112
# 1     v2   90112
# 17     v    9112
# 6     _i     395
# 14   _i1     395
# 19   _oh     288
# 24   Out     288
# 5    _i2     137
# 3   lsos     136

When I put the above function in say abc.py and run

import abc
abc.lsos()

           # name  size
# 8  __builtins__  6240
# 0          lsos   136
# 6      __file__   123
# 2    __cached__    97
# 1    __loader__    56
# 4      __spec__    56
# 5      __name__    54
# 3   __package__    49
# 7       __doc__    16

None of the large v appears.

Edit 2: Because there's problem of accessing globals() in a module, I just pass globals() to the module, here is what I am using now:

#abc.py
def lsos(all_obj = globals(),n=10):

    import sys

    object_name = list(all_obj)
    object_size = [ round(sys.getsizeof(all_obj[x])/1024.0/1024.0,4) for x in object_name]
    object_id = [id(all_obj[x]) for x in object_name]

    d = [(a,b,c) for a,b,c in zip(object_name, object_size, object_id)]
    d.sort(key = lambda x:(x[1],x[2]), reverse=True)
    dprint = d[0:min(len(d), n)]

    #print formating
    name_width_max = max([len(x[0]) for x in dprint])
    print(("{:<" + str(name_width_max +2) + "}{:11}{}").format("name","size_Mb","id"))
    fmt = '{{:<{}}}'.format(name_width_max+2) +"  "+ "{: 5.4f}" +"  "+ "{:d}"
    for line in dprint:
        print( fmt.format(*line))

    return

Then it can be called by

import abc
abc.lsos(globals())

Answer 1

The simplest is to use Pympler:

from operator import itemgetter

from pympler import tracker

mem = tracker.SummaryTracker()
print(sorted(mem.create_summary(), reverse=True, key=itemgetter(2))[:10])

Output:

[["<class 'float", 5004102, 120098448],
 ["<class 'list", 74279, 48527480],
 ["<class 'str", 214166, 23782488],
 ["<class 'dict", 14710, 7109016],
 ["<class 'code", 27702, 3991737],
 ["<class 'type", 3480, 3714520],
 ["<class 'jedi.parser.python.tree.Operator", 24936, 2393856],
 ["<class 'jedi.parser.python.tree.Name", 19965, 1916640],
 ["<class 'jedi.parser.python.tree.PythonNode", 23550, 1884000],
 ["<class 'int", 47671, 1382592]]

Of course, you can also create a pandas dataframe and work with this:

memory = pd.DataFrame(mem.create_summary(), columns=['object', 'number_of_objects', 'memory'])
memory['mem_per_object'] = memory['memory'] / memory['number_of_objects']
print(memory.sort_values('memory', ascending=False).head(10))
print(memory.sort_values('mem_per_object', ascending=False).head(10))

Output:

                                           object  number_of_objects     memory  mem_per_object
11                                  <class 'float            5004112  120098688       24.000000
42                                   <class 'list              74322   48532112      652.997928
2                                     <class 'str             214308   23797202      111.042061
44                                   <class 'dict              14738    7116184      482.845976
10                                   <class 'code              27702    3991737      144.095625
59                                   <class 'type               3480    3715616     1067.705747
9421     <class 'jedi.parser.python.tree.Operator              24928    2393088       96.000000
9422         <class 'jedi.parser.python.tree.Name              19962    1916352       96.000000
9420   <class 'jedi.parser.python.tree.PythonNode              23544    1883520       80.000000
10637           <class 'pandas.core.series.Series                102    1721291    16875.401961


                                                  object  number_of_objects       memory  mem_per_object
237                           <class '_io.BufferedWriter                  3   393744   131248.000000
11518                <class 'pandas.core.frame.DataFrame                 24  1709443    71226.791667
12358            <class 'matplotlib.colors._ColorMapping                  1    36984    36984.000000
8946   <class 'pytz.lazy.LazySet.__new__.<locals>.Laz...                  2    66000    33000.000000
10637                  <class 'pandas.core.series.Series                102  1721291    16875.401961
235                           <class '_io.BufferedReader                  1    16560    16560.000000
11599     <class 'pandas.core.indexes.numeric.Int64Index                 11   129184    11744.000000
12719          <class 'matplotlib._cm._deprecation_datad                  2     9440     4720.000000
8945   <class 'pytz.lazy.LazyList.__new__.<locals>.La...                  2     9248     4624.000000
1594                               <class 'random.Random                  1     2560     2560.000000