_ArrayMemoryError: Unable to allocate

Question

I am trying to process data but i constantly run on this Error: numpy.core._exceptions._ArrayMemoryError: Unable to allocate 8.00 GiB for an array with shape (32761, 32761) and data type float64 this is my code:

import numpy as np
import csv
import tkinter as tk
from tkinter import filedialog

"""
Importing data 
"""

root= tk.Tk()

canvas1 = tk.Canvas(root, width = 300, height = 300, bg = 'gray1', relief = 'raised')
canvas1.pack()

def getCSV ():
    global df
    
    import_file_path = filedialog.askopenfilename()
    df = np.genfromtxt(import_file_path, delimiter=' ')
    
    print (df)
    
browseButton_CSV = tk.Button(text="      Import CSV File     ", command=getCSV, bg='OrangeRed4', fg='black', font=('helvetica', 12, 'bold'))
canvas1.create_window(150, 150, window=browseButton_CSV)

root.mainloop()

x, y, mag = df[:,0], df[:,1], df[:,3]
Xshape, Yshape, MAGshape = np.shape(x), np.shape(y), np.shape(mag)


def fftfreqs(x, y, shape, windowLen):
    """
    Get two 2D-arrays with wavenumbers [rads/km] in x, y directions.
    """
    nx = ny = shape[0]
    
    dx = (x.max() - x.min())/(nx - 1)           # Spacing
    fx = 2*np.pi*np.fft.fftfreq(windowLen[0], dx)
    
    dy = (y.max() - y.min())/(ny - 1)           # Spacing
    fy = 2*np.pi*np.fft.fftfreq(windowLen[1], dy)
    
    return np.meshgrid(fy, fx)[::-1]


"""
Calculation of power spectrum density
"""
shap = (np.shape(x)[0], np.shape(y)[0])
kx, ky = fftfreqs(x, y, shap, shap)
pds = (abs(np.fft.fft2(np.reshape(mag, (1,shap[0])))))**2

"""
Calculation of Radially Averaged Power Spectrum
"""

nx, ny = pds.shape

max_radius = min(kx.max(), ky.max())

ring_width = max(np.unique(kx)[np.unique(kx) > 0][0], np.unique(ky)[np.unique(ky) > 0][0])

k = np.sqrt(kx**2 + ky**2)
pds_radial = []
k_radial = []
radius_i = -1
while True:
    radius_i += 1
    if radius_i*ring_width > max_radius:
        break
    else:
        if radius_i == 0:
            inside = k <= 0.5*ring_width
        else:
            inside = np.logical_and(k > (radius_i - 0.5)*ring_width, k <= (radius_i + 0.5)*ring_width)
        pds_radial.append(pds[inside].mean())
        k_radial.append(radius_i*ring_width)

i am running it on a 8GB RAM SYSTEM but i tried also to run it on GOOGLE COLAB but with the same result. Thanks in advance

Answer 1

I am far from an expert with fast-Fourier transforms, so I will not be able to tell you if what you are doing makes sense. However, I think your MemoryError comes from the fact that you are dealing with more than just one (32761, 32761) array in the same command. So, probably you are able to allocate the first one, but maybe not the second one; you get the idea. Have a look at what I am suggesting below and let me know if that improves things for you.

import numpy as np
import tkinter as tk
from tkinter import filedialog


def getCSV():
    global df
    import_file_path = filedialog.askopenfilename()
    df = np.genfromtxt(import_file_path, delimiter=' ')


def fftfreqs():
    """
    Get two 2D-arrays with wavenumbers [rads/km] in x, y directions.
    """
    nx = ny = df.shape[0]

    dx = (x.max() - x.min()) / (nx - 1)  # Spacing
    fx = 2 * np.pi * np.fft.fftfreq(nx, dx)

    dy = (y.max() - y.min()) / (ny - 1)  # Spacing
    fy = 2 * np.pi * np.fft.fftfreq(ny, dy)

    return np.meshgrid(fy, fx)[::-1]


"""
Importing data
"""

root = tk.Tk()

canvas1 = tk.Canvas(root, width=300, height=300, bg='gray1', relief='raised')
canvas1.pack()

browseButton_CSV = tk.Button(text="      Import CSV File     ", command=getCSV,
                             bg='OrangeRed4', fg='black',
                             font=('helvetica', 12, 'bold'))
canvas1.create_window(150, 150, window=browseButton_CSV)

root.mainloop()

x, y, mag = df[:, 0], df[:, 1], df[:, 3]

"""
Calculation of power spectrum density
"""
kx, ky = fftfreqs()
pds = np.fft.fft2(np.reshape(mag, (1, df.shape[0]))) ** 2

"""
Calculation of Radially Averaged Power Spectrum
"""

max_radius = min(kx.max(), ky.max())
print(kx.shape)  # (32761, 32761)
print(ky.shape)  # (32761, 32761)
# Do not do this
ring_width = max(np.unique(kx)[np.unique(kx) > 0]
                 [0], np.unique(ky)[np.unique(ky) > 0][0])
# Do something like this instead
kx_unique = np.unique(kx)
# Process kx_unique
del kx_unique
ky_unique = np.unique(ky)
# Process ky_unique
del ky_unique
ring_width = max(, )

k = np.sqrt(kx**2 + ky**2)
pds_radial = []
k_radial = []
radius_i = -1
while True:
    radius_i += 1
    if radius_i * ring_width > max_radius:
        break
    else:
        if radius_i == 0:
            inside = k <= 0.5 * ring_width
        else:
            inside = np.logical_and(
                k > (radius_i - 0.5) * ring_width,
                k <= (radius_i + 0.5) * ring_width)
        pds_radial.append(pds[inside].mean())
        k_radial.append(radius_i * ring_width)