Python - plotting large number of lines

Question

I am trying to read in a file containing XY endpoints of line segments and a value associated with the segment, then plot the line segments colored by the value given. The problem I am having is that there is potentially hundreds of thousands to millions of line segments and when I attempt to read in these larger files I run into a memory error. Is there a more memory efficient way of doing this?

import matplotlib.pyplot as plt
import matplotlib.colors as colors
import matplotlib.cm as cmx
import sys
import csv

if len(sys.argv) > 1:
    flofile = sys.argv[1]
else:
    flofile = "GU3\GU3.flo"

fig = plt.figure()
ax = fig.add_subplot(111)
jet = cm = plt.get_cmap('jet')
cNorm = colors.Normalize(vmin=0)
scalarMap = cmx.ScalarMappable(norm=cNorm,cmap=jet)
with open(flofile) as FLO:
    title = FLO.readline()
    limits = [float(tp) for tp in FLO.readline().split()]
    FLO.readline()#headers
    for line in FLO:
        if 'WELLS' in line: break        
        frac = ([float(tp) for tp in line.split()])
        ax.plot([frac[0],frac[2]],[frac[1],frac[3]],color=colorVal)


#ax.plot(*call_list)
scalarMap._A = []
plt.colorbar(scalarMap)
plt.xlim([0,limits[0]])
plt.ylim([0,limits[1]])

plt.show()

This code works for small files. Thanks.

Answer 1

I would look into LineCollection (doc).

import matplotlib
import matplotlib.pyplot as plt    
import random


s = (600,400)
N = 100000

segs = []
colors = []
my_cmap = plt.get_cmap('jet')
for i in range(N):
    x1 = random.random() * s[0]
    y1 = random.random() * s[1]
    x2 = random.random() * s[0]
    y2 = random.random() * s[1]
    c  = random.random()
    colors.append(my_cmap(c))
    segs.append(((x1, y1), (x2, y2)))

ln_coll = matplotlib.collections.LineCollection(segs, colors=colors)

ax = plt.gca()
ax.add_collection(ln_coll)
ax.set_xlim(0, 600)    
ax.set_ylim(0, 400)
plt.draw()
 

It wil also take a list of numpy arrays for the first arguement.

Answer 2

You might consider doing the plotting on a bitmap image first, which doesn't have the memory problem, and after that fine tune the plot/image with matplotlib. As an example:

from PIL import Image
from PIL import ImageDraw
import random
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

s = (500,500)
N = 100000

im = Image.new('RGBA', s, (255,255,255,255))
draw = ImageDraw.Draw(im)

for i in range(N):
    x1 = random.random() * s[0]
    y1 = random.random() * s[1]
    x2 = random.random() * s[0]
    y2 = random.random() * s[1]
    c  = random.random() * 256
    draw.line(((x1,y1),(x2,y2)), fill=(0, 255 - int(c), int(c), 255), width=1)

plt.imshow(np.asarray(im), extent=(-1,1,-1,1), aspect='equal', origin='lower')
plt.show()