How to read a CSV file in reverse order in Python?

Question

I know how to do it for a TXT file, but now I am having some trouble doing it for a CSV file.

How can I read a CSV file from the bottom in Python?

Answer 1

Pretty much the same way as for a text file: read the whole thing into a list and then go backwards:

import csv
with open('test.csv', 'r') as textfile:
    for row in reversed(list(csv.reader(textfile))):
        print ', '.join(row)

If you want to get fancy, you could write a lot of code that reads blocks starting at the end of the file and working backwards, emitting a line at a time, and then feed that to csv.reader, but that will only work with a file that can be seeked, i.e. disk files but not standard input.

---

Some of us have files that do not fit into memory, could anyone come with a solution that does not require storing the entire file in memory?

That's a bit trickier. Luckily, all csv.reader expects is an iterator-like object that returns a string (line) per call to next(). So we grab the technique Darius Bacon presented in "Most efficient way to search the last x lines of a file in python" to read the lines of a file backwards, without having to pull in the whole file:

import os

def reversed_lines(file):
    "Generate the lines of file in reverse order."
    part = ''
    for block in reversed_blocks(file):
        for c in reversed(block):
            if c == '\n' and part:
                yield part[::-1]
                part = ''
            part += c
    if part: yield part[::-1]

def reversed_blocks(file, blocksize=4096):
    "Generate blocks of file's contents in reverse order."
    file.seek(0, os.SEEK_END)
    here = file.tell()
    while 0 < here:
        delta = min(blocksize, here)
        here -= delta
        file.seek(here, os.SEEK_SET)
        yield file.read(delta)

and feed reversed_lines into the code to reverse the lines before they get to csv.reader, removing the need for reversed and list:

import csv
with open('test.csv', 'r') as textfile:
    for row in csv.reader(reversed_lines(textfile)):
        print ', '.join(row)

There is a more Pythonic solution possible, which doesn't require a character-by-character reversal of the block in memory (hint: just get a list of indices where there are line ends in the block, reverse it, and use it to slice the block), and uses chain out of itertools to glue the line clusters from successive blocks together, but that's left as an exercise for the reader.

---

It's worth noting that the reversed_lines() idiom above only works if the columns in the CSV file don't contain newlines.

Aargh! There's always something. Luckily, it's not too bad to fix this:

def reversed_lines(file):
    "Generate the lines of file in reverse order."
    part = ''
    quoting = False
    for block in reversed_blocks(file):
        for c in reversed(block):
            if c == '"':
                quoting = not quoting
            elif c == '\n' and part and not quoting:
                yield part[::-1]
                part = ''
            part += c
    if part: yield part[::-1]

Of course, you'll need to change the quote character if your CSV dialect doesn't use ".

Answer 2

Building on @mike-desimone 's answer. Here's a solution that provides the same structure as a python file object but is read in reverse, line by line:

import os

class ReversedFile(object):
    def __init__(self, f, mode='r'):
        """
        Wraps a file object with methods that make it be read in reverse line-by-line

        if ``f`` is a filename opens a new file object

        """
        if mode != 'r':
            raise ValueError("ReversedFile only supports read mode (mode='r')")

        if not type(f) == file:
            # likely a filename
            f = open(f)

        self.file = f
        self.lines = self._reversed_lines()

    def _reversed_lines(self):
        "Generate the lines of file in reverse order."
        part = ''
        for block in self._reversed_blocks():
            for c in reversed(block):
                if c == '\n' and part:
                    yield part[::-1]
                    part = ''
                part += c
        if part: yield part[::-1]

    def _reversed_blocks(self, blocksize=4096):
        "Generate blocks of file's contents in reverse order."
        file = self.file

        file.seek(0, os.SEEK_END)
        here = file.tell()
        while 0 < here:
            delta = min(blocksize, here)
            here -= delta
            file.seek(here, os.SEEK_SET)
            yield file.read(delta)


    def __getattribute__(self, name):
        """ 
        Allows for the underlying file attributes to come through

        """ 
        try:
            # ReversedFile attribute
            return super(ReversedFile, self).__getattribute__(name)
        except AttributeError:
            # self.file attribute
            return getattr(self.file, name)

    def __iter__(self):
        """ 
        Creates iterator

        """ 
        return self

    def seek(self):
        raise NotImplementedError('ReversedFile does not support seek')

    def next(self):
        """
        Next item in the sequence

        """
        return self.lines.next()

    def read(self):
        """
        Returns the entire contents of the file reversed line by line

        """
        contents = ''

        for line in self:
            contents += line

        return contents

    def readline(self):
        """
        Returns the next line from the bottom

        """
        return self.next()

    def readlines(self):
        """
        Returns all remaining lines from the bottom of the file in reverse

        """
        return [x for x in self]