Read nth line from file efficiently in Python

Question

I have a large txt that contains about 100.000.000 rows (I cannot read it to the memory as a whole). I would like to read n-th row efficiently. I found this How can I get python to read every nth line of a .txt file? and I constructed this function:

from itertools import islice

def read_n_line(file: str, n: int, encoding='utf-8') -> str:
    with open(file, encoding=encoding) as f:
        return next(islice(f, n - 1, n))

The problem is that my function is fast (0.5 seconds) for n = 1000, but slow (15 seconds) for n = 10.000.000. Can I somehow improve my function to be fast for all n, please?

Answer 1

For sufficiently large files, it may be more efficient to use a Numba-based approach:

import numba as nb


@nb.njit
def process(
    block,
    n,
    last_nl_pos,
    nl_pos,
    nl_count,
    offset,
    nl=ord("\n")
):
    nl = ord("\n")
    for i, c in enumerate(block, offset):
        if c == nl:
            found = True
            last_nl_pos = nl_pos
            nl_pos = i
            nl_count += 1
            if nl_count == n:
                break
    return last_nl_pos, nl_pos, nl_count


def read_nth_line_nb(
    filepath: str,
    n: int,
    encoding="utf-8",
    size=2 ** 22,  # 4 MiB
) -> str:
    with open(filepath, "rb") as file_obj:
        last_nl_pos = nl_pos = -1
        nl_count = -1
        offset = 0
        while True:
            block = file_obj.read(size)
            if block:
                last_nl_pos, nl_pos, nl_count = process(
                    block, n, last_nl_pos, nl_pos, nl_count, offset
                )
                offset += size
                if nl_count == n:
                    file_obj.seek(last_nl_pos + 1)
                    return file_obj.read(nl_pos - last_nl_pos).decode(encoding)
            else:
                return

This essentially processes the file in blocks, keeping tracks of how many and where the new lines are (and how far is the block on the file).

For comparison I use the itertools.islice() approach:

import itertools


def read_nth_line_isl(filepath: str, n: int, encoding="utf-8") -> str:
    with open(filepath, "r", encoding=encoding) as file_obj:
        return next(itertools.islice(file_obj, n, n + 1), None)

as well as the naïve looping:

def read_nth_line_loop(filepath: str, n: int, encoding="utf-8") -> str:
    with open(filepath, "r", encoding=encoding) as file_obj:
        for i, line in enumerate(file_obj):
            if i == n:
                return line
    return None

Assuming some files were created with the following:

import random
import string


def write_some_file(filepath: str, n: int, m: int = 10, l: int = 100, encoding="utf-8") -> None:
    with open(filepath, "w", encoding=encoding) as file_obj:
        for i in range(n):
            line = "".join(random.choices(string.ascii_letters, k=random.randrange(m, l)))
            file_obj.write(f"{i:0{k}d} - {line}\n")


k = 9
for i in range(1, k):
    n_max = 10 ** i
    print(n_max)
    write_some_file(f"test{n_max:0{k}d}.txt", n_max)

It is possible to test that they all give the same result:

funcs = read_nth_line_isl, read_nth_line_loop, read_nth_line_nb
k = 9
n_max = 10 ** 5
filename = f"test{n_max:0{k}d}.txt"
for func in funcs:
    print(f"{func.__name__:>20}  {func(filename, n_max - 1)!r}")
#    read_nth_line_isl  '000099999 - sWBnniKkpROZYlqfFLbSttEwYCjXfhQSapkkqxjePpGbobJzgaJTCOCSyHQEcLppZ\n'
#   read_nth_line_loop  '000099999 - sWBnniKkpROZYlqfFLbSttEwYCjXfhQSapkkqxjePpGbobJzgaJTCOCSyHQEcLppZ\n'
#     read_nth_line_nb  '000099999 - sWBnniKkpROZYlqfFLbSttEwYCjXfhQSapkkqxjePpGbobJzgaJTCOCSyHQEcLppZ\n'

The timings can be computed with:

k = 9
timings = {}
for i in range(1, k - 1):
    n_max = 10 ** i
    filename = f"test{n_max:0{k}d}.txt"
    print(filename)
    timings[i] = []
    base = funcs[0](filename, n_max - 1)
    for func in funcs:
        res = func(filename, n_max - 1)
        is_good = base == res
        if i < 6:
            timed = %timeit -r 12 -n 12 -q -o func(filename, n_max - 1)
        else:
            timed = %timeit -r 1 -n 1 -q -o func(filename, n_max - 1)
        timing = timed.best * 1e3
        timings[i].append(timing if is_good else None)
        print(f"{func.__name__:>24}  {is_good!s:5}  {timing:10.3f} ms")

and plotted with:

import pandas as pd
import matplotlib.pyplot as plt


df = pd.DataFrame(data=timings, index=[func.__name__ for func in funcs]).transpose()
df.plot(marker='o', xlabel='log₁₀(Input Size) / #', ylabel='Best timing / µs', figsize=(6, 4), logy=True)
fig = plt.gcf()
fig.patch.set_facecolor('white')

to obtain:

Indicating the Numba-based approach to be marginally faster (some 5-15%) for sufficiently large inputs (above 10⁵).