Counting frequency of words in documents using python regex

Question

Created a python module which reads in a file, removes the stop words and outputs a python dictionary with the word and its frequency (How many times it occurred in the document).

def run():
filelist = os.listdir(path)
regex = re.compile(r'.*<div class="body">(.*?)</div>.*', re.DOTALL | re.IGNORECASE)
reg1 = re.compile(r'<\/?[ap][^>]*>', re.DOTALL | re.IGNORECASE)
quotereg = re.compile(r'&quot;', re.DOTALL | re.IGNORECASE)
puncreg = re.compile(r'[^\w]', re.DOTALL | re.IGNORECASE)
f = open(stopwordfile, 'r')
stopwords = f.read().lower().split()
totalfreq = {}

filewords = {}
htmlfiles = []
for file in filelist:
    if file[-5:] == '.html':
        htmlfiles.append(file)

for file in htmlfiles:
    f = open(path + file, 'r')
    words = f.read().lower()
    words = regex.findall(words)[0]
    words = quotereg.sub(' ', words)
    words = reg1.sub(' ', words)
    words = puncreg.sub(' ', words)
    words = words.strip().split()

    for w in stopwords:
        while w in words:
            words.remove(w)

     freq = {}
    for w in words:
       if w in freq:
           totalfreq[w] = totalfreq[w] + 1
           freq[w] = freq[w] + 1
       else:
           totalfreq[w] = 1
           freq[w] = 1
           filewords[file] = freq
    
  
    print totalfreq

This prints all the 'non-stop' words within that file and the frequency in which they occur in the file: The output looks like:

{{'saturday': 1, 'irish': 1, 'family': 1, 'give': 1, 'year': 2, 'weekend': 1, 'steve': 1, 'guests': 1, 'questions': 1, 'in': 2, 'effort': 1, 'partner': 1, 'extinction': 1, 'dress': 1, 'children': 4, 'utans': 1, '27': 1, 'raise': 1, 'closet': 1, 'haired': 2, 'make': 1, 'humphreys': 1, 'relatives': 1, 'zoo': 5, 'endangered': 1, 'sunday': 1, 'special': 1, 'answer': 1, 'public': 1, 'awareness': 1, 'planned': 1, 'activities': 1, 'rhiona': 1, 'orangutans': 4, 'plans': 1, 'leonie': 1, 'orang': 1, 'yesterday': 2, 'free': 2, 'hand': 1, 'wild': 1, 'independent': 1, 'part': 1, 'preparing': 1, 'revealed': 1, 'day': 1, 'man': 1, 'picture': 1, 'keane': 1, 'animals': 1, '14': 1, 'kevin': 1, '16': 1, '32': 1, 'age': 1, 'sibu': 1, 'dublin': 2, 'keepers': 1, 'face': 1, 'mujur': 1, 'red': 2, 'orangutan': 1, 'species': 1, 'entry': 1, 'efforts': 1, 'shows': 1, '11am': 1, 'influx': 1, '3pm': 1}

{'newest': 1, 'birth': 2, 'orang': 1, 'month': 1, 'steve': 1, 'questions': 1, 'utans': 1, 'children': 4, 'staff': 1, 'limelight': 1, '27': 1, 'based': 1, 'concerned': 1, 'sunday': 1, '3pm': 1, 'finally': 1, '4': 1, 'maeve': 1, 'awareness': 1, 'gave': 1, 'activities': 1, 'giraffe': 1, 'facebook': 1, 'preparing': 1, 'background': 1, 'nurturing': 1, 'day': 1, 'debut': 1, 'rothschild': 1, 'keepers': 1, 'email': 1, 'steps': 1, '11am': 1, 'page': 1, 'picture': 1, 'born': 1, 'result': 1, 'year': 2, 'saturday': 1, 'special': 1, 'closet': 1, 'haired': 2, 'section': 1, 'bennet': 2, 'mum': 3, 'mujur': 1, 'conditions': 1, 'public': 1, 'red': 2, 'shows': 1, 'orangutans': 4, 'free': 2, 'keeper': 1, 'november': 1, 'care': 1, 'sending': 1, 'great': 1, 'origins': 1, '32': 1, 'invited': 1, 'dublin': 2, 'planned': 1, 'orangutan': 1, 'efforts': 1, 'influx': 1, 'named': 1, 'family': 1, 'delighted': 1, 'weather': 1, 'guests': 1, 'extinction': 1, 'post': 1, 'impressed': 1, 'raise': 1, 'revealed': 1, 'remained': 1, 'humphreys': 1, 'confident': 1, 'calf': 3, 'entrance': 1, 'shane': 1, 'part': 1, 'helen': 1, 'attentive': 1, 'effort': 1, 'case': 1, 'made': 2, 'animals': 1, '14': 1, '16': 1, 'ms': 1, 'wild': 1, 'savanna': 1, 'irish': 1, 'give': 1, 'resident': 1, 'suggestions': 1, 'slip': 1, 'in': 2, 'partner': 1, 'dress': 1, 'species': 1, 'kevin': 1, 'rhiona': 1, 'make': 1, 'zoo': 3, 'endangered': 1, 'relatives': 1, 'answer': 1, 'poor': 1, 'independent': 1, 'plans': 1, 'leonie': 1, 'time': 1, 'yesterday': 1, 'hand': 1, 'hickey': 1, 'weekend': 1, 'man': 1, 'sibu': 1, 'age': 1, 'steady': 2, 'face': 1, 'confinement': 1, 'african': 2, 'entry': 1, 'keane': 1, 'clarke': 2, 'left': 1}

But I need both totals to be added together from both files or numerous amounts of files to give a total count of the word eg "zoo" in all files. 1st file zoo=5 2nd file zoo=3 total =8.

I can't seem to work out how I count the words for many files rather than just one at a time.

Any ideas?!

Answer 1

The backslash in '<\/?[ap][^>]*>' is useless because '/' isn't a special character

'[^\w]' is '\W' By the way '[^\w]+' will be more efficient than just one '[^\w]'

re.DOTALL is useless with r'<\/?[ap][^>]*>' since there is no dot in this RE

If you do words = f.read().lower() to lower the letters, you don't need re.IGNORECASE

REs for substitution can be put in one RE: reg123 = re.compile(r'(</?[ap][^>]*>|&quot;|\W+)')

file is not a good name for a file's name, it override the name of an existing built-in function

replacing the lines of code to obtain htmfiles by a generator expression is better

I don't understand why '[0]' in words = regex.findall(words)[0]

You can also group the words of stopwords in the RE used to replace with ' ' :

stopwords = '|'.join(f.read().lower().split())

to be included in the RE for substitution

The indentation of filewords[file] = freq is bad

.

I propose you the following improvement; I didn't test it, 'cause I ain't the files to treat. It certainly isn't perfect. Ask for unclear points.

def run():

    from collection import difaultdict

    with open(stopwordfile, 'r') as f:
        stopwords = '|'.join(f.read().lower().split())

    regex = re.compile(r'.*<div class="body">(.*?)</div>.*', re.DOTALL)
    reg123 = re.compile(r'(</?[ap][^>]*>|&quot;|\W+|'+stopwords+')')

    totalfreq = defaultdict(int)
    filewords = {}

    for filename in (fn for fn in os.listdir(path) if fn[-5:] == '.html'):
        with open(path + filename, 'r') as f:
            ch = regex.findall(f.read().lower())[0]
            ch = reg123.sub(' ', ch)
            words = ch.strip().split()

        freq = defaultdict(int)
        for w in words:
            totalfreq[w] += 1
            freq[w] += 1
        filewords[filename] = freq

    print totalfreq

I didn't understand very well your question. Please give precisions