Python - single vs multiline REGEX

Question

Considering the following text pattern,

#goals: the process report timestamp, eg. 2011-09-21 15:45:00 and the first two stats in succ. statistics line, eg: 1438 1439

input_text = '''
# Process_Name     ( 23387) Report at 2011-09-21 15:45:00.001    Type:  Periodic    #\n
some line 1\n
some line 2\n
some other lines\n
succ. statistics |     1438     1439  99 |   3782245    3797376  99 |\n
some lines\n
Process_Name     ( 23387) Report at 2011-09-21 15:50:00.001    Type:  Periodic    #\n
some line 1\n
some line 2\n
some other lines\n
succ. statistics |     1436     1440  99 |   3782459    3797523  99 |\n
repeat the pattern several hundred times...
'''

I got it working when iterating line to line,

def parse_file(file_handler, patterns):

    results = []
    for line in file_handler:
        for key in patterns.iterkeys():
            result = re.match(patterns[key], line)
            if result:
                results.append( result )

return results

patterns = {
    'report_date_time': re.compile('^# Process_Name\s*\(\s*\d+\) Report at (.*)\.[0-9]   {3}\s+Type:\s*Periodic\s*#\s*.*$'),
    'serv_term_stats': re.compile('^succ. statistics \|\s+(\d+)\s+   (\d+)+\s+\d+\s+\|\s+\d+\s+\d+\s+\d+\s+\|\s*$'),
    }
results = parse_file(fh, patterns)

returning

[('2011-09-21 15:40:00',),
('1425', '1428'),
('2011-09-21 15:45:00',),
('1438', '1439')]

but having a list of tuples output as my goal,

[('2011-09-21 15:40:00','1425', '1428'),
('2011-09-21 15:45:00', '1438', '1439')]

I tried several combos with the initial patterns and a lazy quantifier between them but can't figure out how to capture the patterns using a multiline REGEX

# .+?   Lazy quantifier "match as few characters as possible (all characters allowed) until reaching the next expression"
pattern = '# Process_Name\s*\(\s*\d+\) Report at (.*)\.[0-9]{3}\s+Type:\s*Periodic.*?succ. statistics) \|\s+(\d+)\s+(\d+)+\s+\d+\s+\|\s+\d+\s+\d+\s+\d+\s+\|\s'
regex = re.compile(pattern, flags=re.MULTILINE)

data = file_handler.read()    
for match in regex.finditer(data):
    results = match.groups()

How can I accomplish this ?

Answer 1

Use re.DOTALL so . will match any character, including newlines:

import re

data = '''
# Process_Name     ( 23387) Report at 2011-09-21 15:45:00.001    Type:  Periodic    #\n
some line 1\n
some line 2\n
some other lines\n
succ. statistics |     1438     1439  99 |   3782245    3797376  99 |\n
some lines\n
repeat the pattern several hundred times...
'''

pattern = r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).*?succ. statistics\s+\|\s+(\d+)\s+(\d+)'
regex = re.compile(pattern, flags=re.MULTILINE|re.DOTALL)

for match in regex.finditer(data):
    results = match.groups()
    print(results)

    # ('2011-09-21', '1438', '1439')