How do I put a task back in the queue if the task fails?

Question

I have a script that looks something like this:

#!/usr/bin/env python
# encoding: utf-8

import time, random, os, multiprocessing

def main():
    NPROCESSES = 5
    pool = multiprocessing.Pool(processes=NPROCESSES)

    a = [1,2,3,4,5,6,7,8,9,0]
    for _ in pool.imap_unordered(do_task, a):
        pass

def do_task(n):
    try:
        might_crash(n)
    except Hell, e:
        print e, " crashed."

def might_crash(n):
    time.sleep(3*random.random())
    if random.randrange( 3 ) == 0:
        raise Hell(n)
    print n

class Hell(Exception):
    pass  

if __name__=="__main__":    
    main()

This script will normally print the values from 'a', but might_crash() will randomly raise an exception.

I want to catch these exceptions and put the current do_task() back in the queue to retry later.

How do I put the current task back in the queue if it should fail?

Answer 1

You could collect results from do_task, check which results are instances of Hell, stuff those tasks into a list new_tasks, and loop until there are no new_tasks:

import time
import random
import os
import multiprocessing as mp

def main():
    NPROCESSES = 5
    pool=mp.Pool(NPROCESSES)
    a = [1,2,3,4,5,6,7,8,9,0]
    new_tasks=a
    while new_tasks:
        a=new_tasks
        new_tasks=[]
        for result in pool.imap_unordered(do_task, a):
            if isinstance(result,Hell):
                new_tasks.append(result.args[0])
            else:
                print(result)

def do_task(n):
    try:
        result=might_crash(n)
    except Hell as e:        
        print("{0} crashed.".format(e.args[0]))
        result=e
    return result

def might_crash(n):
    time.sleep(3*random.random())
    if random.randrange( 3 ) == 0:
        raise Hell(n)
    return '{0} done'.format(n)

class Hell(Exception):
    pass  

if __name__=="__main__":    
    main()

yields

1 done
6 crashed.
4 done
7 crashed.
5 done
9 done
3 done
2 crashed.
8 done
0 crashed.
0 crashed.
2 done
7 crashed.
6 done
0 done
7 done