How to make sure the MultiResourceItemReader refreshes the resources each time the job is executed

Question

I have a java application which makes use of spring task scheduling and batch jobs. I am relying on MultiResourceItemReader in the job to read some files from a directory, process them, and remove those files. An external process is responsible to put new files periodically in that directory. But the problem is that each time the job runs, it tries to read the same file resources which were present at the time of launching the application and hence fails because those resources are gone and new files are there now.

The question is, how do I configure the application so that the resources property is evaluated for every scheduled execution of the given job.

The relevant beans are pasted below:

  <bean id="multiResourceReader" class="org.springframework.batch.item.file.MultiResourceItemReader">
    <property name="resources" value="file:/opt/data/*.csv" />
    <property name="delegate" ref="testFlatFileItemReader" />
  </bean>

  <batch:job id="MyJob">
    <batch:step id="readandstore">
      <batch:tasklet>
        <batch:chunk reader="multiResourceReader" writer="oracleItemWriter" commit-interval="10" />
      </batch:tasklet>
    </batch:step>
  </batch:job>

  <bean id="runScheduler" class="com.myapp.Scheduler">
    <property name="jobLauncher" ref="jobLauncher" />
    <property name="job" ref="MyJob" />
  </bean>

  <task:scheduled-tasks>
    <task:scheduled ref="runScheduler" method="run" cron="*/30 * * * * *" />
  </task:scheduled-tasks>

Answer 1

I managed to solve the problem I posted myself after having received no response on it for almost a week.

I removed the resources property of multiResourceReader, and added a StepListener to the listeners under <batch:tasklet> which implements StepExecutionListener. This listener has two methods, one is called before the step execution and the other after the step execution. The StepListener accepts a property named filePattern the value for which will be similar to what you'd have provided to resources property of multiResourceReader before. Here is how the updated beans look like:

  <batch:job id="MyJob">
    <batch:step id="readandstore">
      <batch:tasklet>
        <batch:chunk reader="multiResourceReader" writer="oracleItemWriter" commit-interval="10" />
        <batch:listeners>
          <batch:listener ref="StepListener" />
        </batch:listeners>
      </batch:tasklet>
    </batch:step>
  </batch:job>

  <bean id="multiResourceReader" class="org.springframework.batch.item.file.MultiResourceItemReader">
    <property name="delegate" ref="csvFileItemReader" />
  </bean>

  <bean id="StepListener" class="com.myapp.StepListener">
    <property name="filePattern" value="file:/opt/data/*.csv" />
  </bean>

And here is how the com.myapp.StepListener looks like:

package com.myapp;

import java.io.File;
import java.io.IOException;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.ExitStatus;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.StepExecutionListener;
import org.apache.commons.io.FileUtils;
import org.springframework.batch.item.file.MultiResourceItemReader;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.core.io.Resource;

public class StepListener implements StepExecutionListener, ApplicationContextAware {

  private static final Logger logger = Logger.getLogger(StepListener.class.getName());
  private Resource[] resources;
  private ApplicationContext applicationContext;
  private String filePattern;

  @Override
  public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
    this.applicationContext = applicationContext;
  }

  public void setFilePattern(String filePattern) {
    this.filePattern = filePattern;
  }

  @Override
  public void beforeStep(StepExecution stepExecution) {

    MultiResourceItemReader reader = (MultiResourceItemReader) applicationContext.getBean("multiResourceReader");
    try {
      resources = applicationContext.getResources(filePattern);
      reader.setResources(resources);
    } catch (IOException ex) {
      logger.log(Level.SEVERE, "Unable to set file resources to bean multiResourceItemReader", ex);
    }
  }

  @Override
  public ExitStatus afterStep(StepExecution stepExecution) {

    if (stepExecution.getExitStatus().equals(ExitStatus.COMPLETED)
            && stepExecution.getStatus().equals(BatchStatus.COMPLETED)
            && resources.length > 0) {

      for (Resource resource : resources) {
        try {
          File oldFile = new File(resource.getFile().getAbsolutePath());
          File newFile = new File(resource.getFile().getAbsolutePath() + ".processed");
          FileUtils.copyFile(oldFile, newFile);
          oldFile.delete();
        } catch (IOException ex) {
          logger.log(Level.SEVERE, "Encountered problem when trying to remove the processed file(s)", ex);
        }
      }
    }

    return stepExecution.getExitStatus();
  }
}

Answer 2

The bean multiResourceReader's scope is singleton, so only 1 will every be created. When that bean gets created the value of resources is resolved.

If you set the scope of the bean multiResourceReader to step it should also fix your problem. Then a new bean will created every time the step is executed.