Search code examples
javaspringspring-batch

Spring batch - processing a subset of multiple items at once


I have a list of numerous items that I split into chunks of 10 each.
But the itemProcessor only processes 1 by 1 item in a chunk.

How can I aggregate all items in each chunk into a list and then make a call (contains that list) to the external API because I don't want 1 API request per 1 item?

@Bean
public Job job(JobBuilderFactory jobs, StepBuilderFactory steps) {
    return jobs.get("job")
            .start(steps.get("step")
                    .<Item, Item>chunk(10)
                    .reader(itemReader())
                    .processor(itemProcessor())
                    .writer(itemWriter())
                    .build())
            .build();
}

Solution

  • Solved it.
    I reconfigure the ItemReader to return the "chunk" that needs to be processed

    Job.java

    @Bean
    public Job job(JobBuilderFactory jobs, StepBuilderFactory steps) {
        return jobs.get("job")
                .start(steps.get("step")
                        .<List<Item>, List<Item>chunk(1) // Use chunk size of 1, as the reader now returns a chunk
                        .reader(itemReader())
                        .processor(itemProcessor())
                        .writer(itemWriter())
                        .build())
                .build();
    }
    
    @Bean
    @StepScope
    public ItemReader<List<Item>> itemReader() {
            // Prepare the items from your data source ...
    
            int chunkSize = 10; // Set your desired chunk size here
            return new ChunkItemReader<>(items, chunkSize);
        }
    

    ChunkItemReader.java

    import org.springframework.batch.item.ItemReader;
    
    import java.util.List;
    
    public class ChunkItemReader<T> implements ItemReader<List<T>> {
    
        private final List<T> items;
        private final int chunkSize;
        private int currentIndex;
    
        public ChunkItemReader(List<T> items, int chunkSize) {
            this.items = items;
            this.chunkSize = chunkSize;
            this.currentIndex = 0;
        }
    
        @Override
        public List<T> read() {
            int endIndex = Math.min(currentIndex + chunkSize, items.size());
            if (currentIndex < endIndex) {
                List<T> chunk = items.subList(currentIndex, endIndex);
                currentIndex = endIndex;
                return chunk;
            }
            return null; // End of data
        }
    }