Spring批处理3600分区在启动前长时间暂停

Question

下面的spring批处理作业带有一个分区步骤，它为一个分区步骤创建3600个分区。我使用的ThreadPoolTaskExecutor的最大池大小为100，队列容量为100（尽管这似乎对速度没有什么影响）。Im使用Visual VM监视线程，我注意到taskExecutor线程在启动作业后超过5分钟才启动。

奇怪的是，如果我将分区的数量限制为100，那么线程启动得相当快，大约在一分钟内完成。

我注意到的另一个问题是，在VisualVM线程可视化中，似乎没有一个以上的数据库连接

请查看下面的批处理作业并告诉我是否丢失了将数据库连接数限制为1的内容？另外，如果我的ThreadPoolTaskExecutor参数不变，为什么添加更多分区会影响性能？难道这些作业不应该一直排在队列中直到有一个线程可以为它们服务吗？

---Spring批处理作业---

<beans xmlns="http://www.springframework.org/schema/beans" xmlns:batch="http://www.springframework.org/schema/batch"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xmlns:context="http://www.springframework.org/schema/context"
   xmlns:util="http://www.springframework.org/schema/util"
   xmlns:lang="http://www.springframework.org/schema/lang"
   xsi:schemaLocation="http://www.springframework.org/schema/batch http://www.springframework.org/schema/batch/spring-batch-3.0.xsd
    http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.0.xsd
    http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-4.0.xsd
    http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-3.0.xsd
    http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-4.0.xsd">


<context:property-placeholder location="${ext.properties.dataManipulation.Properties}"/>

<import resource="${ext.properties.dataManipulation.Connection}"/>

<import resource="flatFileLineProperties.xml"/>

<!-- JobRepository and JobLauncher are configuration/setup classes -->
<bean id="jobRepository"
      class="org.springframework.batch.core.repository.support.MapJobRepositoryFactoryBean"/>

<bean id="transactionManager"
      class="org.springframework.batch.support.transaction.ResourcelessTransactionManager"/>

<bean id="jobLauncher"
      class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
    <property name="jobRepository" ref="jobRepository"/>
</bean>



<util:map id="keys" map-class="java.util.HashMap">
    <entry key="SAILING_ID" value="ASCENDING" value-type="org.springframework.batch.item.database.Order"/>
    <entry key="RES_ID" value="ASCENDING" value-type="org.springframework.batch.item.database.Order" />
</util:map>

<!-- Here is my partioned step -->
<bean id="reservationsItemReader" class="org.springframework.batch.item.database.JdbcPagingItemReader" scope="step">
    <property name="dataSource" ref="dataSource" />
    <property name="queryProvider">
        <bean class="org.springframework.batch.item.database.support.SqlPagingQueryProviderFactoryBean">
            <property name="dataSource" ref="dataSource" />
            <property name="selectClause">
                <value>
                    <![CDATA[ 
                    GUEST_ID,
                    FIRST_NAME,
                    LAST_NAME,
                    TITLE,
                    HOUSEHOLD_NAME,
                    SAILING_ID,
                    RES_ID
                ]]>
                </value>
            </property>
            <property name="fromClause" value="FROM RESERVATION "/>
            <property name="whereClause" >
                <value>
                    <![CDATA[ AND SAIL_ID = :sailId
                    ]]>
                </value>
            </property>
            <!--<property name="sortKey" value="SAILING_ID" />-->
            <property name="sortKeys" ref="keys"/>
       <!--<property name="sortKeys" ref="sortKeys"/>-->
        </bean>
    </property>
    <property name="parameterValues">
        <map>
            <!--<entry key="shipCode" value="#{stepExecutionContext[shipCode]}" />-->
            <entry key="sailId" value="#{stepExecutionContext[sailId]}" />
        </map>
    </property>
    <!--property name="pageSize" value="500000" /-->

    <property name="pageSize" value="40000" />
    <property name="rowMapper">
        <bean class="com.ncl.endeca.mapper.ColumnToHashMapper" />
    </property>
</bean>

<bean id="sortKeys" class="java.util.HashMap" scope="prototype" >
    <constructor-arg>
        <map key-type="java.lang.String" value-type="org.springframework.batch.item.database.Order">
            <entry key="SAILING_ID" value="ASCENDING" />
            <entry key="RES_ID" value="ASCENDING" />
        </map>
    </constructor-arg>
</bean>

<util:list id="client_fields" value-type="java.lang.String"> 
    <value>FIRST_NAME</value>
    <value>LAST_NAME</value>
    <value>TITLE</value>
    <value>HOUSEHOLD_NAME</value>

</util:list>

<bean id="reservationsItemWriter" class="com.ncl.endeca.writer.ReservationWriter" scope="step">
    <property name="guestFields" ref="client_fields" />
    <property name="outPrefix" value="${file.out.prefix}" />
    <property name="shipCode" value="#{stepExecutionContext[shipCode]}" />
    <property name="sailId" value="#{stepExecutionContext[sailId]}" />
    <property name="soldOutSailings" ref="soldOutSailingsList" />
</bean>

<bean id="yearsAgo" class="java.lang.Integer">
    <constructor-arg>
        <value>${yearsAgo}</value>
    </constructor-arg>
</bean>
<bean id="yearsAhead" class="java.lang.Integer">
    <constructor-arg>
        <value>${yearsAhead}</value>
    </constructor-arg>
</bean>

<bean id="resPartitioner" class="com.ncl.endeca.partition.ReservationPartitioner">
    <property name="yearsAgo" ref="yearsAgo" />
    <property name="yearsAhead" ref="yearsAhead" />
    <property name="batchLimit" value="${batch.limit}" />
    <property name="dataSource" ref="dataSource"/>
</bean>

<bean id="taskExecutor"
      class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
    <property name="corePoolSize" value="${batch.corePoolSize}" />
    <property name="maxPoolSize" value="${batch.maxPoolSize}" />
    <property name="queueCapacity" value="${batch.queueCapacity}" />
</bean>


<!--<bean id="taskExecutor" class="org.springframework.core.task.SimpleAsyncTaskExecutor"/>-->



<!-- each thread will run this job, with different stepExecutionContext values. -->
<step id="slave" xmlns="http://www.springframework.org/schema/batch" >
    <flow parent="readReservations"/>
</step>

<!--<bean id="countrySpecificCompletionPolicy" class="org.springframework.batch.core.resource.StepExecutionSimpleCompletionPolicy">-->
    <!--<property name="keyName" value="sailId"/>-->
<!--</bean>-->

<batch:flow id="readReservations">
    <batch:step id="reservations" xmlns="http://www.springframework.org/schema/batch" >
        <tasklet throttle-limit="${batch.corePoolSize}">
            <chunk reader="reservationsItemReader" writer="reservationsItemWriter" commit-interval="50000" />
        </tasklet>
    </batch:step>
</batch:flow>

<!-- Actual Job -->
<batch:job id="dataManipulationJob">

    <batch:step id="masterStep">
        <batch:partition step="slave" partitioner="resPartitioner">
            <batch:handler grid-size="100" task-executor="taskExecutor" />
        </batch:partition>
    </batch:step>
</batch:job>

我尝试了BasicDataSource和Hikari连接，但当我监视VisualVM时，池大小没有影响

----connection.xml----

<?xml version="1.0" encoding="UTF-8"?>

<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:jdbc="http://www.springframework.org/schema/jdbc"
   xmlns:batch="http://www.springframework.org/schema/batch"
   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xmlns:context="http://www.springframework.org/schema/context"
   xmlns:util="http://www.springframework.org/schema/util"
   xmlns:lang="http://www.springframework.org/schema/lang"
   xsi:schemaLocation="http://www.springframework.org/schema/batch http://www.springframework.org/schema/batch/spring-batch-3.0.xsd
    http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.0.xsd
    http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-4.0.xsd
    http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-3.0.xsd
    http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang-4.0.xsd">

<bean id="hikariConfig" class="com.zaxxer.hikari.HikariConfig">
    <property name="poolName" value="springHikariCP" />
    <property name="connectionTestQuery" value="SELECT 10 from dual" />
    <property name="dataSourceClassName" value="${hibernate.dataSourceClassName}" />
    <property name="maximumPoolSize" value="${batch.maxPoolSize}" />
    <property name="idleTimeout" value="${hibernate.hikari.idleTimeout}" />

    <property name="dataSourceProperties">
        <props>
            <prop key="url">${dataSource.url}</prop>
            <prop key="user">${dataSource.username}</prop>
            <prop key="password">${dataSource.password}</prop>
        </props>
    </property>
</bean>

<!-- HikariCP configuration -->
<!--<bean id="dataSource" class="com.zaxxer.hikari.HikariDataSource" destroy-method="close">-->
    <!--<constructor-arg ref="hikariConfig" />-->
<!--</bean>-->

<!--<bean id="dataSource" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close" scope="step">-->
    <!--<property name="driverClassName" value="${hibernate.dataSourceClassName}" />-->
    <!--<property name="url" value="${dataSource.url}" />-->
    <!--<property name="username" value="${dataSource.username}" />-->
    <!--<property name="password" value="${dataSource.password}" />-->
    <!--<property name="testWhileIdle" value="false"/>-->
    <!--<property name="maxActive" value="${batch.corePoolSize}"/>-->
<!--</bean>-->

    <!-- connect to database -->
<bean id="dataSource"
      class="org.springframework.jdbc.datasource.DriverManagerDataSource" >
    <property name="driverClassName" value="oracle.jdbc.OracleDriver" />
    <property name="url" value="jdbc:oracle:thin:@******" />
    <property name="username" value="****" />
    <property name="password" value="****" />
</bean>

Answer 1

创建分区的时间取决于ReservationPartitioner的性能以及分区的数量。创建3600个分区意味着创建3600个stepexecution/executioncontext对象并将它们持久化在相应的表中。对于如此多的分区，这可能需要时间。

关于数据库连接，您将MapJobRepositoryFactoryBean与ResourceLestransactionManager一起使用，因此没有与Spring批处理元数据的数据库交互。根据您的配置与数据库交互的唯一组件是JDBCPagingItemReader（我不知道您的ReservationWriter)类型，因此看到单个数据库连接并不奇怪。