问题：

spring批处理从命令行读取jobParameters并在作业配置中使用它

曹普松

2023-03-14

@SpringBootApplication
public class CoreApplication implements ApplicationRunner {

    @Autowired
    JobLauncher jobLauncher;

    @Autowired
    Job processJob;

    @Value("${rundate}")
    private String run_date;

    private static final Logger logger = LoggerFactory.getLogger(CoreApplication.class);

    public static void main(String[] args) {
        SpringApplication.run(CoreApplication.class, args);
    }

    @Override
    public void run(ApplicationArguments args) throws Exception {

        JobParameters jobParameters = new JobParametersBuilder()
                .addLong("JobID", System.currentTimeMillis())
                .addString("RunDate", run_date)
                .toJobParameters();

        try {
            jobLauncher.run(processJob, jobParameters);
        } catch (Exception e) {
            logger.error("Exception while running a batch job {}", e.getMessage());
        }

    }

}

spring.datasource.url=jdbc:postgresql://dbhost:1000/db
spring.datasource.username=username
spring.datasource.password=password
spring.datasource.platform=postgresql
spring.batch.job.enabled=false
local.directory="/my/local/path/"
file.name="file_name_20200601.csv"
remote.directory="/remote/ftp/location"
remote.host="remotehost"
remote.port=22
remote.user="remoteuser"
private.key.location="/key/file/location"

我的批处理配置：

@Configuration
@EnableBatchProcessing
@EnableIntegration
@EnableAutoConfiguration
public class BatchConfiguration {
    private Logger logger = LoggerFactory.getLogger(BatchConfiguration.class);

    @Autowired
    public JobBuilderFactory jobBuilderFactory;

    @Autowired
    public StepBuilderFactory stepBuilderFactory;

    @Bean
    public Job ftpJob() {
        return jobBuilderFactory.get("FTP Job")
                .incrementer(new RunIdIncrementer())
                .start(getFilesFromFTPServer())
                .build();
    }

    @Bean
    public Step getFilesFromFTPServer() {
        return stepBuilderFactory.get("Get file from server")
                .tasklet(new RemoteFileInboundTasklet())
                .build();

    }
}

我的任务：

公共类RemoteFileInboundTasklet实现Tasklet{

private Logger logger = LoggerFactory.getLogger(RemoteFileInboundTasklet.class);

@Value("${file.name}")
private String fileNamePattern;

private String clientName;
private boolean deleteLocalFiles = true;
private boolean retryIfNotFound = false;

@Value("${local.directory}")
private String local_directory_value;

private File localDirectory;
private int downloadFileAttempts = 12;
private long retryIntervalMilliseconds = 300000;

@Value("${remote.directory}")
private String remoteDirectory;

@Value("${remote.host}")
private String remoteHost;

@Value("${remote.user}")
private String remoteUser;

@Value("${remote.port}")
private int remotePort;

@Value("${private.key.location}")
private String private_key_file;

public SessionFactory<ChannelSftp.LsEntry> clientSessionFactory() {
    DefaultSftpSessionFactory ftpSessionFactory = new DefaultSftpSessionFactory();
    ftpSessionFactory.setHost(remoteHost);
    ftpSessionFactory.setPort(remotePort);
    ftpSessionFactory.setUser(remoteUser);
    ftpSessionFactory.setPrivateKey(new FileSystemResource(private_key_file));
    ftpSessionFactory.setAllowUnknownKeys(true);
    return ftpSessionFactory;
}

private SessionFactory sessionFactory = clientSessionFactory();

public SftpInboundFileSynchronizer sftpInboundFileSynchronizer() {
    SftpInboundFileSynchronizer sftpInboundFileSynchronizer = new SftpInboundFileSynchronizer(sessionFactory);
    sftpInboundFileSynchronizer.setDeleteRemoteFiles(false);
    sftpInboundFileSynchronizer.setRemoteDirectory(remoteDirectory);
    return sftpInboundFileSynchronizer;
}

private SftpInboundFileSynchronizer ftpInboundFileSynchronizer = sftpInboundFileSynchronizer();

private SftpInboundFileSynchronizingMessageSource sftpInboundFileSynchronizingMessageSource;

public boolean isDeleteLocalFiles() {
    return deleteLocalFiles;
}

public void setDeleteLocalFiles(boolean deleteLocalFiles) {
    this.deleteLocalFiles = deleteLocalFiles;
}

public SftpInboundFileSynchronizer getFtpInboundFileSynchronizer() {
    return ftpInboundFileSynchronizer;
}

public void setFtpInboundFileSynchronizer(SftpInboundFileSynchronizer ftpInboundFileSynchronizer) {
    this.ftpInboundFileSynchronizer = ftpInboundFileSynchronizer;
}

public SessionFactory getSessionFactory() {
    return sessionFactory;
}

public void setSessionFactory(SessionFactory sessionFactory) {
    this.sessionFactory = sessionFactory;
}

public SftpInboundFileSynchronizingMessageSource getSftpInboundFileSynchronizingMessageSource() {
    return sftpInboundFileSynchronizingMessageSource;
}

public void setSftpInboundFileSynchronizingMessageSource(SftpInboundFileSynchronizingMessageSource sftpInboundFileSynchronizingMessageSource) {
    this.sftpInboundFileSynchronizingMessageSource = sftpInboundFileSynchronizingMessageSource;
}

public String getRemoteDirectory() {
    return remoteDirectory;
}

public void setRemoteDirectory(String remoteDirectory) {
    this.remoteDirectory = remoteDirectory;
}

private SFTPGateway sftpGateway;


@ServiceActivator(inputChannel = "sftpChannel")
public MessageHandler clientMessageHandler() {
    SftpOutboundGateway sftpOutboundGateway = new SftpOutboundGateway(clientSessionFactory(), "mget", "payload");
    sftpOutboundGateway.setAutoCreateLocalDirectory(true);
    sftpOutboundGateway.setLocalDirectory(new File(local_directory_value));
    sftpOutboundGateway.setFileExistsMode(FileExistsMode.REPLACE_IF_MODIFIED);
    sftpOutboundGateway.setFilter(new AcceptOnceFileListFilter<>());
    return sftpOutboundGateway;
}

private void deleteLocalFiles()
{
    if (deleteLocalFiles)
    {
        localDirectory = new File(local_directory_value);
        SimplePatternFileListFilter filter = new SimplePatternFileListFilter(fileNamePattern);
        List<File> matchingFiles = filter.filterFiles(localDirectory.listFiles());
        if (CollectionUtils.isNotEmpty(matchingFiles))
        {
            for (File file : matchingFiles)
            {
                FileUtils.deleteQuietly(file);
            }
        }
    }
}

@Override
public RepeatStatus execute(StepContribution stepContribution, ChunkContext chunkContext) throws Exception {

    deleteLocalFiles();
    ftpInboundFileSynchronizer.synchronizeToLocalDirectory(localDirectory);
    if (retryIfNotFound) {

        SimplePatternFileListFilter filter = new SimplePatternFileListFilter(fileNamePattern);
        int attemptCount = 1;
        while (filter.filterFiles(localDirectory.listFiles()).size() == 0 && attemptCount <= downloadFileAttempts) {

            logger.info("File(s) matching " + fileNamePattern + " not found on remote site. Attempt " + attemptCount + " out of " + downloadFileAttempts);
            Thread.sleep(retryIntervalMilliseconds);
            ftpInboundFileSynchronizer.synchronizeToLocalDirectory(localDirectory);
            attemptCount++;
        }

        if (attemptCount >= downloadFileAttempts && filter.filterFiles(localDirectory.listFiles()).size() == 0) {
            throw new FileNotFoundException("Could not find remote file(s) matching " + fileNamePattern + " after " + downloadFileAttempts + " attempts.");
        }
    }
    return RepeatStatus.FINISHED;
}

public String getFileNamePattern() {
    return fileNamePattern;
}

public void setFileNamePattern(String fileNamePattern) {
    this.fileNamePattern = fileNamePattern;
}

public String getClientName() {
    return clientName;
}

public void setClientName(String clientName) {
    this.clientName = clientName;
}

public boolean isRetryIfNotFound() {
    return retryIfNotFound;
}

public void setRetryIfNotFound(boolean retryIfNotFound) {
    this.retryIfNotFound = retryIfNotFound;
}

public File getLocalDirectory() {
    return localDirectory;
}

public void setLocalDirectory(File localDirectory) {
    this.localDirectory = localDirectory;
}

public int getDownloadFileAttempts() {
    return downloadFileAttempts;
}

public void setDownloadFileAttempts(int downloadFileAttempts) {
    this.downloadFileAttempts = downloadFileAttempts;
}

public long getRetryIntervalMilliseconds() {
    return retryIntervalMilliseconds;
}

public void setRetryIntervalMilliseconds(long retryIntervalMilliseconds) {
    this.retryIntervalMilliseconds = retryIntervalMilliseconds;
}

mvn clean package

Caused by: org.springframework.beans.BeanInstantiationException: Failed to instantiate [org.springframework.batch.core.Step]: Factory method 'getFilesFromFTPServer' threw exception; nested exception is java.lang.IllegalArgumentException: Path must not be null
    at org.springframework.beans.factory.support.SimpleInstantiationStrategy.instantiate(SimpleInstantiationStrategy.java:185) ~[spring-beans-5.2.6.RELEASE.jar:5.2.6.RELEASE]
    at org.springframework.beans.factory.support.ConstructorResolver.instantiate(ConstructorResolver.java:651) ~[spring-beans-5.2.6.RELEASE.jar:5.2.6.RELEASE]
    ... 122 common frames omitted
Caused by: java.lang.IllegalArgumentException: Path must not be null
    at org.springframework.util.Assert.notNull(Assert.java:198) ~[spring-core-5.2.6.RELEASE.jar:5.2.6.RELEASE]
    at org.springframework.core.io.FileSystemResource.<init>(FileSystemResource.java:80) ~[spring-core-5.2.6.RELEASE.jar:5.2.6.RELEASE]
    at com.my.batch.core.tasklet.RemoteFileInboundTasklet.clientSessionFactory(RemoteFileInboundTasklet.java:78) ~[classes/:na]
    at com.my.batch.core.tasklet.RemoteFileInboundTasklet.<init>(RemoteFileInboundTasklet.java:83) ~[classes/:na]
    at com.my.batch.core.BatchConfiguration.getFilesFromFTPServer(BatchConfiguration.java:71) ~[classes/:na]
    at com.my.batch.core.BatchConfiguration$$EnhancerBySpringCGLIB$$17d8a6d9.CGLIB$getFilesFromFTPServer$1(<generated>) ~[classes/:na]

ftpSessionFactory.setPrivateKey(new FileSystemResource(private_key_file));

我尝试在配置中将我的tasklet声明为bean并重新构建包。然而，它却给出了同样的错误。

更改后的我的application.properties文件：

spring.datasource.url=jdbc:postgresql://dbhost:1000/db
spring.datasource.username=username
spring.datasource.password=password
spring.datasource.platform=postgresql
spring.batch.job.enabled=false
local.directory=/my/local/path/
file.name=file_name_20200601.csv
remote.directory=/remote/ftp/location
remote.host=remotehost
remote.port=22
remote.user=remoteuser
private.key.location=/key/file/location

任务没有改变。

@Configuration
@EnableBatchProcessing
@EnableIntegration
@EnableAutoConfiguration
public class BatchConfiguration {
    private Logger logger = LoggerFactory.getLogger(BatchConfiguration.class);

    @Autowired
    public JobBuilderFactory jobBuilderFactory;

    @Autowired
    public StepBuilderFactory stepBuilderFactory;

    @Bean
    public RemoteFileInboundTasklet remoteFileInboundTasklet() {
        return new RemoteFileInboundTasklet();
    }

    @Bean
    public Job ftpJob() {
        return jobBuilderFactory.get("FTP Job")
                .incrementer(new RunIdIncrementer())
                .start(getFilesFromFTPServer())
                .build();
    }

    @Bean
    public Step getFilesFromFTPServer() {
        return stepBuilderFactory.get("Get file from server")
                .tasklet(remoteFileInboundTasklet())
                .build();

    }
}

import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;

@ConfigurationProperties(prefix = "ftp")
@Configuration("coreFtpProperties")
public class CoreFtp {
    private String host;
    private String port;
    private String user;
    private String passwordKey;
    private String localDirectory;
    private String remoteDirectory;
    private String fileName;

    public String getHost() {
        return host;
    }

    public void setHost(String host) {
        this.host = host;
    }

    public String getPort() {
        return port;
    }

    public void setPort(String port) {
        this.port = port;
    }

    public String getUser() {
        return user;
    }

    public void setUser(String user) {
        this.user = user;
    }

    public String getPasswordKey() {
        return passwordKey;
    }

    public void setPasswordKey(String passwordKey) {
        this.passwordKey = passwordKey;
    }

    public String getLocalDirectory() {
        return localDirectory;
    }

    public void setLocalDirectory(String localDirectory) {
        this.localDirectory = localDirectory;
    }

    public String getRemoteDirectory() {
        return remoteDirectory;
    }

    public void setRemoteDirectory(String remoteDirectory) {
        this.remoteDirectory = remoteDirectory;
    }

    public String getFileName() {
        return fileName;
    }

    public void setFileName(String fileName) {
        this.fileName = fileName;
    }
}

spring.datasource.url=jdbc:postgresql://dbhost:1000/db
spring.datasource.username=username
spring.datasource.password=password
spring.datasource.platform=postgresql
spring.batch.job.enabled=false
ftp.local_directory=/my/local/path/
ftp.file_name=file_name_20200601.csv
ftp.remote_directory=/remote/ftp/location
ftp.host=remotehost
ftp.port=22
ftp.user=remoteuser
ftp.password_key=/key/file/location

@Configuration
@EnableBatchProcessing
@EnableIntegration
public class BatchConfiguration {
    private Logger logger = LoggerFactory.getLogger(BatchConfiguration.class);

    @Autowired
    public JobBuilderFactory jobBuilderFactory;

    @Autowired
    public StepBuilderFactory stepBuilderFactory;

    @Autowired
    private CoreFtp coreFtpProperties;


    @Bean
    public RemoteFileInboundTasklet remoteFileInboundTasklet() {
        RemoteFileInboundTasklet ftpTasklet = new RemoteFileInboundTasklet();
        ftpTasklet.setRetryIfNotFound(true);
        ftpTasklet.setDownloadFileAttempts(3);
        ftpTasklet.setRetryIntervalMilliseconds(10000);
        ftpTasklet.setFileNamePattern(coreFtpProperties.getFileName());
        ftpTasklet.setRemoteDirectory(coreFtpProperties.getRemoteDirectory());
        ftpTasklet.setLocalDirectory(new File(coreFtpProperties.getLocalDirectory()));
        ftpTasklet.setSessionFactory(clientSessionFactory());
        ftpTasklet.setFtpInboundFileSynchronizer(sftpInboundFileSynchronizer());
        ftpTasklet.setSftpInboundFileSynchronizingMessageSource(new SftpInboundFileSynchronizingMessageSource(sftpInboundFileSynchronizer()));

        return ftpTasklet;
    }

    @Bean
    public SftpInboundFileSynchronizer sftpInboundFileSynchronizer() {
        SftpInboundFileSynchronizer sftpInboundFileSynchronizer = new SftpInboundFileSynchronizer(clientSessionFactory());
        sftpInboundFileSynchronizer.setDeleteRemoteFiles(false);
        sftpInboundFileSynchronizer.setRemoteDirectory(coreFtpProperties.getRemoteDirectory());
        return sftpInboundFileSynchronizer;
    }

    @Bean(name = "clientSessionFactory")
    public SessionFactory<LsEntry> clientSessionFactory() {
        DefaultSftpSessionFactory ftpSessionFactory = new DefaultSftpSessionFactory();
        ftpSessionFactory.setHost(coreFtpProperties.getHost());
        ftpSessionFactory.setPort(Integer.parseInt(coreFtpProperties.getPort()));
        ftpSessionFactory.setUser(coreFtpProperties.getUser());
        ftpSessionFactory.setPrivateKey(new FileSystemResource(coreFtpProperties.getPasswordKey()));
        ftpSessionFactory.setPassword("");
        ftpSessionFactory.setAllowUnknownKeys(true);
        return ftpSessionFactory;
    }

    @Bean
    @ServiceActivator(inputChannel = "sftpChannel")
    public MessageHandler clientMessageHandler() {
        SftpOutboundGateway sftpOutboundGateway = new SftpOutboundGateway(clientSessionFactory(), "mget", "payload");
        sftpOutboundGateway.setAutoCreateLocalDirectory(true);
        sftpOutboundGateway.setLocalDirectory(new File(coreFtpProperties.getLocalDirectory()));
        sftpOutboundGateway.setFileExistsMode(FileExistsMode.REPLACE_IF_MODIFIED);
        sftpOutboundGateway.setFilter(new AcceptOnceFileListFilter<>());
        return sftpOutboundGateway;
    }


    @Bean
    public Job ftpJob() {
        return jobBuilderFactory.get("FTP Job")
                .incrementer(new RunIdIncrementer())
                .start(getFilesFromFTPServer())
                .build();
    }

    @Bean
    public Step getFilesFromFTPServer() {
        return stepBuilderFactory.get("Get file from server")
                .tasklet(remoteFileInboundTasklet())
                .build();

    }



}

因此，相应地，我的Tasklet更改为：

public class RemoteFileInboundTasklet implements Tasklet {

    private Logger logger = LoggerFactory.getLogger(RemoteFileInboundTasklet.class);

    private String fileNamePattern;

    private String clientName;
    private boolean deleteLocalFiles = true;
    private boolean retryIfNotFound = false;

    private File localDirectory;

    private int downloadFileAttempts = 12;
    private long retryIntervalMilliseconds = 300000;

    private String remoteDirectory;

    private SessionFactory sessionFactory;
    private SftpInboundFileSynchronizer ftpInboundFileSynchronizer;

    private SftpInboundFileSynchronizingMessageSource sftpInboundFileSynchronizingMessageSource;

    public boolean isDeleteLocalFiles() {
        return deleteLocalFiles;
    }

    public void setDeleteLocalFiles(boolean deleteLocalFiles) {
        this.deleteLocalFiles = deleteLocalFiles;
    }

    public SftpInboundFileSynchronizer getFtpInboundFileSynchronizer() {
        return ftpInboundFileSynchronizer;
    }

    public void setFtpInboundFileSynchronizer(SftpInboundFileSynchronizer ftpInboundFileSynchronizer) {
        this.ftpInboundFileSynchronizer = ftpInboundFileSynchronizer;
    }

    public SessionFactory getSessionFactory() {
        return sessionFactory;
    }

    public void setSessionFactory(SessionFactory sessionFactory) {
        this.sessionFactory = sessionFactory;
    }

    public SftpInboundFileSynchronizingMessageSource getSftpInboundFileSynchronizingMessageSource() {
        return sftpInboundFileSynchronizingMessageSource;
    }

    public void setSftpInboundFileSynchronizingMessageSource(SftpInboundFileSynchronizingMessageSource sftpInboundFileSynchronizingMessageSource) {
        this.sftpInboundFileSynchronizingMessageSource = sftpInboundFileSynchronizingMessageSource;
    }



    public String getRemoteDirectory() {
        return remoteDirectory;
    }

    public void setRemoteDirectory(String remoteDirectory) {
        this.remoteDirectory = remoteDirectory;
    }

    private SFTPGateway sftpGateway;


    private void deleteLocalFiles()
    {
        if (deleteLocalFiles)
        {
            SimplePatternFileListFilter filter = new SimplePatternFileListFilter(fileNamePattern);
            List<File> matchingFiles = filter.filterFiles(localDirectory.listFiles());
            if (CollectionUtils.isNotEmpty(matchingFiles))
            {
                for (File file : matchingFiles)
                {
                    FileUtils.deleteQuietly(file);
                }
            }
        }
    }

    @Override
    public RepeatStatus execute(StepContribution stepContribution, ChunkContext chunkContext) throws Exception {

        deleteLocalFiles();

        ftpInboundFileSynchronizer.synchronizeToLocalDirectory(localDirectory);

        if (retryIfNotFound) {

            SimplePatternFileListFilter filter = new SimplePatternFileListFilter(fileNamePattern);
            int attemptCount = 1;
            while (filter.filterFiles(localDirectory.listFiles()).size() == 0 && attemptCount <= downloadFileAttempts) {

                logger.info("File(s) matching " + fileNamePattern + " not found on remote site. Attempt " + attemptCount + " out of " + downloadFileAttempts);
                Thread.sleep(retryIntervalMilliseconds);
                ftpInboundFileSynchronizer.synchronizeToLocalDirectory(localDirectory);
                attemptCount++;
            }

            if (attemptCount >= downloadFileAttempts && filter.filterFiles(localDirectory.listFiles()).size() == 0) {
                throw new FileNotFoundException("Could not find remote file(s) matching " + fileNamePattern + " after " + downloadFileAttempts + " attempts.");
            }
        }
        return RepeatStatus.FINISHED;
    }
}

基于以上更改，我能够编译代码并创建必要的Jar，并使用Jar运行代码。

云宾鸿

2023-03-14

您声明了一个beanjobExecutionListener()，在这个bean中创建了新的FileSystemResource(config_file_path)；。config_file_path是从作业参数@value（“#{jobparameters['configfilePath']}”）注入的，这些参数在配置时不可用，只有在运行作业/步骤时才可用。这称为后期绑定。

因此，在您的示例中，当Spring尝试创建beanjobexecutionlistener()时，它尝试注入config_file_path但当时它为空（此时Spring只是创建bean来配置应用程序上下文），并且作业尚未运行，因此beforejob方法尚未执行。这就是您有NullPointerException的原因。在jobExecutionListener()bean上添加@jobscope应该可以解决这个问题，但我不建议这样做。原因是您试图以错误的方式和错误的位置配置某些属性，因此我将修复该设计，而不是通过添加注释来解决该问题。

作业参数用于业务参数，而不用于技术细节。在您的示例中，rundate是作业参数的好选择，而configfilePath则不是。此外，既然使用了Spring，为什么要注入文件路径，然后执行properties=PropertiesLoadeRutils.LoadProperties（资源）；和Integer.ParseInt（Properties.GetProperty（“Remote.Port”））；？如果告诉Spring在需要的地方注入属性，它就会为您这样做。

@Bean
public Tasklet myTasklet() {
   return new RemoteFileInboundTasklet()
}

@Bean
public Step getFilesFromFTPServer() {
    return stepBuilderFactory.get("Get file from server")
            .tasklet(myTasklet())
            .build();

}

spring批处理从命令行读取jobParameters并在作业配置中使用它

共有1个答案

相关问答

相关文章

相关阅读

相关工具

相关文档