java 用openCV定时拍照画面有变化就报错，什么问题，怎么解决？

Question

java 用openCV定时每10s拍一次照片，当摄像头前没有变化时，都能拍照成功
当摄像头前有变化，就会报错，先报错grab() Error: Could not grab frame. (Has start() been called?)，
然后我重新grabber.start()又会报错read() Error: Could not read frame in start().

package org.jeecg.bussiness.pressMachine.model.frontCamera.service;

//import com.github.sarxos.webcam.Webcam;
//import com.github.sarxos.webcam.WebcamResolution;
//import com.github.sarxos.webcam.WebcamUtils;
//import com.github.sarxos.webcam.util.ImageUtils;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.bytedeco.ffmpeg.global.avutil;
import org.bytedeco.javacv.*;
import org.bytedeco.opencv.global.opencv_imgproc;
import org.bytedeco.opencv.opencv_core.IplImage;
import org.bytedeco.opencv.opencv_core.Mat;
import org.bytedeco.opencv.opencv_core.Scalar;
import org.jeecg.bussiness.pressMachine.model.frontCamera.FrontCameraService;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.FileOutputStream;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Date;

import static org.bytedeco.opencv.global.opencv_core.cvFlip;

@Slf4j
@Service(value = "frontCameraService")
public class FrontCameraServiceImpl implements FrontCameraService {

    @Value("${testBlock.cameraDownloadPathPrefix}")
    private String cameraDownloadPathPrefix;
//    static Webcam webcam = null;

    private LocalDateTime lastPictureTime = LocalDateTime.now();
//    static {
//        try {
//            webcam = Webcam.getDefault();
//            // 设置摄像头捕获的图像尺寸为VGA
//            webcam.setViewSize(WebcamResolution.VGA.getSize());
//            webcam.getDevice().setResolution(WebcamResolution.FHD.getSize());
//        }catch (Exception e){
//            log.error(e.getMessage(),e);
//        }
//    }

    private static int count = 0;


    @Override
    public synchronized String takePhoto() {
//        this.openWebcam();
        String fileFolderName = this.getAndCreatePhotoPathUlr(null);
        fileFolderName +="/"+System.currentTimeMillis()+"."+IMG_TYPE;
//        WebcamUtils.capture(webcam, fileFolderName, ImageUtils.FORMAT_JPG);
//        new GrabImageFromCamera().action(10);

        this.openPicture();
        try {
            grabAndOutput(fileFolderName);
        } catch (Exception e) {
            log.error(e.getMessage(),e);
            safeRelease();
            count = 999;
            fileFolderName="";
        }
        return fileFolderName ;
    }

    private void openPicture(){
        try {
            if(count >= 300 || grabber == null || lastPictureTime.compareTo(LocalDateTime.now().plusHours(-1)) <=0 ){
                safeRelease();
                init();
                count = 0;
                log.error("重启摄像头");
            }
        } catch (Exception e) {
            log.error(e.getMessage(),e);
        }
        count++;
        lastPictureTime = LocalDateTime.now();
    }

//    private synchronized void openWebcam(){
//        if(count >= 300){
//            webcam.close();
//            try {
//                Thread.sleep(1000);
//            } catch (InterruptedException e) {
//                log.error(e.getMessage(),e);
//            }
//            webcam = Webcam.getDefault();
//            // 设置摄像头捕获的图像尺寸为VGA
//            webcam.setViewSize(WebcamResolution.VGA.getSize());
//            webcam.getDevice().setResolution(WebcamResolution.FHD.getSize());
//            count = 0;
//            log.error("重新开启摄像头");
//        }
//        count++;
//    }

    private String getAndCreatePhotoPathUlr(String testBlockId){
        String fileFolderName = cameraDownloadPathPrefix+"/frontCamera/";
        File file=new File(fileFolderName);
        if  (!file .exists()  && !file.isDirectory()) {
            file .mkdir();
        }
        String dateStr = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
        fileFolderName = fileFolderName+"/"+dateStr+"/";
        file=new File(fileFolderName);
        if  (!file .exists()  && !file.isDirectory()) {
            file .mkdir();
        }

        if(testBlockId != null){
            fileFolderName = fileFolderName+"/"+ testBlockId;
            file=new File(fileFolderName);
            if  (!file .exists()  && !file.isDirectory()) {
                file .mkdir();
            }
        }

        return fileFolderName;
    }



    //===============================新版本拍照开始======================

    /**
     * 摄像头序号，如果只有一个摄像头，那就是0
     */
    protected static final int CAMERA_INDEX = 0;

    /**
     * 帧抓取器
     */
    protected FrameGrabber grabber;

    /**
     * 输出帧率
     */
    @Getter
    private final double frameRate = 30;

    /**
     * 摄像头视频的宽
     */
    @Getter
    private final int cameraImageWidth = 1280;

    /**
     * 摄像头视频的高
     */
    @Getter
    private final int cameraImageHeight = 720;

    /**
     * 转换器
     */
    private final OpenCVFrameConverter.ToIplImage openCVConverter = new OpenCVFrameConverter.ToIplImage();

    /**
     * 实例化、初始化输出操作相关的资源
     */
    protected void initOutput() throws Exception {
        // 啥也不用做
    }

    /**
     * 输出
     */
    protected void output(Frame frame,String filePath) throws Exception {
        // 图片的保存位置
        String imagePath = filePath;

        // 把帧对象转为Image对象
        BufferedImage bufferedImage = converter.getBufferedImage(frame);

        // 保存图片
        ImageIO.write(bufferedImage, IMG_TYPE, new FileOutputStream(imagePath));

        log.info("保存完成：{}", imagePath);
    }

    /**
     * 释放输出操作相关的资源
     */
    protected void releaseOutputResource() {
        // 啥也不用做
    }

    /**
     * 两帧之间的间隔时间
     * @return
     */
    protected int getInterval() {
        // 假设一秒钟15帧，那么两帧间隔就是(1000/15)毫秒
        return (int)(1000/ frameRate);
    }

    /**
     * 实例化帧抓取器，默认OpenCVFrameGrabber对象，
     * 子类可按需要自行覆盖
     * @throws FFmpegFrameGrabber.Exception
     */
    protected void instanceGrabber() throws FrameGrabber.Exception {
        grabber = new OpenCVFrameGrabber(CAMERA_INDEX);
    }

    /**
     * 用帧抓取器抓取一帧，默认调用grab()方法，
     * 子类可以按需求自行覆盖
     * @return
     */
    protected Frame grabFrame() throws FrameGrabber.Exception {
        return grabber.grab();
    }

    /**
     * 初始化帧抓取器
     * @throws Exception
     */
    protected void initGrabber() throws Exception {
        // 实例化帧抓取器
        if(grabber !=null){
            grabber.close();
            grabber.release();
        }
        instanceGrabber();

        // 摄像头有可能有多个分辨率，这里指定
        // 可以指定宽高，也可以不指定反而调用grabber.getImageWidth去获取，
        grabber.setImageWidth(cameraImageWidth);
        grabber.setImageHeight(cameraImageHeight);
        grabber.setMaxDelay(5000);
        // 开启抓取器
        grabber.start();
    }

    /**
     * 预览和输出
     * @param
     * @throws Exception
     */
    private void grabAndOutput(String picturePath) throws Exception {
        // 添加水印时用到的时间工具
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

//        long endTime = System.currentTimeMillis() + 1000L *grabSeconds;

        // 两帧输出之间的间隔时间，默认是1000除以帧率，子类可酌情修改
        int interVal = getInterval();

        // 水印在图片上的位置
//        org.bytedeco.opencv.opencv_core.Point point = new org.bytedeco.opencv.opencv_core.Point(15, 35);

        Frame captureFrame;
        IplImage img;
        Mat mat;

        // 超过指定时间就结束循环
//        while (System.currentTimeMillis()<endTime) {
            // 取一帧
            captureFrame = grabFrame();
//            grabber.stop();


        if (null==captureFrame) {
                log.error("帧对象为空");
                return;
//                break;
            }

            // 将帧对象转为IplImage对象
            img = openCVConverter.convert(captureFrame);

            // 镜像翻转
            cvFlip(img, img, 1);

            // IplImage转mat
            mat = new Mat(img);

            // 在图片上添加水印，水印内容是当前时间，位置是左上角
//            opencv_imgproc.putText(mat,
//                    simpleDateFormat.format(new Date()),
//                    point,
//                    opencv_imgproc.CV_FONT_VECTOR0,
//                    0.8,
//                    new Scalar(0, 200, 255, 0),
//                    1,
//                    0,
//                    false);

            // 子类输出
            output(openCVConverter.convert(mat),picturePath);

            // 适当间隔，让肉感感受不到闪屏即可
            if(interVal>0) {
                Thread.sleep(interVal);
            }
//        }
        mat.release();
        log.info("输出结束");
    }

    /**
     * 释放所有资源
     */
    private void safeRelease() {
        try {
            // 子类需要释放的资源
            releaseOutputResource();
        } catch (Exception exception) {
            log.error("do releaseOutputResource error", exception);
        }

        if (null!=grabber) {
            try {
                grabber.close();
            } catch (Exception exception) {
                log.error("close grabber error", exception);
            }
        }
    }

    /**
     * 整合了所有初始化操作
     * @throws Exception
     */
    private void init() throws Exception {
        long startTime = System.currentTimeMillis();

        // 设置ffmepg日志级别
        avutil.av_log_set_level(avutil.AV_LOG_INFO);
        FFmpegLogCallback.set();

        // 实例化、初始化帧抓取器
        initGrabber();

        // 实例化、初始化输出操作相关的资源，
        // 具体怎么输出由子类决定，例如窗口预览、存视频文件等
        initOutput();

        log.info("初始化完成，耗时[{}]毫秒，帧率[{}]，图像宽度[{}]，图像高度[{}]",
                System.currentTimeMillis()-startTime,
                frameRate,
                cameraImageWidth,
                cameraImageHeight);
    }

    /**
     * 执行抓取和输出的操作
     */
    public void action(int grabSeconds) {
        try {
            // 初始化操作
            init();
            // 持续拉取和推送
            log.error("-",new Date());
//            grabAndOutput(grabSeconds);
            log.error("-",new Date());
        } catch (Exception exception) {
            log.error("execute action error", exception);
        } finally {
            // 无论如何都要释放资源
            safeRelease();
        }
    }
    //==============实现类 开始====
    /**
     * 当前进程已经存储的图片数量
     */
    private int saveNums = 0;

    // 转换工具
    private Java2DFrameConverter converter = new Java2DFrameConverter();

    // 图片格式
    private final static String IMG_TYPE = "jpg";
    //==============实现类 结束====

    //===============================新版本拍照结束======================
}

Answer 1

1.保持摄像头持续运行： 可以在程序启动时初始化摄像头，并保持摄像头持续运行，而不是每次拍照前重新初始化。这样可以避免频繁的初始化操作。
2.使用线程处理拍照任务： 使用单独的线程来处理拍照任务，这样可以在后台持续抓取帧，而主线程可以处理其他任务。
3.优化摄像头参数： 检查并优化摄像头的参数设置，确保摄像头在启动时不会加载过多的资源。
以下是一个示例代码：

import org.bytedeco.javacv.*;
import org.bytedeco.opencv.opencv_core.IplImage;

import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.FileOutputStream;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;

import static org.bytedeco.opencv.global.opencv_core.cvFlip;

public class FrontCameraServiceImpl {

    private static final int CAMERA_INDEX = 0;
    private OpenCVFrameGrabber grabber;
    private LocalDateTime lastPictureTime = LocalDateTime.now();
    private static int count = 0;
    private final static String IMG_TYPE = "jpg";
    private String cameraDownloadPathPrefix = "/path/to/save/images";

    public FrontCameraServiceImpl() throws FrameGrabber.Exception {
        grabber = new OpenCVFrameGrabber(CAMERA_INDEX);
        grabber.start();
    }

    public synchronized String takePhoto() {
        String fileFolderName = getAndCreatePhotoPathUlr(null);
        fileFolderName += "/" + System.currentTimeMillis() + "." + IMG_TYPE;

        try {
            grabAndOutput(fileFolderName);
        } catch (Exception e) {
            System.err.println("Error: " + e.getMessage());
            safeRelease();
            count = 999;
            fileFolderName = "";
        }
        return fileFolderName;
    }

    private void grabAndOutput(String fileFolderName) throws Exception {
        Frame frame = grabber.grab();
        if (frame == null) {
            throw new Exception("grab() Error: Could not grab frame. (Has start() been called?)");
        }
        OpenCVFrameConverter.ToIplImage converter = new OpenCVFrameConverter.ToIplImage();
        IplImage img = converter.convert(frame);
        if (img != null) {
            cvFlip(img, img, 1);
            BufferedImage bi = img.getBufferedImage();
            File output = new File(fileFolderName);
            ImageIO.write(bi, "jpg", output);
        }
    }

    private void safeRelease() {
        if (grabber != null) {
            try {
                grabber.stop();
                grabber.release();
            } catch (Exception e) {
                System.err.println("Error: " + e.getMessage());
            }
        }
    }

    private String getAndCreatePhotoPathUlr(String testBlockId) {
        String fileFolderName = cameraDownloadPathPrefix + "/frontCamera/";
        File file = new File(fileFolderName);
        if (!file.exists() && !file.isDirectory()) {
            file.mkdir();
        }
        String dateStr = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
        fileFolderName = fileFolderName + "/" + dateStr + "/";
        file = new File(fileFolderName);
        if (!file.exists() && !file.isDirectory()) {
            file.mkdir();
        }

        if (testBlockId != null) {
            fileFolderName = fileFolderName + "/" + testBlockId;
            file = new File(fileFolderName);
            if (!file.exists() && !file.isDirectory()) {
                file.mkdir();
            }
        }

        return fileFolderName;
    }

    public static void main(String[] args) {
        try {
            FrontCameraServiceImpl cameraService = new FrontCameraServiceImpl();
            while (true) {
                cameraService.takePhoto();
                Thread.sleep(30000); // 每30秒拍一次照片
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}