将大型json文件从Firebase存储传输到Firestore

Question

我需要帮助使用Firebase函数将大型JSON文件从Firebase存储流到Firestore。

我想将几个大的换行JSON文件(11x700MB)传输到FireStore。我正在尝试从Firebase存储加载它们，对文件进行流式处理，并将内容写入Firestore集合。

const functions = require('firebase-functions');


const admin = require('firebase-admin');
admin.initializeApp(functions.config().firebase);
const es = require('event-stream')
const Parser = require('newline-json').Parser
const gcs = require('@google-cloud/storage')();
const path = require('path');

// [START function]
exports.generateData = functions.storage.object().onChange(event => {
  const object = event.data; // The Storage object.

  const fileBucket = object.bucket; // The Storage bucket that contains the file.
  const filePath = object.name; // File path in the bucket.
  const contentType = object.contentType; // File content type.
  const resourceState = object.resourceState; // The resourceState is 'exists' or 'not_exists' (for file/folder deletions).
  const metageneration = object.metageneration; // Number of times metadata has been generated. New objects have a value of 1.

  // Exit if this is triggered on a file that is not JSON.
  if (!contentType.endsWith('json')) {
    console.log('This is not a json file.');
    return;
  }

  // Exit if this is a move or deletion event.
  if (resourceState === 'not_exists') {
    console.log('This is a deletion event.');
    return;
  }

  // Exit if file exists but is not new and is only being triggered
  // because of a metadata change.
  if (resourceState === 'exists' && metageneration > 1) {
    console.log('This is a metadata change event.');
    return;
  }

  // Download file from bucket.
  const bucket = gcs.bucket(fileBucket);

let buf = []

  const getStream = function () {
      let stream = bucket.file(filePath).createReadStream().on('error', () => { console.log('Read Error')}).on('end', () => {console.log('Successful Read')})
      let parser = new Parser()
      return stream.pipe(parser)
  }

  getStream()
   .pipe(es.mapSync(function (data) {
     buf.push(data)
     pump()
   }))
   .on('end', () => {
     console.log("Strem Finished")
     return true
   })
   .on('error', () => {
     console.log('Stream Error')
     return false
   })

   function pump() {
     let pos;

     while((pos = buf.length) >= 1) {
       processLine(buf.pop(0))
     }
   }

   function processLine(line) {
     admin.firestore().collection('test').add(line)
   }

});

我现在不知道该怎么做，但我很感激你的帮助。

Answer 1

云函数的最大执行时间为540秒，因此很可能不适合您的需求。考虑设置一个小型GCE实例来执行迁移。