可能函数无法生成，错误我不明白

Question

请看下面我在main.py中提供的内容

# Imports the BigQuery client library
    from google.cloud import bigquery

    def extract_data(request):
        # Replace these values according to your project
        project_name = "whr-asia-datalake-nonprod" 
        bucket_name = "whr-asia-datalake-dev-standard/outbound/Adobe/" 
        dataset_name = "WHR_DATALAKE" 
        table_name = "CONSUMER_EXTRACT" 
        destination_uri = "gs://{}/{}".format(bucket_name,"Consumer_master.csv")

        bq_client = bigquery.Client(project=project_name)

        dataset = bq_client.dataset(dataset_name, project=project_name)
        table_to_export = dataset.table(table_name)

        job_config = bigquery.job.ExtractJobConfig()
        
        extract_job = bq_client.extract_table(
            table_to_export,
            destination_uri,
            # Location must match that of the source table.
            location="asia-south1",
            job_config=job_config,
        )  
        return "Job with ID {} started exporting data from {}.{} to {}".format(extract_job.job_id, dataset_name, table_name, destination_uri)

在requirements.txt中，我提供了如下内容

google-cloud-bigquery

我的云函数将从一个pub子主题触发，该子主题创建一个事件，一旦我需要从中提取数据的大查询表被加载。

2021-02-13T11:23:38.518815ZCloud FunctionsUpdateFunctionasia-south1:[e-mail] {@type: type.googleapis.com/google.cloud.audit.AuditLog, authenticationInfo: {…}, methodName: google.cloud.functions.v1.CloudFunctionsService.UpdateFunction, resourceName: projects/whr-asia-datalake-nonprod/locations/asia-south1/functions/Consumer_Extract_Bigquery, serviceName: cloudfunctions.google…
{@type: type.googleapis.com/google.cloud.audit.AuditLog, authenticationInfo: {…}, methodName: google.cloud.functions.v1.CloudFunctionsService.UpdateFunction, resourceName: projects/whr-asia-datalake-nonprod/locations/asia-south1/functions/Consumer_Extract_Bigquery, serviceName: cloudfunctions.google…

我可以通过在main.py中添加下面的内容来解决这个问题

# Imports the BigQuery client library
from google.cloud import bigquery
def extract_data(event, context):
 # Replace these values according to your project
    project_name = "whr-asia-datalake-nonprod"
    bucket_name = "whr-asia-datalake-dev-standard/outbound/Adobe/"
    dataset_name = "WHR_DATALAKE"
    table_name = "CONSUMER_EXTRACT"
    destination_uri = "gs://{}/{}".format(bucket_name,"Consumer_master.csv")
    bq_client = bigquery.Client(project=project_name)
    dataset = bq_client.dataset(dataset_name, project=project_name)
    table_to_export = dataset.table(table_name)
    job_config = bigquery.job.ExtractJobConfig()
    extract_job = bq_client.extract_table(
    table_to_export,
    destination_uri,
    # Location must match that of the source table.
    location="asia-south1",
    job_config=job_config,
    )
    return "Job with ID {} started exporting data from {}.{} to {}".format(extract_job.job_id, dataset_name, table_name, destination_uri)

并在requirements.txt中添加以下内容

google-cloud-bigquery==1
google-cloud-bigquery-datatransfer==1

Answer 1

我终于成功地做到了这一点。请在下面找到将通过酒吧子事件触发的云函数中的代码。

from google.cloud import bigquery
def extract_data(event, context):
 # Replace these values according to your project
    project_name = "whr-asia-datalake-nonprod"
    bucket_name = "whr-asia-datalake-dev-standard/outbound/Adobe/"
    dataset_name = "WHR_DATALAKE"
    table_name = "C4C_CONSUMER_CURATED"
    destination_uri = "gs://{}/{}".format(bucket_name,"Consumer_master_test.txt")
    bq_client = bigquery.Client(project=project_name)
    dataset = bq_client.dataset(dataset_name, project=project_name)
    table_to_export = dataset.table(table_name)
    job_config = bigquery.job.ExtractJobConfig()
    job_config.field_delimiter = "|"
    job_config.print_header = True

    extract_job = bq_client.extract_table(
    table_to_export,
    destination_uri,
    # Location must match that of the source table.
    location="asia-south1",
    job_config=job_config,
    )
    return "Job with ID {} started exporting data from {}.{} to {}".format(extract_job.job_id, dataset_name, table_name, destination_uri)

我能够为BigQuery.job.ExtractJobConfig提供其他属性，如管道分隔符和标头。

感谢@Maxim。你的帖子帮了我大忙。