无法访问pyspark中的本地文件

Question

import os
import pyspark.sql.functions as F
from os import listdir, path

from pyspark import SparkConf, SparkContext

import argparse
from pyspark import SparkFiles
from pyspark.sql import SparkSession

def main():
    spark = SparkSession \
    .builder \
    .appName("Spark File load example") \
    .config("spark.jars","/u/user/someuser/sqljdbc4.jar") \
    .config("spark.dynamicAllocation.enabled","true") \
    .config("spark.shuffle.service.enabled","true") \
    .config("hive.exec.dynamic.partition", "true") \
    .config("hive.exec.dynamic.partition.mode", "nonstrict") \
    .config("spark.sql.shuffle.partitions","50") \
    .config("hive.metastore.uris", "thrift://******.hpc.****.com:9083") \
    .enableHiveSupport() \
    .getOrCreate()

    spark.sparkContext.addFile("/u/user/vikrant/testdata/EMPFILE1.csv")


    inputfilename=getinputfile(spark)
    print("input file path is:",inputfilename)
    data = processfiledata(spark,inputfilename)
    data.show()
    spark.stop()

def getinputfile(spark):

    spark_files_dir = SparkFiles.getRootDirectory()
    print("spark_files_dir:",spark_files_dir)
    inputfile = [filename
                   for filename in listdir(spark_files_dir)
                   if filename.endswith('EMPFILE1.csv')]
    if len(inputfile) != 0:
        path_to_input_file = path.join(spark_files_dir, inputfile[0])
    else:
        print("file path not found",path_to_input_file)

    print("inputfile name:",inputfile)
    return path_to_input_file


    def processfiledata(spark,inputfilename):

        dataframe= spark.read.format("csv").option("header","false").load(inputfilename)
        return dataframe

if __name__ == "__main__":
     main()

Below is my shell script-->
    spark-submit --master yarn --deploy-mode client PysparkMainModulenew.py --files /u/user/vikrant/testdata/EMPFILE1.csv

('SPARK_FILES_DIR:'，U'/H/TMP/Spark-76BDBD48-CBB4-4E8F-971A-383B899F79B0/userfiles-EE6DCDEC-B320-433B-8491-311927C75FE2')('inputfile name:'，[U'empfile1.csv'])('input file path:'，U'/H/TMP/Spark-76BDBD48-CBB4-4E8F-971A-383B899F79B0/Userfiles-EE6DCD-B320-433B-8491-31927C75FE 27C75FE2/empfile1.csv')回溯（最近的调用为last）：文件“/u/user/vikrant/testdata/pysparkmainmodulenew.py”，第57行，在main（）文件“/u/user/vikrant/testdata/pysparkmainmodulenew.py”中，第31行，在main data=processfiledata（spark,inputfilename）文件“/u/user/vikrant/testdata/pysparkmainmodulenew.py”中，第53行，在processfiledata current/spark2-client/python/lib/pyspark.zip/pyspark/sql/readwriter.py“，第166行，加载文件”/usr/hdp/current/spark2-client/python/lib/pyspark/zip/pyspark/sql/readwriter.py“，第1160行，调用文件”/usr/hdp/current/spark2-client/python/lib/pyspark/sql/utils.py“，第6行9，在deco pyspark.sql.utils.analysisException:U'路径不存在:hdfs://hdd2cluster/h/tmp/spark-76bdbd48-cbb4-4e8f-971a-383b899f79b0/userfiles-ee6dcdec-b320-433b-8491-311927c75fe2/empfile1.csv；‘

Answer 1

你有这样的东西。这是行不通的，因为您需要在--files选项之后放置pysparkmainmodulenew.py。所以，这个

spark-submit --master yarn --deploy-mode client PysparkMainModulenew.py --files /u/user/vikrant/testdata/EMPFILE1.csv

应该是，

spark-submit --master yarn --deploy-mode client --files /u/user/vikrant/testdata/EMPFILE1.csv PysparkMainModulenew.py

在这种情况下，不需要使用addfile。您可以将pysparkmainmodulenew.py和empfile1.csv复制到同一个文件夹中。并且，所有内容都应该在--files选项之后。

spark-submit --master yarn --deploy-mode client --files /u/user/vikrant/testdata/EMPFILE1.csv /u/user/vikrant/testdata/PysparkMainModulenew.py