sparkException：任务不可序列化-传递RDD

Question

我已经上了三节课

任务未序列化

public class Person implements Serializable 
  {
      private String name;
      private int age;

      public String getName() 
      {
        return name;
      }

      public void setAge(int age) 
      {
        this.age = age;
      }
    }

public class sqlTestv2 implements Serializable

{

    private int appID;
    private int dataID;
    private JavaSparkContext sc;



    public JavaRDD<Person> getDataRDD()
    {

    JavaRDD<String> test = sc.textFile("hdfs://localhost:8020/user/cloudera/people.txt");

         JavaRDD<Person> people = test.map(
                    new Function<String, Person>() {
                    public Person call(String line) throws Exception {
                      String[] parts = line.split(",");

                      Person person = new Person();
                      person.setName(parts[0]);
                      person.setAge(Integer.parseInt(parts[1].trim()));

                      return person;
                    }
                  });


        return people;



    }

}

public class sqlTestv1 implements Serializable

{

    public static void main(String[] arg) throws Exception 
    {
          SparkConf conf = new SparkConf().setMaster("local").setAppName("wordCount");
          JavaSparkContext sc = new JavaSparkContext(conf);
          SQLContext sqlContext = new org.apache.spark.sql.SQLContext(sc);
          sqlTestv2 v2=new sqlTestv2(1,1,sc);
          JavaRDD<Person> test=v2.getDataRDD();


          DataFrame schemaPeople = sqlContext.createDataFrame(test, Person.class);
          schemaPeople.registerTempTable("people");

          DataFrame df = sqlContext.sql("SELECT age FROM people");

          df.show();





    }

}

Answer 1

堆栈告诉你答案。您要传递给SQLTestV2的是JavaSparkContext。您应该将sc传递到方法中，而不是类中