火花提交pyspark脚本超出了最大递归深度

问题描述

我能够通过spark提交yarn-cluster模式提交org.apache.spark.examples.SparkPi示例jar，并且成功，但是pyspark中的以下代码段失败，最大递归深度超过错误。

package com.example.pricetag.fragments

import android.app.AlertDialog
import android.content.Context
import android.content.DialogInterface
import android.os.Bundle
import android.view.LayoutInflater
import android.view.View
import android.view.ViewGroup
import android.widget.EditText
import android.widget.ImageButton
import android.widget.TextView
import androidx.core.content.ContextCompat
import androidx.fragment.app.Fragment
import androidx.recyclerview.widget.DividerItemdecoration
import androidx.recyclerview.widget.linearlayoutmanager
import androidx.recyclerview.widget.RecyclerView
import com.example.pricetag.R
import com.example.pricetag.fragments.DTO.ItemList
import com.google.android.material.floatingactionbutton.FloatingActionButton
import kotlinx.android.synthetic.main.fragment_shopping_list.*


lateinit var dbHandler: DBHandler

class shopping_list() : Fragment() {

    override fun onCreateView(
        inflater: LayoutInflater,container: ViewGroup?,savedInstanceState: Bundle?
    ): View? {
        
    
        val view = inflater.inflate(R.layout.fragment_shopping_list,container,false)
        dbHandler = DBHandler(view.context)

        val addItem_button = view.findViewById<FloatingActionButton>(R.id.fab_shoppiglist)


        addItem_button.setonClickListener {
            val dialog = AlertDialog.Builder(getContext())
            val tempView = layoutInflater.inflate(R.layout.dialog_shoppinglist,null)
            val text_result = tempView.findViewById<EditText>(R.id.et_Itemlist)
            dialog.setView(tempView)
            dialog.setPositiveButton("Add") { _: DialogInterface,_: Int ->
                if (text_result.text.isNotEmpty()) {
                    val itemList = ItemList()
                    itemList.name = text_result.text.toString()
                    dbHandler.addItem(itemList)
                    refreshList()
                }


            }
            dialog.setNeutralButton("DeleteList") { _: DialogInterface,_: Int ->
                    dbHandler.delete()
                    refreshList()
            }
            dialog.setNegativeButton("Cancel"){ _: DialogInterface,_: Int->

            }
            dialog.show()
        }

        return view
    }

    override fun onActivityCreated(savedInstanceState: Bundle?) {
        super.onActivityCreated(savedInstanceState)
        rv_shoppinglist.layoutManager = linearlayoutmanager(activity)
        val verticaldecoration = DividerItemdecoration(
            rv_shoppinglist.getContext(),DividerItemdecoration.VERTICAL
        )
        val verticalDivider =
            ContextCompat.getDrawable(activity!!,R.drawable.rv_divider)
        verticaldecoration.setDrawable(verticalDivider!!)
        rv_shoppinglist.addItemdecoration(verticaldecoration)

    }


    override fun onResume() {
        refreshList()
        super.onResume()
    }

    private fun refreshList(){
        rv_shoppinglist.adapter = ShoppinglistAdpter(this.requireContext(),dbHandler,dbHandler.getItem())
    }


    class ShoppinglistAdpter(val context: Context,dbHandler: DBHandler,val list: MutableList<ItemList>): RecyclerView.Adapter<ShoppinglistAdpter.ViewHolder>(){

        override fun onCreateViewHolder(p0: ViewGroup,pl: Int): ViewHolder {
            return ViewHolder(LayoutInflater.from(context).inflate(R.layout.rv_child_shoppinglist,p0,false))
        }

        override fun getItemCount(): Int {
            return list.size
        }

        override fun onBindViewHolder(holder: ViewHolder,p1: Int) {
            holder.itemName.text = list[p1].name
            holder.itemAmount.text = list[p1].amount.toString()
            holder.addButton.setonClickListener{
                dbHandler.changeAmount(list[p1].id.toInt())
            }

        }

        class ViewHolder(v : View) : RecyclerView.ViewHolder(v){
            val itemName : TextView = v.findViewById(R.id.tv_item_name)
            val itemAmount : TextView = v.findViewById(R.id.tv_item_amount)
            val addButton : ImageButton = v.findViewById(R.id.IB_add)

        }

    }

}

我根据Pyspark on yarn-cluster mode的建议添加了pyspark_python env。

test.py

spark-submit --master yarn --deploy-mode cluster --executor-memory 1G --num-executors 4 --conf spark.yarn.appMasterEnv.PYSPARK_PYTHON="/usr/bin/python2.7" test.py --verbose

如何解决此问题？

from pyspark import SparkContext
from pyspark.sql import HiveContext

sc_new = SparkContext()
sqlContext = HiveContext(sc_new)
sqlContext.setConf("spark.sql.hive.convertmetastoreOrc","false")
txt = sqlContext.sql( "SELECT 1")
txt.show(2000000,False)

• 运行Spark版本1.6.0
• 配置单元1.1.0版
• Hadoop版本：2.6.0-cdh5.13.0

解决方法

通过调用txt.show(2000000,False)来进行py4j来进行to-and-fro jvm-python-object-jvm调用，其中结果没有那么多行。 我相信您最多可以拨入show()的费用是2000欧元。 为什么您只需要SELECT 1时为什么需要显示 2000000 条记录？