[Solved] Jupyter Notebook Error: SparkException: Python worker failed to connect back

[Solved] Jupyter Notebook Error: Sparkexception: Python Worker Failed To Connect Back

report errors

---------------------------------------------------------------------------
Py4JJavaError                             Traceback (most up-to-date name final)
<ipython-input-24-bafca16b0526> in <module>
      8     return jobitem, ratingsRDD
      9 jobitem, jobRDD = preparJobknowledge(sc)
---> 10 jobRDD.acquire() 

G:Projectspython-3.6.4-amd64libsite-packagespysparkrdd.py in acquire(self)
    947         """
    948         with SCCallSiteSync(self.context) as css:
--> 949             sock_info = self.ctx._jvm.PythonRDD.acquireAndServe(self._jrdd.rdd())
    950         return checklist(_load_from_socket(sock_info, self._jrdd_deserializer))
    951 

G:Projectspython-3.6.4-amd64libsite-packagespy4jjava_gateway.py in __call__(self, *args)
   1303         reply = self.gateway_client.send_command(command)
   1304         return_value = get_return_value(
-> 1305             reply, self.gateway_client, self.target_id, self.title)
   1306 
   1307         for temp_arg in temp_args:

G:Projectspython-3.6.4-amd64libsite-packagespy4jprotocol.py in get_return_value(reply, gateway_client, target_id, title)
    326                 elevate Py4JJavaError(
    327                     "An error occurred while calling {0}{1}{2}.n".
--> 328                     format(target_id, ".", title), worth)
    329             else:
    330                 elevate Py4JError(

Py4JJavaError: An error occurred whereas calling z:org.apache.spark.api.python.PythonRDD.acquireAndServe.
: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 1 instances, most up-to-date failure: Lost job 0.0 in stage 0.0 (TID 0) (192.168.101.68 executor driver): org.apache.spark.SparkException: Python worker failed to connect back.
	at org.apache.spark.api.python.PythonWorkerManufacturing unit.createSimpleWorker(PythonWorkerManufacturing unit.scala:182)
	at org.apache.spark.api.python.PythonWorkerManufacturing unit.create(PythonWorkerManufacturing unit.scala:107)
	at org.apache.spark.SparkEnv.createPythonWorker(SparkEnv.scala:119)
	at org.apache.spark.api.python.BasePythonRunner.compute(PythonRunner.scala:145)
	at org.apache.spark.api.python.PythonRDD.compute(PythonRDD.scala:65)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.OutcomeTask.runTask(OutcomeTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.web.SocketTimeoutException: Accept timed out
	at java.web.DualStackPlainSocketImpl.waitForNewConnection(Native Method)
	at java.web.DualStackPlainSocketImpl.socketAccept(DualStackPlainSocketImpl.java:135)
	at java.web.AbstractPlainSocketImpl.settle for(AbstractPlainSocketImpl.java:409)
	at java.web.PlainSocketImpl.settle for(PlainSocketImpl.java:199)
	at java.web.ServerSocket.implAccept(ServerSocket.java:545)
	at java.web.ServerSocket.settle for(ServerSocket.java:513)
	at org.apache.spark.api.python.PythonWorkerManufacturing unit.createSimpleWorker(PythonWorkerManufacturing unit.scala:174)
	... 14 extra

Driver stacktrace:
	at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2253)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(DAGScheduler.scala:2202)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$tailored(DAGScheduler.scala:2201)
	at scala.assortment.mutable.ResizableArray.foreach(ResizableArray.scala:62)
	at scala.assortment.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
	at scala.assortment.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
	at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2201)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$deal withTaskSetFailed$1(DAGScheduler.scala:1078)
	at org.apache.spark.scheduler.DAGScheduler.$anonfun$deal withTaskSetFailed$1$tailored(DAGScheduler.scala:1078)
	at scala.Option.foreach(Option.scala:407)
	at org.apache.spark.scheduler.DAGScheduler.deal withTaskSetFailed(DAGScheduler.scala:1078)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2440)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2382)
	at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2371)
	at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)
	at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:868)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2202)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2223)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2242)
	at org.apache.spark.SparkContext.runJob(SparkContext.scala:2267)
	at org.apache.spark.rdd.RDD.$anonfun$acquire$1(RDD.scala:1030)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
	at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
	at org.apache.spark.rdd.RDD.withScope(RDD.scala:414)
	at org.apache.spark.rdd.RDD.acquire(RDD.scala:1029)
	at org.apache.spark.api.python.PythonRDD$.acquireAndServe(PythonRDD.scala:180)
	at org.apache.spark.api.python.PythonRDD.acquireAndServe(PythonRDD.scala)
	at solar.replicate.NativeMethodAccessorImpl.invoke0(Native Method)
	at solar.replicate.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at solar.replicate.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.replicate.Method.invoke(Method.java:498)
	at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
	at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
	at py4j.Gateway.invoke(Gateway.java:282)
	at py4j.instructions.AbstractCommand.invokeMethod(AbstractCommand.java:132)
	at py4j.instructions.CallCommand.execute(CallCommand.java:79)
	at py4j.GatewayConnection.run(GatewayConnection.java:238)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.spark.SparkException: Python worker failed to connect back.
	at org.apache.spark.api.python.PythonWorkerManufacturing unit.createSimpleWorker(PythonWorkerManufacturing unit.scala:182)
	at org.apache.spark.api.python.PythonWorkerManufacturing unit.create(PythonWorkerManufacturing unit.scala:107)
	at org.apache.spark.SparkEnv.createPythonWorker(SparkEnv.scala:119)
	at org.apache.spark.api.python.BasePythonRunner.compute(PythonRunner.scala:145)
	at org.apache.spark.api.python.PythonRDD.compute(PythonRDD.scala:65)
	at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)
	at org.apache.spark.rdd.RDD.iterator(RDD.scala:337)
	at org.apache.spark.scheduler.OutcomeTask.runTask(OutcomeTask.scala:90)
	at org.apache.spark.scheduler.Task.run(Task.scala:131)
	at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
	at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
	at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 extra
Caused by: java.web.SocketTimeoutException: Accept timed out
	at java.web.DualStackPlainSocketImpl.waitForNewConnection(Native Method)
	at java.web.DualStackPlainSocketImpl.socketAccept(DualStackPlainSocketImpl.java:135)
	at java.web.AbstractPlainSocketImpl.settle for(AbstractPlainSocketImpl.java:409)
	at java.web.PlainSocketImpl.settle for(PlainSocketImpl.java:199)
	at java.web.ServerSocket.implAccept(ServerSocket.java:545)
	at java.web.ServerSocket.settle for(ServerSocket.java:513)
	at org.apache.spark.api.python.PythonWorkerManufacturing unit.createSimpleWorker(PythonWorkerManufacturing unit.scala:174)
	... 14 extra

Solution:

READ :  java – new data source and connection pool are not displayed in the JDBC resources and Connection pool – Code Utility

The following variable environments are configured:

# Windows Hadoop variable environments are configured
HADOOP_HOME = F:hadoop-common-2.2.0-bin-masterhadoop-common-2.2.0-bin-master

# Windows JDKvariable environments are configured
JAVA_HOME = F:jdk-8u121-windows-x64_8.0.1210.13

# Windows Pysparkvariable environments are configured
PYSPARK_DRIVER_PYTHON = jupyter
PYSPARK_DRIVER_PYTHON_OPTS = pocket book
PYSPARK_PYTHON = python

Remember to restart the pc after the configuration is accomplished!

Read More:

This entry was posted in Python and tagged # Python, Jupyter, jupyter pocket book Error, PySpark on by Robins.

Leave a Reply

Your email address will not be published. Required fields are marked *