I encountered this error while trying to create spark session in a python virtual environment using VS Code as IDE. The code I ran and the output is below, please help.
**Code**
# Create SparkSession
spark = SparkSession.builder\
.master("local")\
.appName("spark-app-version-x")\
.getOrCreate()SparkConf for existing SparkContext, as it's shared
271 # by all sessions.
272 session = SparkSession(sc, options=self._options)
File c:\Users\Oluwa\Documents\DATA PROJECTS\Hands-On Real Time PySpark Project for Beginners\Lab PySpark_1\PSP_1-Venv\Lib\site-packages\pyspark\context.py:483, in SparkContext.getOrCreate(cls, conf)
481 with SparkContext._lock:
482 if SparkContext._active_spark_context is None:
--> 483 SparkContext(conf=conf or SparkConf())
484 assert SparkContext._active_spark_context is not None
485 return SparkContext._active_spark_context
File c:\Users\Oluwa\Documents\DATA PROJECTS\Hands-On Real Time PySpark Project for Beginners\Lab PySpark_1\PSP_1-Venv\Lib\site-packages\pyspark\context.py:197, in SparkContext.__init__(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls, udf_profiler_cls)
195 SparkContext._ensure_initialized(self, gateway=gateway, conf=conf)
196 try:
--> 197 self._do_init(
198 master,
199 appName,
200 sparkHome,
201 pyFiles,
202 environment,
203 batchSize,
204 serializer,
205 conf,
206 jsc,
207 profiler_cls,
208 udf_profiler_cls,
209 )
210 except BaseException:
211 # If an error occurs, clean up in order to allow future SparkContext creation:
212 self.stop()
File c:\Users\Oluwa\Documents\DATA PROJECTS\Hands-On Real Time PySpark Project for Beginners\Lab PySpark_1\PSP_1-Venv\Lib\site-packages\pyspark\context.py:282, in SparkContext._do_init(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, jsc, profiler_cls, udf_profiler_cls)
279 self.environment["PYTHONHASHSEED"] = os.environ.get("PYTHONHASHSEED", "0")
281 # Create the Java SparkContext through Py4J
--> 282 self._jsc = jsc or self._initialize_context(self._conf._jconf)
283 # Reset the SparkConf to the one actually used by the SparkContext in JVM.
284 self._conf = SparkConf(_jconf=self._jsc.sc().conf())
File c:\Users\Oluwa\Documents\DATA PROJECTS\Hands-On Real Time PySpark Project for Beginners\Lab PySpark_1\PSP_1-Venv\Lib\site-packages\pyspark\context.py:402, in SparkContext._initialize_context(self, jconf)
398 """
399 Initialize SparkContext in function to allow subclass specific initialization
400 """
401 assert self._jvm is not None
--> 402 return self._jvm.JavaSparkContext(jconf)
File c:\Users\Oluwa\Documents\DATA PROJECTS\Hands-On Real Time PySpark Project for Beginners\Lab PySpark_1\PSP_1-Venv\Lib\site-packages\py4j\java_gateway.py:1585, in JavaClass.__call__(self, *args)
1579 command = proto.CONSTRUCTOR_COMMAND_NAME +\
1580 self._command_header +\
1581 args_command +\
1582 proto.END_COMMAND_PART
1584 answer = self._gateway_client.send_command(command)
-> 1585 return_value = get_return_value(
1586 answer, self._gateway_client, None, self._fqn)
1588 for temp_arg in temp_args:
1589 temp_arg._detach()
File c:\Users\Oluwa\Documents\DATA PROJECTS\Hands-On Real Time PySpark Project for Beginners\Lab PySpark_1\PSP_1-Venv\Lib\site-packages\py4j\protocol.py:326, in get_return_value(answer, gateway_client, target_id, name)
324 value = OUTPUT_CONVERTER[type](answer[2:], gateway_client)
325 if answer == REFERENCE_TYPE:
--> 326 raise Py4JJavaError(
327 "An error occurred while calling {0}{1}{2}.\n".
328 format(target_id, ".", name), value)
329 else:
330 raise Py4JError(
331 "An error occurred while calling {0}{1}{2}. Trace:\n{3}\n".
332 format(target_id, ".", name, value))
Py4JJavaError: An error occurred while calling None.org.apache.spark.api.java.JavaSparkContext.
: java.lang.ExceptionInInitializerError
at org.apache.spark.unsafe.array.ByteArrayMethods.(ByteArrayMethods.java:56)
at org.apache.spark.memory.MemoryManager.defaultPageSizeBytes$lzycompute(MemoryManager.scala:264)
at org.apache.spark.memory.MemoryManager.defaultPageSizeBytes(MemoryManager.scala:254)
at org.apache.spark.memory.MemoryManager.$anonfun$pageSizeBytes$1(MemoryManager.scala:273)
at scala.runtime.java8.JFunction0$mcJ$sp.apply(JFunction0$mcJ$sp.java:23)
at scala.Option.getOrElse(Option.scala:189)
at org.apache.spark.memory.MemoryManager.(MemoryManager.scala:273)
at org.apache.spark.memory.UnifiedMemoryManager.(UnifiedMemoryManager.scala:58)
at org.apache.spark.memory.UnifiedMemoryManager$.apply(UnifiedMemoryManager.scala:207)
at org.apache.spark.SparkEnv$.create(SparkEnv.scala:320)
at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:194)
at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:279)
at org.apache.spark.SparkContext.(SparkContext.scala:464)
at org.apache.spark.api.java.JavaSparkContext.(JavaSparkContext.scala:58)
at java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62)
at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:502)
at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:486)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:238)
at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80)
at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69)
at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
at java.base/java.lang.Thread.run(Thread.java:1570)
Caused by: java.lang.IllegalStateException: java.lang.NoSuchMethodException: java.nio.DirectByteBuffer.(long,int)
at org.apache.spark.unsafe.Platform.(Platform.java:113)
... 25 more
Caused by: java.lang.NoSuchMethodException: java.nio.DirectByteBuffer.(long,int)
at java.base/java.lang.Class.getConstructor0(Class.java:3784)
at java.base/java.lang.Class.getDeclaredConstructor(Class.java:2955)
at org.apache.spark.unsafe.Platform.(Platform.java:71)
... 25 more
Asked by tomiealff
(1 rep)
Jul 22, 2024, 12:49 PM