Glue Spark ジョブや Jupyter Notebook の glueContext で spark.conf.set() する方法
import sys from awsglue.transforms import * from awsglue.utils import getResolvedOptions from pyspark.context import SparkContext, SparkConf ★ from awsglue.context import GlueContext from awsglue.job import Job conf = SparkConf().set("spark.sql.parquet.enableVectorizedReader", "false") ★ sc = SparkContext.getOrCreate(conf=conf) ★ glueContext = GlueContext(sc) spark = glueContext.spark_session job = Job(glueContext) job.init('job01')
