PySpark で Parquet から行数をカウントしたメモ

from pyspark.sql import SparkSession
spark = SparkSession.builder.           \
  appName("ExamplePySparkSubmitTask").  \
  config("spark.databricks.hive.metastore.glueCatalog.enabled", "true"). \
  enableHiveSupport(). \
  getOrCreate()

print(spark.sparkContext.getConf().get("spark.databricks.hive.metastore.glueCatalog.enabled"))
sql("SELECT COUNT(*) FROM default.table1").show()