Untitled

from pyspark.sql import SparkSession
spark = SparkSession.builder.getOrCreate()

# 1. PARTITION SPEC KONTROLÜ (En önemli!)
spark.sql("DESCRIBE EXTENDED iceberg.sompodatabase2.T001PSRCVP").show(100, truncate=False)

# 2. PARTITION FIELD'LARI GÖR
spark.sql("""
    SELECT * FROM iceberg.sompodatabase2.T001PSRCVP.partitions
    ORDER BY partition
""").show(50, truncate=False)

# 3. HER PARTITION'DAKİ DOSYA SAYISI
spark.sql("""
    SELECT 
        partition,
        COUNT(*) as file_count,
        ROUND(SUM(file_size_in_bytes) / 1024 / 1024 / 1024, 2) as size_gb,
        ROUND(AVG(file_size_in_bytes) / 1024 / 1024, 2) as avg_file_mb
    FROM iceberg.sompodatabase2.T001PSRCVP.files
    GROUP BY partition
    ORDER BY partition DESC
    LIMIT 20
""").show(truncate=False)

# 4. PARTITION PRUNING TEST (EXPLAIN ile)
spark.sql("""
    EXPLAIN EXTENDED
    SELECT COUNT(*) 
    FROM iceberg.sompodatabase2.T001PSRCVP
    WHERE CONFIRM_DATE >= '2024-10-01' 
      AND CONFIRM_DATE < '2024-11-01'
""").show(truncate=False)