Would someone be able to explain how to create date partitioned table while using a loadjob in google Bigquery using JobConfig.
I couldnt understand the documentation, if someone could explain with an example it would be very helpful.
Edited: So I thought I figured out the object thanks to @irvifa, but I am still not able to create a TimePartitioned Table, here is the code am trying to use.
import pandas
from google.cloud import bigquery
def load_df(self, df):
project_id="ProjectID"
dataset_id="Dataset"
table_id="TableName"
table_ref=project_id+"."+dataset_id+"."+table_id
time_partitioning = bigquery.table.TimePartitioning(field="PartitionColumn")
job_config = bigquery.LoadJobConfig(
schema="Schema",
destinationTable=table_ref
write_disposition="WRITE_TRUNCATE",
timePartitioning=time_partitioning
)
Job = Client.load_table_from_dataframe(df, table_ref,
job_config=job_config)
Job.result()
I don't know whether it will help, but you can use the following sample to load job with partition:
from datetime import datetime, time
from concurrent import futures
import math
from pathlib import Path
from google.cloud import bigquery
def run_query(self, query_job_config):
time_partitioning = bigquery.table.TimePartitioning(field="partition_date")
job_config = bigquery.QueryJobConfig()
job_config.destination = query_job_config['destination_dataset_table']
job_config.time_partitioning = time_partitioning
job_config.use_legacy_sql = False
job_config.allow_large_results = True
job_config.write_disposition = 'WRITE_APPEND'
sql = query_job_config['sql']
query_job = self.client.query(sql, job_config=job_config)
query_job.result()