PostgreSQL increasing the performance of a PL/pgSQL Function

Question

I have a PostgreSQL function script that is used for simulation of certain tables based on the number of records which is passed as the argument to the function. What is happening while executing the script is that it has to check for the existing records and then insert, i.e., Incremental Load, due to which the time taken is very high. How can I improve the performance of this script? I am putting up my script here:

CREATE OR REPLACE FUNCTION ccdb.perf_test_new(records bigint)
  RETURNS void AS
$BODY$

DECLARE 
i bigint;
count bigint;
bill_mnth integer;
j integer;
cin_num bigint;
BEGIN

count := records;
bill_mnth := 201400;


FOR i IN 1..count
LOOP


INSERT INTO ccdb.consumer_index_details_new
    (
        cin
        ,pole_lmdt
        ,meter_lmdt
        ,dtr_lmdt
        ,r_apdrp_town_flag
        ,town
        ,creation_dt
        ,created_by
    )
SELECT nextval('ccdb.cin_gen')
       ,now()
       ,now()
       ,now()
       ,1
       ,'Kottayam'
       ,now()
       ,'System';



INSERT INTO ccdb.consumers_new
    (   
    cin
    ,consumer_num
    ,first_name
    ,org_unit_id
    ,source_system_flag
    ,cust_area_type
    ,cons_cat_flag
    ,legacy_consumer_num
    ,cust_connection_id
    ,purpose_id
    ,tariff_id
    ,connected_load
    ,connected_load_uom
    ,consumer_phase
    ,supply_voltage
    ,connection_date
    ,bill_freq_id
    ,conn_status
    ,conn_category_group_id
    ,conn_cat_subgroup_id
    ,conn_address
    ,billing_address_1
    ,billing_address_dist
    ,permanent_address
    ,communication_address
    ,conn_owner_address
    ,pricing_type_flag
    ,conn_owner_customer_id
    ,ownership_flag
    ,district_id
    ,sanction_load
    ,sanction_load_uom
    ,advance_amount
    ,total_arrear
    ,dc_date
    ,creation_dt
    ,created_by
    ,cdemand_unit
    )
SELECT 
    ci.cin
    ,(SELECT CASE WHEN MAX(consumer_num) IS NULL THEN 1146341200001 ELSE ((MAX(consumer_num))+1) END FROM ccdb.consumers_new)
    ,'Perf_Test'||i
    ,4634
    ,1
    ,2
    ,1
    ,16876
    ,(SELECT CASE WHEN MAX(cust_connection_id) IS NULL THEN 1146340000001 ELSE ((MAX(cust_connection_id))+1) END FROM ccdb.consumers_new)
    ,15
    ,1
    ,800
    ,'W'
    ,1
    ,300
    ,now()
    ,2
    ,1
    ,1
    ,1001
    ,'SAPEER MANZIL,,KARAPUZHA P O,,KOTTAYAM,'
    ,'SAPEER MANZIL,,KARAPUZHA P O,,KOTTAYAM,'      
    ,'KOTTAYAM'
    ,'SAPEER MANZIL,,KARAPUZHA P O,,KOTTAYAM,'
    ,'SAPEER MANZIL,,KARAPUZHA P O,,KOTTAYAM,'
    ,'SAPEER MANZIL,,KARAPUZHA P O,,KOTTAYAM,'
    ,1
    ,(SELECT CASE WHEN MAX(conn_owner_customer_id) IS NULL THEN 1146340001 ELSE ((MAX(conn_owner_customer_id))+1) END FROM ccdb.consumers_new)
    ,1
    ,5
    ,600
    ,'W'
    ,45
    ,3
    ,now()
    ,now()
    ,'System'
    ,'KVA'
FROM ccdb.consumer_index_details_new ci
WHERE cin > (SELECT MAX(cin) FROM ccdb.consumers_new);


SELECT MAX(cin) INTO cin_num FROM ccdb.consumers_new;


FOR j IN 1..5
LOOP



    INSERT INTO ccdb.bills_new
                (
                    bill_id
                    ,source_system_id
                    ,mbc_bill_id
                    ,mbc_bill_no
                    ,cin
                    ,cust_connection_id
                    ,consumer_number
                    ,cust_type_flag
                    ,bill_type_group_code
                    ,bill_type_code
                    ,bill_month
                    ,total_consumption
                    ,bill_date
                    ,due_date
                    ,dc_date
                    ,org_unit_id
                    ,parent_bill_id
                    ,category_flag
                    ,status_flag
                    ,conn_cat_subgroup_id
                    ,dispute_flag
                    ,inst_flag
                    ,approved_date
                    ,bill_amt
                    ,paid_amt
                    ,creation_dt
                    ,created_by
                )
                SELECT
                    nextval('ccdb.bills_seq')
                    ,1
                    ,(SELECT CASE WHEN MAX(mbc_bill_id) IS NULL THEN 1000000001 ELSE (MAX(mbc_bill_id))+1 END FROM ccdb.bills_new)
                    ,(SELECT CASE WHEN MAX(mbc_bill_no) IS NULL THEN 4634000000001 ELSE (MAX(mbc_bill_no))+1 END FROM ccdb.bills_new)
                    ,c.cin
                    ,c.cust_connection_id
                    ,c.consumer_num
                    ,1
                    ,'EB'
                    ,'RgCC'
                    ,bill_mnth+j
                    ,400
                    ,now()
                    ,now()
                    ,now()
                    ,4634
                    ,currval('ccdb.bills_seq')
                    ,1
                    ,3
                    ,c.conn_cat_subgroup_id
                    ,0
                    ,0
                    ,now()
                    ,1000
                    ,700
                    ,now()
                    ,'System'
                FROM ccdb.consumers_new c
                WHERE c.cin = cin_num;


        INSERT INTO ccdb.bill_head_details_new
        (
            bill_id
            ,charge_head_code
            ,amount_billed
            ,amount_paid
            ,ccdb_update_time
            ,creation_dt
            ,created_by
            ,tariff_id
            ,demand_date
        )
        SELECT
            bill_id
            ,'EB'
            ,1000
            ,700
            ,now()
            ,now()
            ,'System'
            ,1
            ,now()
        FROM ccdb.bills_new
        WHERE bill_id > (SELECT MAX(bill_id) FROM ccdb.bill_head_details_new);


END LOOP;



INSERT INTO ccdb.payments_new
            (
                payment_id
                ,receipt_id
                ,source_system_flag
                ,cin
                ,consumer_nbr
                ,cust_connection_id
                ,cust_type_flg
                ,receipt_type_id
                ,mop_code
                ,coll_effect_date
                ,coll_entry_date
                ,receipt_num
                ,receipt_amt
                ,receipt_loc_flg
                ,receipt_date
                ,cancel_flag
                ,acc_type_id
                ,cust_section_code
                ,coll_section_code
                ,remarks
                ,pm_paydate
                ,pm_amount
                ,creation_dt
                ,created_by
            )
SELECT 
    nextval('ccdb.payments_seq')
    ,'REC/35747/2013'
    ,1
    ,c.cin
    ,c.consumer_num
    ,c.cust_connection_id
    ,1
    ,27
    ,2
    ,now()
    ,now()
    ,(SELECT CASE WHEN MAX(CAST(receipt_num AS bigint)) IS NULL THEN 102820110000001 ELSE (MAX(CAST(receipt_num AS bigint)))+1 END FROM ccdb.payments_new)
    ,700
    ,1
    ,now()
    ,0
    ,1
    ,4634
    ,4634
    ,'ONCOUNTER'
    ,now()
    ,700
    ,now()
    ,'System'
FROM ccdb.consumers_new c
JOIN ccdb.bills_new b ON c.consumer_num = b.consumer_number
AND c.cin > (SELECT MAX(cin) FROM ccdb.payments_new);

INSERT INTO ccdb.payment_head_dtls_new
        (
            payment_id
            ,mbc_receipt_id
            ,charge_head_code
            ,amount
            ,tariff_id
            ,creation_dt
            ,created_by
        )
SELECT
        payment_id
        ,receipt_id
        ,'EB'
        ,700
        ,1
        ,now()
        ,'System'
FROM ccdb.payments_new
WHERE payment_id > (SELECT MAX(payment_id) FROM ccdb.payment_head_dtls_new);


END LOOP;

END;

$BODY$
  LANGUAGE plpgsql VOLATILE
 COST 100;
ALTER FUNCTION ccdb.perf_test_new(bigint)
  OWNER TO postgres;

When I execute the function with no records in the table for about 10,000 records, it gets completed in about 7 minutes, but when I try to run it for another 10,000 records after the initial insertion, it continues for about 5-6 hours and still it doesn't get completed and I have to kill the job. I am relatively new to psql, can anyone suggest me on how can I increase the performance for this kind of scripts?

Thanks.

Answer 1

What you can do for your loop is that you can use an explicit cursor and try to improve on your performance.