Search code examples
javapostgresqljdbcgreenplumbulk-load

Java JDBC Postgres copyIn not recognizing end of line and padding double quotes


I am trying to load data from Oracle to Greenplum using Java. I store the result set as comma separated values in to byte array input stream and then load it using copy in.

import java.sql.*; 
import au.com.bytecode.opencsv.CSVWriter;
import java.io.*;
import org.postgresql.copy.CopyManager;
import org.postgresql.core.BaseConnection;

public class ORtoGP {   
        public static void main(String[] args) throws SQLException {
            try {
                String dbURL = "jdbc:oracle:thin:@(DESCRIPTION = (ADDRESS_LIST = (ADDRESS = (PROTOCOL = TCP)(HOST = xxxxxx)(PORT = 1521))) (CONNECT_DATA = (SERVICE_NAME = xxxxxx) (SRVR = DEDICATED)))";
                String strUserID = "xxxxxx";
                String strPassword = "xxxxxx";
                Connection myConnection=DriverManager.getConnection(dbURL,strUserID,strPassword);
                Statement sqlStatement = myConnection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_READ_ONLY);
                String readRecordSQL = "select id,name from table where rownum <= 10 ";
                ResultSet rs = sqlStatement.executeQuery(readRecordSQL); 

                StringWriter stringWriter = new StringWriter();
                CSVWriter csvWriter = new CSVWriter(stringWriter);

                rs.first(); 
                csvWriter.writeAll(rs, true);
                String orresult = stringWriter.toString();
                System.out.println(orresult);

                byte[] bytes = orresult.getBytes();
                ByteArrayInputStream orinput = new ByteArrayInputStream(bytes); 


                String dbURL1 = "jdbc:postgresql://xxxxx:5432/xxxxx";
                String user = "xxxx";
                String pass = "xxxx";
                Connection conn2 = DriverManager.getConnection(dbURL1, user, pass);

                CopyManager copyManager = new CopyManager((BaseConnection) conn2);
                copyManager.copyIn("copy java_test from stdin with DELIMITER ','",orinput);

                rs.close();
                myConnection.close();
                csvWriter.close();

            } catch (Exception e) {
                System.out.println(e);
            }       
        }
    }

However, I run in to two issues:

  1. While bulk loading the data, the process is unable to identify end of line. So it gives this error. "ERROR: extra data after last expected column "
  2. Also it tries to load the data including double quotes around values.

Solution

  • According to the documentation the default format is text, which does not handle quoting.

    You need to specify FORMAT csv in your command.