提交达梦数据

src/main/java/com/citygis/service/BigDataService.java

@@ -298,6 +298,9 @@ public class BigDataService {
 
     }
 
+
+
+//  -=================================  拉取數據
     @Test
     public void testQuery100w() throws SQLException, ClassNotFoundException {
         System.out.println("开始时间"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
@@ -587,6 +590,479 @@ public class BigDataService {
                 }
 
 
+//                StringBuffer value = new StringBuffer();
+//                for (int i = 0; i < size; i++) {
+//                    value.append(rs2.getString(i + 1)).append("\t");
+//                }
+//                // 获取输出结果并关闭连接
+//                System.out.println(value.toString());
+                rowCount++;
+            }
+            ps_sqlserver.close();
+
+//            System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  查询总行数" + rowCount);
+
+            rs2.close();
+            ps2.close();
+
+            pageIndex++;
+            begin = pageSize*(pageIndex-1);
+        }
+        conn.close();
+        System.out.println("结束时间"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+
+    }
+
+//  -=================================  拉取數據
+
+
+
+    @Test
+    public void testdm100w() throws SQLException, ClassNotFoundException {
+        System.out.println("开始时间"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        Class.forName("org.apache.hive.jdbc.HiveDriver");
+        String jdbcUrl = "jdbc:hive2://10.1.161.88:10000/default";
+        Connection conn = DriverManager.getConnection(jdbcUrl);
+
+        Class.forName("dm.jdbc.driver.DmDriver");
+        String jdbcUrl2 = "jdbc:dm://127.0.0.1:5236/test?zeroDateTimeBehavior=convertToNull&useUnicode=true&characterEncoding=utf-8";
+        Connection conn2 = DriverManager.getConnection(jdbcUrl2,"SYSDBA","SYSDBA");
+
+        PreparedStatement ps_dm = conn2.prepareStatement("SELECT count(1) FROM \"test\".\"TABLE_1_100w\"");
+        ResultSet rs_dm = ps_dm.executeQuery();
+
+        int total = 0;
+        while(rs_dm.next()) {
+            total = (int) rs_dm.getLong(1);
+        }
+        System.out.println("总数据量："+total);
+        ps_dm.close();
+//        Integer total = sqlServiceMapper.getCount();
+//        System.out.println(total);
+
+        Map<String, Object> map = new HashMap<>();
+        int pageIndex = 1;
+        int pageSize = 150000;
+        int begin = pageSize*(pageIndex-1);
+        map.put("begin",begin);
+        map.put("pageSize",pageSize);
+
+        while (begin <= total) {
+            System.out.println(begin);
+            map.put("begin",begin);
+
+            PreparedStatement ps_list = conn2.prepareStatement("SELECT * FROM \"test\".\"TABLE_1_100w\" LIMIT ?,?");
+            ps_list.setInt(1,begin);
+            ps_list.setInt(2,pageSize);
+
+            System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  第" + pageIndex + "次查询");
+
+            ResultSet rs_list = ps_list.executeQuery();
+//            List<Map<String, Object>> all = sqlServiceMapper.getAll(map);
+
+            PreparedStatement ps = conn.prepareStatement("insert into test_dm_100 values (?,?,?,?,?,?,?,?,?,?,?,?)");
+
+            while(rs_list.next()){
+                ps.setString(1, rs_list.getString(1));
+                ps.setInt(2, rs_list.getInt(2));
+                ps.setString(3, rs_list.getString(3));
+                ps.setBigDecimal(4, rs_list.getBigDecimal(4));
+                ps.setDate(5, rs_list.getDate(5));
+                ps.setString(6, rs_list.getString(6));
+                ps.setString(7, rs_list.getString(7));
+                ps.setString(8, rs_list.getString(8));
+                ps.setString(9, rs_list.getString(9));
+                ps.setString(10, rs_list.getString(10));
+                ps.setString(11, rs_list.getString(11));
+                ps.setString(12, rs_list.getString(12));
+                ps.addBatch();
+            }
+
+
+            System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  addBatch" + pageIndex);
+
+            int[] execute = ps.executeBatch();
+            ps.close();
+
+            pageIndex++;
+            begin = pageSize*(pageIndex-1);
+        }
+        conn.close();
+        System.out.println("结束时间"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+
+    }
+
+    @Test
+    public void testdm500w() throws SQLException, ClassNotFoundException {
+        System.out.println("开始时间"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        Class.forName("org.apache.hive.jdbc.HiveDriver");
+        String jdbcUrl = "jdbc:hive2://10.1.161.88:10000/default";
+        Connection conn = DriverManager.getConnection(jdbcUrl);
+
+        Class.forName("dm.jdbc.driver.DmDriver");
+        String jdbcUrl2 = "jdbc:dm://127.0.0.1:5236/test?zeroDateTimeBehavior=convertToNull&useUnicode=true&characterEncoding=utf-8";
+        Connection conn2 = DriverManager.getConnection(jdbcUrl2,"SYSDBA","SYSDBA");
+
+        PreparedStatement ps_dm = conn2.prepareStatement("SELECT count(1) FROM \"test\".\"TABLE_2_500w\"");
+        ResultSet rs_dm = ps_dm.executeQuery();
+
+        int total = 0;
+        while(rs_dm.next()) {
+            total = (int) rs_dm.getLong(1);
+        }
+        System.out.println("总数据量："+total);
+        ps_dm.close();
+//        Integer total = sqlServiceMapper.getCount();
+//        System.out.println(total);
+
+        Map<String, Object> map = new HashMap<>();
+        int pageIndex = 1;
+        int pageSize = 150000;
+        int begin = pageSize*(pageIndex-1);
+        map.put("begin",begin);
+        map.put("pageSize",pageSize);
+
+        while (begin <= total) {
+            System.out.println(begin);
+            map.put("begin",begin);
+
+            PreparedStatement ps_list = conn2.prepareStatement("SELECT * FROM \"test\".\"TABLE_2_500w\" LIMIT ?,?");
+            ps_list.setInt(1,begin);
+            ps_list.setInt(2,pageSize);
+
+            System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  第" + pageIndex + "次查询");
+
+            ResultSet rs_list = ps_list.executeQuery();
+//            List<Map<String, Object>> all = sqlServiceMapper.getAll(map);
+
+            PreparedStatement ps = conn.prepareStatement("insert into test_dm_500 values (?,?,?,?,?,?,?,?,?,?,?,?)");
+
+            while(rs_list.next()){
+                ps.setString(1, rs_list.getString(1));
+                ps.setInt(2, rs_list.getInt(2));
+                ps.setString(3, rs_list.getString(3));
+                ps.setBigDecimal(4, rs_list.getBigDecimal(4));
+                ps.setDate(5, rs_list.getDate(5));
+                ps.setString(6, rs_list.getString(6));
+                ps.setString(7, rs_list.getString(7));
+                ps.setString(8, rs_list.getString(8));
+                ps.setString(9, rs_list.getString(9));
+                ps.setString(10, rs_list.getString(10));
+                ps.setString(11, rs_list.getString(11));
+                ps.setString(12, rs_list.getString(12));
+                ps.addBatch();
+            }
+
+
+            System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  addBatch" + pageIndex);
+
+            int[] execute = ps.executeBatch();
+            ps.close();
+
+            pageIndex++;
+            begin = pageSize*(pageIndex-1);
+        }
+        conn.close();
+        System.out.println("结束时间"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+
+    }
+
+
+
+
+
+
+    @Test
+    public void testQueryDm100w() throws SQLException, ClassNotFoundException {
+        System.out.println("开始时间"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        Class.forName("org.apache.hive.jdbc.HiveDriver");
+        String jdbcUrl = "jdbc:hive2://10.1.161.88:10000/default";
+        Connection conn = DriverManager.getConnection(jdbcUrl);
+
+        PreparedStatement ps = conn.prepareStatement("SELECT count(1) FROM test_dm_100");
+        ResultSet rs = ps.executeQuery();
+
+        int total = 0;
+        while(rs.next()) {
+            total = (int) rs.getLong(1);
+        }
+        System.out.println("总数据量："+total);
+        ps.close();
+
+        int rowCount = 0;
+//        Map<String, Object> map = new HashMap<>();
+        int pageIndex = 1;
+        int pageSize = 1000000;
+        int begin = pageSize*(pageIndex-1);
+//        map.put("begin",begin);
+//        map.put("pageSize",pageSize);
+
+        System.out.println("开始查询"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        while (begin < total) {
+//        while (begin <= total) {
+//            System.out.println(begin);
+            PreparedStatement ps2 = conn.prepareStatement("SELECT * FROM test_dm_100 LIMIT ?,?");
+            ps2.setInt(1,begin);
+            ps2.setInt(2,pageSize);
+
+            ResultSet rs2 = ps2.executeQuery();
+            ResultSetMetaData rsmd = rs2.getMetaData();
+            int size = rsmd.getColumnCount();
+            while(rs2.next()) {
+//                StringBuffer value = new StringBuffer();
+//                for (int i = 0; i < size; i++) {
+//                    value.append(rs2.getString(i + 1)).append("\t");
+//                }
+//                // 获取输出结果并关闭连接
+//                System.out.println(value.toString());
+                rowCount++;
+            }
+
+            System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  查询总行数" + rowCount);
+
+            rs2.close();
+            ps2.close();
+
+            pageIndex++;
+            begin = pageSize*(pageIndex-1);
+        }
+        conn.close();
+        System.out.println("结束时间"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+
+    }
+
+    @Test
+    public void testQueryDm500w() throws SQLException, ClassNotFoundException {
+        System.out.println("开始时间"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        Class.forName("org.apache.hive.jdbc.HiveDriver");
+        String jdbcUrl = "jdbc:hive2://10.1.161.88:10000/default";
+        Connection conn = DriverManager.getConnection(jdbcUrl);
+
+        PreparedStatement ps = conn.prepareStatement("SELECT count(1) FROM test_dm_500");
+        ResultSet rs = ps.executeQuery();
+
+        int total = 0;
+        while(rs.next()) {
+            total = (int) rs.getLong(1);
+        }
+        System.out.println("总数据量："+total);
+        ps.close();
+
+        int rowCount = 0;
+//        Map<String, Object> map = new HashMap<>();
+        int pageIndex = 1;
+        int pageSize = 1000000;
+        int begin = pageSize*(pageIndex-1);
+//        map.put("begin",begin);
+//        map.put("pageSize",pageSize);
+
+        System.out.println("开始查询"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        while (begin < total) {
+//        while (begin <= total) {
+//            System.out.println(begin);
+            PreparedStatement ps2 = conn.prepareStatement("SELECT * FROM test_dm_500 LIMIT ?,?");
+            ps2.setInt(1,begin);
+            ps2.setInt(2,pageSize);
+
+            ResultSet rs2 = ps2.executeQuery();
+            ResultSetMetaData rsmd = rs2.getMetaData();
+            int size = rsmd.getColumnCount();
+            while(rs2.next()) {
+//                StringBuffer value = new StringBuffer();
+//                for (int i = 0; i < size; i++) {
+//                    value.append(rs2.getString(i + 1)).append("\t");
+//                }
+//                // 获取输出结果并关闭连接
+//                System.out.println(value.toString());
+                rowCount++;
+            }
+
+            System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  查询总行数" + rowCount);
+
+            rs2.close();
+            ps2.close();
+
+            pageIndex++;
+            begin = pageSize*(pageIndex-1);
+        }
+        conn.close();
+        System.out.println("结束时间"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+
+    }
+
+    @Test
+    public void testQueryAndDMInsert100w() throws SQLException, ClassNotFoundException {
+        System.out.println("开始时间"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        Class.forName("org.apache.hive.jdbc.HiveDriver");
+        String jdbcUrl = "jdbc:hive2://10.1.161.88:10000/default";
+        Connection conn = DriverManager.getConnection(jdbcUrl);
+
+        Class.forName("dm.jdbc.driver.DmDriver");
+        String jdbcUrl2 = "jdbc:dm://127.0.0.1:5236/test?zeroDateTimeBehavior=convertToNull&useUnicode=true&characterEncoding=utf-8";
+        Connection conn2 = DriverManager.getConnection(jdbcUrl2,"SYSDBA","SYSDBA");
+
+
+
+        PreparedStatement ps = conn.prepareStatement("SELECT count(1) FROM test_dm_100");
+        ResultSet rs = ps.executeQuery();
+
+        int total = 0;
+        while(rs.next()) {
+            total = (int) rs.getLong(1);
+        }
+        System.out.println("总数据量："+total);
+        ps.close();
+
+        int rowCount = 0;
+//        Map<String, Object> map = new HashMap<>();
+        int pageIndex = 1;
+        int pageSize = 1000000;
+        int begin = pageSize*(pageIndex-1);
+//        map.put("begin",begin);
+//        map.put("pageSize",pageSize);
+
+        int totalInsert = 0;
+
+        System.out.println("开始查询"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        while (begin < total) {
+//        while (begin <= total) {
+//            System.out.println(begin);
+            PreparedStatement ps2 = conn.prepareStatement("SELECT * FROM test_dm_100 LIMIT ?,?");
+            ps2.setInt(1,begin);
+            ps2.setInt(2,pageSize);
+
+            System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  第" + pageIndex + "次查询");
+
+            ResultSet rs2 = ps2.executeQuery();
+            ResultSetMetaData rsmd = rs2.getMetaData();
+            int size = rsmd.getColumnCount();
+            PreparedStatement ps_sqlserver = conn2.prepareStatement("insert into \"test\".\"TABLE_1_100w_in\" values (?,?,?,?,?,?,?,?,?,?,?,?)");
+            int insertcount = 0;
+            while(rs2.next()) {
+
+                ps_sqlserver.setString(1, rs2.getString(1));
+                ps_sqlserver.setInt(2, rs2.getInt(2));
+                ps_sqlserver.setString(3, rs2.getString(3));
+                ps_sqlserver.setBigDecimal(4, rs2.getBigDecimal(4));
+                ps_sqlserver.setDate(5, rs2.getDate(5));
+                ps_sqlserver.setString(6, rs2.getString(6));
+                ps_sqlserver.setString(7, rs2.getString(7));
+                ps_sqlserver.setString(8, rs2.getString(8));
+                ps_sqlserver.setString(9, rs2.getString(9));
+                ps_sqlserver.setString(10, rs2.getString(10));
+                ps_sqlserver.setString(11, rs2.getString(11));
+                ps_sqlserver.setString(12, rs2.getString(12));
+                ps_sqlserver.addBatch();
+
+                insertcount++;
+                totalInsert++;
+
+                if(insertcount == 100000){
+                    insertcount = 0;
+                    ps_sqlserver.executeBatch();
+                    ps_sqlserver.clearBatch();
+                    System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  插入总行数" + totalInsert);
+                }
+
+
+//                StringBuffer value = new StringBuffer();
+//                for (int i = 0; i < size; i++) {
+//                    value.append(rs2.getString(i + 1)).append("\t");
+//                }
+//                // 获取输出结果并关闭连接
+//                System.out.println(value.toString());
+                rowCount++;
+            }
+            ps_sqlserver.close();
+
+//            System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  查询总行数" + rowCount);
+
+            rs2.close();
+            ps2.close();
+
+            pageIndex++;
+            begin = pageSize*(pageIndex-1);
+        }
+        conn.close();
+        System.out.println("结束时间"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+
+    }
+
+    @Test
+    public void testQueryAndDMInsert500w() throws SQLException, ClassNotFoundException {
+        System.out.println("开始时间"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        Class.forName("org.apache.hive.jdbc.HiveDriver");
+        String jdbcUrl = "jdbc:hive2://10.1.161.88:10000/default";
+        Connection conn = DriverManager.getConnection(jdbcUrl);
+
+        Class.forName("dm.jdbc.driver.DmDriver");
+        String jdbcUrl2 = "jdbc:dm://127.0.0.1:5236/test?zeroDateTimeBehavior=convertToNull&useUnicode=true&characterEncoding=utf-8";
+        Connection conn2 = DriverManager.getConnection(jdbcUrl2,"SYSDBA","SYSDBA");
+
+
+
+        PreparedStatement ps = conn.prepareStatement("SELECT count(1) FROM test_dm_500");
+        ResultSet rs = ps.executeQuery();
+
+        int total = 0;
+        while(rs.next()) {
+            total = (int) rs.getLong(1);
+        }
+        System.out.println("总数据量："+total);
+        ps.close();
+
+        int rowCount = 0;
+//        Map<String, Object> map = new HashMap<>();
+        int pageIndex = 1;
+        int pageSize = 1000000;
+        int begin = pageSize*(pageIndex-1);
+//        map.put("begin",begin);
+//        map.put("pageSize",pageSize);
+
+        int totalInsert = 0;
+
+        System.out.println("开始查询"+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        while (begin < total) {
+//        while (begin <= total) {
+//            System.out.println(begin);
+            PreparedStatement ps2 = conn.prepareStatement("SELECT * FROM test_dm_500 LIMIT ?,?");
+            ps2.setInt(1,begin);
+            ps2.setInt(2,pageSize);
+
+            System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  第" + pageIndex + "次查询");
+
+            ResultSet rs2 = ps2.executeQuery();
+            ResultSetMetaData rsmd = rs2.getMetaData();
+            int size = rsmd.getColumnCount();
+            PreparedStatement ps_sqlserver = conn2.prepareStatement("insert into \"test\".\"TABLE_2_500w_in\" values (?,?,?,?,?,?,?,?,?,?,?,?)");
+            int insertcount = 0;
+            while(rs2.next()) {
+
+                ps_sqlserver.setString(1, rs2.getString(1));
+                ps_sqlserver.setInt(2, rs2.getInt(2));
+                ps_sqlserver.setString(3, rs2.getString(3));
+                ps_sqlserver.setBigDecimal(4, rs2.getBigDecimal(4));
+                ps_sqlserver.setDate(5, rs2.getDate(5));
+                ps_sqlserver.setString(6, rs2.getString(6));
+                ps_sqlserver.setString(7, rs2.getString(7));
+                ps_sqlserver.setString(8, rs2.getString(8));
+                ps_sqlserver.setString(9, rs2.getString(9));
+                ps_sqlserver.setString(10, rs2.getString(10));
+                ps_sqlserver.setString(11, rs2.getString(11));
+                ps_sqlserver.setString(12, rs2.getString(12));
+                ps_sqlserver.addBatch();
+
+                insertcount++;
+                totalInsert++;
+
+                if(insertcount == 100000){
+                    insertcount = 0;
+                    ps_sqlserver.executeBatch();
+                    ps_sqlserver.clearBatch();
+                    System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  插入总行数" + totalInsert);
+                }
+
+
 //                StringBuffer value = new StringBuffer();
 //                for (int i = 0; i < size; i++) {
 //                    value.append(rs2.getString(i + 1)).append("\t");