提交达梦数据

src/main/java/com/citygis/service/KafkaTest.java

@@ -2,9 +2,11 @@ package com.citygis.service;
 
 import org.apache.kafka.clients.producer.*;
 import org.apache.kafka.clients.consumer.*;
+import org.junit.Test;
 import sun.text.resources.FormatData;
 
 import javax.xml.crypto.Data;
+import java.sql.*;
 import java.text.SimpleDateFormat;
 import java.util.Collections;
 import java.util.Date;
@@ -14,9 +16,12 @@ class KafkaTest {
     private static final String TOPIC = "test_topic_json_500w";
     private static final String BOOTSTRAP_SERVERS = "10.1.161.88:9092";
 
-    public static void main(String[] args) {
+    public static void main(String[] args) throws SQLException, ClassNotFoundException {
 
-        producer();
+//        producer();
+//        testQuery500w();
+//        testQueryfei100w();
+        testQueryfei500w();
 
     }
 
@@ -69,4 +74,242 @@ class KafkaTest {
             }
         }
     }
+
+//            test_result_json_100w  非结构
+//            test_result_json_500w
+
+//            test_result_500w  结构
+//            test_result
+
+    public static void testQuery100w() throws SQLException, ClassNotFoundException {
+        System.out.println("开始时间" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        Class.forName("org.apache.hive.jdbc.HiveDriver");
+        String jdbcUrl = "jdbc:hive2://10.1.161.88:10000/default";
+        Connection conn = DriverManager.getConnection(jdbcUrl);
+
+        PreparedStatement ps = conn.prepareStatement("SELECT count(1) FROM test_result_json_100w");
+        ResultSet rs = ps.executeQuery();
+
+        int total = 0;
+        while (rs.next()) {
+            total = (int) rs.getLong(1);
+        }
+        System.out.println("总数据量：" + total);
+        ps.close();
+
+        int rowCount = 0;
+//        Map<String, Object> map = new HashMap<>();
+        int pageIndex = 1;
+        int pageSize = 1000000;
+        int begin = pageSize * (pageIndex - 1);
+//        map.put("begin",begin);
+//        map.put("pageSize",pageSize);
+
+        System.out.println("开始查询" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        while (begin < total) {
+//        while (begin <= total) {
+//            System.out.println(begin);
+            PreparedStatement ps2 = conn.prepareStatement("SELECT * FROM test_result_json_100w LIMIT ?,?");
+            ps2.setInt(1, begin);
+            ps2.setInt(2, pageSize);
+
+            ResultSet rs2 = ps2.executeQuery();
+            ResultSetMetaData rsmd = rs2.getMetaData();
+            int size = rsmd.getColumnCount();
+            while (rs2.next()) {
+//                StringBuffer value = new StringBuffer();
+//                for (int i = 0; i < size; i++) {
+//                    value.append(rs2.getString(i + 1)).append("\t");
+//                }
+//                // 获取输出结果并关闭连接
+//                System.out.println(value.toString());
+                rowCount++;
+            }
+
+            System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  查询总行数" + rowCount);
+
+            rs2.close();
+            ps2.close();
+
+            pageIndex++;
+            begin = pageSize * (pageIndex - 1);
+        }
+        conn.close();
+        System.out.println("结束时间" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+
+    }
+
+    public static void testQuery500w() throws SQLException, ClassNotFoundException {
+        System.out.println("开始时间" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        Class.forName("org.apache.hive.jdbc.HiveDriver");
+        String jdbcUrl = "jdbc:hive2://10.1.161.88:10000/default";
+        Connection conn = DriverManager.getConnection(jdbcUrl);
+
+        PreparedStatement ps = conn.prepareStatement("SELECT count(1) FROM test_result_json_500w");
+        ResultSet rs = ps.executeQuery();
+
+        int total = 0;
+        while (rs.next()) {
+            total = (int) rs.getLong(1);
+        }
+        System.out.println("总数据量：" + total);
+        ps.close();
+
+        int rowCount = 0;
+//        Map<String, Object> map = new HashMap<>();
+        int pageIndex = 1;
+        int pageSize = 1000000;
+        int begin = pageSize * (pageIndex - 1);
+//        map.put("begin",begin);
+//        map.put("pageSize",pageSize);
+
+        System.out.println("开始查询" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        while (begin < total) {
+//        while (begin <= total) {
+//            System.out.println(begin);
+            PreparedStatement ps2 = conn.prepareStatement("SELECT * FROM test_result_json_500w LIMIT ?,?");
+            ps2.setInt(1, begin);
+            ps2.setInt(2, pageSize);
+
+            ResultSet rs2 = ps2.executeQuery();
+            ResultSetMetaData rsmd = rs2.getMetaData();
+            int size = rsmd.getColumnCount();
+            while (rs2.next()) {
+//                StringBuffer value = new StringBuffer();
+//                for (int i = 0; i < size; i++) {
+//                    value.append(rs2.getString(i + 1)).append("\t");
+//                }
+//                // 获取输出结果并关闭连接
+//                System.out.println(value.toString());
+                rowCount++;
+            }
+
+            System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  查询总行数" + rowCount);
+
+            rs2.close();
+            ps2.close();
+
+            pageIndex++;
+            begin = pageSize * (pageIndex - 1);
+        }
+        conn.close();
+        System.out.println("结束时间" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+
+    }
+
+    public static void testQueryfei500w() throws SQLException, ClassNotFoundException {
+        System.out.println("开始时间" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        Class.forName("org.apache.hive.jdbc.HiveDriver");
+        String jdbcUrl = "jdbc:hive2://10.1.161.88:10000/default";
+        Connection conn = DriverManager.getConnection(jdbcUrl);
+
+        PreparedStatement ps = conn.prepareStatement("SELECT count(1) FROM test_result_500w");
+        ResultSet rs = ps.executeQuery();
+
+        int total = 0;
+        while (rs.next()) {
+            total = (int) rs.getLong(1);
+        }
+        System.out.println("总数据量：" + total);
+        ps.close();
+
+        int rowCount = 0;
+//        Map<String, Object> map = new HashMap<>();
+        int pageIndex = 1;
+        int pageSize = 1000000;
+        int begin = pageSize * (pageIndex - 1);
+//        map.put("begin",begin);
+//        map.put("pageSize",pageSize);
+
+        System.out.println("开始查询" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        while (begin < total) {
+//        while (begin <= total) {
+//            System.out.println(begin);
+            PreparedStatement ps2 = conn.prepareStatement("SELECT * FROM test_result_500w LIMIT ?,?");
+            ps2.setInt(1, begin);
+            ps2.setInt(2, pageSize);
+
+            ResultSet rs2 = ps2.executeQuery();
+            ResultSetMetaData rsmd = rs2.getMetaData();
+            int size = rsmd.getColumnCount();
+            while (rs2.next()) {
+//                StringBuffer value = new StringBuffer();
+//                for (int i = 0; i < size; i++) {
+//                    value.append(rs2.getString(i + 1)).append("\t");
+//                }
+//                // 获取输出结果并关闭连接
+//                System.out.println(value.toString());
+                rowCount++;
+            }
+
+            System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  查询总行数" + rowCount);
+
+            rs2.close();
+            ps2.close();
+
+            pageIndex++;
+            begin = pageSize * (pageIndex - 1);
+        }
+        conn.close();
+        System.out.println("结束时间" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+
+    }
+
+    public static void testQueryfei100w() throws SQLException, ClassNotFoundException {
+        System.out.println("开始时间" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        Class.forName("org.apache.hive.jdbc.HiveDriver");
+        String jdbcUrl = "jdbc:hive2://10.1.161.88:10000/default";
+        Connection conn = DriverManager.getConnection(jdbcUrl);
+
+        PreparedStatement ps = conn.prepareStatement("SELECT count(1) FROM test_result");
+        ResultSet rs = ps.executeQuery();
+
+        int total = 0;
+        while (rs.next()) {
+            total = (int) rs.getLong(1);
+        }
+        System.out.println("总数据量：" + total);
+        ps.close();
+
+        int rowCount = 0;
+//        Map<String, Object> map = new HashMap<>();
+        int pageIndex = 1;
+        int pageSize = 1000000;
+        int begin = pageSize * (pageIndex - 1);
+//        map.put("begin",begin);
+//        map.put("pageSize",pageSize);
+
+        System.out.println("开始查询" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+        while (begin < total) {
+//        while (begin <= total) {
+//            System.out.println(begin);
+            PreparedStatement ps2 = conn.prepareStatement("SELECT * FROM test_result LIMIT ?,?");
+            ps2.setInt(1, begin);
+            ps2.setInt(2, pageSize);
+
+            ResultSet rs2 = ps2.executeQuery();
+            ResultSetMetaData rsmd = rs2.getMetaData();
+            int size = rsmd.getColumnCount();
+            while (rs2.next()) {
+//                StringBuffer value = new StringBuffer();
+//                for (int i = 0; i < size; i++) {
+//                    value.append(rs2.getString(i + 1)).append("\t");
+//                }
+//                // 获取输出结果并关闭连接
+//                System.out.println(value.toString());
+                rowCount++;
+            }
+
+            System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()) + "  查询总行数" + rowCount);
+
+            rs2.close();
+            ps2.close();
+
+            pageIndex++;
+            begin = pageSize * (pageIndex - 1);
+        }
+        conn.close();
+        System.out.println("结束时间" + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS").format(new java.util.Date()));
+
+    }
 }