最简单实现高并发插入数万条数据(可同步可异步)

获取数据| 整理数据

        
        public static int count = 0;
        
        String uniqueTimeId = IDUtils.getInstance().getUniqueTimeId();

        String sqlof1 = "select vc_scode from ( "+sqlExpr+" )";

        String sqlof2 = SqlExprConstant.pool_sql.replace(SqlExprConstant.POOL_TEMPLATE, poolId);
        
        String joiner = " minus ";
        
        //查询筛选结果
        List<Map<String, Object>> reduce1 = riskruleService.executeSql(sqlof1+joiner+sqlof2);
        

        //插入数据库  121_证券筛选执行结果
        List<String> keys1 = reduce1.stream().filter(map -> !Objects.isNull(map.get("VC_SCODE")))
                                      .map(i ->(String)i.get("VC_SCODE"))
                                      .distinct().collect(Collectors.toList());
                                      
        
        //并发插入数据
        CurrencyAdd(keys1.size(), 100, keys1, uniqueTimeId, "1");       

高并发插入上万条数据

    /**
     * 高并发插入上万条数据
     * @param totalCount
     * @param threadTotal
     * @throws InterruptedException
     * @author Bruce
     */
    public void  CurrencyAdd(int totalCount ,int threadTotal,List<String> keys,String uniqueTimeId,String cCompType) throws InterruptedException {
        ExecutorService executorService = Executors.newCachedThreadPool();
        // 限制同时执行的线程数
        final Semaphore semaphore = new Semaphore(threadTotal);
        final CountDownLatch countDownLatch = new CountDownLatch(totalCount);

        for (int i = 0; i < totalCount; i++) {
            executorService.execute(() -> {
                try {
                    semaphore.acquire();
                    add(cCompType);
                    //根据下标获取取值插入
                    log.info("current count no is --->{}",count);
                    CompSecuFilterRes res = new CompSecuFilterRes();
                    String code = "";
                    
                    code = keys.get(count-1);
                    
                    res.setVcScode(code );
                    res.setVcFilterSeqno(uniqueTimeId);
                    res.setCCompType(cCompType);
                    res.setDMdftime(new Date());
                    riskruleService.addCompSecuFilterRes(res);
                    semaphore.release();
                } catch (InterruptedException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
                countDownLatch.countDown();

            });
        }

        //异步不等待执行过程就把这行注掉
        //countDownLatch.await();
        executorService.shutdown();
        log.info("count{}", count);
    }
    
    private synchronized static void add(String type ) {
        
        count++;
       
    }

日志信息

...

2019-03-27 at 18:38:37 CST INFO  com.xx.xxx.controller.xxxxController 442 lambda$4 - current count no is --->97
2019-03-27 at 18:38:37 CST DEBUG org.apache.ibatis.logging.jdbc.BaseJdbcLogger 159 debug - ==>  Preparing: insert into T_COMP_SECU_FILTER_RES (VC_FILTER_SEQNO, C_REC_TYPE, VC_SCODE, C_COMP_TYPE, D_MDFTIME) values (?, ?, ?, ?, ?) 
2019-03-27 at 18:38:37 CST DEBUG org.apache.ibatis.logging.jdbc.BaseJdbcLogger 159 debug - ==> Parameters: 1553683109754193(String), null, 011801593YH(String), 2(String), 2019-03-27(Date)
2019-03-27 at 18:38:37 CST DEBUG org.apache.ibatis.logging.jdbc.BaseJdbcLogger 159 debug - <==    Updates: 1
2019-03-27 at 18:38:37 CST INFO  com.xx.xxx.controller.xxxxController 442 lambda$4 - current count no is --->98
2019-03-27 at 18:38:37 CST DEBUG org.apache.ibatis.logging.jdbc.BaseJdbcLogger 159 debug - ==>  Preparing: insert into T_COMP_SECU_FILTER_RES (VC_FILTER_SEQNO, C_REC_TYPE, VC_SCODE, C_COMP_TYPE, D_MDFTIME) values (?, ?, ?, ?, ?) 
2019-03-27 at 18:38:37 CST DEBUG org.apache.ibatis.logging.jdbc.BaseJdbcLogger 159 debug - ==> Parameters: 1553683109754193(String), null, 041758030YH(String), 1(String), 2019-03-27(Date)
2019-03-27 at 18:38:37 CST DEBUG org.apache.ibatis.logging.jdbc.BaseJdbcLogger 159 debug - <==    Updates: 1
2019-03-27 at 18:38:37 CST INFO  com.xx.xxx.controller.xxxxController 442 lambda$4 - current count no is --->99

...

转载请注明来源。
我的博客：http://blog.northpark.cn
本文同步地址：http://blog.northpark.cn/2019/03/28/%E6%9C%80%E7%AE%80%E5%8D%95%E5%AE%9E%E7%8E%B0%E9%AB%98%E5%B9%B6%E5%8F%91%E6%8F%92%E5%85%A5%E6%95%B0%E4%B8%87%E6%9D%A1%E6%95%B0%E6%8D%AE(%E5%8F%AF%E5%90%8C%E6%AD%A5%E5%8F%AF%E5%BC%82%E6%AD%A5)/