[17]JAVAEE-HTTP协议

Clickhouse数据库部署、Python3压测实践

一、Clickhouse数据库部署

• 版本：yandex/clickhouse-server:latest

• 部署方式：docker

• 内容

  version: "3"
  
  services:
    clickhouse:
      image: yandex/clickhouse-server:latest
      container_name: clickhouse    
      ports:
        - "8123:8123"
        - "9000:9000"
        - "9009:9009"
        - "9004:9004"
      volumes:
        - ./data/config:/var/lib/clickhouse
      ulimits:
        nproc: 65535
        nofile:
          soft: 262144
          hard: 262144
      healthcheck:
        test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"]
        interval: 30s
        timeout: 5s
        retries: 3
      deploy:
        resources:
          limits:
            cpus: '4'
            memory: 4096M
          reservations:
            memory: 4096M
  

• 建表语句

  CREATE TABLE test_table (id int,
      feild1 String, feild2 String, feild3 String
      , feild4 String, feild5 String, feild6 String
      , feild7 String, feild8 String, feild9 String
      , feild10 String, feild11 String, feild12 String
      , feild13 String, feild14 String, feild15 String
      , feild16 String, feild17 String, feild18 String
      , feild19 String, feild20 String
      ) ENGINE = MergeTree：
  

二、Python3插入数据压测

• 关键库：clickhouse_driver、 concurrent.futures

• 代码：

  import random
  import time
  from clickhouse_driver import Client
  from concurrent.futures import ThreadPoolExecutor, as_completed
  
  
  client = Client(host='ip')
  
  # 采用多个连接，避免单个连接被打死
  clients = [
      Client(host='ip'),
      Client(host='ip'),
      Client(host='ip'),
      Client(host='ip')
  ]
  
  
  # 采用批量插入，经过测试，单条并发插入支持差，每秒只能执行2-5次insert
  def task(i):
      sql = "INSERT INTO ck_table (id, feild1, feild2,feild3,feild4,feild5,feild6,feild7,feild8,feild9,feild10,feild11,feild12,feild13,feild14,feild15,feild16,feild17,feild18,feild19,feild20) VALUES"
      values = []
      for i in range(1000):
          values.append((random.randint(1,10000000),"feild1-"+str((random.randint(1,10000000))),"feild2-"+str(i),"feild3-"+str(i), "feild4-"+str(i), "feild5-"+str(i), "feild6-"+str(i), "feild7-"+str(i)
                         , "feild8-"+str(i), "feild9-"+str(i), "feild10-"+str(i), "feild11-"+str(i), "feild12-"+str(i), "feild13-"+str(i), "feild14-"+str(i)
                         , "feild15-"+str(i), "feild16-"+str(i), "feild17-"+str(i), "feild18-"+str(i), "feild19-"+str(i)
                         , "feild20-"+str(i)
                         ))
      clid = random.randint(1, len(clients)-1)
      clients[clid].execute(sql, values)
      return '第',clid, "插入",i, '条数据成功'
  
  
  if __name__ == '__main__':
      print ("程序开始运行")
      exec = ThreadPoolExecutor(max_workers=2)
      #ress = []
      start_time = time.perf_counter()
      for j in range(4000000):  # 总共需要执行的次数
          res = exec.submit(task,j)
          #ress.append(res)
      # for i in as_completed(ress):
      #     print("执行状态",i.result())
      print("执行耗时", time.perf_counter()-start_time,"s")
  

三、Python3查询数据测试

• 关键库：clickhouse_driver、 concurrent.futures

• 代码

  import time
  from concurrent.futures import ThreadPoolExecutor, as_completed
  from clickhouse_driver import Client
  
  client = Client(host='10.10.16.110')
  
  query_sql = """select * from ck_table where feild2='feild2-1009' """
  
  
  def new_task(i):
      count_sql = """ select count(*) from ck_table"""
      time.sleep(1)
      return "执行第",i,"个任务",client.execute(count_sql)
  
  
  if __name__ == '__main__':
      print ("程序开始运行")
      thd_ques = []
      exec = ThreadPoolExecutor(max_workers=1)
      ress = []
      start_time = time.perf_counter()
      for j in range(1000):
          res = exec.submit(new_task,j)
          ress.append(res)
      for i in as_completed(ress):
          print("执行状态",i.result())
      print("执行耗时", time.perf_counter()-start_time,"s")
  
  

四、测试结论