目录
前言:
创建sparkSession
因为这个不是重点,所以先简单创建下,实际是需要按照你的需求创建的
// 创建SparkSession
val spark = SparkSession.builder()
.appName("Write to MySQL")
.config("spark.sql.catalogImplementation", "hive")
.getOrCreate()
一.spark读出
1. spark 读出 MySQL表数据
//读取数据
spark.read.format("jdbc")
.option("url","jdbc:mysql://address-ip:3306/ds_db01??characterEncoding=UTF-8") //url
.option("driver","com.mysql.jdbc.Driver") //驱动
.option("user","root") //用户名
.option("password","123456") //密码
.option("dbtable","product_info") //表
.load().createOrReplaceTempView("v")
spark.sql("select * from v") //查询
1.2 spark 读出 ClickHouse表数据
// 以jdbc为连接方式进行连接clickhouse
val frame = sc.read.format("jdbc")
.option("driver","ru.yandex.clickhouse.ClickHouseDriver" )// 配置driver
.option("url", "jdbc:clickhouse:/address-ip:8123/shtd_result")// 配置url
.option("dbtable", "cityavgcmpprovince")
.option("user", "default")
.option("password", "123456")
.load()
二.spark写入
1. spark 写入 MySQL表数据
// 将数据写入MySQL表
processedData.write
.format("jdbc")
.option("driver", "com.mysql.jdbc.Driver")
.option("url", "jdbc:mysql://mysql-host:3306/database")
.option("dbtable", "mysql_table")
.option("user", "username")
.option("password", "password")
.mode(SaveMode.Append)
.save()
2.spark 写入 ClickHouse表数据
// 将数据写入ClickHouse表
processeData.write
.format("jdbc")
.option("driver", "ru.yandex.clickhouse.ClickHouseDriver")
.option("url", "jdbc:clickhouse://clickhouse-host:8123/database")
.option("dbtable", "clickhouse_table")
.option("user", "username")
.option("password", "password")
.mode(SaveMode.Append)
.save()