随笔记录程序员成长之路
配置spark streaming 的Log4j， 需要在spark-submit的--conf选项中加入jvm参数， 在--files中传入properties文件绝对路径
spark2-submit --master yarn --deploy-mode cluster \
--conf"spark.ui.port=4050" \
--files ./sparklog.properties#sparklog.properties,./hive-site.xml#hive-site.xml \
--conf spark.executor.extraJavaOptions="-Dlog4j.configuration=./sparklog.properties -Dlog_name=routing -Djava.security.auth.login.config=./flafka_jaas_spark.conf" \
--conf spark.driver.extraJavaOptions="-Dlog4j.configuration=./sparklog.properties -Dlog_name=routing -Djava.security.auth.login.config=./flafka_jaas_spark.conf" \
以下是log4j的properties内容
# Spark Streaming Logging Configuration
# See also: http://spark.apache.org/docs/2.0.2/running-on-yarn.html#debugging-your-application
log4j.rootLogger=INFO, stderr, stdout
# application namespace configuration
log4j.logger.com.anz.enbd.respbank.dataalbum=stderr
# Write all logs to standard Spark stderr file
log4j.appender.stderr=org.apache.log4j.RollingFileAppender
log4j.appender.stderr.file=/tmp/delevin/log/${log_name}.log
log4j.appender.stderr.threshold=INFO
log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
log4j.appender.stderr.layout.ConversionPattern=%d %p %c %m %n
log4j.appender.stderr.maxFileSize=50MB
log4j.appender.stderr.maxBackupIndex=10
log4j.appender.stderr.encoding=UTF-8
# Write application logs to stdout file
log4j.appender.stdout=org.apache.log4j.RollingFileAppender
log4j.appender.stdout.append=true
log4j.appender.stdout.file=${spark.yarn.app.container.log.dir}/stdout
log4j.appender.stdout.threshold=INFO
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p %c %m %n
log4j.appender.stdout.maxFileSize=50MB
log4j.appender.stdout.maxBackupIndex=10
log4j.appender.stdout.encoding=UTF-8
网友评论