在 SpringBoot 应用中使用 Logback 将日志发送到 ELK

发表于 分类于

RedisAppender

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.classic.spi.IThrowableProxy;
import ch.qos.logback.classic.spi.ThrowableProxyUtil;
import ch.qos.logback.core.UnsynchronizedAppenderBase;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import io.lettuce.core.RedisClient;
import io.lettuce.core.RedisURI;
import io.lettuce.core.api.async.RedisAsyncCommands;
import lombok.Getter;
import lombok.Setter;

import java.time.Duration;
import java.util.Date;

@Getter
@Setter
public class RedisAppender extends UnsynchronizedAppenderBase<ILoggingEvent> {

    private String host;
    private Integer port;
    private String password;
    private String key;

    private RedisClient redisClient;
    private RedisAsyncCommands<String, String> async;

    @Override
    protected void append(ILoggingEvent event) {
        if (redisClient == null) {
            return;
        }
        LoggerEntity entity = new LoggerEntity();
        entity.setTimestamp(new Date(event.getTimeStamp()));
        entity.setLevel(event.getLevel().toString());
        entity.setCaller(event.getLoggerName());
        entity.setThread(event.getThreadName());
        entity.setMessage(event.getFormattedMessage());
        IThrowableProxy throwableProxy = event.getThrowableProxy();
        if (throwableProxy != null) {
            entity.setThrowable(ThrowableProxyUtil.asString(throwableProxy));
        }
        async.rpush(key, JSONUtil.toJsonStr(entity));
    }

    @Override
    public void start() {
        super.start();
        initRedis();
    }

    private void initRedis() {
        if (StrUtil.isEmpty(host) || StrUtil.isEmpty(key)) {
            return;
        }
        try {
            RedisURI uri = RedisURI
                .builder()
                .withHost(host)
                .withPort(port)
                .withPassword(password.toCharArray())
                .withTimeout(Duration.ofSeconds(10))
                .build();
            redisClient = RedisClient.create(uri);
            async = redisClient.connect().async();
        } catch (Exception e) {
            System.out.printf("Initialize Logger Redis Exception: %s\n", e.getMessage());
        }
    }

    @Override
    public void stop() {
        closeRedis();
        super.stop();
    }

    private void closeRedis() {
        if (redisClient != null) {
            redisClient.close();
        }
    }

}

logback-spring.xml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
<?xml version="1.0" encoding="UTF-8"?>
<configuration>

    <include resource="org/springframework/boot/logging/logback/base.xml"/>

    <appender name="REDIS" class="xxx.RedisAppender">
        <host>${LOGGER_REDIS_HOST:-}</host>
        <port>${LOGGER_REDIS_PORT:-6379}</port>
        <password>${LOGGER_REDIS_PASSWORD:-}</password>
        <key>${LOGGER_REDIS_KEY:-log:test}</key>
    </appender>

    <root level="info">
        <appender-ref ref="CONSOLE"/>
        <!-- <appender-ref ref="FILE"/> -->
        <appender-ref ref="REDIS"/>
    </root>

</configuration>

logstash

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
input {
    redis {
        id => "test"
        host => "xxxx"
        port => "xxxx"
        password => "p1ssw0rd"
        data_type => "list"
        threads => 4
        key => "log:test"
        type => "test"
    }
}

filter {
    date {
        match => ["ts", "UNIX", "UNIX_MS", "ISO8601"]
        timezone => "Asia/Shanghai"
        target => "@timestamp"
        remove_field => ["ts"]
    }
    ruby {
        code => '
          if event.get("msg") and event.get("msg").length > 512*1024
            event.set("msg", "IGNORED LARGE MSG")
          end
        '
    }
}

output {
    elasticsearch {
        id => "log"
        hosts => "es-master:9200"
        user => "logstash_internal"
        password => "${LOGSTASH_INTERNAL_PASSWORD}"
        index => "logstash-%{type}-%{+YYYY.MM.dd}"
    }
}