Kafka Fundamental Hands On

목차

[Phase 1 : 기본 커맨드 (토픽 생성, 수정, 삭제, Producer, Consumer 구동하기)]

# 토픽 리스트 조회
$ sudo kafka-topics --bootstrap-server localhost:9092 --list
 
# 토픽 생성
$ sudo kafka-topics --create --bootstrap-server localhost:9092 --topic hands-on-jaeshim --partitions 5 --replication-factor 4 --config min.insync.replicas=2
Created topic hands-on-jaeshim.
 
# 토픽 확인
$ sudo kafka-topics     --describe     --bootstrap-server localhost:9092     --topic hands-on-jaeshim
Topic: hands-on-jaeshim TopicId: 6OgnY5iNS1-tLG8NQ4BrYA PartitionCount: 5       ReplicationFactor: 4    Configs: min.insync.replicas=2,segment.bytes=1073741824
        Topic: hands-on-jaeshim Partition: 0    Leader: 1       Replicas: 1,5,4,2       Isr: 1,5,4,2    Offline:
        Topic: hands-on-jaeshim Partition: 1    Leader: 4       Replicas: 4,1,2,3       Isr: 4,1,2,3    Offline:
        Topic: hands-on-jaeshim Partition: 2    Leader: 2       Replicas: 2,4,3,5       Isr: 2,4,3,5    Offline:
        Topic: hands-on-jaeshim Partition: 3    Leader: 3       Replicas: 3,2,5,1       Isr: 3,2,5,1    Offline:
        Topic: hands-on-jaeshim Partition: 4    Leader: 5       Replicas: 5,3,1,4       Isr: 5,3,1,4    Offline:
 
# 토픽 파티션 수 변경해보기
$ sudo kafka-topics --bootstrap-server localhost:9092 --alter --topic hands-on-jaeshim --partitions 10
 
# Producer 구동해보기
$ kafka-console-producer --bootstrap-server localhost:9092 --topic hands-on-jaeshim
> 1
> 2
> 3
> 4
> 5
 
 
# Consumer 구동해보기
## consumer.properties 작성
$ vim consumer.properties
 
partition.assignment.strategy=org.apache.kafka.clients.consumer.RangeAssignor
client.id=jaeshim-client
auto.offset.reset=earliest
 
## Consumer 구동하기
$ sudo kafka-console-consumer --bootstrap-server localhost:9092 --topic hands-on-jaeshim --group handson-jaeshim-group --consumer.config consumer.properties
1
2
3
4
5
 
## Consumer Group 확인하기
$  kafka-consumer-groups --describe --group handson-jaeshim-group --bootstrap-server localhost:9092
 
GROUP                 TOPIC               PARTITION  CURRENT-OFFSET  LOG-END-OFFSET  LAG             CONSUMER-ID                                   HOST            CLIENT-ID
handson-jaeshim-group hands-on-jaeshim    7          0               0               0               jaeshim-533a5867-b7fc-45eb-b2 7e-f0a0acc1216f /172.30.1.35    jaeshim
handson-jaeshim-group hands-on-jaeshim    4          0               0               0               jaeshim-533a5867-b7fc-45eb-b2 7e-f0a0acc1216f /172.30.1.35    jaeshim
handson-jaeshim-group hands-on-jaeshim    5          0               0               0               jaeshim-533a5867-b7fc-45eb-b2 7e-f0a0acc1216f /172.30.1.35    jaeshim
handson-jaeshim-group hands-on-jaeshim    8          0               0               0               jaeshim-533a5867-b7fc-45eb-b2 7e-f0a0acc1216f /172.30.1.35    jaeshim
handson-jaeshim-group hands-on-jaeshim    0          0               0               0               jaeshim-533a5867-b7fc-45eb-b2 7e-f0a0acc1216f /172.30.1.35    jaeshim
handson-jaeshim-group hands-on-jaeshim    9          0               0               0               jaeshim-533a5867-b7fc-45eb-b2 7e-f0a0acc1216f /172.30.1.35    jaeshim
handson-jaeshim-group hands-on-jaeshim    3          0               0               0               jaeshim-533a5867-b7fc-45eb-b2 7e-f0a0acc1216f /172.30.1.35    jaeshim
handson-jaeshim-group hands-on-jaeshim    6          0               0               0               jaeshim-533a5867-b7fc-45eb-b2 7e-f0a0acc1216f /172.30.1.35    jaeshim
handson-jaeshim-group hands-on-jaeshim    1          0               0               0               jaeshim-533a5867-b7fc-45eb-b2 7e-f0a0acc1216f /172.30.1.35    jaeshim
handson-jaeshim-group hands-on-jaeshim    2          0               0               0               jaeshim-533a5867-b7fc-45eb-b2 7e-f0a0acc1216f /172.30.1.35    jaeshim
 
# Consumer Group 삭제하기
$ kafka-consumer-groups --bootstrap-server "localhost:9092" --delete --group handson-jaeshim-group
Deletion of requested consumer groups ('handson-jaeshim-group') was successful.
 
# Topic 삭제하기
$ sudo kafka-topics --bootstrap-server localhost:9092 --delete --topic hands-on-jaeshim

[Phase 2 : Producer Partitioner 차이 느껴보기]

Round-Robin, Sticky

 

## Round Robin
# 토픽 생성
$ sudo kafka-topics --create --bootstrap-server localhost:9092 --topic jaeshim-handson-rr --partitions 5 --replication-factor 2 --config min.insync.replicas=1
 
# 구버전 Kafka 클라이언트 사용을 위해 이동
$ cd /hands-on/kafka_2.12-2.1.0/bin
 
 
# 메시지 생성
$ /home/kafka/hands-on/kafka_2.12-2.1.0/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic "jaeshim-handson-rr"
> 1
> 2
> 3
> 4
> 5
 
# 확인
$ sudo kafka-run-class kafka.tools.GetOffsetShell   --broker-list localhost:9092 --topic jaeshim-handson-rr
jaeshim-handson-rr:0:1
jaeshim-handson-rr:1:1
jaeshim-handson-rr:2:1
jaeshim-handson-rr:3:1
jaeshim-handson-rr:4:1
 
## Sticky
# 토픽 생성
$ sudo kafka-topics --create --bootstrap-server localhost:9092 --topic jaeshim-handson-stcky --partitions 5 --replication-factor 2 --config min.insync.replicas=1
 
# 메시지 생성
$  sudo kafka-console-producer --bootstrap-server localhost:9092 --topic "jaeshim-handson-stcky"
> 1
> 2
> 3
> 4
> 5
 
# 확인
$ sudo kafka-run-class kafka.tools.GetOffsetShell   --broker-list localhost:9092 --topic jaeshim-handson-stcky
 
# 토픽 삭제
$ sudo kafka-topics --bootstrap-server localhost:9092 --delete --topic jaeshim-handson-rr
$ sudo kafka-topics --bootstrap-server localhost:9092 --delete --topic jaeshim-handson-stcky

[Phase 3 : Consumer Group Assignment Strategy 비교해보기]

# 테스트용 토픽 2개 생성 (Partition 2)
$ sudo kafka-topics --create --bootstrap-server localhost:9092 --topic hands-on-jaeshim-1 --partitions 2 --replication-factor 2 --config min.insync.replicas=1
$ sudo kafka-topics --create --bootstrap-server localhost:9092 --topic hands-on-jaeshim-2 --partitions 2 --replication-factor 2 --config min.insync.replicas=1
 
# 토픽 생성 확인
$ sudo kafka-topics     --describe     --bootstrap-server localhost:9092     --topic hands-on-jaeshim-1
$ sudo kafka-topics     --describe     --bootstrap-server localhost:9092     --topic hands-on-jaeshim-2
 
# Assignment Strategy = Range 적용해서 Consumer 구동해보기
$ sudo kafka-console-consumer --bootstrap-server localhost:9092 --whitelist 'hands-on-jaeshim-1|hands-on-jaeshim-2' --group hands-on-jaeshim-group --consumer.config consumer1.properties
$ sudo kafka-console-consumer --bootstrap-server localhost:9092 --whitelist 'hands-on-jaeshim-1|hands-on-jaeshim-2' --group hands-on-jaeshim-group --consumer.config consumer2.properties
$ sudo kafka-console-consumer --bootstrap-server localhost:9092 --whitelist 'hands-on-jaeshim-1|hands-on-jaeshim-2' --group hands-on-jaeshim-group --consumer.config consumer3.properties
 
# Consumer-Group 확인
$ kafka-consumer-groups --describe --group hands-on-jaeshim-group --bootstrap-server localhost:9092
 
# Assignment Strategy = Round Robin 적용해서 Consumer 구동해보기 (순서대로 Assignor 변경)
$ vim consumer1.properties
$ vim consumer2.properties
$ vim consumer3.properties

#consumer.properties내용
partition.assignment.strategy=org.apache.kafka.clients.consumer.RoundRobinAssignor
#partition.assignment.strategy=org.apache.kafka.clients.consumer.StickyAssignor

 
# Assignment Strategy = RoundRobin 적용해서 Consumer 구동해보기
$ sudo kafka-console-consumer --bootstrap-server localhost:9092 --whitelist 'hands-on-jaeshim-1|hands-on-jaeshim-2' --group hands-on-jaeshim-group --consumer.config consumer1.properties
$ sudo kafka-console-consumer --bootstrap-server localhost:9092 --whitelist 'hands-on-jaeshim-1|hands-on-jaeshim-2' --group hands-on-jaeshim-group --consumer.config consumer2.properties
$ sudo kafka-console-consumer --bootstrap-server localhost:9092 --whitelist 'hands-on-jaeshim-1|hands-on-jaeshim-2' --group hands-on-jaeshim-group --consumer.config consumer3.properties  
 
# Consumer-Group 확인
$ kafka-consumer-groups --describe --group hands-on-jaeshim-group --bootstrap-server localhost:9092
 
 
# Consumer-Group 삭제
$ kafka-consumer-groups --bootstrap-server "localhost:9092" --delete --group hands-on-jaeshim-group

[Phase 4 : DEBUG 레벨로 Consumer 동작 상세히 보기 (Rebalancing, Heartbeat) ]

# debug consumer 경로로 이동
$ cd console-consumer-debug
 
# 스크립트 확인
$ vim kafka-console-consumer-debug.sh
 
if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
    export KAFKA_HEAP_OPTS="-Xmx512M"
fi
export KAFKA_OPTS="-Dlog4j.configuration=file:log4j.properties"
exec /usr/bin/kafka-run-class kafka.tools.ConsoleConsumer "$@"
 
# log4j.properties 확인
$ vim log4j.properties
 
# Consumer Group 가동하기
$ sudo  ./kafka-console-consumer-debug.sh --bootstrap-server localhost:9092 --whitelist 'hands-on-jaeshim-1|hands-on-jaeshim-2' --group hands-on-jaeshim-group --consumer.config consumer1.properties
$ sudo  ./kafka-console-consumer-debug.sh --bootstrap-server localhost:9092 --whitelist 'hands-on-jaeshim-1|hands-on-jaeshim-2' --group hands-on-jaeshim-group --consumer.config consumer2.properties
$ sudo  ./kafka-console-consumer-debug.sh --bootstrap-server localhost:9092 --whitelist 'hands-on-jaeshim-1|hands-on-jaeshim-2' --group hands-on-jaeshim-group --consumer.config consumer3.properties
 
# 로그 파일로 추출하기
$ sudo  ./kafka-console-consumer-debug.sh --bootstrap-server localhost:9092 --whitelist 'hands-on-jaeshim-1|hands-on-jaeshim-2' --group hands-on-jaeshim-group --consumer.config consumer3.properties 2> consumer3.log

[Phase 5 : Trouble Shooting (UnderReplicated, UnderMinISR)]

PerfProducer 세팅

# 20분간 produce
$ sudo kafka-producer-perf-test \
    --topic hands-on-jaeshim \
    --num-records 12000 \
    --record-size 1 \
    --throughput 10 \
    --producer-props acks=-1 \
    bootstrap.servers=localhost:9092

UnderReplicated

# 토픽 생성
$ sudo kafka-topics --create --bootstrap-server localhost:9092 --topic hands-on-jaeshim --partitions 5 --replication-factor 5 --config min.insync.replicas=4
 
# 토픽 확인
$ sudo kafka-topics --describe --bootstrap-server localhost:9092 --topic hands-on-jaeshim
 
# perf-producer 세팅
 
 
# broker down 진행
# grafana, control-center 확인
 
# under replicated 확인
$ sudo kafka-topics --describe --bootstrap-server localhost:9092 --under-replicated-partitions | grep "hands-on-jaeshim"
 
# under min isr 확인
$ sudo kafka-topics --describe --bootstrap-server localhost:9092 --under-min-isr-partitions | grep "hands-on-jaeshim"
 
# unavailable 확인
$ sudo kafka-topics --describe --bootstrap-server localhost:9092 --unavailable-partitions | grep "hands-on-jaeshim"
 
# offset 개수 확인
$ watch 'sudo kafka-run-class kafka.tools.GetOffsetShell   --broker-list localhost:9092 --topic hands-on-jaeshim'
 
# broker up

UnderMinISR

# min isr 증가
$ sudo kafka-configs \
--bootstrap-server localhost:9092 \
--entity-type topics \
--entity-name hands-on-jaeshim \
--alter \
--add-config min.insync.replicas=5
 
# 토픽 확인
$ sudo kafka-topics --describe --bootstrap-server localhost:9092 --topic hands-on-jaeshim
 
# perf-producer 세팅
 
# broker down 진행
# under replicated 확인
$ sudo kafka-topics --describe --bootstrap-server localhost:9092 --under-replicated-partitions | grep "hands-on-jaeshim"
 
# under min isr 확인
$ sudo kafka-topics --describe --bootstrap-server localhost:9092 --under-min-isr-partitions | grep "hands-on-jaeshim"
 
# unavailable 확인
$ sudo kafka-topics --describe --bootstrap-server localhost:9092 --unavailable-partitions | grep "hands-on-jaeshim"
 
# offset 개수 확인
$ watch 'sudo kafka-run-class kafka.tools.GetOffsetShell   --broker-list localhost:9092 --topic hands-on-jaeshim'
 
# min isr 수정
$ sudo kafka-configs \
--bootstrap-server localhost:9092 \
--entity-type topics \
--entity-name hands-on-jaeshim \
--alter \
--add-config min.insync.replicas=4
 
# 토픽 삭제
$ sudo kafka-topics --bootstrap-server localhost:9092 --delete --topic hands-on-jaeshim
 
----------------------------------------------------------------------------------------------------------
## under-min-isr 상황에서 producer retries = 0으로 줘서 메시지 유실되는 상황 짐작해보기.
 
# 토픽 생성
$ sudo kafka-topics --create --bootstrap-server localhost:9092 --topic hands-on-jaeshim --partitions 5 --replication-factor 2 --config min.insync.replicas=2
 
# 토픽 확인
$ sudo kafka-topics --describe --bootstrap-server localhost:9092 --topic hands-on-jaeshim
 
# perf-producer 세팅
$ sudo kafka-producer-perf-test     --topic hands-on-jaeshim     --num-records 12000     --rerd-size 1     --throughput 10     --producer-props acks=-1     bootstrap.servers=localhost:9092 retries=0
 
# broker down 진행
# under replicated 확인
$ sudo kafka-topics --describe --bootstrap-server localhost:9092 --under-replicated-partitions | grep "hands-on-jaeshim"
 
# under min isr 확인
$ sudo kafka-topics --describe --bootstrap-server localhost:9092 --under-min-isr-partitions | grep "hands-on-jaeshim"
 
# unavailable 확인
$ sudo kafka-topics --describe --bootstrap-server localhost:9092 --unavailable-partitions | grep "hands-on-jaeshim"
 
# offset 개수 확인
$ watch 'sudo kafka-run-class kafka.tools.GetOffsetShell   --broker-list localhost:9092 --topic hands-on-jaeshim'