// --------------------- partition discovery loop ---------------------
List discoveredPartitions;
// throughout the loop, we always eagerly check if we are still running before
// performing the next operation, so that we can escape the loop as soon as possible
while (running) {
if (LOG.isDebugEnabled()) {
LOG.debug(“Consumer subtask {} is trying to discover new partitions …”, getRuntimeContext().getIndexOfThisSubtask());
}
try {
discoveredPartitions = partitionDiscoverer.discoverPartitions();
} catch (AbstractPartitionDiscoverer.WakeupException | AbstractPartitionDiscoverer.ClosedException e) {
// the partition discoverer may have been closed or woken up before or during the discovery;
// this would only happen if the consumer was canceled; simply escape the loop
break;
}
// no need to add the discovered partitions if we were closed during the meantime
if (running && !discoveredPartitions.isEmpty()) {
kafkaFetcher.addDiscoveredPartitions(discoveredPartitions);
}
// do not waste any time sleeping if we’re not running anymore
if (running && discoveryIntervalMillis != 0) {
try {
Thread.sleep(discoveryIntervalMillis);
} catch (InterruptedException iex) {
// may be interrupted if the consumer was canceled midway; simply escape the loop
break;
}
}
}
} catch (Exception e) {
discoveryLoopErrorRef.set(e);
} finally {
// calling cancel will also let the fetcher loop escape
// (if not running, cancel() was already called)
if (running) {
cancel();
}
}
}
}, "Kafka Partition Discovery for " + getRuntimeContext().getTaskNameWithSubtasks());
它定义了一个线程池对象，去动态发现kafka新增的topic（支持正则形式指定消费的topic），或者动态发现kafka新增的分区。
接着肯定是启动动态发现分区或者topic线程，并且启动kafkaFetcher。
discoveryLoopThread.start();
kafkaFetcher.runFetchLoop();
// --------------------------------------------------------------------
// make sure that the partition discoverer is properly closed
partitionDiscoverer.close();
discoveryLoopThread.join();
接着，我们进入kafkaFetcher的runFetchLoop方法，映入眼帘的是
// kick off the actual Kafka consumer
consumerThread.start();
这个线程是在构建kafka09Fetcher的时候创建的
this.consumerThread = new KafkaConsumerThread(
LOG,
handover,
kafkaProperties,
unassignedPartitionsQueue,
createCallBridge(),
getFetcherName() + " for " + taskNameWithSubtasks,
pollTimeout,
useMetrics,
consumerMetricGroup,
subtaskMetricGroup);
KafkaConsumerThread 继承自Thread，然后在其run方法里，首先看到的是
// this is the means to talk to FlinkKafkaConsumer’s main thread
final Handover handover = this.handover;
这个handover的作用呢暂且不提，接着分析run方法里面内容
1，获取消费者
try {
this.consumer = getConsumer(kafkaProperties);
}
2，检测分区并且会重分配新增的分区
try {
if (hasAssignedPartitions) {
newPartitions = unassignedPartitionsQueue.pollBatch();
}
else {
// if no assigned partitions block until we get at least one
// instead of hot spinning this loop. We rely on a fact that
// unassignedPartitionsQueue will be closed on a shutdown, so
// we don’t block indefinitely
newPartitions = unassignedPartitionsQueue.getBatchBlocking();
}
if (newPartitions != null) {
reassignPartitions(newPartitions);
}
3，消费数据
// get the next batch of records, unless we did not manage to hand the old batch over
if (records == null) {
try {
records = consumer.poll(pollTimeout);
}
catch (WakeupException we) {
continue;
}
}
4，通过handover将数据发出去
try {
handover.produce(records);
records = null;
}
由于被kafkaConsumerThread打断了kafkaFetcher的runFetchLoop方法的分析，我们在这里继续
1，拉取handover.producer生产的数据
while (running) {
// this blocks until we get the next records
// it automatically re-throws exceptions encountered in the consumer thread
先自我介绍一下，小编13年上海交大毕业，曾经在小公司待过，也去过华为、OPPO等大厂，18年进入阿里一直到现在。
深知大多数初中级Java工程师，想要提升技能，往往是自己摸索成长或者是报班学习，但对于培训机构动则近万的学费，着实压力不小。自己不成体系的自学效果低效又漫长，而且极易碰到天花板技术停滞不前！
因此收集整理了一份《Java开发全套学习资料》送给大家，初衷也很简单，就是希望能够帮助到想自学提升又不知道该从何学起的朋友，同时减轻大家的负担。
由于文件比较大，这里只是将部分目录截图出来，每个节点里面都包含大厂面经、学习笔记、源码讲义、实战项目、讲解视频
如果你觉得这些内容对你有帮助，可以扫码领取！
总结
在清楚了各个大厂的面试重点之后，就能很好的提高你刷题以及面试准备的效率，接下来小编也为大家准备了最新的互联网大厂资料。
nity.csdnimg.cn/images/e5c14a7895254671a72faed303032d36.jpg" alt=“img” style=“zoom: 33%;” />
总结
在清楚了各个大厂的面试重点之后，就能很好的提高你刷题以及面试准备的效率，接下来小编也为大家准备了最新的互联网大厂资料。
[外链图片转存中…(img-750j7NCN-1711366040862)]
[外链图片转存中…(img-1tu3Kleq-1711366040862)]
[外链图片转存中…(img-Al2LSd1a-1711366040862)]
[外链图片转存中…(img-H5YUZS84-1711366040863)]
需要更多Java资料的小伙伴可以帮忙点赞+关注，点击传送门，即可免费领取！