"Failed to execute requests in bulk" in skywalking-oap-server.log

[AndySach]

Which version of SkyWalking, OS and JRE?
SkyWalking9.2.0 CentOS8 Elasticsearch7.10.0 jdk1.8.0_131

What happened?
Following error keeps appearing in skywalking-oap-server.log file. Any idea how to solve it?

2022-12-23 17:06:00,495 - org.apache.skywalking.library.elasticsearch.bulk.BulkProcessor - 155 [armeria-eventloop-epoll-5-3] ERROR [] - Failed to execute requests in bulk
java.util.concurrent.CompletionException: java.lang.RuntimeException: {"error":{"root_cause":[{"type":"null_pointer_exception","reason":null}],"type":"null_pointer_exception","reason":null},"status":500}
at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:273) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:280) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture.uniAccept(CompletableFuture.java:659) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture$UniAccept.tryFire(CompletableFuture.java:632) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) ~[?:1.8.0_131]
at com.linecorp.armeria.common.util.UnmodifiableFuture.doComplete(UnmodifiableFuture.java:152) ~[armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.util.UnmodifiableFuture.lambda$wrap$0(UnmodifiableFuture.java:103) ~[armeria-1.16.0.jar:?]
at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:822) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:797) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) ~[?:1.8.0_131]
at com.linecorp.armeria.common.stream.DeferredStreamMessage.lambda$delegate$0(DeferredStreamMessage.java:132) ~[armeria-1.16.0.jar:?]
at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:822) [?:1.8.0_131]
at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:797) [?:1.8.0_131]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) [?:1.8.0_131]
at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) [?:1.8.0_131]
at com.linecorp.armeria.common.stream.StreamMessageCollector.onComplete(StreamMessageCollector.java:67) [armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.AbstractStreamMessage$CloseEvent.notifySubscriber(AbstractStreamMessage.java:258) [armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.DefaultStreamMessage.notifySubscriberOfCloseEvent0(DefaultStreamMessage.java:300) [armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.DefaultStreamMessage.notifySubscriberOfCloseEvent(DefaultStreamMessage.java:292) [armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.DefaultStreamMessage.handleCloseEvent(DefaultStreamMessage.java:429) [armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.DefaultStreamMessage.notifySubscriber0(DefaultStreamMessage.java:372) [armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.DefaultStreamMessage.notifySubscriber(DefaultStreamMessage.java:328) [armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.DefaultStreamMessage.addObjectOrEvent(DefaultStreamMessage.java:314) [armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.DefaultStreamMessage.close(DefaultStreamMessage.java:436) [armeria-1.16.0.jar:?]
at com.linecorp.armeria.client.HttpResponseDecoder$HttpResponseWrapper.closeAction(HttpResponseDecoder.java:335) [armeria-1.16.0.jar:?]
at com.linecorp.armeria.client.HttpResponseDecoder$HttpResponseWrapper.cancelTimeoutOrLog(HttpResponseDecoder.java:370) [armeria-1.16.0.jar:?]
at com.linecorp.armeria.client.HttpResponseDecoder$HttpResponseWrapper.close(HttpResponseDecoder.java:318) [armeria-1.16.0.jar:?]
at com.linecorp.armeria.client.HttpResponseDecoder$HttpResponseWrapper.close(HttpResponseDecoder.java:308) [armeria-1.16.0.jar:?]
at com.linecorp.armeria.client.Http1ResponseDecoder.channelRead(Http1ResponseDecoder.java:245) [armeria-1.16.0.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:436) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:327) [netty-codec-4.1.77.Final.jar:4.1.77.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:299) [netty-codec-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:251) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) [netty-handler-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.handler.flush.FlushConsolidationHandler.channelRead(FlushConsolidationHandler.java:152) [netty-handler-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:800) [netty-transport-classes-epoll-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:487) [netty-transport-classes-epoll-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:385) [netty-transport-classes-epoll-4.1.77.Final.jar:4.1.77.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:995) [netty-common-4.1.77.Final.jar:4.1.77.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.77.Final.jar:4.1.77.Final]
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [netty-common-4.1.77.Final.jar:4.1.77.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
Caused by: java.lang.RuntimeException: {"error":{"root_cause":[{"type":"null_pointer_exception","reason":null}],"type":"null_pointer_exception","reason":null},"status":500}
at org.apache.skywalking.library.elasticsearch.bulk.BulkProcessor.lambda$null$3(BulkProcessor.java:144) ~[library-elasticsearch-client-9.2.0.jar:9.2.0]
at java.util.concurrent.CompletableFuture.uniAccept(CompletableFuture.java:656) ~[?:1.8.0_131]
... 57 more
2022-12-23 17:06:00,495 - org.apache.skywalking.oap.server.receiver.trace.provider.handler.v8.grpc.TraceSegmentReportServiceHandler - 75 [grpcServerPool-1-thread-11] ERROR [] - java.lang.RuntimeException: {"error":{"root_cause":[{"type":"null_pointer_exception","reason":null}],"type":"null_pointer_exception","reason":null},"status":500}
java.util.concurrent.CompletionException: java.lang.RuntimeException: {"error":{"root_cause":[{"type":"null_pointer_exception","reason":null}],"type":"null_pointer_exception","reason":null},"status":500}
at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:273) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:280) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture.uniAccept(CompletableFuture.java:659) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture$UniAccept.tryFire(CompletableFuture.java:632) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) ~[?:1.8.0_131]
at com.linecorp.armeria.common.util.UnmodifiableFuture.doComplete(UnmodifiableFuture.java:152) ~[armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.util.UnmodifiableFuture.lambda$wrap$0(UnmodifiableFuture.java:103) ~[armeria-1.16.0.jar:?]
at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:822) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:797) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) ~[?:1.8.0_131]
at com.linecorp.armeria.common.stream.DeferredStreamMessage.lambda$delegate$0(DeferredStreamMessage.java:132) ~[armeria-1.16.0.jar:?]
at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:822) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:797) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_131]
at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1962) ~[?:1.8.0_131]
at com.linecorp.armeria.common.stream.StreamMessageCollector.onComplete(StreamMessageCollector.java:67) ~[armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.AbstractStreamMessage$CloseEvent.notifySubscriber(AbstractStreamMessage.java:258) ~[armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.DefaultStreamMessage.notifySubscriberOfCloseEvent0(DefaultStreamMessage.java:300) ~[armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.DefaultStreamMessage.notifySubscriberOfCloseEvent(DefaultStreamMessage.java:292) ~[armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.DefaultStreamMessage.handleCloseEvent(DefaultStreamMessage.java:429) ~[armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.DefaultStreamMessage.notifySubscriber0(DefaultStreamMessage.java:372) ~[armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.DefaultStreamMessage.notifySubscriber(DefaultStreamMessage.java:328) ~[armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.DefaultStreamMessage.addObjectOrEvent(DefaultStreamMessage.java:314) ~[armeria-1.16.0.jar:?]
at com.linecorp.armeria.common.stream.DefaultStreamMessage.close(DefaultStreamMessage.java:436) ~[armeria-1.16.0.jar:?]
at com.linecorp.armeria.client.HttpResponseDecoder$HttpResponseWrapper.closeAction(HttpResponseDecoder.java:335) ~[armeria-1.16.0.jar:?]
at com.linecorp.armeria.client.HttpResponseDecoder$HttpResponseWrapper.cancelTimeoutOrLog(HttpResponseDecoder.java:370) ~[armeria-1.16.0.jar:?]
at com.linecorp.armeria.client.HttpResponseDecoder$HttpResponseWrapper.close(HttpResponseDecoder.java:318) ~[armeria-1.16.0.jar:?]
at com.linecorp.armeria.client.HttpResponseDecoder$HttpResponseWrapper.close(HttpResponseDecoder.java:308) ~[armeria-1.16.0.jar:?]
at com.linecorp.armeria.client.Http1ResponseDecoder.channelRead(Http1ResponseDecoder.java:245) ~[armeria-1.16.0.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:436) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:327) ~[netty-codec-4.1.77.Final.jar:4.1.77.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:299) ~[netty-codec-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:251) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:280) ~[netty-handler-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.handler.flush.FlushConsolidationHandler.channelRead(FlushConsolidationHandler.java:152) ~[netty-handler-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:800) ~[netty-transport-classes-epoll-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:487) ~[netty-transport-classes-epoll-4.1.77.Final.jar:4.1.77.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:385) ~[netty-transport-classes-epoll-4.1.77.Final.jar:4.1.77.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:995) ~[netty-common-4.1.77.Final.jar:4.1.77.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.77.Final.jar:4.1.77.Final]
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[netty-common-4.1.77.Final.jar:4.1.77.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
Caused by: java.lang.RuntimeException: {"error":{"root_cause":[{"type":"null_pointer_exception","reason":null}],"type":"null_pointer_exception","reason":null},"status":500}
at org.apache.skywalking.library.elasticsearch.bulk.BulkProcessor.lambda$null$3(BulkProcessor.java:144) ~[library-elasticsearch-client-9.2.0.jar:9.2.0]
at java.util.concurrent.CompletableFuture.uniAccept(CompletableFuture.java:656) ~[?:1.8.0_131]
... 57 more

--------- application.yml snippet
storage:
selector: ${SW_STORAGE:elasticsearch}
elasticsearch:
namespace: ${SW_NAMESPACE:"skywalking_product"}
clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:es-cn-.elasticsearch..com:9200}
protocol: ${SW_STORAGE_ES_HTTP_PROTOCOL:"http"}
connectTimeout: ${SW_STORAGE_ES_CONNECT_TIMEOUT:3000}
socketTimeout: ${SW_STORAGE_ES_SOCKET_TIMEOUT:30000}
responseTimeout: ${SW_STORAGE_ES_RESPONSE_TIMEOUT:15000}
numHttpClientThread: ${SW_STORAGE_ES_NUM_HTTP_CLIENT_THREAD:0}
user: ${SW_ES_USER:""}
password: ${SW_ES_PASSWORD:"****"}
trustStorePath: ${SW_STORAGE_ES_SSL_JKS_PATH:""}
trustStorePass: ${SW_STORAGE_ES_SSL_JKS_PASS:""}
secretsManagementFile: ${SW_ES_SECRETS_MANAGEMENT_FILE:""} # Secrets management file in the properties format includes the username, password, which are managed by 3rd party tool.
indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:6} # Shard number of new indexes/day index.
indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0} # Replicas number of new indexes
# Super data set has been defined in the codes, such as trace segments.The following 3 config would be improve es performance when storage super size data in es.
superDatasetDayStep: ${SW_SUPERDATASET_STORAGE_DAY_STEP:-1} # Represent the number of days in the super size dataset record index, the default value is the same as dayStep when the value is less than 0
superDatasetIndexShardsFactor: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_SHARDS_FACTOR:5} # This factor provides more shards for the super data set, shards number = indexShardsNumber * superDatasetIndexShardsFactor. Also, this factor effects Zipkin and Jaeger traces.
superDatasetIndexReplicasNumber: ${SW_STORAGE_ES_SUPER_DATASET_INDEX_REPLICAS_NUMBER:0} # Represent the replicas number in the super size dataset record index, the default value is 0.
indexTemplateOrder: ${SW_STORAGE_ES_INDEX_TEMPLATE_ORDER:0} # the order of index template
bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:5000} # Execute the async bulk record data every ${SW_STORAGE_ES_BULK_ACTIONS} requests
# flush the bulk every 10 seconds whatever the number of requests
# INT(flushInterval * 2/3) would be used for index refresh period.
flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:15}
concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests
resultWindowMaxSize: ${SW_STORAGE_ES_QUERY_MAX_WINDOW_SIZE:10000}
metadataQueryMaxSize: ${SW_STORAGE_ES_QUERY_MAX_SIZE:10000}
scrollingBatchSize: ${SW_STORAGE_ES_SCROLLING_BATCH_SIZE:5000}
segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200}
profileTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_PROFILE_TASK_SIZE:200}
profileDataQueryBatchSize: ${SW_STORAGE_ES_QUERY_PROFILE_DATA_BATCH_SIZE:100}
oapAnalyzer: ${SW_STORAGE_ES_OAP_ANALYZER:"{"analyzer":{"oap_analyzer":{"type":"stop"}}}"} # the oap analyzer.
oapLogAnalyzer: ${SW_STORAGE_ES_OAP_LOG_ANALYZER:"{"analyzer":{"oap_log_analyzer":{"type":"standard"}}}"} # the oap log analyzer. It could be customized by the ES analyzer configuration to support more language log formats, such as Chinese log, Japanese log and etc.
advanced: ${SW_STORAGE_ES_ADVANCED:""}
# Enable shard metrics and records indices into multi-physical indices, one index template per metric/meter aggregation function or record.
logicSharding: ${SW_STORAGE_ES_LOGIC_SHARDING:false}

[shihui-tang]

@AndySach Hi, any feedback about this issue ?

[Karl-Sparticle]

Same issue for me.

When my data retention exceeds 25 days and Elasticsearch is using HDD.
Queries easily timeout, leading to this problem.

[wu-sheng]

I can't see the reason, as the L75 of the source codes would not be NPE.

skywalking/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/handler/v8/grpc/TraceSegmentReportServiceHandler.java

Line 75 in 6b49098

errorCounter.inc();

Unless you are using some old versions, and L75 points to another line of codes.

[kanggeyyds]

I also have the same problem

2024-09-26 15:16:15,268 org.apache.skywalking.library.elasticsearch.ElasticSearch 95 [armeria-eventloop-epoll-9-4] WARN [] - [creqId=0dd088de, preqId=ad7e3273, chanId=78d2e214, laddr=xxxxxx:45748, raddr=xxxxxxx:9200][http://xxxxxxxx:9200/_bulk#POST] Request: {startTime=2024-09-26T07:16:00.255Z(1727334960255658), length=7858KiB(8047271B), duration=8818µs(8818275ns), scheme=none+h1c, name=POST, headers=[:method=POST, :path=/_bulk, :authority=xxxxxxxxx:9200, content-type=application/json, content-length=8047271, authorization=Basic xxxxxxxxxxxxxxxxxxxxxxx, user-agent=armeria/1.24.3]}
2024-09-26 15:16:15,268 org.apache.skywalking.library.elasticsearch.ElasticSearch 156 [armeria-eventloop-epoll-9-4] WARN [] - [creqId=0dd088de, preqId=ad7e3273, chanId=78d2e214, laddr=172.21.11.174:45748, raddr=25.213.85.172:9200][http://xxxxxxx:9200/_bulk#POST] Response: {startTime=2024-09-26T07:16:15.267Z(1727334975267830), length=0B, duration=0ns, totalDuration=15012ms(15012170976ns), cause=com.linecorp.armeria.client.ResponseTimeoutException, headers=[:status=0]}
com.linecorp.armeria.client.ResponseTimeoutException: null
at com.linecorp.armeria.client.ResponseTimeoutException.get(ResponseTimeoutException.java:38) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.internal.common.CancellationScheduler.invokeTask(CancellationScheduler.java:470) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.internal.common.CancellationScheduler.lambda$init0$2(CancellationScheduler.java:123) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.common.DefaultContextAwareRunnable.run(DefaultContextAwareRunnable.java:45) [armeria-1.24.3.jar:?]
at io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:98) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:153) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:406) [netty-transport-classes-epoll-4.1.86.Final.jar:4.1.86.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at java.lang.Thread.run(Thread.java:840) [?:?]
2024-09-26 15:16:15,272 org.apache.skywalking.library.elasticsearch.bulk.BulkProcessor 191 [armeria-eventloop-epoll-9-4] ERROR [] - [9.6.0-4141f56] Failed to execute requests in bulk
java.util.concurrent.CompletionException: com.linecorp.armeria.client.ResponseTimeoutException
at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:332) ~[?:?]
at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:347) ~[?:?]
at java.util.concurrent.CompletableFuture$UniAccept.tryFire(CompletableFuture.java:708) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?]
at com.linecorp.armeria.common.stream.AggregationSupport.lambda$aggregate$2(AggregationSupport.java:130) ~[armeria-1.24.3.jar:?]
at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:934) ~[?:?]
at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:911) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?]
at com.linecorp.armeria.common.stream.DeferredStreamMessage.lambda$delegate$0(DeferredStreamMessage.java:135) ~[armeria-1.24.3.jar:?]
at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:934) ~[?:?]
at java.util.concurrent.CompletableFuture.uniHandleStage(CompletableFuture.java:950) ~[?:?]
at java.util.concurrent.CompletableFuture.handle(CompletableFuture.java:2340) ~[?:?]
at com.linecorp.armeria.common.stream.DeferredStreamMessage.delegate(DeferredStreamMessage.java:131) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.common.DeferredHttpResponse.delegate(DeferredHttpResponse.java:48) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.common.DeferredHttpResponse.lambda$delegateWhenComplete$0(DeferredHttpResponse.java:59) ~[armeria-1.24.3.jar:?]
at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:934) ~[?:?]
at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:911) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
at com.linecorp.armeria.client.retry.RetryingClient.handleRetryDecision(RetryingClient.java:468) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.client.retry.RetryingClient.lambda$handleResponseWithoutContent$6(RetryingClient.java:357) ~[armeria-1.24.3.jar:?]
at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:934) ~[?:?]
at java.util.concurrent.CompletableFuture.uniHandleStage(CompletableFuture.java:950) ~[?:?]
at java.util.concurrent.CompletableFuture.handle(CompletableFuture.java:2340) ~[?:?]
at java.util.concurrent.CompletableFuture.handle(CompletableFuture.java:144) ~[?:?]
at com.linecorp.armeria.client.retry.RetryingClient.handleResponseWithoutContent(RetryingClient.java:355) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.client.retry.RetryingClient.lambda$handleResponse$9(RetryingClient.java:425) ~[armeria-1.24.3.jar:?]
at java.util.concurrent.CompletableFuture$UniAccept.tryFire(CompletableFuture.java:718) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?]
at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147) ~[?:?]
at com.linecorp.armeria.common.logging.DefaultRequestLog$RequestLogFuture.completeLog(DefaultRequestLog.java:1455) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.common.logging.DefaultRequestLog.completeSatisfiedFutures(DefaultRequestLog.java:419) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.common.logging.DefaultRequestLog.updateFlags(DefaultRequestLog.java:407) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.common.logging.DefaultRequestLog.endResponse0(DefaultRequestLog.java:1388) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.common.logging.DefaultRequestLog.endResponse0(DefaultRequestLog.java:1357) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.common.logging.DefaultRequestLog.endResponse(DefaultRequestLog.java:1343) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.client.HttpResponseDecoder$HttpResponseWrapper$1.run(HttpResponseDecoder.java:419) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.internal.common.CancellationScheduler.invokeTask(CancellationScheduler.java:484) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.internal.common.CancellationScheduler.lambda$init0$2(CancellationScheduler.java:123) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.common.DefaultContextAwareRunnable.run(DefaultContextAwareRunnable.java:45) [armeria-1.24.3.jar:?]
at io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:98) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:153) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:406) [netty-transport-classes-epoll-4.1.86.Final.jar:4.1.86.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [netty-common-4.1.86.Final.jar:4.1.86.Final]
at java.lang.Thread.run(Thread.java:840) [?:?]
Caused by: com.linecorp.armeria.client.ResponseTimeoutException
at com.linecorp.armeria.client.ResponseTimeoutException.get(ResponseTimeoutException.java:38) ~[armeria-1.24.3.jar:?]
at com.linecorp.armeria.internal.common.CancellationScheduler.invokeTask(CancellationScheduler.java:470) ~[armeria-1.24.3.jar:?]
... 12 more