From 2631642038f781a86a2005efc4fe2d226226daed Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 13 Oct 2026 22:55:49 +0100 Subject: [PATCH 1/4] HDDS-17399. Make BlockInputStream positioned reads lock-free after initialization readPositioned copied chunkOffsets, chunkStreams and length under the BlockInputStream lock, and readChunkAt called the synchronized checkOpen() before each attempt. readWithStrategy keeps the same lock during the datanode RPC, so a positioned read waited for any sequential read on the same block. initialize() sets these fields before the volatile write of initialized and nothing changes them after that, so read them without the lock. Remove the checkOpen() calls from the positioned read path: MultipartInputStream, the only caller, checks its own volatile closed flag first. initialize() and refreshBlockInfo() stay synchronized. --- .../hdds/scm/storage/BlockInputStream.java | 24 ++++--- .../scm/storage/TestBlockInputStream.java | 63 +++++++++++++++++++ .../scm/storage/TestMultipartInputStream.java | 14 +++++ 3 files changed, 88 insertions(+), 13 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java index 90accc18137f..a567c42f1cd1 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java @@ -425,6 +425,9 @@ protected synchronized int readWithStrategy(ByteReaderStrategy strategy) * without mutating this stream's cursor ({@code chunkIndex}, * {@code blockPosition}) or the sequential chunk streams' buffered state. * Reuses each chunk's range reader, whose buffers and client references belong to the request. + * Takes this stream's lock only to initialize it or to refresh the block info after a read error, + * so it does not wait for a sequential read in progress. + * It does not check that this stream is open: {@link MultipartInputStream} checks that before it calls this method. * * @return bytes copied into {@code dst}, or {@link #EOF} at EOF */ @@ -440,20 +443,16 @@ public int readPositioned(long blockRelativePosition, ByteBuffer dst) throws IOE if (!initialized) { initialize(); } - final long[] offsets; - final List streams; - final long blockLength; - synchronized (this) { - checkOpen(); - if (chunkOffsets == null || blockData == null || blockRelativePosition >= length) { - return EOF; - } - - offsets = chunkOffsets; - streams = chunkStreams; - blockLength = length; + // initialized stays false for a block with no chunks. + if (!initialized || blockRelativePosition >= length) { + return EOF; } + // initialize() sets these fields before it sets initialized, and nothing changes them after that. + final long[] offsets = chunkOffsets; + final List streams = chunkStreams; + final long blockLength = length; + int index = MultipartInputStream.binarySearchOffsetIndex(offsets, blockRelativePosition); long pos = blockRelativePosition; @@ -476,7 +475,6 @@ public int readPositioned(long blockRelativePosition, ByteBuffer dst) throws IOE private int readChunkAt(ChunkInputStream chunk, long chunkOffset, int readLength, ByteBuffer dst) throws IOException { int preadRetries = 0; while (true) { - checkOpen(); final ByteBuffer target = dst.duplicate(); target.limit(target.position() + readLength); final int numBytesRead; diff --git a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java index 506d26ac1d8d..9106c5004fad 100644 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java @@ -43,8 +43,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import java.util.stream.Stream; @@ -548,6 +553,16 @@ public void testPositionedReadAtEof() throws Exception { assertEquals(-1, blockStream.readPositioned(blockSize + 1, ByteBuffer.allocate(1))); } + @Test + public void testPositionedReadOnBlockWithNoChunks() throws Exception { + BlockID blockID = new BlockID(new ContainerBlockID(1, 1)); + Pipeline pipeline = MockPipeline.createSingleNodePipeline(); + try (BlockInputStream subject = new DummyBlockInputStream(blockID, CHUNK_SIZE, pipeline, null, null, + refreshFunction, new ArrayList<>(), new HashMap<>(), clientConfig)) { + assertEquals(-1, subject.readPositioned(0, ByteBuffer.allocate(1))); + } + } + @Test public void testConcurrentPositionedRead() throws Exception { try (BlockInputStream largeBlockStream = createLargeBlockStream()) { @@ -563,6 +578,54 @@ public void testConcurrentPositionedRead() throws Exception { } } + @Test + public void testPositionedReadDoesNotWaitForSequentialRead() throws Exception { + CountDownLatch sequentialReadStarted = new CountDownLatch(1); + CountDownLatch releaseSequentialRead = new CountDownLatch(1); + BlockID blockID = new BlockID(new ContainerBlockID(1, 1)); + ExecutorService executor = Executors.newFixedThreadPool(2); + // The sequential read holds the BlockInputStream lock while the read of chunk-0 is blocked. + Pipeline pipeline = MockPipeline.createSingleNodePipeline(); + try (BlockInputStream subject = new DummyBlockInputStream(blockID, blockSize, pipeline, null, null, + refreshFunction, chunks, chunkDataMap, clientConfig) { + @Override + protected ChunkInputStream createChunkInputStream(ChunkInfo chunkInfo) { + if (!chunkInfo.getChunkName().equals("chunk-0")) { + return super.createChunkInputStream(chunkInfo); + } + return new DummyChunkInputStream(chunkInfo, null, null, false, chunkDataMap.get("chunk-0"), null) { + @Override + protected ByteBuffer[] readChunk(ChunkInfo readChunkInfo) { + sequentialReadStarted.countDown(); + try { + releaseSequentialRead.await(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + return super.readChunk(readChunkInfo); + } + }; + } + }) { + try { + Future sequentialRead = executor.submit(() -> subject.read(new byte[CHUNK_SIZE], 0, CHUNK_SIZE)); + assertTrue(sequentialReadStarted.await(10, TimeUnit.SECONDS)); + + ByteBuffer buf = ByteBuffer.allocate(CHUNK_SIZE); + Future positionedRead = executor.submit(() -> subject.readPositioned(2 * CHUNK_SIZE, buf)); + assertEquals(CHUNK_SIZE, positionedRead.get(5, TimeUnit.SECONDS)); + matchWithInputData(buf.array(), 2 * CHUNK_SIZE, CHUNK_SIZE); + + releaseSequentialRead.countDown(); + assertEquals(CHUNK_SIZE, sequentialRead.get(10, TimeUnit.SECONDS)); + } finally { + releaseSequentialRead.countDown(); + executor.shutdownNow(); + } + } + } + private BlockInputStream createLargeBlockStream() throws Exception { refreshFunction = mock(Function.class); BlockID blockID = new BlockID(new ContainerBlockID(1, 2)); diff --git a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestMultipartInputStream.java b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestMultipartInputStream.java index 3ab9e171f49d..a556accdc7b0 100644 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestMultipartInputStream.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestMultipartInputStream.java @@ -32,6 +32,7 @@ import com.google.common.primitives.Bytes; import java.io.EOFException; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -93,6 +94,19 @@ public void testPositionedReadPartialAtEof() throws Exception { } } + @Test + public void testPositionedReadAfterCloseFails() throws Exception { + BlockInputStream blockPart = mock(BlockInputStream.class); + when(blockPart.getLength()).thenReturn((long) PART_SIZE); + StreamBlockInputStream streamPart = stubStreamBlockPart(generateRandomData(PART_SIZE)); + MultipartInputStream multipartStream = new MultipartInputStream("test-key", Arrays.asList(blockPart, streamPart)); + multipartStream.close(); + assertThrows(IOException.class, () -> multipartStream.read(0, ByteBuffer.allocate(1))); + assertThrows(IOException.class, () -> multipartStream.read(PART_SIZE, ByteBuffer.allocate(1))); + verify(blockPart, never()).read(anyLong(), any(ByteBuffer.class)); + verify(streamPart, never()).read(anyLong(), any(ByteBuffer.class)); + } + @Test public void testStreamBlockPositionedReadPartialAtEof() throws Exception { int fileLen = 1024; From a81d037f1ea92ab27a0f3cfa2631be21d9f2bd24 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 20 Oct 2026 15:42:50 +0800 Subject: [PATCH 2/4] HDDS-17399. Make StreamBlockInputStream positioned reads lock-free readPositioned took the StreamBlockInputStream lock on each call: it called the synchronized updateBlockLength(), then ran checkOpen() and copied xceiverClientFactory in a synchronized block. The sequential read path keeps the same lock while it waits for data from the datanode, so a positioned read waited for any sequential read in progress on the same stream. - needUpdateBlockLength is now volatile. updateBlockLength() sets blockLength before it clears the flag, so readPositioned calls it only while the flag is set and then reads blockLength without the lock. - The constructor sets xceiverClientFactory, so make it final and let readPositioned use it without the lock. close() now sets a closed flag that only the lock guards, instead of setting the factory to null. - readPositioned does not check the flag: MultipartInputStream, its only caller, checks its own volatile closed flag before it calls the part. --- .../scm/storage/StreamBlockInputStream.java | 28 +++---- .../storage/TestStreamBlockInputStream.java | 76 ++++++++++++++++++- 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBlockInputStream.java index a0aac296fca8..9ec80c49893c 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/StreamBlockInputStream.java @@ -79,15 +79,16 @@ public class StreamBlockInputStream extends BlockExtendedInputStream { private final BlockID blockID; private long blockLength; // The block is under construction, so its length from OM may be stale; fetch it from the datanode once. - private boolean needUpdateBlockLength; + private volatile boolean needUpdateBlockLength; private final int responseDataSize; private final long preReadSize; private final Duration readTimeout; private final long readTimeoutNanos; private final AtomicReference pipelineRef = new AtomicReference<>(); private final AtomicReference> tokenRef = new AtomicReference<>(); - private XceiverClientFactory xceiverClientFactory; + private final XceiverClientFactory xceiverClientFactory; private XceiverClientGrpc xceiverClient; + private boolean closed; private ReadBuffer readBuffer; private long position = 0; @@ -185,6 +186,9 @@ synchronized int readFully(ByteBuffer targetBuf, boolean preRead) throws IOExcep /** * Positioned read served by a dedicated one-shot streaming read which never touches the * cursor, the pre-read buffer or the sequential reader. + * Once the block length is known, it does not take this stream's lock, so it does not wait for a sequential read + * in progress. It does not check that this stream is open: {@link MultipartInputStream} checks that before it + * calls this method. * * @param blockOffset the offset within the block to read from. * @param dst the buffer to read into. @@ -195,21 +199,19 @@ public int readPositioned(long blockOffset, ByteBuffer dst) throws IOException { if (!dst.hasRemaining()) { return 0; } - updateBlockLength(); + // updateBlockLength() sets blockLength before it clears the volatile flag, so blockLength is safe to read here. + if (needUpdateBlockLength) { + updateBlockLength(); + } if (blockOffset < 0 || blockOffset >= blockLength) { return EOF; } - final XceiverClientFactory factory; - synchronized (this) { - checkOpen(); - factory = xceiverClientFactory; - } final int length = Math.toIntExact(Math.min(dst.remaining(), blockLength - blockOffset)); final int startPosition = dst.position(); int callRetries = 0; while (true) { final Pipeline pipeline = pipelineRef.get(); - final XceiverClientGrpc client = acquireGrpcClientForReadData(factory, pipeline); + final XceiverClientGrpc client = acquireGrpcClientForReadData(xceiverClientFactory, pipeline); final StreamingReader reader = new StreamingReader(client); try { // On failure initStreamRead releases its own permit, so the reader is closed only after it succeeds. @@ -223,7 +225,7 @@ public int readPositioned(long blockOffset, ByteBuffer dst) throws IOException { dst.position(startPosition); handlePreadException(e, reader, blockOffset, callRetries++); } finally { - factory.releaseClientForReadData(client, false); + xceiverClientFactory.releaseClientForReadData(client, false); } } } @@ -405,7 +407,7 @@ private static void closeStream(StreamingReader reader) { } protected synchronized void checkOpen() throws IOException { - if (xceiverClientFactory == null) { + if (closed) { throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED + " Block: " + blockID); } } @@ -549,7 +551,7 @@ private void recordFailedStreamingDatanode(StreamingReader reader) { } protected synchronized void releaseClient() { - if (xceiverClientFactory != null && xceiverClient != null) { + if (xceiverClient != null) { closeReader("releaseClient"); xceiverClientFactory.releaseClientForReadData(xceiverClient, false); xceiverClient = null; @@ -559,7 +561,7 @@ protected synchronized void releaseClient() { @Override public synchronized void close() throws IOException { releaseClient(); - xceiverClientFactory = null; + closed = true; } private void refreshBlockInfo(IOException cause) throws IOException { diff --git a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestStreamBlockInputStream.java b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestStreamBlockInputStream.java index 7bdf10b495f7..422e8e61e587 100644 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestStreamBlockInputStream.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestStreamBlockInputStream.java @@ -85,6 +85,8 @@ import org.apache.ratis.thirdparty.io.grpc.StatusRuntimeException; import org.apache.ratis.thirdparty.io.grpc.stub.ClientCallStreamObserver; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; /** @@ -312,6 +314,72 @@ public void testConcurrentPositionedRead() throws Exception { } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testPositionedReadDoesNotWaitForSequentialRead(boolean underConstruction) throws Exception { + byte[] data = new byte[100]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) i; + } + XceiverClientGrpc client = mock(XceiverClientGrpc.class); + Pipeline pipeline = mockStandalonePipeline(); + // For an under-construction block the length from OM is stale, and the sequential read gets it from the datanode. + BlockLocationInfo blockInfo = newBlockInfo(new BlockID(1, 26), underConstruction ? 2 : data.length); + blockInfo.setUnderConstruction(underConstruction); + stubGetBlock(client, pipeline, data.length); + XceiverClientFactory factory = mock(XceiverClientFactory.class); + when(factory.acquireClientForReadData(any())).thenReturn(client); + Map readers = new ConcurrentHashMap<>(); + doAnswer(inv -> { + StreamingReaderSpi reader = inv.getArgument(1); + StreamingReadResponse response = new StreamingReadResponse( + MockDatanodeDetails.randomDatanodeDetails(), mock(ClientCallStreamObserver.class)); + reader.setStreamingReadResponse(response); + readers.put(response, reader); + return null; + }).when(client).initStreamRead(any(), any(), any()); + // The first request is the sequential read. It gets no data until the test sends it, + // so it waits in poll() and holds the StreamBlockInputStream lock. + AtomicReference sequentialReader = new AtomicReference<>(); + CountDownLatch sequentialReadStarted = new CountDownLatch(1); + doAnswer(inv -> { + ContainerCommandRequestProto request = inv.getArgument(0); + StreamingReaderSpi reader = readers.get(inv.getArgument(1)); + if (sequentialReader.compareAndSet(null, reader)) { + sequentialReadStarted.countDown(); + return null; + } + int offset = Math.toIntExact(request.getReadBlock().getOffset()); + int end = Math.toIntExact(offset + request.getReadBlock().getLength()); + reader.onNext(buildResponseProto(Arrays.copyOfRange(data, offset, end), offset)); + reader.onCompleted(); + return null; + }).when(client).streamRead(any(), any()); + OzoneClientConfig config = newStreamReadConfig(); + config.setStreamReadTimeout(Duration.ofSeconds(30)); + ExecutorService executor = Executors.newFixedThreadPool(2); + try (StreamBlockInputStream stream = new StreamBlockInputStream(blockInfo, pipeline, null, factory, NO_REFRESH, + config)) { + try { + byte[] sequential = new byte[10]; + Future sequentialRead = executor.submit(() -> stream.read(sequential, 0, sequential.length)); + assertTrue(sequentialReadStarted.await(10, TimeUnit.SECONDS)); + + ByteBuffer positioned = ByteBuffer.allocate(10); + Future positionedRead = executor.submit(() -> stream.readPositioned(50, positioned)); + assertEquals(10, positionedRead.get(10, TimeUnit.SECONDS)); + assertArrayEquals(Arrays.copyOfRange(data, 50, 60), positioned.array()); + + sequentialReader.get().onNext(buildResponseProto(Arrays.copyOfRange(data, 0, 10), 0)); + assertEquals(10, sequentialRead.get(10, TimeUnit.SECONDS)); + assertArrayEquals(Arrays.copyOfRange(data, 0, 10), sequential); + } finally { + executor.shutdownNow(); + } + } + verify(client, times(underConstruction ? 1 : 0)).sendCommand(any(ContainerCommandRequestProto.class), any()); + } + @Test public void testReleasesStreamPermitAtBlockEof() throws Exception { OzoneClientConfig clientConfig = newStreamReadConfig(); @@ -514,6 +582,11 @@ public void testCompletedBlockDoesNotGetBlockFromDatanode() throws Exception { private XceiverClientGrpc mockGetBlockClient(byte[] data, Pipeline pipeline) throws Exception { XceiverClientGrpc client = mockOffsetAwareStreamingReadClient(data, mock(ClientCallStreamObserver.class)); + stubGetBlock(client, pipeline, data.length); + return client; + } + + private static void stubGetBlock(XceiverClientGrpc client, Pipeline pipeline, long blockSize) throws Exception { when(client.getPipeline()).thenReturn(pipeline); DatanodeDetails datanode = pipeline.getFirstNode(); when(pipeline.getClosestNode(any())).thenReturn(datanode); @@ -525,10 +598,9 @@ private XceiverClientGrpc mockGetBlockClient(byte[] data, Pipeline pipeline) thr .setGetBlock(ContainerProtos.GetBlockResponseProto.newBuilder() .setBlockData(ContainerProtos.BlockData.newBuilder() .setBlockID(request.getGetBlock().getBlockID()) - .setSize(data.length))) + .setSize(blockSize))) .build(); }); - return client; } private static BlockLocationInfo newBlockInfo(BlockID blockID, long length) { From 11289511c9486a5e1b8ddd4517a14c263ad769ae Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 5 Nov 2026 14:42:59 +0800 Subject: [PATCH 3/4] HDDS-17399. Verify GetBlock through the per-datanode sendCommand after HDDS-16419 --- .../hadoop/hdds/scm/storage/TestStreamBlockInputStream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestStreamBlockInputStream.java b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestStreamBlockInputStream.java index f4695a0d9d96..f7593dea023d 100644 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestStreamBlockInputStream.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestStreamBlockInputStream.java @@ -377,7 +377,7 @@ public void testPositionedReadDoesNotWaitForSequentialRead(boolean underConstruc executor.shutdownNow(); } } - verify(client, times(underConstruction ? 1 : 0)).sendCommand(any(ContainerCommandRequestProto.class), any()); + verify(client, times(underConstruction ? 1 : 0)).sendCommand(any(ContainerCommandRequestProto.class), any(), any()); } @Test From 3766e603814ccbd123969b53df72ba07fe467723 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Thu, 5 Nov 2026 14:44:47 +0800 Subject: [PATCH 4/4] HDDS-17399. Keep the BlockInputStream chunks in one immutable object initialize() now publishes the block data, the block length, the chunk offsets and the chunk streams as one BlockChunks object in a volatile field. It replaces the initialized flag and the separate fields, so a positioned read takes one consistent snapshot without the lock, and final fields make sure that nothing changes after initialize(). This is the block and chunk part of the state split proposed in HDDS-16325. A block with no chunks still counts as not initialized, so the next read gets its block data again. A sequential read of such a block now returns EOF instead of throwing NullPointerException. --- .../hdds/scm/storage/BlockInputStream.java | 135 ++++++++++-------- .../scm/storage/TestBlockInputStream.java | 3 +- 2 files changed, 77 insertions(+), 61 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java index 4e3fea6f3711..e5fe518bc607 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java @@ -26,6 +26,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -60,7 +61,6 @@ public class BlockInputStream extends BlockExtendedInputStream { public static final Logger LOG = LoggerFactory.getLogger(BlockInputStream.class); private final BlockID blockID; - private long length; private final BlockLocationInfo blockInfo; private final AtomicReference pipelineRef = new AtomicReference<>(); @@ -72,24 +72,13 @@ public class BlockInputStream extends BlockExtendedInputStream { private XceiverClientShortCircuit xceiverClientShortCircuit; private final AtomicBoolean fallbackToGrpc = new AtomicBoolean(false); private volatile FileInputStream blockFileInputStream; - private volatile boolean initialized = false; + // Set by initialize(). Positioned reads read it without the lock. + private volatile BlockChunks blockChunks; // TODO: do we need to change retrypolicy based on exception. private final RetryPolicy retryPolicy; private int retries; - // List of ChunkInputStreams, one for each chunk in the block - private List chunkStreams; - - // chunkOffsets[i] stores the index of the first data byte in - // chunkStream i w.r.t the block data. - // Let’s say we have chunk size as 40 bytes. And let's say the parent - // block stores data from index 200 and has length 400. - // The first 40 bytes of this block will be stored in chunk[0], next 40 in - // chunk[1] and so on. But since the chunkOffsets are w.r.t the block only - // and not the key, the values in chunkOffsets will be [0, 40, 80,....]. - private long[] chunkOffsets = null; - // Index of the chunkStream corresponding to the current position of the // BlockInputStream i.e offset of the data to be read next from this block private int chunkIndex; @@ -109,8 +98,6 @@ public class BlockInputStream extends BlockExtendedInputStream { private final Function refreshFunction; - private BlockData blockData; - public BlockInputStream( BlockLocationInfo blockInfo, Pipeline pipeline, @@ -120,7 +107,6 @@ public BlockInputStream( OzoneClientConfig config) throws IOException { this.blockInfo = blockInfo; this.blockID = blockInfo.getBlockID(); - this.length = blockInfo.getLength(); pipelineRef.set(setPipeline(pipeline)); tokenRef.set(token); this.verifyChecksum = config.isChecksumVerify(); @@ -136,16 +122,16 @@ public BlockInputStream( public synchronized void initialize() throws IOException { // Pre-check that the stream has not been intialized already - if (initialized) { + if (isInitialized()) { return; } - List chunks = null; + BlockData blockData = null; + long length = blockInfo.getLength(); IOException catchEx = null; do { try { blockData = getBlockData(); - chunks = blockData.getChunksList(); if (blockInfo.isUnderConstruction()) { // use the block length from DN if block is under construction. length = blockData.getSize(); @@ -170,27 +156,28 @@ public synchronized void initialize() throws IOException { } } while (shouldRetryRead(catchEx, retryPolicy, ++retries)); - if (chunks == null) { + if (blockData == null) { throw catchEx; } else { // Reset retry count if we get chunks successfully. retries = 0; } - if (!chunks.isEmpty()) { - // For each chunk in the block, create a ChunkInputStream and compute - // its chunkOffset - this.chunkOffsets = new long[chunks.size()]; - long tempOffset = 0; - - this.chunkStreams = new ArrayList<>(chunks.size()); - for (int i = 0; i < chunks.size(); i++) { - addStream(chunks.get(i)); - chunkOffsets[i] = tempOffset; - tempOffset += chunks.get(i).getLen(); - } + // For each chunk in the block, create a ChunkInputStream and compute + // its chunkOffset + final List chunks = blockData.getChunksList(); + final long[] chunkOffsets = new long[chunks.size()]; + long tempOffset = 0; + + final List chunkStreams = new ArrayList<>(chunks.size()); + for (int i = 0; i < chunks.size(); i++) { + chunkStreams.add(createChunkInputStream(chunks.get(i))); + chunkOffsets[i] = tempOffset; + tempOffset += chunks.get(i).getLen(); + } + blockChunks = new BlockChunks(blockData, length, chunkOffsets, chunkStreams); - initialized = true; + if (!chunks.isEmpty()) { this.chunkIndex = 0; if (blockPosition > 0) { @@ -201,6 +188,12 @@ public synchronized void initialize() throws IOException { } } + /** A block with no chunks is not initialized, so the next read gets its block data again. */ + private boolean isInitialized() { + final BlockChunks chunks = blockChunks; + return chunks != null && !chunks.streams.isEmpty(); + } + private synchronized void refreshBlockInfo(IOException cause) throws IOException { refreshBlockInfo(cause, blockID, pipelineRef, tokenRef, refreshFunction); } @@ -318,14 +311,9 @@ private void acquireClient() throws IOException { } /** - * Append another ChunkInputStream to the end of the list. Note that the - * ChunkInputStream is only created here. The chunk will be read from the - * Datanode only when a read operation is performed on for that chunk. + * Create a ChunkInputStream for the chunk. The chunk will be read from the + * Datanode only when a read operation is performed on that chunk. */ - protected synchronized void addStream(ChunkInfo chunkInfo) { - chunkStreams.add(createChunkInputStream(chunkInfo)); - } - protected ChunkInputStream createChunkInputStream(ChunkInfo chunkInfo) { if (blockFileInputStream != null) { // a non-empty blockFileInputStream means we have a direct local block replica to read from @@ -341,11 +329,12 @@ protected ChunkInputStream createChunkInputStream(ChunkInfo chunkInfo) { protected synchronized int readWithStrategy(ByteReaderStrategy strategy) throws IOException { Preconditions.checkArgument(strategy != null); - if (!initialized) { + if (!isInitialized()) { initialize(); } checkOpen(); + final List chunkStreams = blockChunks.streams; int totalReadLen = 0; int len = strategy.getTargetLength(); while (len > 0) { @@ -430,18 +419,17 @@ public int readPositioned(long blockRelativePosition, ByteBuffer dst) throws IOE return EOF; } - if (!initialized) { + if (!isInitialized()) { initialize(); } - // initialized stays false for a block with no chunks. - if (!initialized || blockRelativePosition >= length) { + final BlockChunks chunks = blockChunks; + if (chunks.streams.isEmpty() || blockRelativePosition >= chunks.length) { return EOF; } - // initialize() sets these fields before it sets initialized, and nothing changes them after that. - final long[] offsets = chunkOffsets; - final List streams = chunkStreams; - final long blockLength = length; + final long[] offsets = chunks.offsets; + final List streams = chunks.streams; + final long blockLength = chunks.length; int index = MultipartInputStream.binarySearchOffsetIndex(offsets, blockRelativePosition); @@ -516,7 +504,7 @@ private int readChunkAt(ChunkInputStream chunk, long chunkOffset, int readLength */ @Override public synchronized void seek(long pos) throws IOException { - if (!initialized) { + if (!isInitialized()) { // Stream has not been initialized yet. Save the position so that it // can be seeked when the stream is initialized. blockPosition = pos; @@ -524,7 +512,10 @@ public synchronized void seek(long pos) throws IOException { } checkOpen(); - if (pos < 0 || pos > length) { + final BlockChunks chunks = blockChunks; + final long[] chunkOffsets = chunks.offsets; + final List chunkStreams = chunks.streams; + if (pos < 0 || pos > chunks.length) { if (pos == 0) { // It is possible for length and pos to be zero in which case // seek should return instead of throwing exception @@ -567,15 +558,15 @@ public synchronized void seek(long pos) throws IOException { @Override public synchronized long getPos() { - if (length == 0) { + if (getLength() == 0) { return 0; } - if (!initialized) { + if (!isInitialized()) { // The stream is not initialized yet. Return the blockPosition return blockPosition; } else { - return chunkOffsets[chunkIndex] + chunkStreams.get(chunkIndex).getPos(); + return blockChunks.offsets[chunkIndex] + blockChunks.streams.get(chunkIndex).getPos(); } } @@ -589,7 +580,7 @@ public synchronized void close() { releaseClient(); xceiverClientFactory = null; - final List inputStreams = this.chunkStreams; + final List inputStreams = getChunkStreams(); if (inputStreams != null) { for (ChunkInputStream is : inputStreams) { is.close(); @@ -633,7 +624,8 @@ public BlockID getBlockID() { @Override public long getLength() { - return length; + final BlockChunks chunks = blockChunks; + return chunks != null ? chunks.length : blockInfo.getLength(); } public synchronized int getChunkIndex() { @@ -654,7 +646,7 @@ public synchronized void unbuffer() { storePosition(); releaseClient(); - final List inputStreams = this.chunkStreams; + final List inputStreams = getChunkStreams(); if (inputStreams != null) { for (ChunkInputStream is : inputStreams) { is.unbuffer(); @@ -668,7 +660,7 @@ private synchronized void storePosition() { private synchronized void handleReadError(IOException cause) throws IOException { releaseClient(); - final List inputStreams = this.chunkStreams; + final List inputStreams = getChunkStreams(); if (inputStreams != null) { for (ChunkInputStream is : inputStreams) { is.releaseClient(); @@ -679,11 +671,34 @@ private synchronized void handleReadError(IOException cause) throws IOException } public synchronized List getChunkStreams() { - return chunkStreams; + final BlockChunks chunks = blockChunks; + return chunks != null ? chunks.streams : null; } public BlockData getStreamBlockData() { - return blockData; + final BlockChunks chunks = blockChunks; + return chunks != null ? chunks.blockData : null; } + /** The block data from the datanode and a ChunkInputStream for each chunk. Nothing changes it after initialize(). */ + private static final class BlockChunks { + private final BlockData blockData; + private final long length; + // offsets[i] stores the index of the first data byte in + // streams[i] w.r.t the block data. + // Let’s say we have chunk size as 40 bytes. And let's say the parent + // block stores data from index 200 and has length 400. + // The first 40 bytes of this block will be stored in chunk[0], next 40 in + // chunk[1] and so on. But since the offsets are w.r.t the block only + // and not the key, the values in offsets will be [0, 40, 80,....]. + private final long[] offsets; + private final List streams; + + BlockChunks(BlockData blockData, long length, long[] offsets, List streams) { + this.blockData = blockData; + this.length = length; + this.offsets = offsets; + this.streams = Collections.unmodifiableList(streams); + } + } } diff --git a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java index 9106c5004fad..f43c02d7b5e8 100644 --- a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java +++ b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java @@ -554,12 +554,13 @@ public void testPositionedReadAtEof() throws Exception { } @Test - public void testPositionedReadOnBlockWithNoChunks() throws Exception { + public void testReadOnBlockWithNoChunks() throws Exception { BlockID blockID = new BlockID(new ContainerBlockID(1, 1)); Pipeline pipeline = MockPipeline.createSingleNodePipeline(); try (BlockInputStream subject = new DummyBlockInputStream(blockID, CHUNK_SIZE, pipeline, null, null, refreshFunction, new ArrayList<>(), new HashMap<>(), clientConfig)) { assertEquals(-1, subject.readPositioned(0, ByteBuffer.allocate(1))); + assertEquals(-1, subject.read(new byte[1], 0, 1)); } }