Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
Expand Down Expand Up @@ -60,7 +61,6 @@ public class BlockInputStream extends BlockExtendedInputStream {
public static final Logger LOG = LoggerFactory.getLogger(BlockInputStream.class);

private final BlockID blockID;
private long length;
private final BlockLocationInfo blockInfo;
private final AtomicReference<Pipeline> pipelineRef =
new AtomicReference<>();
Expand All @@ -72,24 +72,13 @@ public class BlockInputStream extends BlockExtendedInputStream {
private XceiverClientShortCircuit xceiverClientShortCircuit;
private final AtomicBoolean fallbackToGrpc = new AtomicBoolean(false);
private volatile FileInputStream blockFileInputStream;
private volatile boolean initialized = false;
// Set by initialize(). Positioned reads read it without the lock.
private volatile BlockChunks blockChunks;
// TODO: do we need to change retrypolicy based on exception.
private final RetryPolicy retryPolicy;

private int retries;

// List of ChunkInputStreams, one for each chunk in the block
private List<ChunkInputStream> chunkStreams;

// chunkOffsets[i] stores the index of the first data byte in
// chunkStream i w.r.t the block data.
// Let’s say we have chunk size as 40 bytes. And let's say the parent
// block stores data from index 200 and has length 400.
// The first 40 bytes of this block will be stored in chunk[0], next 40 in
// chunk[1] and so on. But since the chunkOffsets are w.r.t the block only
// and not the key, the values in chunkOffsets will be [0, 40, 80,....].
private long[] chunkOffsets = null;

// Index of the chunkStream corresponding to the current position of the
// BlockInputStream i.e offset of the data to be read next from this block
private int chunkIndex;
Expand All @@ -109,8 +98,6 @@ public class BlockInputStream extends BlockExtendedInputStream {

private final Function<BlockID, BlockLocationInfo> refreshFunction;

private BlockData blockData;

public BlockInputStream(
BlockLocationInfo blockInfo,
Pipeline pipeline,
Expand All @@ -120,7 +107,6 @@ public BlockInputStream(
OzoneClientConfig config) throws IOException {
this.blockInfo = blockInfo;
this.blockID = blockInfo.getBlockID();
this.length = blockInfo.getLength();
pipelineRef.set(setPipeline(pipeline));
tokenRef.set(token);
this.verifyChecksum = config.isChecksumVerify();
Expand All @@ -136,18 +122,18 @@ public BlockInputStream(
public synchronized void initialize() throws IOException {

// Pre-check that the stream has not been intialized already
if (initialized) {
if (isInitialized()) {
return;
}
// A closed stream has no client to get the block with.
checkOpen();

List<ChunkInfo> chunks = null;
BlockData blockData = null;
long length = blockInfo.getLength();
IOException catchEx = null;
do {
try {
blockData = getBlockData();
chunks = blockData.getChunksList();
if (blockInfo.isUnderConstruction()) {
// use the block length from DN if block is under construction.
length = blockData.getSize();
Expand All @@ -172,27 +158,28 @@ public synchronized void initialize() throws IOException {
}
} while (shouldRetryRead(catchEx, retryPolicy, ++retries));

if (chunks == null) {
if (blockData == null) {
throw catchEx;
} else {
// Reset retry count if we get chunks successfully.
retries = 0;
}
Comment on lines 131 to 166

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's also add an getBlockDataWithRetries() method

    checkOpen();

    final BlockData blockData = getBlockDataWithRetries();
    final long length = blockInfo.isUnderConstruction() ? blockData.getSize() : blockInfo.getLength();

    // For each chunk in the block, create a ChunkInputStream and compute its chunkOffset
    final List<ChunkInfo> chunks = blockData.getChunksList();
  private BlockData getBlockDataWithRetries() throws IOException {
    for (;;) {
      final IOException exception;
      try {
        final BlockData data = getBlockData();
        retries = 0;
        return data;

        // If we get a StorageContainerException or an IOException due to datanodes are unreachable,
        // refresh pipeline and retry. Otherwise, retry according to the retry policy.
      } catch (SCMSecurityException ex) {
        throw ex;
      } catch (StorageContainerException e) {
        refreshBlockInfo(e);
        exception = e;
      } catch (IOException e) {
        LOG.debug("Retry to get chunk info fail", e);
        if (isConnectivityIssue(e)) {
          refreshBlockInfo(e);
        }
        exception = e;
      }

      if (!shouldRetryRead(exception, retryPolicy, ++retries)) {
        throw exception;
      }
    }
  }


if (!chunks.isEmpty()) {
// For each chunk in the block, create a ChunkInputStream and compute
// its chunkOffset
this.chunkOffsets = new long[chunks.size()];
long tempOffset = 0;

this.chunkStreams = new ArrayList<>(chunks.size());
for (int i = 0; i < chunks.size(); i++) {
addStream(chunks.get(i));
chunkOffsets[i] = tempOffset;
tempOffset += chunks.get(i).getLen();
}
// For each chunk in the block, create a ChunkInputStream and compute
// its chunkOffset
final List<ChunkInfo> chunks = blockData.getChunksList();
final long[] chunkOffsets = new long[chunks.size()];
long tempOffset = 0;

initialized = true;
final List<ChunkInputStream> chunkStreams = new ArrayList<>(chunks.size());
for (int i = 0; i < chunks.size(); i++) {
chunkStreams.add(createChunkInputStream(chunks.get(i)));
chunkOffsets[i] = tempOffset;
tempOffset += chunks.get(i).getLen();
}
blockChunks = new BlockChunks(blockData, length, chunkOffsets, chunkStreams);
Comment on lines +170 to +180

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's add a newBlockChunks(..) method:

  private BlockChunks newBlockChunks(BlockData blockData, long length) {
    final List<ChunkInfo> chunks = blockData.getChunksList();
    final long[] chunkOffsets = new long[chunks.size()];
    long tempOffset = 0;

    // For each chunk, create a ChunkInputStream and compute its chunkOffset
    final List<ChunkInputStream> chunkStreams = new ArrayList<>(chunks.size());
    for (int i = 0; i < chunks.size(); i++) {
      chunkStreams.add(createChunkInputStream(chunks.get(i)));
      chunkOffsets[i] = tempOffset;
      tempOffset += chunks.get(i).getLen();
    }
    return new BlockChunks(blockData, length, chunkOffsets, chunkStreams);
  }


if (!chunks.isEmpty()) {
this.chunkIndex = 0;

if (blockPosition > 0) {
Expand All @@ -203,6 +190,12 @@ public synchronized void initialize() throws IOException {
}
}

/** A block with no chunks is not initialized, so the next read gets its block data again. */
private boolean isInitialized() {
final BlockChunks chunks = blockChunks;
return chunks != null && !chunks.streams.isEmpty();
}

private synchronized void refreshBlockInfo(IOException cause) throws IOException {
refreshBlockInfo(cause, blockID, pipelineRef, tokenRef, refreshFunction);
}
Expand Down Expand Up @@ -320,14 +313,9 @@ private void acquireClient() throws IOException {
}

/**
* Append another ChunkInputStream to the end of the list. Note that the
* ChunkInputStream is only created here. The chunk will be read from the
* Datanode only when a read operation is performed on for that chunk.
* Create a ChunkInputStream for the chunk. The chunk will be read from the
* Datanode only when a read operation is performed on that chunk.
*/
protected synchronized void addStream(ChunkInfo chunkInfo) {
chunkStreams.add(createChunkInputStream(chunkInfo));
}

protected ChunkInputStream createChunkInputStream(ChunkInfo chunkInfo) {
if (blockFileInputStream != null) {
// a non-empty blockFileInputStream means we have a direct local block replica to read from
Expand All @@ -343,11 +331,12 @@ protected ChunkInputStream createChunkInputStream(ChunkInfo chunkInfo) {
protected synchronized int readWithStrategy(ByteReaderStrategy strategy)
throws IOException {
Preconditions.checkArgument(strategy != null);
if (!initialized) {
if (!isInitialized()) {
initialize();
}

checkOpen();
final List<ChunkInputStream> chunkStreams = blockChunks.streams;
int totalReadLen = 0;
int len = strategy.getTargetLength();
while (len > 0) {
Expand Down Expand Up @@ -417,6 +406,9 @@ protected synchronized int readWithStrategy(ByteReaderStrategy strategy)
* without mutating this stream's cursor ({@code chunkIndex},
* {@code blockPosition}) or the sequential chunk streams' buffered state.
* Reuses each chunk's range reader, whose buffers and client references belong to the request.
* Takes this stream's lock only to initialize it or to refresh the block info after a read error,
* so it does not wait for a sequential read in progress.
* It does not check that this stream is open: {@link MultipartInputStream} checks that before it calls this method.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Could this state the contract instead of naming a caller? EC calls this directly too.

Suggested change
* It does not check that this stream is open: {@link MultipartInputStream} checks that before it calls this method.
* It does not check that this stream is open, so callers must check that first.

*
* @return bytes copied into {@code dst}, or {@link #EOF} at EOF
*/
Expand All @@ -429,23 +421,18 @@ public int readPositioned(long blockRelativePosition, ByteBuffer dst) throws IOE
return EOF;
}

if (!initialized) {
if (!isInitialized()) {
initialize();
}
final long[] offsets;
final List<ChunkInputStream> streams;
final long blockLength;
synchronized (this) {
checkOpen();
if (chunkOffsets == null || blockData == null || blockRelativePosition >= length) {
return EOF;
}

offsets = chunkOffsets;
streams = chunkStreams;
blockLength = length;
final BlockChunks chunks = blockChunks;
if (chunks.streams.isEmpty() || blockRelativePosition >= chunks.length) {
return EOF;
}

final long[] offsets = chunks.offsets;
final List<ChunkInputStream> streams = chunks.streams;
final long blockLength = chunks.length;

int index = MultipartInputStream.binarySearchOffsetIndex(offsets, blockRelativePosition);

long pos = blockRelativePosition;
Expand All @@ -468,7 +455,6 @@ public int readPositioned(long blockRelativePosition, ByteBuffer dst) throws IOE
private int readChunkAt(ChunkInputStream chunk, long chunkOffset, int readLength, ByteBuffer dst) throws IOException {
int preadRetries = 0;
while (true) {
checkOpen();

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any reason to drop checkOpen() here instead of making it lock-free? On EC failover, a short-circuit pread now retries and sleeps on a closed stream. A volatile closed check before shouldRetryRead() would avoid that. What do you think?

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why removing checkOpen()?

final ByteBuffer target = dst.duplicate();
target.limit(target.position() + readLength);
final int numBytesRead;
Expand Down Expand Up @@ -520,15 +506,18 @@ private int readChunkAt(ChunkInputStream chunk, long chunkOffset, int readLength
*/
@Override
public synchronized void seek(long pos) throws IOException {
if (!initialized) {
if (!isInitialized()) {
// Stream has not been initialized yet. Save the position so that it
// can be seeked when the stream is initialized.
blockPosition = pos;
return;
}

checkOpen();
if (pos < 0 || pos > length) {
final BlockChunks chunks = blockChunks;
final long[] chunkOffsets = chunks.offsets;
final List<ChunkInputStream> chunkStreams = chunks.streams;
if (pos < 0 || pos > chunks.length) {
Comment on lines -531 to +520

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's add some methods to BlockChunks:

    chunkIndex = chunks.searchChunkIndex(chunkIndex, pos);
    chunks.resetPosition(chunkIndex, chunkIndexOfPrevPosition, pos);
//BlockChunks
    void resetPosition(int chunkIndex, int chunkIndexOfPrevPosition, long pos) throws IOException {
      // Reset the previous chunkStream's position
      streams.get(chunkIndexOfPrevPosition).resetPosition();

      // Reset all the chunkStreams above the chunkIndex. We do this to reset
      // any previous reads which might have updated the chunkPosition.
      for (int index =  chunkIndex + 1; index < streams.size(); index++) {
        streams.get(index).seek(0);
      }
      // seek to the proper offset in the ChunkInputStream
      streams.get(chunkIndex).seek(pos - offsets[chunkIndex]);
    }

    int searchChunkIndex(int chunkIndex, long pos) {
      return searchStreamIndex(chunkIndex, pos, streams, offsets, ChunkInputStream::getLength);
    }
  }

  static <T extends Seekable> int searchStreamIndex(int index, long pos, List<T> streams, long[] offsets,
      ToLongFunction<T> getLength) {
    if (index >= streams.size()) {
      index = Arrays.binarySearch(offsets, pos);
    } else if (pos < offsets[index]) {
      index = Arrays.binarySearch(offsets, 0, index, pos);
    } else if (pos >= offsets[index] + getLength.applyAsLong(streams.get(index))) {
      index = Arrays.binarySearch(offsets, index + 1, streams.size(), pos);
    }
    if (index < 0) {
      // Binary search returns -insertionPoint - 1  if element is not present in the array.
      // Adjust the chunkIndex accordingly so that chunkIndex = insertionPoint - 1
      index = -index - 2;
    }
    return index;
  }

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

MultipartInputStream can also use the new method

+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/MultipartInputStream.java
@@ -145,23 +145,7 @@ public synchronized void seek(long pos) throws IOException {
     }
 
     // 1. Update the partIndex
-    if (partIndex >= partStreams.size()) {
-      partIndex = Arrays.binarySearch(partOffsets, pos);
-    } else if (pos < partOffsets[partIndex]) {
-      partIndex =
-          Arrays.binarySearch(partOffsets, 0, partIndex, pos);
-    } else if (pos >= partOffsets[partIndex] + partStreams
-        .get(partIndex).getLength()) {
-      partIndex = Arrays.binarySearch(partOffsets, partIndex + 1,
-          partStreams.size(), pos);
-    }
-    if (partIndex < 0) {
-      // Binary search returns -insertionPoint - 1  if element is not present
-      // in the array. insertionPoint is the point at which element would be
-      // inserted in the sorted array. We need to adjust the blockIndex
-      // accordingly so that partIndex = insertionPoint - 1
-      partIndex = -partIndex - 2;
-    }
+    partIndex = BlockInputStream.searchStreamIndex(partIndex, pos, partStreams, partOffsets, PartInputStream::getLength);
 

if (pos == 0) {
// It is possible for length and pos to be zero in which case
// seek should return instead of throwing exception
Expand Down Expand Up @@ -571,15 +560,15 @@ public synchronized void seek(long pos) throws IOException {

@Override
public synchronized long getPos() {
if (length == 0) {
if (getLength() == 0) {
return 0;
}

if (!initialized) {
if (!isInitialized()) {
// The stream is not initialized yet. Return the blockPosition
return blockPosition;
} else {
return chunkOffsets[chunkIndex] + chunkStreams.get(chunkIndex).getPos();
return blockChunks.offsets[chunkIndex] + blockChunks.streams.get(chunkIndex).getPos();
}
}

Expand All @@ -593,7 +582,7 @@ public synchronized void close() {
releaseClient();
xceiverClientFactory = null;

final List<ChunkInputStream> inputStreams = this.chunkStreams;
final List<ChunkInputStream> inputStreams = getChunkStreams();
if (inputStreams != null) {
for (ChunkInputStream is : inputStreams) {
is.close();
Expand Down Expand Up @@ -637,7 +626,8 @@ public BlockID getBlockID() {

@Override
public long getLength() {
return length;
final BlockChunks chunks = blockChunks;
return chunks != null ? chunks.length : blockInfo.getLength();
}

public synchronized int getChunkIndex() {
Expand All @@ -658,7 +648,7 @@ public synchronized void unbuffer() {
storePosition();
releaseClient();

final List<ChunkInputStream> inputStreams = this.chunkStreams;
final List<ChunkInputStream> inputStreams = getChunkStreams();
if (inputStreams != null) {
for (ChunkInputStream is : inputStreams) {
is.unbuffer();
Expand All @@ -672,7 +662,7 @@ private synchronized void storePosition() {

private synchronized void handleReadError(IOException cause) throws IOException {
releaseClient();
final List<ChunkInputStream> inputStreams = this.chunkStreams;
final List<ChunkInputStream> inputStreams = getChunkStreams();
if (inputStreams != null) {
for (ChunkInputStream is : inputStreams) {
is.releaseClient();
Expand All @@ -683,11 +673,34 @@ private synchronized void handleReadError(IOException cause) throws IOException
}

public synchronized List<ChunkInputStream> getChunkStreams() {
return chunkStreams;
final BlockChunks chunks = blockChunks;
return chunks != null ? chunks.streams : null;
}

public BlockData getStreamBlockData() {
return blockData;
final BlockChunks chunks = blockChunks;
return chunks != null ? chunks.blockData : null;
}

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

getStreamBlockData() is called multiple times in KeyValueHandler.reconcileChunksPerBlock(..) except for tests. reconcileChunksPerBlock(..) also calls getChunkStreams(). We should refactor the code to get BlockChunks only

We should get BlockChunks once and keep using it in reconcileChunksPerBlock(..).


/** The block data from the datanode and a ChunkInputStream for each chunk. Nothing changes it after initialize(). */
private static final class BlockChunks {
private final BlockData blockData;
private final long length;
// offsets[i] stores the index of the first data byte in
// streams[i] w.r.t the block data.
// Let’s say we have chunk size as 40 bytes. And let's say the parent
// block stores data from index 200 and has length 400.
// The first 40 bytes of this block will be stored in chunk[0], next 40 in
// chunk[1] and so on. But since the offsets are w.r.t the block only
// and not the key, the values in offsets will be [0, 40, 80,....].
private final long[] offsets;
private final List<ChunkInputStream> streams;

BlockChunks(BlockData blockData, long length, long[] offsets, List<ChunkInputStream> streams) {
this.blockData = blockData;
this.length = length;
this.offsets = offsets;
this.streams = Collections.unmodifiableList(streams);
}
}
}
Loading