Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,17 @@
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.time.Instant;
import java.time.format.DateTimeParseException;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.accumulo.server.conf.codec.VersionedProperties.PropertyMetadata;
import org.checkerframework.checker.nullness.qual.NonNull;

/**
Expand Down Expand Up @@ -132,9 +135,9 @@ abstract void encodePayload(final OutputStream out, final VersionedProperties vP

var timestamp = TIMESTAMP_FORMATTER.parse(dis.readUTF(), Instant::from);

Map<String,String> props = decodePayload(bis, encodingOpts);
Payload props = decodePayload(bis, encodingOpts);

return new VersionedProperties(version, timestamp, props);
return new VersionedProperties(version, timestamp, props.properties(), props.metadata());
} catch (NullPointerException | DateTimeParseException ex) {
throw new IllegalArgumentException("Invalid data cannot decode byte array", ex);
}
Expand Down Expand Up @@ -185,16 +188,18 @@ public static Instant readTimestamp(final byte[] bytes) {
}

/**
* Decode the payload and any optional encoding specific metadata and return a map of the property
* name, value pairs.
* Decode the payload and any optional encoding specific metadata.
*
* @param inStream an input stream
* @param encodingOpts the general encoding options.
* @return a map of properties name, value pairs.
* @return the decoded payload
* @throws IOException if an exception occurs reading from the input stream.
*/
abstract Map<String,String> decodePayload(final InputStream inStream,
final EncodingOptions encodingOpts) throws IOException;
abstract Payload decodePayload(final InputStream inStream, final EncodingOptions encodingOpts)
throws IOException;

record Payload(Map<String,String> properties, Map<String,PropertyMetadata> metadata) {
}

/**
* Read the property map from a data input stream as UTF strings. The input stream should be
Expand Down Expand Up @@ -246,4 +251,49 @@ void writeMapAsUTF(final DataOutputStream dos, final Map<String,String> aMap) th
dos.flush();
}

/**
* Read optional property metadata written after the property map.
*
* @return the property metadata map, or an empty map when absent.
*/
Map<String,PropertyMetadata> readMetadataAsUTF(DataInputStream dis) throws IOException {
if (dis.available() == 0) {
return Map.of();
}

int items;
try {
items = dis.readInt();
} catch (EOFException e) {
return Map.of();
}

Map<String,PropertyMetadata> metadata = new HashMap<>(items);
for (int i = 0; i < items; i++) {
String key = dis.readUTF();
Instant created = TIMESTAMP_FORMATTER.parse(dis.readUTF(), Instant::from);
Instant modified = TIMESTAMP_FORMATTER.parse(dis.readUTF(), Instant::from);
metadata.put(key, new PropertyMetadata(created, modified));
}
return metadata;
}

/**
* Write the property metadata to the data output stream.
*
* @param dos a data output stream
* @param metadata the property metadata map.
*/
void writeMetadataAsUTF(final DataOutputStream dos, final Map<String,PropertyMetadata> metadata)
throws IOException {
dos.writeInt(metadata.size());

for (Entry<String,PropertyMetadata> e : metadata.entrySet()) {
dos.writeUTF(e.getKey());
dos.writeUTF(TIMESTAMP_FORMATTER.format(e.getValue().created()));
dos.writeUTF(TIMESTAMP_FORMATTER.format(e.getValue().modified()));
}
dos.flush();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import org.apache.accumulo.server.conf.codec.VersionedProperties.PropertyMetadata;

/**
* Initial property encoding that (optionally) uses gzip to compress the property map. The encoding
* version supported is EncodingVersion.V1_0.
Expand All @@ -53,6 +55,7 @@ void encodePayload(final OutputStream out, final VersionedProperties vProps,
DataOutputStream zdos = new DataOutputStream(gzipOut)) {

writeMapAsUTF(zdos, props);
writeMetadataAsUTF(zdos, vProps.getMetadata());

// finalize compression
gzipOut.flush();
Expand All @@ -61,6 +64,7 @@ void encodePayload(final OutputStream out, final VersionedProperties vProps,
} else {
try (DataOutputStream dos = new DataOutputStream(out)) {
writeMapAsUTF(dos, props);
writeMetadataAsUTF(dos, vProps.getMetadata());
}
}

Expand All @@ -72,21 +76,24 @@ boolean checkCanDecodeVersion(final EncodingOptions encodingOpts) {
}

@Override
Map<String,String> decodePayload(final InputStream inStream, final EncodingOptions encodingOpts)
throws IOException {
VersionedPropCodec.Payload decodePayload(final InputStream inStream,
final EncodingOptions encodingOpts) throws IOException {
// read the property map keys, values
Map<String,String> aMap;
Map<String,PropertyMetadata> metadata;
if (encodingOpts.isCompressed()) {
// Read and uncompress an input stream compressed with GZip
try (GZIPInputStream gzipIn = new GZIPInputStream(inStream);
DataInputStream zdis = new DataInputStream(gzipIn)) {
aMap = readMapAsUTF(zdis);
metadata = readMetadataAsUTF(zdis);
}
} else {
try (DataInputStream dis = new DataInputStream(inStream)) {
aMap = readMapAsUTF(dis);
metadata = readMetadataAsUTF(dis);
}
}
return aMap;
return new Payload(aMap, metadata);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;

import org.checkerframework.checker.nullness.qual.NonNull;
Expand Down Expand Up @@ -55,6 +56,15 @@ public class VersionedProperties {
private final long dataVersion;
private final Instant timestamp;
private final Map<String,String> props;
private final Map<String,PropertyMetadata> metadata;

public record PropertyMetadata(Instant created, Instant modified) {

public PropertyMetadata {
requireNonNull(created, "A created timestamp must be supplied");
requireNonNull(modified, "A modified timestamp must be supplied");
}
}

/**
* Instantiate an initial instance with default version info and empty map.
Expand All @@ -70,7 +80,7 @@ public VersionedProperties() {
* have been previously validated (if required)
*/
public VersionedProperties(Map<String,String> props) {
this(INIT_VERSION, Instant.now(), props);
this(INIT_VERSION, Instant.now(), props, null);
}

/**
Expand All @@ -83,10 +93,41 @@ public VersionedProperties(Map<String,String> props) {
*/
public VersionedProperties(final long zkDataVersion, final Instant timestamp,
final Map<String,String> props) {
this(zkDataVersion, timestamp, props, null);
}

/**
* Instantiate an instance with the provided properties and metadata.
*
* @param zkDataVersion the ZooKeeper node data version.
* @param timestamp timestamp of this version.
* @param props optional map of property key, value pairs.
* @param metadata optional map of metadata for property keys.
*/
public VersionedProperties(final long zkDataVersion, final Instant timestamp,
final Map<String,String> props, final Map<String,PropertyMetadata> metadata) {
// convert the signed integer ZK version to an unsigned value stored in a long.
this.dataVersion = zkDataVersion & 0xffffffffL;
this.timestamp = requireNonNull(timestamp, "A timestamp must be supplied");
this.props = props == null ? Map.of() : Map.copyOf(props);
this.metadata = initMetadata(this.props, metadata, timestamp);
}

/**
* Initialize the metadata map for the given properties.
*/
private static Map<String,PropertyMetadata> initMetadata(Map<String,String> props,
Map<String,PropertyMetadata> metadata, Instant timestamp) {
if (props.isEmpty()) {
return Map.of();
}

var propMetadata = new HashMap<String,PropertyMetadata>();
for (String key : props.keySet()) {
propMetadata.put(key, metadata == null ? new PropertyMetadata(timestamp, timestamp)
: metadata.getOrDefault(key, new PropertyMetadata(timestamp, timestamp)));
}
return Map.copyOf(propMetadata);
}

/**
Expand Down Expand Up @@ -126,6 +167,13 @@ public Instant getTimestamp() {
return timestamp;
}

/**
* Get an unmodifiable map with metadata for each property key.
*/
public @NonNull Map<String,PropertyMetadata> getMetadata() {
return metadata;
}

/**
* The timestamp formatted as an ISO 8601 string with format of
* {@code YYYY-MM-DDTHH:mm:ss.SSSSSSZ}
Expand All @@ -151,9 +199,7 @@ public String getTimestampISO() {
* @return A new instance of this class with the property added or updated.
*/
public VersionedProperties addOrUpdate(final String key, final String value) {
var updated = new HashMap<>(props);
updated.put(key, value);
return new VersionedProperties(dataVersion, Instant.now(), updated);
return addOrUpdate(Map.of(key, value));
}

/**
Expand All @@ -168,9 +214,20 @@ public VersionedProperties addOrUpdate(final String key, final String value) {
* @return A new instance of this class with the properties added or updated.
*/
public VersionedProperties addOrUpdate(final Map<String,String> updates) {
Instant now = Instant.now();
var updated = new HashMap<>(props);
var updatedMetadata = new HashMap<>(metadata);
updates.forEach((key, value) -> {
PropertyMetadata existing = metadata.get(key);
if (existing == null) {
updatedMetadata.put(key, new PropertyMetadata(now, now));
} else {
updatedMetadata.put(key, new PropertyMetadata(existing.created(),
value.equals(props.get(key)) ? existing.modified() : now));
}
});
updated.putAll(updates);
return new VersionedProperties(dataVersion, Instant.now(), updated);
return new VersionedProperties(dataVersion, now, updated, updatedMetadata);
}

/**
Expand All @@ -181,8 +238,19 @@ public VersionedProperties addOrUpdate(final Map<String,String> updates) {
* @return A new instance of this class with the replaced properties.
*/
public VersionedProperties replaceAll(final Map<String,String> updates) {
// Constructor will copy the map to a new map already
return new VersionedProperties(dataVersion, Instant.now(), updates);
Instant now = Instant.now();
var updatedMetadata = new HashMap<String,PropertyMetadata>();
updates.forEach((key, value) -> {
PropertyMetadata existing = metadata.get(key);
if (existing == null) {
updatedMetadata.put(key, new PropertyMetadata(now, now));
} else {
updatedMetadata.put(key, new PropertyMetadata(existing.created(),
value.equals(props.get(key)) ? existing.modified() : now));
}
});
// Constructor will copy the maps already
return new VersionedProperties(dataVersion, now, updates, updatedMetadata);
}

/**
Expand All @@ -197,9 +265,12 @@ public VersionedProperties replaceAll(final Map<String,String> updates) {
* @return A new instance of this class.
*/
public VersionedProperties remove(Collection<String> keys) {
Instant now = Instant.now();
var updated = new HashMap<>(props);
updated.keySet().removeAll(keys);
return new VersionedProperties(dataVersion, Instant.now(), updated);
var updatedMetadata = new HashMap<>(metadata);
updatedMetadata.keySet().removeAll(keys);
return new VersionedProperties(dataVersion, now, updated, updatedMetadata);
}

/**
Expand All @@ -218,14 +289,14 @@ public String print(boolean prettyPrint) {
.append(prettyPrint ? "\n" : ", ");

Map<String,String> sorted = new TreeMap<>(props);
sorted.forEach((k, v) -> {
for (Entry<String,String> e : sorted.entrySet()) {
if (prettyPrint) {
// indent if pretty
sb.append(" ");
}
sb.append(k).append("=").append(v);
sb.append(e.getKey()).append("=").append(e.getValue());
sb.append(prettyPrint ? "\n" : ", ");
});
}
return sb.toString();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -465,7 +465,14 @@ private void printSortedProps(final PrintWriter writer,
writer.println("-- none --");
} else {
TreeMap<String,String> sorted = new TreeMap<>(pMap);
sorted.forEach((name, value) -> writer.printf("%s%s=%s\n", INDENT, name, value));
sorted.forEach((name, value) -> {
var metadata = p.getMetadata().get(name);
writer.printf("%s%s=%s\n", INDENT, name, value);
writer.printf("%s%screated: %s\n", INDENT, INDENT,
tsFormat.format(metadata.created()));
writer.printf("%s%smodified: %s\n", INDENT, INDENT,
tsFormat.format(metadata.modified()));
});
}
writer.println();
}
Expand Down
Loading