Support calculating into lists and objects

src/Altinn.App.Core/Features/DataProcessing/DataModelFieldCalculator.cs

@@ -1,8 +1,9 @@
-using System.Text.Json;
 using Altinn.App.Core.Internal.App;
 using Altinn.App.Core.Internal.Data;
 using Altinn.App.Core.Internal.Expressions;
 using Altinn.App.Core.Models;
+using Altinn.App.Core.Models.Calculation;
+using Altinn.App.Core.Models.Expressions;
 using Altinn.App.Core.Models.Layout;
 using Altinn.Platform.Storage.Interface.Models;
 using Microsoft.Extensions.Logging;
@@ -12,12 +13,6 @@
 
 internal sealed class DataModelFieldCalculator
 {
-    private static readonly JsonSerializerOptions _jsonSerializerOptions = new()
-    {
-        ReadCommentHandling = JsonCommentHandling.Skip,
-        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
-    };
-
     private readonly ILogger<DataModelFieldCalculator> _logger;
     private readonly IAppResources _appResourceService;
     private readonly IDataElementAccessChecker _dataElementAccessChecker;
@@ -41,32 +36,31 @@
         using var activity = _telemetry?.StartCalculateActivity(dataAccessor.Instance.Id, taskId);
         foreach (var (dataType, dataElement) in dataAccessor.GetDataElementsWithFormDataForTask(taskId))
         {
             if (await _dataElementAccessChecker.CanRead(dataAccessor.Instance, dataType) is false)
             {
                 continue;
             }
 
-            var calculationConfig = _appResourceService.GetCalculationConfiguration(dataType.Id);
-            if (!string.IsNullOrEmpty(calculationConfig))
+            var calculationSchema = _appResourceService.GetCalculationConfiguration(dataType.Id);
+            if (calculationSchema is not null)
             {
-                await CalculateFormData(dataAccessor, dataElement, calculationConfig);
+                await CalculateFormData(dataAccessor, dataElement, calculationSchema);
             }
         }
     }
 
-    internal async Task CalculateFormData(
+    private async Task CalculateFormData(
         IInstanceDataAccessor dataAccessor,
         DataElement dataElement,
-        string rawCalculationConfig
+        CalculationSchema calculationSchema
     )
     {
         DataElementIdentifier dataElementIdentifier = dataElement;
-        var dataModelFieldCalculations = ParseDataModelFieldCalculationConfig(rawCalculationConfig);
         var formDataWrapper = await dataAccessor.GetFormDataWrapper(dataElement);
 
-        foreach (var (baseField, calculation) in dataModelFieldCalculations)
+        foreach (var calculation in calculationSchema.Calculations)
         {
-            var resolvedFields = formDataWrapper.GetResolvedKeys(baseField);
+            var resolvedFields = formDataWrapper.GetResolvedKeys(calculation.Field);
             foreach (var resolvedField in resolvedFields)
             {
                 var resolvedFieldReference = new DataReference()
@@ -88,7 +82,7 @@
                     formDataWrapper,
                     resolvedFieldReference,
                     positionalArguments,
-                    calculation
+                    calculation.Expression
                 );
             }
         }
@@ -100,14 +94,14 @@
         IFormDataWrapper formDataWrapper,
         DataReference resolvedField,
         ExpressionValue[] positionalArguments,
-        DataModelFieldCalculation calculation
+        Expression calculation
     )
     {
         try
         {
             var calculationResult = await ExpressionEvaluator.EvaluateExpressionToExpressionValue(
                 dataAccessor,
-                calculation.Expression,
+                calculation,
                 context,
                 positionalArguments
             );
@@ -121,78 +115,10 @@
                 );
             }
         }
         catch (Exception e)
         {
             _logger.LogError(e, "Error while evaluating calculation for field {Field}", resolvedField.Field);
             throw;
         }
     }
-
-    private Dictionary<string, DataModelFieldCalculation> ParseDataModelFieldCalculationConfig(
-        string rawCalculationConfig
-    )
-    {
-        JsonDocument calculationConfigDocument;
-        try
-        {
-            calculationConfigDocument = JsonDocument.Parse(
-                rawCalculationConfig,
-                new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip }
-            );
-        }
-        catch (JsonException e)
-        {
-            _logger.LogError(e, "Failed to parse calculation configuration JSON");
-            return new Dictionary<string, DataModelFieldCalculation>();
-        }
-        using (calculationConfigDocument)
-        {
-            var dataModelFieldCalculations = new Dictionary<string, DataModelFieldCalculation>();
-            var hasCalculations = calculationConfigDocument.RootElement.TryGetProperty(
-                "calculations",
-                out JsonElement calculationsObject
-            );
-            if (hasCalculations)
-            {
-                foreach (var calculationArray in calculationsObject.EnumerateObject())
-                {
-                    var field = calculationArray.Name;
-                    var calculation = calculationArray.Value;
-                    var resolvedDataModelFieldCalculation = ResolveDataModelFieldCalculation(field, calculation);
-                    if (resolvedDataModelFieldCalculation == null)
-                    {
-                        _logger.LogError("Calculation for field {Field} could not be resolved", field);
-                        continue;
-                    }
-                    dataModelFieldCalculations[field] = resolvedDataModelFieldCalculation;
-                }
-            }
-            return dataModelFieldCalculations;
-        }
-    }
-
-    private DataModelFieldCalculation? ResolveDataModelFieldCalculation(string field, JsonElement definition)
-    {
-        var dataModelFieldCalculationDefinition = definition.Deserialize<RawDataModelFieldCalculation>(
-            _jsonSerializerOptions
-        );
-        if (dataModelFieldCalculationDefinition == null)
-        {
-            _logger.LogError("Calculation for field {Field} could not be parsed", field);
-            return null;
-        }
-
-        if (dataModelFieldCalculationDefinition.Expression == null)
-        {
-            _logger.LogError("Calculation for field {Field} is missing expression", field);
-            return null;
-        }
-
-        var dataModelFieldCalculation = new DataModelFieldCalculation
-        {
-            Expression = dataModelFieldCalculationDefinition.Expression.Value,
-        };
-
-        return dataModelFieldCalculation;
-    }
 }

src/Altinn.App.Core/Helpers/Extensions/Utf8JsonReaderExtentions.cs

@@ -1,3 +1,4 @@
+using System.Text;
 using System.Text.Json;
 
 namespace Altinn.App.Core.Helpers.Extensions;
@@ -13,16 +14,22 @@
         Copy(ref reader, writer);
         writer.Flush();
 
-        return System.Text.Encoding.UTF8.GetString(stream.ToArray());
+        return Encoding.UTF8.GetString(stream.ToArray());
+    }
+
+    internal static void WriteRawFormattedValue(this Utf8JsonWriter writer, string json)
+    {
+        var jsonReader = new Utf8JsonReader(Encoding.UTF8.GetBytes(json), isFinalBlock: true, state: default);
+        jsonReader.Read(); // Need to read first token to initialize the reader
+        Copy(ref jsonReader, writer);
     }
 
     private static void Copy(ref Utf8JsonReader reader, Utf8JsonWriter writer)
     {
         switch (reader.TokenType)
         {
             case JsonTokenType.None:
-                writer.WriteNullValue();
-                break;
+                throw new JsonException("Reader is not initialized");
             case JsonTokenType.StartObject:
                 writer.WriteStartObject();
                 while (reader.Read())
@@ -41,9 +48,13 @@
                             writer.WriteEndObject();
                             return;
                         default:
-                            throw new JsonException($"Something is wrong, did not expect {reader.TokenType} here2");
+                            throw new JsonException($"Something is wrong, did not expect {reader.TokenType} here");
                     }
                 }
+                if (reader.TokenType != JsonTokenType.EndObject)
+                {
+                    throw new JsonException("Something is wrong, did not find end of object");
+                }
                 break;
             case JsonTokenType.StartArray:
                 writer.WriteStartArray();
@@ -52,6 +63,10 @@
                     Copy(ref reader, writer);
                 }
                 writer.WriteEndArray();
+                if (reader.TokenType != JsonTokenType.EndArray)
+                {
+                    throw new JsonException("Something is wrong, did not find end of array");
+                }
                 break;
             case JsonTokenType.Comment:
                 writer.WriteCommentValue(reader.ValueSpan);
@@ -60,7 +75,14 @@
                 writer.WriteStringValue(reader.ValueSpan);
                 break;
             case JsonTokenType.Number:
-                writer.WriteNumberValue(reader.GetDouble());
+                if (reader.HasValueSequence)
+                {
+                    writer.WriteRawValue(reader.ValueSequence);
+                }
+                else
+                {
+                    writer.WriteRawValue(reader.ValueSpan);
+                }
                 break;
             case JsonTokenType.True:
                 writer.WriteBooleanValue(true);

src/Altinn.App.Core/Implementation/AppResourcesSI.cs

@@ -5,6 +5,7 @@
 using Altinn.App.Core.Helpers;
 using Altinn.App.Core.Internal.App;
 using Altinn.App.Core.Models;
+using Altinn.App.Core.Models.Calculation;
 using Altinn.App.Core.Models.Layout;
 using Altinn.App.Core.Models.Layout.Components;
 using Altinn.Platform.Storage.Interface.Models;
@@ -78,7 +79,7 @@
             return null;
         }
 
         await using FileStream fileStream = new(fullFileName, FileMode.Open, FileAccess.Read);
         TextResource textResource =
             await System.Text.Json.JsonSerializer.DeserializeAsync<TextResource>(fileStream, _jsonSerializerOptions)
             ?? throw new System.Text.Json.JsonException("Failed to deserialize text resource");
@@ -218,7 +219,7 @@
     }
 
     /// <inheritdoc />
     [Obsolete("Use GetLayoutsForSet or GetLayoutModelForTask instead")]
     public string GetLayouts()
     {
         using var activity = _telemetry?.StartGetLayoutsActivity();
@@ -295,7 +296,7 @@
 
         PathHelper.EnsureLegalPath(Path.Join(_settings.AppBasePath, _settings.UiFolder), layoutsPath);
 
         if (Directory.Exists(layoutsPath))
         {
             foreach (string file in Directory.GetFiles(layoutsPath))
             {
@@ -407,9 +408,9 @@
         PathHelper.EnsureLegalPath(Path.Join(_settings.AppBasePath, _settings.UiFolder), filename);
 
         string? filedata = null;
         if (File.Exists(filename))
         {
             filedata = File.ReadAllText(filename, Encoding.UTF8);
         }
 
         return filedata;
@@ -428,9 +429,9 @@
 
         PathHelper.EnsureLegalPath(Path.Join(_settings.AppBasePath, _settings.UiFolder), filename);
 
         if (File.Exists(filename))
         {
             var fileData = File.ReadAllText(filename, Encoding.UTF8);
             LayoutSettings? layoutSettings = JsonConvert.DeserializeObject<LayoutSettings>(fileData);
             return layoutSettings;
         }
@@ -465,9 +466,9 @@
     private static byte[] ReadFileByte(string fileName)
     {
         byte[]? filedata = null;
         if (File.Exists(fileName))
         {
             filedata = File.ReadAllBytes(fileName);
         }
 
 #nullable disable
@@ -534,26 +535,27 @@
         string? filedata = null;
         if (File.Exists(filename))
         {
             filedata = File.ReadAllText(filename, Encoding.UTF8);
         }
 
         return filedata;
     }
 
     /// <inheritdoc />
-    public string? GetCalculationConfiguration(string dataTypeId)
+    public CalculationSchema? GetCalculationConfiguration(string dataTypeId)
     {
         using var activity = _telemetry?.StartGetCalculationConfigurationActivity();
         string legalPath = Path.Join(_settings.AppBasePath, _settings.ModelsFolder);
         string filename = Path.Join(legalPath, $"{dataTypeId}.{_settings.CalculationConfigurationFileName}");
         PathHelper.EnsureLegalPath(legalPath, filename);
 
-        string? fileData = null;
-        if (File.Exists(filename))
+        if (!File.Exists(filename))
         {
-            fileData = File.ReadAllText(filename, Encoding.UTF8);
+            return null;
         }
 
-        return fileData;
+        return System.Text.Json.JsonSerializer.Deserialize<CalculationSchema>(
+            File.ReadAllText(filename, Encoding.UTF8)
+        );
     }
 }

src/Altinn.App.Core/Internal/App/IAppResources.cs

@@ -1,4 +1,5 @@
 using Altinn.App.Core.Models;
+using Altinn.App.Core.Models.Calculation;
 using Altinn.App.Core.Models.Layout;
 using Altinn.Platform.Storage.Interface.Models;
 
@@ -82,7 +83,7 @@
     /// Gets the layouts for the app.
     /// </summary>
     /// <returns>A dictionary of FormLayout objects serialized to JSON</returns>
     [Obsolete("Use GetLayoutsForSet or GetLayoutModelForTask instead")]
     string GetLayouts();
 
     /// <summary>
@@ -134,7 +135,7 @@
     /// <summary>
     /// Gets the full layout model for the optional set
     /// </summary>
     [Obsolete("Use GetLayoutModelForTask instead", false)]
     LayoutModel GetLayoutModel(string? layoutSetId = null);
 
     /// <summary>
@@ -177,5 +178,5 @@
     /// Gets the calculation configuration for a given data type
     /// </summary>
     /// <returns>The calculation configuration in JSON format represented as string</returns>
-    string? GetCalculationConfiguration(string dataTypeId);
+    CalculationSchema? GetCalculationConfiguration(string dataTypeId);
 }

src/Altinn.App.Core/Internal/Expressions/ExpressionValue.cs

@@ -1,3 +1,4 @@
+using System.Collections;
 using System.Diagnostics;
 using System.Globalization;
 using System.Numerics;
@@ -12,7 +13,7 @@
 /// Discriminated union for the JSON types that can be arguments and result of expressions
 /// </summary>
 [JsonConverter(typeof(ExpressionTypeUnionConverter))]
-[DebuggerDisplay("{ToString(),nq}")]
+[DebuggerDisplay("{ToStringForText(),nq}")]
 public readonly struct ExpressionValue : IEquatable<ExpressionValue>
 {
     private readonly string? _stringValue = null;
@@ -22,12 +23,12 @@
     private readonly double _numberValue = 0;
 
     /// <summary>
-    /// Constructor for NULL value (structs require a public parameterless constructor)
+    /// Constructor for Undefined value (structs require a public parameterless constructor)
     /// </summary>
     public ExpressionValue()
-        : this(JsonValueKind.Null) { }
+        : this(JsonValueKind.Undefined) { }
 
     private ExpressionValue(JsonValueKind valueKind)
     {
         ValueKind = valueKind;
     }
@@ -221,10 +222,10 @@
     /// <summary>
     /// Convert the value to the relevant CLR type
     /// </summary>
     [Obsolete(
         "ToObject is not type safe and should be avoided. Use the type-specific properties or TryDeserialize<T> instead.",
         error: false
     )]
     public object? ToObject() =>
         ValueKind switch
         {
@@ -257,7 +258,7 @@
     /// <summary>
     /// Get the value as a string (or throw if it isn't a string ValueKind)
     /// </summary>
     public string String
     {
         get
         {
@@ -354,11 +355,11 @@
             JsonValueKind.Null => "null",
             JsonValueKind.Undefined => "undefined",
             JsonValueKind.True => "true",
             JsonValueKind.False => "false",
             JsonValueKind.String => JsonSerializer.Serialize(String, _unsafeSerializerOptionsForSerializingDates),
             JsonValueKind.Number => Number.ToString(CultureInfo.InvariantCulture),
             JsonValueKind.Object or JsonValueKind.Array => _stringValueNotNull,
             _ => throw new InvalidOperationException($"Invalid value kind {ValueKind}"),
         };
 
     /// <summary>
@@ -410,7 +411,7 @@
     {
         throw new NotImplementedException("Equals is not used for ExpressionValue");
         // First compare value kinds
         // if (_valueKind != other._valueKind)
         //     return false;
 
         // // Then compare actual values based on the kind
@@ -439,7 +440,7 @@
     {
         throw new NotImplementedException("GetHashCode is not implemented for ExpressionValue");
         // return ValueKind switch
         // {
         //     JsonValueKind.Null => 0,
         //     JsonValueKind.True => 1,
         //     JsonValueKind.False => 0,
@@ -556,7 +557,7 @@
     /// <param name="result">The result (null or default if unsuccessful), but note that null might also be a valid result</param>
     /// <param name="type">The type to convert to</param>
     /// <returns>Whether the conversion was successful</returns>
     public bool TryDeserialize(Type type, out object? result)
     {
         // Value types can be Nullable<T>, so assign underlyingType accordingly
         Type underlyingType;
@@ -622,6 +623,28 @@
                     return false;
                 }
             }
+            case JsonValueKind.Array when underlyingType.IsAssignableTo(typeof(IEnumerable)):
+            case JsonValueKind.Object:
+                try
+                {
+                    // For complex types we serialize the expressionValue to json
+                    // and then deserialize to the target type.
+                    // This allows us to leverage the normal JSON deserialization rules and also handle cases where the
+                    // ValueKind doesn't exactly match the target type (e.g., deserialize a JsonObject to a Dictionary<string, object> or similar)
+                    var json = JsonSerializer.SerializeToUtf8Bytes(this);
+                    result = JsonSerializer.Deserialize(json, type);
+                    return true;
+                }
+                catch (JsonException)
+                {
+                    result = null;
+                    return false;
+                }
+                catch (NotSupportedException)
+                {
+                    result = null;
+                    return false;
+                }
         }
 
         // Add special handling for bool to support loose conversion rules
@@ -673,7 +696,7 @@
 
     private static bool IsSupportedNumericType(Type type)
     {
         // TODO: consider supporting enums as numeric types as well, but currently we
         // don't use C# enums in datamodels, so it isn't very urgent.
         return type == typeof(double)
             || type == typeof(int)
@@ -707,7 +730,7 @@
         return new(doc.RootElement);
     }
 
-    internal void WriteJson(Utf8JsonWriter writer, JsonSerializerOptions options)
+    internal void WriteJson(Utf8JsonWriter writer)
     {
         switch (ValueKind)
         {
@@ -729,7 +752,8 @@
                 break;
             case JsonValueKind.Object:
             case JsonValueKind.Array:
-                writer.WriteRawValue(_stringValueNotNull);
+                // writer.WriteRawFormattedValue(_stringValueNotNull);
+                JsonSerializer.Serialize(writer, JsonElement);
                 break;
             default:
                 throw new JsonException();
@@ -768,5 +792,5 @@
 
     /// <inheritdoc />
     public override void Write(Utf8JsonWriter writer, ExpressionValue value, JsonSerializerOptions options) =>
-        value.WriteJson(writer, options);
+        value.WriteJson(writer);
 }

src/Altinn.App.Core/Models/Calculation/CalculationItem.cs

@@ -0,0 +1,23 @@
+using System.Text.Json.Serialization;
+using Altinn.App.Core.Models.Expressions;
+
+namespace Altinn.App.Core.Models.Calculation;
+
+/// <summary>
+/// Calculation item in the calculation configuration
+/// </summary>
+public class CalculationItem
+{
+    /// <summary>
+    /// The base field to be calculated.
+    /// Note that missing indexes will be added to the field name when calculating array items. For example, if the field is "myArray[].myField", the calculation will be applied to all items in the array.
+    /// </summary>
+    [JsonPropertyName("field")]
+    public required string Field { get; init; }
+
+    /// <summary>
+    /// The expression to be used for the calculation. Note that this will be run in the context of the field, so you can use relative paths in the expression.
+    /// </summary>
+    [JsonPropertyName("expression")]
+    public required Expression Expression { get; init; }
+}