fix: Export binary-tagged UTF-8 attributes over OTLP

common/lib/opentelemetry/common/utilities.rb

@@ -54,9 +54,9 @@ def time_in_nanoseconds(timestamp = Time.now)
       #
       # @param [String] string The string to be utf8 encoded
       # @param [optional boolean] binary This option is for displaying binary data
-      # @param [optional String] placeholder The fallback string to be used if encoding fails
+      # @param [String, nil] placeholder The fallback value to be used if encoding fails
       #
-      # @return [String]
+      # @return [String, nil]
       def utf8_encode(string, binary: false, placeholder: STRING_PLACEHOLDER)
         string = string.to_s
 
@@ -66,6 +66,11 @@ def utf8_encode(string, binary: false, placeholder: STRING_PLACEHOLDER)
           string.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
         elsif string.encoding == ::Encoding::UTF_8
           string
+        elsif string.encoding == ::Encoding::ASCII_8BIT
+          utf8_string = string.dup.force_encoding(::Encoding::UTF_8)
+          raise Encoding::InvalidByteSequenceError, 'binary string is not valid UTF-8' unless utf8_string.valid_encoding?
+
+          utf8_string
         else
           string.encode(::Encoding::UTF_8)
         end

common/test/opentelemetry/common/utilities_test.rb

@@ -76,6 +76,22 @@ def shutdown(timeout: nil); end
       assert_equal('?', common_utils.utf8_encode(time_bomb, placeholder: '?'))
     end
 
+    it 'preserves valid UTF-8 bytes from a binary-encoded string' do
+      city = 'Montréal'.dup.force_encoding(::Encoding::ASCII_8BIT)
+
+      encoded = common_utils.utf8_encode(city)
+
+      assert_equal('Montréal', encoded)
+      assert_equal(::Encoding::UTF_8, encoded.encoding)
+      assert_equal(::Encoding::ASCII_8BIT, city.encoding)
+    end
+
+    it 'does not validate an already UTF-8-tagged string' do
+      invalid = "\xC3".dup.force_encoding(::Encoding::UTF_8)
+
+      assert_same(invalid, common_utils.utf8_encode(invalid, placeholder: '?'))
+    end
+
     it 'with binary data' do
       byte_array = (+"keep what\xC2 is valid").force_encoding(::Encoding::ASCII_8BIT)
 

exporter/otlp-common/lib/opentelemetry/exporter/otlp/common.rb

@@ -5,6 +5,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 require 'opentelemetry'
+require 'opentelemetry/common'
 require 'opentelemetry/exporter/otlp/common/version'
 
 require 'google/rpc/status_pb'
@@ -152,9 +153,10 @@ def as_otlp_span_kind(kind)
         end
 
         def as_otlp_key_value(key, value)
+          key = OpenTelemetry::Common::Utilities.utf8_encode(key, placeholder: 'Encoding Error')
           Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value(value))
         rescue Encoding::UndefinedConversionError => e
-          encoded_value = value.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�')
+          encoded_value = value.to_s.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�')
           OpenTelemetry.handle_error(exception: e, message: "encoding error for key #{key} and value #{encoded_value}")
           Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value('Encoding Error'))
         end
@@ -163,7 +165,7 @@ def as_otlp_any_value(value)
           result = Opentelemetry::Proto::Common::V1::AnyValue.new
           case value
           when String
-            result.string_value = value
+            result.string_value = OpenTelemetry::Common::Utilities.utf8_encode(value, placeholder: value)
           when Integer
             result.int_value = value
           when Float

exporter/otlp-common/opentelemetry-exporter-otlp-common.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'googleapis-common-protos-types', '~> 1.3'
   spec.add_dependency 'google-protobuf', '>= 3.18'
   spec.add_dependency 'opentelemetry-api', '~> 1.1'
+  spec.add_dependency 'opentelemetry-common', '~> 0.20'
 
   if spec.respond_to?(:metadata)
     spec.metadata['changelog_uri'] = "https://rubydoc.info/gems/#{spec.name}/#{spec.version}/file/CHANGELOG.md"

exporter/otlp-common/test/opentelemetry/exporter/otlp/common/common_test.rb

@@ -61,6 +61,46 @@
       _(result.resource_spans).must_be_empty
     end
 
+    it 'exports valid UTF-8 bytes from binary-encoded attribute strings' do
+      city = 'Montréal'.dup.force_encoding(::Encoding::ASCII_8BIT)
+      span_data = OpenTelemetry::TestHelpers.create_span_data(
+        total_recorded_attributes: 1,
+        attributes: { 'city' => city }
+      )
+
+      etsr = OpenTelemetry::Exporter::OTLP::Common.as_etsr([span_data])
+      exported_span = etsr.resource_spans.first.scope_spans.first.spans.first
+
+      _(exported_span.attributes.first.value.string_value).must_equal('Montréal')
+    end
+
+    it 'safely exports attributes with invalid UTF-8 keys' do
+      invalid_key = "\xC2".dup.force_encoding(::Encoding::ASCII_8BIT)
+      span_data = OpenTelemetry::TestHelpers.create_span_data(
+        total_recorded_attributes: 1,
+        attributes: { invalid_key => 'value' }
+      )
+
+      etsr = OpenTelemetry::Exporter::OTLP::Common.as_etsr([span_data])
+      exported_attribute = etsr.resource_spans.first.scope_spans.first.spans.first.attributes.first
+
+      _(exported_attribute.key).must_equal('Encoding Error')
+      _(exported_attribute.value.string_value).must_equal('value')
+    end
+
+    it 'safely exports arrays containing invalid UTF-8 strings' do
+      invalid_value = "\xC2".dup.force_encoding(::Encoding::ASCII_8BIT)
+      span_data = OpenTelemetry::TestHelpers.create_span_data(
+        total_recorded_attributes: 1,
+        attributes: { 'values' => [invalid_value] }
+      )
+
+      etsr = OpenTelemetry::Exporter::OTLP::Common.as_etsr([span_data])
+      exported_value = etsr.resource_spans.first.scope_spans.first.spans.first.attributes.first.value
+
+      _(exported_value.string_value).must_equal('Encoding Error')
+    end
+
     it 'batches per resource and instrumentation scope' do
       # Test resource batching
       resource_one = OpenTelemetry::SDK::Resources::Resource.create('k1' => 'v1')

exporter/otlp-logs/lib/opentelemetry/exporter/otlp/logs/logs_exporter.rb

@@ -316,9 +316,10 @@ def as_otlp_log_record(log_record_data)
           end
 
           def as_otlp_key_value(key, value)
+            key = OpenTelemetry::Common::Utilities.utf8_encode(key, placeholder: 'Encoding Error')
             Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value(value))
           rescue Encoding::UndefinedConversionError => e
-            encoded_value = value.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�')
+            encoded_value = value.to_s.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�')
             OpenTelemetry.handle_error(exception: e, message: "encoding error for key #{key} and value #{encoded_value}")
             Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value('Encoding Error'))
           end
@@ -327,7 +328,7 @@ def as_otlp_any_value(value) # rubocop:disable Metrics/CyclomaticComplexity
             result = Opentelemetry::Proto::Common::V1::AnyValue.new
             case value
             when String
-              result.string_value = value
+              result.string_value = OpenTelemetry::Common::Utilities.utf8_encode(value, placeholder: value)
             when Integer
               result.int_value = value
             when Float

exporter/otlp-logs/test/opentelemetry/exporter/otlp/logs_exporter_test.rb

@@ -649,6 +649,23 @@
       OpenTelemetry.logger = logger
     end
 
+    it 'exports valid UTF-8 bytes from binary-encoded strings' do
+      city = 'Montréal'.dup.force_encoding(::Encoding::ASCII_8BIT)
+
+      value = exporter.send(:as_otlp_any_value, city)
+
+      _(value.string_value).must_equal('Montréal')
+      _(value.string_value.encoding).must_equal(::Encoding::UTF_8)
+    end
+
+    it 'safely exports arrays containing invalid UTF-8 strings' do
+      invalid_value = "\xC2".dup.force_encoding(::Encoding::ASCII_8BIT)
+
+      attribute = exporter.send(:as_otlp_key_value, 'values', [invalid_value])
+
+      _(attribute.value.string_value).must_equal('Encoding Error')
+    end
+
     it 'logs rpc.Status on bad request' do
       log_stream = StringIO.new
       logger = OpenTelemetry.logger

exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/util.rb

@@ -34,9 +34,10 @@ def around_request
 
           # Converts a key/value pair to an OTLP proto KeyValue.
           def as_otlp_key_value(key, value)
+            key = OpenTelemetry::Common::Utilities.utf8_encode(key, placeholder: 'Encoding Error')
             Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value(value))
           rescue Encoding::UndefinedConversionError => e
-            encoded_value = value.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�')
+            encoded_value = value.to_s.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�')
             OpenTelemetry.handle_error(exception: e, message: "encoding error for key #{key} and value #{encoded_value}")
             Opentelemetry::Proto::Common::V1::KeyValue.new(key: key, value: as_otlp_any_value('Encoding Error'))
           end
@@ -46,7 +47,7 @@ def as_otlp_any_value(value)
             result = Opentelemetry::Proto::Common::V1::AnyValue.new
             case value
             when String
-              result.string_value = value
+              result.string_value = OpenTelemetry::Common::Utilities.utf8_encode(value, placeholder: value)
             when Integer
               result.int_value = value
             when Float

exporter/otlp-metrics/test/opentelemetry/exporter/otlp/metrics/metrics_exporter_test.rb

@@ -603,6 +603,23 @@
       OpenTelemetry.logger = logger
     end
 
+    it 'exports valid UTF-8 bytes from binary-encoded attribute strings' do
+      city = 'Montréal'.dup.force_encoding(::Encoding::ASCII_8BIT)
+
+      value = exporter.send(:as_otlp_any_value, city)
+
+      _(value.string_value).must_equal('Montréal')
+      _(value.string_value.encoding).must_equal(::Encoding::UTF_8)
+    end
+
+    it 'safely exports arrays containing invalid UTF-8 strings' do
+      invalid_value = "\xC2".dup.force_encoding(::Encoding::ASCII_8BIT)
+
+      attribute = exporter.send(:as_otlp_key_value, 'values', [invalid_value])
+
+      _(attribute.value.string_value).must_equal('Encoding Error')
+    end
+
     it 'is able to encode NumberDataPoint with Integer or Float value' do
       stub_request(:post, 'http://localhost:4318/v1/metrics').to_return(status: 200)
 

exporter/otlp/test/opentelemetry/exporter/otlp/exporter_test.rb

@@ -627,6 +627,34 @@
       OpenTelemetry.logger = logger
     end
 
+    it 'exports valid UTF-8 bytes from binary-encoded attribute strings' do
+      city = 'Montréal'.dup.force_encoding(::Encoding::ASCII_8BIT)
+      span_data = OpenTelemetry::TestHelpers.create_span_data(
+        total_recorded_attributes: 1,
+        attributes: { 'city' => city }
+      )
+
+      encoded_data = exporter.send(:encode, [span_data])
+      decoded = Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode(encoded_data)
+      exported_span = decoded.resource_spans.first.scope_spans.first.spans.first
+
+      _(exported_span.attributes.first.value.string_value).must_equal('Montréal')
+    end
+
+    it 'safely exports arrays containing invalid UTF-8 strings' do
+      invalid_value = "\xC2".dup.force_encoding(::Encoding::ASCII_8BIT)
+      span_data = OpenTelemetry::TestHelpers.create_span_data(
+        total_recorded_attributes: 1,
+        attributes: { 'values' => [invalid_value] }
+      )
+
+      encoded_data = exporter.send(:encode, [span_data])
+      decoded = Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode(encoded_data)
+      exported_value = decoded.resource_spans.first.scope_spans.first.spans.first.attributes.first.value
+
+      _(exported_value.string_value).must_equal('Encoding Error')
+    end
+
     it 'logs rpc.Status on bad request' do
       log_stream = StringIO.new
       logger = OpenTelemetry.logger