Ignore URL components during identifier extraction

lib/identifiers/arxiv_id.rb

@@ -7,7 +7,7 @@ class ArxivId
       \.
       \d{4,5}           # Zero-padded sequence number of 4- or 5-digits
       (?:v\d+)?         # Literal v followed by version number of 1 or more digits
-      (?=$|[[:space:]])    # Look-ahead for end of string or whitespace
+      (?=$|[[:space:]?#])  # Look-ahead for end, whitespace, query or fragment
     }xi
     PRE_2007_REGEXP = %r{
       (?<=^|[[:space:]/])  # Look-behind for the start of the string, whitespace or a forward slash
@@ -19,7 +19,7 @@ class ArxivId
       (?:0[1-9]|1[012]) # Month
       \d{3}             # Number
       (?:v\d+)?         # Literal v followed by version number of 1 or more digits
-      (?=$|[[:space:]])    # Look-ahead for end of string or whitespace
+      (?=$|[[:space:]?#])  # Look-ahead for end, whitespace, query or fragment
     }xi
 
     def self.extract(str)

lib/identifiers/doi.rb

@@ -28,10 +28,16 @@ class DOI
       )
     }x
 
+    RESOLVER_URL_QUERY_OR_FRAGMENT_REGEXP = %r{
+      (https?://(?:www\.|dx\.)?doi\.org/[^\s?#]+)
+      [?#]\S+
+    }ix
+
     def self.extract(str, options = {})
       strict = options.fetch(:strict, false)
 
-      dois = str.to_s.downcase.scan(REGEXP)
+      text = str.to_s.gsub(RESOLVER_URL_QUERY_OR_FRAGMENT_REGEXP, '\\1')
+      dois = text.downcase.scan(REGEXP)
       dois = dois.map { |doi| doi.gsub(/\.+$/, '') } unless strict
 
       dois

lib/identifiers/handle.rb

@@ -1,7 +1,13 @@
 module Identifiers
   class Handle
+    RESOLVER_URL_QUERY_OR_FRAGMENT_REGEXP = %r{
+      (https?://hdl\.handle\.net/[^\s?#]+)
+      [?#]\S+
+    }ix
+
     def self.extract(str)
-      str.to_s.scan(%r{\b[0-9.]+/[^[:space:]]+\b}i)
+      text = str.to_s.gsub(RESOLVER_URL_QUERY_OR_FRAGMENT_REGEXP, '\\1')
+      text.scan(%r{\b[0-9.]+/[^[:space:]]+\b}i)
     end
   end
 end

lib/identifiers/repec_id.rb

@@ -3,7 +3,7 @@ class RepecId
     def self.extract(str)
       str
         .to_s
-        .scan(/\brepec:[^[:space:]]+\b/i)
+        .scan(/\brepec:[^[:space:]?#&]+\b/i)
         .map { |repec| "RePEc:#{repec.split(':', 2).last}" }
     end
   end

spec/identifiers/arxiv_id_spec.rb

@@ -14,6 +14,15 @@
       expect(described_class.extract('Example: arXiv:1501.00001v2')).to contain_exactly('1501.00001v2')
     end
 
+    it 'extracts arXiv IDs from URLs with queries and fragments' do
+      str = <<~TEXT
+        https://arxiv.org/abs/1501.00001v2?utm_source=readme
+        https://arxiv.org/abs/math.GT/0309136#section
+      TEXT
+
+      expect(described_class.extract(str)).to contain_exactly('1501.00001v2', 'math.GT/0309136')
+    end
+
     it 'does not extract IDs from DOIs that end in a valid arXiv ID' do
       expect(described_class.extract('10.1049/el.2013.3006')).to be_empty
     end

spec/identifiers/doi_spec.rb

@@ -23,6 +23,21 @@ def each_doi(file)
         expect(described_class.extract(str, options)).to contain_exactly('10.1049/el.2013.3006')
       end
 
+      it 'discards queries and fragments from DOI resolver URLs' do
+        str = <<~TEXT
+          https://doi.org/10.1000/PAPER.1?utm_source=readme
+          https://dx.doi.org/10.1000/PAPER.2#abstract
+        TEXT
+
+        expect(described_class.extract(str, options)).to contain_exactly('10.1000/paper.1', '10.1000/paper.2')
+      end
+
+      it 'retains question marks and hashes in bare DOI suffixes' do
+        str = '10.1000/PAPER?appendix 10.1000/PAPER#figure'
+
+        expect(described_class.extract(str, options)).to contain_exactly('10.1000/paper?appendix', '10.1000/paper#figure')
+      end
+
       it 'downcases the DOIs extracted' do
         str = 'This is an example of a DOI: 10.1097/01.ASW.0000443266.17665.19'
 

spec/identifiers/handle_spec.rb

@@ -13,6 +13,21 @@
     expect(described_class.extract(str)).to contain_exactly('2117/83545it.ly/1UtXnTW')
   end
 
+  it 'discards queries and fragments from Handle resolver URLs' do
+    str = <<~TEXT
+      https://hdl.handle.net/10149/596901?utm_source=readme
+      http://hdl.handle.net/10251/79612#details
+    TEXT
+
+    expect(described_class.extract(str)).to contain_exactly('10149/596901', '10251/79612')
+  end
+
+  it 'retains question marks and hashes in bare Handle suffixes' do
+    str = '10149/report?appendix 10251/report#figure'
+
+    expect(described_class.extract(str)).to contain_exactly('10149/report?appendix', '10251/report#figure')
+  end
+
   it 'extracts Handles separated by Unicode whitespace' do
     str = '10149/596901 10251/79612'
 

spec/identifiers/repec_id_spec.rb

@@ -19,6 +19,20 @@
     expect(described_class.extract(str)).to contain_exactly('RePEc:wbk:wbpubs:2266', 'RePEc:inn:wpaper:2016-03')
   end
 
+  it 'stops before URL queries, fragments, and parameters' do
+    str = <<~TEXT
+      RePEc:wbk:wbpubs:2266?utm_source=readme
+      RePEc:inn:wpaper:2016-03#section
+      https://socionet.example/item?h=RePEc:rpc:rdfdoc:redif&utm_source=readme
+    TEXT
+
+    expect(described_class.extract(str)).to contain_exactly(
+      'RePEc:wbk:wbpubs:2266',
+      'RePEc:inn:wpaper:2016-03',
+      'RePEc:rpc:rdfdoc:redif'
+    )
+  end
+
   it 'extracts nothing when given empty arguments' do
     expect(described_class.extract(nil)).to be_empty
   end