Feat: incremental glacier directory creation

docs/_code/prepare_hef.py

@@ -46,7 +46,7 @@
 out = workflow.calibrate_inversion_from_volume(gdir, vol_ref_m3=refv)
 np.testing.assert_allclose(refv, out['vol_oggm_m3'], rtol=0.01)
 
-cl = gdir.read_pickle('inversion_input')[-1]
+cl = gdir.read_store('inversion_input')[-1]
 mbmod = massbalance.ConstantMassBalance(gdir, y0=1985)
 mbx = mbmod.get_annual_mb(cl['hgt']) * cfg.SEC_IN_YEAR * cfg.PARAMS['ice_density']
 fdf = pd.DataFrame(index=np.arange(len(mbx))*cl['dx'])

docs/api.rst

@@ -510,7 +510,7 @@ level 5.
                                               prepro_border=80)
     gdir = gdirs[0]  # init_glacier_directories always returns a list
 
-    fls = gdir.read_pickle('model_flowlines')
+    fls = gdir.read_store('model_flowlines')
     fls
     [fl.order for fl in fls]
 

docs/whats-new.rst

@@ -678,6 +678,11 @@ break some code, but remain minor in nature:
   resulting thickness fields. This would yield to weird effects at glacier boundaries
   and seemed overkill since topography is smoothed quite aggressively as well (:pull:`1773`).
   By `Fabien Maussion <https://github.com/fmaussion>`_
+- ``read_pickle`` and ``write_pickle`` are now deprecated in favour of
+  ``read_store`` and ``write_store``. The latter uses npz files inside glacier
+  directories instead of pickles. Users should migrate to the new functions, as
+  compatibility with pickles is not guaranteed in the future. (:pull:`1908`).
+  By `Nicolas Gampierakis <https://github.com/gampnico>`_.
 
 
 v1.6.2 (August 25, 2024)

oggm/cfg.py

@@ -309,6 +309,10 @@ def _log_param_change(self, key, value):
 _doc = "A dict containing all observations used during the OGGM workflow."
 BASENAMES['observations'] = ('observations.yml', _doc)
 
+_doc = "A store of npz files containing previously pickled data."
+BASENAMES['data_store'] = ('data_store', _doc)
+
+
 
 def set_logging_config(logging_level='INFO'):
     """Set the global logger parameters.

oggm/cli/benchmark.py

@@ -73,8 +73,9 @@ def run_benchmark(rgi_version=None, rgi_reg=None, border=None,
     # Initialize OGGM and set up the run parameters
     cfg.initialize(logging_level=logging_level, params=override_params)
 
-    # Use multiprocessing?
-    cfg.PARAMS['use_multiprocessing'] = platform.system() != 'Darwin'
+    # Allow multiprocessing override in tests/platform
+    if 'use_multiprocessing' not in override_params:
+        cfg.PARAMS['use_multiprocessing'] = platform.system() != 'Darwin'
 
     # How many grid points around the glacier?
     # Make it large if you expect your glaciers to grow large