Example notebook

.github/PULL_REQUEST_TEMPLATE

@@ -4,6 +4,10 @@
 * [ ] I have added [a changelog entry](https://github.com/LiberTEM/LiberTEM/tree/master/docs/source/changelog) for my contribution
 * [ ] I have added/updated documentation for all user-facing changes
 * [ ] I have added/updated test cases
+#
+## Reviewer Checklist:
+
+* [ ] `/azp run libertem.libertem-holo` passed
 
 <!--
 

.github/workflows/ci.yml

@@ -27,7 +27,7 @@ jobs:
             - version: "3.13"
               toxenv: "313"
             - version: "3.14"
-              toxenv: "313"
+              toxenv: "314"
         os: [ubuntu-latest, windows-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:

.gitignore

@@ -21,11 +21,12 @@ Miniconda3-latest-Linux-x86_64.sh
 MANIFEST
 .vscode
 pip-wheel-metadata
-data
+./data
 *Thumbs.db
 *.DS_Store
 *.idea
 .mypy_cache/
 .benchmarks
 src/libertem_holo/_baked_revision.py
 *.dm4
+junit.xml

README.rst

@@ -64,12 +64,28 @@ collected from a magnetic TEM lamella used for the example is available
 
 Installation
 ------------
+
+If you are using conda, for example via `miniforge <https://conda-forge.org/download/>`_,
+installation into a new environment can be done as follows:
+
 .. code-block:: shell
 
   $ conda create -n holo python=3.12
   $ conda activate holo
   $ pip install libertem-holo
 
+For GPU support, install the :code:`gpu` extra, and cupy. For example
+
+.. code-block:: shell
+
+  $ conda create -n holo python=3.12
+  $ conda activate holo
+  $ pip install libertem-holo[gpu] cupy-cuda13x 'cuda-toolkit[all]<14'
+
+Make sure to pick a CUDA version that works with your GPU and its drivers.
+For more details, please check the
+`CuPy installation guide <https://docs.cupy.dev/en/latest/install.html>`_.
+
 Input File formats
 ------------------
 LiberTEM-holo was designed specfically for working on larger stacks of images

azure-pipelines-data.yml

@@ -51,11 +51,11 @@ stages:
         displayName: 'install tox'
 
       - ${{ if eq(parameters.recreate_tox, true) }}:
-        - bash: $(Agent.TempDirectory)/venv/bin/tox -e notebooks -r -- examples/*.ipynb -v
+        - bash: $(Agent.TempDirectory)/venv/bin/tox -e notebooks -r -- notebooks/*.ipynb -v
           displayName: 'Run nbval tests $(Agent.OS) (recreating)'
 
       - ${{ if eq(parameters.recreate_tox, false) }}:
-        - bash: $(Agent.TempDirectory)/venv/bin/tox -e notebooks -- examples/*.ipynb -v
+        - bash: $(Agent.TempDirectory)/venv/bin/tox -e notebooks -- notebooks/*.ipynb -v
           displayName: 'Run nbval tests $(Agent.OS)'
 
       - bash: ./scripts/codecov.sh -f ./coverage.xml
@@ -110,6 +110,32 @@ stages:
           env:
             CODECOV_TOKEN: $(CODECOV_TOKEN)
 
+    - job: gpu_cupy_tests
+      pool: GPUTests
+      variables:
+        TESTDATA_BASE_PATH: '/data/'
+        TOXENV: py313-cupy
+      steps:
+      - task: UsePythonVersion@0
+        displayName: 'Use Python 3.13'
+        inputs:
+          versionSpec: '3.13'
+
+      - bash: python3 -m venv $(Agent.TempDirectory)/venv
+        displayName: 'create venv'
+
+      - bash: $(Agent.TempDirectory)/venv/bin/pip install -U tox tox-uv uv
+        displayName: 'install tox'
+
+      - bash: $(Agent.TempDirectory)/venv/bin/tox -r
+        displayName: 'Run tox tests $(TOXENV) $(Agent.OS)'
+
+      - bash: ./scripts/codecov.sh -f ./coverage.xml
+        displayName: 'Submit coverage to codecov.io'
+        ${{ if and(not(eq(variables['Build.Reason'], 'PullRequest')), eq(variables['Build.SourceBranch'], 'refs/heads/master')) }}:
+          env:
+            CODECOV_TOKEN: $(CODECOV_TOKEN)
+
     - job: data_tests
       pool: DataAccess
       strategy:

conftest.py

@@ -3,6 +3,9 @@
 Use this file to define fixtures to use
 in both doctests and regular tests.
 """
+import os
+import pathlib
+
 import numpy as np
 import pytest
 from libertem.api import Context
@@ -108,3 +111,17 @@ def auto_ds(doctest_namespace, holo_data):
 def auto_ctx(doctest_namespace):
     ctx = Context(executor=InlineJobExecutor())
     doctest_namespace["ctx"] = ctx
+
+
+@pytest.fixture
+def dm_testdata_path(scope='module'):
+    base_path = os.environ.get('TESTDATA_BASE_PATH')
+    if base_path is None:
+        pytest.skip('need test data, TESTDATA_BASE_PATH is not set')
+    return pathlib.Path(base_path) / 'dm'
+
+
+def pytest_collectstart(collector):
+    # nbval: ignore some output types
+    if collector.fspath and collector.fspath.ext == '.ipynb':
+        collector.skip_compare += 'text/html', 'application/javascript', 'stderr',

docs/source/conf.py

@@ -190,7 +190,7 @@
 #  dir menu entry, description, category)
 texinfo_documents = [
     (master_doc, 'LiberTEM-holo', 'LiberTEM-holo Documentation',
-     author, 'LiberTEM-holo', 'One line description of project.',
+     author, 'LiberTEM-holo', 'Open-source electron holography reconstruction.',
      'Miscellaneous'),
 ]
 
@@ -207,6 +207,16 @@
 # is not set up correctly
 doctest_test_doctest_blocks = ''
 
+doctest_global_setup = '''
+import os
+import pathlib
+base_path = os.environ.get('TESTDATA_BASE_PATH')
+if base_path is None or len(base_path) == 0:
+    path_to_data = None
+else:
+    path_to_data = pathlib.Path(base_path) / 'dm' / '3D' / 'alpha-50_obj.dm3'
+'''
+
 # -- Options for todo extension ----------------------------------------------
 
 # If true, `todo` and `todoList` produce output, else they produce nothing.

docs/source/development.rst

@@ -0,0 +1,15 @@
+Development
+===========
+
+Running notebooks in CI
+-----------------------
+
+Notebooks are tested as part of the CI run. When adding a notebook,
+you can generate the notebook outputs in a clean environment using tox:
+
+.. code-block:: shell
+
+    $ tox -e notebooks_gen
+    $ cp notebooks/generated/*.ipynb notebooks
+
+

docs/source/index.rst

@@ -16,14 +16,16 @@ Documentation
    :caption: Contents:
 
    usage
+   io
    changelog
    acknowledgments
 
 .. toctree::
-    :maxdepth: 2
-    :caption: Reference
+   :maxdepth: 2
+   :caption: Reference
 
-    reference/index
+   reference/index
+   development
 
 
 Indices and tables

docs/source/io.rst

@@ -0,0 +1,139 @@
+.. testsetup:: *
+
+   import os
+   import pathlib
+   base_path = os.environ.get('TESTDATA_BASE_PATH')
+   if base_path is None:
+      path_to_data = None
+   else:
+      path_to_data = pathlib.Path(base_path) / 'dm' / '3D' / 'alpha-50_obj.dm3'
+
+
+Input and Output
+================
+
+Loading holograms
+-----------------
+
+In our :ref:`I/O module <io api>`, we provide helpers for both loading data,
+and after reconstruction, saving the results in a simple numpy npz file.
+
+.. testcode:: load_data
+   :skipif: path_to_data is None
+
+   from libertem_holo.base.io import InputData
+
+   # insert the full path to a dm3 or dm4 file here:
+   input_data = InputData.load_from_dm(path_to_data)
+
+
+The data itself and the most important metadata is directly available:
+
+.. testcode:: load_data
+   :skipif: path_to_data is None
+
+   print(f"shape: {input_data.data.shape}")
+   print(f"pixel size: {input_data.pixelsize} nm")
+   print(f"total expsure time: {input_data.exposure_time} s")
+
+This code will print, for example:
+
+.. testoutput:: load_data
+
+   shape: (20, 3838, 3710)
+   pixel size: 0.1671157330274582 nm
+   total expsure time: 120.0 s
+
+We make a few assumptions:
+
+- Pixel sizes are given in nm/px and are the same in X and Y direction
+- The data is either a single hologram or a 3D stack of holograms
+
+.. note::
+
+   If you are using UDFs for reconstruction, you need to use
+   :ref:`the I/O functionality of LiberTEM <libertem:dataset api>` to load data.
+   This is especially useful for reconstructing very large stacks.
+
+
+Saving and loading results
+--------------------------
+
+Once you have reconstructed your data, you can save the results in a convenient
+format:
+
+.. testsetup:: results
+
+    import numpy as np
+    import tempfile
+    import pathlib
+    from libertem_holo.base.io import InputData, Results
+    from libertem_holo.base.utils import HoloParams
+
+    wave = np.random.random((128, 128)) + 1j * np.random.random((128, 128))
+    phase = np.random.random((128, 128))
+    brightfield = np.random.random((128, 128))
+    temp_dir = tempfile.TemporaryDirectory()
+    save_path = pathlib.Path(temp_dir.name) / 'test-1.npz'
+    holo = np.random.random((256, 256))
+    input_data = InputData(
+        data=holo,
+        exposure_time=24.6,
+        pixelsize=0.1,
+    )
+    holo_params = HoloParams.from_hologram(
+        holo,
+        out_shape=(holo.shape[0] // 2, holo.shape[1] // 2),
+    )
+
+.. testcode:: results
+   :skipif: path_to_data is None
+
+   from libertem_holo.base.io import Results
+
+   res = Results(
+       complex_wave=wave,
+       unwrapped_phase=phase,
+       brightfield=brightfield,
+       metadata={"custom": 12.34},
+   )
+   # save_path is the full or relative path that ends in .npz
+   # where your data will be saved:
+   print(f"saving to {save_path}")
+   res.save(save_path)
+
+.. testoutput:: results
+   :options: +ELLIPSIS
+
+   saving to ...
+
+You can also include metadata from the input data and reconstruction parameters
+in the results:
+
+.. testcode:: results
+   :skipif: path_to_data is None
+
+   res.metadata_from_input(
+       input_data=input_data,
+       params=holo_params,  # a HoloParams instance
+   )
+   res.save(save_path)
+   print(f"effective pixel size: {res.metadata['effective_pixelsize']} nm")
+   print(f"stack shape: {res.metadata['stack_shape']}")
+   print(f"total exposure time: {res.metadata['exposure_time']} s")
+   print(f"a custom value: {res.metadata['custom']}")
+
+This code will print, for example:
+
+.. testoutput:: results
+
+    effective pixel size: 0.2 nm
+    stack shape: [256, 256]
+    total exposure time: 24.6 s
+    a custom value: 12.34
+
+The effective pixel size is the pixel size in the resulting phase image,
+which is the original pixel size adjusted for the chosen output shape.
+In case of stack reconstruction, the exposure time read from the input metadata
+is multiplied by the number of frames in the stack.
+

docs/source/reference/index.rst

@@ -39,3 +39,11 @@ Utility functions
 
 .. automodule:: libertem_holo.base.utils
     :members:
+
+.. _`io api`:
+
+Input/output
+~~~~~~~~~~~~
+
+.. automodule:: libertem_holo.base.io
+    :members:

nbval_sanitize.cfg

@@ -0,0 +1,27 @@
+[cpu_times]
+regex: CPU times: user (\d+min ?)?([\d\.e+]+ ?[nmµ]?s)?, sys: (\d+min ?)?([\d\.e+]+ ?[mnµ]?s)?, total: (\d+min ?)?([\d\.e+]+ ?[nmµ]?s)?
+replace: CPU_TIMES
+
+[cpu_times_win]
+regex: CPU times: total: (\d+min ?)?([\d\.e+]+ ?[nmµ]?s)?
+replace: CPU_TIMES
+
+[wall_time]
+regex: Wall time: (\d+min ?)?([\d\.e+]+ ?[nmµ]?s)?
+replace: WALL_TIME
+
+[default_repr]
+regex: <[a-zA-Z\.]+ at 0x[0-9a-f]+>
+replace: DEFAULT_REPR
+
+[figure_size]
+regex: Figure size [0-9]+x[0-9]+
+replace: FIGURE
+
+[num_partitions]
+regex: 'name': 'Number of partitions', 'value': '[0-9]+'
+replace: NUM_PART
+
+[progress_bar]
+regex: Partitions [0-9]+(\([0-9]+\))?\/[0-9]+, Frames:\s*[0-9]+\%.*$
+replace: PROGRESS_TEXT