Migrate from conda to pixi and from Sphinx to MkDocs

.gitattributes

@@ -0,0 +1,6 @@
+# SPDX-FileCopyrightText: Contributors to technology-data <https://github.com/pypsa/technology-data>
+#
+# SPDX-License-Identifier: GPL-3.0-only
+
+# SCM syntax highlighting & preventing 3-way merges
+pixi.lock merge=binary linguist-language=YAML linguist-generated=true

.github/ISSUE_TEMPLATE/bug_report.md

@@ -19,7 +19,7 @@ assignees: ''
 ## Error Message
 
 *If applicable, paste any terminal output to help illustrating your problem.*
-*In some cases it may also be useful to share your list of installed packages: `conda list`.*
+*In some cases it may also be useful to share your list of installed packages: `pixi list`.*
 
 ```
 <paste here>

.github/pull_request_template.md

@@ -7,6 +7,6 @@ Closes # (if applicable).
 
 - [ ] Code changes are sufficiently documented; i.e. new functions contain docstrings and further explanations may be given in `doc`.
 - [ ] Data source for new technologies is clearly stated.
-- [ ] Newly introduced dependencies are added to `environment.yaml` (if applicable).
-- [ ] A note for the release notes `doc/release_notes.rst` of the upcoming release is included.
+- [ ] Newly introduced dependencies are added to `pixi.toml` and `pixi run sync-locks` was run (if applicable).
+- [ ] A note for the release notes `doc/release_notes.md` of the upcoming release is included.
 - [ ] I consent to the release of this PR's code under the GPLv3 license.

.github/workflows/ci.yaml

@@ -7,6 +7,7 @@ on:
     branches: [master]
   schedule:
   - cron: "0 5 * * 0"
+  workflow_dispatch:
 
 
 # Cancel any in-progress runs when a new run is triggered
@@ -27,23 +28,21 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v7
 
-    - name: Setup micromamba
-      uses: mamba-org/setup-micromamba@v3
+    - name: Setup Pixi
+      uses: prefix-dev/setup-pixi@v0.11.2
       with:
-        micromamba-version: latest
-        environment-file: environment.yaml
-        log-level: debug
-        init-shell: bash
-        cache-environment: true
-        cache-downloads: true
+        environments: test
+        cache: true
+        cache-write: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
 
     - name: Run unit tests
       run: |
-        make unit-test
+        pixi run unit-tests
 
     - name: Test snakemake workflows
       run: |
-        make test
+        pixi run -e test snakemake --cores all -f compile_cost_assumptions --configfile config.yaml
+        pixi run -e test snakemake --cores all -f compile_cost_assumptions_usa --configfile config.yaml
 
     - name: Upload artifacts
       uses: actions/upload-artifact@v7
@@ -56,7 +55,7 @@ jobs:
         git config --global --add safe.directory $GITHUB_WORKSPACE
         git add outputs/
         if ! git diff --cached --quiet; then
-          echo "Generated outputs differ from the branch. Please commit rerun `snakemake` and commit the changes to the `outputs/*.csv` files."
+          echo 'Generated outputs differ from the branch. Please rerun the workflow with pixi and commit the changes to the outputs/*.csv files.'
           git diff --cached
           exit 1
         else

.github/workflows/update-lockfile.yaml

@@ -0,0 +1,43 @@
+name: Update locked envs
+on:
+  schedule:
+  - cron: "0 8 1,16 * *" # Bi-weekly
+  workflow_dispatch:
+
+jobs:
+  update-lockfiles:
+    if: ${{ github.ref == 'refs/heads/master' }}
+    name: Update lockfiles
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+    - uses: actions/checkout@v7
+
+    - name: Setup Pixi
+      uses: prefix-dev/setup-pixi@v0.11.2
+
+    - name: Full resolve
+      run: |
+        rm pixi.lock
+        pixi install --all
+        git checkout pixi.toml
+
+    - name: Export conda environment files
+      run: pixi run sync-locks
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v8
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        branch: update-locked-environment
+        title: "[github-actions.ci] Update locked envs"
+        body: |
+          Automatically generated PR to update the pixi lockfile.
+
+          **Note: Do not merge without manual test execution.**
+        commit-message: "Update locked environment files for all platforms"

.gitignore

@@ -4,7 +4,7 @@
 
 .DS_Store
 .snakemake
-_build
+/site
 .idea
 .vscode
 gurobi.log
@@ -25,3 +25,5 @@ gurobi.log
 *.cb2
 .*.lb
 .claude/*
+.pixi/*
+!.pixi/config.toml

docs/requirements.txt → .pixi/config.toml

@@ -2,7 +2,4 @@
 #
 # SPDX-License-Identifier: GPL-3.0-only
 
-sphinx==8.1.3
-sphinx-book-theme==1.1.3
-sphinxcontrib-bibtex==2.6.3
-myst-parser==4.0.0
+pinning-strategy = "latest-up"

.pre-commit-config.yaml

@@ -31,7 +31,7 @@ repos:
   hooks:
   - id: codespell
     args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore,vor,pris,WEGE,Wege,Eletricity'] # Ignore capital case words, e.g. country codes
-    types_or: [python, rst, markdown]
+    types_or: [python, markdown]
     files: ^(scripts|doc)/
 
   # YAML formatting

.readthedocs.yml

@@ -4,14 +4,20 @@
 
 version: 2
 
-sphinx:
-  configuration: docs/conf.py
+mkdocs:
+  configuration: mkdocs.yml
 
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   tools:
-    python: "3.11"
-
-python:
-  install:
-  - requirements: docs/requirements.txt
+    python: "3.12"
+  jobs:
+    create_environment:
+    - asdf plugin add pixi
+    - asdf install pixi latest
+    - asdf global pixi latest
+    install:
+    - pixi install -e doc --frozen
+    build:
+      html:
+      - pixi run build-docs $READTHEDOCS_OUTPUT

.syncignore

@@ -13,5 +13,6 @@ __pycache__
 *.pyc
 *.pyo
 *.ipynb
-docs
+doc
 latex_tables
+.pixi

Snakefile

@@ -32,8 +32,6 @@ rule compile_cost_assumptions:
         expand("outputs/costs_{year}.csv", year=config["years"]),
     log:
         pathlib.Path("logs", "compile_cost_assumptions.log"),
-    conda:
-        "environment.yaml"
     threads: 1
     resources:
         mem=500,
@@ -56,8 +54,6 @@ rule compile_cost_assumptions_usa:
         expand("outputs/US/costs_{year}.csv", year=config["years"]),
     log:
         pathlib.Path("logs", "compile_cost_assumptions_usa.log"),
-    conda:
-        "environment.yaml"
     threads: 1
     resources:
         mem=500,
@@ -73,7 +69,6 @@ rule compile_cost_assumptions_usa:
 #         energy_prices = "inputs/Fraunhofer_ISE_energy_prices.csv"
 #     threads: 1
 #     resources: mem=500
-#     conda: "environment.yaml"
 #     script: "scripts/convert_pdf_fraunhofer_to_dataframe.py"
 
 
@@ -82,8 +77,6 @@ rule convert_EWG:
         EWG="docu/EWG_LUT_100RE_All_Sectors_Global_Report_2019.pdf",
     output:
         costs="inputs/EWG_costs.csv",
-    conda:
-        "environment.yaml"
     threads: 1
     resources:
         mem=500,

doc/addnew.md

@@ -0,0 +1,79 @@
+<!-- SPDX-FileCopyrightText: Contributors to technology-data <https://github.com/pypsa/technology-data> -->
+<!-- SPDX-License-Identifier: GPL-3.0-only -->
+
+# How to add a new technology
+
+## Add a new technology from the Danish Energy Agency (DEA)
+
+You can add a new technology from the Danish Energy Agency database which are
+saved in `inputs/*.xlsx`. One can add the wished technology name (e.g. "onwind")
+as a key and the excel sheet name of the technology in DEA (e.g. '20 Onshore turbines')
+as a value to the dictionary `sheet_names` in the `compile_cost_assumptions.py`.
+For using the technology data uncertainty of DEA, add to the dictionary
+`uncrtnty_lookup` the columns in the excel sheet which contain the uncertainty
+estimations (e.g. `'onwind':'J:K'`)
+It is recommended to still check in the output csv if all new technology data is
+compiled correctly.
+
+```python
+--8<-- "scripts/compile_cost_assumptions.py:DEA-sheet-names"
+```
+
+If you want to extract further parameters from the DEA excel sheets you can
+add them to the `parameter` list in the function `get_data_DEA()`.
+
+## Add a new technology from another source
+
+A new technology can be added to the technology-data with an additional function in the `compile_cost_assumptions.py`.
+In this script a pandas Dataframe called `costs` is created for every year
+specified in the `config.yaml`. `costs` has a MultiIndex (`['technology', 'parameter']`)
+and four columns (`['value', 'unit', 'source', 'further description']`).
+
+```python
+--8<-- "scripts/compile_cost_assumptions.py:RTD-target-multiindex-df"
+```
+
+The function for the newly added technology should extend this pandas Dataframe
+`costs`. This is done for example for solar PV costs in the function [add_solar_from_other(costs)](https://github.com/PyPSA/technology-data/blob/2f4da6d75f07ef9457f4070b26d690f1e5e932a5/scripts/compile_cost_assumptions.py#L396)
+
+```python
+--8<-- "scripts/compile_cost_assumptions.py:add-solar-from-others"
+```
+
+If a new technology is added, existing parameter names (e.g. "investment") and
+units (e.g. EUR/MW) should be used. If the energy output is not distinct, clarify
+e.g. EUR/MW_el or EUR/MW_th. Efficiencies for converting/using hydrogen or
+methane are given for Lower Heating Values (LHV).
+All parameters and corresponding units are
+summarised in the table below. It is not necessary to add for one technology all
+parameters, e.g. if there are only investment costs for a technology only those
+can be added without specifying e.g. VOM.
+
+{{ read_csv("doc/parameter.csv", keep_default_na=False) }}
+
+Try to make the source of the data as clear as possible (e.g. including page
+number, DOI) and add the source to the [source_dict](https://github.com/PyPSA/technology-data/blob/2f4da6d75f07ef9457f4070b26d690f1e5e932a5/scripts/compile_cost_assumptions.py#L35)
+in the `compile_cost_assumptions.py`. Convert the units within the script (this can be done [in part (2)](https://github.com/PyPSA/technology-data/blob/2f4da6d75f07ef9457f4070b26d690f1e5e932a5/scripts/compile_cost_assumptions.py#L1016) in the script)
+to be as transparent as possible.
+
+The inflation rate should be considered for the investment costs,
+as it is done for example for the cost assumptions from DIW from 2010
+
+```python
+--8<-- "scripts/compile_cost_assumptions.py:unify-diw-inflation"
+```
+
+The output cost assumptions are given for different years. So either the added
+cost assumptions have to be interpolated for different in the `config.yaml`
+specified years, as done e.g. with the technology data from DEA [here](https://github.com/PyPSA/technology-data/blob/2f4da6d75f07ef9457f4070b26d690f1e5e932a5/scripts/compile_cost_assumptions.py#L259)
+
+```python
+--8<-- "scripts/compile_cost_assumptions.py:RTD-interpolation-example"
+```
+
+or the technology data is assumed to be constant for the different years, as e.g.
+done if other electrolyzer data is assumed in this [function](https://github.com/PyPSA/technology-data/blob/2f4da6d75f07ef9457f4070b26d690f1e5e932a5/scripts/compile_cost_assumptions.py#L443).
+
+```python
+--8<-- "scripts/compile_cost_assumptions.py:add-h2-from-other"
+```

doc/assets/stylesheets/extra.css

@@ -0,0 +1,13 @@
+/*
+ * SPDX-FileCopyrightText: Contributors to technology-data <https://github.com/pypsa/technology-data>
+ *
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+:root > * {
+  --md-primary-fg-color:        #4b73d7;
+  --md-primary-fg-color--light: #749bfb;
+  --md-primary-fg-color--dark:  #3d5696;
+  --md-primary-bg-color:           white;
+  --md-accent-fg-color: rgb(75, 115, 215, 0.5);
+  --md-typeset-a-color:    #4b73d7;
+}

doc/index.md

@@ -0,0 +1,40 @@
+<!-- SPDX-FileCopyrightText: Contributors to technology-data <https://github.com/pypsa/technology-data> -->
+<!-- SPDX-License-Identifier: GPL-3.0-only -->
+
+# Technology-data base
+
+![GitHub release (latest by date including pre-releases)](https://img.shields.io/github/v/release/pypsa/technology-data?include_prereleases)
+[![Documentation Status](https://readthedocs.org/projects/technology-data/badge/?version=latest)](https://technology-data.readthedocs.io/en/latest/?badge=latest)
+![GitHub](https://img.shields.io/github/license/pypsa/technology-data)
+![GitHub repo size](https://img.shields.io/github/repo-size/pypsa/technology-data)
+[![Chat on Gitter](https://badges.gitter.im/PyPSA/community.svg)](https://gitter.im/PyPSA/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
+
+This script compiles assumptions on energy system technologies (such as costs, efficiencies, lifetimes, etc.) for chosen years (e.g. [2020, 2030, 2050]) from a variety
+of sources into CSV files to be read by energy system modelling software. The merged outputs have standardized cost years, technology names, units and source information.
+
+This project is maintained by the [Department of Digital Transformation in Energy Systems](https://www.ensys.tu-berlin.de) at the [Technische Universität Berlin](https://www.tu.berlin)
+and [Open Energy Transition (OET)](https://openenergytransition.org).
+
+## Documentation
+
+**Getting Started**
+
+* [Installation](installation.md)
+
+**Structure**
+
+* [Structure of the repository](structure.md)
+
+**How to add a new technology**
+
+* [How to add a new technology](addnew.md)
+
+**References**
+
+* [Release Notes](release_notes.md)
+
+## Licence
+
+The code in technology-data is released as free software under the [GPLv3](http://www.gnu.org/licenses/gpl-3.0.en.html), see
+[LICENSE](https://github.com/PyPSA/technology-data/blob/master/LICENSE.txt).
+However, different licenses and terms of use may apply to the various input data.