fix: align appended emissions records to existing CSV headers

codecarbon/output_methods/file.py

@@ -1,6 +1,6 @@
 import csv
 import os
-from typing import List
+from typing import List, Optional
 
 import pandas as pd
 
@@ -51,23 +51,31 @@ def __init__(
             f"Emissions data (if any) will be saved to file {os.path.abspath(self.save_file_path)}"
         )
 
-    def has_valid_headers(self, data: EmissionsData) -> bool:
+    def has_valid_headers(
+        self, data: EmissionsData, headers: Optional[List[str]] = None
+    ) -> bool:
         """
         Checks self.save_file_path has headers matching those from passed data.
 
         Args:
             data: EmissionsData object with valid headers.
+            headers: header row already read from the file; read it if None.
 
         Returns:
             True if the file has valid headers, False otherwise.
         """
-        with open(self.save_file_path) as csv_file:
-            reader = csv.reader(csv_file)
-            try:
-                headers = next(reader)
-            except StopIteration:
+        if headers is None:
+            headers = self._read_headers()
+            if headers is None:
                 return True
-            return sorted(headers) == sorted(data.values.keys())
+        return sorted(headers) == sorted(data.values.keys())
+
+    def _read_headers(self) -> Optional[List[str]]:
+        """
+        Read the header row of self.save_file_path, or None if the file is empty.
+        """
+        with open(self.save_file_path, newline="") as csv_file:
+            return next(csv.reader(csv_file), None)
 
     def out(self, total: EmissionsData, _):
         """
@@ -91,7 +99,8 @@ def out(self, total: EmissionsData, _):
             )
             file_exists = False
 
-        headers_match = file_exists and self.has_valid_headers(total)
+        headers = self._read_headers() if file_exists else None
+        headers_match = file_exists and self.has_valid_headers(total, headers)
         if file_exists and not headers_match:
             logger.warning("The CSV format has changed, backing up old emission file.")
             backup(self.save_file_path)
@@ -104,6 +113,11 @@ def out(self, total: EmissionsData, _):
         elif self.on_csv_write == "append":
             # Never drop all-NA columns here: the append is headerless, so column
             # identity is positional and a missing column shifts every value after it.
+            # Header names may be in any order (see has_valid_headers), so write the
+            # values in the file's order. reindex() drops or NaN-fills mismatched
+            # columns silently: it is only safe because has_valid_headers() above
+            # guarantees the file and the record have the same set of columns.
+            new_df = new_df.reindex(columns=headers)
             new_df.to_csv(self.save_file_path, mode="a", header=False, index=False)
         else:
             df = pd.read_csv(self.save_file_path)

tests/output_methods/test_file.py

@@ -1,7 +1,9 @@
+import csv
 import os
 import shutil
 import tempfile
 import unittest
+from dataclasses import replace
 from unittest.mock import patch
 
 import pandas as pd
@@ -87,6 +89,32 @@ def test_has_valid_headers_different_order_success(self):
 
         self.assertTrue(file_output.has_valid_headers(self.emissions_data))
 
+    def test_file_output_preserves_reordered_header_alignment(self):
+        for mode in ("append", "update"):
+            with self.subTest(mode=mode):
+                output = FileOutput(f"{mode}.csv", self.temp_dir, on_csv_write=mode)
+                output.out(self.emissions_data, None)
+                df = pd.read_csv(output.save_file_path)
+                df[list(reversed(df.columns))].to_csv(
+                    output.save_file_path, index=False
+                )
+                next_run = replace(
+                    self.emissions_data, run_id="next_run", project_name="next_project"
+                )
+
+                output.out(next_run, None)
+
+                with open(output.save_file_path, newline="") as file:
+                    rows = list(csv.DictReader(file))
+                self.assertEqual(len(rows), 2)
+                self.assertEqual(
+                    rows[-1],
+                    {
+                        key: str(value) if value is not None else ""
+                        for key, value in next_run.values.items()
+                    },
+                )
+
     def test_has_valid_headers_failure(self):
         file_output = FileOutput("test.csv", self.temp_dir)
         file_output.out(self.emissions_data, None)