Add isolated sessions: a fresh Python process per session, from a prepared template

CHANGELOG.md

@@ -1,6 +1,57 @@
 # Changelog
 
-## 5.0.1 (2026-09-22)
+## 5.1.0 (unreleased)
+
+### Added
+
+- **Isolated sessions** - `py_session:template/1` prepares a Python
+  environment once (interpreter, `paths`, `imports`, `preload`, `env`,
+  `hash_seed`, `rlimits`) and `py_session:new/1` gives a fresh child process
+  per session that starts from it and shares no state with any other
+  session: module globals, `sys.modules`, environment, threads, working
+  directory. By default a session is forked from a zygote that already ran
+  the imports and preload, so it is ready in a few milliseconds instead of
+  the ~50 ms of a new interpreter plus its imports; `start => spawn` starts
+  a new interpreter per session (optionally from a `warm` pool) for code
+  that cannot be forked. A session is an isolated context: calls, callbacks,
+  calls back into the same session, interrupts, loops and `pass_fd` work
+  unchanged. `close/1` kills it; a session whose process dies answers with
+  the reason until closed and is never restarted. `run/5` runs one call in a
+  new session; `refresh/1` rebuilds the template after a deploy; `info/1`
+  reports zygotes, live sessions and forks. `start => reimport` runs each
+  call in a fresh module dictionary on a worker or owngil context instead,
+  the way Temporal's Python SDK isolates a workflow run: the function's
+  module is imported again per run, and the standard library, `imports` and
+  `passthrough` modules are shared. On such a template `new/1` gives a
+  session that keeps its fresh modules and `__main__` over several calls,
+  used like any context (`py_context:call/eval/exec`, `py:call`). A timeout
+  there does not interrupt, since the context is shared with other
+  sessions. See `docs/sessions.md`.
+- `clear_env` and `hash_seed` options for isolated contexts: the child sees
+  only the variables named in `env`, and every child built with the same
+  seed hashes strings and orders sets the same way.
+
+### Fixed
+
+- On an owngil context, the process-local environment of an Erlang process
+  (`py:call/eval/exec` on the context) was kept until the context stopped:
+  300 short-lived processes each leaving 1 MB grew the node by 300 MB. The
+  environment now goes back to its context thread when the process exits
+  and is released before the context's next request. The per-process
+  namespaces of an owngil context's event loop (`py_event_loop:exec/eval`)
+  were not released either; they now are.
+- A process that ran an event loop task with its process-local environment
+  (`py_event_loop:create_task` after `py:exec`, in any mode) left that
+  environment registered on the loop for the loop's lifetime. The
+  registration is now dropped, and the environment released, when the
+  process exits.
+- On an owngil context, a Python function that called `erlang.call`, where
+  the Erlang callback called the same context again, hung until the request
+  timeout. The context thread waited for the callback on its pipe while the
+  nested call sat in its queue. It now waits inline and serves the nested
+  call, as worker contexts do since 5.0.1.
+
+
 
 ### Fixed
 

c_src/py_callback.c

@@ -1657,9 +1657,14 @@ static PyObject *erlang_call_impl(PyObject *self, PyObject *args) {
     bool has_context_suspension = (tl_current_context != NULL && tl_allow_suspension &&
                                    !loop_running);
     bool has_context_handler = (tl_current_context != NULL && tl_current_context->has_callback_handler);
+    /* An owngil context also has a callback handler (for its other
+     * threads), but a request with a caller must wait inline too: on the
+     * handler pipe a callback calling back into this context would queue
+     * behind the request that waits for it. */
     bool has_context_inline = (tl_current_context != NULL && !tl_allow_suspension &&
                                tl_current_context->has_current_caller &&
-                               !has_context_handler && !loop_running);
+                               (!has_context_handler || tl_current_context->is_subinterp) &&
+                               !loop_running);
 
     if (has_context_inline) {
         Py_ssize_t nargs = PyTuple_Size(args);
@@ -3794,7 +3799,7 @@ static int create_erlang_module(void) {
         PyDict_SetItemString(log_globals, "__builtins__", builtins);
 
         /* Import erlang module into globals so the code can reference it */
-        PyObject *sys_modules = PySys_GetObject("modules");
+        PyObject *sys_modules = PyImport_GetModuleDict();  /* the interpreter's table, a dict even when sys.modules is replaced */
         if (sys_modules != NULL) {
             PyObject *erlang_mod = PyDict_GetItemString(sys_modules, "erlang");
             if (erlang_mod != NULL) {
@@ -3896,7 +3901,7 @@ static int create_erlang_module(void) {
         PyDict_SetItemString(ext_globals, "__builtins__", builtins);
 
         /* Import erlang module into globals so the code can reference it */
-        PyObject *sys_modules = PySys_GetObject("modules");
+        PyObject *sys_modules = PyImport_GetModuleDict();  /* the interpreter's table, a dict even when sys.modules is replaced */
         if (sys_modules != NULL) {
             PyObject *erlang_mod = PyDict_GetItemString(sys_modules, "erlang");
             if (erlang_mod != NULL) {
@@ -3953,7 +3958,7 @@ static int create_erlang_module(void) {
         PyDict_SetItemString(atom_globals, "__builtins__", builtins);
 
         /* Import erlang module into globals so the code can reference it */
-        PyObject *sys_modules = PySys_GetObject("modules");
+        PyObject *sys_modules = PyImport_GetModuleDict();  /* the interpreter's table, a dict even when sys.modules is replaced */
         if (sys_modules != NULL) {
             PyObject *erlang_mod = PyDict_GetItemString(sys_modules, "erlang");
             if (erlang_mod != NULL) {
@@ -4043,7 +4048,7 @@ static int create_erlang_module(void) {
         PyDict_SetItemString(sd_globals, "__builtins__", builtins);
 
         /* Import erlang module into globals so the code can reference it */
-        PyObject *sys_modules = PySys_GetObject("modules");
+        PyObject *sys_modules = PyImport_GetModuleDict();  /* the interpreter's table, a dict even when sys.modules is replaced */
         if (sys_modules != NULL) {
             PyObject *erlang_mod = PyDict_GetItemString(sys_modules, "erlang");
             if (erlang_mod != NULL) {

c_src/py_event_loop.c

@@ -93,7 +93,7 @@
  */
 
 /** @brief Name for the PyCapsule storing event loop pointer */
 static const char *EVENT_LOOP_CAPSULE_NAME = "erlang_python.event_loop";
 
 /* Capsule over a loop resource for ErlangEventLoop (defined with the
  * capsule helpers below). */
@@ -105,7 +105,7 @@
 static _Atomic uint64_t g_next_timer_ref = 1;
 
 /** @brief Module attribute name for storing the event loop */
 static const char *EVENT_LOOP_ATTR_NAME = "_loop";
 
 /* ============================================================================
  * Module State Structure
@@ -529,17 +529,37 @@
         pthread_mutex_unlock(&loop->namespaces_mutex);
         PyGILState_Release(gstate);
     } else {
-        /* Subinterpreter or runtime not running: just free structs */
+        /* Subinterpreter or runtime not running. Namespaces created in the
+         * subinterpreter went (or go) with it; those created in the main
+         * interpreter (event_loop_exec/eval) are released under the main
+         * GIL, taken after the lock is dropped. */
         pthread_mutex_lock(&loop->namespaces_mutex);
+        process_namespace_t *ns_list = loop->namespaces_head;
+        loop->namespaces_head = NULL;
+        pthread_mutex_unlock(&loop->namespaces_mutex);
 
-        process_namespace_t *ns = loop->namespaces_head;
+        bool main_gil = runtime_is_running() &&
+                        PyGILState_GetThisThreadState() == NULL && !PyGILState_Check();
+        PyGILState_STATE gstate = PyGILState_UNLOCKED;
+        if (main_gil) {
+            gstate = PyGILState_Ensure();
+        }
+        process_namespace_t *ns = ns_list;
         while (ns != NULL) {
             process_namespace_t *next = ns->next;
-            /* Skip Py_XDECREF - can't safely acquire GIL */
+            if (main_gil && ns->interp_id == 0) {
+                Py_XDECREF(ns->globals);
+                Py_XDECREF(ns->locals);
+                Py_XDECREF(ns->module_cache);
+            }
             enif_free(ns);
             ns = next;
         }
-        loop->namespaces_head = NULL;
+        if (main_gil) {
+            PyGILState_Release(gstate);
+        }
+
+        pthread_mutex_lock(&loop->namespaces_mutex);
 
         /* Clean up PID-to-env mappings */
         pid_env_mapping_t *mapping = loop->pid_env_head;
@@ -679,6 +699,28 @@
  * Per-Process Namespace Management
  * ============================================================================ */
 
+/**
+ * @brief Unlink the pid-to-env mapping of an exited process.
+ *
+ * Call with namespaces_mutex held; release the returned env (if any) with
+ * enif_release_resource after dropping it, since the env destructor may
+ * take the GIL (GIL before namespaces_mutex).
+ */
+static void *take_pid_env(erlang_event_loop_t *loop, const ErlNifPid *pid) {
+    pid_env_mapping_t **pp = &loop->pid_env_head;
+    while (*pp != NULL) {
+        if (enif_compare_pids(&(*pp)->pid, pid) == 0) {
+            pid_env_mapping_t *m = *pp;
+            void *env_res = m->env;
+            *pp = m->next;
+            enif_free(m);
+            return env_res;
+        }
+        pp = &(*pp)->next;
+    }
+    return NULL;
+}
+
 /**
  * @brief Down callback for event loop resources (process monitor)
  *
@@ -693,26 +735,53 @@
     erlang_event_loop_t *loop = (erlang_event_loop_t *)obj;
 
     /*
-     * For subinterpreters (interp_id != 0), we can't use PyGILState_Ensure.
-     * Just remove from the list without Py_DECREF - the Python objects will
-     * be cleaned up when the interpreter is destroyed.
+     * Subinterpreter loop. Its namespaces are not all its own: event_loop_exec
+     * and event_loop_eval create theirs in the main interpreter. Unlink under
+     * the lock; hand subinterpreter dicts to the OWN_GIL context thread while
+     * the lock guarantees that context is still serving; release
+     * main-interpreter dicts under the main GIL once the lock is dropped
+     * (GIL before namespaces_mutex, as everywhere else).
      */
     if (!runtime_is_running() || loop->interp_id != 0) {
+        process_namespace_t *main_ns = NULL;
         pthread_mutex_lock(&loop->namespaces_mutex);
+        void *dead_env = take_pid_env(loop, pid);
 
         process_namespace_t **pp = &loop->namespaces_head;
         while (*pp != NULL) {
             if (enif_compare_pids(&(*pp)->owner_pid, pid) == 0) {
                 process_namespace_t *to_free = *pp;
                 *pp = to_free->next;
-                /* Skip Py_XDECREF - can't safely acquire GIL for subinterp */
-                enif_free(to_free);
+                if (!runtime_is_running()) {
+                    enif_free(to_free);
+                } else if (to_free->interp_id == 0) {
+                    main_ns = to_free;
+                } else {
+                    if (loop->gc_ctx != NULL) {
+                        env_gc_push(loop->gc_ctx, to_free->globals, to_free->locals,
+                                    to_free->module_cache);
+                    }
+                    /* else: they go with the interpreter */
+                    enif_free(to_free);
+                }
                 break;
             }
             pp = &(*pp)->next;
         }
 
         pthread_mutex_unlock(&loop->namespaces_mutex);
+
+        if (main_ns != NULL) {
+            PyGILState_STATE gstate = PyGILState_Ensure();
+            Py_XDECREF(main_ns->globals);
+            Py_XDECREF(main_ns->locals);
+            Py_XDECREF(main_ns->module_cache);
+            PyGILState_Release(gstate);
+            enif_free(main_ns);
+        }
+        if (dead_env != NULL) {
+            enif_release_resource(dead_env);
+        }
         return;
     }
 
@@ -724,6 +793,8 @@
     PyGILState_STATE gstate = PyGILState_Ensure();
     pthread_mutex_lock(&loop->namespaces_mutex);
 
+    void *dead_env = take_pid_env(loop, pid);
+
     /* Find and remove namespace for this pid */
     process_namespace_t **pp = &loop->namespaces_head;
     while (*pp != NULL) {
@@ -742,6 +813,10 @@
     }
 
     pthread_mutex_unlock(&loop->namespaces_mutex);
+    if (dead_env != NULL) {
+        /* The env destructor takes the GIL again: re-entrant here */
+        enif_release_resource(dead_env);
+    }
     PyGILState_Release(gstate);
 }
 
@@ -812,6 +887,10 @@
     }
 
     ns->owner_pid = *pid;
+    {
+        PyInterpreterState *creator = PyInterpreterState_Get();
+        ns->interp_id = creator != NULL ? PyInterpreterState_GetID(creator) : 0;
+    }
     ns->globals = PyDict_New();
     ns->locals = PyDict_New();
     ns->module_cache = PyDict_New();
@@ -1294,7 +1373,7 @@
         if (!enif_get_atom(env, argv[1], atom_buf, sizeof(atom_buf), ERL_NIF_LATIN1)) {
             return make_error(env, "invalid_id");
         }
         strncpy(loop->loop_id, atom_buf, sizeof(loop->loop_id) - 1);
         loop->loop_id[sizeof(loop->loop_id) - 1] = '\0';
     } else {
         size_t copy_len = id_bin.size < sizeof(loop->loop_id) - 1 ?
@@ -2480,9 +2559,9 @@
 
         if (entry->callable == NULL) {
             /* Found empty slot */
             strncpy(entry->module_name, module, CALLABLE_NAME_MAX - 1);
             entry->module_name[CALLABLE_NAME_MAX - 1] = '\0';
             strncpy(entry->func_name, func, CALLABLE_NAME_MAX - 1);
             entry->func_name[CALLABLE_NAME_MAX - 1] = '\0';
             Py_INCREF(callable);
             entry->callable = callable;
@@ -2623,8 +2702,8 @@
  * @param env_res Environment resource (will be kept via enif_keep_resource)
  * @return true on success, false on allocation failure
  */
-static bool register_pid_env(erlang_event_loop_t *loop, const ErlNifPid *pid,
-                              void *env_res) {
+static bool register_pid_env(ErlNifEnv *caller_env, erlang_event_loop_t *loop,
+                              const ErlNifPid *pid, void *env_res) {
     pthread_mutex_lock(&loop->namespaces_mutex);
 
     /* Check if mapping already exists */
@@ -2649,6 +2728,14 @@
     mapping->pid = *pid;
     mapping->env = env_res;
     mapping->refcount = 1;
+
+    /* Dropped with its env when the process exits (event_loop_down). A
+     * process already gone gets no mapping. */
+    if (enif_monitor_process(caller_env, loop, pid, &mapping->monitor) != 0) {
+        enif_free(mapping);
+        pthread_mutex_unlock(&loop->namespaces_mutex);
+        return false;
+    }
     mapping->next = loop->pid_env_head;
     loop->pid_env_head = mapping;
 
@@ -2662,6 +2749,9 @@
 /**
  * @brief Look up env for a PID
  *
+ * The env is returned with a reference of its own (the process may exit
+ * and drop the mapping meanwhile): release it with enif_release_resource.
+ *
  * @param loop Event loop containing the mapping registry
  * @param pid PID to look up
  * @return Environment resource or NULL if not found
@@ -2673,6 +2763,7 @@
     while (mapping != NULL) {
         if (enif_compare_pids(&mapping->pid, pid) == 0) {
             void *env_res = mapping->env;
+            enif_keep_resource(env_res);
             pthread_mutex_unlock(&loop->namespaces_mutex);
             return env_res;
         }
@@ -2720,7 +2811,7 @@
     }
 
     /* Register the env for this PID (increments refcount if exists) */
-    if (!register_pid_env(loop, &caller_pid, env_res)) {
+    if (!register_pid_env(env, loop, &caller_pid, env_res)) {
         return make_error(env, "env_registration_failed");
     }
 
@@ -3296,9 +3387,6 @@
             continue;
         }
 
-        /* Look up env by PID (registered via submit_task_with_env) */
-        py_env_resource_t *task_env = (py_env_resource_t *)lookup_pid_env(loop, &caller_pid);
-
         /* Convert module/func to C strings (stack buffers to avoid alloc overhead) */
         char module_name[CALLABLE_NAME_MAX];
         char func_name[CALLABLE_NAME_MAX];
@@ -3317,6 +3405,9 @@
         /* Look up function - check task_env first, then process namespace, then import */
         PyObject *func = NULL;
 
+        /* Env registered via submit_task_with_env, held for this lookup only */
+        py_env_resource_t *task_env = (py_env_resource_t *)lookup_pid_env(loop, &caller_pid);
+
         /* First, check the passed env's globals (from py:exec) */
         if (task_env != NULL && task_env->globals != NULL) {
             if (strcmp(module_name, "__main__") == 0 ||
@@ -3328,6 +3419,11 @@
             }
         }
 
+        if (task_env != NULL) {
+            enif_release_resource(task_env);
+            task_env = NULL;
+        }
+
         /* Fallback to process namespace and cache/import */
         if (func == NULL) {
             func = get_function_for_task(loop, ns, module_name, func_name);

c_src/py_event_loop.h

@@ -121,6 +121,11 @@ typedef struct process_namespace {
     /** @brief Module import cache for this process */
     PyObject *module_cache;
 
+    /** @brief Interpreter the dicts were created in (0 = main). Not always
+     * the loop's: event_loop_exec/eval run on the main interpreter even for
+     * a subinterpreter loop. */
+    int64_t interp_id;
+
     /** @brief Monitor for detecting process death */
     ErlNifMonitor monitor;
 
@@ -145,6 +150,10 @@ typedef struct pid_env_mapping {
     /** @brief Reference count for this mapping (multiple tasks may use it) */
     int refcount;
 
+    /** @brief Monitor on the owning process: the mapping and its env are
+     * dropped when it exits (event_loop_down) */
+    ErlNifMonitor monitor;
+
     /** @brief Next mapping in linked list */
     struct pid_env_mapping *next;
 } pid_env_mapping_t;
@@ -470,6 +479,11 @@ typedef struct erlang_event_loop {
     /** @brief Mutex protecting namespace registry */
     pthread_mutex_t namespaces_mutex;
 
+    /** @brief OWN_GIL context whose thread releases the namespaces of dead
+     * processes (NULL for the main interpreter, and once the context stops
+     * serving). Protected by namespaces_mutex. */
+    struct py_context *gc_ctx;
+
     /* ========== PID-to-Env Mapping Registry ========== */
     /* Protected by namespaces_mutex (shared with namespace registry) */