Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions GLOSSARY.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,15 @@
**转发任务**:将一个文件或文件夹条目交付给一组已接受目标的整体工作;单个目标完成不等于整个任务完成。
_Avoid_:将单条数据流称为整个任务

**共享上行转发**:同一条目的各已接受目标属于同一个转发任务,共享源端的发送进度和用户暂停状态。
_Avoid_:把一个目标的数据流当作可独立暂停的任务

**一对一转发**:一个转发任务只面向一个目标;同一条目发往多个目标时,各目标的任务可分别暂停和继续。
_Avoid_:把共享上行转发的一条下游数据流称为一对一转发任务

**转发队列预算**:同一中转服务的所有转发任务合计可积压的待下发数据额度,不是预先占用的内存,也不是服务的全部内存用量。
_Avoid_:把队列预算称作进程内存上限

**任务交付完成**:转发任务的所有已接受目标均已真实确认收到完整文件的状态。
_Avoid_:发送完毕、单目标完成、仅凭文件大小认定成功

Expand Down
1 change: 1 addition & 0 deletions PyFlow/flow_setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,1110 +33,1111 @@
from . import command_control_extension_tcp, forward_extension_tcp
from .network_api.connect_tcp import TCP_Client_Base, TCP_Server_Base

config_file_name="setup.json"
flow_setup_root=os.path.dirname(os.path.abspath(__file__))
config_file_dir=os.path.join(flow_setup_root, config_file_name)
project_root = os.path.dirname(flow_setup_root)

SERVER_DEFAULTS = {
"host": "127.0.0.1",
"port": 65432,
"max_clients": 10,
"port_add_step": 1,
"port_range_num": 100,
"max_file_transfer_thread_num": 10,
"is_hand_alloc_port": False,
"is_input_command_in_console": True,
"max_custom_workers": 10,
"is_extend_command": False,
"is_enable_encrypto": True,
"is_custom_keys": None,
"max_mem_buff": 2048,
"recv_messages_chunk_len": 4096,
"file_transfer_chunk_len": 65536,
"is_asynic_clients_io": False,
"is_debug": False,
"is_print_log": True,
}

CLIENT_DEFAULTS = {
"host": None,
"client_host": "127.0.0.1",
"port": 65432,
"client_port": None,
"timeout": None,
"port_add_step": 1,
"max_thread_num": 10,
"is_input_command_in_console": True,
"is_wait_server": True,
"max_custom_workers": 10,
"is_extend_command": False,
"is_enable_encrypto": True,
"is_custom_keys": None,
"max_mem_buff": 2048,
"recv_messages_chunk_len": 4096,
"file_transfer_chunk_len": 65536,
"is_debug": False,
"is_print_log": True,
"is_forward_one_to_one": False,
}


def parse_addr_port(addr_port):
host, port_str = addr_port.strip().split(":")
return host, int(port_str)


def load_existing_config():
if os.path.exists(config_file_dir):
with open(config_file_dir, "r", encoding="utf-8") as f:
return json.load(f)
return {"servers": [], "clients": []}


def complete_server_config(cfg):
full = SERVER_DEFAULTS.copy()
full.update(cfg)
return full


def complete_client_config(cfg):
full = CLIENT_DEFAULTS.copy()
full.update(cfg)
return full


def save_config(servers, clients):
server_config=[]
client_config=[]
if servers:
pass
else:
servers = []
if clients:
pass
else:
clients = []
for server in servers:
server_config.append(complete_server_config(server))
for client in clients:
client_config.append(complete_client_config(client))
data = {
"servers": server_config,
"clients": client_config,
}
with open(config_file_dir, "w", encoding="utf-8") as f:
json.dump(data, f, indent=4, ensure_ascii=False)


def launch_instance(config, instance_type):
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False, encoding="utf-8") as f:
json.dump(config, f)
config_file_path = f.name
module = "PyFlow.flow_setup"
python = sys.executable
launch_arg = f"--launch_{instance_type}"
system = platform.system()
try:
if system == "Windows":
cmd = (
f'start cmd /k {python} -m {module} {launch_arg} '
f'--config-file "{config_file_path}"'
)
subprocess.Popen(cmd, shell=True, cwd=project_root)
elif system == "Linux":
terminals = ["gnome-terminal", "xterm", "x-terminal-emulator"]
launched = False
for term in terminals:
if shutil.which(term):
cmd = (
f'{term} -- {python} -m {module} {launch_arg} '
f'--config-file "{config_file_path}"'
)
subprocess.Popen(cmd, shell=True, cwd=project_root)
launched = True
break
if not launched:
subprocess.Popen(
[python, "-m", module, launch_arg, "--config-file", config_file_path],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
stdin=subprocess.DEVNULL,
start_new_session=True,
cwd=project_root,
)
elif system == "Darwin":
cmd = (
f'open -a Terminal.app {python} -m {module} {launch_arg} '
f'--config-file "{config_file_path}"'
)
subprocess.Popen(cmd, shell=True, cwd=project_root)
else:
subprocess.Popen(
[python, "-m", module, launch_arg, "--config-file", config_file_path],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
stdin=subprocess.DEVNULL,
start_new_session=True,
cwd=project_root,
)
except Exception as e:
print(f"Failed to launch instance: {e}")
traceback.print_exc()
try:
os.unlink(config_file_path)
except:
pass


# ---- web tool launchers (transfer_web) --------------------------------------

def launch_web_tool(kind):
"""Launch the transfer_web launcher (``kind`` = "server" or "client").

The web tool is a Flask app that opens a browser UI, so it runs in
its own process (a terminal window when one is available, otherwise
detached) and the launcher returns immediately.
"""
module = (
"PyFlow.transfer_web.setup_server" if kind == "server" else "PyFlow.transfer_web.setup_client"
)
python = sys.executable
system = platform.system()
try:
if system == "Windows":
cmd = f"start cmd /k {python} -m {module}"
subprocess.Popen(cmd, shell=True, cwd=project_root)
elif system == "Linux":
terminals = ["gnome-terminal", "xterm", "x-terminal-emulator"]
launched = False
for term in terminals:
if shutil.which(term):
cmd = f"{term} -- {python} -m {module}"
subprocess.Popen(cmd, shell=True, cwd=project_root)
launched = True
break
if not launched:
subprocess.Popen(
[python, "-m", module],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
stdin=subprocess.DEVNULL,
start_new_session=True,
cwd=project_root,
)
elif system == "Darwin":
cmd = f"open -a Terminal.app {python} -m {module}"
subprocess.Popen(cmd, shell=True, cwd=project_root)
else:
subprocess.Popen(
[python, "-m", module],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
stdin=subprocess.DEVNULL,
start_new_session=True,
cwd=project_root,
)
except Exception as e:
print(f"Failed to launch web {kind}: {e}")
traceback.print_exc()


# ---- instance editor: reduce/change existing instances ----------------------

_MOUSE_WHEEL_UP = 64
_MOUSE_WHEEL_DOWN = 65
_DOUBLE_CLICK_SECONDS = 0.3
_ESC_SEQUENCE_WAIT = 0.08

EDITOR_USAGE = """\
Instance editor usage (instance menu):
up/down arrows or j/k ......... move the selection (first instance selected
by default, works without a UI)
mouse click ................... select the clicked instance
double click / Enter .......... open the selected instance's config editor
Delete / Backspace, dd ........ delete the selected instance
:w ............................ write setup.json (save without exiting)
:wq ........................... write setup.json and exit the editor
:q ............................ exit (refused while there are unsaved changes)
:q! ........................... exit and discard all changes
Fix_Config ................... write setup.json and exit (same as :wq);
at field-value prompts it returns to the list
Setup ......................... launch every instance from setup.json and
exit (works at every prompt)
Quit .......................... exit the setup program immediately
Help .......................... show this help
Config editor (opened with Enter): type a field index or press Enter to edit
the selected field, j/k move the selection, Esc/back returns to the menu.
Config fields (empty input keeps the current value):
server: host, port, max_clients, port_add_step, port_range_num,
max_file_transfer_thread_num, is_hand_alloc_port,
is_input_command_in_console, max_custom_workers, is_extend_command,
is_enable_encrypto, is_custom_keys, max_mem_buff,
recv_messages_chunk_len, file_transfer_chunk_len,
is_asynic_clients_io, is_debug, is_print_log
client: host, client_host, port, client_port, timeout, port_add_step,
max_thread_num, is_input_command_in_console, is_wait_server,
max_custom_workers, is_extend_command, is_enable_encrypto,
is_custom_keys, max_mem_buff, recv_messages_chunk_len,
file_transfer_chunk_len, is_debug, is_print_log
Booleans accept true/false/1/0/y/n; integers are parsed with int(). Type
"none" to reset a nullable field (host/client_port/timeout/is_custom_keys).
Help, Fix_Config, Setup and Quit work at every prompt, including field
values.
"""

COLLECT_USAGE = """\
Setup flow usage:
0/1 ......... instance type (0 = server, 1 = client)
host:port ... bind address, e.g. 127.0.0.1:65432; clients are also asked
for the server host:port they connect to
Y/N ......... confirmations: y = yes, n = no (case-insensitive)
Help ........ show this help
Fix_Config .. jump to the instance editor, where startup parameters can
be changed (works at every prompt)
Setup ....... launch every instance from setup.json and exit (works at
every prompt)
Add_Extension ... add extension protocol files to the project (works at
every prompt)
Delete_Extension ... remove registered extension files from the project
(works at every prompt)
Quit ........ exit the setup program immediately
Help, Fix_Config, Add_Extension, Delete_Extension, Setup and Quit work at
every prompt. In the instance editor (reached with Y on the reduce question)
'Help' prints all config fields and the vim-style commands.
"""


class _EnterEditor(Exception):
"""Entering Fix_Config at any prompt jumps to the instance editor."""


class _LaunchFromSetup(Exception):
"""Entering Setup at any prompt launches every instance from setup.json."""


class _QuitFlow(Exception):
"""Entering Quit at any prompt exits the setup program immediately."""


def _launch_all_from_setup():
"""Start every instance configured in setup.json; True when launched.

Used by the global ``Setup`` command: the current interactive flow is
abandoned and the configured instances are started as-is. Returns False
(with a message) when setup.json is missing, unreadable or empty, so the
caller can keep asking instead of silently doing nothing.
"""
if not os.path.exists(config_file_dir):
print("setup.json not found - nothing to launch")
return False
try:
with open(config_file_dir, "r", encoding="utf-8") as f:
config_data = json.load(f)
except Exception as e: # noqa: BLE001
print(f"setup.json could not be read: {e}")
return False
servers = config_data.get("servers", [])
clients = config_data.get("clients", [])
if not servers and not clients:
print("setup.json contains no instances - nothing to launch")
return False
for cfg in servers:
launch_instance(cfg, "server")
for cfg in clients:
launch_instance(cfg, "client")
return True


def _input(prompt, help_text=COLLECT_USAGE):
"""Ask one question; 'Help' reprints usage, 'Fix_Config' enters the
instance editor, 'Setup' launches every instance from setup.json,
'Add_Extension' adds extension files, 'Delete_Extension' removes
registered extension files, 'Quit' exits the program.

The returned line has surrounding whitespace stripped; the caller
decides case handling for its own answers.
"""
while True:
answer = input(prompt).strip()
low = answer.lower()
if low == "help":
print(help_text, flush=True)
continue
if low == "fix_config":
raise _EnterEditor()
if low == "setup":
if _launch_all_from_setup():
raise _LaunchFromSetup()
continue # nothing usable in setup.json: ask again
if low == "quit":
raise _QuitFlow()
if low == "add_extension":
paths_input = input(
"Enter extension path(s) to add (separated by space): "
).strip()
if paths_input:
try:
add_extension.add_extension(paths_input.split())
print("Extension(s) added successfully.")
except Exception as e:
print(f"Failed to add extension(s): {e}")
continue
if low == "delete_extension":
if not os.path.exists(add_extension.added_extensions_log_file):
print("No extension registration file found - nothing to delete.")
continue
paths_input = input(
"Enter extension path(s) to delete (separated by space): "
).strip()
if paths_input:
try:
add_extension.remove_extension(paths_input.split())
print("Extension(s) deleted successfully.")
except Exception as e:
print(f"Failed to delete extension(s): {e}")
continue
return answer


def _stdin_is_tty():
try:
return sys.stdin.isatty()
except Exception:
return False


def _raw_mode_active():
# Interactive terminals get real keys; pipes, CI and tests (which set
# PYTEST_CURRENT_TEST) fall back to line commands (plain input()).
if not _stdin_is_tty():
return False
if "PYTEST_CURRENT_TEST" in os.environ:
return False
return _termios is not None or _msvcrt is not None


def _enter_raw():
"""Switch stdin to raw byte mode with mouse tracking; return saved attrs."""
if _termios is None or _tty is None or not _stdin_is_tty():
return None
fd = sys.stdin.fileno()
saved_attr = _termios.tcgetattr(fd)
_tty.setraw(fd)
sys.stdout.write("\x1b[?1000h\x1b[?1006h\x1b[?25l")
sys.stdout.flush()
return saved_attr


_pending_bytes = bytearray()


def _leave_raw(saved_attr):
if _termios is None or saved_attr is None:
return
sys.stdout.write("\x1b[?1000l\x1b[?1006l\x1b[?25h")
sys.stdout.flush()
_termios.tcsetattr(sys.stdin.fileno(), _termios.TCSADRAIN, saved_attr)


def _read_posix_key():
fd = sys.stdin.fileno()
first = os.read(fd, 1)
if first != b"\x1b":
return first
seq = first
while True:
assert _select is not None
ready, _, _ = _select.select([fd], [], [], _ESC_SEQUENCE_WAIT)
if not ready:
break
chunk = os.read(fd, 4096)
if not chunk:
break
seq += chunk
return seq


def _decode_posix_seq(seq): # noqa: PLR0911
if not seq:
return ("eof",)
if seq == b"\x1b[A":
return ("up",)
if seq == b"\x1b[B":
return ("down",)
if seq in (b"\x1b[3~", b"\x1b[3;2~", b"\x1b[3;5~"):
return ("delete",)
if seq in (b"\r", b"\n"):
return ("enter",)
if seq in (b"\x7f", b"\x08"):
return ("backspace",)
if seq == b"\x1b":
return ("esc",)
if seq == b"\x03":
raise KeyboardInterrupt
if seq.startswith(b"\x1b[<") and seq[-1:] in (b"M", b"m"):
try:
button, x, y = (int(p) for p in seq[3:-1].decode("ascii", "replace").split(";"))
return ("mouse", button, x, y)
except ValueError:
pass
if len(seq) == 1:
return ("char", seq.decode("utf-8", "replace"))
if seq.startswith(b"\x1b"):
# Unknown escape sequence: keep the trailing bytes for the next reads
# and surface the leading ESC, so pasted text after ESC is not lost.
_pending_bytes[0:0] = seq[1:]
return ("esc",)
return None


def _read_win_key():
if _msvcrt is None:
return None
ch = _msvcrt.getwch() # ty: ignore[unresolved-attribute]
if ch in ("\x00", "\xe0"):
second = _msvcrt.getwch() # ty: ignore[unresolved-attribute]
return ({"H": "up", "P": "down", "S": "delete"}.get(second, "unknown"),)
if ch == "\r":
return ("enter",)
if ch == "\x08":
return ("backspace",)
if ch == "\x1b":
return ("esc",)
if ch == "\x03":
raise KeyboardInterrupt
return ("char", ch)


def _read_key():
if _pending_bytes:
return ("char", bytes([_pending_bytes.pop(0)]).decode("utf-8", "replace"))
if _termios is not None and _stdin_is_tty():
return _decode_posix_seq(_read_posix_key())
if _msvcrt is not None and _stdin_is_tty():
return _read_win_key()
return None


def _menu_action_from_line(command): # noqa: PLR0911, PLR0912
low = command.lower()
if command == "":
return ("edit", None)
if low in ("j", "down", "+"):
return ("select", 1)
if low in ("k", "up", "-"):
return ("select", -1)
if low in ("dd", "del", "delete", "backspace"):
return ("delete", None)
if low == ":wq":
return ("write_quit", None)
if low == ":w":
return ("write", None)
if low == ":q!":
return ("abort", None)
if low == ":q":
return ("quit", None)
if low in (":help", ":h", "help", "h"):
return ("help", None)
if low == "fix_config":
return ("write_quit", None)
if low == "setup":
return ("setup", None)
if low == "quit":
return ("quit_flow", None)
if command.isdigit():
return ("select_abs", int(command))
return ("noop", command)


def _detail_command(command, num_fields): # noqa: PLR0911
low = command.lower()
if command == "":
return ("edit", None)
if command.isdigit():
index = int(command)
if 0 <= index < num_fields:
return ("edit", index)
return ("noop", command)
if low in ("j", "down"):
return ("select", 1)
if low in ("k", "up"):
return ("select", -1)
if low in ("b", "back", "esc"):
return ("back", None)
if low == "fix_config":
return ("back", None)
if low == "setup":
return ("setup", None)
if low == "quit":
return ("quit_flow", None)
if low in ("help", "h"):
return ("help", None)
return ("noop", command)


def _instance_label(kind, cfg):
if kind == "server":
return f"{cfg.get('host', '?')}:{cfg.get('port', '?')}"
return (
f"{cfg.get('client_host', '?')}:{cfg.get('client_port', '?')}"
f" -> {cfg.get('host', '?')}:{cfg.get('port', '?')}"
)


def _instance_entries(servers, clients):
entries = [("server", s) for s in servers]
entries.extend(("client", c) for c in clients)
return entries


def _terminal_width():
"""Current terminal width in columns (80 when it cannot be determined)."""
try:
return shutil.get_terminal_size().columns
except Exception:
return 80


def _row_to_entry(y, entry_rows):
"""Map a 1-based screen row back to the entry index it belongs to."""
for index, start in enumerate(entry_rows):
end = entry_rows[index + 1] if index + 1 < len(entry_rows) else start + 1
if start <= y < end:
return index
return None


def _print_menu(servers, clients, selection, raw, typed, status_msg): # noqa: PLR0913, PLR0917
"""Render the instance menu with dynamically aligned columns.

Column widths (index, kind, label) are derived from the current
content; rows wider than the terminal wrap with the continuation
indented to the label column. Returns the 1-based screen row where
every entry starts, used to map mouse clicks back to entries.
"""
entries = _instance_entries(servers, clients)
width = _terminal_width()
index_w = len(str(max(len(entries) - 1, 0))) if entries else 1
kind_w = max((len(k) for k, _ in entries), default=6)
lines = []
if raw:
lines.append("\x1b[2J\x1b[H")
lines.append("=== Instance Manager === (:wq save&exit | :q! / Fix_Config | Help)")
entry_rows = []
screen_row = 2 # row 1 is the title
for index, (kind, cfg) in enumerate(entries):
marker = "> " if index == selection else " "
label = _instance_label(kind, cfg)
prefix = f"{marker}[{index:>{index_w}}] [{kind:<{kind_w}}] "
entry_rows.append(screen_row)
pieces = textwrap.wrap(label, max(width - len(prefix), 10)) or [""]
lines.append(prefix + pieces[0])
for extra in pieces[1:]:
lines.append(" " * len(prefix) + extra)
screen_row += len(pieces)
if not entries:
lines.append(" (no instances)")
if typed:
lines.append(f"command: {typed}")
if status_msg:
lines.append(status_msg)
for line in lines:
print(line, flush=True)
return entry_rows


def _delete_at(servers, clients, selection):
entries = _instance_entries(servers, clients)
if not entries or not (0 <= selection < len(entries)):
return None
kind, cfg = entries[selection]
if kind == "server":
del servers[selection]
else:
del clients[selection - len(servers)]
return kind, cfg


def _parse_field_value(default, raw_value): # noqa: PLR0911
if isinstance(default, bool):
low = raw_value.lower()
if low in ("true", "1", "y", "yes"):
return True, None
if low in ("false", "0", "n", "no"):
return False, None
return None, f"invalid boolean {raw_value!r} (use true/false/1/0/y/n)"
if isinstance(default, int):
try:
return int(raw_value), None
except ValueError:
return None, f"invalid integer {raw_value!r}"
if default is None and raw_value.lower() in ("none", "null"):
return None, None
return raw_value, None


def _edit_field_value(cfg, field):
key, default = field
current = cfg.get(key, default)
while True:
raw_value = _input(
f"{key} (current={current!r}, empty keeps): ", help_text=EDITOR_USAGE
)
if raw_value == "":
return False
value, error = _parse_field_value(default, raw_value)
if error:
print(error, flush=True)
continue
cfg[key] = value
print(f"{key} = {value!r}", flush=True)
return True


def _edit_instance_detail(kind, cfg, raw, saved_attr): # noqa: PLR0912, PLR0915
"""Edit every startup field of one instance; returns True when changed."""
defaults = SERVER_DEFAULTS if kind == "server" else CLIENT_DEFAULTS
fields = list(defaults.items())
selection = 0
changed = False
typed = ""
status_msg = ""
while True:
if selection >= len(fields):
selection = len(fields) - 1
width = _terminal_width()
idx_w = len(str(len(fields) - 1))
key_w = max(len(key) for key, _ in fields)
lines = []
if raw:
lines.append("\x1b[2J\x1b[H")
lines.append(f"=== Editing {kind}: {_instance_label(kind, cfg)} ===")
lines.append("(field index/Enter = edit, j/k = move, Esc/back = return, Fix_Config)")
for index, (key, default) in enumerate(fields):
marker = "> " if index == selection else " "
value = str(cfg.get(key, default))
prefix = f"{marker}[{index:>{idx_w}}] {key:<{key_w}} = "
pieces = textwrap.wrap(value, max(width - len(prefix), 10)) or [""]
lines.append(prefix + pieces[0])
for extra in pieces[1:]:
lines.append(" " * len(prefix) + extra)
if typed:
lines.append(f"command: {typed}")
if status_msg:
lines.append(status_msg)
for line in lines:
print(line, flush=True)
status_msg = ""
if raw:
key = None
while key is None:
key = _read_key()
kind_key = key[0]
if kind_key == "eof":
return changed # stdin closed: leave the field editor
if kind_key == "char":
typed += key[1]
continue
if kind_key == "backspace":
typed = typed[:-1]
continue
if kind_key == "esc":
if typed:
typed = ""
continue
return changed
if kind_key == "enter":
command, typed = typed, ""
else:
typed = ""
if kind_key == "up":
selection = (selection - 1) % len(fields)
elif kind_key == "down":
selection = (selection + 1) % len(fields)
continue
else:
command = input("Field> ").strip()
name, arg = _detail_command(command, len(fields))
if name == "edit":
index = selection if arg is None else arg
_leave_raw(saved_attr)
try:
if _edit_field_value(cfg, fields[index]):
changed = True
except _EnterEditor:
# Fix_Config at the value prompt: keep the edits made so far
# and return to the instance menu.
return changed
finally:
saved_attr = _enter_raw()
selection = index
elif name == "select":
selection = (selection + arg) % len(fields)
elif name == "back":
return changed
elif name == "setup":
if _launch_all_from_setup():
raise _LaunchFromSetup()
status_msg = "nothing to launch (setup.json missing or empty)"
elif name == "quit_flow":
raise _QuitFlow()
elif name == "help":
print(EDITOR_USAGE, flush=True)
if raw:
_read_key() # let the user read the help before the redraw
else:
status_msg = f"unknown command {command!r} (field index, j/k, back)"


def _menu_driver(servers, clients, raw, saved_attr): # noqa: PLR0912, PLR0915
"""Selection menu over existing instances; returns 'saved' or 'discarded'."""
selection = 0
modified = False
status_msg = ""
typed = ""
last_click = (None, 0.0)
while True:
entries = _instance_entries(servers, clients)
if entries and selection >= len(entries):
selection = len(entries) - 1
entry_rows = _print_menu(servers, clients, selection, raw, typed, status_msg)
status_msg = ""
if raw:
key = None
while key is None:
key = _read_key()
kind = key[0]
if kind == "eof":
return "discarded" # stdin closed: leave the editor
if kind == "char":
typed += key[1]
action = None
elif kind == "enter":
action = _menu_action_from_line(typed) if typed else ("edit", None)
typed = ""
else:
if kind == "esc" and typed:
status_msg = "command cancelled"
typed = ""
if kind in ("up", "down"):
action = ("select", 1 if kind == "down" else -1)
elif kind in ("delete", "backspace"):
action = ("delete", None)
elif kind == "mouse":
button, _, y = key[1], key[2], key[3]
if button == _MOUSE_WHEEL_UP:
action = ("select", -1)
elif button == _MOUSE_WHEEL_DOWN:
action = ("select", 1)
elif button == 0:
row_index = _row_to_entry(y, entry_rows)
if row_index is None:
action = None
else:
now = time.monotonic()
if (
row_index == last_click[0]
and now - last_click[1] < _DOUBLE_CLICK_SECONDS
):
selection = row_index
action = ("edit", None) # double click opens the editor
else:
selection = row_index
action = ("select_abs", row_index)
last_click = (row_index, now)
else:
action = None
else:
action = None
else:
action = _menu_action_from_line(input("Menu> ").strip())
if action is None:
continue
name, arg = action
if name == "select":
if entries and arg is not None:
selection = (selection + arg) % len(entries)
elif name == "select_abs":
if arg is not None and 0 <= arg < len(entries):
selection = arg
else:
status_msg = f"invalid index: {arg}"
elif name == "delete":
deleted = _delete_at(servers, clients, selection)
if deleted:
modified = True
status_msg = f"deleted [{deleted[0]}] {_instance_label(*deleted)}"
elif name == "edit":
if entries:
kind, cfg = entries[selection]
if _edit_instance_detail(kind, cfg, raw, saved_attr):
modified = True
elif name == "help":
print(EDITOR_USAGE, flush=True)
if raw:
_read_key() # let the user read the help before the redraw
elif name == "setup":
if _launch_all_from_setup():
raise _LaunchFromSetup()
status_msg = "nothing to launch (setup.json missing or empty)"
elif name == "quit_flow":
raise _QuitFlow()
elif name == "write":
save_config(servers, clients)
modified = False
status_msg = "saved"
elif name == "write_quit":
save_config(servers, clients)
return "saved"
elif name == "quit":
if modified:
status_msg = "No write since last change (use :q! to discard)"
else:
return "discarded"
elif name == "abort":
return "discarded"
elif name == "noop":
status_msg = f"unknown command: {arg}"


def edit_existing_instances(servers, clients):
"""Vim-style editor to delete/change existing instances.

Returns (status, servers, clients):
status == "saved" -> setup.json was written (:w / :wq); keep the
returned edited lists
status == "discarded" -> the editor was exited without saving
(:q! / :q) and the original lists are
returned unchanged
"""
edited_servers = [dict(s) for s in servers]
edited_clients = [dict(c) for c in clients]
raw = _raw_mode_active()
saved_attr = _enter_raw() if raw else None
try:
status = _menu_driver(edited_servers, edited_clients, raw, saved_attr)
finally:
_leave_raw(saved_attr)
if status == "saved":
return "saved", edited_servers, edited_clients
return "discarded", servers, clients


def _add_instances_loop(servers, clients):
while True:
print("\n--- Add New Instance ---")
while True:
type_choice = _input("Select type (0=Server, 1=Client): ")
if type_choice in ("0", "1"):
break
print("Invalid input, please enter 0 or 1")
is_server = type_choice == "0"
while True:
setup_addr = _input("Enter bind address and port (format host:port): ")
try:
host, port = parse_addr_port(setup_addr)
break
except:
print("Invalid format, please retry")
if is_server:
server_len=len(servers)
server_index=0
config = {"host": host, "port": port}
for server in servers:
if config==server:
break
server_index+=1
if server_len==server_index:
servers.append(config)
print(f"Server config set to: {host}:{port}")
else:
while True:
conn_addr = _input(
"Enter server address and port to connect (format host:port): "
)
try:
srv_host, srv_port = parse_addr_port(conn_addr)
break
except:
print("Invalid format, please retry")
config = {"client_host": host, "client_port": port, "host": srv_host, "port": srv_port}
client_len=len(clients)
client_index=0
for client in clients:
if config==client:
break
client_index+=1
if client_len==client_index:
clients.append(config)
print(f"Client config set to: local {host}:{port} -> server {srv_host}:{srv_port}")
cont = _input("Continue adding more instances? (Y/N): ")
if cont.lower() != "y":
break
return servers, clients


def _collect_after_editor(servers, clients):
"""Open the instance editor over existing configs, then keep collecting.

Fix_Config entered at any prompt funnels here. After the editor is left
with :wq / Fix_Config the user is asked whether new instances are wanted.
"""
while True:
try:
status, servers, clients = edit_existing_instances(servers, clients)
if status == "saved":
choice = _input("Add new instances? (Y/N): ")
if choice.lower() != "y":
return servers, clients
return _add_instances_loop(servers, clients)
except _EnterEditor:
continue # Fix_Config at the follow-up prompt: edit again


def interactive_collect():
config_data = load_existing_config()
servers = config_data["servers"]
clients = config_data["clients"]
try:
if servers or clients:
# Instances already exist: first ask whether to delete/change
# them. An empty config skips this question and goes straight to
# the add flow. Help/Fix_Config work at every prompt via _input().
choice = _input("Delete or change existing instances or configs? (Y/N): ")
if choice.lower() == "y":
return _collect_after_editor(servers, clients)
return _add_instances_loop(servers, clients)
except _EnterEditor:
# Fix_Config entered at any prompt: jump to the instance editor,
# where startup parameters can be changed, then keep collecting.
return _collect_after_editor(servers, clients)


def generate_configs_from_args(args):
if args.setup_num > 1:
print("Warning: --setup_num is ignored because only one instance per type is allowed.")
host, port = parse_addr_port(args.setup_addr_port)
if args.type == 0:
config = {"host": host, "port": port}
return [config], []
else:
srv_host, srv_port = parse_addr_port(args.connect_addr_port)
config = {"client_host": host, "client_port": port, "host": srv_host, "port": srv_port}
return [], [config]


def _load_extensions(instance, instance_type):
"""Register the extension protocols on an instance (is_extend_command)."""
if instance_type == "server":
command_control_extension_tcp.setup_server_commands(instance)
forward_extension_tcp.setup_server_commands(instance)
else:
command_control_extension_tcp.setup_client_commands(instance)
forward_extension_tcp.setup_client_commands(instance)


def _keep_alive(instance):
"""Keep the process alive while the instance runs in a background thread.

Used when is_input_command_in_console=False: the instance runs in a
daemon thread and the main thread only waits for it to stop.
"""
try:
while instance.running:
time.sleep(1)
except KeyboardInterrupt:
stop = getattr(instance, "stop", None) or getattr(instance, "close", None)
if stop is not None:
try:
stop()
except Exception:
traceback.print_exc()


def run_launched_instance(instance_type, config_file_path):
try:
with open(config_file_path, "r", encoding="utf-8") as f:
config = json.load(f)
try:
os.unlink(config_file_path)
except:
pass
if instance_type == "server":
server = TCP_Server_Base(**config)
if config.get("is_extend_command", False):
_load_extensions(server, "server")
try:
add_extension.load_registered_extensions(server, "server")
except ImportError as e:
print(f"Failed to load registered extensions: {e}")
if config.get("is_input_command_in_console", True):
server.start_TCP_Server()
else:
threading.Thread(target=server.start_TCP_Server, daemon=True).start()
_keep_alive(server)
else:
client = TCP_Client_Base(**config)
if config.get("is_extend_command", False):
_load_extensions(client, "client")
try:
add_extension.load_registered_extensions(client, "client")
except ImportError as e:
print(f"Failed to load registered extensions: {e}")
if config.get("is_input_command_in_console", True):
client.start_TCP_client()
else:
threading.Thread(target=client.start_TCP_client, daemon=True).start()
_keep_alive(client)
except Exception as e:
print(f"Failed to start instance: {e}")
traceback.print_exc()
input("Press any key to exit...")
sys.exit(1)


def main(): # noqa: PLR0911, PLR0912, PLR0915
if "--launch_server" in sys.argv:
idx = sys.argv.index("--launch_server")
try:
cfg_idx = sys.argv.index("--config-file", idx)
if cfg_idx + 1 < len(sys.argv):
config_file = sys.argv[cfg_idx + 1]
run_launched_instance("server", config_file)
else:
print("Error: missing --config-file argument")
sys.exit(1)
except ValueError:
print("Error: missing --config-file argument")
sys.exit(1)
return
if "--launch_client" in sys.argv:
idx = sys.argv.index("--launch_client")
try:
cfg_idx = sys.argv.index("--config-file", idx)
if cfg_idx + 1 < len(sys.argv):
config_file = sys.argv[cfg_idx + 1]
run_launched_instance("client", config_file)
else:
print("Error: missing --config-file argument")
sys.exit(1)
except ValueError:
print("Error: missing --config-file argument")
sys.exit(1)
return
parser = argparse.ArgumentParser(description="Flow Setup Launcher")
parser.add_argument("--type", type=int, choices=[0, 1], help="0=Server, 1=Client")
parser.add_argument(
"--web_server", action="store_true", help="Launch the web TCP server tool (browser UI)"
)
parser.add_argument(
"--web_client", action="store_true", help="Launch the web TCP client tool (browser UI)"
)
parser.add_argument("--setup_addr_port", type=str, help="Bind address and port (host:port)")
parser.add_argument(
"--connect_addr_port", type=str, help="Server address and port to connect (client required)"
)
parser.add_argument("--delete", type=str, nargs="+", help="Remove registered extension protocol file(s) by path")

Check failure on line 1140 in PyFlow/flow_setup.py

View workflow job for this annotation

GitHub Actions / Lint

ruff (unformatted)

PyFlow/flow_setup.py:36:17: unformatted: File would be reformatted
parser.add_argument("--add", type=str, nargs="+", help="Add extension protocol file(s) by path")
parser.add_argument(
"--setup_num", type=int, default=1, help="Number of instances to launch (only 1 is allowed)"
Expand Down
402 changes: 249 additions & 153 deletions PyFlow/network_api/connect_tcp.py

Large diffs are not rendered by default.

107 changes: 107 additions & 0 deletions docs/plans/forward-one-to-one-plan.md

Large diffs are not rendered by default.

223 changes: 223 additions & 0 deletions docs/plans/forward-one-to-one-report.md

Large diffs are not rendered by default.

44 changes: 44 additions & 0 deletions docs/plans/forward-one-to-one-requirements.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# 原生 TCP 一对一文件转发 — 需求卡

状态:**需求方已确认本卡,仅授权编写[更改计划](forward-one-to-one-plan.md),未批准实施**。沿用[文件转发词汇表](../../GLOSSARY.md)。

## 已明确的目标

- 在发送端 `TCP_Client_Base` 新增可选参数 `is_forward_one_to_one=False`,现有调用保持共享上行拓扑、控制和交付语义。同一条目向多个目标仍只上传一份,用户暂停/继续仍作用于整个转发任务。
- 开启时,由发送端将同一条目的多目标请求拆为逐目标的单目标转发。已获准使用资源的任务可并行传输;不得等前一个目标传完整个文件才启动下一个。B 的用户暂停或继续不改变 C 的用户暂停状态。
- Q3 已选:并发单位是「文件条目 × 去重后的目标」,不是每个目标一条串行文件队列。x、y 同时发往 B、C 会产生四个单目标任务;即便 x→B 正暂停,只要有名额,y→B 也可开始。
- Q4 已选:开启模式的发送端复用现有 `max_thread_num`/文件发送名额,与同一实例内已受该限制的普通批量发送共同限流;多次转发命令不各自乘一份额度。满额等待,不提前开数据连接、占数据端口或给每个等待项都起线程;活跃但暂停的任务不释放仍在占用的资源和名额。
- Q5 已选:同一新模式命令中,若可从请求判定两个条目会写同一目标的同一目的路径,则保留第一次、跳过后者并给可定位提示;不更名、不并发写同一路径。不同目标或不同相对目录不构成此冲突。
- Q6 已选:没有命令级父任务、总进度或整批 ACK。每个获准单目标任务沿用自己的 upstream FID、child FID、进度、用户控制和最终确认;B 的成功清理不能删除 C 的失败/恢复依据。已有公开入口的返回值、查询和五字段进度格式不改。
- Q7 澄清已选:排队项仅属于提交命令时的实际控制会话;未调度不分 FID、不开数据端口、不建进度行、不持久化。旧会话结束就取消尚未发起的项并提示,不能借用重连后的 socket;不自动重发等待项。已在途的中断传输走既有进度恢复。
- Q8 已选:获准进入既有单目标握手即按在途处理,可在首条 `/forward_item` 发出前绑定 FID 和已有五字段上传行(跟踪关闭不新建行)。不要因没有正文或握手结果不确定而删掉已成立任务的恢复依据。只拆首次请求,不因重连或构造参数变化重新拆分历史 route。
- Server 不接收新的模式参数或模式标记。交给它的仍是既有单目标或多目标转发请求;不另造一套文件传输协议。
- 沿用原生流式转发的资源保护、控制会话、握手与真实收件确认,不用取消背压或提前报成功换取独立性。

## 范围与数据边界

- 范围是 `PyFlow/network_api/connect_tcp.py` 的原生 `/forward_file`、`/forward_folder`(文件夹内实际文件)和共用入口,包括现有 Web 单目标调用;不扩展 Web 选目标界面。构造参数置于现有可选参数末尾;普通配置沿用 Client 构造入口,不新增线上字段。
- 原生转发不在 Server 落文件;不绕过统一控制消息和文件数据加密路径,不新增文件正文、密钥或完整负载日志。普通直传、消息转发、独立落盘扩展均不是本次新数据通路。
- 不改 CI、依赖、覆盖率策略;不删或弱化既有测试。风险较高的部分是多任务并存时的资源计账、传输身份和清理。
- 不承诺不同进程/发送者在接收盘上的写路径互斥、远端文件系统别名识别、源文件传输中变更的快照一致性、进程内存硬限或带宽公平。不会把 Server 的普通文件发送名额说成现存的原生中转任务数上限。

## 资源口径:配置值、计账范围、实际用量分开

现有[传输说明](../source/File_Transfer/File_Transfer.rst)和 `_forward_pump` 的实现是每次中转各有队列计数,允许一次入队超出水位;这既不是预分配,也不是进程 RAM 上限。

Q1 澄清和 Q2 已确认:同一 Server 实例中所有并发原生转发(不同文件、目标、命令或源端,含旧模式)共用既有 `max_mem_buff` **一份严格的待下发字节额度**。例如 64 MiB 已由 x→B 用满,即使 y→C 的队列为空,也须等待有足够额度才入队。不乘大配置值、不预分配、不跨任务共享正文;只协调计账、等待和释放。用户暂停/继续只改变所属任务状态,不能越过资源等待。Server 因此要改通用背压,不增模式参数。

## 执行路径与验收证据(均未执行)

| 路径 | 必须实际观察的结果 |
|---|---|
| P1 默认、显式 False、原本就只有一个目标 | 同一文件的默认多目标仍只有一次上行、一个原任务;全部已接受目标真实 ACK 才收尾。单目标复用原握手,无模式字段。 |
| P2 输入和落点:原生文件/目录、多条目、目标列表、destination、Web 既有入口 | 引号/空格和嵌套相对目录保留;重复目标只交付一次,自己不分 FID,不同端口的合法目标照常;同命令目的路径冲突跳过后者。不可达/Server 目标及无效文件不妨碍其他项。 |
| P3 一对一命令、多个文件/目标、并发名额 | x、y 发 B、C 是四个独立任务;受限时不超同一 Client 现有名额。阻住 x→B 的正文或 ACK 后 C 能完成,有空位时 y→B 不等 x→B。未调度不占数据端口/进度行。 |
| P4 用户暂停与继续 | 源端、Server、目标端仍沿原 FID 和角色路由,由源端仲裁;B 的暂停/继续不改变独立 C 的状态,同一 B 数据连接可继续。在途字节可排空。False 模式按原任务联动。 |
| P5 Server 内存预算 | 同一 Server 上来自不同请求/源的单目标、多目标及恢复并存时,所有排队/writer 未交付的计费字节之和始终不超过额度。块大于预算、扇出大于预算、元信息、慢 writer、失败/取消/停止都要有界、有进展、最终归零;自动背压不发用户控制。 |
| P6 错误、握手和控制会话更替 | 乱序端口、单项协商拒绝或数据失败不串 FID、不取消健康任务;旧会话的等待项撤销、已准入 owner 收尾,不能借新 socket。用户手动关闭同样释放等待和名额。 |
| P7 真实交付与恢复 | B 完成只清 B;C 中断/重连按原 upstream/child FID、实际文件 offset 和原路径重放。空文件、offset=0/EOF、最终 ACK 丢失沿既有规则;跟踪关闭不伪造持久进度;旧多目标 route 不迁移成多条新 route。 |
| P8 环境和回归 | 明文/加密、线程/异步 Server、系统端口/手动端口生命周期;普通直传、消息、落盘扩展不进入新分支。检查临时连接/端口、线程、路由、预算归属和真实内容,不以文件存在或一次 pytest 通过代替证据。 |

高风险:跨中转的严格预算、并发握手/快照和会话取消。文件、阶段、逐路径机械验收命令见[更改计划](forward-one-to-one-plan.md);本阶段不实施、不提交。
47 changes: 37 additions & 10 deletions docs/source/File_Transfer/File_Transfer.rst
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,29 @@ its structure under
omitted the targets
write to their default ``file_transfer_dir``.

By default a file entry is uploaded once for all accepted targets; it has
one user pause state and waits for all their final acknowledgements. To send
independent copies, set ``is_forward_one_to_one=True`` on the **sending
client**. For example, forwarding ``x`` and ``y`` to B and C schedules
``x → B``, ``x → C``, ``y → B`` and ``y → C``. Each pair has its own data
connection, FID and final ACK. Pausing ``x → B`` does not pause the others.

Repeated targets are used once. If two entries in the same command name the
same destination file, the first wins; different relative directories do not
collide. A pair waits for one of the client's existing ``max_thread_num``
file slots **before** opening a connection or recording progress. A paused
pair keeps its slot. Disconnect cancels pairs still waiting; it does not move
them to the new session. An interrupted upload with retained progress replays
the original FID and route, not a new batch. There is no batch ACK.

When tracking is enabled, forwarding ``x`` to B and C in one-to-one mode
creates two upload rows with the same source path. Use the **FID and logical
target**, not the pathname, to distinguish them. If B confirms delivery and
C disconnects, only B's row is removed. On reconnect C keeps its child FID
and destination path; the source resumes its original upload from C's disk
offset. Even if C already has every byte, reaching EOF is not a final ACK.
With the default shared upload, B's ACK alone does not retire the route.

The data path reuses the protocol's own transfer
machinery:

Expand All @@ -545,16 +568,20 @@ machinery:

### Memory Bounding and Flow Control

Because uploader, server and targets may have different
bandwidths, data can
pile up in the server's memory. ``TCP_Server_Base``
takes a ``max_mem_buff`` parameter (in MiB, default 2048) for the pump's
accounted queued bytes. Above the high-water mark, its reader waits until
writers drain to the low-water mark (half the limit); TCP flow control carries
the pressure upstream. Automatic pressure does not send user pause/start lines,
so draining cannot cancel a user pause and user start cannot bypass the gate.
One enqueue can overshoot by its fan-out bytes. This is not a total process RAM
ceiling; socket, crypto and in-flight writer buffers are separate.
``TCP_Server_Base.max_mem_buff`` is one queue budget for **all native forwards
on that server**, including resumed transfers. The constructor takes MiB
(default 2048); the effective limit is at least one byte. Each downstream copy
of the file header and body is charged before it enters its queue and stays
charged until the writer returns from sending it or discards it. A large
chunk or fan-out is sent in parts rather than exceeding the limit.

When the budget is full, relay readers wait. Returning credit lets another
route proceed without waiting for the whole file to finish. TCP carries this
pressure upstream without sending user pause/start commands: draining a queue
cannot cancel a user pause, and user start cannot bypass the budget. This is
not preallocated memory or a process RAM limit. Temporary reads, encryption
buffers and OS socket buffers are outside the budget. The guarantee requires
an updated server; the client's setting does not set the server's limit.

.. _concurrency-and-threading:

Expand Down
6 changes: 5 additions & 1 deletion docs/source/Instance_Setup/Instance_Setup.rst
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,11 @@ the client entry carries ``host``, ``client_host``, ``port``,
``is_wait_server``, ``max_custom_workers``,
``is_extend_command``, ``is_enable_encrypto``,
``is_custom_keys``, ``max_mem_buff``, ``recv_messages_chunk_len``,
``file_transfer_chunk_len``, ``is_debug`` and ``is_print_log``.
``file_transfer_chunk_len``, ``is_debug``, ``is_print_log`` and
``is_forward_one_to_one``. Set ``"is_forward_one_to_one": true`` on a
sending client to forward each file entry separately to each target. Omit
it or use ``false`` to keep the default shared upload. No server or target
setting is required.

Custom Parameters
-----------------
Expand Down
14 changes: 12 additions & 2 deletions docs/source/Network_APIs/TCP_Client_APIs.rst
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@ the TCP server and exchange data with the server.
is_debug: Any=False,
is_print_log: Any=True,
is_enable_file_transfer_progress_bar: Any=True,
is_enable_real_time_network_speed: Any=True) -> None:
is_enable_real_time_network_speed: Any=True,
is_auto_reconnect: Any=False,
is_forward_one_to_one: Any=False) -> None:
...

The TCP Client Setup API is defined in the ``TCP_Client_Base`` class.
Expand All @@ -44,7 +46,8 @@ The parameters of the ``__init__`` method are as follows:
- ``client_port``: The local port number to bind the client socket to (``None`` means let the OS choose).
- ``timeout``: The socket timeout in seconds for connection and receive operations. ``None`` (the default) is not "no timeout": the socket then uses a fixed 5-second timeout, and receive timeouts are swallowed by the receive loop (it simply continues).
- ``port_add_step``: The step size for incrementing the port number (used only in manual allocation mode).
- ``max_thread_num``: The maximum number of threads for concurrent file transfer operations.
- ``max_thread_num``: File send slots shared by ordinary batch sends and
opt-in one-to-one forwards. This is not a process-wide thread limit.
- ``is_input_command_in_console``: A flag indicating whether to input commands in the console (interactive mode).
- ``is_wait_server``: A flag indicating whether to keep retrying connection until the server is available.
- ``max_custom_workers``: The maximum number of custom worker threads.
Expand Down Expand Up @@ -99,6 +102,11 @@ The parameters of the ``__init__`` method are as follows:
``recv_network_speed`` (bytes per second, a rolling average over
``NETWORK_SPEED_WINDOW`` seconds). With ``False`` nothing is measured and
every speed value is ``None``.
- ``is_forward_one_to_one``: On the sender only. ``True`` creates one native
forward per file entry and distinct target, each with its own upload FID,
pause state and final ACK. First sends and forwarded replays share file
slots. A paused transfer keeps its slot. Unstarted pairs belong only to the
original control connection and are cancelled when it closes.

Every parameter has default values:

Expand Down Expand Up @@ -142,6 +150,8 @@ Every parameter has default values:
resume the interrupted transfers, see :ref:`transfer-pause-and-reconnect`.
The dead session's socket is closed abortively first, so a configured
``client_port`` can be bound again — Windows refuses it otherwise)
- ``is_forward_one_to_one``: Default is ``False`` (one upload per file entry
shared by all accepted targets). Existing routes keep their original shape.

The TCP Client Setup API will initialize all the necessary
parameters and resources for the TCP client.
Expand Down
9 changes: 5 additions & 4 deletions docs/source/Network_APIs/TCP_Server_APIs.rst
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,11 @@ The parameters of the ``__init__`` method are as follows:
user-supplied RSA keys. Both files must exist, parse as PEM, and pair
with each other; an invalid pair is silently ignored and the default
key lookup is used instead (``None`` keeps the default lookup).
- ``max_mem_buff``: The forward pump's queued-byte high-water mark, in MiB and
stored as bytes. The reader waits for low-water drain, applying TCP flow
control independently of user pause/start. One fan-out enqueue may overshoot;
it is not a total process RAM ceiling. The default is 2048.
- ``max_mem_buff``: Shared queue budget for all native file and folder relays
on this server, in MiB (default 2048). Each downstream copy of a header or
body segment counts until its send returns or the copy is discarded. When
the shared budget is full, readers wait without changing user pause/start
state. This limits pending delivery bytes, not total process RAM.
- ``recv_messages_chunk_len``: The number of bytes read per receive on the
control connection, covering commands and messages. The default is 4096.
- ``file_transfer_chunk_len``: The chunk size in bytes for streaming a file
Expand Down
11 changes: 11 additions & 0 deletions docs/source/changes/forward-one-to-one.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Native TCP file forwarding: shared server queue budget

`max_mem_buff` now bounds the total pending downstream bytes of all native file and folder relays on one `TCP_Server_Base`. Previously each relay could buffer up to its own high-water mark plus one fan-out enqueue. Operators running concurrent relays should size **one** budget for their server. The constructor still takes MiB (default 2048); no Client or wire change is required. Updating only the Client does not change an older Server's behavior.

Each header or body segment is charged per downstream copy, including a writer's current send. A full budget delays readers; it does not pause the user task or complete it early. The ordinary transfer path, encryption, and final recipient acknowledgements are unchanged. This is not a cap on process RSS.

## Opt-in one-to-one scheduling on the Client

`TCP_Client_Base(..., is_forward_one_to_one=True)` makes each file entry and distinct target a separate transfer. The sender uses its existing `max_thread_num` file slots for initial sends and forwarded replays. Each pair keeps its own transfer ID and holds one slot through pause and final ACK. Items waiting at disconnect are not persisted or submitted on a replacement connection. `False` remains the default, and neither setting changes a stored route or the wire format.

With tracking enabled, two rows for the same source pathname can therefore be correct. A confirmed delivery to B does not retire C's interrupted transfer. C resumes on its retained route and destination, including when it has reached EOF but has not acknowledged delivery. In the default shared mode, the source still waits for every accepted target's ACK.
Loading
Loading