Update: tests and gitignore

.gitignore

@@ -1,6 +1,10 @@
 # subject
 en.subject.pdf
 
+# md files
+*.md
+!README.md
+
 # Prerequisites
 *.d
 
@@ -26,7 +30,6 @@ en.subject.pdf
 *.dll
 *.so.*
 
-
 # Fortran module files
 *.mod
 *.smod
@@ -51,6 +54,11 @@ build/
 build_debug/
 Build/
 build-*/
+dist/
+.dist
+
+# Byte compiled
+__pycache__/
 
 # CMake generated files
 CMakeFiles/
@@ -59,6 +67,10 @@ cmake_install.cmake
 install_manifest.txt
 compile_commands.json
 
+# OS generated files
+.DS_Store
+.DS_Store?
+
 # Temporary files
 *.tmp
 *.log
@@ -75,3 +87,8 @@ vcpkg_installed/
 Testing/
 .cache/
 reviewNotes.txt
+
+# test outputs
+tests/dynamic_tester/artifacts/
+tests/checks/.*.test
+received.bin

tests/HOWTO.txt

@@ -0,0 +1,133 @@
+:::::::::::::::::::
+	ft_irc tests
+:::::::::::::::::::
+
+Notation
+--------
+<value>     required, replace it
+[value]     optional
+ a | b       choose one
+<name>...   one or more, separated by spaces
+
+Defaults, set once for everything below: host 127.0.0.1, port 6667 (the subject's
+port; also the one irssi is told to use), password pass. Files for the bonus
+transfer are names under files/ (files/test.txt -> test.txt).
+Run every command from the repo root.
+
+Requirements (macOS and Linux)
+------------------------------
+c++        # C++ tests are built with the project flags: -Wall -Wextra -Werror -pedantic -std=c++98
+python3    # 3.8 or newer, standard library only bash, make, nc
+optional   # tmux and irssi (client windows), strace on Linux (syscall figures in the monitor)
+Port 6667 must be free; the tester names the process using it otherwise.
+The server is built by the tests themselves (make, make bonus).
+macOS, before the load suite:ulimit -n 2048
+
+==============================================================================
+Everything
+==============================================================================
+  make -C tests                       checks, then the dynamic tester; totals and the report link at the end
+  make -C tests checks                the checks only
+  make -C tests tester                the dynamic tester only
+  make -C tests clean                 remove test binaries and artifacts
+
+==============================================================================
+tests/checks
+==============================================================================
+Fixed pass/fail tests. C++ tests are compiled with the project flags against
+the server sources; scripts get their own fresh server on a free port.
+
+  make -C tests/checks                             all groups
+  make -C tests/checks <basic|networking|client|bonus>
+  make -C tests/checks <name>                      one test
+  bash tests/checks/run.sh <name>...               several tests in one run (any number)
+  bash tests/checks/run.sh                         usage and all names
+
+  group       name          checks
+  ---------   -----------   -------------------------------------------------------------
+  basic       basic         Makefile rules and flags, no relink, only allowed functions,
+                            one poll(), no select/epoll/kqueue/fork/errno, fcntl form
+  networking  socket        line framing, 510-byte limit, POLLOUT send path
+              flood         a member stops reading while the channel is flooded: others
+                            still served, the member is kept, every line delivered later
+              probe         odd input: NUL, binary, CR CR LF, bare LF, 511 and 4096-byte lines
+              throw         MODE on a missing channel, +o by a non-operator: server stays up
+              overlong      417 on an over-length line, close after the third one in a row
+              longline      10 KB without CRLF from three clients, a fourth stays responsive
+              churn         20 members, broadcast storm, JOIN/PART churn, RST disconnects
+              closing       what the client gets before its link is closed: 464, ERROR
+  client      parser        message grammar: prefix, command, params, trailing
+              reply         numeric reply formatting
+              state         client state: nick casefolding, invites, prefix, flags
+              registration  PASS/NICK/USER/CAP in every order, 431 432 433 461 462 464
+              messaging     PING/PONG, QUIT, NOTICE, PRIVMSG and their numerics
+              operator      482/442 for non-operators; KICK INVITE TOPIC MODE i t k o l
+  bonus       bot           bot dialogue: !help !version !whoami !send !reject !accept, CTCP
+              transfer      a DCC transfer through the bot, bytes compared with the file
+
+  Output      [ name ]  what it checks
+              [section]
+                [PASS] check          [FAIL] check
+              one PASS/FAIL line per test and "Results: N pass / M fail / T total" at the end.
+              Exit status 1 if anything failed.
+  ----------------------------------------------------------------------------
+  Against a server you started yourself (./ircserv <port> <password>):
+  bash tests/checks/networking/session.sh [host] [port] [password] [nick]                  raw IRC session, already registered
+  python3 tests/checks/networking/<probe|throw|overlong|longline|churn|closing>.py [port] [password]
+  bash tests/checks/bonus/bot.sh [host] [port] [password] [file]                          ./ircserv_bonus
+  bash tests/checks/bonus/transfer.sh [host] [port] [password] [nick] [file] [botnick]    ./ircserv_bonus
+
+==============================================================================
+tests/dynamic_tester
+==============================================================================
+Starts the server on port 6667, drives it with real clients over TCP and
+writes artifacts/report.html.
+
+  make -C tests/dynamic_tester                               menu; Enter runs everything
+  make -C tests/dynamic_tester <mandatory|robustness|load|bonus|compare>
+  make -C tests/dynamic_tester live                          everything, server and client windows shown while it runs
+  make -C tests/dynamic_tester debug                         everything, every line sent and received printed
+  make -C tests/dynamic_tester open                          open the report (macOS)
+  make -C tests/dynamic_tester <suite> [PORT=<port>] [PASSWORD=<password>]
+  make -C tests/dynamic_tester external [PORT=<port>] [PASSWORD=<password>]     a server you started yourself
+  make -C tests/dynamic_tester config                        the expectations in tester.conf and their values
+  make -C tests/dynamic_tester <suite> SET="<key>=<value> [<key>=<value>]..."   override an expectation for one run
+
+-----------------------------------------------------------------------------
+  Result      [PASS] / [FAIL]   subject requirement
+              [NOTE]            behaviour the subject leaves open; the check says what the server does
+              [INFO]            a measurement
+              [SKIP]            could not run
+              "Results: N pass / N fail / N notes / N info / N skip / T total", then the report link.
+------------------------------------------------------------------------------
+  Report      artifacts/report.html: one page, every scenario with its checks (sent, received,
+              expected, why), the client windows and the server log. Scenarios with a FAIL or
+              NOTE are expanded. artifacts/transcripts/<scenario>.txt: the raw lines.
+------------------------------------------------------------------------------
+  tester.conf what the tester expects where the subject leaves a choice (line endings,
+              over-long lines, a client that never reads, MODE numerics ...). Edit it to match
+              the server; a wrong entry shows as NOTE or FAIL in the affected checks.
+
+==============================================================================
+  Interactive
+==============================================================================
+  make -C tests/dynamic_tester lab          # client windows in one terminal, server log, walkthrough
+  make -C tests/dynamic_tester clients      # irssi or nc client windows on a launched server
+  make -C tests/dynamic_tester <irssi|nc>   # same, program preselected
+  make -C tests/dynamic_tester monitor      # live dashboard; starts the server itself
+  make -C tests/dynamic_tester attach [PORT=<port>] [PASSWORD=<password>]
+  											# live dashboard on a running server
+
+==============================================================================
+  Monitor while the tester runs: two terminals from the repo root.
+==============================================================================
+    terminal 1:  make -C tests/dynamic_tester attach
+    										# waits for port 6667, follows every server restart
+    terminal 2:  make -C tests/dynamic_tester
+    										# or any suite
+
+  Keys in the monitor: Tab moves clients -> log -> menu bar; in the menu bar Left/Right pick,
+  Enter or Up/Down change, b back; Up/Down Left/Right scroll the focused pane; q quits.
+
+------------------------------------------------------------------------------
+  Details: tests/dynamic_tester/README.md

tests/Makefile

@@ -0,0 +1,43 @@
+# ft_irc tests
+#
+#   make -C tests                 checks (all groups), then the dynamic tester (all suites)
+#   make -C tests checks          tests/checks: fixed C++/python/bash tests: basic networking client bonus
+#   make -C tests tester          tests/dynamic_tester: protocol suites over TCP, HTML report
+#   make -C tests lab             interactive lab (several client windows, server log, walkthrough)
+#   make -C tests clients         irssi or nc client windows on a launched server
+#   make -C tests monitor         live dashboard of the server
+#
+# Each part has its own Makefile with more targets:  make -C tests/checks help   make -C tests/dynamic_tester help
+
+SHELL := /bin/bash
+REPORT := $(CURDIR)/dynamic_tester/artifacts/report.html
+
+.PHONY: all checks tester lab clients monitor clean
+
+all:
+	@[ -t 1 ] && export FORCE_COLOR=1; t=$$(mktemp); \
+	 $(MAKE) --no-print-directory -C "$(CURDIR)/checks" all 2>&1 | tee "$$t"; r1=$${PIPESTATUS[0]}; \
+	 c=$$(sed $$'s/\e\\[[0-9;]*m//g' "$$t" | grep -E '^ *Results:' | tail -1 | sed 's/^ *Results: *//'); \
+	 $(MAKE) --no-print-directory -C "$(CURDIR)/dynamic_tester" all 2>&1 | tee "$$t"; r2=$${PIPESTATUS[0]}; \
+	 d=$$(sed $$'s/\e\\[[0-9;]*m//g' "$$t" | grep -E '^ *Results:' | tail -1 | sed 's/^ *Results: *//'); rm -f "$$t"; \
+	 url=$$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).as_uri())' "$(REPORT)"); \
+	 echo; echo "================================ tests ================================"; \
+	 printf '  %-16s %s\n' "checks" "$${c:-did not run}"; \
+	 printf '  %-16s %s\n' "dynamic tester" "$${d:-did not run}"; \
+	 printf '  %-16s %s\n' "report" "$$url"; \
+	 printf '  %-16s %s\n' "" "(Cmd-click / Ctrl-click the link, or: make -C tests/dynamic_tester open)"; \
+	 echo "======================================================================="; \
+	 [ $$r1 = 0 ] && [ $$r2 = 0 ]
+
+checks:
+	@[ -t 1 ] && export FORCE_COLOR=1; $(MAKE) --no-print-directory -C "$(CURDIR)/checks" all
+
+tester:
+	@$(MAKE) --no-print-directory -C "$(CURDIR)/dynamic_tester" all
+
+lab clients monitor:
+	@$(MAKE) --no-print-directory -C "$(CURDIR)/dynamic_tester" $@
+
+clean:
+	@$(MAKE) --no-print-directory -C "$(CURDIR)/checks" clean
+	@$(MAKE) --no-print-directory -C "$(CURDIR)/dynamic_tester" clean

tests/checks/Makefile

@@ -0,0 +1,29 @@
+# ft_irc checks: basic, networking, client, bonus. macOS and Linux.
+#
+#   make -C tests/checks              everything: basic networking client bonus
+#   make -C tests/checks basic        Makefile rules, flags, no relink, allowed functions only, one poll(),
+#                                     no select/epoll/kqueue/fork/errno, fcntl form            (basic/checks.sh)
+#   make -C tests/checks networking   framing, 510 limit, POLLOUT send path, stalled member kept (C++);
+#                                     odd input, crash regressions, over-length lines, killed
+#                                     and many clients (python, fresh server each)             (networking/)
+#   make -C tests/checks client       registration, PRIVMSG/NOTICE/PING/QUIT, channel operator
+#                                     commands and numerics, parser, replies (C++, no sockets) (client/)
+#   make -C tests/checks bonus        bot dialogue, real DCC transfer (bash, fresh ircserv_bonus) (bonus/)
+#   make -C tests/checks socket       one test by name: basic socket flood probe throw overlong longline
+#                                     churn closing parser reply state registration messaging operator bot transfer
+#
+# By hand: bash tests/checks/run.sh <group|name...>   (no argument = usage)
+
+SHELL := /bin/bash
+NAMES := basic socket flood probe throw overlong longline churn closing parser reply state registration messaging operator bot transfer
+
+.PHONY: all basic networking client bonus $(NAMES) clean help
+
+all networking bonus $(NAMES):
+	@bash "$(CURDIR)/run.sh" $@
+
+clean:
+	@rm -f "$(CURDIR)"/.*.test "$(CURDIR)"/bonus/received.bin
+
+help:
+	@sed -n '2,15p' "$(CURDIR)/Makefile" | sed 's/^# \{0,1\}//'

tests/checks/basic/checks.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Evaluation sheet "Basic checks", verified on the sources:
+#   - Makefile has NAME/all/clean/fclean/re, builds with the required flags,
+#     executable named ircserv, a second make does not relink
+#   - only functions from the subject's allowed list (no read/write/open/sleep/...)
+#   - exactly one poll() (or equivalent) call site, no select/epoll/kqueue
+#   - no fork, no errno-driven logic
+#   - every fcntl() is fcntl(fd, F_SETFL, O_NONBLOCK)
+# Usage: bash tests/checks/basic/checks.sh
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT="$(cd "$HERE/../../.." && pwd)"
+fail=0
+ok()   { echo "  ok   $1"; }
+bad()  { echo "  FAIL $1"; fail=1; }
+code() { # source lines with comments and string literals stripped, prefixed file:line
+    find "$ROOT/srcs" "$ROOT/includes" -type f \( -name '*.cpp' -o -name '*.hpp' -o -name '*.tpp' -o -name '*.ipp' \) |
+    while read -r f; do
+        awk -v F="${f#$ROOT/}" '
+            BEGIN { inc = 0 }
+            {
+                line = $0
+                if (inc) { if (sub(/.*\*\//, "", line)) inc = 0; else next }
+                while (match(line, /\/\*/)) { if (match(line, /\/\*.*\*\//)) { line = substr(line, 1, RSTART-1) substr(line, RSTART+RLENGTH) } else { line = substr(line, 1, RSTART-1); inc = 1; break } }
+                sub(/\/\/.*/, "", line)
+                gsub(/"([^"\\]|\\.)*"/, "\"\"", line)
+                print F ":" NR ": " line
+            }' "$f"
+    done
+}
+SRC=$(code)
+
+echo "[Makefile and build]"
+for rule in all clean fclean re; do
+    grep -qE "^$rule[[:space:]]*:" "$ROOT/Makefile" && ok "rule $rule" || bad "rule $rule missing"
+done
+grep -qE '^[[:space:]]*NAME[[:space:]]*[:?]?=[[:space:]]*ircserv' "$ROOT/Makefile" && ok "NAME = ircserv" || bad "NAME is not ircserv"
+grep -qE '^\s*CXXFLAGS.*-Wall.*-Wextra.*-Werror' "$ROOT/Makefile" && ok "flags -Wall -Wextra -Werror" || bad "flags -Wall -Wextra -Werror missing"
+grep -qE '\-std=c\+\+98' "$ROOT/Makefile" && ok "flag -std=c++98" || bad "flag -std=c++98 missing"
+if make -s -C "$ROOT" >/dev/null 2>&1 && [ -x "$ROOT/ircserv" ]; then ok "make builds ./ircserv"; else bad "make did not produce ./ircserv"; fi
+second=$(make -C "$ROOT" 2>&1 | grep -cE '(c\+\+|clang|g\+\+).* -o ')
+[ "$second" -eq 0 ] && ok "a second make does not relink" || bad "a second make recompiled/relinked ($second command(s))"
+
+echo "[allowed functions only]"
+# not in the subject's list: read/write (send/recv are), open, stat, sleep/usleep, system, pipe, dup, exec*, ioctl
+forb=$(printf '%s\n' "$SRC" | grep -E '(^|[^A-Za-z0-9_:.>])(read|write|open|stat|sleep|usleep|system|pipe|dup|dup2|execve|execl|execvp|ioctl|select)[[:space:]]*\(' | grep -vE 'std::|\.read\(|\.write\(|\.open\(|is_open\(')
+if [ -z "$forb" ]; then ok "no function outside the subject's list (read/write/open/stat/sleep/system/pipe/dup/exec/ioctl)"; else bad "functions outside the allowed list:"; printf '%s\n' "$forb" | cut -d: -f1,2,3 | sed 's/^/       /'; fi
+
+echo "[one poll()]"
+polls=$(printf '%s\n' "$SRC" | grep -E '(^|[^A-Za-z0-9_:.>])poll[[:space:]]*\(')
+n=$(printf '%s\n' "$polls" | grep -c .)
+if [ "$n" -eq 1 ]; then ok "exactly one poll() call site: $(printf '%s\n' "$polls" | cut -d: -f1,2)"; else bad "$n poll() call sites:"; printf '%s\n' "$polls" | cut -d: -f1,2 | sed 's/^/       /'; fi
+if printf '%s\n' "$SRC" | grep -qE '(^|[^A-Za-z0-9_])(select|epoll_wait|epoll_create|kqueue|kevent)[[:space:]]*\('; then bad "select/epoll/kqueue found"; else ok "no select/epoll/kqueue"; fi
+
+echo "[no fork, no errno]"
+if printf '%s\n' "$SRC" | grep -qE '(^|[^A-Za-z0-9_])fork[[:space:]]*\('; then bad "fork() found"; else ok "no fork()"; fi
+if printf '%s\n' "$SRC" | grep -qE '(^|[^A-Za-z0-9_])errno([^A-Za-z0-9_]|$)'; then bad "errno used:"; printf '%s\n' "$SRC" | grep -E '(^|[^A-Za-z0-9_])errno' | cut -d: -f1,2 | sed 's/^/       /'; else ok "errno is not used"; fi
+
+echo "[fcntl form]"
+fc=$(printf '%s\n' "$SRC" | grep -E '(^|[^A-Za-z0-9_])fcntl[[:space:]]*\(')
+badfc=$(printf '%s\n' "$fc" | grep -vE 'fcntl[[:space:]]*\([^,]+,[[:space:]]*F_SETFL[[:space:]]*,[[:space:]]*O_NONBLOCK[[:space:]]*\)' | grep .)
+if [ -z "$badfc" ]; then ok "every fcntl() is fcntl(fd, F_SETFL, O_NONBLOCK) ($(printf '%s\n' "$fc" | grep -c .) call(s))"; else bad "fcntl() used another way:"; printf '%s\n' "$badfc" | sed 's/^/       /'; fi
+
+echo "[non-blocking sockets]"
+printf '%s\n' "$SRC" | grep -q 'O_NONBLOCK' && ok "O_NONBLOCK is set" || bad "O_NONBLOCK never set"
+
+exit $fail