diff --git a/src/lib.rs b/src/lib.rs index a3b7dae4..c151056d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -212,12 +212,15 @@ impl BindingsGenerator<'_> { let mut packages = Vec::new(); for &path in self.wit_paths { - packages.push((path, resolve.push_path(path)?.0)); + packages.push((path, push_wit_path(&mut resolve, path)?)); } if packages.is_empty() { // If no WIT directory was provided as a parameter, use ./wit by default. - packages.push((Path::new("wit"), resolve.push_path("wit")?.0)); + packages.push(( + Path::new("wit"), + push_wit_path(&mut resolve, Path::new("wit"))?, + )); } let worlds = select_worlds(&resolve, self.worlds, &packages)?; @@ -973,6 +976,54 @@ impl ComponentGenerator<'_> { } } +fn is_wit_file(path: &Path) -> bool { + path.extension().and_then(|ext| ext.to_str()) == Some("wit") +} + +/// Parse WIT from `path`, omitting `//` and `/* … */` comments so generated +/// Python docstrings only include WIT documentation comments (`///` / `/**`). +fn push_wit_path(resolve: &mut Resolve, path: &Path) -> Result { + if path.is_file() && is_wit_file(path) { + let contents = fs::read_to_string(path) + .with_context(|| format!("failed to read {}", path.display()))?; + resolve.push_str(path, &summary::strip_wit_non_doc_comments(&contents)) + } else if path.is_dir() { + let tmp = tempfile::tempdir()?; + copy_dir_keeping_doc_comments(path, tmp.path())?; + Ok(resolve.push_path(tmp.path())?.0) + } else { + Ok(resolve.push_path(path)?.0) + } +} + +fn copy_dir_keeping_doc_comments(src: &Path, dst: &Path) -> Result<()> { + fs::create_dir_all(dst).with_context(|| format!("failed to create {}", dst.display()))?; + for entry in + fs::read_dir(src).with_context(|| format!("failed to read directory {}", src.display()))? + { + let entry = entry?; + let src_path = entry.path(); + let dst_path = dst.join(entry.file_name()); + if src_path.is_dir() { + copy_dir_keeping_doc_comments(&src_path, &dst_path)?; + } else if is_wit_file(&src_path) { + let contents = fs::read_to_string(&src_path) + .with_context(|| format!("failed to read {}", src_path.display()))?; + fs::write(&dst_path, summary::strip_wit_non_doc_comments(&contents)) + .with_context(|| format!("failed to write {}", dst_path.display()))?; + } else { + fs::copy(&src_path, &dst_path).with_context(|| { + format!( + "failed to copy {} to {}", + src_path.display(), + dst_path.display() + ) + })?; + } + } + Ok(()) +} + fn parse_features(features: &[&str]) -> IndexSet { features .iter() diff --git a/src/summary.rs b/src/summary.rs index da8f50f2..70671220 100644 --- a/src/summary.rs +++ b/src/summary.rs @@ -3222,6 +3222,97 @@ fn bindings_module_import(name: &str, alias: &str) -> String { } } +/// Drop WIT `//` and `/* … */` comments while keeping documentation comments +/// (`///` and `/** … */`). Newlines inside removed comments are preserved so +/// parse error locations stay aligned with the original file. +pub(crate) fn strip_wit_non_doc_comments(src: &str) -> String { + let mut out = String::with_capacity(src.len()); + let mut rest = src; + let mut in_string = false; + + while !rest.is_empty() { + if in_string { + let c = rest.chars().next().unwrap(); + out.push(c); + rest = &rest[c.len_utf8()..]; + if c == '\\' { + if let Some(next) = rest.chars().next() { + out.push(next); + rest = &rest[next.len_utf8()..]; + } + } else if c == '"' { + in_string = false; + } + continue; + } + + if rest.starts_with("///") { + let end = rest.find('\n').map_or(rest.len(), |i| i + 1); + out.push_str(&rest[..end]); + rest = &rest[end..]; + continue; + } + + if rest.starts_with("//") { + if let Some(i) = rest.find('\n') { + out.push('\n'); + rest = &rest[i + 1..]; + } else { + rest = ""; + } + continue; + } + + if rest.starts_with("/**") && !rest.starts_with("/**/") { + let end = block_comment_end(rest); + out.push_str(&rest[..end]); + rest = &rest[end..]; + continue; + } + + if rest.starts_with("/*") { + let end = block_comment_end(rest); + out.extend(rest[..end].chars().filter(|&c| c == '\n')); + rest = &rest[end..]; + continue; + } + + let c = rest.chars().next().unwrap(); + if c == '"' { + in_string = true; + } + out.push(c); + rest = &rest[c.len_utf8()..]; + } + + out +} + +fn block_comment_end(src: &str) -> usize { + let mut depth = 0; + let mut rest = src; + let mut offset = 0; + while !rest.is_empty() { + if rest.starts_with("/*") { + depth += 1; + offset += 2; + rest = &rest[2..]; + } else if rest.starts_with("*/") { + depth -= 1; + offset += 2; + rest = &rest[2..]; + if depth == 0 { + return offset; + } + } else { + let c = rest.chars().next().unwrap(); + offset += c.len_utf8(); + rest = &rest[c.len_utf8()..]; + } + } + offset +} + fn docstring(docs: Option<&str>, indent_level: usize, error: Option<&str>) -> String { let docs = match ( docs, @@ -3304,6 +3395,107 @@ mod tests { Ok(()) } + #[test] + fn strip_wit_non_doc_comments_keeps_docs_only() { + let src = r#"package demo:docs; + +// package-level line comment +interface example { + // plain comment + /// real docs + foo: func(); + + /// kept + // dropped + bar: func(); + + /* block comment */ + /** block docs */ + baz: func(); +} + +world example { + // world comment + export example; +} +"#; + let stripped = super::strip_wit_non_doc_comments(src); + assert!( + !stripped.contains("package-level line comment"), + "{stripped}" + ); + assert!(!stripped.contains("plain comment"), "{stripped}"); + assert!(!stripped.contains("dropped"), "{stripped}"); + assert!(!stripped.contains("block comment"), "{stripped}"); + assert!(!stripped.contains("world comment"), "{stripped}"); + assert!(stripped.contains("/// real docs"), "{stripped}"); + assert!(stripped.contains("/// kept"), "{stripped}"); + assert!(stripped.contains("/** block docs */"), "{stripped}"); + + let url_src = r#"package demo:docs; +world example { + // ignore + /// see "http://example.com/x" + export echo: func(); +} +"#; + let url_stripped = super::strip_wit_non_doc_comments(url_src); + assert!( + url_stripped.contains(r#"/// see "http://example.com/x""#), + "{url_stripped}" + ); + assert!(!url_stripped.contains("// ignore"), "{url_stripped}"); + } + + #[test] + fn stripped_wit_docs_exclude_line_comments() { + let src = r#"package demo:docs; + +interface types { + // plain comment + /// real docs + foo: func(); + + // some comment + /// plus documentation + comment-and-doc: func(); + + /// documentation plus + // another comment. + doc-and-comment: func(); + + // only a comment + plain: func(); +} + +world example { + export types; +} +"#; + let stripped = super::strip_wit_non_doc_comments(src); + let mut resolve = wit_parser::Resolve::default(); + resolve.push_str("docs.wit", &stripped).unwrap(); + let iface = resolve + .interfaces + .iter() + .find_map(|(_, iface)| (iface.name.as_deref() == Some("types")).then_some(iface)) + .unwrap(); + + assert_eq!( + iface.functions["foo"].docs.contents.as_deref(), + Some("real docs") + ); + assert_eq!( + iface.functions["comment-and-doc"].docs.contents.as_deref(), + Some("plus documentation") + ); + assert_eq!( + iface.functions["doc-and-comment"].docs.contents.as_deref(), + Some("documentation plus") + ); + assert_eq!(iface.functions["plain"].docs.contents, None); + } + #[test] fn versions_are_valid_module_name_components() -> Result<()> { for version in ["1.2.3", "0.1.7", "0.0.3", "1.0.0-alpha.1", "1.0.0+build.5"] { diff --git a/tests/bindings.rs b/tests/bindings.rs index 4b7b8e11..c1490010 100644 --- a/tests/bindings.rs +++ b/tests/bindings.rs @@ -266,6 +266,99 @@ if not any(("'''" in doc and '\\"""' in doc) for doc in backslash_docs): Ok(()) } +#[test] +fn bindings_docstrings_omit_wit_line_comments() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + fs::write( + dir.path().join("example.wit"), + r#"package demo:docs; + +// package of named fields (line comment) +interface documented { + /// documented function + doc-func: func(); + + // some comment + /// plus documentation + comment-and-doc: func(); + + /// documentation plus + // another comment. + doc-and-comment: func(); + + // only a line comment + plain: func(); +} + +world example { + // world line comment + export documented; +} +"#, + )?; + + cargo::cargo_bin_cmd!("componentize-py") + .current_dir(dir.path()) + .args(["-d", "example.wit", "-w", "example", "bindings", "."]) + .assert() + .success(); + + assert!(predicate::path::is_dir().eval(&dir.path().join("wit"))); + + Command::new("python3") + .current_dir(dir.path()) + .args([ + "-c", + r#" +import ast +import sys +from pathlib import Path + +docs_by_name = {} +for path in Path(".").rglob("*.py"): + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Module)): + doc = ast.get_docstring(node) + if doc: + key = getattr(node, "name", "") + docs_by_name.setdefault(key, []).append(doc) + +all_docs = "\n".join(doc for docs in docs_by_name.values() for doc in docs) +for forbidden in ( + "package of named fields", + "some comment", + "another comment", + "only a line comment", + "world line comment", +): + if forbidden in all_docs: + sys.stderr.write("line comment leaked into docstrings: %r\n%s\n" % (forbidden, all_docs)) + sys.exit(1) + +def has_doc(name, text): + return any(text in doc for doc in docs_by_name.get(name, [])) + +if not has_doc("doc_func", "documented function"): + sys.stderr.write("doc_func lost /// docs: %r\n" % docs_by_name.get("doc_func")) + sys.exit(1) +if not has_doc("comment_and_doc", "plus documentation"): + sys.stderr.write("comment_and_doc lost /// docs: %r\n" % docs_by_name.get("comment_and_doc")) + sys.exit(1) +if not has_doc("doc_and_comment", "documentation plus"): + sys.stderr.write("doc_and_comment lost /// docs: %r\n" % docs_by_name.get("doc_and_comment")) + sys.exit(1) +if docs_by_name.get("plain"): + sys.stderr.write("plain unexpectedly has a docstring: %r\n" % docs_by_name.get("plain")) + sys.exit(1) +"#, + ]) + .assert() + .success(); + + Ok(()) +} + fn generate_bindings(path: &Path, world: &str) -> Result { Ok(cargo::cargo_bin_cmd!("componentize-py") .current_dir(path)