format: allow identifiers to start with $

CHANGELOG.md

@@ -83,6 +83,19 @@ The sections do not signal obligations; the prefixes do.
     `patternProperties`, `enum` and `const`. The `$` spellings, such as
     `"$wordsize"` and `{ "$sum": [...] }`, are no longer format terms.
 
+- An identifier may now start with `$`, so a compiler can use a
+  source-language name such as Solidity's `$balance` as it is. This relies on
+  the `~` sigil of [#323]: a name that starts with `$` cannot be confused with
+  a format term ([#325]).
+  - Schemas: **ethdebug/format/pointer/identifier**
+  - Producers: optional: a name may now start with `$`
+    (**ethdebug/format/pointer/identifier** `pattern`
+    `^[a-zA-Z$_\-]+[a-zA-Z0-9$_\-]*$`).
+  - Consumers: required: accept a name that starts with `$`, which
+    **ethdebug/format/pointer/identifier**'s `pattern` now allows. Such a
+    string is a name, not a format term: `"$wordsize"` is a variable and
+    `{ ".offset": "$this" }` looks up a region named `$this`.
+
 - The variables of a scope (`define`/`in`) are visible only within its `in`.
   Pointers outside the scope, such as later members of an enclosing group, do
   not see them, and a variable defined with an outer variable's identifier
@@ -704,3 +717,4 @@ First published version of the specification.
 [#317]: https://github.com/ethdebug/format/pull/317
 [#318]: https://github.com/ethdebug/format/pull/318
 [#323]: https://github.com/ethdebug/format/pull/323
+[#325]: https://github.com/ethdebug/format/pull/325

packages/format/CHANGELOG.md

@@ -6,6 +6,12 @@ tracked in the root [`CHANGELOG.md`](../../CHANGELOG.md).
 
 ## Unreleased
 
+### Changed
+
+- `Pointer.isIdentifier` accepts a name that starts with `$`, and still
+  rejects one that starts with `~`. See the root
+  [`CHANGELOG.md`](../../CHANGELOG.md) ([#325]).
+
 ### Breaking
 
 - The bundled schemas write the pointer vocabulary with `~` instead of `$`
@@ -139,6 +145,7 @@ First publication.
 [#303]: https://github.com/ethdebug/format/pull/303
 [#305]: https://github.com/ethdebug/format/pull/305
 [#323]: https://github.com/ethdebug/format/pull/323
+[#325]: https://github.com/ethdebug/format/pull/325
 [`0ef2f37`]: https://github.com/ethdebug/format/commit/0ef2f37
 [`10ab103`]: https://github.com/ethdebug/format/commit/10ab103
 [`21e532e`]: https://github.com/ethdebug/format/commit/21e532e

packages/format/src/types/pointer/pointer.test.ts

@@ -8,12 +8,22 @@ import { Pointer, isPointer } from "./pointer.js";
 const identifierSchema = "schema:ethdebug/format/pointer/identifier";
 
 describe("Pointer.isIdentifier", () => {
-  const valid = ["a", "a0", "-$", "__init__", "_x", "A", "foo-bar", "x$y"];
+  const valid = [
+    "a",
+    "a0",
+    "-$",
+    "__init__",
+    "_x",
+    "A",
+    "foo-bar",
+    "x$y",
+    "$x", // leading $, as Solidity allows
+    "$wordsize", // a source name, not the ~wordsize constant
+  ];
 
   const invalid = [
     "", // empty
     "0abc", // leading digit
-    "$x", // leading $
     "~x", // leading ~, reserved for the format's own terms
     "~wordsize", // the constant, not an identifier
     "a b", // space
@@ -55,10 +65,10 @@ describe("the ~ sigil", () => {
     expect(Pointer.Expression.Arithmetic.isSum({ "~sum": [1, 2] })).toBe(true);
   });
 
-  it("does not read the old $-prefixed terms", () => {
+  it("reads $-prefixed names as identifiers, not format terms", () => {
     expect(Pointer.Expression.isConstant("$wordsize")).toBe(false);
-    expect(Pointer.isExpression("$wordsize")).toBe(false);
-    expect(Pointer.Expression.isReference("$this")).toBe(false);
+    expect(Pointer.Expression.isVariable("$wordsize")).toBe(true);
+    expect(Pointer.Expression.isReference("$this")).toBe(true);
     expect(Pointer.isExpression({ $sum: [1, 2] })).toBe(false);
   });
 });

packages/format/src/types/pointer/pointer.ts

@@ -6,7 +6,7 @@ export const isPointer = (value: unknown): value is Pointer =>
 export namespace Pointer {
   export type Identifier = string;
   export const isIdentifier = (value: unknown): value is Identifier =>
-    typeof value === "string" && /^[a-zA-Z_-]+[a-zA-Z0-9$_-]*$/.test(value);
+    typeof value === "string" && /^[a-zA-Z$_-]+[a-zA-Z0-9$_-]*$/.test(value);
 
   export type Region =
     | Region.Stack

packages/pointers/CHANGELOG.md

@@ -6,6 +6,13 @@ itself are tracked in the root [`CHANGELOG.md`](../../CHANGELOG.md).
 
 ## Unreleased
 
+### Changed
+
+- `evaluate` and `dereference` resolve variable, region and template names
+  that start with `$`: `"$wordsize"` is a variable, and `{ ".offset": "$this" }`
+  looks up a region named `$this`. The hint for an old `$` term now comes with
+  an unknown variable or a region that is not found ([#325]).
+
 ### Fixed
 
 - `dereference` scopes the variables of a `define` to its `in`. They used to
@@ -129,3 +136,4 @@ First publication.
 [#316]: https://github.com/ethdebug/format/pull/316
 [#317]: https://github.com/ethdebug/format/pull/317
 [#323]: https://github.com/ethdebug/format/pull/323
+[#325]: https://github.com/ethdebug/format/pull/325

packages/pointers/src/dereference/index.test.ts

@@ -816,17 +816,71 @@ describe("dereference", () => {
     expect(regions.lookup["my-slot-value"]).toBeDefined();
   });
 
-  it("rejects a $ operator", async () => {
-    const pointer = {
-      location: "memory",
-      offset: { $sum: [1, 2] },
-      length: 0x20,
-    } as unknown as Pointer;
+  describe("names that start with $", () => {
+    it("resolves a region named $this, not the current region", async () => {
+      const pointer: Pointer = {
+        group: [
+          { name: "$this", location: "memory", offset: 0x40, length: 0x20 },
+          {
+            location: "memory",
+            offset: { ".offset": "$this" },
+            length: 0x10,
+          },
+        ],
+      };
 
-    const cursor = await dereference(pointer);
+      const cursor = await dereference(pointer);
 
-    await expect(cursor.view(state)).rejects.toThrow(
-      "Unexpected unknown kind of pointer",
-    );
+      const { regions } = await cursor.view(state);
+
+      expect(regions[1]).toEqual({
+        location: "memory",
+        offset: Data.fromNumber(0x40),
+        length: Data.fromNumber(0x10),
+      });
+    });
+
+    it("dereferences a define and a template named with $", async () => {
+      const pointer: Pointer = {
+        define: { $wordsize: 7 },
+        in: {
+          templates: {
+            $template: {
+              expect: ["$wordsize"],
+              for: {
+                location: "memory",
+                offset: "$wordsize",
+                length: "$wordsize",
+              },
+            },
+          },
+          in: { template: "$template" },
+        },
+      };
+
+      const cursor = await dereference(pointer);
+
+      const { regions } = await cursor.view(state);
+
+      expect(regions[0]).toEqual({
+        location: "memory",
+        offset: Data.fromNumber(7),
+        length: Data.fromNumber(7),
+      });
+    });
+
+    it("rejects a $ operator", async () => {
+      const pointer = {
+        location: "memory",
+        offset: { $sum: [1, 2] },
+        length: 0x20,
+      } as unknown as Pointer;
+
+      const cursor = await dereference(pointer);
+
+      await expect(cursor.view(state)).rejects.toThrow(
+        "Unexpected unknown kind of pointer",
+      );
+    });
   });
 });

packages/pointers/src/evaluate.test.ts

@@ -415,8 +415,29 @@ describe("evaluate", () => {
   });
 });
 
-describe("evaluate, with the old $ sigil", () => {
-  const options: EvaluateOptions = { state, variables: {}, regions: {} };
+describe("evaluate, with names that start with $", () => {
+  const region = (name: string, offset: number): Cursor.Region => ({
+    name,
+    location: "memory",
+    offset: Data.fromNumber(offset),
+    length: Data.fromNumber(0x20),
+  });
+
+  const options: EvaluateOptions = {
+    state,
+    variables: { $wordsize: Value.integer(7n) },
+    regions: { $this: region("$this", 0x40), "~this": region("~this", 0x80) },
+  };
+
+  it("reads $wordsize as a variable, not the constant", async () => {
+    expect(await evaluate("$wordsize", options)).toEqual(Value.integer(7n));
+  });
+
+  it("looks up a region named $this, not the current region", async () => {
+    expect(await evaluate({ ".offset": "$this" }, options)).toEqual(
+      Value.integer(0x40n),
+    );
+  });
 
   it("rejects a $ operator, with a hint", async () => {
     const expression = { $sum: [1, 2] } as unknown as Pointer.Expression;
@@ -426,19 +447,21 @@ describe("evaluate, with the old $ sigil", () => {
     );
   });
 
-  it("hints at ~ for a $ constant or reference", async () => {
-    await expect(evaluate("$wordsize", options)).rejects.toThrow(
-      'expression: "$wordsize"; did you mean `~wordsize`?',
+  it("hints at ~ for an unknown $ variable or region", async () => {
+    const empty = { ...options, variables: {}, regions: {} };
+
+    await expect(evaluate("$wordsize", empty)).rejects.toThrow(
+      "Unknown variable with identifier $wordsize; did you mean `~wordsize`?",
     );
-    await expect(evaluate({ ".offset": "$this" }, options)).rejects.toThrow(
-      '{".offset":"$this"}; did you mean `~this`?',
+    await expect(evaluate({ ".offset": "$this" }, empty)).rejects.toThrow(
+      "Region not found: $this; did you mean `~this`?",
     );
   });
 
   it("gives no hint for other names", async () => {
-    await expect(evaluate("$balance", options)).rejects.toThrow(
-      /^Unexpected runtime failure to recognize kind of expression: "\$balance"$/,
-    );
+    await expect(
+      evaluate("$balance", { ...options, variables: {} }),
+    ).rejects.toThrow(/^Unknown variable with identifier \$balance$/);
   });
 });
 

packages/pointers/src/evaluate.ts

@@ -121,10 +121,13 @@ export async function evaluate(
     return evaluateRead(expression, options);
   }
 
+  const keys =
+    expression && typeof expression === "object" ? Object.keys(expression) : [];
+
   throw new Error(
     `Unexpected runtime failure to recognize kind of expression: ${JSON.stringify(
       expression,
-    )}${sigilHint(...namesIn(expression))}`,
+    )}${sigilHint(...keys)}`,
   );
 }
 
@@ -133,30 +136,10 @@ const formerTerm = new RegExp(
     "keccak256|concat|wordsized|sized\\d+)$",
 );
 
-/**
- * The strings in an unrecognized expression that may be old `$` terms: the
- * expression itself, or an object's keys and string values (as in
- * `{ ".offset": "$this" }`)
- */
-function namesIn(expression: unknown): string[] {
-  if (typeof expression === "string") {
-    return [expression];
-  }
-  if (expression && typeof expression === "object") {
-    return [
-      ...Object.keys(expression),
-      ...Object.values(expression).filter(
-        (value): value is string => typeof value === "string",
-      ),
-    ];
-  }
-  return [];
-}
-
 /**
  * For an error message: if one of `names` is a format term written with
  * the old `$` sigil, suggest its `~` spelling. The value still fails;
- * `$` is not an alias.
+ * `$` is not an alias, because a name may start with `$`.
  */
 function sigilHint(...names: string[]): string {
   const name = names.find((name) => formerTerm.test(name));
@@ -238,7 +221,9 @@ async function evaluateVariable(
 ): Promise<Value> {
   const value = variables[identifier];
   if (typeof value === "undefined") {
-    throw new Error(`Unknown variable with identifier ${identifier}`);
+    throw new Error(
+      `Unknown variable with identifier ${identifier}${sigilHint(identifier)}`,
+    );
   }
 
   return value;
@@ -334,7 +319,7 @@ async function evaluateLookup<O extends Pointer.Expression.Lookup.Operation>(
   const identifier = lookup[operation];
   const region = regions[identifier];
   if (!region) {
-    throw new Error(`Region not found: ${identifier}`);
+    throw new Error(`Region not found: ${identifier}${sigilHint(identifier)}`);
   }
 
   const property = Pointer.Expression.Lookup.propertyFrom(operation);
@@ -359,7 +344,7 @@ async function evaluateRead(
   const identifier = expression["~read"];
   const region = regions[identifier];
   if (!region) {
-    throw new Error(`Region not found: ${identifier}`);
+    throw new Error(`Region not found: ${identifier}${sigilHint(identifier)}`);
   }
 
   return Value.bytes(await read(region, options));

schemas/pointer/identifier.schema.yaml

@@ -5,14 +5,18 @@ title: ethdebug/format/pointer/identifier
 description: |
   An identifier for use within the context of a root pointer
 
-  An identifier **must not** start with `~`: that prefix is reserved for the
-  format's own terms in pointer expressions (e.g., `~wordsize`, `~this`,
-  `~sum`), so a name can never be confused with one of them.
+  The allowed characters seek to admit identifiers from common existing
+  languages as they are, so a compiler can use a source-language name
+  without renaming it. An identifier **must not** start with `~`: that
+  prefix is reserved for the format's own terms in pointer expressions
+  (e.g., `~wordsize`, `~this`, `~sum`), so a name can never be confused with
+  one of them.
 type: string
-pattern: "^[a-zA-Z_\\-]+[a-zA-Z0-9$_\\-]*$"
+pattern: "^[a-zA-Z$_\\-]+[a-zA-Z0-9$_\\-]*$"
 
 examples:
   - a
   - a0
   - -$
   - __init__
+  - $balance