bugc: string calldata references

packages/bugc/CHANGELOG.md

@@ -9,6 +9,16 @@ support. Changes to the specification itself are tracked in the root
 
 ### Added
 
+- `string calldata`, a string in calldata, as `bytes calldata` is bytes
+  in calldata: `let name: string calldata = msg.data[a:b] as string calldata;`
+  refers to the calldata and copies nothing. Bytes and a string in
+  calldata convert to each other with a cast (`as string calldata`,
+  `as bytes calldata`). A `string calldata` has `.length`, and copies
+  to a `string` in memory or storage (`let m: string = name;`,
+  `name as string`, `players[msg.sender].name = name;`). As `string`
+  is, it is not indexed or sliced, and as `bytes calldata` is, it is
+  only the type of a `let` or a cast. A local's pointer is that of
+  bytes in calldata, and its type is `{ "kind": "string" }` ([#379]).
 - A struct copied from storage to memory may have `string` and `bytes`
   fields: each field's word holds the address of its own copy. A struct
   in memory assigned to storage (`players[msg.sender] = player;`) is
@@ -23,6 +33,9 @@ support. Changes to the specification itself are tracked in the root
 
 ### Changed
 
+- `==`, `!=` and a mapping key with bytes or a string in calldata are
+  now type errors. Before, they compared or hashed the calldata
+  reference's word (offset and length), not the bytes ([#379]).
 - A struct member of a struct in storage now takes all the slots its
   struct needs, as in Solidity, and the member after it starts at the
   next free slot. Before, it took one slot, so the members after a
@@ -36,6 +49,9 @@ support. Changes to the specification itself are tracked in the root
 
 ### Fixed
 
+- `push` of bytes in calldata onto an `array<bytes>` in storage now
+  stores the bytes. Before, it stored the calldata reference's word
+  (offset and length) ([#379]).
 - Reading a `string` or `bytes` from storage as a whole, as in
   `let m: string = motd;`, now copies it to memory, decoded from its
   storage encoding. This also corrects an index (`data[3]`), a slice
@@ -509,3 +525,4 @@ First publication.
 [#370]: https://github.com/ethdebug/format/pull/370
 [#371]: https://github.com/ethdebug/format/pull/371
 [#377]: https://github.com/ethdebug/format/pull/377
+[#379]: https://github.com/ethdebug/format/pull/379

packages/bugc/src/ast/spec.ts

@@ -489,8 +489,10 @@ export namespace Type {
       };
     }
 
+    /** `string`, or `string calldata` (`location`) */
     export interface String extends Elementary.Base {
       kind: "type:elementary:string";
+      location?: "calldata";
     }
 
     export const isString = (type: Type.Base): type is Type.Elementary.String =>
@@ -499,8 +501,14 @@ export namespace Type {
     export function string(
       id: Id,
       loc?: SourceLocation,
+      location?: "calldata",
     ): Type.Elementary.String {
-      return { id, kind: "type:elementary:string", loc: loc ?? null };
+      return {
+        id,
+        kind: "type:elementary:string",
+        ...(location ? { location } : {}),
+        loc: loc ?? null,
+      };
     }
 
     export interface Fixed extends Elementary.Base {

packages/bugc/src/evmgen/debug/local-variables.soundness.test.ts

@@ -476,6 +476,37 @@ code {
   };
 }
 
+/**
+ * A `string calldata` local refers to the calldata as `bytes calldata`
+ * does; its pointer reads exactly the string's bytes
+ */
+function namedProgram(text: string): LocalsProgram {
+  return {
+    name: `a string calldata local of ${text.length} bytes`,
+    source: `name SetName;
+storage { [0] motd: string; [1] r: uint256; }
+create { r = 1; }
+code {
+  let offset = msg.data[4:36] as bytes32 as uint256;
+  let n = msg.data[4 + offset:36 + offset] as bytes32 as uint256;
+  let name: string calldata =
+    msg.data[36 + offset:36 + offset + n] as string calldata;
+  motd = name;
+  if (r > 0) { r = name.length; }
+}`,
+    calldata: setMotd(text),
+    locals: {
+      offset: { values: [32n] },
+      n: { values: [BigInt(text.length)] },
+      name: {
+        shape: { kind: "calldata" },
+        values: [textBytes(text)],
+        everyLevel: true,
+      },
+    },
+  };
+}
+
 const programs: LocalsProgram[] = [
   {
     name: "straight line",
@@ -1109,6 +1140,9 @@ code {
     locals: { b: { shape: { kind: "calldata" }, values: ["0x"] } },
   },
   ...annotations.map(setMotdProgram),
+  ...["", "a".repeat(31), "b".repeat(32), "c".repeat(33), motd].map(
+    namedProgram,
+  ),
   {
     // A slice longer than a word: its pointer reads every byte
     name: "a long bytes slice",
@@ -1216,6 +1250,40 @@ describe.each(levels)("locals hold the program's values at O%i", (level) => {
   );
 });
 
+describe.each(levels)("a string calldata local at O%i", (level) => {
+  it("has a string type and the pointer of bytes in calldata", async () => {
+    const pointerOf = async (type: string) => {
+      const { program } = await traceLocals(
+        `name SetName;
+storage { [0] motd: string; }
+code {
+  let name: ${type} = msg.data[4:msg.data.length] as ${type};
+  motd = name as string;
+}`,
+        level,
+        setMotd(motd),
+      );
+      const entries = program.instructions
+        .flatMap(({ context }) => localsOf(context))
+        .filter((entry) => entry.identifier === "name" && entry.pointer);
+      expect(entries.length).toBeGreaterThan(0);
+      return entries;
+    };
+
+    const strings = await pointerOf("string calldata");
+    const bytes = await pointerOf("bytes calldata");
+    for (const entry of strings) {
+      expect(entry.type).toEqual({ kind: "string" });
+    }
+    for (const entry of bytes) {
+      expect(entry.type).toEqual({ kind: "bytes" });
+    }
+    expect(strings.map((entry) => entry.pointer)).toEqual(
+      bytes.map((entry) => entry.pointer),
+    );
+  });
+});
+
 describe("variables before a function's first statement", () => {
   // Contexts are postconditions: the step after an instruction shows
   // its context. Every step lists the storage variable; from the

packages/bugc/src/evmgen/string-calldata.test.ts

@@ -0,0 +1,109 @@
+import { describe, it, expect } from "vitest";
+import { keccak256 } from "ethereum-cryptography/keccak";
+
+import { executeProgram } from "#test/evm/behavioral";
+import {
+  expectEncoded,
+  hash,
+  mappingSlot,
+  text,
+  word as storageWord,
+} from "#test/evm/storage";
+
+const levels = [0, 1, 2, 3] as const;
+
+/** The sender of every transaction the executor sends */
+const sender = 1n;
+
+const word = (n: bigint) => n.toString(16).padStart(64, "0");
+
+/** Calldata for `setName(string)`, ABI-encoded, without the 0x */
+function setName(name: string): string {
+  const data = Buffer.from(name).toString("hex");
+  const padded = data.padEnd(Math.ceil(data.length / 64) * 64, "0");
+  return "deadbeef" + word(32n) + word(BigInt(name.length)) + padded;
+}
+
+/**
+ * A `string calldata` local from the ABI offset and length words of a
+ * string argument, copied to storage (a variable and a struct field)
+ * and to memory
+ */
+const source = `name Names;
+define { struct Player { score: uint256; name: string; }; }
+storage {
+  [0] motd: string; [1] len: uint256; [2] players: mapping<address, Player>;
+  [3] copy: string; [4] raw: bytes; [5] first: uint256; [6] h: bytes32;
+  [7] other: string;
+}
+code {
+  let offset = msg.data[4:36] as bytes32 as uint256;
+  let n = msg.data[4 + offset:36 + offset] as bytes32 as uint256;
+  let name: string calldata =
+    msg.data[36 + offset:36 + offset + n] as string calldata;
+  let alias = name;
+  len = alias.length;
+  motd = name;
+  players[msg.sender].name = name;
+  let m = name as string;
+  copy = m;
+  let s: string = name;
+  other = s;
+  raw = name as bytes;
+  let b = name as bytes calldata;
+  if (b.length > 0) { first = b[0]; }
+  h = keccak256(name);
+}`;
+
+const lengths = [0, 31, 32, 33, 65];
+
+describe.each(levels)("a string in calldata at O%i", (level) => {
+  it.each(lengths)("copies %i bytes to storage and memory", async (length) => {
+    const name = text(length);
+    const result = await executeProgram(source, {
+      calldata: setName(name),
+      optimizationLevel: level,
+    });
+    expect(result.callSuccess).toBe(true);
+    expect(await result.getStorage(1n)).toBe(BigInt(length));
+    await expectEncoded(result, 0n, name);
+    await expectEncoded(result, mappingSlot(2n, sender) + 1n, name);
+    await expectEncoded(result, 3n, name);
+    await expectEncoded(result, 4n, name);
+    await expectEncoded(result, 7n, name);
+    expect(await result.getStorage(5n)).toBe(
+      length > 0 ? BigInt(name.charCodeAt(0)) : 0n,
+    );
+    expect(await result.getStorage(6n)).toBe(
+      BigInt("0x" + Buffer.from(keccak256(Buffer.from(name))).toString("hex")),
+    );
+  });
+});
+
+describe.each(levels)("a push from calldata at O%i", (level) => {
+  for (const [type, memory] of [
+    ["string calldata", "string"],
+    ["bytes calldata", "bytes"],
+  ]) {
+    it.each(lengths)(`pushes ${type} of %i bytes`, async (length) => {
+      const name = text(length);
+      const result = await executeProgram(
+        `name Push;
+storage { [0] items: array<${memory}>; }
+code {
+  let offset = msg.data[4:36] as bytes32 as uint256;
+  let n = msg.data[4 + offset:36 + offset] as bytes32 as uint256;
+  let c: ${type} = msg.data[36 + offset:36 + offset + n] as ${type};
+  items.push(c);
+  items.push(c);
+}`,
+        { calldata: setName(name), optimizationLevel: level },
+      );
+      expect(result.callSuccess).toBe(true);
+      expect(await result.getStorage(0n)).toBe(2n);
+      const base = hash(storageWord(0n));
+      await expectEncoded(result, base, name);
+      await expectEncoded(result, base + 1n, name);
+    });
+  }
+});

packages/bugc/src/irgen/generate/calldata.ts

@@ -127,10 +127,19 @@ export function* emitCopyToMemory(
   return Ir.Value.temp(base, Ir.Type.ref("memory", origin));
 }
 
-/** A value, or, if it is bytes in calldata, a copy of them in memory */
+/**
+ * A value, or, if it is bytes or a string in calldata, a copy of them
+ * in memory, typed `bytes` or `string`
+ */
 export function* emitInMemory(
   value: Ir.Value,
   node: Ast.Node,
 ): Process<Ir.Value> {
-  return isCalldata(value) ? yield* emitCopyToMemory(value, node) : value;
+  if (!isCalldata(value)) return value;
+  const { origin } = value.type;
+  return yield* emitCopyToMemory(
+    value,
+    node,
+    origin === "synthetic" ? undefined : Type.Elementary.inMemory(origin),
+  );
 }

packages/bugc/src/irgen/generate/expressions/access.ts

@@ -96,9 +96,9 @@ const makeBuildMemberAccess = (
             (Type.Elementary.isBytes(objectType) ||
               Type.Elementary.isString(objectType))))
       ) {
-        // Bytes in calldata: the length is in their word
+        // Bytes or a string in calldata: the length is in their word
         if (
-          Type.Elementary.Bytes.isCalldata(objectType) &&
+          Type.Elementary.isCalldata(objectType) &&
           !Ast.Expression.Special.isMsgData(expr.object)
         ) {
           const word = yield* buildExpression(expr.object, {

packages/bugc/src/irgen/generate/expressions/call.ts

@@ -167,9 +167,13 @@ function* buildPush(
   ) => Process<Ir.Value>,
 ): Process<Ir.Value> {
   const marker = Ir.Value.constant(0n, Ir.Type.Scalar.uint256);
+  // A pushed value's bytes are in memory
   const value =
     expr.arguments.length > 0
-      ? yield* buildExpression(expr.arguments[0], { kind: "rvalue" })
+      ? yield* emitInMemory(
+          yield* buildExpression(expr.arguments[0], { kind: "rvalue" }),
+          expr.arguments[0],
+        )
       : undefined;
 
   const chain = yield* findStorageAccessChain(callee.object);

packages/bugc/src/irgen/generate/statements/assign.ts

@@ -63,10 +63,11 @@ function* buildLValue(
   }
 
   // Evaluate the value expression with the appropriate context. Bytes
-  // in calldata copy to memory, unless the target is in calldata too.
+  // or a string in calldata copy to memory, unless the target is in
+  // calldata too.
   let value = yield* buildExpression(valueExpr, context);
   const targetType = yield* Process.Types.nodeType(target);
-  if (!targetType || !Type.Elementary.Bytes.isCalldata(targetType)) {
+  if (!targetType || !Type.Elementary.isCalldata(targetType)) {
     value = yield* emitInMemory(value, valueExpr);
   }
 

packages/bugc/src/irgen/type.ts

@@ -57,8 +57,8 @@ export function fromBugType(type: BugType): Ir.Type {
           return Ir.Type.ref(type.location ?? "memory", type);
         }
       case "string":
-        // Strings are always memory references
-        return Ir.Type.ref("memory", type);
+        // Strings are memory references, or calldata ones
+        return Ir.Type.ref(type.location ?? "memory", type);
       default:
         assertExhausted(type);
     }

packages/bugc/src/parser/parser.ts

@@ -376,21 +376,24 @@ let statement: P.Parser<Ast.Statement>;
  * Type Parsers
  */
 
-// Elementary types with location. Dynamic bytes may be followed by
-// `calldata`: bytes in calldata
+// Elementary types with location. Dynamic bytes and string may be
+// followed by `calldata`: bytes or a string in calldata
 const elementaryType = located(
   Lang.elementaryType.chain((name: string) =>
-    name === "bytes"
+    name === "bytes" || name === "string"
       ? Lang.keyword("calldata")
           .atMost(1)
-          .map((calldata) =>
-            Ast.Type.Elementary.bytes(
-              PENDING_ID,
-              undefined,
-              undefined,
-              calldata.length > 0 ? "calldata" : undefined,
-            ),
-          )
+          .map((calldata) => {
+            const location = calldata.length > 0 ? "calldata" : undefined;
+            return name === "bytes"
+              ? Ast.Type.Elementary.bytes(
+                  PENDING_ID,
+                  undefined,
+                  undefined,
+                  location,
+                )
+              : Ast.Type.Elementary.string(PENDING_ID, undefined, location);
+          })
       : P.succeed(elementaryTypeOf(name)),
   ),
 );

packages/bugc/src/typechecker/assignable.ts

@@ -22,6 +22,16 @@ export function isAssignable(target: Type, value: Type): boolean {
     return true;
   }
 
+  // A string in calldata copies to a string in memory or storage
+  if (
+    Type.Elementary.String.isCalldata(value) &&
+    Type.isElementary(target) &&
+    Type.Elementary.isString(target) &&
+    target.location === undefined
+  ) {
+    return true;
+  }
+
   // Numeric types can be implicitly converted (with range checks)
   if (
     Type.isElementary(target) &&